TЗ на выполнение работ по настройки серверов проекта u4u.
Альфа версия для обсуждения цены и сроков выполнения работ
1. Что сейчас есть Описание существующей конфигурации.
1.1 Окружение и сервисы(API) которые используем.
Google Сloud.
google cloud storage
google cloud logging
google cloud error-reporting
1.2 Используемые технологии
Microservice Architecture.: Docker(docker-composer)
Databases: PostgreSql, Cassandra, MongoDb, Redis
Message broker: RabbitMq
Programming languages(frameworks): Python(Django, DjangorestFramework, Tornado, Сelery ). Golang
1.3 Конфигурация существующей системы (docker-composer.yml)
version: '2'
networks:
default:
external:
name: testing
services:
postgresql:
restart: on-failure
image: postgres:9.5
env_file: .env-testing
volumes:
- testing-postgres-db-data:/var/lib/postgresql/data/
ports:
- "5432:5432"
logging:
driver: gcplogs
postgresql-order:
restart: on-failure
image: postgres:9.5
environment:
- POSTGRES_USER=xxxx
- POSTGRES_DB=xxxx
- POSTGRES_PASSWORD=xxxx
- POSTGRES_HOST=postgresql-order
- POSTGRES_PORT=5432
volumes:
- testing-postgres-order-db:/var/lib/postgresql/data/
ports:
- "5433:5432"
logging:
driver: gcplogs
postgresql-promo:
restart: on-failure
image: postgres:9.5
environment:
- POSTGRES_USER=xxxxxx
- POSTGRES_DB=xxxxx
- POSTGRES_PASSWORD=xxxxx
- POSTGRES_HOST=postgresql-promo
- POSTGRES_PORT=5432
volumes:
- testing-postgres-promo-db:/var/lib/postgresql/data/
ports:
- "9193:5432"
logging:
driver: gcplogs
mongo:
restart: on-failure
image: mongo:3.3
volumes:
- testing-mongo-db-data:/data/db/
logging:
driver: gcplogs
mongo-auth:
restart: on-failure
image: mongo:3.3
volumes:
- mongo-auth-db-data:/data/db/
logging:
driver: gcplogs
rabbitmq:
restart: on-failure
image: rabbitmq:3-management
env_file: .env
ports:
- "9095:15672"
logging:
driver: gcplogs
postfix:
image: alterrebe/postfix-relay
env_file: .env-testing
logging:
driver: gcplogs
genpdf:
restart: on-failure
image: center.u4u.online/testing-genpdf
env_file: .env-testing
volumes:
- media:/opt/src/media
links:
- rabbitmq:rabbitmq
logging:
driver: gcplogs
front:
restart: on-failure
image: center.u4u.online/testing-front
command: bash -c "nginx"
logging:
driver: gcplogs
wa:
restart: on-failure
image: center.u4u.online/testing-wa
env_file: .env-testing
volumes:
- media:/opt/wa/media
links:
- postgresql
- genpdf
- auth
logging:
driver: gcplogs
ua:
restart: on-failure
image: center.u4u.online/testing-ua
env_file: .env-testing
volumes:
- media:/opt/media
links:
- mongo
- rabbitmq
- genpdf
- photod
logging:
driver: gcplogs
auth:
restart: on-failure
image: center.u4u.online/testing-auth
env_file: .env-testing
volumes:
- media:/opt/src/media
links:
- mongo
logging:
driver: gcplogs
order:
restart: on-failure
image: center.u4u.online/testing-order
env_file: .env-testing
volumes:
- media:/opt/src/media
links:
- postgresql-order
- genpdf
- auth
ports:
- "8088:80"
logging:
driver: gcplogs
promo:
restart: on-failure
image: center.u4u.online/testing-promo
env_file: .env
volumes:
- media:/opt/src/media
links:
- postgresql-promo
ports:
- "9192:80"
logging:
driver: gcplogs
mongodata_photod:
image: mongo:3.3
volumes:
- testing-mongodata-photod-db:/data/db
command: --break-mongo
logging:
driver: gcplogs
mongo-photod:
image: mongo:3.3
volumes_from:
- mongodata_photod
command: --smallfiles --rest
logging:
driver: gcplogs
redis-photorec:
image: center.u4u.online/redis-photorec
ports:
- "6379:6379"
logging:
driver: gcplogs
photorec:
restart: on-failure
image: center.u4u.online/testing-photorec
env_file: .env-testing
volumes:
- media:/opt/media
links:
- mongo
- rabbitmq
- redis-photorec
logging:
driver: gcplogs
photod:
restart: on-failure
image: center.u4u.online/testing-photod
cpu_shares: 768
cpuset: 0-2
env_file: .env-testing
links:
- mongo-photod
- rabbitmq
- redis-photorec
volumes_from:
- wa
logging:
driver: gcplogs
tsm:
restart: on-failure
image: center.u4u.online/testing-tsm
env_file: .env-testing
links:
- rabbitmq
- photod
volumes:
- ./stats:/go/src/maker_maket/stat
logging:
driver: gcplogs
textgen:
restart: on-failure
image: center.u4u.online/testing-textgen
env_file: .env
links:
- rabbitmq
volumes_from:
- media-access
logging:
driver: gcplogs
media-access:
image: center.u4u.online/media
volumes:
- media:/www/media/
logging:
driver: gcplogs
cassandra:
restart: on-failure
image: cassandra:2.2.7
volumes:
- testing-cassandra-db-data:/var/lib/cassandra
logging:
driver: gcplogs
kong:
restart: on-failure
image: center.u4u.online/tmp-kong
env_file: .env-testing
links:
- cassandra
- front
- wa
- ua
- auth
- order
- textgen
ports:
- "80:8000"
- "443:8443"
logging:
driver: gcplogs
kong-dev:
restart: on-failure
image: center.u4u.online/kong-dev
env_file: .env-testing
links:
- cassandra
- front
- wa
- ua
- auth
- textgen
ports:
- "8080:8000"
logging:
driver: gcplogs
ua-backup:
image: center.u4u.online/mongo-backup-to-gs
links:
- mongo
environment:
- MONGO_URL=mongo
- MONGO_USER=
- MONGO_PASSWORD=xxxx
- GS_URL=u4u-backup/testing
- GS_ACCESS_KEY=xxxxx
- GS_SECRET_KEY=xxxxxx
logging:
driver: gcplogs
photod-backup:
image: center.u4u.online/mongo-backup-to-gs
links:
- mongo-photod
environment:
- MONGO_URL=mongo-photod
- MONGO_USER=
- MONGO_PASSWORD=
- GS_URL=u4u-backup/testing
- GS_ACCESS_KEY=xxxxx
- GS_SECRET_KEY=xxxxx
logging:
driver: gcplogs
wa-backup:
image: center.u4u.online/postgres-backup-to-gs
links:
- postgresql
environment:
- POSTGRES_HOST=xxxx
- POSTGRES_PORT=5432
- POSTGRES_DB=xxxx
- POSTGRES_USER=xxxx
- POSTGRES_PASSWORD=xxxxx
- GS_URL=u4u-backup/testing
- GS_ACCESS_KEY=xxxxx
- GS_SECRET_KEY=xxxxx
logging:
driver: gcplogs
order-backup:
image: center.u4u.online/postgres-backup-to-gs
links:
- postgresql-order
environment:
- POSTGRES_HOST=postgresql-order
- POSTGRES_PORT=5432
- POSTGRES_DB=xxxx
- POSTGRES_USER=xxxx
- POSTGRES_PASSWORD=xxxx
- GS_URL=u4u-backup/testing
- GS_ACCESS_KEY=xxxx
- GS_SECRET_KEY=xxxxx
logging:
driver: gcplogs
media-backup:
image: center.u4u.online/media-backup-to-gs
volumes_from:
- media-access
environment:
- GS_URL=u4u-backup/testing
- GS_ACCESS_KEY=xxxxxx
- GS_SECRET_KEY=xxxxxxx
logging:
driver: gcplogs
volumes:
media:
driver: local
testing-mongodata-photod-db:
driver: local
testing-postgres-db-data:
driver: local
testing-postgres-order-db:
driver: local
testing-postgres-promo-db:
driver: local
testing-mongo-db-data:
driver: local
mongo-auth-db-data:
driver: local
testing-cassandra-db-data:
driver: local
2. Что хотим получить.
Добиться отказоустойчивости.
Быстрой выкладки наработак приложений сервисов.
Мониторин работоспособности и корректное выполнение резервного копирования.
Примерный вариант для обсуждения.
2.1 Миграция на CI/СD
Поддержка 3 окружений develop, testing, production(stage)
Предполаем использовать:
Google Cloud Kubernetes clusters
Google Cloud Container Registry
Google Cloud Source code
Возможно GitLab как альтернатива.
2.2 Выполнение BackUp с возможностью проверить сделанный BackUp данных
2.3 Мониторинг работы.
Prometheus
Stackdriver Monitoring
2.4 Логирование работы прилогения.
В существующей системе уже реализовано
c использованием API google-cloud-logging, google-cloud-error-reporting