From ae406a39791728747e2e40dd83c7cb2331b5d918 Mon Sep 17 00:00:00 2001 From: Finn Stutzenstein Date: Thu, 1 Apr 2021 09:05:07 +0200 Subject: [PATCH] OS4: prod setup --- .gitmodules | 16 +- docker/build.sh | 6 +- docker/docker-compose.dev.yml | 16 ++ docker/docker-compose.yml.m4 | 65 ++++-- docker/docker-stack.yml.m4 | 408 +++++++++++++++++----------------- docker/services.env | 6 + openslides-autoupdate-service | 2 +- openslides-backend | 2 +- openslides-client | 2 +- openslides-manage-service | 2 +- openslides-permission-service | 2 +- proxy/Caddyfile | 10 +- 12 files changed, 297 insertions(+), 240 deletions(-) diff --git a/.gitmodules b/.gitmodules index f0abd9602..b65e85bc3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,31 +1,31 @@ [submodule "openslides-datastore-service"] path = openslides-datastore-service - url = git@github.com:OpenSlides/openslides-datastore-service.git + url = https://github.com/OpenSlides/openslides-datastore-service.git branch = master [submodule "openslides-client"] path = openslides-client - url = git@github.com:OpenSlides/openslides-client.git + url = https://github.com/OpenSlides/openslides-client.git branch = master [submodule "openslides-backend"] path = openslides-backend - url = git@github.com:OpenSlides/openslides-backend.git + url = https://github.com/OpenSlides/openslides-backend.git branch = master [submodule "openslides-autoupdate-service"] path = openslides-autoupdate-service - url = git@github.com:OpenSlides/openslides-autoupdate-service.git + url = https://github.com/OpenSlides/openslides-autoupdate-service.git [submodule "openslides-auth-service"] path = openslides-auth-service - url = git@github.com:OpenSlides/openslides-auth-service.git + url = https://github.com/OpenSlides/openslides-auth-service.git branch = master [submodule "openslides-media-service"] path = openslides-media-service - url = git@github.com:OpenSlides/openslides-media-service.git + url = https://github.com/OpenSlides/openslides-media-service.git branch = openslides4-dev [submodule "openslides-permission-service"] path = openslides-permission-service - url = git@github.com:OpenSlides/openslides-permission-service.git + url = https://github.com/OpenSlides/openslides-permission-service.git branch = master [submodule "openslides-manage-service"] path = openslides-manage-service - url = git@github.com:OpenSlides/openslides-manage-service.git + url = https://github.com/OpenSlides/openslides-manage-service.git branch = main diff --git a/docker/build.sh b/docker/build.sh index 065d635bd..158ce695e 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -10,6 +10,8 @@ TARGETS=( [backend]="$HOME/../openslides-backend/" [auth]="$HOME/../openslides-auth-service/" [autoupdate]="$HOME/../openslides-autoupdate-service/" + [permission]="$HOME/../openslides-permission-service/" + [manage]="$HOME/../openslides-manage-service/" [datastore-reader]="$HOME/../openslides-datastore-service/reader" [datastore-writer]="$HOME/../openslides-datastore-service/writer" [media]="$HOME/../openslides-media-service/" @@ -19,11 +21,11 @@ TARGETS=( ) DOCKER_REPOSITORY="openslides" -DOCKER_TAG="latest" +DOCKER_TAG="latest-4" CONFIG="/etc/osinstancectl" OPTIONS=() BUILT_IMAGES=() -DEFAULT_TARGETS=(proxy client backend auth autoupdate datastore-reader datastore-writer media) +DEFAULT_TARGETS=(proxy client backend auth autoupdate permission manage datastore-reader datastore-writer media) usage() { cat << EOF diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 95bfbb36f..23d48631b 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -8,6 +8,7 @@ services: environment: - DATASTORE_ENABLE_DEV_ENVIRONMENT=1 - NUM_WORKERS=8 + - OPENSLIDES_DEVELOPMENT=1 volumes: - ../openslides-datastore-service/shared/shared:/app/shared - ../openslides-datastore-service/reader/reader:/app/reader @@ -27,6 +28,7 @@ services: - DATASTORE_ENABLE_DEV_ENVIRONMENT=1 - COMMAND=create_initial_data - DATASTORE_INITIAL_DATA_FILE=https://raw.githubusercontent.com/OpenSlides/OpenSlides/openslides4-dev/docs/example-data.json + - OPENSLIDES_DEVELOPMENT=1 ports: - 9011:9011 postgres: @@ -41,6 +43,8 @@ services: - backend - autoupdate env_file: services.env + environment: + - OPENSLIDES_DEVELOPMENT=1 volumes: - ../openslides-client/client/src:/app/src backend: @@ -51,6 +55,8 @@ services: - auth - permission env_file: services.env + environment: + - OPENSLIDES_DEVELOPMENT=1 volumes: - ../openslides-backend/openslides_backend:/app/openslides_backend ports: @@ -61,6 +67,8 @@ services: - datastore-reader - message-bus env_file: services.env + environment: + - OPENSLIDES_DEVELOPMENT=1 volumes: - ../openslides-autoupdate-service/cmd:/root/cmd - ../openslides-autoupdate-service/internal:/root/internal @@ -69,6 +77,8 @@ services: depends_on: - datastore-reader env_file: services.env + environment: + - OPENSLIDES_DEVELOPMENT=1 volumes: - ../openslides-permission-service/cmd:/app/cmd - ../openslides-permission-service/internal:/app/internal @@ -79,6 +89,8 @@ services: - datastore-reader - cache env_file: services.env + environment: + - OPENSLIDES_DEVELOPMENT=1 volumes: - ../openslides-auth-service/auth/src:/app/src ports: @@ -91,6 +103,8 @@ services: - backend - postgres env_file: services.env + environment: + - OPENSLIDES_DEVELOPMENT=1 volumes: - ../openslides-media-service/src:/app/src manage: @@ -99,6 +113,8 @@ services: - auth - datastore-writer env_file: services.env + environment: + - OPENSLIDES_DEVELOPMENT=1 ports: - "8001:8001" message-bus: diff --git a/docker/docker-compose.yml.m4 b/docker/docker-compose.yml.m4 index acb62e60b..af9ac0d54 100644 --- a/docker/docker-compose.yml.m4 +++ b/docker/docker-compose.yml.m4 @@ -14,43 +14,46 @@ define(`ifenvelse', `ifelse(read_env(`$1'),, `$2', read_env(`$1'))') define(`BACKEND_IMAGE', ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl ifenvelse(`DOCKER_OPENSLIDES_BACKEND_NAME', openslides-backend):dnl -ifenvelse(`DOCKER_OPENSLIDES_BACKEND_TAG', latest)) +ifenvelse(`DOCKER_OPENSLIDES_BACKEND_TAG', latest-4)) define(`PROXY_IMAGE', ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl ifenvelse(`DOCKER_OPENSLIDES_PROXY_NAME', openslides-proxy):dnl -ifenvelse(`DOCKER_OPENSLIDES_PROXY_TAG', latest)) +ifenvelse(`DOCKER_OPENSLIDES_PROXY_TAG', latest-4)) define(`CLIENT_IMAGE', ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl ifenvelse(`DOCKER_OPENSLIDES_CLIENT_NAME', openslides-client):dnl -ifenvelse(`DOCKER_OPENSLIDES_CLIENT_TAG', latest)) +ifenvelse(`DOCKER_OPENSLIDES_CLIENT_TAG', latest-4)) define(`AUTH_IMAGE', ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl ifenvelse(`DOCKER_OPENSLIDES_AUTH_NAME', openslides-auth):dnl -ifenvelse(`DOCKER_OPENSLIDES_AUTH_TAG', latest)) +ifenvelse(`DOCKER_OPENSLIDES_AUTH_TAG', latest-4)) define(`AUTOUPDATE_IMAGE', ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl ifenvelse(`DOCKER_OPENSLIDES_AUTOUPDATE_NAME', openslides-autoupdate):dnl -ifenvelse(`DOCKER_OPENSLIDES_AUTOUPDATE_TAG', latest)) +ifenvelse(`DOCKER_OPENSLIDES_AUTOUPDATE_TAG', latest-4)) define(`DATASTORE_READER_IMAGE', ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_READER_NAME', openslides-datastore-reader):dnl -ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_READER_TAG', latest)) +ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_READER_TAG', latest-4)) define(`DATASTORE_WRITER_IMAGE', ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_WRITER_NAME', openslides-datastore-writer):dnl -ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_WRITER_TAG', latest)) +ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_WRITER_TAG', latest-4)) define(`MEDIA_IMAGE', ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl ifenvelse(`DOCKER_OPENSLIDES_MEDIA_NAME', openslides-media):dnl -ifenvelse(`DOCKER_OPENSLIDES_MEDIA_TAG', latest)) +ifenvelse(`DOCKER_OPENSLIDES_MEDIA_TAG', latest-4)) define(`MANAGE_IMAGE', ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl ifenvelse(`DOCKER_OPENSLIDES_MANAGE_NAME', openslides-manage):dnl -ifenvelse(`DOCKER_OPENSLIDES_MANAGE_TAG', latest)) +ifenvelse(`DOCKER_OPENSLIDES_MANAGE_TAG', latest-4)) +define(`PERMISSION_IMAGE', +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_PERMISSION_NAME', openslides-permission):dnl +ifenvelse(`DOCKER_OPENSLIDES_PERMISSION_TAG', latest-4)) define(`PROJECT_DIR', ifdef(`PROJECT_DIR',PROJECT_DIR,.)) define(`ADMIN_SECRET_AVAILABLE', `syscmd(`test -f 'PROJECT_DIR`/secrets/admin.env')sysval') -define(`USER_SECRET_AVAILABLE', `syscmd(`test -f 'PROJECT_DIR`/secrets/user.env')sysval') divert(0)dnl dnl ---------------------------------------- # This configuration was created from a template file. Before making changes, @@ -88,6 +91,9 @@ services: - datastore-reader - datastore-writer env_file: services.env + environment: + - AUTH_TOKEN_KEY=test123 + - AUTH_COOKIE_KEY=test123 networks: - frontend - backend @@ -103,6 +109,7 @@ services: - backend - datastore-reader - postgres + datastore-writer: image: DATASTORE_WRITER_IMAGE depends_on: @@ -118,6 +125,7 @@ services: - DATASTORE_INITIAL_DATA_FILE=/data/initial-data.json volumes: - ./initial-data.json:/data/initial-data.json + postgres: image: postgres:11 environment: @@ -133,6 +141,9 @@ services: - datastore-reader - message-bus env_file: services.env + environment: + - AUTH_KEY_TOKEN=test123 + - AUTH_KEY_COOKIE=test123 networks: - frontend - backend @@ -145,13 +156,15 @@ services: - message-bus - cache env_file: services.env + environment: + - AUTH_TOKEN_KEY=test123 + - AUTH_COOKIE_KEY=test123 networks: - datastore-reader - frontend - message-bus - auth - volumes: - - ./keys:/keys + cache: image: redis:latest networks: @@ -183,6 +196,26 @@ services: - backend - auth + manage-setup: + image: MANAGE_IMAGE + entrypoint: /root/entrypoint-setup + depends_on: + - manage + env_file: services.env + networks: + - backend + ifelse(ADMIN_SECRET_AVAILABLE, 0,secrets: + - admin) + + permission: + image: PERMISSION_IMAGE + depends_on: + - datastore-reader + env_file: services.env + networks: + - backend + - auth + # Setup: host <-uplink-> proxy <-frontend-> services that are reachable from the client <-backend-> services that are internal-only # There are special networks for some services only, e.g. postgres only for the postgresql, datastore reader and datastore writer networks: @@ -200,8 +233,6 @@ networks: auth: internal: true -dnl secrets: -dnl ifelse(ADMIN_SECRET_AVAILABLE, 0,os_admin: -dnl file: ./secrets/admin.env) -dnl ifelse(USER_SECRET_AVAILABLE, 0,os_user: -dnl file: ./secrets/user.env) +ifelse(ADMIN_SECRET_AVAILABLE, 0,secrets: + admin: + file: ./secrets/admin.env) diff --git a/docker/docker-stack.yml.m4 b/docker/docker-stack.yml.m4 index 640843363..60de1e15d 100644 --- a/docker/docker-stack.yml.m4 +++ b/docker/docker-stack.yml.m4 @@ -12,21 +12,48 @@ define(`read_env', `esyscmd(`printf "%s" "$$1"')') define(`ifenvelse', `ifelse(read_env(`$1'),, `$2', read_env(`$1'))') define(`BACKEND_IMAGE', -ifenvelse(`DOCKER_OPENSLIDES_BACKEND_NAME', openslides/openslides-server):dnl -ifenvelse(`DOCKER_OPENSLIDES_BACKEND_TAG', latest)) -define(`FRONTEND_IMAGE', -ifenvelse(`DOCKER_OPENSLIDES_FRONTEND_NAME', openslides/openslides-client):dnl -ifenvelse(`DOCKER_OPENSLIDES_FRONTEND_TAG', latest)) - -define(`PRIMARY_DB', `ifenvelse(`PGNODE_REPMGR_PRIMARY', pgnode1)') - -define(`PGBOUNCER_NODELIST', -`ifelse(read_env(`PGNODE_2_ENABLED'), 1, `,pgnode2')`'dnl -ifelse(read_env(`PGNODE_3_ENABLED'), 1, `,pgnode3')') +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_BACKEND_NAME', openslides-backend):dnl +ifenvelse(`DOCKER_OPENSLIDES_BACKEND_TAG', latest-4)) +define(`PROXY_IMAGE', +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_PROXY_NAME', openslides-proxy):dnl +ifenvelse(`DOCKER_OPENSLIDES_PROXY_TAG', latest-4)) +define(`CLIENT_IMAGE', +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_CLIENT_NAME', openslides-client):dnl +ifenvelse(`DOCKER_OPENSLIDES_CLIENT_TAG', latest-4)) +define(`AUTH_IMAGE', +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_AUTH_NAME', openslides-auth):dnl +ifenvelse(`DOCKER_OPENSLIDES_AUTH_TAG', latest-4)) +define(`AUTOUPDATE_IMAGE', +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_AUTOUPDATE_NAME', openslides-autoupdate):dnl +ifenvelse(`DOCKER_OPENSLIDES_AUTOUPDATE_TAG', latest-4)) +define(`DATASTORE_READER_IMAGE', +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_READER_NAME', openslides-datastore-reader):dnl +ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_READER_TAG', latest-4)) +define(`DATASTORE_WRITER_IMAGE', +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_WRITER_NAME', openslides-datastore-writer):dnl +ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_WRITER_TAG', latest-4)) +define(`MEDIA_IMAGE', +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_MEDIA_NAME', openslides-media):dnl +ifenvelse(`DOCKER_OPENSLIDES_MEDIA_TAG', latest-4)) +define(`MANAGE_IMAGE', +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_MANAGE_NAME', openslides-manage):dnl +ifenvelse(`DOCKER_OPENSLIDES_MANAGE_TAG', latest-4)) +define(`PERMISSION_IMAGE', +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_PERMISSION_NAME', openslides-permission):dnl +ifenvelse(`DOCKER_OPENSLIDES_PERMISSION_TAG', latest-4)) define(`PROJECT_DIR', ifdef(`PROJECT_DIR',PROJECT_DIR,.)) -define(`ADMIN_SECRET_AVAILABLE', `syscmd(`test -f 'PROJECT_DIR`/secrets/adminsecret.env')sysval') -define(`USER_SECRET_AVAILABLE', `syscmd(`test -f 'PROJECT_DIR`/secrets/usersecret.env')sysval') +define(`ADMIN_SECRET_AVAILABLE', `syscmd(`test -f 'PROJECT_DIR`/secrets/admin.env')sysval') divert(0)dnl dnl ---------------------------------------- # This configuration was created from a template file. Before making changes, @@ -35,242 +62,217 @@ dnl ---------------------------------------- # place for customizations instead. version: '3.4' -x-osserver: - &default-osserver - image: BACKEND_IMAGE - networks: - - front - - back -x-osserver-env: &default-osserver-env - AMOUNT_REPLICAS: ifenvelse(`REDIS_RO_SERVICE_REPLICAS', 3) - AUTOUPDATE_DELAY: ifenvelse(`AUTOUPDATE_DELAY', 1) - CONNECTION_POOL_LIMIT: ifenvelse(`CONNECTION_POOL_LIMIT', 100) - DATABASE_HOST: "ifenvelse(`DATABASE_HOST', pgbouncer)" - DATABASE_PASSWORD: "ifenvelse(`DATABASE_PASSWORD', openslides)" - DATABASE_PORT: ifenvelse(`DATABASE_PORT', 5432) - DATABASE_USER: "ifenvelse(`DATABASE_USER', openslides)" - DEFAULT_FROM_EMAIL: "ifenvelse(`DEFAULT_FROM_EMAIL', noreply@example.com)" - DJANGO_LOG_LEVEL: "ifenvelse(`DJANGO_LOG_LEVEL', INFO)" - EMAIL_HOST: "ifenvelse(`EMAIL_HOST', postfix)" - EMAIL_HOST_PASSWORD: "ifenvelse(`EMAIL_HOST_PASSWORD',)" - EMAIL_HOST_USER: "ifenvelse(`EMAIL_HOST_USER',)" - EMAIL_PORT: ifenvelse(`EMAIL_PORT', 25) - ENABLE_ELECTRONIC_VOTING: "ifenvelse(`ENABLE_ELECTRONIC_VOTING', False)" - ENABLE_SAML: "ifenvelse(`ENABLE_SAML', False)" - INSTANCE_DOMAIN: "ifenvelse(`INSTANCE_DOMAIN', http://example.com:8000)" - JITSI_DOMAIN: "ifenvelse(`JITSI_DOMAIN',)" - JITSI_ROOM_PASSWORD: "ifenvelse(`JITSI_ROOM_PASSWORD',)" - JITSI_ROOM_NAME: "ifenvelse(`JITSI_ROOM_NAME',)" - OPENSLIDES_LOG_LEVEL: "ifenvelse(`OPENSLIDES_LOG_LEVEL', INFO)" - REDIS_CHANNLES_HOST: "ifenvelse(`REDIS_CHANNLES_HOST', redis-channels)" - REDIS_CHANNLES_PORT: ifenvelse(`REDIS_CHANNLES_PORT', 6379) - REDIS_HOST: "ifenvelse(`REDIS_HOST', redis)" - REDIS_PORT: ifenvelse(`REDIS_PORT', 6379) - REDIS_SLAVE_HOST: "ifenvelse(`REDIS_SLAVE_HOST', redis-slave)" - REDIS_SLAVE_PORT: ifenvelse(`REDIS_SLAVE_PORT', 6379) - REDIS_SLAVE_WAIT_TIMEOUT: ifenvelse(`REDIS_SLAVE_WAIT_TIMEOUT', 10000) - RESET_PASSWORD_VERBOSE_ERRORS: "ifenvelse(`RESET_PASSWORD_VERBOSE_ERRORS', False)" -x-pgnode: &default-pgnode - image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-repmgr:latest - networks: - - dbnet - labels: - org.openslides.role: "postgres" - deploy: - replicas: 1 -x-pgnode-env: &default-pgnode-env - REPMGR_RECONNECT_ATTEMPTS: 30 - REPMGR_RECONNECT_INTERVAL: 10 - REPMGR_WAL_ARCHIVE: "ifenvelse(`PGNODE_WAL_ARCHIVING', on)" - services: - server: - << : *default-osserver - # Below is the default command. You can uncomment it to override the - # number of workers, for example: - # command: "gunicorn -w 8 --preload -b 0.0.0.0:8000 - # -k uvicorn.workers.UvicornWorker openslides.asgi:application" - # - # Uncomment the following line to use daphne instead of gunicorn: - # command: "daphne -b 0.0.0.0 -p 8000 openslides.asgi:application" - environment: - << : *default-osserver-env - secrets: - - django - ifelse(read_env(`ENABLE_SAML'), `True',- saml_cert - - saml_key - - saml_config) + proxy: + image: PROXY_IMAGE + networks: + - uplink + - frontend + ports: + - "127.0.0.1:ifenvelse(`EXTERNAL_HTTP_PORT', 8000):8000" deploy: restart_policy: condition: on-failure delay: 5s - replicas: ifenvelse(`OPENSLIDES_BACKEND_SERVICE_REPLICAS', 1) - - server-setup: - << : *default-osserver - entrypoint: /usr/local/sbin/entrypoint-db-setup - environment: - << : *default-osserver-env - secrets: - - django - ifelse(ADMIN_SECRET_AVAILABLE, 0,- os_admin) - ifelse(USER_SECRET_AVAILABLE, 0,- os_user) - ifelse(read_env(`ENABLE_SAML'), `True',- saml_cert - - saml_key - - saml_config) + replicas: ifenvelse(`OPENSLIDES_PROXY_REPLICAS', 1) client: - image: FRONTEND_IMAGE + image: CLIENT_IMAGE networks: - - front - ports: - - "0.0.0.0:ifenvelse(`EXTERNAL_HTTP_PORT', 8000):80" + - frontend + deploy: + restart_policy: + condition: on-failure + delay: 5s + replicas: ifenvelse(`OPENSLIDES_CLIENT_REPLICAS', 1) + + backend: + image: BACKEND_IMAGE + env_file: services.env + environment: + - AUTH_TOKEN_KEY=test123 + - AUTH_COOKIE_KEY=test123 + networks: + - frontend + - backend + deploy: + restart_policy: + condition: on-failure + delay: 5s + replicas: ifenvelse(`OPENSLIDES_BACKEND_REPLICAS', 1) + + datastore-reader: + image: DATASTORE_READER_IMAGE + env_file: services.env + environment: + - NUM_WORKERS=8 + networks: + - backend + - datastore-reader + - postgres + deploy: + restart_policy: + condition: on-failure + delay: 5s + replicas: ifenvelse(`OPENSLIDES_DATASTORE_READER_REPLICAS', 1) + + datastore-writer: + image: DATASTORE_WRITER_IMAGE + env_file: services.env + networks: + - backend + - postgres + - message-bus + environment: + - COMMAND=create_initial_data + - DATASTORE_INITIAL_DATA_FILE=/data/initial-data.json + volumes: + - ./initial-data.json:/data/initial-data.json deploy: - replicas: ifenvelse(`OPENSLIDES_FRONTEND_SERVICE_REPLICAS', 1) restart_policy: condition: on-failure delay: 5s - pgnode1: - << : *default-pgnode + postgres: + image: postgres:11 environment: - << : *default-pgnode-env - REPMGR_NODE_ID: 1 - REPMGR_PRIMARY: ifelse(PRIMARY_DB, pgnode1, `# This is the primary', PRIMARY_DB) + - POSTGRES_USER=openslides + - POSTGRES_PASSWORD=openslides + - POSTGRES_DB=openslides + networks: + - postgres deploy: - placement: - constraints: ifenvelse(`PGNODE_1_PLACEMENT_CONSTR', [node.labels.openslides-db == dbnode1]) - volumes: - - "dbdata1:/var/lib/postgresql" -ifelse(read_env(`PGNODE_2_ENABLED'), 1, `' - pgnode2: - << : *default-pgnode - environment: - << : *default-pgnode-env - REPMGR_NODE_ID: 2 - REPMGR_PRIMARY: ifelse(PRIMARY_DB, pgnode2, `# This is the primary', PRIMARY_DB) - deploy: - placement: - constraints: ifenvelse(`PGNODE_2_PLACEMENT_CONSTR', [node.labels.openslides-db == dbnode2]) - volumes: - - "dbdata2:/var/lib/postgresql") -ifelse(read_env(`PGNODE_3_ENABLED'), 1, `' - pgnode3: - << : *default-pgnode - environment: - << : *default-pgnode-env - REPMGR_NODE_ID: 3 - REPMGR_PRIMARY: ifelse(PRIMARY_DB, pgnode3, `# This is the primary', PRIMARY_DB) - deploy: - placement: - constraints: ifenvelse(`PGNODE_3_PLACEMENT_CONSTR', [node.labels.openslides-db == dbnode3]) - volumes: - - "dbdata3:/var/lib/postgresql") + restart_policy: + condition: on-failure + delay: 5s - pgbouncer: + autoupdate: + image: AUTOUPDATE_IMAGE + env_file: services.env environment: - - PG_NODE_LIST=pgnode1`'PGBOUNCER_NODELIST - image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-pgbouncer:latest + - AUTH_KEY_TOKEN=test123 + - AUTH_KEY_COOKIE=test123 networks: - back: - aliases: - - db - - postgres - dbnet: + - frontend + - backend + - message-bus deploy: restart_policy: condition: on-failure - delay: 10s - placement: - constraints: ifenvelse(`PGBOUNCER_PLACEMENT_CONSTR', [node.role == manager]) - postfix: - image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-postfix:latest + delay: 5s + replicas: ifenvelse(`OPENSLIDES_AUTOUPDATE_REPLICAS', 1) + + auth: + image: AUTH_IMAGE + env_file: services.env environment: - MYHOSTNAME: "ifenvelse(`POSTFIX_MYHOSTNAME', localhost)" - RELAYHOST: "ifenvelse(`POSTFIX_RELAYHOST', localhost)" + - AUTH_TOKEN_KEY=test123 + - AUTH_COOKIE_KEY=test123 networks: - - back + - datastore-reader + - frontend + - message-bus + - auth deploy: restart_policy: condition: on-failure delay: 5s - replicas: 1 - placement: - constraints: [node.role == manager] - redis: - image: redis:alpine + replicas: ifenvelse(`OPENSLIDES_AUTH_REPLICAS', 1) + + cache: + image: redis:latest networks: - back: - aliases: - - rediscache + - auth deploy: - replicas: 1 restart_policy: condition: on-failure delay: 5s - redis-slave: - image: redis:alpine - command: ["redis-server", "--save", "", "--slaveof", "redis", "6379"] + + message-bus: + image: redis:latest networks: - back: - aliases: - - rediscache-slave + - message-bus deploy: - replicas: ifenvelse(`REDIS_RO_SERVICE_REPLICAS', 3) - restart_policy: - condition: on-failure - delay: 5s - redis-channels: - image: redis:alpine - networks: - back: - deploy: - replicas: 1 restart_policy: condition: on-failure delay: 5s + media: - image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-media-service:latest - environment: - - CHECK_REQUEST_URL=server:8000/check-media/ + image: MEDIA_IMAGE + env_file: services.env + networks: + - frontend + - backend + - postgres deploy: - replicas: ifenvelse(`MEDIA_SERVICE_REPLICAS', 8) restart_policy: condition: on-failure - delay: 10s - networks: - front: - back: - # Override command to run more workers per task - # command: ["gunicorn", "-w", "4", "--preload", "-b", - # "0.0.0.0:8000", "src.mediaserver:app"] + delay: 5s + replicas: ifenvelse(`OPENSLIDES_MEDIA_REPLICAS', 1) -volumes: - dbdata1: -ifelse(read_env(`PGNODE_2_ENABLED'), 1, ` dbdata2:') -ifelse(read_env(`PGNODE_3_ENABLED'), 1, ` dbdata3:') + manage: + image: MANAGE_IMAGE + env_file: services.env + networks: + - backend + - auth + deploy: + restart_policy: + condition: on-failure + delay: 5s + + manage-setup: + image: MANAGE_IMAGE + entrypoint: /root/entrypoint-setup + env_file: services.env + networks: + - backend + ifelse(ADMIN_SECRET_AVAILABLE, 0,secrets: + - admin) + deploy: + restart_policy: + condition: on-failure + delay: 5s + + permission: + image: PERMISSION_IMAGE + env_file: services.env + networks: + - backend + - auth + deploy: + restart_policy: + condition: on-failure + delay: 5s + replicas: ifenvelse(`OPENSLIDES_PERMISSION_REPLICAS', 1) networks: - front: - back: + uplink: + frontend: driver_opts: encrypted: "" - dbnet: + internal: true + backend: driver_opts: encrypted: "" + internal: true + postgres: + driver_opts: + encrypted: "" + internal: true + datastore-reader: + driver_opts: + encrypted: "" + internal: true + message-bus: + driver_opts: + encrypted: "" + internal: true + auth: + driver_opts: + encrypted: "" + internal: true -secrets: - django: - file: ./secrets/django.env - ifelse(ADMIN_SECRET_AVAILABLE, 0,os_admin: - file: ./secrets/adminsecret.env) - ifelse(USER_SECRET_AVAILABLE, 0,os_user: - file: ./secrets/usersecret.env) - ifelse(read_env(`ENABLE_SAML'), `True', saml_cert: - file: ./secrets/saml/sp.crt - saml_key: - file: ./secrets/saml/sp.key - saml_config: - file: ./secrets/saml/saml_settings.json) - -# vim: set sw=2 et: +ifelse(ADMIN_SECRET_AVAILABLE, 0,secrets: + admin: + file: ./secrets/admin.env) diff --git a/docker/services.env b/docker/services.env index cd51ccfea..08b5c8c5d 100644 --- a/docker/services.env +++ b/docker/services.env @@ -12,6 +12,9 @@ ACTION_PORT=9002 PRESENTER_HOST=backend PRESENTER_PORT=9003 +AUTOUPDATE_HOST=autoupdate +AUTOUPDATE_PORT=9012 + PERMISSION_HOST=permission PERMISSION_PORT=9005 @@ -24,3 +27,6 @@ MEDIA_HOST=media MEDIA_PORT=9006 MEDIA_DATABASE_HOST=postgres MEDIA_DATABASE_NAME=openslides + +MANAGE_HOST=manage +MANAGE_PORT=9008 diff --git a/openslides-autoupdate-service b/openslides-autoupdate-service index fb6e25d7a..d28465081 160000 --- a/openslides-autoupdate-service +++ b/openslides-autoupdate-service @@ -1 +1 @@ -Subproject commit fb6e25d7a88ec8202b5080b5563e95451b6071c3 +Subproject commit d284650811d2ae0bb512c4db268952862b5722b4 diff --git a/openslides-backend b/openslides-backend index acef4bbf4..a24b735b4 160000 --- a/openslides-backend +++ b/openslides-backend @@ -1 +1 @@ -Subproject commit acef4bbf409f53f90f34f68a6ab2c5794f023981 +Subproject commit a24b735b482be4ff5f5425f2e92dd85f805f353d diff --git a/openslides-client b/openslides-client index 88e620ec4..412741773 160000 --- a/openslides-client +++ b/openslides-client @@ -1 +1 @@ -Subproject commit 88e620ec4efd634f8fbbffad9c35d4a541a69fcd +Subproject commit 412741773c15a0d4515c12910416a16a50faada8 diff --git a/openslides-manage-service b/openslides-manage-service index a40e5bd94..df61ded33 160000 --- a/openslides-manage-service +++ b/openslides-manage-service @@ -1 +1 @@ -Subproject commit a40e5bd940c41a1eb98533a01f046c0061e2d866 +Subproject commit df61ded339c1cb07e46876d4e463c5f9812d25cc diff --git a/openslides-permission-service b/openslides-permission-service index e30d35768..c33b68b0c 160000 --- a/openslides-permission-service +++ b/openslides-permission-service @@ -1 +1 @@ -Subproject commit e30d357684526c139a397e11ed77ab5befcf2598 +Subproject commit c33b68b0c701f7fc503096c1d89d6c82e5a50232 diff --git a/proxy/Caddyfile b/proxy/Caddyfile index 752e2fafd..b9113a306 100644 --- a/proxy/Caddyfile +++ b/proxy/Caddyfile @@ -1,12 +1,12 @@ import endpoint - reverse_proxy /system/action/* backend:9002 - reverse_proxy /system/presenter/* backend:9003 - reverse_proxy /system/autoupdate/* autoupdate:9012 { + reverse_proxy /system/action* backend:9002 + reverse_proxy /system/presenter* backend:9003 + reverse_proxy /system/autoupdate* autoupdate:9012 { flush_interval -1 } - reverse_proxy /system/auth/* auth:9004 - reverse_proxy /system/media/* media:9006 + reverse_proxy /system/auth* auth:9004 + reverse_proxy /system/media* media:9006 reverse_proxy client:9001