OS4: prod setup

.gitmodules

@@ -1,31 +1,31 @@
 [submodule "openslides-datastore-service"]
 	path = openslides-datastore-service
-	url = git@github.com:OpenSlides/openslides-datastore-service.git
+	url = https://github.com/OpenSlides/openslides-datastore-service.git
 	branch = master
 [submodule "openslides-client"]
 	path = openslides-client
-	url = git@github.com:OpenSlides/openslides-client.git
+	url = https://github.com/OpenSlides/openslides-client.git
 	branch = master
 [submodule "openslides-backend"]
 	path = openslides-backend
-	url = git@github.com:OpenSlides/openslides-backend.git
+	url = https://github.com/OpenSlides/openslides-backend.git
 	branch = master
 [submodule "openslides-autoupdate-service"]
 	path = openslides-autoupdate-service
-	url = git@github.com:OpenSlides/openslides-autoupdate-service.git
+	url = https://github.com/OpenSlides/openslides-autoupdate-service.git
 [submodule "openslides-auth-service"]
 	path = openslides-auth-service
-	url = git@github.com:OpenSlides/openslides-auth-service.git
+	url = https://github.com/OpenSlides/openslides-auth-service.git
 	branch = master
 [submodule "openslides-media-service"]
 	path = openslides-media-service
-	url = git@github.com:OpenSlides/openslides-media-service.git
+	url = https://github.com/OpenSlides/openslides-media-service.git
 	branch = openslides4-dev
 [submodule "openslides-permission-service"]
 	path = openslides-permission-service
-	url = git@github.com:OpenSlides/openslides-permission-service.git
+	url = https://github.com/OpenSlides/openslides-permission-service.git
 	branch = master
 [submodule "openslides-manage-service"]
 	path = openslides-manage-service
-	url = git@github.com:OpenSlides/openslides-manage-service.git
+	url = https://github.com/OpenSlides/openslides-manage-service.git
 	branch = main

docker/build.sh

@@ -10,6 +10,8 @@ TARGETS=(
   [backend]="$HOME/../openslides-backend/"
   [auth]="$HOME/../openslides-auth-service/"
   [autoupdate]="$HOME/../openslides-autoupdate-service/"
+  [permission]="$HOME/../openslides-permission-service/"
+  [manage]="$HOME/../openslides-manage-service/"
   [datastore-reader]="$HOME/../openslides-datastore-service/reader"
   [datastore-writer]="$HOME/../openslides-datastore-service/writer"
   [media]="$HOME/../openslides-media-service/"
@@ -19,11 +21,11 @@ TARGETS=(
 )
 
 DOCKER_REPOSITORY="openslides"
-DOCKER_TAG="latest"
+DOCKER_TAG="latest-4"
 CONFIG="/etc/osinstancectl"
 OPTIONS=()
 BUILT_IMAGES=()
-DEFAULT_TARGETS=(proxy client backend auth autoupdate datastore-reader datastore-writer media)
+DEFAULT_TARGETS=(proxy client backend auth autoupdate permission manage datastore-reader datastore-writer media)
 
 usage() {
   cat << EOF

docker/docker-compose.dev.yml

@@ -8,6 +8,7 @@ services:
     environment:
       - DATASTORE_ENABLE_DEV_ENVIRONMENT=1
       - NUM_WORKERS=8
+      - OPENSLIDES_DEVELOPMENT=1
     volumes:
       - ../openslides-datastore-service/shared/shared:/app/shared
       - ../openslides-datastore-service/reader/reader:/app/reader
@@ -27,6 +28,7 @@ services:
       - DATASTORE_ENABLE_DEV_ENVIRONMENT=1
       - COMMAND=create_initial_data
       - DATASTORE_INITIAL_DATA_FILE=https://raw.githubusercontent.com/OpenSlides/OpenSlides/openslides4-dev/docs/example-data.json
+      - OPENSLIDES_DEVELOPMENT=1
     ports:
       - 9011:9011
   postgres:
@@ -41,6 +43,8 @@ services:
       - backend
       - autoupdate
     env_file: services.env
+    environment:
+      - OPENSLIDES_DEVELOPMENT=1
     volumes:
       - ../openslides-client/client/src:/app/src
   backend:
@@ -51,6 +55,8 @@ services:
       - auth
       - permission
     env_file: services.env
+    environment:
+      - OPENSLIDES_DEVELOPMENT=1
     volumes:
       - ../openslides-backend/openslides_backend:/app/openslides_backend
     ports:
@@ -61,6 +67,8 @@ services:
       - datastore-reader
       - message-bus
     env_file: services.env
+    environment:
+      - OPENSLIDES_DEVELOPMENT=1
     volumes:
       - ../openslides-autoupdate-service/cmd:/root/cmd
       - ../openslides-autoupdate-service/internal:/root/internal
@@ -69,6 +77,8 @@ services:
     depends_on:
       - datastore-reader
     env_file: services.env
+    environment:
+      - OPENSLIDES_DEVELOPMENT=1
     volumes:
       - ../openslides-permission-service/cmd:/app/cmd
       - ../openslides-permission-service/internal:/app/internal
@@ -79,6 +89,8 @@ services:
       - datastore-reader
       - cache
     env_file: services.env
+    environment:
+      - OPENSLIDES_DEVELOPMENT=1
     volumes:
       - ../openslides-auth-service/auth/src:/app/src
     ports:
@@ -91,6 +103,8 @@ services:
       - backend
       - postgres
     env_file: services.env
+    environment:
+      - OPENSLIDES_DEVELOPMENT=1
     volumes:
       - ../openslides-media-service/src:/app/src
   manage:
@@ -99,6 +113,8 @@ services:
     - auth
     - datastore-writer
     env_file: services.env
+    environment:
+      - OPENSLIDES_DEVELOPMENT=1
     ports:
     - "8001:8001"
   message-bus:

docker/docker-compose.yml.m4

@@ -14,43 +14,46 @@ define(`ifenvelse', `ifelse(read_env(`$1'),, `$2', read_env(`$1'))')
 define(`BACKEND_IMAGE',
 ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
 ifenvelse(`DOCKER_OPENSLIDES_BACKEND_NAME', openslides-backend):dnl
-ifenvelse(`DOCKER_OPENSLIDES_BACKEND_TAG', latest))
+ifenvelse(`DOCKER_OPENSLIDES_BACKEND_TAG', latest-4))
 define(`PROXY_IMAGE',
 ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
 ifenvelse(`DOCKER_OPENSLIDES_PROXY_NAME', openslides-proxy):dnl
-ifenvelse(`DOCKER_OPENSLIDES_PROXY_TAG', latest))
+ifenvelse(`DOCKER_OPENSLIDES_PROXY_TAG', latest-4))
 define(`CLIENT_IMAGE',
 ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
 ifenvelse(`DOCKER_OPENSLIDES_CLIENT_NAME', openslides-client):dnl
-ifenvelse(`DOCKER_OPENSLIDES_CLIENT_TAG', latest))
+ifenvelse(`DOCKER_OPENSLIDES_CLIENT_TAG', latest-4))
 define(`AUTH_IMAGE',
 ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
 ifenvelse(`DOCKER_OPENSLIDES_AUTH_NAME', openslides-auth):dnl
-ifenvelse(`DOCKER_OPENSLIDES_AUTH_TAG', latest))
+ifenvelse(`DOCKER_OPENSLIDES_AUTH_TAG', latest-4))
 define(`AUTOUPDATE_IMAGE',
 ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
 ifenvelse(`DOCKER_OPENSLIDES_AUTOUPDATE_NAME', openslides-autoupdate):dnl
-ifenvelse(`DOCKER_OPENSLIDES_AUTOUPDATE_TAG', latest))
+ifenvelse(`DOCKER_OPENSLIDES_AUTOUPDATE_TAG', latest-4))
 define(`DATASTORE_READER_IMAGE',
 ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
 ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_READER_NAME', openslides-datastore-reader):dnl
-ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_READER_TAG', latest))
+ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_READER_TAG', latest-4))
 define(`DATASTORE_WRITER_IMAGE',
 ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
 ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_WRITER_NAME', openslides-datastore-writer):dnl
-ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_WRITER_TAG', latest))
+ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_WRITER_TAG', latest-4))
 define(`MEDIA_IMAGE',
 ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
 ifenvelse(`DOCKER_OPENSLIDES_MEDIA_NAME', openslides-media):dnl
-ifenvelse(`DOCKER_OPENSLIDES_MEDIA_TAG', latest))
+ifenvelse(`DOCKER_OPENSLIDES_MEDIA_TAG', latest-4))
 define(`MANAGE_IMAGE',
 ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
 ifenvelse(`DOCKER_OPENSLIDES_MANAGE_NAME', openslides-manage):dnl
-ifenvelse(`DOCKER_OPENSLIDES_MANAGE_TAG', latest))
+ifenvelse(`DOCKER_OPENSLIDES_MANAGE_TAG', latest-4))
+define(`PERMISSION_IMAGE',
+ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
+ifenvelse(`DOCKER_OPENSLIDES_PERMISSION_NAME', openslides-permission):dnl
+ifenvelse(`DOCKER_OPENSLIDES_PERMISSION_TAG', latest-4))
 
 define(`PROJECT_DIR', ifdef(`PROJECT_DIR',PROJECT_DIR,.))
 define(`ADMIN_SECRET_AVAILABLE', `syscmd(`test -f 'PROJECT_DIR`/secrets/admin.env')sysval')
-define(`USER_SECRET_AVAILABLE', `syscmd(`test -f 'PROJECT_DIR`/secrets/user.env')sysval')
 divert(0)dnl
 dnl ----------------------------------------
 # This configuration was created from a template file.  Before making changes,
@@ -88,6 +91,9 @@ services:
       - datastore-reader
       - datastore-writer
     env_file: services.env
+    environment:
+      - AUTH_TOKEN_KEY=test123
+      - AUTH_COOKIE_KEY=test123
     networks:
       - frontend
       - backend
@@ -103,6 +109,7 @@ services:
       - backend
       - datastore-reader
       - postgres
+
   datastore-writer:
     image: DATASTORE_WRITER_IMAGE
     depends_on:
@@ -118,6 +125,7 @@ services:
       - DATASTORE_INITIAL_DATA_FILE=/data/initial-data.json
     volumes:
       - ./initial-data.json:/data/initial-data.json
+
   postgres:
     image: postgres:11
     environment:
@@ -133,6 +141,9 @@ services:
       - datastore-reader
       - message-bus
     env_file: services.env
+    environment:
+      - AUTH_KEY_TOKEN=test123
+      - AUTH_KEY_COOKIE=test123
     networks:
       - frontend
       - backend
@@ -145,13 +156,15 @@ services:
       - message-bus
       - cache
     env_file: services.env
+    environment:
+      - AUTH_TOKEN_KEY=test123
+      - AUTH_COOKIE_KEY=test123
     networks:
       - datastore-reader
       - frontend
       - message-bus
       - auth
-    volumes:
+
-      - ./keys:/keys
   cache:
     image: redis:latest
     networks:
@@ -183,6 +196,26 @@ services:
     - backend
     - auth
 
+  manage-setup:
+    image: MANAGE_IMAGE
+    entrypoint: /root/entrypoint-setup
+    depends_on:
+    - manage
+    env_file: services.env
+    networks:
+    - backend
+    ifelse(ADMIN_SECRET_AVAILABLE, 0,secrets:
+      - admin)
+
+  permission:
+    image: PERMISSION_IMAGE
+    depends_on:
+    - datastore-reader
+    env_file: services.env
+    networks:
+    - backend
+    - auth
+
 # Setup: host <-uplink-> proxy <-frontend-> services that are reachable from the client <-backend-> services that are internal-only
 # There are special networks for some services only, e.g. postgres only for the postgresql, datastore reader and datastore writer
 networks:
@@ -200,8 +233,6 @@ networks:
   auth:
     internal: true
 
-dnl secrets:
+ifelse(ADMIN_SECRET_AVAILABLE, 0,secrets:
-dnl   ifelse(ADMIN_SECRET_AVAILABLE, 0,os_admin:
+  admin:
-dnl     file: ./secrets/admin.env)
+    file: ./secrets/admin.env)
-dnl   ifelse(USER_SECRET_AVAILABLE, 0,os_user:
-dnl     file: ./secrets/user.env)

docker/docker-stack.yml.m4

@@ -12,21 +12,48 @@ define(`read_env', `esyscmd(`printf "%s" "$$1"')')
 define(`ifenvelse', `ifelse(read_env(`$1'),, `$2', read_env(`$1'))')
 
 define(`BACKEND_IMAGE',
-ifenvelse(`DOCKER_OPENSLIDES_BACKEND_NAME', openslides/openslides-server):dnl
+ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
-ifenvelse(`DOCKER_OPENSLIDES_BACKEND_TAG', latest))
+ifenvelse(`DOCKER_OPENSLIDES_BACKEND_NAME', openslides-backend):dnl
-define(`FRONTEND_IMAGE',
+ifenvelse(`DOCKER_OPENSLIDES_BACKEND_TAG', latest-4))
-ifenvelse(`DOCKER_OPENSLIDES_FRONTEND_NAME', openslides/openslides-client):dnl
+define(`PROXY_IMAGE',
-ifenvelse(`DOCKER_OPENSLIDES_FRONTEND_TAG', latest))
+ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
-
+ifenvelse(`DOCKER_OPENSLIDES_PROXY_NAME', openslides-proxy):dnl
-define(`PRIMARY_DB', `ifenvelse(`PGNODE_REPMGR_PRIMARY', pgnode1)')
+ifenvelse(`DOCKER_OPENSLIDES_PROXY_TAG', latest-4))
-
+define(`CLIENT_IMAGE',
-define(`PGBOUNCER_NODELIST',
+ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
-`ifelse(read_env(`PGNODE_2_ENABLED'), 1, `,pgnode2')`'dnl
+ifenvelse(`DOCKER_OPENSLIDES_CLIENT_NAME', openslides-client):dnl
-ifelse(read_env(`PGNODE_3_ENABLED'), 1, `,pgnode3')')
+ifenvelse(`DOCKER_OPENSLIDES_CLIENT_TAG', latest-4))
+define(`AUTH_IMAGE',
+ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
+ifenvelse(`DOCKER_OPENSLIDES_AUTH_NAME', openslides-auth):dnl
+ifenvelse(`DOCKER_OPENSLIDES_AUTH_TAG', latest-4))
+define(`AUTOUPDATE_IMAGE',
+ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
+ifenvelse(`DOCKER_OPENSLIDES_AUTOUPDATE_NAME', openslides-autoupdate):dnl
+ifenvelse(`DOCKER_OPENSLIDES_AUTOUPDATE_TAG', latest-4))
+define(`DATASTORE_READER_IMAGE',
+ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
+ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_READER_NAME', openslides-datastore-reader):dnl
+ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_READER_TAG', latest-4))
+define(`DATASTORE_WRITER_IMAGE',
+ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
+ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_WRITER_NAME', openslides-datastore-writer):dnl
+ifenvelse(`DOCKER_OPENSLIDES_DATASTORE_WRITER_TAG', latest-4))
+define(`MEDIA_IMAGE',
+ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
+ifenvelse(`DOCKER_OPENSLIDES_MEDIA_NAME', openslides-media):dnl
+ifenvelse(`DOCKER_OPENSLIDES_MEDIA_TAG', latest-4))
+define(`MANAGE_IMAGE',
+ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
+ifenvelse(`DOCKER_OPENSLIDES_MANAGE_NAME', openslides-manage):dnl
+ifenvelse(`DOCKER_OPENSLIDES_MANAGE_TAG', latest-4))
+define(`PERMISSION_IMAGE',
+ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
+ifenvelse(`DOCKER_OPENSLIDES_PERMISSION_NAME', openslides-permission):dnl
+ifenvelse(`DOCKER_OPENSLIDES_PERMISSION_TAG', latest-4))
 
 define(`PROJECT_DIR', ifdef(`PROJECT_DIR',PROJECT_DIR,.))
-define(`ADMIN_SECRET_AVAILABLE', `syscmd(`test -f 'PROJECT_DIR`/secrets/adminsecret.env')sysval')
+define(`ADMIN_SECRET_AVAILABLE', `syscmd(`test -f 'PROJECT_DIR`/secrets/admin.env')sysval')
-define(`USER_SECRET_AVAILABLE', `syscmd(`test -f 'PROJECT_DIR`/secrets/usersecret.env')sysval')
 divert(0)dnl
 dnl ----------------------------------------
 # This configuration was created from a template file.  Before making changes,
@@ -35,242 +62,217 @@ dnl ----------------------------------------
 # place for customizations instead.
 version: '3.4'
 
-x-osserver:
-  &default-osserver
-  image: BACKEND_IMAGE
-  networks:
-    - front
-    - back
-x-osserver-env: &default-osserver-env
-    AMOUNT_REPLICAS: ifenvelse(`REDIS_RO_SERVICE_REPLICAS', 3)
-    AUTOUPDATE_DELAY: ifenvelse(`AUTOUPDATE_DELAY', 1)
-    CONNECTION_POOL_LIMIT: ifenvelse(`CONNECTION_POOL_LIMIT', 100)
-    DATABASE_HOST: "ifenvelse(`DATABASE_HOST', pgbouncer)"
-    DATABASE_PASSWORD: "ifenvelse(`DATABASE_PASSWORD', openslides)"
-    DATABASE_PORT: ifenvelse(`DATABASE_PORT', 5432)
-    DATABASE_USER: "ifenvelse(`DATABASE_USER', openslides)"
-    DEFAULT_FROM_EMAIL: "ifenvelse(`DEFAULT_FROM_EMAIL', noreply@example.com)"
-    DJANGO_LOG_LEVEL: "ifenvelse(`DJANGO_LOG_LEVEL', INFO)"
-    EMAIL_HOST: "ifenvelse(`EMAIL_HOST', postfix)"
-    EMAIL_HOST_PASSWORD: "ifenvelse(`EMAIL_HOST_PASSWORD',)"
-    EMAIL_HOST_USER: "ifenvelse(`EMAIL_HOST_USER',)"
-    EMAIL_PORT: ifenvelse(`EMAIL_PORT', 25)
-    ENABLE_ELECTRONIC_VOTING: "ifenvelse(`ENABLE_ELECTRONIC_VOTING', False)"
-    ENABLE_SAML: "ifenvelse(`ENABLE_SAML', False)"
-    INSTANCE_DOMAIN: "ifenvelse(`INSTANCE_DOMAIN', http://example.com:8000)"
-    JITSI_DOMAIN: "ifenvelse(`JITSI_DOMAIN',)"
-    JITSI_ROOM_PASSWORD: "ifenvelse(`JITSI_ROOM_PASSWORD',)"
-    JITSI_ROOM_NAME: "ifenvelse(`JITSI_ROOM_NAME',)"
-    OPENSLIDES_LOG_LEVEL: "ifenvelse(`OPENSLIDES_LOG_LEVEL', INFO)"
-    REDIS_CHANNLES_HOST: "ifenvelse(`REDIS_CHANNLES_HOST', redis-channels)"
-    REDIS_CHANNLES_PORT: ifenvelse(`REDIS_CHANNLES_PORT', 6379)
-    REDIS_HOST: "ifenvelse(`REDIS_HOST', redis)"
-    REDIS_PORT: ifenvelse(`REDIS_PORT', 6379)
-    REDIS_SLAVE_HOST: "ifenvelse(`REDIS_SLAVE_HOST', redis-slave)"
-    REDIS_SLAVE_PORT: ifenvelse(`REDIS_SLAVE_PORT', 6379)
-    REDIS_SLAVE_WAIT_TIMEOUT: ifenvelse(`REDIS_SLAVE_WAIT_TIMEOUT', 10000)
-    RESET_PASSWORD_VERBOSE_ERRORS: "ifenvelse(`RESET_PASSWORD_VERBOSE_ERRORS', False)"
-x-pgnode: &default-pgnode
-  image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-repmgr:latest
-  networks:
-    - dbnet
-  labels:
-    org.openslides.role: "postgres"
-  deploy:
-    replicas: 1
-x-pgnode-env: &default-pgnode-env
-  REPMGR_RECONNECT_ATTEMPTS: 30
-  REPMGR_RECONNECT_INTERVAL: 10
-  REPMGR_WAL_ARCHIVE: "ifenvelse(`PGNODE_WAL_ARCHIVING', on)"
-
 services:
-  server:
+  proxy:
-    << : *default-osserver
+    image: PROXY_IMAGE
-    # Below is the default command.  You can uncomment it to override the
+    networks:
-    # number of workers, for example:
+      - uplink
-    # command: "gunicorn -w 8 --preload -b 0.0.0.0:8000
+      - frontend
-    #   -k uvicorn.workers.UvicornWorker openslides.asgi:application"
+    ports:
-    #
+      - "127.0.0.1:ifenvelse(`EXTERNAL_HTTP_PORT', 8000):8000"
-    # Uncomment the following line to use daphne instead of gunicorn:
-    # command: "daphne -b 0.0.0.0 -p 8000 openslides.asgi:application"
-    environment:
-      << : *default-osserver-env
-    secrets:
-      - django
-      ifelse(read_env(`ENABLE_SAML'), `True',- saml_cert
-      - saml_key
-      - saml_config)
     deploy:
       restart_policy:
         condition: on-failure
         delay: 5s
-      replicas: ifenvelse(`OPENSLIDES_BACKEND_SERVICE_REPLICAS', 1)
+      replicas: ifenvelse(`OPENSLIDES_PROXY_REPLICAS', 1)
-
-  server-setup:
-    << : *default-osserver
-    entrypoint: /usr/local/sbin/entrypoint-db-setup
-    environment:
-      << : *default-osserver-env
-    secrets:
-      - django
-      ifelse(ADMIN_SECRET_AVAILABLE, 0,- os_admin)
-      ifelse(USER_SECRET_AVAILABLE, 0,- os_user)
-      ifelse(read_env(`ENABLE_SAML'), `True',- saml_cert
-      - saml_key
-      - saml_config)
 
   client:
-    image: FRONTEND_IMAGE
+    image: CLIENT_IMAGE
     networks:
-      - front
+      - frontend
-    ports:
-      - "0.0.0.0:ifenvelse(`EXTERNAL_HTTP_PORT', 8000):80"
     deploy:
-      replicas: ifenvelse(`OPENSLIDES_FRONTEND_SERVICE_REPLICAS', 1)
       restart_policy:
         condition: on-failure
         delay: 5s
+      replicas: ifenvelse(`OPENSLIDES_CLIENT_REPLICAS', 1)
 
-  pgnode1:
+  backend:
-    << : *default-pgnode
+    image: BACKEND_IMAGE
+    env_file: services.env
     environment:
-      << : *default-pgnode-env
+      - AUTH_TOKEN_KEY=test123
-      REPMGR_NODE_ID: 1
+      - AUTH_COOKIE_KEY=test123
-      REPMGR_PRIMARY: ifelse(PRIMARY_DB, pgnode1, `# This is the primary', PRIMARY_DB)
-    deploy:
-      placement:
-        constraints: ifenvelse(`PGNODE_1_PLACEMENT_CONSTR', [node.labels.openslides-db == dbnode1])
-    volumes:
-      - "dbdata1:/var/lib/postgresql"
-ifelse(read_env(`PGNODE_2_ENABLED'), 1, `'
-  pgnode2:
-    << : *default-pgnode
-    environment:
-      << : *default-pgnode-env
-      REPMGR_NODE_ID: 2
-      REPMGR_PRIMARY: ifelse(PRIMARY_DB, pgnode2, `# This is the primary', PRIMARY_DB)
-    deploy:
-      placement:
-        constraints: ifenvelse(`PGNODE_2_PLACEMENT_CONSTR', [node.labels.openslides-db == dbnode2])
-    volumes:
-      - "dbdata2:/var/lib/postgresql")
-ifelse(read_env(`PGNODE_3_ENABLED'), 1, `'
-  pgnode3:
-    << : *default-pgnode
-    environment:
-      << : *default-pgnode-env
-      REPMGR_NODE_ID: 3
-      REPMGR_PRIMARY: ifelse(PRIMARY_DB, pgnode3, `# This is the primary', PRIMARY_DB)
-    deploy:
-      placement:
-        constraints: ifenvelse(`PGNODE_3_PLACEMENT_CONSTR', [node.labels.openslides-db == dbnode3])
-    volumes:
-      - "dbdata3:/var/lib/postgresql")
-
-  pgbouncer:
-    environment:
-      - PG_NODE_LIST=pgnode1`'PGBOUNCER_NODELIST
-    image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-pgbouncer:latest
     networks:
-      back:
+      - frontend
-        aliases:
+      - backend
-          - db
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+      replicas: ifenvelse(`OPENSLIDES_BACKEND_REPLICAS', 1)
+
+  datastore-reader:
+    image: DATASTORE_READER_IMAGE
+    env_file: services.env
+    environment:
+      - NUM_WORKERS=8
+    networks:
+      - backend
+      - datastore-reader
       - postgres
-      dbnet:
-    deploy:
-      restart_policy:
-        condition: on-failure
-        delay: 10s
-      placement:
-        constraints: ifenvelse(`PGBOUNCER_PLACEMENT_CONSTR', [node.role == manager])
-  postfix:
-    image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-postfix:latest
-    environment:
-      MYHOSTNAME: "ifenvelse(`POSTFIX_MYHOSTNAME', localhost)"
-      RELAYHOST: "ifenvelse(`POSTFIX_RELAYHOST', localhost)"
-    networks:
-      - back
     deploy:
       restart_policy:
         condition: on-failure
         delay: 5s
-      replicas: 1
+      replicas: ifenvelse(`OPENSLIDES_DATASTORE_READER_REPLICAS', 1)
-      placement:
-        constraints: [node.role == manager]
-  redis:
-    image: redis:alpine
-    networks:
-      back:
-        aliases:
-          - rediscache
-    deploy:
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-        delay: 5s
-  redis-slave:
-    image: redis:alpine
-    command: ["redis-server", "--save", "", "--slaveof", "redis", "6379"]
-    networks:
-      back:
-        aliases:
-          - rediscache-slave
-    deploy:
-      replicas: ifenvelse(`REDIS_RO_SERVICE_REPLICAS', 3)
-      restart_policy:
-        condition: on-failure
-        delay: 5s
-  redis-channels:
-    image: redis:alpine
-    networks:
-      back:
-    deploy:
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-        delay: 5s
-  media:
-    image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-media-service:latest
-    environment:
-      - CHECK_REQUEST_URL=server:8000/check-media/
-    deploy:
-      replicas: ifenvelse(`MEDIA_SERVICE_REPLICAS', 8)
-      restart_policy:
-        condition: on-failure
-        delay: 10s
-    networks:
-      front:
-      back:
-    # Override command to run more workers per task
-    # command: ["gunicorn", "-w", "4", "--preload", "-b",
-    #   "0.0.0.0:8000", "src.mediaserver:app"]
 
-volumes:
+  datastore-writer:
-  dbdata1:
+    image: DATASTORE_WRITER_IMAGE
-ifelse(read_env(`PGNODE_2_ENABLED'), 1, `  dbdata2:')
+    env_file: services.env
-ifelse(read_env(`PGNODE_3_ENABLED'), 1, `  dbdata3:')
+    networks:
+      - backend
+      - postgres
+      - message-bus
+    environment:
+      - COMMAND=create_initial_data
+      - DATASTORE_INITIAL_DATA_FILE=/data/initial-data.json
+    volumes:
+      - ./initial-data.json:/data/initial-data.json
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+
+  postgres:
+    image: postgres:11
+    environment:
+      - POSTGRES_USER=openslides
+      - POSTGRES_PASSWORD=openslides
+      - POSTGRES_DB=openslides
+    networks:
+      - postgres
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+
+  autoupdate:
+    image: AUTOUPDATE_IMAGE
+    env_file: services.env
+    environment:
+      - AUTH_KEY_TOKEN=test123
+      - AUTH_KEY_COOKIE=test123
+    networks:
+      - frontend
+      - backend
+      - message-bus
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+      replicas: ifenvelse(`OPENSLIDES_AUTOUPDATE_REPLICAS', 1)
+
+  auth:
+    image: AUTH_IMAGE
+    env_file: services.env
+    environment:
+      - AUTH_TOKEN_KEY=test123
+      - AUTH_COOKIE_KEY=test123
+    networks:
+      - datastore-reader
+      - frontend
+      - message-bus
+      - auth
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+      replicas: ifenvelse(`OPENSLIDES_AUTH_REPLICAS', 1)
+
+  cache:
+    image: redis:latest
+    networks:
+      - auth
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+
+  message-bus:
+    image: redis:latest
+    networks:
+      - message-bus
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+
+  media:
+    image: MEDIA_IMAGE
+    env_file: services.env
+    networks:
+      - frontend
+      - backend
+      - postgres
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+      replicas: ifenvelse(`OPENSLIDES_MEDIA_REPLICAS', 1)
+
+  manage:
+    image: MANAGE_IMAGE
+    env_file: services.env
+    networks:
+    - backend
+    - auth
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+
+  manage-setup:
+    image: MANAGE_IMAGE
+    entrypoint: /root/entrypoint-setup
+    env_file: services.env
+    networks:
+    - backend
+    ifelse(ADMIN_SECRET_AVAILABLE, 0,secrets:
+      - admin)
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+
+  permission:
+    image: PERMISSION_IMAGE
+    env_file: services.env
+    networks:
+    - backend
+    - auth
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+      replicas: ifenvelse(`OPENSLIDES_PERMISSION_REPLICAS', 1)
 
 networks:
-  front:
+  uplink:
-  back:
+  frontend:
     driver_opts:
       encrypted: ""
-  dbnet:
+    internal: true
+  backend:
     driver_opts:
       encrypted: ""
+    internal: true
+  postgres:
+    driver_opts:
+      encrypted: ""
+    internal: true
+  datastore-reader:
+    driver_opts:
+      encrypted: ""
+    internal: true
+  message-bus:
+    driver_opts:
+      encrypted: ""
+    internal: true
+  auth:
+    driver_opts:
+      encrypted: ""
+    internal: true
 
-secrets:
+ifelse(ADMIN_SECRET_AVAILABLE, 0,secrets:
-  django:
+  admin:
-    file: ./secrets/django.env
+    file: ./secrets/admin.env)
-  ifelse(ADMIN_SECRET_AVAILABLE, 0,os_admin:
-    file: ./secrets/adminsecret.env)
-  ifelse(USER_SECRET_AVAILABLE, 0,os_user:
-    file: ./secrets/usersecret.env)
-  ifelse(read_env(`ENABLE_SAML'), `True', saml_cert:
-    file: ./secrets/saml/sp.crt
-  saml_key:
-    file: ./secrets/saml/sp.key
-  saml_config:
-    file: ./secrets/saml/saml_settings.json)
-
-# vim: set sw=2 et:

docker/services.env

@@ -12,6 +12,9 @@ ACTION_PORT=9002
 PRESENTER_HOST=backend
 PRESENTER_PORT=9003
 
+AUTOUPDATE_HOST=autoupdate
+AUTOUPDATE_PORT=9012
+
 PERMISSION_HOST=permission
 PERMISSION_PORT=9005
 
@@ -24,3 +27,6 @@ MEDIA_HOST=media
 MEDIA_PORT=9006
 MEDIA_DATABASE_HOST=postgres
 MEDIA_DATABASE_NAME=openslides
+
+MANAGE_HOST=manage
+MANAGE_PORT=9008

openslides-autoupdate-service

@@ -1 +1 @@
-Subproject commit fb6e25d7a88ec8202b5080b5563e95451b6071c3
+Subproject commit d284650811d2ae0bb512c4db268952862b5722b4

openslides-backend

@@ -1 +1 @@
-Subproject commit acef4bbf409f53f90f34f68a6ab2c5794f023981
+Subproject commit a24b735b482be4ff5f5425f2e92dd85f805f353d

openslides-client

@@ -1 +1 @@
-Subproject commit 88e620ec4efd634f8fbbffad9c35d4a541a69fcd
+Subproject commit 412741773c15a0d4515c12910416a16a50faada8

openslides-manage-service

@@ -1 +1 @@
-Subproject commit a40e5bd940c41a1eb98533a01f046c0061e2d866
+Subproject commit df61ded339c1cb07e46876d4e463c5f9812d25cc

openslides-permission-service

@@ -1 +1 @@
-Subproject commit e30d357684526c139a397e11ed77ab5befcf2598
+Subproject commit c33b68b0c701f7fc503096c1d89d6c82e5a50232

proxy/Caddyfile

@@ -1,12 +1,12 @@
 import endpoint
 
-    reverse_proxy /system/action/* backend:9002
+    reverse_proxy /system/action* backend:9002
-    reverse_proxy /system/presenter/* backend:9003
+    reverse_proxy /system/presenter* backend:9003
-    reverse_proxy /system/autoupdate/* autoupdate:9012 {
+    reverse_proxy /system/autoupdate* autoupdate:9012 {
         flush_interval -1
     }
-    reverse_proxy /system/auth/* auth:9004
+    reverse_proxy /system/auth* auth:9004
-    reverse_proxy /system/media/* media:9006
+    reverse_proxy /system/media* media:9006
 
     reverse_proxy client:9001