Merge pull request #5409 from FinnStutzenstein/integration

Integration of the backend and autoupdate service

.gitignore

@@ -6,6 +6,7 @@
 .DS_Store
 .idea
 .env
+*.code-workspace
 
 # Old OS3 files and folders
 .coverage

.gitmodules

@@ -6,3 +6,10 @@
 	path = openslides-client
 	url = git@github.com:OpenSlides/openslides-client.git
 	branch = master
+[submodule "openslides-backend"]
+	path = openslides-backend
+	url = git@github.com:OpenSlides/openslides-backend.git
+	branch = master
+[submodule "openslides-autoupdate-service"]
+	path = openslides-autoupdate-service
+	url = git@github.com:OpenSlides/openslides-autoupdate-service.git

DEVELOPMENT.md

@@ -21,16 +21,18 @@ current commit:
 
     $ git diff --cached
 
-Than, commit changes and create a pull request.
+Then, commit changes and create a pull request.
 
 ## Work in submodules
 
 - Create your own fork at github.
 - Remove the upstream (main) repo as the origin in the submodule:
+
     $ cd <submodule>
     $ git remote remove origin
 
 - Add your fork and the main repo as origin and upstream
+
     $ git remote add origin <your fork>
     $ git remote add upstream <main repo>
 
@@ -102,9 +104,18 @@ Or a direct push on master:
     $ git commit -am "Updated my-service"
     $ git push origin master
 
-## A useful command
+## Working with Submodules
 
 After working in many services with different branches, this command checks
-out `master` (or the given branch in the .gitmodules) in all submodules:
+out `master` (or the given branch in the .gitmodules) in all submodules and
+pulls master from upstream (This requres to have `upstream`set up as a remote
+in all submodules):
+
+    $ git submodule foreach -q --recursive 'git checkout $(git config -f $toplevel/.gitmodules submodule.$name.branch || echo master); git pull upstream $(git config -f $toplevel/.gitmodules submodule.$name.branch || echo master)'
+
+When changing the branch in the main repo (this one), the submodules do not
+automatically gets changed. THis ocmmand checks out all submodules to the given
+commits in the main repo:
+
+    $ git submodule update
 
-    $ git submodule foreach -q --recursive 'git checkout $(git config -f $toplevel/.gitmodules submodule.$name.branch || echo master)'

Makefile

@@ -11,11 +11,15 @@ build-dev:
 	make -C haproxy build-dev
 
 run-dev: | build-dev
-	docker-compose -f docker-compose.yml -f docker-compose.dev.yml -p 127.0.0.1:8000:8000/tcp up
+	docker-compose -f docker-compose.yml -f docker-compose.dev.yml up
+
+copy-node-modules:
+	docker-compose -f docker-compose.yml -f docker-compose.dev.yml exec client bash -c "cp -r /app/node_modules/ /app/src/"
+	mv openslides-client/client/src/node_modules/ openslides-client/client/
 
 build-prod:
 	git submodule status | awk '{ gsub(/[^0-9a-f]/, "", $$1); gsub("-","_",$$2); print toupper($$2)"_COMMIT_HASH="$$1 }' > .env
 	docker-compose -f docker-compose.yml -f docker-compose.prod.yml build
 
 run-prod: | build-prod
-	docker-compose -f docker-compose.yml -f docker-compose.prod.yml -p 127.0.0.1:8000:8000/tcp up
+	docker-compose -f docker-compose.yml -f docker-compose.prod.yml up

docker-compose.dev.yml

@@ -11,11 +11,20 @@ services:
             - ./openslides-datastore-service/shared/shared:/app/shared
             - ./openslides-datastore-service/writer/writer:/app/writer
             - ./cli:/app/cli
-    haproxy:
-        image: openslides-haproxy-dev
-        volumes:
-            - ./haproxy/src:/usr/local/etc/haproxy
     client:
         image: openslides-client-dev
         volumes:
             - ./openslides-client/client/src:/app/src
+    backend:
+        image: openslides-backend-dev
+        volumes:
+            - ./openslides-backend/openslides_backend:/srv/code/openslides_backend
+    autoupdate:
+        image: openslides-autoupdate-dev
+        volumes:
+            - ./openslides-autoupdate-service/cmd:/app/cmd
+            - ./openslides-autoupdate-service/internal:/app/internal
+    haproxy:
+        image: openslides-haproxy-dev
+        volumes:
+            - ./haproxy/src:/usr/local/etc/haproxy

docker-compose.yml

@@ -6,6 +6,7 @@ services:
             - datastore-postgresql
         env_file: services.env
         networks:
+            - backend
             - datastore-postgresql
     datastore-writer:
         depends_on:
@@ -13,6 +14,7 @@ services:
             - message-bus
         env_file: services.env
         networks:
+            - backend
             - datastore-postgresql
             - message-bus
     datastore-postgresql:
@@ -28,8 +30,32 @@ services:
     
     # CLIENT
     client:
-        networks:
+        depends_on:
             - backend
+            - autoupdate
+        networks:
+            - frontend
+
+    # BACKEND
+    backend:
+        depends_on:
+            - datastore-reader
+            - datastore-writer
+        env_file: services.env
+        networks:
+            - frontend
+            - backend
+
+    # AUTOUPDATE
+    autoupdate:
+        depends_on:
+            - datastore-reader
+            - message-bus
+        env_file: services.env
+        networks:
+            - frontend
+            - backend
+            - message-bus
 
     # SHARED
     message-bus:
@@ -41,17 +67,23 @@ services:
     haproxy:
         depends_on:
             - client
+            - backend
+            - autoupdate
         ports:
             - "8000:8000"
         networks:
             - uplink
-            - backend
+            - frontend
 
+# Setup: host <-uplink-> haproxy <-frontend-> services that are reachable from the client <-backend-> services that are internal-only
+# There are special networks for some services only, e.g. datastore-postgresql only for the postgresql, datastore reader and datastore writer
 networks:
     uplink:
-    datastore-postgresql:
+    frontend:
-        internal: true
-    message-bus:
         internal: true
     backend:
         internal: true
+    datastore-postgresql:
+        internal: true
+    message-bus:
+        internal: true

haproxy/src/dev-haproxy.cfg

@@ -1,5 +1,5 @@
 backend backend_client
     mode http
-    server client client:4200 resolvers docker_resolver no-check
+    server client client:9001 resolvers docker_resolver no-check
     timeout server 60s
     timeout connect 60s

haproxy/src/haproxy.cfg

@@ -3,15 +3,18 @@ frontend http
     bind *:8000
     mode http
     option http-keep-alive
-    default_backend backend_client
+    default_backend backend_client  # this is defined in the dev-*/prod-* file
 
     timeout client 60s
 
-    #acl auth path_beg -i /auth/
+    acl action path_beg -i /system/action
-    #use_backend backend_auth if auth
+    use_backend backend_action if action
 
-    #acl todolist path_beg -i /todo
+    acl presenter path_beg -i /system/presenter
-    #use_backend backend_todolist if todolist
+    use_backend backend_presenter if presenter
+
+    acl autoupdate path_beg -i /system/autoupdate
+    use_backend backend_autoupdate if autoupdate
 
     stats enable
     stats uri /stats
@@ -21,18 +24,20 @@ frontend http
 resolvers docker_resolver
     nameserver dns 127.0.0.11:53
 
-# backend backend_auth
+backend backend_action
-#     mode http
+    mode http
+    server action backend:9002 resolvers docker_resolver check
+    timeout connect 60s
+    timeout server 60s
 
-#     server auth1 auth:8080 resolvers docker_resolver check
+backend backend_presenter
+    mode http
+    server presenter backend:9003 resolvers docker_resolver check
+    timeout connect 60s
+    timeout server 60s
 
-#     timeout connect 60s
+backend backend_autoupdate
-#     timeout server 60s
+    mode http
-
+    server autoupdate autoupdate:9012 resolvers docker_resolver check
-# backend backend_todolist
+    timeout connect 60s
-#     mode http
+    timeout server 60s
-
-#     server todolist1 todolist:8000 resolvers docker_resolver check
-
-#     timeout connect 60s
-#     timeout server 60s

haproxy/src/prod-haproxy.cfg

@@ -1,5 +1,5 @@
 backend backend_client
     mode http
-    server client client:80 resolvers docker_resolver check
+    server client client:9001 resolvers docker_resolver check
     timeout server 60s
     timeout connect 60s

openslides-autoupdate-service

@@ -0,0 +1 @@
+Subproject commit c90af9929ac64415317866fc8f493aff9fca8420

openslides-backend

@@ -0,0 +1 @@
+Subproject commit 3968ecd4cf1fbd997bdc2f4c6e8d4b699aba6f44

openslides-client

@@ -1 +1 @@
-Subproject commit cddc802cee1adf796f4170200aab26012311d32c
+Subproject commit 453ffe428722da38cc6f06360726089b884d5c35

openslides-datastore-service

@@ -1 +1 @@
-Subproject commit 4325e10af1f7ecad911c9f3ded195ba08f7eef86
+Subproject commit a0d8a222429e7efd048730b823a3247d2f44a4ff

services.env

@@ -1,6 +1,15 @@
 MESSAGE_BUS_HOST=message-bus
 MESSAGE_BUS_PORT=6379
+
+DATASTORE_DATABASE_HOST=datastore-postgresql
+# see https://github.com/OpenSlides/openslides-datastore-service/issues/56
+
+DATASTORE_READER_HOST=datastore-reader
+DATASTORE_READER_PORT=9010
 DATASTORE_WRITER_HOST=datastore-writer
-DATASTORE_WRITER_PORT=8000
+DATASTORE_WRITER_PORT=9011
-DATASTORE_WRITER_HOST=datastore-reader
+
-DATASTORE_WRITER_PORT=8001
+ACTION_HOST=backend
+ACTION_PORT=9002
+PRESENTER_HOST=backend
+PRESENTER_PORT=9003