Integrate the autoupdate and backend service

2020-06-10 13:58:26 +02:00 · 2020-06-10 13:58:26 +02:00 · fe8a74ddf4
parent b6a9d7dee8
commit fe8a74ddf4
14 changed files with 120 additions and 40 deletions
--- a/.gitignore
+++ b/.gitignore
@ -6,6 +6,7 @@
 .DS_Store
 .idea
 .env
+*.code-workspace

 # Old OS3 files and folders
 .coverage
--- a/.gitmodules
+++ b/.gitmodules
@ -6,3 +6,10 @@
 	path = openslides-client
 	url = git@github.com:OpenSlides/openslides-client.git
 	branch = master
+[submodule "openslides-backend"]
+	path = openslides-backend
+	url = git@github.com:OpenSlides/openslides-backend.git
+	branch = master
+[submodule "openslides-autoupdate-service"]
+	path = openslides-autoupdate-service
+	url = git@github.com:OpenSlides/openslides-autoupdate-service.git
--- a/DEVELOPMENT.md
+++ b/DEVELOPMENT.md
@ -21,16 +21,18 @@ current commit:

    $ git diff --cached

-Than, commit changes and create a pull request.
+Then, commit changes and create a pull request.

 ## Work in submodules

 - Create your own fork at github.
 - Remove the upstream (main) repo as the origin in the submodule:
+
    $ cd <submodule>
    $ git remote remove origin

 - Add your fork and the main repo as origin and upstream
+
    $ git remote add origin <your fork>
    $ git remote add upstream <main repo>

@ -102,9 +104,18 @@ Or a direct push on master:
    $ git commit -am "Updated my-service"
    $ git push origin master

-## A useful command
+## Working with Submodules

 After working in many services with different branches, this command checks
-out `master` (or the given branch in the .gitmodules) in all submodules:
+out `master` (or the given branch in the .gitmodules) in all submodules and
+pulls master from upstream (This requres to have `upstream`set up as a remote
+in all submodules):
+
+    $ git submodule foreach -q --recursive 'git checkout $(git config -f $toplevel/.gitmodules submodule.$name.branch || echo master); git pull upstream $(git config -f $toplevel/.gitmodules submodule.$name.branch || echo master)'
+
+When changing the branch in the main repo (this one), the submodules do not
+automatically gets changed. THis ocmmand checks out all submodules to the given
+commits in the main repo:
+
+    $ git submodule update

-    $ git submodule foreach -q --recursive 'git checkout $(git config -f $toplevel/.gitmodules submodule.$name.branch || echo master)'
--- a/8
+++ b/8
@ -11,11 +11,15 @@ build-dev:
 	make -C haproxy build-dev

 run-dev: | build-dev
-	docker-compose -f docker-compose.yml -f docker-compose.dev.yml -p 127.0.0.1:8000:8000/tcp up
+	docker-compose -f docker-compose.yml -f docker-compose.dev.yml up
+
+copy-node-modules:
+	docker-compose -f docker-compose.yml -f docker-compose.dev.yml exec client bash -c "cp -r /app/node_modules/ /app/src/"
+	mv openslides-client/client/src/node_modules/ openslides-client/client/

 build-prod:
 	git submodule status | awk '{ gsub(/[^0-9a-f]/, "", $$1); gsub("-","_",$$2); print toupper($$2)"_COMMIT_HASH="$$1 }' > .env
 	docker-compose -f docker-compose.yml -f docker-compose.prod.yml build

 run-prod: | build-prod
-	docker-compose -f docker-compose.yml -f docker-compose.prod.yml -p 127.0.0.1:8000:8000/tcp up
+	docker-compose -f docker-compose.yml -f docker-compose.prod.yml up
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@ -11,11 +11,20 @@ services:
            - ./openslides-datastore-service/shared/shared:/app/shared
            - ./openslides-datastore-service/writer/writer:/app/writer
            - ./cli:/app/cli
-    haproxy:
-        image: openslides-haproxy-dev
-        volumes:
-            - ./haproxy/src:/usr/local/etc/haproxy
    client:
        image: openslides-client-dev
        volumes:
            - ./openslides-client/client/src:/app/src
+    backend:
+        image: openslides-backend-dev
+        volumes:
+            - ./openslides-backend/openslides_backend:/srv/code/openslides_backend
+    autoupdate:
+        image: openslides-autoupdate-dev
+        volumes:
+            - ./openslides-autoupdate-service/cmd:/app/cmd
+            - ./openslides-autoupdate-service/internal:/app/internal
+    haproxy:
+        image: openslides-haproxy-dev
+        volumes:
+            - ./haproxy/src:/usr/local/etc/haproxy
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -6,6 +6,7 @@ services:
            - datastore-postgresql
        env_file: services.env
        networks:
+            - backend
            - datastore-postgresql
    datastore-writer:
        depends_on:
@ -13,6 +14,7 @@ services:
            - message-bus
        env_file: services.env
        networks:
+            - backend
            - datastore-postgresql
            - message-bus
    datastore-postgresql:
@ -28,8 +30,32 @@ services:
    
    # CLIENT
    client:
-        networks:
+        depends_on:
            - backend
+            - autoupdate
+        networks:
+            - frontend
+
+    # BACKEND
+    backend:
+        depends_on:
+            - datastore-reader
+            - datastore-writer
+        env_file: services.env
+        networks:
+            - frontend
+            - backend
+
+    # AUTOUPDATE
+    autoupdate:
+        depends_on:
+            - datastore-reader
+            - message-bus
+        env_file: services.env
+        networks:
+            - frontend
+            - backend
+            - message-bus

    # SHARED
    message-bus:
@ -41,17 +67,23 @@ services:
    haproxy:
        depends_on:
            - client
+            - backend
+            - autoupdate
        ports:
            - "8000:8000"
        networks:
            - uplink
-            - backend
+            - frontend

+# Setup: host <-uplink-> haproxy <-frontend-> services that are reachable from the client <-backend-> services that are internal-only
+# There are special networks for some services only, e.g. datastore-postgresql only for the postgresql, datastore reader and datastore writer
 networks:
    uplink:
-    datastore-postgresql:
-        internal: true
-    message-bus:
+    frontend:
        internal: true
    backend:
        internal: true
+    datastore-postgresql:
+        internal: true
+    message-bus:
+        internal: true
--- a/haproxy/src/dev-haproxy.cfg
+++ b/haproxy/src/dev-haproxy.cfg
@ -1,5 +1,5 @@
 backend backend_client
    mode http
-    server client client:4200 resolvers docker_resolver no-check
+    server client client:9001 resolvers docker_resolver no-check
    timeout server 60s
    timeout connect 60s
--- a/haproxy/src/haproxy.cfg
+++ b/haproxy/src/haproxy.cfg
@ -3,15 +3,18 @@ frontend http
    bind *:8000
    mode http
    option http-keep-alive
-    default_backend backend_client
+    default_backend backend_client  # this is defined in the dev-*/prod-* file

    timeout client 60s

-    #acl auth path_beg -i /auth/
-    #use_backend backend_auth if auth
+    acl action path_beg -i /system/action
+    use_backend backend_action if action

-    #acl todolist path_beg -i /todo
-    #use_backend backend_todolist if todolist
+    acl presenter path_beg -i /system/presenter
+    use_backend backend_presenter if presenter
+
+    acl autoupdate path_beg -i /system/autoupdate
+    use_backend backend_autoupdate if autoupdate

    stats enable
    stats uri /stats
@ -21,18 +24,20 @@ frontend http
 resolvers docker_resolver
    nameserver dns 127.0.0.11:53

-# backend backend_auth
-#     mode http
+backend backend_action
+    mode http
+    server action backend:9002 resolvers docker_resolver check
+    timeout connect 60s
+    timeout server 60s

-#     server auth1 auth:8080 resolvers docker_resolver check
+backend backend_presenter
+    mode http
+    server presenter backend:9003 resolvers docker_resolver check
+    timeout connect 60s
+    timeout server 60s

-#     timeout connect 60s
-#     timeout server 60s
-
-# backend backend_todolist
-#     mode http
-
-#     server todolist1 todolist:8000 resolvers docker_resolver check
-
-#     timeout connect 60s
-#     timeout server 60s
+backend backend_autoupdate
+    mode http
+    server autoupdate autoupdate:9012 resolvers docker_resolver check
+    timeout connect 60s
+    timeout server 60s
--- a/haproxy/src/prod-haproxy.cfg
+++ b/haproxy/src/prod-haproxy.cfg
@ -1,5 +1,5 @@
 backend backend_client
    mode http
-    server client client:80 resolvers docker_resolver check
+    server client client:9001 resolvers docker_resolver check
    timeout server 60s
    timeout connect 60s
--- a/1
+++ b/1
@ -0,0 +1 @@
+Subproject commit c90af9929ac64415317866fc8f493aff9fca8420
--- a/1
+++ b/1
@ -0,0 +1 @@
+Subproject commit 3968ecd4cf1fbd997bdc2f4c6e8d4b699aba6f44
--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit cddc802cee1adf796f4170200aab26012311d32c
+Subproject commit 453ffe428722da38cc6f06360726089b884d5c35
--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit 4325e10af1f7ecad911c9f3ded195ba08f7eef86
+Subproject commit a0d8a222429e7efd048730b823a3247d2f44a4ff
--- a/services.env
+++ b/services.env
@ -1,6 +1,15 @@
 MESSAGE_BUS_HOST=message-bus
 MESSAGE_BUS_PORT=6379
+
+DATASTORE_DATABASE_HOST=datastore-postgresql
+# see https://github.com/OpenSlides/openslides-datastore-service/issues/56
+
+DATASTORE_READER_HOST=datastore-reader
+DATASTORE_READER_PORT=9010
 DATASTORE_WRITER_HOST=datastore-writer
-DATASTORE_WRITER_PORT=8000
-DATASTORE_WRITER_HOST=datastore-reader
-DATASTORE_WRITER_PORT=8001
+DATASTORE_WRITER_PORT=9011
+
+ACTION_HOST=backend
+ACTION_PORT=9002
+PRESENTER_HOST=backend
+PRESENTER_PORT=9003
				`@ -0,0 +1 @@`
				`Subproject commit c90af9929ac64415317866fc8f493aff9fca8420`
				`@ -0,0 +1 @@`
				`Subproject commit 3968ecd4cf1fbd997bdc2f4c6e8d4b699aba6f44`