From fe8a74ddf421c713664b5eb36406f5b8ed746b10 Mon Sep 17 00:00:00 2001 From: FinnStutzenstein Date: Wed, 10 Jun 2020 13:58:26 +0200 Subject: [PATCH] Integrate the autoupdate and backend service --- .gitignore | 1 + .gitmodules | 7 ++++++ DEVELOPMENT.md | 19 ++++++++++++---- Makefile | 8 +++++-- docker-compose.dev.yml | 17 ++++++++++---- docker-compose.yml | 42 ++++++++++++++++++++++++++++++----- haproxy/src/dev-haproxy.cfg | 2 +- haproxy/src/haproxy.cfg | 41 +++++++++++++++++++--------------- haproxy/src/prod-haproxy.cfg | 2 +- openslides-autoupdate-service | 1 + openslides-backend | 1 + openslides-client | 2 +- openslides-datastore-service | 2 +- services.env | 15 ++++++++++--- 14 files changed, 120 insertions(+), 40 deletions(-) create mode 160000 openslides-autoupdate-service create mode 160000 openslides-backend diff --git a/.gitignore b/.gitignore index 7541e6e14..01b1434d6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ .DS_Store .idea .env +*.code-workspace # Old OS3 files and folders .coverage diff --git a/.gitmodules b/.gitmodules index 856f19fa6..a96849a93 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,3 +6,10 @@ path = openslides-client url = git@github.com:OpenSlides/openslides-client.git branch = master +[submodule "openslides-backend"] + path = openslides-backend + url = git@github.com:OpenSlides/openslides-backend.git + branch = master +[submodule "openslides-autoupdate-service"] + path = openslides-autoupdate-service + url = git@github.com:OpenSlides/openslides-autoupdate-service.git diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index a61a7a922..a72fa9cbe 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -21,16 +21,18 @@ current commit: $ git diff --cached -Than, commit changes and create a pull request. +Then, commit changes and create a pull request. ## Work in submodules - Create your own fork at github. - Remove the upstream (main) repo as the origin in the submodule: + $ cd $ git remote remove origin - Add your fork and the main repo as origin and upstream + $ git remote add origin $ git remote add upstream
@@ -102,9 +104,18 @@ Or a direct push on master: $ git commit -am "Updated my-service" $ git push origin master -## A useful command +## Working with Submodules After working in many services with different branches, this command checks -out `master` (or the given branch in the .gitmodules) in all submodules: +out `master` (or the given branch in the .gitmodules) in all submodules and +pulls master from upstream (This requres to have `upstream`set up as a remote +in all submodules): + + $ git submodule foreach -q --recursive 'git checkout $(git config -f $toplevel/.gitmodules submodule.$name.branch || echo master); git pull upstream $(git config -f $toplevel/.gitmodules submodule.$name.branch || echo master)' + +When changing the branch in the main repo (this one), the submodules do not +automatically gets changed. THis ocmmand checks out all submodules to the given +commits in the main repo: + + $ git submodule update - $ git submodule foreach -q --recursive 'git checkout $(git config -f $toplevel/.gitmodules submodule.$name.branch || echo master)' diff --git a/Makefile b/Makefile index 1d8b8df5b..0eba1d0b6 100644 --- a/Makefile +++ b/Makefile @@ -11,11 +11,15 @@ build-dev: make -C haproxy build-dev run-dev: | build-dev - docker-compose -f docker-compose.yml -f docker-compose.dev.yml -p 127.0.0.1:8000:8000/tcp up + docker-compose -f docker-compose.yml -f docker-compose.dev.yml up + +copy-node-modules: + docker-compose -f docker-compose.yml -f docker-compose.dev.yml exec client bash -c "cp -r /app/node_modules/ /app/src/" + mv openslides-client/client/src/node_modules/ openslides-client/client/ build-prod: git submodule status | awk '{ gsub(/[^0-9a-f]/, "", $$1); gsub("-","_",$$2); print toupper($$2)"_COMMIT_HASH="$$1 }' > .env docker-compose -f docker-compose.yml -f docker-compose.prod.yml build run-prod: | build-prod - docker-compose -f docker-compose.yml -f docker-compose.prod.yml -p 127.0.0.1:8000:8000/tcp up + docker-compose -f docker-compose.yml -f docker-compose.prod.yml up diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index b76412922..f6d1b7c8d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -11,11 +11,20 @@ services: - ./openslides-datastore-service/shared/shared:/app/shared - ./openslides-datastore-service/writer/writer:/app/writer - ./cli:/app/cli - haproxy: - image: openslides-haproxy-dev - volumes: - - ./haproxy/src:/usr/local/etc/haproxy client: image: openslides-client-dev volumes: - ./openslides-client/client/src:/app/src + backend: + image: openslides-backend-dev + volumes: + - ./openslides-backend/openslides_backend:/srv/code/openslides_backend + autoupdate: + image: openslides-autoupdate-dev + volumes: + - ./openslides-autoupdate-service/cmd:/app/cmd + - ./openslides-autoupdate-service/internal:/app/internal + haproxy: + image: openslides-haproxy-dev + volumes: + - ./haproxy/src:/usr/local/etc/haproxy \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 3a497c73e..e5de08742 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: - datastore-postgresql env_file: services.env networks: + - backend - datastore-postgresql datastore-writer: depends_on: @@ -13,6 +14,7 @@ services: - message-bus env_file: services.env networks: + - backend - datastore-postgresql - message-bus datastore-postgresql: @@ -28,8 +30,32 @@ services: # CLIENT client: - networks: + depends_on: - backend + - autoupdate + networks: + - frontend + + # BACKEND + backend: + depends_on: + - datastore-reader + - datastore-writer + env_file: services.env + networks: + - frontend + - backend + + # AUTOUPDATE + autoupdate: + depends_on: + - datastore-reader + - message-bus + env_file: services.env + networks: + - frontend + - backend + - message-bus # SHARED message-bus: @@ -41,17 +67,23 @@ services: haproxy: depends_on: - client + - backend + - autoupdate ports: - "8000:8000" networks: - uplink - - backend + - frontend +# Setup: host <-uplink-> haproxy <-frontend-> services that are reachable from the client <-backend-> services that are internal-only +# There are special networks for some services only, e.g. datastore-postgresql only for the postgresql, datastore reader and datastore writer networks: uplink: - datastore-postgresql: - internal: true - message-bus: + frontend: internal: true backend: internal: true + datastore-postgresql: + internal: true + message-bus: + internal: true \ No newline at end of file diff --git a/haproxy/src/dev-haproxy.cfg b/haproxy/src/dev-haproxy.cfg index b0207b05b..aada18779 100644 --- a/haproxy/src/dev-haproxy.cfg +++ b/haproxy/src/dev-haproxy.cfg @@ -1,5 +1,5 @@ backend backend_client mode http - server client client:4200 resolvers docker_resolver no-check + server client client:9001 resolvers docker_resolver no-check timeout server 60s timeout connect 60s diff --git a/haproxy/src/haproxy.cfg b/haproxy/src/haproxy.cfg index 9504c44e4..8e991d65e 100644 --- a/haproxy/src/haproxy.cfg +++ b/haproxy/src/haproxy.cfg @@ -3,15 +3,18 @@ frontend http bind *:8000 mode http option http-keep-alive - default_backend backend_client + default_backend backend_client # this is defined in the dev-*/prod-* file timeout client 60s - #acl auth path_beg -i /auth/ - #use_backend backend_auth if auth + acl action path_beg -i /system/action + use_backend backend_action if action - #acl todolist path_beg -i /todo - #use_backend backend_todolist if todolist + acl presenter path_beg -i /system/presenter + use_backend backend_presenter if presenter + + acl autoupdate path_beg -i /system/autoupdate + use_backend backend_autoupdate if autoupdate stats enable stats uri /stats @@ -21,18 +24,20 @@ frontend http resolvers docker_resolver nameserver dns 127.0.0.11:53 -# backend backend_auth -# mode http +backend backend_action + mode http + server action backend:9002 resolvers docker_resolver check + timeout connect 60s + timeout server 60s -# server auth1 auth:8080 resolvers docker_resolver check +backend backend_presenter + mode http + server presenter backend:9003 resolvers docker_resolver check + timeout connect 60s + timeout server 60s -# timeout connect 60s -# timeout server 60s - -# backend backend_todolist -# mode http - -# server todolist1 todolist:8000 resolvers docker_resolver check - -# timeout connect 60s -# timeout server 60s +backend backend_autoupdate + mode http + server autoupdate autoupdate:9012 resolvers docker_resolver check + timeout connect 60s + timeout server 60s diff --git a/haproxy/src/prod-haproxy.cfg b/haproxy/src/prod-haproxy.cfg index 3215e51d5..c03d7244d 100644 --- a/haproxy/src/prod-haproxy.cfg +++ b/haproxy/src/prod-haproxy.cfg @@ -1,5 +1,5 @@ backend backend_client mode http - server client client:80 resolvers docker_resolver check + server client client:9001 resolvers docker_resolver check timeout server 60s timeout connect 60s diff --git a/openslides-autoupdate-service b/openslides-autoupdate-service new file mode 160000 index 000000000..c90af9929 --- /dev/null +++ b/openslides-autoupdate-service @@ -0,0 +1 @@ +Subproject commit c90af9929ac64415317866fc8f493aff9fca8420 diff --git a/openslides-backend b/openslides-backend new file mode 160000 index 000000000..3968ecd4c --- /dev/null +++ b/openslides-backend @@ -0,0 +1 @@ +Subproject commit 3968ecd4cf1fbd997bdc2f4c6e8d4b699aba6f44 diff --git a/openslides-client b/openslides-client index cddc802ce..453ffe428 160000 --- a/openslides-client +++ b/openslides-client @@ -1 +1 @@ -Subproject commit cddc802cee1adf796f4170200aab26012311d32c +Subproject commit 453ffe428722da38cc6f06360726089b884d5c35 diff --git a/openslides-datastore-service b/openslides-datastore-service index 4325e10af..a0d8a2224 160000 --- a/openslides-datastore-service +++ b/openslides-datastore-service @@ -1 +1 @@ -Subproject commit 4325e10af1f7ecad911c9f3ded195ba08f7eef86 +Subproject commit a0d8a222429e7efd048730b823a3247d2f44a4ff diff --git a/services.env b/services.env index 296145ebf..7e1335fc6 100644 --- a/services.env +++ b/services.env @@ -1,6 +1,15 @@ MESSAGE_BUS_HOST=message-bus MESSAGE_BUS_PORT=6379 + +DATASTORE_DATABASE_HOST=datastore-postgresql +# see https://github.com/OpenSlides/openslides-datastore-service/issues/56 + +DATASTORE_READER_HOST=datastore-reader +DATASTORE_READER_PORT=9010 DATASTORE_WRITER_HOST=datastore-writer -DATASTORE_WRITER_PORT=8000 -DATASTORE_WRITER_HOST=datastore-reader -DATASTORE_WRITER_PORT=8001 \ No newline at end of file +DATASTORE_WRITER_PORT=9011 + +ACTION_HOST=backend +ACTION_PORT=9002 +PRESENTER_HOST=backend +PRESENTER_PORT=9003 \ No newline at end of file