From 4484d1e4ef5274e6868fe2b1840814673d2e0282 Mon Sep 17 00:00:00 2001 From: FinnStutzenstein Date: Thu, 13 Feb 2020 10:01:33 +0100 Subject: [PATCH] Integrating the client and datastore into the main repo: - Added each service as a submodule for development - Added conventions for naming environment variables - Added required targets in makefiles of services --- .gitignore | 3 +- .gitmodules | 8 +++ DEVELOPMENT.md | 97 ++++++++++++++++++++++++++++++++++++ Makefile | 20 ++++++++ README.md | 11 +++- docker-compose.dev.yml | 14 ++++++ docker-compose.prod.yml | 11 ++++ docker-compose.yml | 56 +++++++++++++++++++++ haproxy/Dockerfile | 4 ++ haproxy/Dockerfile.dev | 4 ++ haproxy/Makefile | 2 + haproxy/src/dev-haproxy.cfg | 5 ++ haproxy/src/haproxy.cfg | 38 ++++++++++++++ haproxy/src/prod-haproxy.cfg | 5 ++ openslides-client | 1 + openslides-datastore-service | 1 + services.env | 4 ++ 17 files changed, 282 insertions(+), 2 deletions(-) create mode 100644 .gitmodules create mode 100644 DEVELOPMENT.md create mode 100644 Makefile create mode 100644 docker-compose.dev.yml create mode 100644 docker-compose.prod.yml create mode 100644 docker-compose.yml create mode 100644 haproxy/Dockerfile create mode 100644 haproxy/Dockerfile.dev create mode 100644 haproxy/Makefile create mode 100644 haproxy/src/dev-haproxy.cfg create mode 100644 haproxy/src/haproxy.cfg create mode 100644 haproxy/src/prod-haproxy.cfg create mode 160000 openslides-client create mode 160000 openslides-datastore-service create mode 100644 services.env diff --git a/.gitignore b/.gitignore index f3b824406..9a94f7cbc 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ __pycache__ bower_components client make -openslides* +openslides_* +openslides personal_data tests diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..856f19fa6 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,8 @@ +[submodule "openslides-datastore-service"] + path = openslides-datastore-service + url = git@github.com:OpenSlides/openslides-datastore-service.git + branch = master +[submodule "openslides-client"] + path = openslides-client + url = git@github.com:OpenSlides/openslides-client.git + branch = master diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 000000000..211e85be9 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,97 @@ +# Development of OpenSlides 4 + +## First time checkout + +After cloning the repository you need to initialize all submodules, before you can start the development setup + + $ git submodule update --init + $ make run-dev + +## Running tests + +To run all tests of all services, execute `run-service-tests`. TODO: Systemtests in this repo. + +## Adding a new Service + + $ git submodule add + +Append `branch = master` to the new entry in the `.gitmodules` file. Verify, +that it is there (the folder should have 160000 permissions: Submodule) with the +current commit: + + $ git diff --cached + +Than, commit changes and create a pull request. + +## Work in submodules + + - Create your own fork at github. + - Remove the upstream (main) repo as the origin in the submodule: + $ cd + $ git remote remove origin + + - Add your fork and the main repo as origin and upstream + $ git remote add origin + $ git remote add upstream
+ +## Requirements for services + +### Environment variables + - `PORT`: The port, the service is running on + - `_HOST`: The host from a required service + - `_PORT`: The port from a required service + +Required services can be `MESSAGE_BUS`, `DATASTORE_WRITER`, `PERMISSION`, `AUTOUPDATE`, +etc. For private services (e.g. a database dedicated to exactly one service), +use the following syntax: `__`, e.g. the +Postgresql user for the datastore: `DATASTORE_POSTGRESQL_USER`. + +### Makefile + +A makefile must be provided at the root-level of the service. The currently +required (phony) targets are: + + - `run-tests`: Execute all tests from the submodule + - `build-dev`: Build an image with the tag `openslides--dev` + +## Developing on a single service + +Go to the serivce and create a new branch (from master): + + $ cd my-service + $ git status # -> on master? + $ git checkout -b my-feature + +Run OpenSlides in development mode (e.g. in a new terminal): + + $ make run-dev + +After making some changes in my-service, create a commit and push to your fork + + $ git add -A + $ git commit -m "A meaningful commit message here" + $ git push origin my-feature + +As the last step, you can create a PR on Github. After merging, these steps are +required to be executed in the main repo: + + $ cd my-service + $ git pull upstream master + $ cd .. + $ git diff # -> commit hash changed for my-service + +If the update commit should be a PR: + $ git checkout -b updated-my-service + $ git commit -am "Updated my-service" + $ git push origin updated-my-service + +Or a direct push on master: + $ git commit -am "Updated my-service" + $ git push origin master + +## A useful command + +After working in many services with different branches, this command checks +out `master` (or the given branch in the .gitmodules) in all submodules: + + $ git submodule foreach -q --recursive 'git checkout $(git config -f $toplevel/.gitmodules submodule.$name.branch || echo master)' diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..290e1d0fc --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +# TESTS + +run-system-tests: + echo "TODO: write complete system tests" + +run-service-tests: + git submodule foreach 'make run-tests' + +build-dev: + git submodule foreach 'make build-dev' + make -C haproxy build-dev + +run-dev: | build-dev + docker-compose -f docker-compose.yml -f docker-compose.dev.yml -p 127.0.0.1:8000:8000/tcp up + +build-prod: + docker-compose -f docker-compose.yml -f docker-compose.prod.yml build + +run-prod: | build-prod + docker-compose -f docker-compose.yml -f docker-compose.prod.yml -p 127.0.0.1:8000:8000/tcp up diff --git a/README.md b/README.md index 29ee4c5cf..7705632a6 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,16 @@ Read more about our [concept of OpenSlides 4.0](https://github.com/OpenSlides/Op ## Installation -__TODO (Installation with docker-compose)__ +Required software: Docker, docker-compose, make, git + +For a non-development setup, clone this repo and run it via docker compose. The make command is a handy shortcut for this: + + $ git clone git@github.com:OpenSlides/OpenSlides.git + $ cd OpenSlides + $ git checkout openslides4-dev # needed, until OS4 is released + $ make run-prod + +For a development setup, refer to [the development docs](DEVELOPMENT.md) ## Used software diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 000000000..6e42ee728 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,14 @@ +version: '3' +services: + datastore-writer: + image: openslides-datastore-writer-dev + volumes: + - ./openslides-datastore-service/writer/writer:/app/writer + haproxy: + image: openslides-haproxy-dev + volumes: + - ./haproxy/src:/usr/local/etc/haproxy + client: + image: openslides-client-dev + volumes: + - ./openslides-client/client/src:/app/src diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 000000000..7e09bb737 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,11 @@ +version: '3' +services: + datastore-writer: + build: https://github.com/OpenSlides/openslides-datastore-service.git#:writer + image: openslides-datastore-writer + client: + build: https://github.com/OpenSlides/openslides-client.git + image: openslides-client + haproxy: + build: ./haproxy + image: openslides-haproxy diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..8e18821ff --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,56 @@ +version: '3' +services: + # DATASTORE SECTION + datastore-writer: + depends_on: + - datastore-postgresql + - message-bus + environment: + - DATASTORE_DATABASE_HOST=datastore-postgresql + - DATASTORE_DATABASE_USER=openslides + - DATASTORE_DATABASE_PASSWORD=openslides + - DATASTORE_DATABASE_NAME=openslides + env_file: services.env + networks: + - datastore-postgresql + - message-bus + datastore-postgresql: + image: sameersbn/postgresql:10 + labels: + org.openslides.role: "postgres" + environment: + - DB_USER=openslides + - DB_PASS=openslides + - DB_NAME=openslides + networks: + - datastore-postgresql + + # CLIENT + client: + networks: + - backend + + # SHARED + message-bus: + image: redis:alpine + networks: + - message-bus + + # UPLINK + haproxy: + depends_on: + - client + ports: + - "8000:8000" + networks: + - uplink + - backend + +networks: + uplink: + datastore-postgresql: + internal: true + message-bus: + internal: true + backend: + internal: true diff --git a/haproxy/Dockerfile b/haproxy/Dockerfile new file mode 100644 index 000000000..99657bdc7 --- /dev/null +++ b/haproxy/Dockerfile @@ -0,0 +1,4 @@ +FROM haproxy:2.0.8-alpine +COPY src/haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg +COPY src/prod-haproxy.cfg /usr/local/etc/haproxy/prod-haproxy.cfg +CMD ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.cfg", "-f", "/usr/local/etc/haproxy/prod-haproxy.cfg"] diff --git a/haproxy/Dockerfile.dev b/haproxy/Dockerfile.dev new file mode 100644 index 000000000..73d2638e0 --- /dev/null +++ b/haproxy/Dockerfile.dev @@ -0,0 +1,4 @@ +FROM haproxy:2.0.8-alpine +COPY src/haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg +COPY src/dev-haproxy.cfg /usr/local/etc/haproxy/dev-haproxy.cfg +CMD ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.cfg", "-f", "/usr/local/etc/haproxy/dev-haproxy.cfg"] diff --git a/haproxy/Makefile b/haproxy/Makefile new file mode 100644 index 000000000..740241b68 --- /dev/null +++ b/haproxy/Makefile @@ -0,0 +1,2 @@ +build-dev: + docker build -t openslides-haproxy-dev -f Dockerfile.dev . diff --git a/haproxy/src/dev-haproxy.cfg b/haproxy/src/dev-haproxy.cfg new file mode 100644 index 000000000..b0207b05b --- /dev/null +++ b/haproxy/src/dev-haproxy.cfg @@ -0,0 +1,5 @@ +backend backend_client + mode http + server client client:4200 resolvers docker_resolver no-check + timeout server 60s + timeout connect 60s diff --git a/haproxy/src/haproxy.cfg b/haproxy/src/haproxy.cfg new file mode 100644 index 000000000..9504c44e4 --- /dev/null +++ b/haproxy/src/haproxy.cfg @@ -0,0 +1,38 @@ + +frontend http + bind *:8000 + mode http + option http-keep-alive + default_backend backend_client + + timeout client 60s + + #acl auth path_beg -i /auth/ + #use_backend backend_auth if auth + + #acl todolist path_beg -i /todo + #use_backend backend_todolist if todolist + + stats enable + stats uri /stats + stats refresh 10s + stats auth admin:admin + +resolvers docker_resolver + nameserver dns 127.0.0.11:53 + +# backend backend_auth +# mode http + +# server auth1 auth:8080 resolvers docker_resolver check + +# timeout connect 60s +# timeout server 60s + +# backend backend_todolist +# mode http + +# server todolist1 todolist:8000 resolvers docker_resolver check + +# timeout connect 60s +# timeout server 60s diff --git a/haproxy/src/prod-haproxy.cfg b/haproxy/src/prod-haproxy.cfg new file mode 100644 index 000000000..3215e51d5 --- /dev/null +++ b/haproxy/src/prod-haproxy.cfg @@ -0,0 +1,5 @@ +backend backend_client + mode http + server client client:80 resolvers docker_resolver check + timeout server 60s + timeout connect 60s diff --git a/openslides-client b/openslides-client new file mode 160000 index 000000000..e75e72b50 --- /dev/null +++ b/openslides-client @@ -0,0 +1 @@ +Subproject commit e75e72b50c66cd59ca6aac8cea1cf8c6c998d954 diff --git a/openslides-datastore-service b/openslides-datastore-service new file mode 160000 index 000000000..51af334dd --- /dev/null +++ b/openslides-datastore-service @@ -0,0 +1 @@ +Subproject commit 51af334ddc7c7a57616263e8d56e27c21c0e5e65 diff --git a/services.env b/services.env new file mode 100644 index 000000000..8ca0dbfbd --- /dev/null +++ b/services.env @@ -0,0 +1,4 @@ +MESSAGE_BUS_HOST=message-bus +MESSAGE_BUS_PORT=6379 +DATASTORE_WRITER_HOST=datastore-writer +DATASTORE_WRITER_PORT=8000 \ No newline at end of file