Merge pull request #5237 from FinnStutzenstein/os4-compose

Integrating the client and datastore into the main repo:

.gitignore

@@ -14,6 +14,7 @@ __pycache__
 bower_components
 client
 make
-openslides*
+openslides_*
+openslides
 personal_data
 tests

.gitmodules

@@ -0,0 +1,8 @@
+[submodule "openslides-datastore-service"]
+	path = openslides-datastore-service
+	url = git@github.com:OpenSlides/openslides-datastore-service.git
+	branch = master
+[submodule "openslides-client"]
+	path = openslides-client
+	url = git@github.com:OpenSlides/openslides-client.git
+	branch = master

DEVELOPMENT.md

@@ -0,0 +1,101 @@
+# Development of OpenSlides 4
+
+## First time checkout
+
+After cloning the repository you need to initialize all submodules, before you can start the development setup
+
+    $ git submodule update --init
+    $ make run-dev
+
+## Running tests
+
+To run all tests of all services, execute `run-service-tests`. TODO: Systemtests in this repo.
+
+## Adding a new Service
+
+    $ git submodule add <git@myrepo.git>
+
+Append `branch = master` to the new entry in the `.gitmodules` file. Verify,
+that it is there (the folder should have 160000 permissions: Submodule) with the
+current commit:
+
+    $ git diff --cached
+
+Than, commit changes and create a pull request.
+
+## Work in submodules
+
+- Create your own fork at github.
+- Remove the upstream (main) repo as the origin in the submodule:
+    $ cd <submodule>
+    $ git remote remove origin
+
+- Add your fork and the main repo as origin and upstream
+    $ git remote add origin <your fork>
+    $ git remote add upstream <main repo>
+
+## Requirements for services
+
+### Environment variables
+
+These environment variables are available:
+
+- `<SERVICE>_HOST`: The host from a required service
+- `<SERVICE>_PORT`: The port from a required service
+
+Required services can be `MESSAGE_BUS`, `DATASTORE_WRITER`, `PERMISSION`, `AUTOUPDATE`,
+etc. For private services (e.g. a database dedicated to exactly one service),
+use the following syntax: `<SERVICE>_<PRIV_SERVICE>_<ATTRIBUTE>`, e.g. the
+Postgresql user for the datastore: `DATASTORE_POSTGRESQL_USER`.
+
+### Makefile
+
+A makefile must be provided at the root-level of the service. The currently
+required (phony) targets are:
+
+- `run-tests`: Execute all tests from the submodule
+- `build-dev`: Build an image with the tag `openslides-<service>-dev`
+
+## Developing on a single service
+
+Go to the serivce and create a new branch (from master):
+
+    $ cd my-service
+    $ git status  # -> on master?
+    $ git checkout -b my-feature
+
+Run OpenSlides in development mode (e.g. in a new terminal):
+
+    $ make run-dev
+
+After making some changes in my-service, create a commit and push to your fork
+
+    $ git add -A
+    $ git commit -m "A meaningful commit message here"
+    $ git push origin my-feature
+
+As the last step, you can create a PR on Github. After merging, these steps are
+required to be executed in the main repo:
+
+    $ cd my-service
+    $ git pull upstream master
+    $ cd ..
+    $ git diff  # -> commit hash changed for my-service
+
+If the update commit should be a PR:
+
+    $ git checkout -b updated-my-service
+    $ git commit -am "Updated my-service"
+    $ git push origin updated-my-service
+
+Or a direct push on master:
+
+    $ git commit -am "Updated my-service"
+    $ git push origin master
+
+## A useful command
+
+After working in many services with different branches, this command checks
+out `master` (or the given branch in the .gitmodules) in all submodules:
+
+    $ git submodule foreach -q --recursive 'git checkout $(git config -f $toplevel/.gitmodules submodule.$name.branch || echo master)'

Makefile

@@ -0,0 +1,20 @@
+# TESTS
+
+run-system-tests:
+	echo "TODO: write complete system tests"
+
+run-service-tests:
+	git submodule foreach 'make run-tests'
+
+build-dev:
+	git submodule foreach 'make build-dev'
+	make -C haproxy build-dev
+
+run-dev: | build-dev
+	docker-compose -f docker-compose.yml -f docker-compose.dev.yml -p 127.0.0.1:8000:8000/tcp up
+
+build-prod:
+	docker-compose -f docker-compose.yml -f docker-compose.prod.yml build
+
+run-prod: | build-prod
+	docker-compose -f docker-compose.yml -f docker-compose.prod.yml -p 127.0.0.1:8000:8000/tcp up

README.md

@@ -18,7 +18,16 @@ Read more about our [concept of OpenSlides 4.0](https://github.com/OpenSlides/Op
 
 ## Installation
 
-__TODO (Installation with docker-compose)__
+Required software: Docker, docker-compose, make, git
+
+For a non-development setup, clone this repo and run it via docker compose. The make command is a handy shortcut for this:
+
+    $ git clone git@github.com:OpenSlides/OpenSlides.git
+    $ cd OpenSlides
+    $ git checkout openslides4-dev  # needed, until OS4 is released
+    $ make run-prod
+
+For a development setup, refer to [the development docs](DEVELOPMENT.md)
 
 
 ## Used software

docker-compose.dev.yml

@@ -0,0 +1,14 @@
+version: '3'
+services:
+    datastore-writer:
+        image: openslides-datastore-writer-dev
+        volumes:
+            - ./openslides-datastore-service/writer/writer:/app/writer
+    haproxy:
+        image: openslides-haproxy-dev
+        volumes:
+            - ./haproxy/src:/usr/local/etc/haproxy
+    client:
+        image: openslides-client-dev
+        volumes:
+            - ./openslides-client/client/src:/app/src

docker-compose.prod.yml

@@ -0,0 +1,11 @@
+version: '3'
+services:
+    datastore-writer:
+        build: https://github.com/OpenSlides/openslides-datastore-service.git#:writer
+        image: openslides-datastore-writer
+    client:
+        build: https://github.com/OpenSlides/openslides-client.git
+        image: openslides-client
+    haproxy:
+        build: ./haproxy
+        image: openslides-haproxy

docker-compose.yml

@@ -0,0 +1,56 @@
+version: '3'
+services:
+    # DATASTORE SECTION
+    datastore-writer:
+        depends_on:
+            - datastore-postgresql
+            - message-bus
+        environment:
+            - DATASTORE_DATABASE_HOST=datastore-postgresql
+            - DATASTORE_DATABASE_USER=openslides
+            - DATASTORE_DATABASE_PASSWORD=openslides
+            - DATASTORE_DATABASE_NAME=openslides
+        env_file: services.env
+        networks:
+            - datastore-postgresql
+            - message-bus
+    datastore-postgresql:
+        image: sameersbn/postgresql:10
+        labels:
+            org.openslides.role: "postgres"
+        environment:
+            - DB_USER=openslides
+            - DB_PASS=openslides
+            - DB_NAME=openslides
+        networks:
+            - datastore-postgresql
+    
+    # CLIENT
+    client:
+        networks:
+            - backend
+
+    # SHARED
+    message-bus:
+        image: redis:alpine
+        networks:
+            - message-bus
+
+    # UPLINK
+    haproxy:
+        depends_on:
+            - client
+        ports:
+            - "8000:8000"
+        networks:
+            - uplink
+            - backend
+
+networks:
+    uplink:
+    datastore-postgresql:
+        internal: true
+    message-bus:
+        internal: true
+    backend:
+        internal: true

haproxy/Dockerfile

@@ -0,0 +1,4 @@
+FROM haproxy:2.0.8-alpine
+COPY src/haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg
+COPY src/prod-haproxy.cfg /usr/local/etc/haproxy/prod-haproxy.cfg
+CMD ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.cfg", "-f", "/usr/local/etc/haproxy/prod-haproxy.cfg"]

haproxy/Dockerfile.dev

@@ -0,0 +1,4 @@
+FROM haproxy:2.0.8-alpine
+COPY src/haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg
+COPY src/dev-haproxy.cfg /usr/local/etc/haproxy/dev-haproxy.cfg
+CMD ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.cfg", "-f", "/usr/local/etc/haproxy/dev-haproxy.cfg"]

haproxy/Makefile

@@ -0,0 +1,2 @@
+build-dev:
+	docker build -t openslides-haproxy-dev -f Dockerfile.dev .

haproxy/src/dev-haproxy.cfg

@@ -0,0 +1,5 @@
+backend backend_client
+    mode http
+    server client client:4200 resolvers docker_resolver no-check
+    timeout server 60s
+    timeout connect 60s

haproxy/src/haproxy.cfg

@@ -0,0 +1,38 @@
+
+frontend http
+    bind *:8000
+    mode http
+    option http-keep-alive
+    default_backend backend_client
+
+    timeout client 60s
+
+    #acl auth path_beg -i /auth/
+    #use_backend backend_auth if auth
+
+    #acl todolist path_beg -i /todo
+    #use_backend backend_todolist if todolist
+
+    stats enable
+    stats uri /stats
+    stats refresh 10s
+    stats auth admin:admin
+
+resolvers docker_resolver
+    nameserver dns 127.0.0.11:53
+
+# backend backend_auth
+#     mode http
+
+#     server auth1 auth:8080 resolvers docker_resolver check
+
+#     timeout connect 60s
+#     timeout server 60s
+
+# backend backend_todolist
+#     mode http
+
+#     server todolist1 todolist:8000 resolvers docker_resolver check
+
+#     timeout connect 60s
+#     timeout server 60s

haproxy/src/prod-haproxy.cfg

@@ -0,0 +1,5 @@
+backend backend_client
+    mode http
+    server client client:80 resolvers docker_resolver check
+    timeout server 60s
+    timeout connect 60s

openslides-client

@@ -0,0 +1 @@
+Subproject commit e75e72b50c66cd59ca6aac8cea1cf8c6c998d954

openslides-datastore-service

@@ -0,0 +1 @@
+Subproject commit 51af334ddc7c7a57616263e8d56e27c21c0e5e65

services.env

@@ -0,0 +1,4 @@
+MESSAGE_BUS_HOST=message-bus
+MESSAGE_BUS_PORT=6379
+DATASTORE_WRITER_HOST=datastore-writer
+DATASTORE_WRITER_PORT=8000