Merge pull request #5958 from FinnStutzenstein/updateAndCaddy

Update submodules and use Caddy

.gitmodules

@@ -24,6 +24,7 @@
 [submodule "openslides-permission-service"]
 	path = openslides-permission-service
 	url = git@github.com:OpenSlides/openslides-permission-service.git
+	branch = master
 [submodule "openslides-manage-service"]
 	path = openslides-manage-service
 	url = git@github.com:OpenSlides/openslides-manage-service.git

Makefile

@@ -6,7 +6,7 @@ run-service-tests:
 
 build-dev:
 	git submodule foreach 'make build-dev'
-	make -C haproxy build-dev
+	make -C proxy build-dev
 
 run-dev: | build-dev
 	docker-compose -f docker/docker-compose.dev.yml up
@@ -18,5 +18,12 @@ copy-node-modules:
 	docker-compose -f docker/docker-compose.dev.yml exec client bash -c "cp -r /app/node_modules/ /app/src/"
 	mv openslides-client/client/src/node_modules/ openslides-client/client/
 
-reload-haproxy:
+reload-proxy:
-	docker-compose -f docker/docker-compose.dev.yml kill -s HUP haproxy
+	docker-compose -f docker/docker-compose.dev.yml exec -w /etc/caddy proxy caddy reload
+
+services-to-master:
+	# Note: This script updates all submodules to upstream/master[1]. For setting the submodules to the linked
+	# commits use `git submodule update`. The `upstream` remote must be set up correctly to point to the main repo.
+	#
+	# [1] ...or main, or whatever branch the OS4 one is. See .gitmodules.
+	git submodule foreach -q --recursive 'git checkout $(git config -f $$toplevel/.gitmodules submodule.$$name.branch || echo master); git pull upstream $$(git config -f $$toplevel/.gitmodules submodule.$$name.branch || echo master)'

README.md

@@ -44,6 +44,7 @@ Setup the repository (may be already done)
 Prod setup. `./build.sh` may take a while.
 
     $ cd docker
+    $ m4 docker-compose.yml.m4 > docker-compose.yml
     $ ./build.sh
     $ ./setup-prod.sh
     $ docker-compose up

docker/build.sh

@@ -5,7 +5,7 @@ set -e
 HOME=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
 declare -A TARGETS
 TARGETS=(
-  [haproxy]="$HOME/../haproxy/"
+  [proxy]="$HOME/../proxy/"
   [client]="$HOME/../openslides-client/"
   [backend]="$HOME/../openslides-backend/"
   [auth]="$HOME/../openslides-auth-service/"
@@ -23,7 +23,7 @@ DOCKER_TAG="latest"
 CONFIG="/etc/osinstancectl"
 OPTIONS=()
 BUILT_IMAGES=()
-DEFAULT_TARGETS=(haproxy client backend auth autoupdate datastore-reader datastore-writer media)
+DEFAULT_TARGETS=(proxy client backend auth autoupdate datastore-reader datastore-writer media)
 
 usage() {
   cat << EOF

docker/docker-compose.dev.yml

@@ -103,8 +103,8 @@ services:
     - "8001:8001"
   message-bus:
     image: redis:latest
-  haproxy:
+  proxy:
-    image: openslides-haproxy-dev
+    image: openslides-proxy-dev
     depends_on:
       - client
       - backend
@@ -112,4 +112,4 @@ services:
     ports:
       - "8000:8000"
     volumes:
-      - ../haproxy/src:/usr/local/etc/haproxy
+      - ../proxy/Caddyfile.dev:/etc/caddy/Caddyfile

docker/docker-compose.yml.m4

@@ -15,10 +15,10 @@ define(`BACKEND_IMAGE',
 ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
 ifenvelse(`DOCKER_OPENSLIDES_BACKEND_NAME', openslides-backend):dnl
 ifenvelse(`DOCKER_OPENSLIDES_BACKEND_TAG', latest))
-define(`HAPROXY_IMAGE',
+define(`PROXY_IMAGE',
 ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
-ifenvelse(`DOCKER_OPENSLIDES_HAPROXY_NAME', openslides-haproxy):dnl
+ifenvelse(`DOCKER_OPENSLIDES_PROXY_NAME', openslides-proxy):dnl
-ifenvelse(`DOCKER_OPENSLIDES_HAPROXY_TAG', latest))
+ifenvelse(`DOCKER_OPENSLIDES_PROXY_TAG', latest))
 define(`CLIENT_IMAGE',
 ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl
 ifenvelse(`DOCKER_OPENSLIDES_CLIENT_NAME', openslides-client):dnl
@@ -60,8 +60,8 @@ dnl ----------------------------------------
 version: '3.4'
 
 services:
-  haproxy:
+  proxy:
-    image: HAPROXY_IMAGE
+    image: PROXY_IMAGE
     depends_on:
       - client
       - backend
@@ -183,7 +183,7 @@ services:
     - backend
     - auth
 
-# Setup: host <-uplink-> haproxy <-frontend-> services that are reachable from the client <-backend-> services that are internal-only
+# Setup: host <-uplink-> proxy <-frontend-> services that are reachable from the client <-backend-> services that are internal-only
 # There are special networks for some services only, e.g. postgres only for the postgresql, datastore reader and datastore writer
 networks:
   uplink:

haproxy/Dockerfile

@@ -1,5 +0,0 @@
-FROM haproxy:2.0-alpine
-COPY src/haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg
-COPY src/haproxy.prod.cfg /usr/local/etc/haproxy/haproxy.prod.cfg
-COPY src/combined.pem /usr/local/etc/haproxy/combined.pem
-CMD ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.cfg", "-f", "/usr/local/etc/haproxy/haproxy.prod.cfg"]

haproxy/Dockerfile.dev

@@ -1,5 +0,0 @@
-FROM haproxy:2.0-alpine
-COPY src/haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg
-COPY src/haproxy.dev.cfg /usr/local/etc/haproxy/haproxy.dev.cfg
-COPY src/combined.pem /usr/local/etc/haproxy/combined.pem
-CMD ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.cfg", "-f", "/usr/local/etc/haproxy/haproxy.dev.cfg"]

haproxy/Makefile

@@ -1,3 +0,0 @@
-build-dev:
-	./prepare-cert.sh
-	docker build -t openslides-haproxy-dev -f Dockerfile.dev .

haproxy/build.sh

@@ -1,3 +0,0 @@
-./prepare-cert.sh
-docker build --tag "${img:-openslides/openslides-haproxy:latest}" \
-    --pull "${OPTIONS[@]}" .

haproxy/prepare-cert.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-
-set -e
-cd "$(dirname "$0")"
-
-# check, if we already generated a cert
-combined="src/combined.pem"
-
-if [[ ! -f $combined ]]; then
-    echo "Creating certificates..."
-    cd src
-    if type 2>&1 >/dev/null openssl ; then
-      echo "Using openssl to generate a certificate."
-      echo "You will need to accept an security exception for the"
-      echo "generated certificate in your browser manually."
-      openssl req -x509 -newkey rsa:4096 -nodes -days 3650 \
-              -subj "/C=DE/O=Selfsigned Test/CN=localhost" \
-              -keyout localhost-key.pem -out localhost.pem
-    else
-      echo >&2 "FATAL: No valid certificate generation tool found!"
-      exit -1
-    fi
-    cat localhost.pem localhost-key.pem > combined.pem
-    echo "done"
-else
-    echo "Certificate exists."
-fi

haproxy/src/haproxy.cfg

@@ -1,87 +0,0 @@
-global
-    log stdout  format raw  local0  debug
-
-defaults
-    option http-use-htx
-    timeout connect 3s
-    timeout client 10s
-    timeout client-fin 10s
-    timeout server 10s
-    timeout server-fin 10s
-    timeout check 2s
-    timeout tunnel 10s
-    timeout queue 2s
-    log global
-    option httplog
-
-# We have to wait for 2.3: https://github.com/haproxy/haproxy/issues/737
-# WebSocket handling is broken in HaProxy 2.x, x<3
-#frontend uplink
-#    mode tcp
-#    bind :8000
-#    tcp-request inspect-delay 2s
-#    tcp-request content accept if HTTP
-#    tcp-request content accept if { req.ssl_hello_type 1 }
-#    use_backend receive_http if HTTP
-#    default_backend receive_https
-#backend receive_http
-#    mode tcp
-#    server loopback-for-http abns@http send-proxy-v2
-#backend receive_https
-#    mode tcp
-#    server loopback-for-https abns@https send-proxy-v2
-
-#frontend http
-#    mode http
-#    bind abns@http accept-proxy
-#    redirect scheme https code 301
-
-frontend https
-    mode http
-    #bind abns@https accept-proxy ssl crt /usr/local/etc/haproxy/combined.pem alpn h2,http/1.1
-    bind *:8000 ssl crt /usr/local/etc/haproxy/combined.pem alpn h2,http/1.1
-    default_backend backend_client  # this is defined in the dev-*/prod-* file
-
-    acl action path_beg -i /system/action
-    use_backend backend_action if action
-
-    acl presenter path_beg -i /system/presenter
-    use_backend backend_presenter if presenter
-
-    acl autoupdate path_beg -i /system/autoupdate
-    use_backend backend_autoupdate if autoupdate
-
-    acl auth path_beg -i /system/auth
-    use_backend backend_auth if auth
-
-    acl media path_beg -i /system/media
-    use_backend backend_media if media
-
-    stats enable
-    stats uri /stats
-    stats refresh 10s
-    stats auth admin:admin
-
-resolvers docker_resolver
-    nameserver dns 127.0.0.11:53
-
-backend backend_action
-    mode http
-    server action backend:9002 resolvers docker_resolver check
-
-backend backend_presenter
-    mode http
-    server presenter backend:9003 resolvers docker_resolver check
-
-backend backend_autoupdate
-    mode http
-    timeout server 1h
-    server autoupdate autoupdate:9012 resolvers docker_resolver check ssl verify none alpn h2
-
-backend backend_auth
-    mode http
-    server auth auth:9004 resolvers docker_resolver check
-
-backend backend_media
-    mode http
-    server media media:9006 resolvers docker_resolver check

haproxy/src/haproxy.dev.cfg

@@ -1,4 +0,0 @@
-backend backend_client
-    mode http
-    timeout tunnel 1h
-    server client client:9001 resolvers docker_resolver no-check

haproxy/src/haproxy.prod.cfg

@@ -1,3 +0,0 @@
-backend backend_client
-    mode http
-    server client client:9001 resolvers docker_resolver check

openslides-autoupdate-service

@@ -1 +1 @@
-Subproject commit 8b1aec26a291d86a42c25920f550b2e321b4a1bd
+Subproject commit fb6e25d7a88ec8202b5080b5563e95451b6071c3

openslides-backend

@@ -1 +1 @@
-Subproject commit 617c098777cbdaac6f32c928c5b7f06cf7c0bb5e
+Subproject commit acef4bbf409f53f90f34f68a6ab2c5794f023981

proxy/Caddyfile

@@ -0,0 +1,13 @@
+import endpoint
+
+    reverse_proxy /system/action/* backend:9002
+    reverse_proxy /system/presenter/* backend:9003
+    reverse_proxy /system/autoupdate/* autoupdate:9012 {
+        flush_interval -1
+    }
+    reverse_proxy /system/auth/* auth:9004
+    reverse_proxy /system/media/* media:9006
+
+    reverse_proxy client:9001
+
+}

proxy/Caddyfile.dev

@@ -0,0 +1,13 @@
+https://:8000 {
+    tls /certs/cert.pem /certs/key.pem
+
+    reverse_proxy /system/action* backend:9002
+    reverse_proxy /system/presenter* backend:9003
+    reverse_proxy /system/autoupdate* autoupdate:9012 {
+        flush_interval -1
+    }
+    reverse_proxy /system/auth* auth:9004
+    reverse_proxy /system/media* media:9006
+
+    reverse_proxy client:9001
+}

proxy/Dockerfile

@@ -0,0 +1,8 @@
+FROM caddy:2.3.0-alpine
+
+COPY Caddyfile /etc/caddy/Caddyfile
+COPY entrypoint /entrypoint
+COPY certs /certs
+
+ENTRYPOINT ["/entrypoint"]
+CMD ["caddy", "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"]

proxy/Dockerfile.dev

@@ -0,0 +1,4 @@
+FROM caddy:2.3.0-alpine
+
+COPY Caddyfile.dev /etc/caddy/Caddyfile
+COPY certs /certs

proxy/Makefile

@@ -0,0 +1,3 @@
+build-dev:
+	./make-localhost-cert.sh
+	docker build -t openslides-proxy-dev -f Dockerfile.dev .

proxy/entrypoint

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -e
+
+if [[ -f "/certs/key.pem" ]] && [[ -f "/certs/cert.pem" ]]; then
+    cat <<EOF >> /etc/caddy/endpoint
+https://:8000 {
+    tls /certs/cert.pem /certs/key.pem
+EOF
+    echo "Configured https"
+else
+    echo "http://:8000 {" > /etc/caddy/endpoint
+    echo "Configured http"
+fi
+
+exec "$@"

proxy/make-localhost-cert.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+cd "$(dirname "$0")"
+
+if [[ -f "certs/key.pem" ]] || [[ -f "certs/cert.pem" ]]; then
+    echo "Certificate already exists."
+    exit 0
+fi
+
+if ! type 2>&1 >/dev/null openssl ; then
+    echo >&2 "Error: openssl not found!"
+    exit 1
+fi
+
+echo "Creating certificates..."
+echo "You will need to accept an security exception for the"
+echo "generated certificate in your browser manually."
+openssl req -x509 -newkey rsa:4096 -nodes -days 3650 \
+        -subj "/C=DE/O=Selfsigned Test/CN=localhost" \
+        -keyout certs/key.pem -out certs/cert.pem
+echo "done"