Merge pull request #5850 from FinnStutzenstein/ownSessionRedis

Own session redis

README.rst

@@ -108,8 +108,7 @@ Used software
 
 OpenSlides uses the following projects or parts of them:
 
-* several Python packages (see ``server/requirements/production.txt`` and
+* several Python packages (see ``server/requirements/production.txt``)
-  ``server/requirements/big_mode.txt``)
 
 * several JavaScript packages (see ``client/package.json``)
 

autoupdate

@@ -1 +1 @@
-Subproject commit 756043511cc00b9fd4b42cc3a7ba0d8c16897895
+Subproject commit dbf4fef1b7ff9f2c1973ac1077d4a07cdf5aa41d

caddy/Caddyfile

@@ -1,8 +1,3 @@
-{
-    # Enable debug output
-    #debug
-}
-
 :8000
 
 reverse_proxy /system/* autoupdate:8002 {

caddy/Caddyfile.dev

@@ -1,5 +1,4 @@
 {
-    # Enable debug output
     debug
 }
 

docker/docker-compose.dev.yml

@@ -34,8 +34,6 @@ services:
     environment:
         - MESSAGE_BUS_HOST=redis
         - MESSAGE_BUS_PORT=6379
-        - WORKER_HOST=server
-        - WORKER_PORT=8000
     depends_on:
         - server
         - redis

docker/docker-compose.yml.m4

@@ -158,10 +158,11 @@ services:
     depends_on:
       - redis
       - server
+    secrets:
+      - django
     environment:
       REDIS_WRITE_HOST: redis
       MESSAGE_BUS_HOST: redis-slave
-      WORKER_HOST: server
     networks:
       - back
 

docker/docker-stack.yml.m4

@@ -156,10 +156,12 @@ services:
   autoupdate:
     image: AUTOUPDATE_IMAGE
     environment:
-      MESSAGE_BUS_HOST: redis
+      REDIS_WRITE_HOST: redis
-      WORKER_HOST: server
+      MESSAGE_BUS_HOST: redis-slave
     networks:
       - back
+    secrets:
+      - django
     deploy:
       replicas: ifenvelse(`OPENSLIDES_AUTOUPDATE_SERVICE_REPLICAS', 1)
       restart_policy:

server/docker/Dockerfile

@@ -51,8 +51,7 @@ RUN apt-get install --no-install-recommends -y \
 RUN rm -rf /var/lib/apt/lists/*
 
 COPY requirements /app/requirements
-RUN pip install -r requirements/production.txt -r requirements/big_mode.txt \
+RUN pip install -r requirements/production.txt -r requirements/saml.txt && \
-    -r requirements/saml.txt && \
     rm -rf /root/.cache/pip
 
 # SAML

server/docker/entrypoint-dev

@@ -2,18 +2,47 @@
 
 set -e
 
+function isSettingsFileOk() {
+    # Forbidden keys to set
+    # - OPENSLIDES_USER_DATA_DIR
+    # - SESSION_ENGINE
+    if grep -q "OPENSLIDES_USER_DATA_DIR.*=" /app/personal_data/var/settings.py; then
+        echo "Found forbidden key OPENSLIDES_USER_DATA_DIR"
+        return 1;
+    fi
+
+    if grep -q "SESSION_ENGINE.*=" /app/personal_data/var/settings.py; then
+        echo "Found forbidden key SESSION_ENGINE"
+        return 1;
+    fi
+
+    return 0;
+}
+
 wait-for-it -t 0 redis:6379
 
 until pg_isready -h postgres -p 5432 -U openslides; do
-  echo "Waiting for Postgres to become available..."
+    echo "Waiting for Postgres to become available..."
-  sleep 3
+    sleep 3
 done
 
 if [[ ! -f "/app/personal_data/var/settings.py" ]]; then
     echo "Create settings"
     python manage.py createsettings
+else
+    echo "Settings exists - checking for invalid configurations"
+    if ! isSettingsFileOk; then
+        echo "Settings are not ok."
+        echo "Saving old settings in settings.py.bak"
+        mv /app/personal_data/var/settings.py /app/personal_data/var/settings.py.bak
+        python manage.py createsettings
+    else
+        echo "Settings are ok."
+    fi
 fi
 
+sed -i -e "s/SECRET_KEY.*$/SECRET_KEY = 'development'/" /app/personal_data/var/settings.py
+
 python -u manage.py migrate
 
 exec "$@"

server/docker/settings.py

@@ -107,16 +107,6 @@ REDIS_ADDRESS = f"redis://{REDIS_HOST}:{REDIS_PORT}/0"
 REDIS_READ_ONLY_ADDRESS = f"redis://{REDIS_SLAVE_HOST}:{REDIS_SLAVE_PORT}/0"
 CONNECTION_POOL_LIMIT = get_env("CONNECTION_POOL_LIMIT", 100, int)
 
-# Session backend
-SESSION_ENGINE = "redis_sessions.session"
-SESSION_REDIS = {
-    "host": REDIS_HOST,
-    "port": REDIS_PORT,
-    "db": 0,
-    "prefix": "session",
-    "socket_timeout": 2,
-}
-
 # SAML integration
 ENABLE_SAML = get_env("ENABLE_SAML", False, bool)
 if ENABLE_SAML:

server/openslides/global_settings.py

@@ -5,6 +5,8 @@ from openslides.utils.plugins import collect_plugins
 
 MODULE_DIR = os.path.realpath(os.path.dirname(os.path.abspath(__file__)))
 
+# This is not set to the docker environment
+OPENSLIDES_USER_DATA_DIR = "/app/personal_data/var"
 
 # Application definition
 
@@ -49,6 +51,8 @@ TEMPLATES = [
     }
 ]
 
+SESSION_ENGINE = "openslides.utils.sessions"
+
 # Email
 # https://docs.djangoproject.com/en/1.10/topics/email/
 
@@ -84,11 +88,19 @@ LOCALE_PATHS = [os.path.join(MODULE_DIR, "locale")]
 
 
 # Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/1.10/howto/static-files/
 
 STATIC_URL = "/static/"
 
-STATICFILES_DIRS = [os.path.join(MODULE_DIR, "static")]
+STATICFILES_DIRS = [os.path.join(MODULE_DIR, "static")] + [
+    os.path.join(OPENSLIDES_USER_DATA_DIR, "static")
+]
+
+# Static files (CSS, JavaScript, Images)
+STATIC_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, "collected-static")
+
+# Files
+# https://docs.djangoproject.com/en/1.10/topics/files/
+MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, "media", "")
 
 
 # Sessions and user authentication

server/openslides/utils/sessions.py

@@ -0,0 +1,122 @@
+# type: ignore
+
+from asgiref.sync import async_to_sync
+from django.conf import settings
+from django.contrib.sessions.backends.base import (
+    VALID_KEY_CHARS,
+    CreateError,
+    SessionBase,
+)
+from django.utils.crypto import get_random_string, salted_hmac
+from django.utils.encoding import force_str
+
+from .redis import get_connection
+
+
+REDIS_SESSION_PREFIX = getattr(settings, "REDIS_SESSION_PREFIX", "session:")
+
+
+def to_redis_key(session_key):
+    return f"{REDIS_SESSION_PREFIX}{session_key}"
+
+
+class SessionStore(SessionBase):
+    """
+    Implements Redis database session store.
+    """
+
+    def __init__(self, session_key=None):
+        super(SessionStore, self).__init__(session_key)
+
+    def _hash(self, value):
+        key_salt = "openslides.utils.sessions.SessionStore"
+        return salted_hmac(key_salt, value).hexdigest()
+
+    def load(self):
+        return async_to_sync(self._load)()
+
+    async def _load(self):
+        async with get_connection(read_only=True) as redis:
+            try:
+                key = to_redis_key(self._get_or_create_session_key())
+                print("laod:", key)
+                session_data = await redis.get(key)
+                x = self.decode(force_str(session_data))
+                print("load result:", x)
+                return x
+            except Exception as e:
+                print("load ex", e)
+                self._session_key = None
+                return {}
+
+    def exists(self, session_key):
+        return async_to_sync(self._load)(session_key)
+
+    async def _exists(self, session_key):
+        async with get_connection(read_only=True) as redis:
+            key = to_redis_key(session_key)
+            print("exists:", key)
+            x = await redis.exists(key)
+            print("exists result:", x)
+            return x
+
+    def create(self):
+        async_to_sync(self._create)()
+
+    async def _create(self):
+        while True:
+            self._session_key = await self._async_get_new_session_key()
+
+            try:
+                await self._save(must_create=True)
+            except CreateError:
+                # Key wasn't unique. Try again.
+                continue
+            self.modified = True
+            return
+
+    def save(self, must_create=False):
+        async_to_sync(self._save)(must_create)
+
+    async def _save(self, must_create=False):
+        async with get_connection() as redis:
+            if self.session_key is None:
+                return await self._create()
+            if must_create and await self._exists(self._get_or_create_session_key()):
+                raise CreateError
+            data = self.encode(self._get_session(no_load=must_create))
+            print("Save:", self._get_or_create_session_key(), data)
+            await redis.setex(
+                to_redis_key(self._get_or_create_session_key()),
+                self.get_expiry_age(),
+                data,
+            )
+
+    def delete(self, session_key=None):
+        async_to_sync(self._delete)(session_key)
+
+    async def _delete(self, session_key=None):
+        if session_key is None:
+            if self.session_key is None:
+                return
+            session_key = self.session_key
+
+        async with get_connection() as redis:
+            try:
+                print("delete:", to_redis_key(session_key))
+                await redis.delete(to_redis_key(session_key))
+            except Exception as e:
+                print("delete ex:", e)
+                pass
+
+    # This must be overwritten to stay inside async code...
+    async def _async_get_new_session_key(self):
+        "Return session key that isn't being used."
+        while True:
+            session_key = get_random_string(32, VALID_KEY_CHARS)
+            if not await self._exists(session_key):
+                return session_key
+
+    @classmethod
+    def clear_expired(cls):
+        pass

server/openslides/utils/settings.py.tpl

@@ -7,29 +7,18 @@ https://github.com/OpenSlides/OpenSlides/blob/master/SETTINGS.rst
 
 import os
 from openslides.global_settings import *
-%(import_function)s
-
-# The directory for user specific data files
-
-OPENSLIDES_USER_DATA_DIR = %(openslides_user_data_dir)s
 
 
 # OpenSlides plugins
-
 # Add plugins to this list (see example entry in comment).
 
 INSTALLED_PLUGINS += (
 #    'plugin_module_name',
 )
-
 INSTALLED_APPS += INSTALLED_PLUGINS
 
 
-# Important settings for production use
-# https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
-
 # SECURITY WARNING: Keep the secret key used in production secret!
-
 SECRET_KEY = %(secret_key)r
 
 # Use 'DEBUG = True' to get more details for server errors.
@@ -60,7 +49,7 @@ DEFAULT_FROM_EMAIL = 'noreply@example.com'
 DATA_UPLOAD_MAX_MEMORY_SIZE = 104857600
 
 # Database
-# https://docs.djangoproject.com/en/1.10/ref/settings/#databases
+# https://docs.djangoproject.com/en/2.2/ref/settings/#databases
 
 # Change this setting to use e. g. PostgreSQL or MySQL.
 
@@ -86,17 +75,6 @@ DATABASES = {
 # Collection Cache
 REDIS_ADDRESS = "redis://redis:6379/0"
 
-# Session backend
-# Redis configuration for django-redis-sessions.
-# https://github.com/martinrusev/django-redis-sessions
-SESSION_ENGINE = 'redis_sessions.session'
-SESSION_REDIS = {
-    'host': 'redis',
-    'port': 6379,
-    'db': 0,
-    'prefix': 'session',
-    'socket_timeout': 2
-}
 
 # SAML integration
 # Please read https://github.com/OpenSlides/OpenSlides/blob/master/openslides/saml/README.md
@@ -120,23 +98,12 @@ ENABLE_CHAT = False
 
 
 # Internationalization
-# https://docs.djangoproject.com/en/1.10/topics/i18n/
+# https://docs.djangoproject.com/en/2.2/topics/i18n/
 TIME_ZONE = 'Europe/Berlin'
 
 
-# Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/1.10/howto/static-files/
-STATICFILES_DIRS = [os.path.join(OPENSLIDES_USER_DATA_DIR, 'static')] + STATICFILES_DIRS
-STATIC_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, 'collected-static')
-
-
-# Files
-# https://docs.djangoproject.com/en/1.10/topics/files/
-MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, 'media', '')
-
-
 # Password validation
-# https://docs.djangoproject.com/en/1.10/topics/auth/passwords/#module-django.contrib.auth.password_validation
+# https://docs.djangoproject.com/en/2.2/topics/auth/passwords/#module-django.contrib.auth.password_validation
 # AUTH_PASSWORD_VALIDATORS = []
 
 

server/requirements.txt

@@ -1,6 +1,5 @@
 # Requirements for OpenSlides in production
 -r requirements/production.txt
--r requirements/big_mode.txt
 
 # Requirements for development and tests in alphabetical order
 -r requirements/development.txt

server/requirements/big_mode.txt

@@ -1,11 +0,0 @@
-# Requirements for Redis and PostgreSQL support
-django-redis-sessions>=0.6.1,<0.7
-psycopg2-binary>=2.7.3.2,<2.9
-aioredis>=1.1.0,<1.3
-
-# Requirements for fast asgi server
-#gunicorn>=19.9.0,<20
-
-# https://github.com/benoitc/gunicorn/issues/1913
-git+https://github.com/FinnStutzenstein/gunicorn.git@fix
-uvicorn[standard]>=0.9,<1.0

server/requirements/production.txt

@@ -6,6 +6,7 @@ autobahn==19.5.1
 asgiref>=3.2.9
 
 # Requirements for OpenSlides in production in alphabetical order
+aioredis>=1.1.0,<1.3
 bleach>=1.5.0,<3.2
 daphne>=2.2,<2.5
 Django>=2.1,<2.3
@@ -14,9 +15,14 @@ jsonfield2>=3.0,<3.1
 attrs>=19.2.0
 jsonschema>=3.0,<3.1
 mypy_extensions>=0.4,<0.5
+psycopg2-binary>=2.7.3.2,<2.9
 PyPDF2>=1.26,<1.27
 roman>=2.0,<3.2
 setuptools>=29.0,<42.0
 typing_extensions>=3.6.6,<3.8
 websockets>=8.0,<9.0
 twisted[tls]>=20.3.0
+uvicorn[standard]>=0.9,<1.0
+
+# https://github.com/benoitc/gunicorn/issues/1913
+git+https://github.com/FinnStutzenstein/gunicorn.git@fix