From 35e8f84fda14bd79239d05087b432f1f7caf0d85 Mon Sep 17 00:00:00 2001 From: Finn Stutzenstein Date: Thu, 4 Feb 2021 17:57:01 +0100 Subject: [PATCH] Own session redis - removed big mode artifacts - removed django_session_redis - Enforce newly created settings.py when invalid options are set: * OPENSLIDES_USER_DATA_DIR * SESSION_ENGINE - Overwrites the secret key during development to "development" --- README.rst | 3 +- autoupdate | 2 +- caddy/Caddyfile | 5 - caddy/Caddyfile.dev | 1 - docker/docker-compose.dev.yml | 2 - docker/docker-compose.yml.m4 | 3 +- docker/docker-stack.yml.m4 | 6 +- server/docker/Dockerfile | 3 +- server/docker/entrypoint-dev | 33 ++++++- server/docker/settings.py | 10 -- server/openslides/global_settings.py | 16 +++- server/openslides/utils/sessions.py | 122 ++++++++++++++++++++++++ server/openslides/utils/settings.py.tpl | 39 +------- server/requirements.txt | 1 - server/requirements/big_mode.txt | 11 --- server/requirements/production.txt | 6 ++ 16 files changed, 185 insertions(+), 78 deletions(-) create mode 100644 server/openslides/utils/sessions.py delete mode 100644 server/requirements/big_mode.txt diff --git a/README.rst b/README.rst index 3aac2f9e4..e3f5e07fe 100644 --- a/README.rst +++ b/README.rst @@ -108,8 +108,7 @@ Used software OpenSlides uses the following projects or parts of them: -* several Python packages (see ``server/requirements/production.txt`` and - ``server/requirements/big_mode.txt``) +* several Python packages (see ``server/requirements/production.txt``) * several JavaScript packages (see ``client/package.json``) diff --git a/autoupdate b/autoupdate index 756043511..dbf4fef1b 160000 --- a/autoupdate +++ b/autoupdate @@ -1 +1 @@ -Subproject commit 756043511cc00b9fd4b42cc3a7ba0d8c16897895 +Subproject commit dbf4fef1b7ff9f2c1973ac1077d4a07cdf5aa41d diff --git a/caddy/Caddyfile b/caddy/Caddyfile index 4122bbacf..b00ddd0aa 100644 --- a/caddy/Caddyfile +++ b/caddy/Caddyfile @@ -1,8 +1,3 @@ -{ - # Enable debug output - #debug -} - :8000 reverse_proxy /system/* autoupdate:8002 { diff --git a/caddy/Caddyfile.dev b/caddy/Caddyfile.dev index bdf5a139d..a75b4c8c2 100644 --- a/caddy/Caddyfile.dev +++ b/caddy/Caddyfile.dev @@ -1,5 +1,4 @@ { - # Enable debug output debug } diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 6acd282b4..809163598 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -34,8 +34,6 @@ services: environment: - MESSAGE_BUS_HOST=redis - MESSAGE_BUS_PORT=6379 - - WORKER_HOST=server - - WORKER_PORT=8000 depends_on: - server - redis diff --git a/docker/docker-compose.yml.m4 b/docker/docker-compose.yml.m4 index 3c9b1a77a..d29eddff7 100644 --- a/docker/docker-compose.yml.m4 +++ b/docker/docker-compose.yml.m4 @@ -158,10 +158,11 @@ services: depends_on: - redis - server + secrets: + - django environment: REDIS_WRITE_HOST: redis MESSAGE_BUS_HOST: redis-slave - WORKER_HOST: server networks: - back diff --git a/docker/docker-stack.yml.m4 b/docker/docker-stack.yml.m4 index 7bb0b024a..95ea3f9a3 100644 --- a/docker/docker-stack.yml.m4 +++ b/docker/docker-stack.yml.m4 @@ -156,10 +156,12 @@ services: autoupdate: image: AUTOUPDATE_IMAGE environment: - MESSAGE_BUS_HOST: redis - WORKER_HOST: server + REDIS_WRITE_HOST: redis + MESSAGE_BUS_HOST: redis-slave networks: - back + secrets: + - django deploy: replicas: ifenvelse(`OPENSLIDES_AUTOUPDATE_SERVICE_REPLICAS', 1) restart_policy: diff --git a/server/docker/Dockerfile b/server/docker/Dockerfile index 5fd29b856..bde5716e5 100644 --- a/server/docker/Dockerfile +++ b/server/docker/Dockerfile @@ -51,8 +51,7 @@ RUN apt-get install --no-install-recommends -y \ RUN rm -rf /var/lib/apt/lists/* COPY requirements /app/requirements -RUN pip install -r requirements/production.txt -r requirements/big_mode.txt \ - -r requirements/saml.txt && \ +RUN pip install -r requirements/production.txt -r requirements/saml.txt && \ rm -rf /root/.cache/pip # SAML diff --git a/server/docker/entrypoint-dev b/server/docker/entrypoint-dev index 25632f973..fb44f22f0 100755 --- a/server/docker/entrypoint-dev +++ b/server/docker/entrypoint-dev @@ -2,18 +2,47 @@ set -e +function isSettingsFileOk() { + # Forbidden keys to set + # - OPENSLIDES_USER_DATA_DIR + # - SESSION_ENGINE + if grep -q "OPENSLIDES_USER_DATA_DIR.*=" /app/personal_data/var/settings.py; then + echo "Found forbidden key OPENSLIDES_USER_DATA_DIR" + return 1; + fi + + if grep -q "SESSION_ENGINE.*=" /app/personal_data/var/settings.py; then + echo "Found forbidden key SESSION_ENGINE" + return 1; + fi + + return 0; +} + wait-for-it -t 0 redis:6379 until pg_isready -h postgres -p 5432 -U openslides; do - echo "Waiting for Postgres to become available..." - sleep 3 + echo "Waiting for Postgres to become available..." + sleep 3 done if [[ ! -f "/app/personal_data/var/settings.py" ]]; then echo "Create settings" python manage.py createsettings +else + echo "Settings exists - checking for invalid configurations" + if ! isSettingsFileOk; then + echo "Settings are not ok." + echo "Saving old settings in settings.py.bak" + mv /app/personal_data/var/settings.py /app/personal_data/var/settings.py.bak + python manage.py createsettings + else + echo "Settings are ok." + fi fi +sed -i -e "s/SECRET_KEY.*$/SECRET_KEY = 'development'/" /app/personal_data/var/settings.py + python -u manage.py migrate exec "$@" diff --git a/server/docker/settings.py b/server/docker/settings.py index ce66c5a5e..633749c26 100644 --- a/server/docker/settings.py +++ b/server/docker/settings.py @@ -107,16 +107,6 @@ REDIS_ADDRESS = f"redis://{REDIS_HOST}:{REDIS_PORT}/0" REDIS_READ_ONLY_ADDRESS = f"redis://{REDIS_SLAVE_HOST}:{REDIS_SLAVE_PORT}/0" CONNECTION_POOL_LIMIT = get_env("CONNECTION_POOL_LIMIT", 100, int) -# Session backend -SESSION_ENGINE = "redis_sessions.session" -SESSION_REDIS = { - "host": REDIS_HOST, - "port": REDIS_PORT, - "db": 0, - "prefix": "session", - "socket_timeout": 2, -} - # SAML integration ENABLE_SAML = get_env("ENABLE_SAML", False, bool) if ENABLE_SAML: diff --git a/server/openslides/global_settings.py b/server/openslides/global_settings.py index c1d0f8bac..adb45929c 100644 --- a/server/openslides/global_settings.py +++ b/server/openslides/global_settings.py @@ -5,6 +5,8 @@ from openslides.utils.plugins import collect_plugins MODULE_DIR = os.path.realpath(os.path.dirname(os.path.abspath(__file__))) +# This is not set to the docker environment +OPENSLIDES_USER_DATA_DIR = "/app/personal_data/var" # Application definition @@ -49,6 +51,8 @@ TEMPLATES = [ } ] +SESSION_ENGINE = "openslides.utils.sessions" + # Email # https://docs.djangoproject.com/en/1.10/topics/email/ @@ -84,11 +88,19 @@ LOCALE_PATHS = [os.path.join(MODULE_DIR, "locale")] # Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.10/howto/static-files/ STATIC_URL = "/static/" -STATICFILES_DIRS = [os.path.join(MODULE_DIR, "static")] +STATICFILES_DIRS = [os.path.join(MODULE_DIR, "static")] + [ + os.path.join(OPENSLIDES_USER_DATA_DIR, "static") +] + +# Static files (CSS, JavaScript, Images) +STATIC_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, "collected-static") + +# Files +# https://docs.djangoproject.com/en/1.10/topics/files/ +MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, "media", "") # Sessions and user authentication diff --git a/server/openslides/utils/sessions.py b/server/openslides/utils/sessions.py new file mode 100644 index 000000000..640dfee42 --- /dev/null +++ b/server/openslides/utils/sessions.py @@ -0,0 +1,122 @@ +# type: ignore + +from asgiref.sync import async_to_sync +from django.conf import settings +from django.contrib.sessions.backends.base import ( + VALID_KEY_CHARS, + CreateError, + SessionBase, +) +from django.utils.crypto import get_random_string, salted_hmac +from django.utils.encoding import force_str + +from .redis import get_connection + + +REDIS_SESSION_PREFIX = getattr(settings, "REDIS_SESSION_PREFIX", "session:") + + +def to_redis_key(session_key): + return f"{REDIS_SESSION_PREFIX}{session_key}" + + +class SessionStore(SessionBase): + """ + Implements Redis database session store. + """ + + def __init__(self, session_key=None): + super(SessionStore, self).__init__(session_key) + + def _hash(self, value): + key_salt = "openslides.utils.sessions.SessionStore" + return salted_hmac(key_salt, value).hexdigest() + + def load(self): + return async_to_sync(self._load)() + + async def _load(self): + async with get_connection(read_only=True) as redis: + try: + key = to_redis_key(self._get_or_create_session_key()) + print("laod:", key) + session_data = await redis.get(key) + x = self.decode(force_str(session_data)) + print("load result:", x) + return x + except Exception as e: + print("load ex", e) + self._session_key = None + return {} + + def exists(self, session_key): + return async_to_sync(self._load)(session_key) + + async def _exists(self, session_key): + async with get_connection(read_only=True) as redis: + key = to_redis_key(session_key) + print("exists:", key) + x = await redis.exists(key) + print("exists result:", x) + return x + + def create(self): + async_to_sync(self._create)() + + async def _create(self): + while True: + self._session_key = await self._async_get_new_session_key() + + try: + await self._save(must_create=True) + except CreateError: + # Key wasn't unique. Try again. + continue + self.modified = True + return + + def save(self, must_create=False): + async_to_sync(self._save)(must_create) + + async def _save(self, must_create=False): + async with get_connection() as redis: + if self.session_key is None: + return await self._create() + if must_create and await self._exists(self._get_or_create_session_key()): + raise CreateError + data = self.encode(self._get_session(no_load=must_create)) + print("Save:", self._get_or_create_session_key(), data) + await redis.setex( + to_redis_key(self._get_or_create_session_key()), + self.get_expiry_age(), + data, + ) + + def delete(self, session_key=None): + async_to_sync(self._delete)(session_key) + + async def _delete(self, session_key=None): + if session_key is None: + if self.session_key is None: + return + session_key = self.session_key + + async with get_connection() as redis: + try: + print("delete:", to_redis_key(session_key)) + await redis.delete(to_redis_key(session_key)) + except Exception as e: + print("delete ex:", e) + pass + + # This must be overwritten to stay inside async code... + async def _async_get_new_session_key(self): + "Return session key that isn't being used." + while True: + session_key = get_random_string(32, VALID_KEY_CHARS) + if not await self._exists(session_key): + return session_key + + @classmethod + def clear_expired(cls): + pass diff --git a/server/openslides/utils/settings.py.tpl b/server/openslides/utils/settings.py.tpl index fd5da32d7..96bd2bf03 100644 --- a/server/openslides/utils/settings.py.tpl +++ b/server/openslides/utils/settings.py.tpl @@ -7,29 +7,18 @@ https://github.com/OpenSlides/OpenSlides/blob/master/SETTINGS.rst import os from openslides.global_settings import * -%(import_function)s - -# The directory for user specific data files - -OPENSLIDES_USER_DATA_DIR = %(openslides_user_data_dir)s # OpenSlides plugins - # Add plugins to this list (see example entry in comment). INSTALLED_PLUGINS += ( # 'plugin_module_name', ) - INSTALLED_APPS += INSTALLED_PLUGINS -# Important settings for production use -# https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ - # SECURITY WARNING: Keep the secret key used in production secret! - SECRET_KEY = %(secret_key)r # Use 'DEBUG = True' to get more details for server errors. @@ -60,7 +49,7 @@ DEFAULT_FROM_EMAIL = 'noreply@example.com' DATA_UPLOAD_MAX_MEMORY_SIZE = 104857600 # Database -# https://docs.djangoproject.com/en/1.10/ref/settings/#databases +# https://docs.djangoproject.com/en/2.2/ref/settings/#databases # Change this setting to use e. g. PostgreSQL or MySQL. @@ -86,17 +75,6 @@ DATABASES = { # Collection Cache REDIS_ADDRESS = "redis://redis:6379/0" -# Session backend -# Redis configuration for django-redis-sessions. -# https://github.com/martinrusev/django-redis-sessions -SESSION_ENGINE = 'redis_sessions.session' -SESSION_REDIS = { - 'host': 'redis', - 'port': 6379, - 'db': 0, - 'prefix': 'session', - 'socket_timeout': 2 -} # SAML integration # Please read https://github.com/OpenSlides/OpenSlides/blob/master/openslides/saml/README.md @@ -120,23 +98,12 @@ ENABLE_CHAT = False # Internationalization -# https://docs.djangoproject.com/en/1.10/topics/i18n/ +# https://docs.djangoproject.com/en/2.2/topics/i18n/ TIME_ZONE = 'Europe/Berlin' -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.10/howto/static-files/ -STATICFILES_DIRS = [os.path.join(OPENSLIDES_USER_DATA_DIR, 'static')] + STATICFILES_DIRS -STATIC_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, 'collected-static') - - -# Files -# https://docs.djangoproject.com/en/1.10/topics/files/ -MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, 'media', '') - - # Password validation -# https://docs.djangoproject.com/en/1.10/topics/auth/passwords/#module-django.contrib.auth.password_validation +# https://docs.djangoproject.com/en/2.2/topics/auth/passwords/#module-django.contrib.auth.password_validation # AUTH_PASSWORD_VALIDATORS = [] diff --git a/server/requirements.txt b/server/requirements.txt index 952727e81..22f31d76f 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,6 +1,5 @@ # Requirements for OpenSlides in production -r requirements/production.txt --r requirements/big_mode.txt # Requirements for development and tests in alphabetical order -r requirements/development.txt diff --git a/server/requirements/big_mode.txt b/server/requirements/big_mode.txt deleted file mode 100644 index fd8ead856..000000000 --- a/server/requirements/big_mode.txt +++ /dev/null @@ -1,11 +0,0 @@ -# Requirements for Redis and PostgreSQL support -django-redis-sessions>=0.6.1,<0.7 -psycopg2-binary>=2.7.3.2,<2.9 -aioredis>=1.1.0,<1.3 - -# Requirements for fast asgi server -#gunicorn>=19.9.0,<20 - -# https://github.com/benoitc/gunicorn/issues/1913 -git+https://github.com/FinnStutzenstein/gunicorn.git@fix -uvicorn[standard]>=0.9,<1.0 diff --git a/server/requirements/production.txt b/server/requirements/production.txt index d622807d8..8e31cb540 100644 --- a/server/requirements/production.txt +++ b/server/requirements/production.txt @@ -6,6 +6,7 @@ autobahn==19.5.1 asgiref>=3.2.9 # Requirements for OpenSlides in production in alphabetical order +aioredis>=1.1.0,<1.3 bleach>=1.5.0,<3.2 daphne>=2.2,<2.5 Django>=2.1,<2.3 @@ -14,9 +15,14 @@ jsonfield2>=3.0,<3.1 attrs>=19.2.0 jsonschema>=3.0,<3.1 mypy_extensions>=0.4,<0.5 +psycopg2-binary>=2.7.3.2,<2.9 PyPDF2>=1.26,<1.27 roman>=2.0,<3.2 setuptools>=29.0,<42.0 typing_extensions>=3.6.6,<3.8 websockets>=8.0,<9.0 twisted[tls]>=20.3.0 +uvicorn[standard]>=0.9,<1.0 + +# https://github.com/benoitc/gunicorn/issues/1913 +git+https://github.com/FinnStutzenstein/gunicorn.git@fix