Merge pull request #5850 from FinnStutzenstein/ownSessionRedis
Own session redis
This commit is contained in:
commit
5b91ba4597
@ -108,8 +108,7 @@ Used software
|
|||||||
|
|
||||||
OpenSlides uses the following projects or parts of them:
|
OpenSlides uses the following projects or parts of them:
|
||||||
|
|
||||||
* several Python packages (see ``server/requirements/production.txt`` and
|
* several Python packages (see ``server/requirements/production.txt``)
|
||||||
``server/requirements/big_mode.txt``)
|
|
||||||
|
|
||||||
* several JavaScript packages (see ``client/package.json``)
|
* several JavaScript packages (see ``client/package.json``)
|
||||||
|
|
||||||
|
@ -1 +1 @@
|
|||||||
Subproject commit 756043511cc00b9fd4b42cc3a7ba0d8c16897895
|
Subproject commit dbf4fef1b7ff9f2c1973ac1077d4a07cdf5aa41d
|
@ -1,8 +1,3 @@
|
|||||||
{
|
|
||||||
# Enable debug output
|
|
||||||
#debug
|
|
||||||
}
|
|
||||||
|
|
||||||
:8000
|
:8000
|
||||||
|
|
||||||
reverse_proxy /system/* autoupdate:8002 {
|
reverse_proxy /system/* autoupdate:8002 {
|
||||||
|
@ -1,5 +1,4 @@
|
|||||||
{
|
{
|
||||||
# Enable debug output
|
|
||||||
debug
|
debug
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -34,8 +34,6 @@ services:
|
|||||||
environment:
|
environment:
|
||||||
- MESSAGE_BUS_HOST=redis
|
- MESSAGE_BUS_HOST=redis
|
||||||
- MESSAGE_BUS_PORT=6379
|
- MESSAGE_BUS_PORT=6379
|
||||||
- WORKER_HOST=server
|
|
||||||
- WORKER_PORT=8000
|
|
||||||
depends_on:
|
depends_on:
|
||||||
- server
|
- server
|
||||||
- redis
|
- redis
|
||||||
|
@ -158,10 +158,11 @@ services:
|
|||||||
depends_on:
|
depends_on:
|
||||||
- redis
|
- redis
|
||||||
- server
|
- server
|
||||||
|
secrets:
|
||||||
|
- django
|
||||||
environment:
|
environment:
|
||||||
REDIS_WRITE_HOST: redis
|
REDIS_WRITE_HOST: redis
|
||||||
MESSAGE_BUS_HOST: redis-slave
|
MESSAGE_BUS_HOST: redis-slave
|
||||||
WORKER_HOST: server
|
|
||||||
networks:
|
networks:
|
||||||
- back
|
- back
|
||||||
|
|
||||||
|
@ -156,10 +156,12 @@ services:
|
|||||||
autoupdate:
|
autoupdate:
|
||||||
image: AUTOUPDATE_IMAGE
|
image: AUTOUPDATE_IMAGE
|
||||||
environment:
|
environment:
|
||||||
MESSAGE_BUS_HOST: redis
|
REDIS_WRITE_HOST: redis
|
||||||
WORKER_HOST: server
|
MESSAGE_BUS_HOST: redis-slave
|
||||||
networks:
|
networks:
|
||||||
- back
|
- back
|
||||||
|
secrets:
|
||||||
|
- django
|
||||||
deploy:
|
deploy:
|
||||||
replicas: ifenvelse(`OPENSLIDES_AUTOUPDATE_SERVICE_REPLICAS', 1)
|
replicas: ifenvelse(`OPENSLIDES_AUTOUPDATE_SERVICE_REPLICAS', 1)
|
||||||
restart_policy:
|
restart_policy:
|
||||||
|
@ -51,8 +51,7 @@ RUN apt-get install --no-install-recommends -y \
|
|||||||
RUN rm -rf /var/lib/apt/lists/*
|
RUN rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
COPY requirements /app/requirements
|
COPY requirements /app/requirements
|
||||||
RUN pip install -r requirements/production.txt -r requirements/big_mode.txt \
|
RUN pip install -r requirements/production.txt -r requirements/saml.txt && \
|
||||||
-r requirements/saml.txt && \
|
|
||||||
rm -rf /root/.cache/pip
|
rm -rf /root/.cache/pip
|
||||||
|
|
||||||
# SAML
|
# SAML
|
||||||
|
@ -2,6 +2,23 @@
|
|||||||
|
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
|
function isSettingsFileOk() {
|
||||||
|
# Forbidden keys to set
|
||||||
|
# - OPENSLIDES_USER_DATA_DIR
|
||||||
|
# - SESSION_ENGINE
|
||||||
|
if grep -q "OPENSLIDES_USER_DATA_DIR.*=" /app/personal_data/var/settings.py; then
|
||||||
|
echo "Found forbidden key OPENSLIDES_USER_DATA_DIR"
|
||||||
|
return 1;
|
||||||
|
fi
|
||||||
|
|
||||||
|
if grep -q "SESSION_ENGINE.*=" /app/personal_data/var/settings.py; then
|
||||||
|
echo "Found forbidden key SESSION_ENGINE"
|
||||||
|
return 1;
|
||||||
|
fi
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
wait-for-it -t 0 redis:6379
|
wait-for-it -t 0 redis:6379
|
||||||
|
|
||||||
until pg_isready -h postgres -p 5432 -U openslides; do
|
until pg_isready -h postgres -p 5432 -U openslides; do
|
||||||
@ -12,8 +29,20 @@ done
|
|||||||
if [[ ! -f "/app/personal_data/var/settings.py" ]]; then
|
if [[ ! -f "/app/personal_data/var/settings.py" ]]; then
|
||||||
echo "Create settings"
|
echo "Create settings"
|
||||||
python manage.py createsettings
|
python manage.py createsettings
|
||||||
|
else
|
||||||
|
echo "Settings exists - checking for invalid configurations"
|
||||||
|
if ! isSettingsFileOk; then
|
||||||
|
echo "Settings are not ok."
|
||||||
|
echo "Saving old settings in settings.py.bak"
|
||||||
|
mv /app/personal_data/var/settings.py /app/personal_data/var/settings.py.bak
|
||||||
|
python manage.py createsettings
|
||||||
|
else
|
||||||
|
echo "Settings are ok."
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
sed -i -e "s/SECRET_KEY.*$/SECRET_KEY = 'development'/" /app/personal_data/var/settings.py
|
||||||
|
|
||||||
python -u manage.py migrate
|
python -u manage.py migrate
|
||||||
|
|
||||||
exec "$@"
|
exec "$@"
|
||||||
|
@ -107,16 +107,6 @@ REDIS_ADDRESS = f"redis://{REDIS_HOST}:{REDIS_PORT}/0"
|
|||||||
REDIS_READ_ONLY_ADDRESS = f"redis://{REDIS_SLAVE_HOST}:{REDIS_SLAVE_PORT}/0"
|
REDIS_READ_ONLY_ADDRESS = f"redis://{REDIS_SLAVE_HOST}:{REDIS_SLAVE_PORT}/0"
|
||||||
CONNECTION_POOL_LIMIT = get_env("CONNECTION_POOL_LIMIT", 100, int)
|
CONNECTION_POOL_LIMIT = get_env("CONNECTION_POOL_LIMIT", 100, int)
|
||||||
|
|
||||||
# Session backend
|
|
||||||
SESSION_ENGINE = "redis_sessions.session"
|
|
||||||
SESSION_REDIS = {
|
|
||||||
"host": REDIS_HOST,
|
|
||||||
"port": REDIS_PORT,
|
|
||||||
"db": 0,
|
|
||||||
"prefix": "session",
|
|
||||||
"socket_timeout": 2,
|
|
||||||
}
|
|
||||||
|
|
||||||
# SAML integration
|
# SAML integration
|
||||||
ENABLE_SAML = get_env("ENABLE_SAML", False, bool)
|
ENABLE_SAML = get_env("ENABLE_SAML", False, bool)
|
||||||
if ENABLE_SAML:
|
if ENABLE_SAML:
|
||||||
|
@ -5,6 +5,8 @@ from openslides.utils.plugins import collect_plugins
|
|||||||
|
|
||||||
MODULE_DIR = os.path.realpath(os.path.dirname(os.path.abspath(__file__)))
|
MODULE_DIR = os.path.realpath(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
|
||||||
|
# This is not set to the docker environment
|
||||||
|
OPENSLIDES_USER_DATA_DIR = "/app/personal_data/var"
|
||||||
|
|
||||||
# Application definition
|
# Application definition
|
||||||
|
|
||||||
@ -49,6 +51,8 @@ TEMPLATES = [
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
|
SESSION_ENGINE = "openslides.utils.sessions"
|
||||||
|
|
||||||
# Email
|
# Email
|
||||||
# https://docs.djangoproject.com/en/1.10/topics/email/
|
# https://docs.djangoproject.com/en/1.10/topics/email/
|
||||||
|
|
||||||
@ -84,11 +88,19 @@ LOCALE_PATHS = [os.path.join(MODULE_DIR, "locale")]
|
|||||||
|
|
||||||
|
|
||||||
# Static files (CSS, JavaScript, Images)
|
# Static files (CSS, JavaScript, Images)
|
||||||
# https://docs.djangoproject.com/en/1.10/howto/static-files/
|
|
||||||
|
|
||||||
STATIC_URL = "/static/"
|
STATIC_URL = "/static/"
|
||||||
|
|
||||||
STATICFILES_DIRS = [os.path.join(MODULE_DIR, "static")]
|
STATICFILES_DIRS = [os.path.join(MODULE_DIR, "static")] + [
|
||||||
|
os.path.join(OPENSLIDES_USER_DATA_DIR, "static")
|
||||||
|
]
|
||||||
|
|
||||||
|
# Static files (CSS, JavaScript, Images)
|
||||||
|
STATIC_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, "collected-static")
|
||||||
|
|
||||||
|
# Files
|
||||||
|
# https://docs.djangoproject.com/en/1.10/topics/files/
|
||||||
|
MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, "media", "")
|
||||||
|
|
||||||
|
|
||||||
# Sessions and user authentication
|
# Sessions and user authentication
|
||||||
|
122
server/openslides/utils/sessions.py
Normal file
122
server/openslides/utils/sessions.py
Normal file
@ -0,0 +1,122 @@
|
|||||||
|
# type: ignore
|
||||||
|
|
||||||
|
from asgiref.sync import async_to_sync
|
||||||
|
from django.conf import settings
|
||||||
|
from django.contrib.sessions.backends.base import (
|
||||||
|
VALID_KEY_CHARS,
|
||||||
|
CreateError,
|
||||||
|
SessionBase,
|
||||||
|
)
|
||||||
|
from django.utils.crypto import get_random_string, salted_hmac
|
||||||
|
from django.utils.encoding import force_str
|
||||||
|
|
||||||
|
from .redis import get_connection
|
||||||
|
|
||||||
|
|
||||||
|
REDIS_SESSION_PREFIX = getattr(settings, "REDIS_SESSION_PREFIX", "session:")
|
||||||
|
|
||||||
|
|
||||||
|
def to_redis_key(session_key):
|
||||||
|
return f"{REDIS_SESSION_PREFIX}{session_key}"
|
||||||
|
|
||||||
|
|
||||||
|
class SessionStore(SessionBase):
|
||||||
|
"""
|
||||||
|
Implements Redis database session store.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, session_key=None):
|
||||||
|
super(SessionStore, self).__init__(session_key)
|
||||||
|
|
||||||
|
def _hash(self, value):
|
||||||
|
key_salt = "openslides.utils.sessions.SessionStore"
|
||||||
|
return salted_hmac(key_salt, value).hexdigest()
|
||||||
|
|
||||||
|
def load(self):
|
||||||
|
return async_to_sync(self._load)()
|
||||||
|
|
||||||
|
async def _load(self):
|
||||||
|
async with get_connection(read_only=True) as redis:
|
||||||
|
try:
|
||||||
|
key = to_redis_key(self._get_or_create_session_key())
|
||||||
|
print("laod:", key)
|
||||||
|
session_data = await redis.get(key)
|
||||||
|
x = self.decode(force_str(session_data))
|
||||||
|
print("load result:", x)
|
||||||
|
return x
|
||||||
|
except Exception as e:
|
||||||
|
print("load ex", e)
|
||||||
|
self._session_key = None
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def exists(self, session_key):
|
||||||
|
return async_to_sync(self._load)(session_key)
|
||||||
|
|
||||||
|
async def _exists(self, session_key):
|
||||||
|
async with get_connection(read_only=True) as redis:
|
||||||
|
key = to_redis_key(session_key)
|
||||||
|
print("exists:", key)
|
||||||
|
x = await redis.exists(key)
|
||||||
|
print("exists result:", x)
|
||||||
|
return x
|
||||||
|
|
||||||
|
def create(self):
|
||||||
|
async_to_sync(self._create)()
|
||||||
|
|
||||||
|
async def _create(self):
|
||||||
|
while True:
|
||||||
|
self._session_key = await self._async_get_new_session_key()
|
||||||
|
|
||||||
|
try:
|
||||||
|
await self._save(must_create=True)
|
||||||
|
except CreateError:
|
||||||
|
# Key wasn't unique. Try again.
|
||||||
|
continue
|
||||||
|
self.modified = True
|
||||||
|
return
|
||||||
|
|
||||||
|
def save(self, must_create=False):
|
||||||
|
async_to_sync(self._save)(must_create)
|
||||||
|
|
||||||
|
async def _save(self, must_create=False):
|
||||||
|
async with get_connection() as redis:
|
||||||
|
if self.session_key is None:
|
||||||
|
return await self._create()
|
||||||
|
if must_create and await self._exists(self._get_or_create_session_key()):
|
||||||
|
raise CreateError
|
||||||
|
data = self.encode(self._get_session(no_load=must_create))
|
||||||
|
print("Save:", self._get_or_create_session_key(), data)
|
||||||
|
await redis.setex(
|
||||||
|
to_redis_key(self._get_or_create_session_key()),
|
||||||
|
self.get_expiry_age(),
|
||||||
|
data,
|
||||||
|
)
|
||||||
|
|
||||||
|
def delete(self, session_key=None):
|
||||||
|
async_to_sync(self._delete)(session_key)
|
||||||
|
|
||||||
|
async def _delete(self, session_key=None):
|
||||||
|
if session_key is None:
|
||||||
|
if self.session_key is None:
|
||||||
|
return
|
||||||
|
session_key = self.session_key
|
||||||
|
|
||||||
|
async with get_connection() as redis:
|
||||||
|
try:
|
||||||
|
print("delete:", to_redis_key(session_key))
|
||||||
|
await redis.delete(to_redis_key(session_key))
|
||||||
|
except Exception as e:
|
||||||
|
print("delete ex:", e)
|
||||||
|
pass
|
||||||
|
|
||||||
|
# This must be overwritten to stay inside async code...
|
||||||
|
async def _async_get_new_session_key(self):
|
||||||
|
"Return session key that isn't being used."
|
||||||
|
while True:
|
||||||
|
session_key = get_random_string(32, VALID_KEY_CHARS)
|
||||||
|
if not await self._exists(session_key):
|
||||||
|
return session_key
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def clear_expired(cls):
|
||||||
|
pass
|
@ -7,29 +7,18 @@ https://github.com/OpenSlides/OpenSlides/blob/master/SETTINGS.rst
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
from openslides.global_settings import *
|
from openslides.global_settings import *
|
||||||
%(import_function)s
|
|
||||||
|
|
||||||
# The directory for user specific data files
|
|
||||||
|
|
||||||
OPENSLIDES_USER_DATA_DIR = %(openslides_user_data_dir)s
|
|
||||||
|
|
||||||
|
|
||||||
# OpenSlides plugins
|
# OpenSlides plugins
|
||||||
|
|
||||||
# Add plugins to this list (see example entry in comment).
|
# Add plugins to this list (see example entry in comment).
|
||||||
|
|
||||||
INSTALLED_PLUGINS += (
|
INSTALLED_PLUGINS += (
|
||||||
# 'plugin_module_name',
|
# 'plugin_module_name',
|
||||||
)
|
)
|
||||||
|
|
||||||
INSTALLED_APPS += INSTALLED_PLUGINS
|
INSTALLED_APPS += INSTALLED_PLUGINS
|
||||||
|
|
||||||
|
|
||||||
# Important settings for production use
|
|
||||||
# https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
|
|
||||||
|
|
||||||
# SECURITY WARNING: Keep the secret key used in production secret!
|
# SECURITY WARNING: Keep the secret key used in production secret!
|
||||||
|
|
||||||
SECRET_KEY = %(secret_key)r
|
SECRET_KEY = %(secret_key)r
|
||||||
|
|
||||||
# Use 'DEBUG = True' to get more details for server errors.
|
# Use 'DEBUG = True' to get more details for server errors.
|
||||||
@ -60,7 +49,7 @@ DEFAULT_FROM_EMAIL = 'noreply@example.com'
|
|||||||
DATA_UPLOAD_MAX_MEMORY_SIZE = 104857600
|
DATA_UPLOAD_MAX_MEMORY_SIZE = 104857600
|
||||||
|
|
||||||
# Database
|
# Database
|
||||||
# https://docs.djangoproject.com/en/1.10/ref/settings/#databases
|
# https://docs.djangoproject.com/en/2.2/ref/settings/#databases
|
||||||
|
|
||||||
# Change this setting to use e. g. PostgreSQL or MySQL.
|
# Change this setting to use e. g. PostgreSQL or MySQL.
|
||||||
|
|
||||||
@ -86,17 +75,6 @@ DATABASES = {
|
|||||||
# Collection Cache
|
# Collection Cache
|
||||||
REDIS_ADDRESS = "redis://redis:6379/0"
|
REDIS_ADDRESS = "redis://redis:6379/0"
|
||||||
|
|
||||||
# Session backend
|
|
||||||
# Redis configuration for django-redis-sessions.
|
|
||||||
# https://github.com/martinrusev/django-redis-sessions
|
|
||||||
SESSION_ENGINE = 'redis_sessions.session'
|
|
||||||
SESSION_REDIS = {
|
|
||||||
'host': 'redis',
|
|
||||||
'port': 6379,
|
|
||||||
'db': 0,
|
|
||||||
'prefix': 'session',
|
|
||||||
'socket_timeout': 2
|
|
||||||
}
|
|
||||||
|
|
||||||
# SAML integration
|
# SAML integration
|
||||||
# Please read https://github.com/OpenSlides/OpenSlides/blob/master/openslides/saml/README.md
|
# Please read https://github.com/OpenSlides/OpenSlides/blob/master/openslides/saml/README.md
|
||||||
@ -120,23 +98,12 @@ ENABLE_CHAT = False
|
|||||||
|
|
||||||
|
|
||||||
# Internationalization
|
# Internationalization
|
||||||
# https://docs.djangoproject.com/en/1.10/topics/i18n/
|
# https://docs.djangoproject.com/en/2.2/topics/i18n/
|
||||||
TIME_ZONE = 'Europe/Berlin'
|
TIME_ZONE = 'Europe/Berlin'
|
||||||
|
|
||||||
|
|
||||||
# Static files (CSS, JavaScript, Images)
|
|
||||||
# https://docs.djangoproject.com/en/1.10/howto/static-files/
|
|
||||||
STATICFILES_DIRS = [os.path.join(OPENSLIDES_USER_DATA_DIR, 'static')] + STATICFILES_DIRS
|
|
||||||
STATIC_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, 'collected-static')
|
|
||||||
|
|
||||||
|
|
||||||
# Files
|
|
||||||
# https://docs.djangoproject.com/en/1.10/topics/files/
|
|
||||||
MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, 'media', '')
|
|
||||||
|
|
||||||
|
|
||||||
# Password validation
|
# Password validation
|
||||||
# https://docs.djangoproject.com/en/1.10/topics/auth/passwords/#module-django.contrib.auth.password_validation
|
# https://docs.djangoproject.com/en/2.2/topics/auth/passwords/#module-django.contrib.auth.password_validation
|
||||||
# AUTH_PASSWORD_VALIDATORS = []
|
# AUTH_PASSWORD_VALIDATORS = []
|
||||||
|
|
||||||
|
|
||||||
|
@ -1,6 +1,5 @@
|
|||||||
# Requirements for OpenSlides in production
|
# Requirements for OpenSlides in production
|
||||||
-r requirements/production.txt
|
-r requirements/production.txt
|
||||||
-r requirements/big_mode.txt
|
|
||||||
|
|
||||||
# Requirements for development and tests in alphabetical order
|
# Requirements for development and tests in alphabetical order
|
||||||
-r requirements/development.txt
|
-r requirements/development.txt
|
||||||
|
@ -1,11 +0,0 @@
|
|||||||
# Requirements for Redis and PostgreSQL support
|
|
||||||
django-redis-sessions>=0.6.1,<0.7
|
|
||||||
psycopg2-binary>=2.7.3.2,<2.9
|
|
||||||
aioredis>=1.1.0,<1.3
|
|
||||||
|
|
||||||
# Requirements for fast asgi server
|
|
||||||
#gunicorn>=19.9.0,<20
|
|
||||||
|
|
||||||
# https://github.com/benoitc/gunicorn/issues/1913
|
|
||||||
git+https://github.com/FinnStutzenstein/gunicorn.git@fix
|
|
||||||
uvicorn[standard]>=0.9,<1.0
|
|
@ -6,6 +6,7 @@ autobahn==19.5.1
|
|||||||
asgiref>=3.2.9
|
asgiref>=3.2.9
|
||||||
|
|
||||||
# Requirements for OpenSlides in production in alphabetical order
|
# Requirements for OpenSlides in production in alphabetical order
|
||||||
|
aioredis>=1.1.0,<1.3
|
||||||
bleach>=1.5.0,<3.2
|
bleach>=1.5.0,<3.2
|
||||||
daphne>=2.2,<2.5
|
daphne>=2.2,<2.5
|
||||||
Django>=2.1,<2.3
|
Django>=2.1,<2.3
|
||||||
@ -14,9 +15,14 @@ jsonfield2>=3.0,<3.1
|
|||||||
attrs>=19.2.0
|
attrs>=19.2.0
|
||||||
jsonschema>=3.0,<3.1
|
jsonschema>=3.0,<3.1
|
||||||
mypy_extensions>=0.4,<0.5
|
mypy_extensions>=0.4,<0.5
|
||||||
|
psycopg2-binary>=2.7.3.2,<2.9
|
||||||
PyPDF2>=1.26,<1.27
|
PyPDF2>=1.26,<1.27
|
||||||
roman>=2.0,<3.2
|
roman>=2.0,<3.2
|
||||||
setuptools>=29.0,<42.0
|
setuptools>=29.0,<42.0
|
||||||
typing_extensions>=3.6.6,<3.8
|
typing_extensions>=3.6.6,<3.8
|
||||||
websockets>=8.0,<9.0
|
websockets>=8.0,<9.0
|
||||||
twisted[tls]>=20.3.0
|
twisted[tls]>=20.3.0
|
||||||
|
uvicorn[standard]>=0.9,<1.0
|
||||||
|
|
||||||
|
# https://github.com/benoitc/gunicorn/issues/1913
|
||||||
|
git+https://github.com/FinnStutzenstein/gunicorn.git@fix
|
||||||
|
Loading…
Reference in New Issue
Block a user