From 10b3bb649762b46ac6f28514ccdb059dee43e6a4 Mon Sep 17 00:00:00 2001 From: Oskar Hahn Date: Mon, 9 Jul 2018 23:22:26 +0200 Subject: [PATCH] Update to channels 2 * geis does not work with channels2 and never will be (it has to be python now) * pytest * rewrote cache system * use username instead of pk for admin user in tests --- .gitignore | 1 + .travis.yml | 10 +- DEVELOPMENT.rst | 28 +- README.rst | 3 +- make/commands.py | 73 +- make/requirements.txt | 4 + openslides/__main__.py | 71 +- openslides/agenda/apps.py | 5 +- .../migrations/0003_auto_20170818_1202.py | 5 +- .../migrations/0005_auto_20180815_1109.py | 5 +- openslides/agenda/models.py | 3 +- openslides/agenda/urls.py | 1 + openslides/asgi.py | 16 +- openslides/assignments/apps.py | 5 +- openslides/assignments/models.py | 3 +- openslides/core/apps.py | 7 +- openslides/core/config.py | 67 +- .../core/management/commands/collectstatic.py | 5 +- .../core/management/commands/getgeiss.py | 91 -- .../migrations/0005_auto_20170412_1258.py | 5 +- openslides/core/signals.py | 11 +- openslides/core/urls.py | 1 + openslides/core/views.py | 4 +- openslides/global_settings.py | 24 +- openslides/mediafiles/apps.py | 5 +- openslides/motions/apps.py | 7 +- openslides/motions/models.py | 7 +- openslides/motions/urls.py | 1 + openslides/motions/views.py | 3 +- openslides/routing.py | 41 +- openslides/topics/apps.py | 5 +- openslides/urls.py | 1 + openslides/users/access_permissions.py | 22 +- openslides/users/apps.py | 7 +- openslides/users/models.py | 13 +- openslides/users/serializers.py | 6 +- openslides/users/signals.py | 8 +- openslides/users/urls.py | 1 + openslides/users/views.py | 27 +- openslides/utils/auth.py | 21 +- openslides/utils/autoupdate.py | 490 ++--------- openslides/utils/cache.py | 815 ++++++++---------- openslides/utils/cache_providers.py | 508 +++++++++++ openslides/utils/collection.py | 104 ++- openslides/utils/consumers.py | 306 +++++++ openslides/utils/main.py | 11 +- openslides/utils/middleware.py | 63 ++ openslides/utils/migrations.py | 2 +- openslides/utils/models.py | 35 +- openslides/utils/rest_api.py | 15 +- openslides/utils/settings.py.tpl | 71 +- openslides/utils/test.py | 27 +- openslides/utils/utils.py | 46 + openslides/utils/validate.py | 1 + requirements.txt | 6 +- requirements_big_mode.txt | 7 +- requirements_production.txt | 6 +- setup.cfg | 7 +- tests/conftest.py | 40 + .../commands/create-example-data.py | 1 + tests/integration/agenda/test_viewset.py | 77 +- tests/integration/assignments/test_viewset.py | 71 +- tests/integration/core/test_views.py | 10 +- tests/integration/core/test_viewset.py | 154 +--- tests/integration/helpers.py | 73 ++ tests/integration/mediafiles/test_viewset.py | 56 +- tests/integration/motions/test_viewset.py | 199 ++--- tests/integration/topics/test_viewset.py | 52 +- tests/integration/users/test_viewset.py | 101 +-- tests/integration/utils/test_collection.py | 10 +- tests/integration/utils/test_consumers.py | 185 ++++ tests/old/config/test_config.py | 9 +- tests/old/motions/test_models.py | 6 - tests/settings.py | 25 +- tests/unit/utils/cache_provider.py | 82 ++ tests/unit/utils/test_cache.py | 483 +++++++++++ 76 files changed, 2834 insertions(+), 1943 deletions(-) create mode 100644 make/requirements.txt delete mode 100644 openslides/core/management/commands/getgeiss.py create mode 100644 openslides/utils/cache_providers.py create mode 100644 openslides/utils/consumers.py create mode 100644 openslides/utils/middleware.py create mode 100644 tests/conftest.py create mode 100644 tests/integration/helpers.py create mode 100644 tests/integration/utils/test_consumers.py create mode 100644 tests/unit/utils/cache_provider.py create mode 100644 tests/unit/utils/test_cache.py diff --git a/.gitignore b/.gitignore index 9460a3600..846299ece 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ debug/* # Unit test and coverage reports .coverage tests/file/* +.pytest_cache # Plugin development openslides_* diff --git a/.travis.yml b/.travis.yml index 29b18cb04..ed4d13386 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,15 +22,9 @@ install: - node_modules/.bin/gulp --production script: - flake8 openslides tests - - isort --check-only --recursive openslides tests + - isort --check-only --diff --recursive openslides tests - python -m mypy openslides/ - node_modules/.bin/gulp jshint - node_modules/.bin/karma start --browsers PhantomJS tests/karma/karma.conf.js - - DJANGO_SETTINGS_MODULE='tests.settings' coverage run ./manage.py test tests.unit - - coverage report --fail-under=35 - - - DJANGO_SETTINGS_MODULE='tests.settings' coverage run ./manage.py test tests.integration - - coverage report --fail-under=73 - - - DJANGO_SETTINGS_MODULE='tests.settings' ./manage.py test tests.old + - pytest --cov --cov-fail-under=70 diff --git a/DEVELOPMENT.rst b/DEVELOPMENT.rst index 80c17e036..696cc4613 100644 --- a/DEVELOPMENT.rst +++ b/DEVELOPMENT.rst @@ -78,7 +78,7 @@ To get help on the command line options run:: Later you might want to restart the server with one of the following commands. -To start OpenSlides with Daphne and one worker and to avoid opening new browser +To start OpenSlides with Daphne and to avoid opening new browser windows run:: $ python manage.py start --no-browser @@ -87,16 +87,10 @@ When debugging something email related change the email backend to console:: $ python manage.py start --debug-email -To start OpenSlides with Daphne and four workers (avoid concurrent write -requests or use PostgreSQL, see below) run:: +To start OpenSlides with Daphne run:: $ python manage.py runserver -To start OpenSlides with Geiss and one worker and to avoid opening new browser -windows (download Geiss and setup Redis before, see below) run:: - - $ python manage.py start --no-browser --use-geiss - Use gulp watch in a second command-line interface:: $ node_modules/.bin/gulp watch @@ -152,8 +146,7 @@ OpenSlides in big mode In the so called big mode you should use OpenSlides with Redis, PostgreSQL and a webserver like Apache HTTP Server or nginx as proxy server in front of your -OpenSlides interface server. Optionally you can use `Geiss -`_ as interface server instead of Daphne. +OpenSlides interface server. 1. Install and configure PostgreSQL and Redis @@ -200,23 +193,12 @@ Populate your new database:: 4. Run OpenSlides ----------------- -First start e. g. four workers (do not use the `--threads` option, because the threads will not spawn across all cores):: - - $ python manage.py runworker& - $ python manage.py runworker& - $ python manage.py runworker& - $ python manage.py runworker& - To start Daphne as protocol server run:: $ export DJANGO_SETTINGS_MODULE=settings $ export PYTHONPATH=personal_data/var/ $ daphne openslides.asgi:channel_layer -To use Geiss instead of Daphne, just download Geiss and start it:: - - $ python manage.py getgeiss - $ ./personal_data/var/geiss 5. Use Nginx (optional) @@ -224,7 +206,7 @@ When using Nginx as a proxy for delivering staticfiles the performance of the se $ python manage.py collectstatic -This is an example configuration for a single Daphne/Geiss listen on port 8000:: +This is an example configuration for a single Daphne listen on port 8000:: server { listen 80; @@ -259,7 +241,7 @@ This is an example configuration for a single Daphne/Geiss listen on port 8000:: } } -Using Nginx as a load balancer is fairly easy. Just start multiple Daphnes/Geiss on different ports, change the `proxy_pass` to `http://openslides/` and add this on top of the Nginx configuration:: +Using Nginx as a load balancer is fairly easy. Just start multiple Daphnes on different ports, change the `proxy_pass` to `http://openslides/` and add this on top of the Nginx configuration:: upstream openslides { server localhost:2001; diff --git a/README.rst b/README.rst index 7a12638d9..bbf0babd1 100644 --- a/README.rst +++ b/README.rst @@ -148,13 +148,12 @@ You should use a webserver like Apache HTTP Server or nginx to serve the static and media files as proxy server in front of your OpenSlides interface server. You also should use a database like PostgreSQL and Redis as channels backend, cache backend and session engine. Finally you should -start some WSGI workers and one or more interface servers (Daphne or Geiss). +start one or more interface servers (Daphne). Please see the respective section in the `DEVELOPMENT.rst `_ and: * https://channels.readthedocs.io/en/latest/deploying.html -* https://github.com/ostcar/geiss * https://docs.djangoproject.com/en/1.10/topics/cache/ * https://github.com/sebleier/django-redis-cache * https://docs.djangoproject.com/en/1.10/ref/settings/#databases diff --git a/make/commands.py b/make/commands.py index 1f03dd323..3e34be355 100644 --- a/make/commands.py +++ b/make/commands.py @@ -1,42 +1,10 @@ -import re - from parser import command, argument, call +import yaml +import requirements FAIL = '\033[91m' SUCCESS = '\033[92m' -RESET = '\033[0m' - - -@argument('module', nargs='?', default='') -@command('test', help='runs the tests') -def test(args=None): - """ - Runs the tests. - """ - module = getattr(args, 'module', '') - if module == '': - module = 'tests' - else: - module = 'tests.{}'.format(module) - return call("DJANGO_SETTINGS_MODULE='tests.settings' coverage run " - "./manage.py test {}".format(module)) - - -@argument('--plain', action='store_true') -@command('coverage', help='Runs all tests and builds the coverage html files') -def coverage(args=None, plain=None): - """ - Runs the tests and creates a coverage report. - - By default it creates a html report. With the argument --plain, it creates - a plain report and fails under a certain amount of untested lines. - """ - if plain is None: - plain = getattr(args, 'plain', False) - if plain: - return call('coverage report -m --fail-under=80') - else: - return call('coverage html') +RESET = '\033[0m' @command('check', help='Checks for pep8 errors in openslides and tests') @@ -54,17 +22,10 @@ def travis(args=None): """ return_codes = [] with open('.travis.yml') as f: - script_lines = False - for line in (line.strip() for line in f.readlines()): - if line == 'script:': - script_lines = True - continue - if not script_lines or not line: - continue - - match = re.search(r'"(.*)"', line) - print('Run: %s' % match.group(1)) - return_code = call(match.group(1)) + travis = yaml.load(f) + for line in travis['script']: + print('Run: {}'.format(line)) + return_code = call(line) return_codes.append(return_code) if return_code: print(FAIL + 'fail!\n' + RESET) @@ -76,7 +37,7 @@ def travis(args=None): @argument('-r', '--requirements', nargs='?', - default='requirements_production.txt') + default='requirements.txt') @command('min_requirements', help='Prints a pip line to install the minimum supported versions of ' 'the requirements.') @@ -85,23 +46,19 @@ def min_requirements(args=None): Prints a pip install command to install the minimal supported versions of a requirement file. - Uses requirements_production.txt by default. + Uses requirements.txt by default. The following line will install the version: pip install $(python make min_requirements) """ - import pip - def get_lowest_versions(requirements_file): - for line in pip.req.parse_requirements(requirements_file, session=pip.download.PipSession()): - for specifier in line.req.specifier: - if specifier.operator == '>=': - min_version = specifier.version - break - else: - raise ValueError('Not supported line {}'.format(line)) - yield '%s==%s' % (line.req.name, min_version) + with open(requirements_file) as f: + for req in requirements.parse(f): + if req.specifier: + for spec, version in req.specs: + if spec == ">=": + yield "{}=={}".format(req.name, version) print(' '.join(get_lowest_versions(args.requirements))) diff --git a/make/requirements.txt b/make/requirements.txt new file mode 100644 index 000000000..d0dd531a7 --- /dev/null +++ b/make/requirements.txt @@ -0,0 +1,4 @@ +# Requirements for the make scripts + +requirements-parser +PyYAML diff --git a/openslides/__main__.py b/openslides/__main__.py index d162cebb2..698a19cca 100644 --- a/openslides/__main__.py +++ b/openslides/__main__.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import os -import subprocess import sys from typing import Dict # noqa @@ -14,7 +13,6 @@ from openslides.utils.main import ( ExceptionArgumentParser, UnknownCommand, get_default_settings_dir, - get_geiss_path, get_local_settings_dir, is_local_installation, open_browser, @@ -145,10 +143,6 @@ def get_parser(): '--local-installation', action='store_true', help='Store settings and user files in a local directory.') - subcommand_start.add_argument( - '--use-geiss', - action='store_true', - help='Use Geiss instead of Daphne as ASGI protocol server.') # Subcommand createsettings createsettings_help = 'Creates the settings file.' @@ -220,56 +214,23 @@ def start(args): # Migrate database call_command('migrate') - if args.use_geiss: - # Make sure Redis is used. - if settings.CHANNEL_LAYERS['default']['BACKEND'] != 'asgi_redis.RedisChannelLayer': - raise RuntimeError("You have to use the ASGI Redis backend in the settings to use Geiss.") + # Open the browser + if not args.no_browser: + open_browser(args.host, args.port) - # Download Geiss and collect the static files. - call_command('getgeiss') - call_command('collectstatic', interactive=False) - - # Open the browser - if not args.no_browser: - open_browser(args.host, args.port) - - # Start Geiss in its own thread - subprocess.Popen([ - get_geiss_path(), - '--host', args.host, - '--port', args.port, - '--static', '/static/:{}'.format(settings.STATIC_ROOT), - '--static', '/media/:{}'.format(settings.MEDIA_ROOT), - ]) - - # Start one worker in this thread. There can be only one worker as - # long as SQLite3 is used. - call_command('runworker') - - else: - # Open the browser - if not args.no_browser: - open_browser(args.host, args.port) - - # Start Daphne and one worker - # - # Use flag --noreload to tell Django not to reload the server. - # Therefor we have to set the keyword noreload to False because Django - # parses this directly to the use_reloader keyword. - # - # Use flag --insecure to serve static files even if DEBUG is False. - # - # Use flag --nothreading to tell Django Channels to run in single - # thread mode with one worker only. Therefor we have to set the keyword - # nothreading to False because Django parses this directly to - # use_threading keyword. - call_command( - 'runserver', - '{}:{}'.format(args.host, args.port), - noreload=False, # Means True, see above. - insecure=True, - nothreading=False, # Means True, see above. - ) + # Start Daphne + # + # Use flag --noreload to tell Django not to reload the server. + # Therefor we have to set the keyword noreload to False because Django + # parses this directly to the use_reloader keyword. + # + # Use flag --insecure to serve static files even if DEBUG is False. + call_command( + 'runserver', + '{}:{}'.format(args.host, args.port), + noreload=False, # Means True, see above. + insecure=True, + ) def createsettings(args): diff --git a/openslides/agenda/apps.py b/openslides/agenda/apps.py index a2550cbfa..ca50d0075 100644 --- a/openslides/agenda/apps.py +++ b/openslides/agenda/apps.py @@ -1,6 +1,5 @@ from django.apps import AppConfig -from ..utils.collection import Collection from ..utils.projector import register_projector_elements @@ -48,7 +47,7 @@ class AgendaAppConfig(AppConfig): def get_startup_elements(self): """ - Yields all collections required on startup i. e. opening the websocket + Yields all Cachables required on startup i. e. opening the websocket connection. """ - yield Collection(self.get_model('Item').get_collection_string()) + yield self.get_model('Item') diff --git a/openslides/agenda/migrations/0003_auto_20170818_1202.py b/openslides/agenda/migrations/0003_auto_20170818_1202.py index 30a872214..940051b43 100644 --- a/openslides/agenda/migrations/0003_auto_20170818_1202.py +++ b/openslides/agenda/migrations/0003_auto_20170818_1202.py @@ -4,8 +4,9 @@ from __future__ import unicode_literals from django.db import migrations -from openslides.utils.migrations import \ - add_permission_to_groups_based_on_existing_permission +from openslides.utils.migrations import ( + add_permission_to_groups_based_on_existing_permission, +) class Migration(migrations.Migration): diff --git a/openslides/agenda/migrations/0005_auto_20180815_1109.py b/openslides/agenda/migrations/0005_auto_20180815_1109.py index 8438d24c4..841ee9104 100644 --- a/openslides/agenda/migrations/0005_auto_20180815_1109.py +++ b/openslides/agenda/migrations/0005_auto_20180815_1109.py @@ -5,8 +5,9 @@ from __future__ import unicode_literals from django.contrib.auth.models import Permission from django.db import migrations, models -from openslides.utils.migrations import \ - add_permission_to_groups_based_on_existing_permission +from openslides.utils.migrations import ( + add_permission_to_groups_based_on_existing_permission, +) def delete_old_can_see_hidden_permission(apps, schema_editor): diff --git a/openslides/agenda/models.py b/openslides/agenda/models.py index fc403a116..8abe6d614 100644 --- a/openslides/agenda/models.py +++ b/openslides/agenda/models.py @@ -7,8 +7,7 @@ from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.db import models, transaction from django.utils import timezone -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy +from django.utils.translation import ugettext as _, ugettext_lazy from openslides.core.config import config from openslides.core.models import Countdown, Projector diff --git a/openslides/agenda/urls.py b/openslides/agenda/urls.py index 959fbd02f..2589aa2c9 100644 --- a/openslides/agenda/urls.py +++ b/openslides/agenda/urls.py @@ -2,6 +2,7 @@ from django.conf.urls import url from . import views + urlpatterns = [ url(r'^docxtemplate/$', views.AgendaDocxTemplateView.as_view(), diff --git a/openslides/asgi.py b/openslides/asgi.py index 441a46beb..10799265a 100644 --- a/openslides/asgi.py +++ b/openslides/asgi.py @@ -1,12 +1,16 @@ -from channels.asgi import get_channel_layer +""" +ASGI entrypoint. Configures Django and then runs the application +defined in the ASGI_APPLICATION setting. +""" + +import django +from channels.routing import get_default_application from .utils.main import setup_django_settings_module + # Loads the openslides setting. You can use your own settings by setting the # environment variable DJANGO_SETTINGS_MODULE setup_django_settings_module() - -channel_layer = get_channel_layer() - -# Use native twisted mode -channel_layer.extensions.append("twisted") +django.setup() +application = get_default_application() diff --git a/openslides/assignments/apps.py b/openslides/assignments/apps.py index 41b9018d3..04e5398b6 100644 --- a/openslides/assignments/apps.py +++ b/openslides/assignments/apps.py @@ -3,7 +3,6 @@ from typing import Dict, List, Union # noqa from django.apps import AppConfig from mypy_extensions import TypedDict -from ..utils.collection import Collection from ..utils.projector import register_projector_elements @@ -41,10 +40,10 @@ class AssignmentsAppConfig(AppConfig): def get_startup_elements(self): """ - Yields all collections required on startup i. e. opening the websocket + Yields all Cachables required on startup i. e. opening the websocket connection. """ - yield Collection(self.get_model('Assignment').get_collection_string()) + yield self.get_model('Assignment') def get_angular_constants(self): assignment = self.get_model('Assignment') diff --git a/openslides/assignments/models.py b/openslides/assignments/models.py index d6bc5cf7c..f3eabc666 100644 --- a/openslides/assignments/models.py +++ b/openslides/assignments/models.py @@ -4,8 +4,7 @@ from typing import Any, Dict, List, Optional # noqa from django.conf import settings from django.contrib.contenttypes.fields import GenericRelation from django.db import models -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_noop +from django.utils.translation import ugettext as _, ugettext_noop from openslides.agenda.models import Item, Speaker from openslides.core.config import config diff --git a/openslides/core/apps.py b/openslides/core/apps.py index 2a3a885d6..da4a8990e 100644 --- a/openslides/core/apps.py +++ b/openslides/core/apps.py @@ -6,7 +6,6 @@ from django.apps import AppConfig from django.conf import settings from django.db.models.signals import post_migrate -from ..utils.collection import Collection from ..utils.projector import register_projector_elements @@ -66,11 +65,11 @@ class CoreAppConfig(AppConfig): def get_startup_elements(self): """ - Yields all collections required on startup i. e. opening the websocket + Yields all Cachables required on startup i. e. opening the websocket connection. """ - for model in ('Projector', 'ChatMessage', 'Tag', 'ProjectorMessage', 'Countdown', 'ConfigStore'): - yield Collection(self.get_model(model).get_collection_string()) + for model_name in ('Projector', 'ChatMessage', 'Tag', 'ProjectorMessage', 'Countdown', 'ConfigStore'): + yield self.get_model(model_name) def get_angular_constants(self): from .config import config diff --git a/openslides/core/config.py b/openslides/core/config.py index e3129eb55..6783ee056 100644 --- a/openslides/core/config.py +++ b/openslides/core/config.py @@ -1,13 +1,25 @@ -from typing import Any, Callable, Dict, Iterable, Optional, TypeVar, Union +from typing import ( + Any, + Callable, + Dict, + Iterable, + Optional, + TypeVar, + Union, + cast, +) +from asgiref.sync import async_to_sync from django.core.exceptions import ValidationError as DjangoValidationError from django.utils.translation import ugettext as _ from mypy_extensions import TypedDict +from ..utils.cache import element_cache from ..utils.collection import CollectionElement from .exceptions import ConfigError, ConfigNotFound from .models import ConfigStore + INPUT_TYPE_MAPPING = { 'string': str, 'text': str, @@ -37,21 +49,42 @@ class ConfigHandler: self.config_variables = {} # type: Dict[str, ConfigVariable] # Index to get the database id from a given config key - self.key_to_id = {} # type: Dict[str, int] + self.key_to_id = None # type: Optional[Dict[str, int]] def __getitem__(self, key: str) -> Any: """ Returns the value of the config variable. """ - # Build the key_to_id dict - self.save_default_values() - if not self.exists(key): raise ConfigNotFound(_('The config variable {} was not found.').format(key)) return CollectionElement.from_values( self.get_collection_string(), - self.key_to_id[key]).get_full_data()['value'] + self.get_key_to_id()[key]).get_full_data()['value'] + + def get_key_to_id(self) -> Dict[str, int]: + """ + Returns the key_to_id dict. Builds it, if it does not exist. + """ + if self.key_to_id is None: + async_to_sync(self.build_key_to_id)() + self.key_to_id = cast(Dict[str, int], self.key_to_id) + return self.key_to_id + + async def build_key_to_id(self) -> None: + """ + Build the key_to_id dict. + + Recreates it, if it does not exists. + + This uses the element_cache. It expects, that the config values are in the database + before this is called. + """ + self.key_to_id = {} + all_data = await element_cache.get_all_full_data() + elements = all_data[self.get_collection_string()] + for element in elements: + self.key_to_id[element['key']] = element['id'] def exists(self, key: str) -> bool: """ @@ -183,19 +216,17 @@ class ConfigHandler: Saves the default values to the database. Does also build the dictonary key_to_id. - - Does nothing on a second run. """ - if not self.key_to_id: - for item in self.config_variables.values(): - try: - db_value = ConfigStore.objects.get(key=item.name) - except ConfigStore.DoesNotExist: - db_value = ConfigStore() - db_value.key = item.name - db_value.value = item.default_value - db_value.save(skip_autoupdate=True) - self.key_to_id[item.name] = db_value.pk + self.key_to_id = {} + for item in self.config_variables.values(): + try: + db_value = ConfigStore.objects.get(key=item.name) + except ConfigStore.DoesNotExist: + db_value = ConfigStore() + db_value.key = item.name + db_value.value = item.default_value + db_value.save(skip_autoupdate=True) + self.key_to_id[db_value.key] = db_value.id def get_collection_string(self) -> str: """ diff --git a/openslides/core/management/commands/collectstatic.py b/openslides/core/management/commands/collectstatic.py index 314dfd204..3fb8cc588 100644 --- a/openslides/core/management/commands/collectstatic.py +++ b/openslides/core/management/commands/collectstatic.py @@ -2,8 +2,9 @@ import os from typing import Any, Dict from django.conf import settings -from django.contrib.staticfiles.management.commands.collectstatic import \ - Command as CollectStatic +from django.contrib.staticfiles.management.commands.collectstatic import ( + Command as CollectStatic, +) from django.core.management.base import CommandError from django.db.utils import OperationalError diff --git a/openslides/core/management/commands/getgeiss.py b/openslides/core/management/commands/getgeiss.py deleted file mode 100644 index cbb9510dc..000000000 --- a/openslides/core/management/commands/getgeiss.py +++ /dev/null @@ -1,91 +0,0 @@ -import distutils -import json -import os -import stat -import sys -from urllib.request import urlopen, urlretrieve - -from django.core.management.base import BaseCommand, CommandError - -from openslides.utils.main import get_geiss_path - - -class Command(BaseCommand): - """ - Command to get the latest release of Geiss from GitHub. - """ - help = 'Get the latest Geiss release from GitHub.' - - FIRST_NOT_SUPPORTED_VERSION = '1.0.0' - - def handle(self, *args, **options): - geiss_github_name = self.get_geiss_github_name() - download_file = get_geiss_path() - - if os.path.isfile(download_file): - # Geiss does probably exist. Do nothing. - # TODO: Add an update flag, that Geiss is downloaded anyway. - return - - release = self.get_release() - download_url = None - for asset in release['assets']: - if asset['name'] == geiss_github_name: - download_url = asset['browser_download_url'] - break - if download_url is None: - raise CommandError("Could not find download URL in release.") - - urlretrieve(download_url, download_file) - - # Set the executable bit on the file. This will do nothing on windows - st = os.stat(download_file) - os.chmod(download_file, st.st_mode | stat.S_IEXEC) - - self.stdout.write(self.style.SUCCESS('Geiss {} successfully downloaded.'.format(release['tag_name']))) - - def get_release(self): - """ - Returns API data for the latest supported Geiss release. - """ - response = urlopen(self.get_geiss_url()).read() - releases = json.loads(response.decode()) - for release in releases: - version = distutils.version.StrictVersion(release['tag_name']) # type: ignore - if version < self.FIRST_NOT_SUPPORTED_VERSION: - break - else: - raise CommandError('Could not find Geiss release.') - return release - - def get_geiss_url(self): - """ - Returns the URL to the API which gives the information which Geiss - binary has to be downloaded. - - Currently this is a static GitHub URL to the repository where Geiss - is hosted at the moment. - """ - # TODO: Use a settings variable or a command line flag in the future. - return 'https://api.github.com/repos/ostcar/geiss/releases' - - def get_geiss_github_name(self): - """ - Returns the name of the Geiss executable for the current operating - system. - - For example geiss_windows_64 on a windows64 platform. - """ - # This will be 32 if the current python interpreter has only - # 32 bit, even if it is run on a 64 bit operating sysem. - bits = '64' if sys.maxsize > 2**32 else '32' - - geiss_names = { - 'linux': 'geiss_linux_{bits}', - 'win32': 'geiss_windows_{bits}.exe', # Yes, it is win32, even on a win64 system! - 'darwin': 'geiss_mac_{bits}'} - - try: - return geiss_names[sys.platform].format(bits=bits) - except KeyError: - raise CommandError("Plattform {} is not supported by Geiss".format(sys.platform)) diff --git a/openslides/core/migrations/0005_auto_20170412_1258.py b/openslides/core/migrations/0005_auto_20170412_1258.py index 68871ce08..fc7e615eb 100644 --- a/openslides/core/migrations/0005_auto_20170412_1258.py +++ b/openslides/core/migrations/0005_auto_20170412_1258.py @@ -4,8 +4,9 @@ from __future__ import unicode_literals from django.db import migrations -from openslides.utils.migrations import \ - add_permission_to_groups_based_on_existing_permission +from openslides.utils.migrations import ( + add_permission_to_groups_based_on_existing_permission, +) class Migration(migrations.Migration): diff --git a/openslides/core/signals.py b/openslides/core/signals.py index ca6f0595e..663b41852 100644 --- a/openslides/core/signals.py +++ b/openslides/core/signals.py @@ -8,6 +8,7 @@ from ..utils.auth import has_perm from ..utils.collection import Collection from .models import ChatMessage + # This signal is send when the migrate command is done. That means it is sent # after post_migrate sending and creating all Permission objects. Don't use it # for other things than dealing with Permission objects. @@ -38,18 +39,18 @@ def delete_django_app_permissions(sender, **kwargs): def get_permission_change_data(sender, permissions, **kwargs): """ - Yields all necessary collections if the respective permissions change. + Yields all necessary Cachables if the respective permissions change. """ core_app = apps.get_app_config(app_label='core') for permission in permissions: if permission.content_type.app_label == core_app.label: if permission.codename == 'can_see_projector': - yield Collection(core_app.get_model('Projector').get_collection_string()) + yield core_app.get_model('Projector') elif permission.codename == 'can_manage_projector': - yield Collection(core_app.get_model('ProjectorMessage').get_collection_string()) - yield Collection(core_app.get_model('Countdown').get_collection_string()) + yield core_app.get_model('ProjectorMessage') + yield core_app.get_model('Countdown') elif permission.codename == 'can_use_chat': - yield Collection(core_app.get_model('ChatMessage').get_collection_string()) + yield core_app.get_model('ChatMessage') def required_users(sender, request_user, **kwargs): diff --git a/openslides/core/urls.py b/openslides/core/urls.py index 351c239d7..826a18964 100644 --- a/openslides/core/urls.py +++ b/openslides/core/urls.py @@ -2,6 +2,7 @@ from django.conf.urls import url from . import views + urlpatterns = [ url(r'^core/servertime/$', views.ServerTime.as_view(), diff --git a/openslides/core/views.py b/openslides/core/views.py index ba2b1f1b6..dfaa981f9 100644 --- a/openslides/core/views.py +++ b/openslides/core/views.py @@ -11,9 +11,7 @@ from django.utils.timezone import now from django.utils.translation import ugettext as _ from mypy_extensions import TypedDict -from .. import __license__ as license -from .. import __url__ as url -from .. import __version__ as version +from .. import __license__ as license, __url__ as url, __version__ as version from ..utils import views as utils_views from ..utils.auth import anonymous_is_enabled, has_perm from ..utils.autoupdate import inform_changed_data, inform_deleted_data diff --git a/openslides/global_settings.py b/openslides/global_settings.py index 1704e9c7e..c8c79a02b 100644 --- a/openslides/global_settings.py +++ b/openslides/global_settings.py @@ -2,6 +2,7 @@ import os from openslides.utils.plugins import collect_plugins + MODULE_DIR = os.path.realpath(os.path.dirname(os.path.abspath(__file__))) @@ -121,31 +122,14 @@ PASSWORD_HASHERS = [ MEDIA_URL = '/media/' -# Cache -# https://docs.djangoproject.com/en/1.10/topics/cache/ - -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'LOCATION': 'openslides-cache', - 'OPTIONS': { - 'MAX_ENTRIES': 10000 - } - } -} - - # Django Channels # http://channels.readthedocs.io/en/latest/ -# https://github.com/ostcar/geiss + +ASGI_APPLICATION = 'openslides.routing.application' CHANNEL_LAYERS = { 'default': { - 'BACKEND': 'asgiref.inmemory.ChannelLayer', - 'ROUTING': 'openslides.routing.channel_routing', - 'CONFIG': { - 'capacity': 1000, - }, + 'BACKEND': 'channels.layers.InMemoryChannelLayer', }, } diff --git a/openslides/mediafiles/apps.py b/openslides/mediafiles/apps.py index fae0cf32b..90d16b3d3 100644 --- a/openslides/mediafiles/apps.py +++ b/openslides/mediafiles/apps.py @@ -1,6 +1,5 @@ from django.apps import AppConfig -from ..utils.collection import Collection from ..utils.projector import register_projector_elements @@ -34,7 +33,7 @@ class MediafilesAppConfig(AppConfig): def get_startup_elements(self): """ - Yields all collections required on startup i. e. opening the websocket + Yields all Cachables required on startup i. e. opening the websocket connection. """ - yield Collection(self.get_model('Mediafile').get_collection_string()) + yield self.get_model('Mediafile') diff --git a/openslides/motions/apps.py b/openslides/motions/apps.py index 5969c8ff3..674c247b2 100644 --- a/openslides/motions/apps.py +++ b/openslides/motions/apps.py @@ -1,7 +1,6 @@ from django.apps import AppConfig from django.db.models.signals import post_migrate -from ..utils.collection import Collection from ..utils.projector import register_projector_elements @@ -60,8 +59,8 @@ class MotionsAppConfig(AppConfig): def get_startup_elements(self): """ - Yields all collections required on startup i. e. opening the websocket + Yields all Cachables required on startup i. e. opening the websocket connection. """ - for model in ('Category', 'Motion', 'MotionBlock', 'Workflow', 'MotionChangeRecommendation'): - yield Collection(self.get_model(model).get_collection_string()) + for model_name in ('Category', 'Motion', 'MotionBlock', 'Workflow', 'MotionChangeRecommendation'): + yield self.get_model(model_name) diff --git a/openslides/motions/models.py b/openslides/motions/models.py index ae058553d..c2b0f2e8c 100644 --- a/openslides/motions/models.py +++ b/openslides/motions/models.py @@ -7,8 +7,11 @@ from django.core.exceptions import ImproperlyConfigured, ValidationError from django.db import IntegrityError, models, transaction from django.db.models import Max from django.utils import formats, timezone -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy, ugettext_noop +from django.utils.translation import ( + ugettext as _, + ugettext_lazy, + ugettext_noop, +) from jsonfield import JSONField from openslides.agenda.models import Item diff --git a/openslides/motions/urls.py b/openslides/motions/urls.py index 6e7c41473..277e22dce 100644 --- a/openslides/motions/urls.py +++ b/openslides/motions/urls.py @@ -2,6 +2,7 @@ from django.conf.urls import url from . import views + urlpatterns = [ url(r'^docxtemplate/$', views.MotionDocxTemplateView.as_view(), diff --git a/openslides/motions/views.py b/openslides/motions/views.py index 29b151acf..13b9d6ba6 100644 --- a/openslides/motions/views.py +++ b/openslides/motions/views.py @@ -8,8 +8,7 @@ from django.db import IntegrityError, transaction from django.db.models.deletion import ProtectedError from django.http import Http404 from django.http.request import QueryDict -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_noop +from django.utils.translation import ugettext as _, ugettext_noop from rest_framework import status from ..core.config import config diff --git a/openslides/routing.py b/openslides/routing.py index a40ffd311..29457c7c3 100644 --- a/openslides/routing.py +++ b/openslides/routing.py @@ -1,31 +1,16 @@ -from channels.routing import include, route +from channels.routing import ProtocolTypeRouter, URLRouter +from django.conf.urls import url -from openslides.utils.autoupdate import ( - send_data_projector, - send_data_site, - ws_add_projector, - ws_add_site, - ws_disconnect_projector, - ws_disconnect_site, - ws_receive_projector, - ws_receive_site, -) +from openslides.utils.consumers import ProjectorConsumer, SiteConsumer +from openslides.utils.middleware import AuthMiddlewareStack -projector_routing = [ - route("websocket.connect", ws_add_projector), - route("websocket.disconnect", ws_disconnect_projector), - route("websocket.receive", ws_receive_projector), -] -site_routing = [ - route("websocket.connect", ws_add_site), - route("websocket.disconnect", ws_disconnect_site), - route("websocket.receive", ws_receive_site), -] - -channel_routing = [ - include(projector_routing, path=r'^/ws/projector/(?P\d+)/$'), - include(site_routing, path=r'^/ws/site/$'), - route("autoupdate.send_data_projector", send_data_projector), - route("autoupdate.send_data_site", send_data_site), -] +application = ProtocolTypeRouter({ + # WebSocket chat handler + "websocket": AuthMiddlewareStack( + URLRouter([ + url(r"^ws/site/$", SiteConsumer), + url(r"^ws/projector/(?P\d+)/$", ProjectorConsumer), + ]) + ) +}) diff --git a/openslides/topics/apps.py b/openslides/topics/apps.py index 6f4f0b075..edc9f572e 100644 --- a/openslides/topics/apps.py +++ b/openslides/topics/apps.py @@ -1,6 +1,5 @@ from django.apps import AppConfig -from ..utils.collection import Collection from ..utils.projector import register_projector_elements @@ -31,7 +30,7 @@ class TopicsAppConfig(AppConfig): def get_startup_elements(self): """ - Yields all collections required on startup i. e. opening the websocket + Yields all Cachables required on startup i. e. opening the websocket connection. """ - yield Collection(self.get_model('Topic').get_collection_string()) + yield self.get_model('Topic') diff --git a/openslides/urls.py b/openslides/urls.py index ad0f03792..0dd87defd 100644 --- a/openslides/urls.py +++ b/openslides/urls.py @@ -6,6 +6,7 @@ from openslides.mediafiles.views import protected_serve from openslides.utils.plugins import get_all_plugin_urlpatterns from openslides.utils.rest_api import router + urlpatterns = get_all_plugin_urlpatterns() urlpatterns += [ diff --git a/openslides/users/access_permissions.py b/openslides/users/access_permissions.py index c1a5ec904..545133b5a 100644 --- a/openslides/users/access_permissions.py +++ b/openslides/users/access_permissions.py @@ -43,16 +43,20 @@ class UserAccessPermissions(BaseAccessPermissions): """ return {key: full_data[key] for key in whitelist} - # We have four sets of data to be sent: - # * full data i. e. all fields, - # * many data i. e. all fields but not the default password, - # * little data i. e. all fields but not the default password, comments and active status, + # We have five sets of data to be sent: + # * full data i. e. all fields (including session_auth_hash), + # * all data i. e. all fields but not session_auth_hash, + # * many data i. e. all fields but not the default password and session_auth_hash, + # * little data i. e. all fields but not the default password, session_auth_hash, comments and active status, # * no data. - # Prepare field set for users with "many" data and with "little" data. - many_data_fields = set(USERCANSEEEXTRASERIALIZER_FIELDS) - many_data_fields.add('groups_id') - many_data_fields.discard('groups') + # Prepare field set for users with "all" data, "many" data and with "little" data. + all_data_fields = set(USERCANSEEEXTRASERIALIZER_FIELDS) + all_data_fields.add('groups_id') + all_data_fields.discard('groups') + all_data_fields.add('default_password') + many_data_fields = all_data_fields.copy() + many_data_fields.discard('default_password') litte_data_fields = set(USERCANSEESERIALIZER_FIELDS) litte_data_fields.add('groups_id') litte_data_fields.discard('groups') @@ -61,7 +65,7 @@ class UserAccessPermissions(BaseAccessPermissions): if has_perm(user, 'users.can_see_name'): if has_perm(user, 'users.can_see_extra_data'): if has_perm(user, 'users.can_manage'): - data = full_data + data = [filtered_data(full, all_data_fields) for full in full_data] else: data = [filtered_data(full, many_data_fields) for full in full_data] else: diff --git a/openslides/users/apps.py b/openslides/users/apps.py index 6c1a5adcf..5dfb93305 100644 --- a/openslides/users/apps.py +++ b/openslides/users/apps.py @@ -2,7 +2,6 @@ from django.apps import AppConfig from django.conf import settings from django.contrib.auth.signals import user_logged_in -from ..utils.collection import Collection from ..utils.projector import register_projector_elements @@ -45,11 +44,11 @@ class UsersAppConfig(AppConfig): def get_startup_elements(self): """ - Yields all collections required on startup i. e. opening the websocket + Yields all Cachables required on startup i. e. opening the websocket connection. """ - for model in ('User', 'Group', 'PersonalNote'): - yield Collection(self.get_model(model).get_collection_string()) + for model_name in ('User', 'Group', 'PersonalNote'): + yield self.get_model(model_name) def get_angular_constants(self): from django.contrib.auth.models import Permission diff --git a/openslides/users/models.py b/openslides/users/models.py index 1e9592a8f..07ba04d60 100644 --- a/openslides/users/models.py +++ b/openslides/users/models.py @@ -2,11 +2,11 @@ import smtplib from random import choice from django.contrib.auth.hashers import make_password -from django.contrib.auth.models import Group as DjangoGroup -from django.contrib.auth.models import GroupManager as _GroupManager from django.contrib.auth.models import ( AbstractBaseUser, BaseUserManager, + Group as DjangoGroup, + GroupManager as _GroupManager, Permission, PermissionsMixin, ) @@ -286,6 +286,15 @@ class User(RESTModelMixin, PermissionsMixin, AbstractBaseUser): return False + @property + def session_auth_hash(self): + """ + Returns the session auth hash of a user as attribute. + + Needed for the django rest framework. + """ + return self.get_session_auth_hash() + class GroupManager(_GroupManager): """ diff --git a/openslides/users/serializers.py b/openslides/users/serializers.py index 65fea361b..d35fb5674 100644 --- a/openslides/users/serializers.py +++ b/openslides/users/serializers.py @@ -1,7 +1,6 @@ from django.contrib.auth.hashers import make_password from django.contrib.auth.models import Permission -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy +from django.utils.translation import ugettext as _, ugettext_lazy from ..utils.autoupdate import inform_changed_data from ..utils.rest_api import ( @@ -13,6 +12,7 @@ from ..utils.rest_api import ( ) from .models import Group, PersonalNote, User + USERCANSEESERIALIZER_FIELDS = ( 'id', 'username', @@ -52,7 +52,7 @@ class UserFullSerializer(ModelSerializer): class Meta: model = User - fields = USERCANSEEEXTRASERIALIZER_FIELDS + ('default_password',) + fields = USERCANSEEEXTRASERIALIZER_FIELDS + ('default_password', 'session_auth_hash') read_only_fields = ('last_email_send',) def validate(self, data): diff --git a/openslides/users/signals.py b/openslides/users/signals.py index a5f72d561..fdbf84d1e 100644 --- a/openslides/users/signals.py +++ b/openslides/users/signals.py @@ -81,7 +81,7 @@ def create_builtin_groups_and_admin(**kwargs): permission_dict['mediafiles.can_see'], permission_dict['motions.can_see'], permission_dict['users.can_see_name'], ) - group_default = Group.objects.create(name='Default') + group_default = Group.objects.create(pk=1, name='Default') group_default.permissions.add(*base_permissions) # Delegates (pk 2) @@ -99,7 +99,7 @@ def create_builtin_groups_and_admin(**kwargs): permission_dict['motions.can_create'], permission_dict['motions.can_support'], permission_dict['users.can_see_name'], ) - group_delegates = Group.objects.create(name='Delegates') + group_delegates = Group.objects.create(pk=2, name='Delegates') group_delegates.permissions.add(*delegates_permissions) # Staff (pk 3) @@ -130,7 +130,7 @@ def create_builtin_groups_and_admin(**kwargs): permission_dict['users.can_manage'], permission_dict['users.can_see_extra_data'], permission_dict['mediafiles.can_see_hidden'],) - group_staff = Group.objects.create(name='Staff') + group_staff = Group.objects.create(pk=3, name='Staff') group_staff.permissions.add(*staff_permissions) # Admin (pk 4) @@ -164,7 +164,7 @@ def create_builtin_groups_and_admin(**kwargs): permission_dict['users.can_manage'], permission_dict['users.can_see_extra_data'], permission_dict['mediafiles.can_see_hidden'],) - group_admin = Group.objects.create(name='Admin') + group_admin = Group.objects.create(pk=4, name='Admin') group_admin.permissions.add(*admin_permissions) # Add users.can_see_name permission to staff/admin diff --git a/openslides/users/urls.py b/openslides/users/urls.py index 5b1bbeddb..872b9245d 100644 --- a/openslides/users/urls.py +++ b/openslides/users/urls.py @@ -2,6 +2,7 @@ from django.conf.urls import url from . import views + urlpatterns = [ # Auth url(r'^login/$', diff --git a/openslides/users/views.py b/openslides/users/views.py index 4077d3c4a..d6993d1e4 100644 --- a/openslides/users/views.py +++ b/openslides/users/views.py @@ -1,10 +1,13 @@ import smtplib from typing import List # noqa +from asgiref.sync import async_to_sync from django.conf import settings -from django.contrib.auth import login as auth_login -from django.contrib.auth import logout as auth_logout -from django.contrib.auth import update_session_auth_hash +from django.contrib.auth import ( + login as auth_login, + logout as auth_logout, + update_session_auth_hash, +) from django.contrib.auth.forms import AuthenticationForm from django.contrib.auth.password_validation import validate_password from django.core import mail @@ -15,13 +18,17 @@ from django.utils.translation import ugettext as _ from ..core.config import config from ..core.signals import permission_change -from ..utils.auth import anonymous_is_enabled, has_perm +from ..utils.auth import ( + anonymous_is_enabled, + has_perm, + user_to_collection_user, +) from ..utils.autoupdate import ( inform_changed_data, inform_data_collection_element_list, ) -from ..utils.cache import restricted_data_cache -from ..utils.collection import CollectionElement +from ..utils.cache import element_cache +from ..utils.collection import Collection, CollectionElement from ..utils.rest_api import ( ModelViewSet, Response, @@ -103,7 +110,7 @@ class UserViewSet(ModelViewSet): del request.data[key] response = super().update(request, *args, **kwargs) # Maybe some group assignments have changed. Better delete the restricted user cache - restricted_data_cache.del_user(user.id) + async_to_sync(element_cache.del_user)(user_to_collection_user(user)) return response def destroy(self, request, *args, **kwargs): @@ -303,7 +310,7 @@ class GroupViewSet(ModelViewSet): # Delete the user chaches of all affected users for user in group.user_set.all(): - restricted_data_cache.del_user(user.id) + async_to_sync(element_cache.del_user)(user_to_collection_user(user)) def diff(full, part): """ @@ -321,8 +328,8 @@ class GroupViewSet(ModelViewSet): collection_elements = [] # type: List[CollectionElement] signal_results = permission_change.send(None, permissions=new_permissions, action='added') for receiver, signal_collections in signal_results: - for collection in signal_collections: - collection_elements.extend(collection.element_generator()) + for cachable in signal_collections: + collection_elements.extend(Collection(cachable.get_collection_string()).element_generator()) inform_data_collection_element_list(collection_elements) # TODO: Some permissions are deleted. diff --git a/openslides/utils/auth.py b/openslides/utils/auth.py index 0fe46c71e..4d818db3b 100644 --- a/openslides/utils/auth.py +++ b/openslides/utils/auth.py @@ -1,9 +1,10 @@ -from typing import Optional, Union +from typing import Dict, Optional, Union, cast from django.contrib.auth import get_user_model from django.contrib.auth.models import AnonymousUser from django.db.models import Model +from .cache import element_cache from .collection import CollectionElement @@ -46,6 +47,18 @@ def anonymous_is_enabled() -> bool: return config['general_system_enable_anonymous'] +async def async_anonymous_is_enabled() -> bool: + """ + Like anonymous_is_enabled but async. + """ + from ..core.config import config + if config.key_to_id is None: + await config.build_key_to_id() + config.key_to_id = cast(Dict[str, int], config.key_to_id) + element = await element_cache.get_element_full_data(config.get_collection_string(), config.key_to_id['general_system_enable_anonymous']) + return False if element is None else element['value'] + + AnyUser = Union[Model, CollectionElement, int, AnonymousUser, None] @@ -75,7 +88,11 @@ def user_to_collection_user(user: AnyUser) -> Optional[CollectionElement]: "Unsupported type for user. Only CollectionElements for users can be" "used. Not {}".format(user.collection_string)) elif isinstance(user, int): - user = CollectionElement.from_values(User.get_collection_string(), user) + # user 0 means anonymous + if user == 0: + user = None + else: + user = CollectionElement.from_values(User.get_collection_string(), user) elif isinstance(user, AnonymousUser): user = None elif isinstance(user, User): diff --git a/openslides/utils/autoupdate.py b/openslides/utils/autoupdate.py index 5f926c954..f68770c13 100644 --- a/openslides/utils/autoupdate.py +++ b/openslides/utils/autoupdate.py @@ -1,366 +1,14 @@ -import json import threading -import time -import warnings -from collections import OrderedDict, defaultdict -from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, Union +from collections import OrderedDict +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union -from channels import Channel, Group -from channels.asgi import get_channel_layer -from channels.auth import channel_session_user, channel_session_user_from_http -from django.apps import apps -from django.core.exceptions import ObjectDoesNotExist -from django.db import transaction +from asgiref.sync import async_to_sync +from channels.layers import get_channel_layer +from django.conf import settings from django.db.models import Model -from ..core.config import config -from ..core.models import Projector -from .auth import anonymous_is_enabled, has_perm, user_to_collection_user -from .cache import restricted_data_cache, websocket_user_cache -from .collection import AutoupdateFormat # noqa -from .collection import ( - ChannelMessageFormat, - Collection, - CollectionElement, - format_for_autoupdate, - from_channel_message, - to_channel_message, -) - - -def send_or_wait(send_func: Any, *args: Any, **kwargs: Any) -> None: - """ - Wrapper for channels' send() method. - - If the method send() raises ChannelFull exception the worker waits for 20 - milliseconds and tries again. After 5 secondes it gives up, drops the - channel message and writes a warning to stderr. - - Django channels' consumer atomicity feature is disabled. - """ - kwargs['immediately'] = True - for i in range(250): - try: - send_func(*args, **kwargs) - except get_channel_layer().ChannelFull: - time.sleep(0.02) - else: - break - else: - warnings.warn( - 'Channel layer is full. Channel message dropped.', - RuntimeWarning - ) - - -@channel_session_user_from_http -def ws_add_site(message: Any) -> None: - """ - Adds the websocket connection to a group specific to the connecting user. - - The group with the name 'user-None' stands for all anonymous users. - - Send all "startup-data" through the connection. - """ - if not anonymous_is_enabled() and not message.user.id: - send_or_wait(message.reply_channel.send, {'accept': False}) - return - - Group('site').add(message.reply_channel) - message.channel_session['user_id'] = message.user.id - # Saves the reply channel to the user. Uses 0 for anonymous users. - websocket_user_cache.add(message.user.id or 0, message.reply_channel.name) - - # Open the websocket connection. - send_or_wait(message.reply_channel.send, {'accept': True}) - - # Collect all elements that shoud be send to the client when the websocket - # connection is established. - user = user_to_collection_user(message.user.id) - user_id = user.id if user is not None else 0 - if restricted_data_cache.exists_for_user(user_id): - output = restricted_data_cache.get_data(user_id) - else: - output = [] - for collection in get_startup_collections(): - access_permissions = collection.get_access_permissions() - restricted_data = access_permissions.get_restricted_data(collection.get_full_data(), user) - - for data in restricted_data: - if data is None: - # We do not want to send 'deleted' objects on startup. - # That's why we skip such data. - continue - - formatted_data = format_for_autoupdate( - collection_string=collection.collection_string, - id=data['id'], - action='changed', - data=data) - - output.append(formatted_data) - # Cache restricted data for user - restricted_data_cache.add_element( - user_id, - collection.collection_string, - data['id'], - formatted_data) - - # Send all data. - if output: - send_or_wait(message.reply_channel.send, {'text': json.dumps(output)}) - - -@channel_session_user -def ws_disconnect_site(message: Any) -> None: - """ - This function is called, when a client on the site disconnects. - """ - Group('site').discard(message.reply_channel) - websocket_user_cache.remove(message.user.id or 0, message.reply_channel.name) - - -@channel_session_user -def ws_receive_site(message: Any) -> None: - """ - If we recieve something from the client we currently just interpret this - as a notify message. - - The server adds the sender's user id (0 for anonymous) and reply - channel name so that a receiver client may reply to the sender or to all - sender's instances. - """ - try: - incomming = json.loads(message.content['text']) - except ValueError: - # Message content is invalid. Just do nothing. - pass - else: - if isinstance(incomming, list): - notify( - incomming, - senderReplyChannelName=message.reply_channel.name, - senderUserId=message.user.id or 0) - - -def notify(incomming: List[Dict[str, Any]], **attributes: Any) -> None: - """ - The incomming should be a list of notify elements. Every item is broadcasted - to the given users, channels or projectors. If none is given, the message is - send to each site client. - """ - # Parse all items - receivers_users = defaultdict(list) # type: Dict[int, List[Any]] - receivers_projectors = defaultdict(list) # type: Dict[int, List[Any]] - receivers_reply_channels = defaultdict(list) # type: Dict[str, List[Any]] - items_for_all = [] - for item in incomming: - if item.get('collection') == 'notify': - use_receivers_dict = False - - for key, value in attributes.items(): - item[key] = value - - # Force the params to be a dict - if not isinstance(item.get('params'), dict): - item['params'] = {} - - users = item.get('users') - if isinstance(users, list): - # Send this item only to all reply channels of some site users. - for user_id in users: - receivers_users[user_id].append(item) - use_receivers_dict = True - - projectors = item.get('projectors') - if isinstance(projectors, list): - # Send this item only to all reply channels of some site users. - for projector_id in projectors: - receivers_projectors[projector_id].append(item) - use_receivers_dict = True - - reply_channels = item.get('replyChannels') - if isinstance(reply_channels, list): - # Send this item only to some reply channels. - for reply_channel_name in reply_channels: - receivers_reply_channels[reply_channel_name].append(item) - use_receivers_dict = True - - if not use_receivers_dict: - # Send this item to all reply channels. - items_for_all.append(item) - - # Send all items - for user_id, channel_names in websocket_user_cache.get_all().items(): - output = receivers_users[user_id] - if len(output) > 0: - for channel_name in channel_names: - send_or_wait(Channel(channel_name).send, {'text': json.dumps(output)}) - - for channel_name, output in receivers_reply_channels.items(): - if len(output) > 0: - send_or_wait(Channel(channel_name).send, {'text': json.dumps(output)}) - - for projector_id, output in receivers_projectors.items(): - if len(output) > 0: - send_or_wait(Group('projector-{}'.format(projector_id)).send, {'text': json.dumps(output)}) - - if len(items_for_all) > 0: - send_or_wait(Group('site').send, {'text': json.dumps(items_for_all)}) - - -@channel_session_user_from_http -def ws_add_projector(message: Any, projector_id: int) -> None: - """ - Adds the websocket connection to a group specific to the projector with the given id. - Also sends all data that are shown on the projector. - """ - user = user_to_collection_user(message.user.id) - - if not has_perm(user, 'core.can_see_projector'): - send_or_wait(message.reply_channel.send, {'text': 'No permissions to see this projector.'}) - else: - try: - projector = Projector.objects.get(pk=projector_id) - except Projector.DoesNotExist: - send_or_wait(message.reply_channel.send, {'text': 'The projector {} does not exist.'.format(projector_id)}) - else: - # At first, the client is added to the projector group, so it is - # informed if the data change. - Group('projector-{}'.format(projector_id)).add(message.reply_channel) - - # Then it is also added to the global projector group which is - # used for broadcasting data. - Group('projector-all').add(message.reply_channel) - - # Now check whether broadcast is active at the moment. If yes, - # change the local projector variable. - if config['projector_broadcast'] > 0: - projector = Projector.objects.get(pk=config['projector_broadcast']) - - # Collect all elements that are on the projector. - output = [] # type: List[AutoupdateFormat] - for requirement in projector.get_all_requirements(): - required_collection_element = CollectionElement.from_instance(requirement) - output.append(required_collection_element.as_autoupdate_for_projector()) - - # Collect all config elements. - config_collection = Collection(config.get_collection_string()) - projector_data = (config_collection.get_access_permissions() - .get_projector_data(config_collection.get_full_data())) - for data in projector_data: - output.append(format_for_autoupdate( - config_collection.collection_string, - data['id'], - 'changed', - data)) - - # Collect the projector instance. - collection_element = CollectionElement.from_instance(projector) - output.append(collection_element.as_autoupdate_for_projector()) - - # Send all the data that were only collected before. - send_or_wait(message.reply_channel.send, {'text': json.dumps(output)}) - - -def ws_disconnect_projector(message: Any, projector_id: int) -> None: - """ - This function is called, when a client on the projector disconnects. - """ - Group('projector-{}'.format(projector_id)).discard(message.reply_channel) - Group('projector-all').discard(message.reply_channel) - - -def ws_receive_projector(message: Any, projector_id: int) -> None: - """ - If we recieve something from the client we currently just interpret this - as a notify message. - - The server adds the sender's projector id and reply channel name so that - a receiver client may reply to the sender or to all sender's instances. - """ - try: - incomming = json.loads(message.content['text']) - except ValueError: - # Message content is invalid. Just do nothing. - pass - else: - if isinstance(incomming, list): - notify( - incomming, - senderReplyChannelName=message.reply_channel.name, - senderProjectorId=projector_id) - - -def send_data_projector(message: ChannelMessageFormat) -> None: - """ - Informs all projector clients about changed data. - """ - collection_elements = from_channel_message(message) - - # Check whether broadcast is active at the moment and set the local - # projector queryset. - if config['projector_broadcast'] > 0: - queryset = Projector.objects.filter(pk=config['projector_broadcast']) - else: - queryset = Projector.objects.all() - - # Loop over all projectors and send data that they need. - for projector in queryset: - output = [] - for collection_element in collection_elements: - if collection_element.is_deleted(): - output.append(collection_element.as_autoupdate_for_projector()) - else: - for element in projector.get_collection_elements_required_for_this(collection_element): - output.append(element.as_autoupdate_for_projector()) - if output: - if config['projector_broadcast'] > 0: - send_or_wait( - Group('projector-all').send, - {'text': json.dumps(output)}) - else: - send_or_wait( - Group('projector-{}'.format(projector.pk)).send, - {'text': json.dumps(output)}) - - -def send_data_site(message: ChannelMessageFormat) -> None: - """ - Informs all site users about changed data. - """ - collection_elements = from_channel_message(message) - - # Send data to site users. - for user_id, channel_names in websocket_user_cache.get_all().items(): - if not user_id: - # Anonymous user - user = None - else: - try: - user = user_to_collection_user(user_id) - except ObjectDoesNotExist: - # The user does not exist. Skip him/her. - continue - - output = [] - for collection_element in collection_elements: - formatted_data = collection_element.as_autoupdate_for_user(user) - if formatted_data['action'] == 'changed': - restricted_data_cache.update_element( - user_id or 0, - collection_element.collection_string, - collection_element.id, - formatted_data) - else: - restricted_data_cache.del_element( - user_id or 0, - collection_element.collection_string, - collection_element.id) - output.append(formatted_data) - - for channel_name in channel_names: - send_or_wait(Channel(channel_name).send, {'text': json.dumps(output)}) +from .cache import element_cache, get_element_id +from .collection import CollectionElement, to_channel_message def to_ordered_dict(d: Optional[Dict]) -> Optional[OrderedDict]: @@ -377,7 +25,7 @@ def to_ordered_dict(d: Optional[Dict]) -> Optional[OrderedDict]: def inform_changed_data(instances: Union[Iterable[Model], Model], information: Dict[str, Any] = None) -> None: """ Informs the autoupdate system and the caching system about the creation or - update of an element. This is done via the AutoupdateBundleMiddleware. + update of an element. The argument instances can be one instance or an iterable over instances. """ @@ -392,37 +40,47 @@ def inform_changed_data(instances: Union[Iterable[Model], Model], information: D # Instance has no method get_root_rest_element. Just ignore it. pass - # Put all collection elements into the autoupdate_bundle. + collection_elements = {} + for root_instance in root_instances: + collection_element = CollectionElement.from_instance( + root_instance, + information=information) + key = root_instance.get_collection_string() + str(root_instance.get_rest_pk()) + str(to_ordered_dict(information)) + collection_elements[key] = collection_element + bundle = autoupdate_bundle.get(threading.get_ident()) if bundle is not None: - # Run autoupdate only if the bundle exists because we are in a request-response-cycle. - for root_instance in root_instances: - collection_element = CollectionElement.from_instance( - root_instance, - information=information) - key = root_instance.get_collection_string() + str(root_instance.get_rest_pk()) + str(to_ordered_dict(information)) - bundle[key] = collection_element + # Put all collection elements into the autoupdate_bundle. + bundle.update(collection_elements) + else: + # Send autoupdate directly + async_to_sync(send_autoupdate)(collection_elements.values()) def inform_deleted_data(elements: Iterable[Tuple[str, int]], information: Dict[str, Any] = None) -> None: """ Informs the autoupdate system and the caching system about the deletion of - elements. This is done via the AutoupdateBundleMiddleware. + elements. The argument information is added to each collection element. """ - # Put all stuff to be deleted into the autoupdate_bundle. + collection_elements = {} # type: Dict[str, Any] + for element in elements: + collection_element = CollectionElement.from_values( + collection_string=element[0], + id=element[1], + deleted=True, + information=information) + key = element[0] + str(element[1]) + str(to_ordered_dict(information)) + collection_elements[key] = collection_element + bundle = autoupdate_bundle.get(threading.get_ident()) if bundle is not None: - # Run autoupdate only if the bundle exists because we are in a request-response-cycle. - for element in elements: - collection_element = CollectionElement.from_values( - collection_string=element[0], - id=element[1], - deleted=True, - information=information) - key = element[0] + str(element[1]) + str(to_ordered_dict(information)) - bundle[key] = collection_element + # Put all collection elements into the autoupdate_bundle. + bundle.update(collection_elements) + else: + # Send autoupdate directly + async_to_sync(send_autoupdate)(collection_elements.values()) def inform_data_collection_element_list(collection_elements: List[CollectionElement], @@ -431,13 +89,18 @@ def inform_data_collection_element_list(collection_elements: List[CollectionElem Informs the autoupdate system about some collection elements. This is used just to send some data to all users. """ - # Put all stuff into the autoupdate_bundle. + elements = {} + for collection_element in collection_elements: + key = collection_element.collection_string + str(collection_element.id) + str(to_ordered_dict(information)) + elements[key] = collection_element + bundle = autoupdate_bundle.get(threading.get_ident()) if bundle is not None: - # Run autoupdate only if the bundle exists because we are in a request-response-cycle. - for collection_element in collection_elements: - key = collection_element.collection_string + str(collection_element.id) + str(to_ordered_dict(information)) - bundle[key] = collection_element + # Put all collection elements into the autoupdate_bundle. + bundle.update(elements) + else: + # Send autoupdate directly + async_to_sync(send_autoupdate)(elements.values()) """ @@ -461,41 +124,48 @@ class AutoupdateBundleMiddleware: response = self.get_response(request) bundle = autoupdate_bundle.pop(thread_id) # type: Dict[str, CollectionElement] - # If currently there is an open database transaction, then the - # send_autoupdate function is only called, when the transaction is - # commited. If there is currently no transaction, then the function - # is called immediately. - transaction.on_commit(lambda: send_autoupdate(bundle.values())) + async_to_sync(send_autoupdate)(bundle.values()) return response -def send_autoupdate(collection_elements: Iterable[CollectionElement]) -> None: +async def send_autoupdate(collection_elements: Iterable[CollectionElement]) -> None: """ Helper function, that sends collection_elements through a channel to the autoupdate system. + Also updates the redis cache. + Does nothing if collection_elements is empty. """ if collection_elements: - send_or_wait( - Channel('autoupdate.send_data_projector').send, - to_channel_message(collection_elements)) - send_or_wait( - Channel('autoupdate.send_data_site').send, - to_channel_message(collection_elements)) + cache_elements = {} # type: Dict[str, Optional[Dict[str, Any]]] + for element in collection_elements: + element_id = get_element_id(element.collection_string, element.id) + if element.is_deleted(): + cache_elements[element_id] = None + else: + cache_elements[element_id] = element.get_full_data() + if not getattr(settings, 'SKIP_CACHE', False): + # Hack for django 2.0 and channels 2.1 to stay in the same thread. + # This is needed for the tests. + change_id = await element_cache.change_elements(cache_elements) + else: + change_id = 1 -def get_startup_collections() -> Generator[Collection, None, None]: - """ - Returns all Collections that should be send to the user at startup - """ - for app in apps.get_app_configs(): - try: - # Get the method get_startup_elements() from an app. - # This method has to return an iterable of Collection objects. - get_startup_elements = app.get_startup_elements - except AttributeError: - # Skip apps that do not implement get_startup_elements. - continue - - yield from get_startup_elements() + channel_layer = get_channel_layer() + # TODO: don't await. They can be send in parallel + await channel_layer.group_send( + "projector", + { + "type": "send_data", + "message": to_channel_message(collection_elements), + }, + ) + await channel_layer.group_send( + "site", + { + "type": "send_data", + "change_id": change_id, + }, + ) diff --git a/openslides/utils/cache.py b/openslides/utils/cache.py index a69447caa..ac2a162fd 100644 --- a/openslides/utils/cache.py +++ b/openslides/utils/cache.py @@ -1,506 +1,417 @@ +import asyncio import json from collections import defaultdict -from typing import ( # noqa +from datetime import datetime +from typing import ( TYPE_CHECKING, Any, Callable, Dict, - Generator, - Iterable, List, Optional, - Set, + Tuple, Type, - Union, ) -from channels import Group -from channels.sessions import session_for_reply_channel +from asgiref.sync import sync_to_async +from channels.db import database_sync_to_async from django.conf import settings -from django.core.cache import cache, caches + +from .cache_providers import ( + BaseCacheProvider, + Cachable, + MemmoryCacheProvider, + RedisCacheProvider, + get_all_cachables, + no_redis_dependency, +) +from .utils import get_element_id, get_user_id, split_element_id + if TYPE_CHECKING: - # Dummy import Collection for mypy - from .collection import Collection # noqa - -UserCacheDataType = Dict[int, Set[str]] + # Dummy import Collection for mypy, can be fixed with python 3.7 + from .collection import CollectionElement # noqa -class BaseWebsocketUserCache: +class ElementCache: """ - Caches the reply channel names of all open websocket connections. The id of - the user that that opened the connection is used as reference. + Cache for the CollectionElements. - This is the Base cache that has to be overriden. - """ - cache_key = 'current_websocket_users' + Saves the full_data and if enabled the restricted data. - def add(self, user_id: int, channel_name: str) -> None: - """ - Adds a channel name to an user id. - """ - raise NotImplementedError() + There is one redis Hash (simular to python dict) for the full_data and one + Hash for every user. - def remove(self, user_id: int, channel_name: str) -> None: - """ - Removes one channel name from the cache. - """ - raise NotImplementedError() + The key of the Hashes is COLLECTIONSTRING:ID where COLLECTIONSTRING is the + collection_string of a collection and id the id of an element. - def get_all(self) -> UserCacheDataType: - """ - Returns all data using a dict where the key is a user id and the value - is a set of channel_names. - """ - raise NotImplementedError() + All elements have to be in the cache. If one element is missing, the cache + is invalid, but this can not be detected. When a plugin with a new + collection is added to OpenSlides, then the cache has to be rebuild manualy. - def save_data(self, data: UserCacheDataType) -> None: - """ - Saves the full data set (like created with build_data) to the cache. - """ - raise NotImplementedError() + There is an sorted set in redis with the change id as score. The values are + COLLETIONSTRING:ID for the elements that have been changed with that change + id. With this key it is possible, to get all elements as full_data or as + restricted_data that are newer then a specific change id. - def build_data(self) -> UserCacheDataType: - """ - Creates all the data, saves it to the cache and returns it. - """ - websocket_user_ids = defaultdict(set) # type: UserCacheDataType - for channel_name in Group('site').channel_layer.group_channels('site'): - session = session_for_reply_channel(channel_name) - user_id = session.get('user_id', None) - websocket_user_ids[user_id or 0].add(channel_name) - self.save_data(websocket_user_ids) - return websocket_user_ids - - def get_cache_key(self) -> str: - """ - Returns the cache key. - """ - return self.cache_key - - -class RedisWebsocketUserCache(BaseWebsocketUserCache): - """ - Implementation of the WebsocketUserCache that uses redis. - - This uses one cache key to store all connected user ids in a set and - for each user another set to save the channel names. + All method of this class are async. You either have to call them with + await in an async environment or use asgiref.sync.async_to_sync(). """ - def add(self, user_id: int, channel_name: str) -> None: + def __init__( + self, + redis: str, + use_restricted_data_cache: bool = False, + cache_provider_class: Type[BaseCacheProvider] = RedisCacheProvider, + cachable_provider: Callable[[], List[Cachable]] = get_all_cachables, + start_time: int = None) -> None: """ - Adds a channel name to an user id. - """ - redis = get_redis_connection() - pipe = redis.pipeline() - pipe.sadd(self.get_cache_key(), user_id) - pipe.sadd(self.get_user_cache_key(user_id), channel_name) - pipe.execute() + Initializes the cache. - def remove(self, user_id: int, channel_name: str) -> None: + When restricted_data_cache is false, no restricted data is saved. """ - Removes one channel name from the cache. - """ - redis = get_redis_connection() - redis.srem(self.get_user_cache_key(user_id), channel_name) + self.use_restricted_data_cache = use_restricted_data_cache + self.cache_provider = cache_provider_class(redis) + self.cachable_provider = cachable_provider + self._cachables = None # type: Optional[Dict[str, Cachable]] - def get_all(self) -> UserCacheDataType: + # Start time is used as first change_id if there is non in redis + if start_time is None: + start_time = int((datetime.utcnow() - datetime(1970, 1, 1)).total_seconds()) + self.start_time = start_time + + # Contains Futures to controll, that only one client updates the restricted_data. + self.restricted_data_cache_updater = {} # type: Dict[int, asyncio.Future] + + @property + def cachables(self) -> Dict[str, Cachable]: """ - Returns all data using a dict where the key is a user id and the value - is a set of channel_names. + Returns all Cachables as a dict where the key is the collection_string of the cachable. """ - redis = get_redis_connection() - user_ids = redis.smembers(self.get_cache_key()) # type: Optional[List[str]] - if user_ids is None: - websocket_user_ids = self.build_data() + # This method is neccessary to lazy load the cachables + if self._cachables is None: + self._cachables = {cachable.get_collection_string(): cachable for cachable in self.cachable_provider()} + return self._cachables + + async def save_full_data(self, db_data: Dict[str, List[Dict[str, Any]]]) -> None: + """ + Saves the full data. + """ + mapping = {} + for collection_string, elements in db_data.items(): + for element in elements: + mapping.update( + {get_element_id(collection_string, element['id']): + json.dumps(element)}) + await self.cache_provider.reset_full_cache(mapping) + + async def build_full_data(self) -> Dict[str, List[Dict[str, Any]]]: + """ + Build or rebuild the full_data cache. + """ + db_data = {} # type: Dict[str, List[Dict[str, Any]]] + for collection_string, cachable in self.cachables.items(): + db_data[collection_string] = await database_sync_to_async(cachable.get_elements)() + await self.save_full_data(db_data) + return db_data + + async def exists_full_data(self) -> bool: + """ + Returns True, if the full_data_cache exists. + """ + return await self.cache_provider.data_exists() + + async def change_elements( + self, elements: Dict[str, Optional[Dict[str, Any]]]) -> int: + """ + Changes elements in the cache. + + elements is a list of the changed elements as dict. When the value is None, + it is interpreded as deleted. The key has to be an element_id. + + Returns the new generated change_id. + """ + if not await self.exists_full_data(): + await self.build_full_data() + + deleted_elements = [] + changed_elements = [] + for element_id, data in elements.items(): + if data: + # The arguments for redis.hset is pairs of key value + changed_elements.append(element_id) + changed_elements.append(json.dumps(data)) + else: + deleted_elements.append(element_id) + + if changed_elements: + await self.cache_provider.add_elements(changed_elements) + if deleted_elements: + await self.cache_provider.del_elements(deleted_elements) + + # TODO: The provider has to define the new change_id with lua. In other + # case it is possible, that two changes get the same id (which + # would not be a big problem). + change_id = await self.get_next_change_id() + + await self.cache_provider.add_changed_elements(change_id, elements.keys()) + return change_id + + async def get_all_full_data(self) -> Dict[str, List[Dict[str, Any]]]: + """ + Returns all full_data. If it does not exist, it is created. + + The returned value is a dict where the key is the collection_string and + the value is a list of data. + """ + if not await self.exists_full_data(): + out = await self.build_full_data() else: - websocket_user_ids = dict() - for redis_user_id in user_ids: - # Redis returns the id as string. So we have to convert it - user_id = int(redis_user_id) - channel_names = redis.smembers(self.get_user_cache_key(user_id)) # type: Optional[List[str]] - if channel_names is not None: - # If channel name is empty, then we can assume, that the user - # has no active connection. - websocket_user_ids[user_id] = set(channel_names) - return websocket_user_ids + out = defaultdict(list) + full_data = await self.cache_provider.get_all_data() + for element_id, data in full_data.items(): + collection_string, __ = split_element_id(element_id) + out[collection_string].append(json.loads(data.decode())) + return dict(out) - def save_data(self, data: UserCacheDataType) -> None: + async def get_full_data( + self, change_id: int = 0) -> Tuple[Dict[str, List[Dict[str, Any]]], List[str]]: """ - Saves the full data set (like created with the method build_data()) to - the cache. + Returns all full_data since change_id. If it does not exist, it is created. + + Returns two values inside a tuple. The first value is a dict where the + key is the collection_string and the value is a list of data. The second + is a list of element_ids with deleted elements. + + Only returns elements with the change_id or newer. When change_id is 0, + all elements are returned. + + Raises a RuntimeError when the lowest change_id in redis is higher then + the requested change_id. In this case the method has to be rerun with + change_id=0. This is importend because there could be deleted elements + that the cache does not know about. """ - redis = get_redis_connection() - pipe = redis.pipeline() + if change_id == 0: + return (await self.get_all_full_data(), []) - # Save all user ids - pipe.delete(self.get_cache_key()) - pipe.sadd(self.get_cache_key(), *data.keys()) + lowest_change_id = await self.get_lowest_change_id() + if change_id < lowest_change_id: + # When change_id is lower then the lowest change_id in redis, we can + # not inform the user about deleted elements. + raise RuntimeError( + "change_id {} is lower then the lowest change_id in redis {}. " + "Catch this exception and rerun the method with change_id=0." + .format(change_id, lowest_change_id)) - for user_id, channel_names in data.items(): - pipe.delete(self.get_user_cache_key(user_id)) - pipe.sadd(self.get_user_cache_key(user_id), *channel_names) - pipe.execute() + if not await self.exists_full_data(): + # If the cache does not exist, create it. + await self.build_full_data() - def get_cache_key(self) -> str: + raw_changed_elements, deleted_elements = await self.cache_provider.get_data_since(change_id) + return ( + {collection_string: [json.loads(value.decode()) for value in value_list] + for collection_string, value_list in raw_changed_elements.items()}, + deleted_elements) + + async def get_element_full_data(self, collection_string: str, id: int) -> Optional[Dict[str, Any]]: """ - Returns the cache key. + Returns one element as full data. + + If the cache is empty, it is created. + + Returns None if the element does not exist. """ - return cache.make_key(self.cache_key) + if not await self.exists_full_data(): + await self.build_full_data() - def get_user_cache_key(self, user_id: int) -> str: - """ - Returns a cache key to save the channel names for a specific user. - """ - return cache.make_key('{}:{}'.format(self.cache_key, user_id)) + element = await self.cache_provider.get_element(get_element_id(collection_string, id)) - -class DjangoCacheWebsocketUserCache(BaseWebsocketUserCache): - """ - Implementation of the WebsocketUserCache that uses the django cache. - - If you use this with the inmemory cache, then you should only use one - worker. - - This uses only one cache key to save a dict where the key is the user id and - the value is a set of channel names. - """ - - def add(self, user_id: int, channel_name: str) -> None: - """ - Adds a channel name for a user using the django cache. - """ - websocket_user_ids = cache.get(self.get_cache_key()) - if websocket_user_ids is None: - websocket_user_ids = dict() - - if user_id in websocket_user_ids: - websocket_user_ids[user_id].add(channel_name) - else: - websocket_user_ids[user_id] = set([channel_name]) - cache.set(self.get_cache_key(), websocket_user_ids) - - def remove(self, user_id: int, channel_name: str) -> None: - """ - Removes one channel name from the django cache. - """ - websocket_user_ids = cache.get(self.get_cache_key()) - if websocket_user_ids is not None and user_id in websocket_user_ids: - websocket_user_ids[user_id].discard(channel_name) - cache.set(self.get_cache_key(), websocket_user_ids) - - def get_all(self) -> UserCacheDataType: - """ - Returns the data using the django cache. - """ - websocket_user_ids = cache.get(self.get_cache_key()) - if websocket_user_ids is None: - return self.build_data() - return websocket_user_ids - - def save_data(self, data: UserCacheDataType) -> None: - """ - Saves the data using the django cache. - """ - cache.set(self.get_cache_key(), data) - - -class FullDataCache: - """ - Caches all data as full data. - - Helps to get all data from one collection. - """ - - base_cache_key = 'full_data_cache' - - def build_for_collection(self, collection_string: str) -> None: - """ - Build the cache for collection from a django model. - - Rebuilds the cache for that collection, if it already exists. - """ - redis = get_redis_connection() - pipe = redis.pipeline() - - # Clear the cache for collection - pipe.delete(self.get_cache_key(collection_string)) - - # Save all elements - from .collection import get_model_from_collection_string - model = get_model_from_collection_string(collection_string) - try: - query = model.objects.get_full_queryset() - except AttributeError: - # If the model des not have to method get_full_queryset(), then use - # the default queryset from django. - query = model.objects - - # Build a dict from the instance id to the full_data - mapping = {instance.pk: json.dumps(model.get_access_permissions().get_full_data(instance)) - for instance in query.all()} - - if mapping: - # Save the dict into a redis map, if there is at least one value - pipe.hmset( - self.get_cache_key(collection_string), - mapping) - - pipe.execute() - - def add_element(self, collection_string: str, id: int, data: Dict[str, Any]) -> None: - """ - Adds one element to the cache. If the cache does not exists for the collection, - it is created. - """ - redis = get_redis_connection() - - # If the cache does not exist for the collection, then create it first. - if not self.exists_for_collection(collection_string): - self.build_for_collection(collection_string) - - redis.hset( - self.get_cache_key(collection_string), - id, - json.dumps(data)) - - def del_element(self, collection_string: str, id: int) -> None: - """ - Removes one element from the cache. - - Does nothing if the cache does not exist. - """ - redis = get_redis_connection() - redis.hdel( - self.get_cache_key(collection_string), - id) - - def exists_for_collection(self, collection_string: str) -> bool: - """ - Returns True if the cache for the collection exists, else False. - """ - redis = get_redis_connection() - return redis.exists(self.get_cache_key(collection_string)) - - def get_data(self, collection_string: str) -> List[Dict[str, Any]]: - """ - Returns all data for the collection. - """ - redis = get_redis_connection() - return [json.loads(element.decode()) for element in redis.hvals(self.get_cache_key(collection_string))] - - def get_element(self, collection_string: str, id: int) -> Dict[str, Any]: - """ - Returns one element from the collection. - - Raises model.DoesNotExist if the element is not in the cache. - """ - redis = get_redis_connection() - element = redis.hget(self.get_cache_key(collection_string), id) if element is None: - from .collection import get_model_from_collection_string - model = get_model_from_collection_string(collection_string) - raise model.DoesNotExist(collection_string, id) + return None return json.loads(element.decode()) - def get_cache_key(self, collection_string: str) -> str: + async def exists_restricted_data(self, user: Optional['CollectionElement']) -> bool: """ - Returns the cache key for a collection. + Returns True, if the restricted_data exists for the user. """ - return cache.make_key('{}:{}'.format(self.base_cache_key, collection_string)) + if not self.use_restricted_data_cache: + return False + + return await self.cache_provider.data_exists(get_user_id(user)) + + async def del_user(self, user: Optional['CollectionElement']) -> None: + """ + Removes one user from the resticted_data_cache. + """ + await self.cache_provider.del_restricted_data(get_user_id(user)) + + async def update_restricted_data( + self, user: Optional['CollectionElement']) -> None: + """ + Updates the restricted data for an user from the full_data_cache. + """ + # TODO: When elements are changed at the same time then this method run + # this could make the cache invalid. + # This could be fixed when get_full_data would be used with a + # max change_id. + if not self.use_restricted_data_cache: + # If the restricted_data_cache is not used, there is nothing to do + return + + # Try to write a special key. + # If this succeeds, there is noone else currently updating the cache. + # TODO: Make a timeout. Else this could block forever + if await self.cache_provider.set_lock_restricted_data(get_user_id(user)): + future = asyncio.Future() # type: asyncio.Future + self.restricted_data_cache_updater[get_user_id(user)] = future + # Get change_id for this user + value = await self.cache_provider.get_change_id_user(get_user_id(user)) + # If the change id is not in the cache yet, use -1 to get all data since 0 + user_change_id = int(value) if value else -1 + change_id = await self.get_current_change_id() + if change_id > user_change_id: + try: + full_data_elements, deleted_elements = await self.get_full_data(user_change_id + 1) + except RuntimeError: + # The user_change_id is lower then the lowest change_id in the cache. + # The whole restricted_data for that user has to be recreated. + full_data_elements = await self.get_all_full_data() + await self.cache_provider.del_restricted_data(get_user_id(user)) + else: + # Remove deleted elements + if deleted_elements: + await self.cache_provider.del_elements(deleted_elements, get_user_id(user)) + + mapping = {} + for collection_string, full_data in full_data_elements.items(): + restricter = self.cachables[collection_string].restrict_elements + elements = await sync_to_async(restricter)(user, full_data) + for element in elements: + mapping.update( + {get_element_id(collection_string, element['id']): + json.dumps(element)}) + mapping['_config:change_id'] = str(change_id) + await self.cache_provider.update_restricted_data(get_user_id(user), mapping) + # Unset the lock + await self.cache_provider.del_lock_restricted_data(get_user_id(user)) + future.set_result(1) + else: + # Wait until the update if finshed + if get_user_id(user) in self.restricted_data_cache_updater: + # The active worker is on the same asgi server, we can use the future + await self.restricted_data_cache_updater[get_user_id(user)] + else: + while await self.cache_provider.get_lock_restricted_data(get_user_id(user)): + await asyncio.sleep(0.01) + + async def get_all_restricted_data(self, user: Optional['CollectionElement']) -> Dict[str, List[Dict[str, Any]]]: + """ + Like get_all_full_data but with restricted_data for an user. + """ + if not self.use_restricted_data_cache: + all_restricted_data = {} + for collection_string, full_data in (await self.get_all_full_data()).items(): + restricter = self.cachables[collection_string].restrict_elements + elements = await sync_to_async(restricter)(user, full_data) + all_restricted_data[collection_string] = elements + return all_restricted_data + + await self.update_restricted_data(user) + + out = defaultdict(list) # type: Dict[str, List[Dict[str, Any]]] + restricted_data = await self.cache_provider.get_all_data(get_user_id(user)) + for element_id, data in restricted_data.items(): + if element_id.decode().startswith('_config'): + continue + collection_string, __ = split_element_id(element_id) + out[collection_string].append(json.loads(data.decode())) + return dict(out) + + async def get_restricted_data( + self, + user: Optional['CollectionElement'], + change_id: int = 0) -> Tuple[Dict[str, List[Dict[str, Any]]], List[str]]: + """ + Like get_full_data but with restricted_data for an user. + """ + if change_id == 0: + # Return all data + return (await self.get_all_restricted_data(user), []) + + if not self.use_restricted_data_cache: + changed_elements, deleted_elements = await self.get_full_data(change_id) + restricted_data = {} + for collection_string, full_data in changed_elements.items(): + restricter = self.cachables[collection_string].restrict_elements + elements = await sync_to_async(restricter)(user, full_data) + restricted_data[collection_string] = elements + return restricted_data, deleted_elements + + lowest_change_id = await self.get_lowest_change_id() + if change_id < lowest_change_id: + # When change_id is lower then the lowest change_id in redis, we can + # not inform the user about deleted elements. + raise RuntimeError( + "change_id {} is lower then the lowest change_id in redis {}. " + "Catch this exception and rerun the method with change_id=0." + .format(change_id, lowest_change_id)) + + # If another coroutine or another daphne server also updates the restricted + # data, this waits until it is done. + await self.update_restricted_data(user) + + raw_changed_elements, deleted_elements = await self.cache_provider.get_data_since(change_id, get_user_id(user)) + return ( + {collection_string: [json.loads(value.decode()) for value in value_list] + for collection_string, value_list in raw_changed_elements.items()}, + deleted_elements) + + async def get_current_change_id(self) -> int: + """ + Returns the current change id. + + Returns start_time if there is no change id yet. + """ + value = await self.cache_provider.get_current_change_id() + if not value: + return self.start_time + # Return the score (second element) of the first (and only) element + return value[0][1] + + async def get_next_change_id(self) -> int: + """ + Returns the next change_id. + + Returns the start time in seconds + 1, if there is no change_id in yet. + """ + current_id = await self.get_current_change_id() + return current_id + 1 + + async def get_lowest_change_id(self) -> int: + """ + Returns the lowest change id. + + Raises a RuntimeError if there is no change_id. + """ + value = await self.cache_provider.get_lowest_change_id() + if not value: + raise RuntimeError('There is no known change_id.') + # Return the score (second element) of the first (and only) element + return value -class DummyFullDataCache: +def load_element_cache(redis_addr: str = '', restricted_data: bool = True) -> ElementCache: """ - Dummy FullDataCache that does nothing. + Generates an element cache instance. """ - def build_for_collection(self, collection_string: str) -> None: - pass + if not redis_addr: + return ElementCache(redis='', cache_provider_class=MemmoryCacheProvider) - def add_element(self, collection_string: str, id: int, data: Dict[str, Any]) -> None: - pass - - def del_element(self, collection_string: str, id: int) -> None: - pass - - def exists_for_collection(self, collection_string: str) -> bool: - return False - - def get_data(self, collection_string: str) -> List[Dict[str, Any]]: - from .collection import get_model_from_collection_string - model = get_model_from_collection_string(collection_string) - try: - query = model.objects.get_full_queryset() - except AttributeError: - # If the model des not have to method get_full_queryset(), then use - # the default queryset from django. - query = model.objects - - return [model.get_access_permissions().get_full_data(instance) - for instance in query.all()] - - def get_element(self, collection_string: str, id: int) -> Dict[str, Any]: - from .collection import get_model_from_collection_string - model = get_model_from_collection_string(collection_string) - try: - query = model.objects.get_full_queryset() - except AttributeError: - # If the model des not have to method get_full_queryset(), then use - # the default queryset from django. - query = model.objects - - return model.get_access_permissions().get_full_data(query.get(pk=id)) + if no_redis_dependency: + raise ImportError("OpenSlides is configured to use redis as cache backend, but aioredis is not installed.") + return ElementCache(redis=redis_addr, use_restricted_data_cache=restricted_data) -class RestrictedDataCache: - """ - Caches all data for a specific users. - - Helps to get all data from all collections for a specific user. - - The cached values are expected to be formatted for outout via websocket. - """ - - base_cache_key = 'restricted_user_cache' - - def update_element(self, user_id: int, collection_string: str, id: int, data: object) -> None: - """ - Adds on element to the cache only if the cache exists for the user. - - Note: This method is not atomic. So in very rare cases it is possible - that the restricted date cache can become corrupt. The best solution would be to - use a lua script instead. See also #3427. - """ - if self.exists_for_user(user_id): - self.add_element(user_id, collection_string, id, data) - - def add_element(self, user_id: int, collection_string: str, id: int, data: object) -> None: - """ - Adds one element to the cache. If the cache does not exists for the user, - it is created. - """ - redis = get_redis_connection() - redis.hset( - self.get_cache_key(user_id), - "{}/{}".format(collection_string, id), - json.dumps(data)) - - def del_element(self, user_id: int, collection_string: str, id: int) -> None: - """ - Removes one element from the cache. - - Does nothing if the cache does not exist. - """ - redis = get_redis_connection() - redis.hdel( - self.get_cache_key(user_id), - "{}/{}".format(collection_string, id)) - - def del_user(self, user_id: int) -> None: - """ - Removes all elements for one user from the cache. - """ - redis = get_redis_connection() - redis.delete(self.get_cache_key(user_id)) - - def del_all(self) -> None: - """ - Deletes all elements from the cache. - - This method uses the redis command SCAN. See - https://redis.io/commands/scan#scan-guarantees for its limitations. If - an element is added to the cache while del_all() is in process, it is - possible, that it is not deleted. - """ - redis = get_redis_connection() - - # Get all keys that start with self.base_cache_key and delete them - match = cache.make_key('{}:*'.format(self.base_cache_key)) - cursor = 0 - while True: - cursor, keys = redis.scan(cursor, match) - for key in keys: - redis.delete(key) - if cursor == 0: - return - - def exists_for_user(self, user_id: int) -> bool: - """ - Returns True if the cache for the user exists, else False. - """ - redis = get_redis_connection() - return redis.exists(self.get_cache_key(user_id)) - - def get_data(self, user_id: int) -> List[object]: - """ - Returns all data for the user. - - The returned value is a list of the elements. - """ - redis = get_redis_connection() - return [json.loads(element.decode()) for element in redis.hvals(self.get_cache_key(user_id))] - - def get_cache_key(self, user_id: int) -> str: - """ - Returns the cache key for a user. - """ - return cache.make_key('{}:{}'.format(self.base_cache_key, user_id)) - - -class DummyRestrictedDataCache: - """ - Dummy RestrictedDataCache that does nothing. - """ - - def update_element(self, user_id: int, collection_string: str, id: int, data: object) -> None: - pass - - def add_element(self, user_id: int, collection_string: str, id: int, data: object) -> None: - pass - - def del_element(self, user_id: int, collection_string: str, id: int) -> None: - pass - - def del_user(self, user_id: int) -> None: - pass - - def del_all(self) -> None: - pass - - def exists_for_user(self, user_id: int) -> bool: - return False - - def get_data(self, user_id: int) -> List[object]: - pass - - -def use_redis_cache() -> bool: - """ - Returns True if Redis is used als caching backend. - """ - try: - from django_redis.cache import RedisCache - except ImportError: - return False - return isinstance(caches['default'], RedisCache) - - -def get_redis_connection() -> Any: - """ - Returns an object that can be used to talk directly to redis. - """ - from django_redis import get_redis_connection - return get_redis_connection("default") - - -if use_redis_cache(): - websocket_user_cache = RedisWebsocketUserCache() # type: BaseWebsocketUserCache - if settings.DISABLE_USER_CACHE: - restricted_data_cache = DummyRestrictedDataCache() # type: Union[RestrictedDataCache, DummyRestrictedDataCache] - else: - restricted_data_cache = RestrictedDataCache() - full_data_cache = FullDataCache() # type: Union[FullDataCache, DummyFullDataCache] -else: - websocket_user_cache = DjangoCacheWebsocketUserCache() - restricted_data_cache = DummyRestrictedDataCache() - full_data_cache = DummyFullDataCache() +redis_address = getattr(settings, 'REDIS_ADDRESS', '') +use_restricted_data = getattr(settings, 'RESTRICTED_DATA_CACHE', True) +element_cache = load_element_cache(redis_addr=redis_address, restricted_data=use_restricted_data) diff --git a/openslides/utils/cache_providers.py b/openslides/utils/cache_providers.py new file mode 100644 index 000000000..1be8d867d --- /dev/null +++ b/openslides/utils/cache_providers.py @@ -0,0 +1,508 @@ +from collections import defaultdict +from typing import Set # noqa +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Generator, + Iterable, + List, + Optional, + Tuple, + Union, +) + +from django.apps import apps + +from .utils import split_element_id, str_dict_to_bytes + + +if TYPE_CHECKING: + # Dummy import Collection for mypy, can be fixed with python 3.7 + from .collection import CollectionElement # noqa + +try: + import aioredis +except ImportError: + no_redis_dependency = True +else: + no_redis_dependency = False + + +class BaseCacheProvider: + """ + Base class for cache provider. + + See RedisCacheProvider as reverence implementation. + """ + full_data_cache_key = 'full_data_cache' + restricted_user_cache_key = 'restricted_data_cache:{user_id}' + change_id_cache_key = 'change_id_cache' + lock_key = '_config:updating' + + def __init__(self, *args: Any) -> None: + pass + + def get_full_data_cache_key(self) -> str: + return self.full_data_cache_key + + def get_restricted_data_cache_key(self, user_id: int) -> str: + return self.restricted_user_cache_key.format(user_id=user_id) + + def get_change_id_cache_key(self) -> str: + return self.change_id_cache_key + + def clear_cache(self) -> None: + raise NotImplementedError("CacheProvider has to implement the method clear_cache().") + + async def reset_full_cache(self, data: Dict[str, str]) -> None: + raise NotImplementedError("CacheProvider has to implement the method reset_full_cache().") + + async def data_exists(self, user_id: Optional[int] = None) -> bool: + raise NotImplementedError("CacheProvider has to implement the method exists_full_data().") + + async def add_elements(self, elements: List[str]) -> None: + raise NotImplementedError("CacheProvider has to implement the method add_elements().") + + async def del_elements(self, elements: List[str], user_id: Optional[int] = None) -> None: + raise NotImplementedError("CacheProvider has to implement the method del_elements().") + + async def add_changed_elements(self, change_id: int, element_ids: Iterable[str]) -> None: + raise NotImplementedError("CacheProvider has to implement the method add_changed_elements().") + + async def get_all_data(self, user_id: Optional[int] = None) -> Dict[bytes, bytes]: + raise NotImplementedError("CacheProvider has to implement the method get_all_data().") + + async def get_data_since(self, change_id: int, user_id: Optional[int] = None) -> Tuple[Dict[str, List[bytes]], List[str]]: + raise NotImplementedError("CacheProvider has to implement the method get_data_since().") + + async def get_element(self, element_id: str) -> Optional[bytes]: + raise NotImplementedError("CacheProvider has to implement the method get_element().") + + async def del_restricted_data(self, user_id: int) -> None: + raise NotImplementedError("CacheProvider has to implement the method del_restricted_data().") + + async def set_lock_restricted_data(self, user_id: int) -> bool: + raise NotImplementedError("CacheProvider has to implement the method set_lock_restricted_data().") + + async def get_lock_restricted_data(self, user_id: int) -> bool: + raise NotImplementedError("CacheProvider has to implement the method get_lock_restricted_data().") + + async def del_lock_restricted_data(self, user_id: int) -> None: + raise NotImplementedError("CacheProvider has to implement the method del_lock_restricted_data().") + + async def get_change_id_user(self, user_id: int) -> Optional[int]: + raise NotImplementedError("CacheProvider has to implement the method get_change_id_user().") + + async def update_restricted_data(self, user_id: int, data: Dict[str, str]) -> None: + raise NotImplementedError("CacheProvider has to implement the method update_restricted_data().") + + async def get_current_change_id(self) -> List[Tuple[str, int]]: + raise NotImplementedError("CacheProvider has to implement the method get_current_change_id().") + + async def get_lowest_change_id(self) -> Optional[int]: + raise NotImplementedError("CacheProvider has to implement the method get_lowest_change_id().") + + +class RedisCacheProvider(BaseCacheProvider): + """ + Cache provider that loads and saves the data to redis. + """ + redis_pool = None # type: Optional[aioredis.RedisConnection] + + def __init__(self, redis: str) -> None: + self.redis_address = redis + + async def get_connection(self) -> 'aioredis.RedisConnection': + """ + Returns a redis connection. + """ + if self.redis_pool is None: + self.redis_pool = await aioredis.create_redis_pool(self.redis_address) + return self.redis_pool + + async def reset_full_cache(self, data: Dict[str, str]) -> None: + """ + Deletes the cache and write new data in it. + """ + # TODO: lua or transaction + redis = await self.get_connection() + await redis.delete(self.get_full_data_cache_key()) + await redis.hmset_dict(self.get_full_data_cache_key(), data) + + async def data_exists(self, user_id: Optional[int] = None) -> bool: + """ + Returns True, when there is data in the cache. + + If user_id is None, the method tests for full_data. If user_id is an int, it tests + for the restricted_data_cache for the user with the user_id. 0 is for anonymous. + """ + redis = await self.get_connection() + if user_id is None: + cache_key = self.get_full_data_cache_key() + else: + cache_key = self.get_restricted_data_cache_key(user_id) + return await redis.exists(cache_key) + + async def add_elements(self, elements: List[str]) -> None: + """ + Add or change elements to the cache. + + elements is a list with an even len. the odd values are the element_ids and the even + values are the elements. The elements have to be encoded, for example with json. + """ + redis = await self.get_connection() + await redis.hmset( + self.get_full_data_cache_key(), + *elements) + + async def del_elements(self, elements: List[str], user_id: Optional[int] = None) -> None: + """ + Deletes elements from the cache. + + elements has to be a list of element_ids. + + If user_id is None, the elements are deleted from the full_data cache. If user_id is an + int, the elements are deleted one restricted_data_cache. 0 is for anonymous. + """ + redis = await self.get_connection() + if user_id is None: + cache_key = self.get_full_data_cache_key() + else: + cache_key = self.get_restricted_data_cache_key(user_id) + await redis.hdel( + cache_key, + *elements) + + async def add_changed_elements(self, change_id: int, element_ids: Iterable[str]) -> None: + """ + Saves which elements are change with a change_id. + + args has to be an even iterable. The odd values have to be a change id (int) and the + even values have to be element_ids. + """ + def zadd_args(change_id: int) -> Generator[Union[int, str], None, None]: + """ + Small helper to generates the arguments for the redis command zadd. + """ + for element_id in element_ids: + yield change_id + yield element_id + + redis = await self.get_connection() + await redis.zadd(self.get_change_id_cache_key(), *zadd_args(change_id)) + # Saves the lowest_change_id if it does not exist + await redis.zadd(self.get_change_id_cache_key(), change_id, '_config:lowest_change_id', exist='ZSET_IF_NOT_EXIST') + + async def get_all_data(self, user_id: Optional[int] = None) -> Dict[bytes, bytes]: + """ + Returns all data from a cache. + + if user_id is None, then the data is returned from the full_data_cache. If it is and + int, it is returned from a restricted_data_cache. 0 is for anonymous. + """ + if user_id is None: + cache_key = self.get_full_data_cache_key() + else: + cache_key = self.get_restricted_data_cache_key(user_id) + redis = await self.get_connection() + return await redis.hgetall(cache_key) + + async def get_element(self, element_id: str) -> Optional[bytes]: + """ + Returns one element from the full_data_cache. + + Returns None, when the element does not exist. + """ + redis = await self.get_connection() + return await redis.hget( + self.get_full_data_cache_key(), + element_id) + + async def get_data_since(self, change_id: int, user_id: Optional[int] = None) -> Tuple[Dict[str, List[bytes]], List[str]]: + """ + Returns all elements since a change_id. + + The returend value is a two element tuple. The first value is a dict the elements where + the key is the collection_string and the value a list of (json-) encoded elements. The + second element is a list of element_ids, that have been deleted since the change_id. + + if user_id is None, the full_data is returned. If user_id is an int, the restricted_data + for an user is used. 0 is for the anonymous user. + """ + # TODO: rewrite with lua to get all elements with one request + redis = await self.get_connection() + changed_elements = defaultdict(list) # type: Dict[str, List[bytes]] + deleted_elements = [] # type: List[str] + for element_id in await redis.zrangebyscore(self.get_change_id_cache_key(), min=change_id): + if element_id.startswith(b'_config'): + continue + element_json = await redis.hget(self.get_full_data_cache_key(), element_id) # Optional[bytes] + if element_json is None: + # The element is not in the cache. It has to be deleted. + deleted_elements.append(element_id) + else: + collection_string, id = split_element_id(element_id) + changed_elements[collection_string].append(element_json) + return changed_elements, deleted_elements + + async def del_restricted_data(self, user_id: int) -> None: + """ + Deletes all restricted_data for an user. 0 is for the anonymous user. + """ + redis = await self.get_connection() + await redis.delete(self.get_restricted_data_cache_key(user_id)) + + async def set_lock_restricted_data(self, user_id: int) -> bool: + """ + Tries to sets a lock for the restricted_data of an user. + + Returns True when the lock could be set. + + Returns False when the lock was already set. + """ + redis = await self.get_connection() + return await redis.hsetnx(self.get_restricted_data_cache_key(user_id), self.lock_key, 1) + + async def get_lock_restricted_data(self, user_id: int) -> bool: + """ + Returns True, when the lock for the restricted_data of an user is set. Else False. + """ + redis = await self.get_connection() + return await redis.hget(self.get_restricted_data_cache_key(user_id), self.lock_key) + + async def del_lock_restricted_data(self, user_id: int) -> None: + """ + Deletes the lock for the restricted_data of an user. Does nothing when the + lock is not set. + """ + redis = await self.get_connection() + await redis.hdel(self.get_restricted_data_cache_key(user_id), self.lock_key) + + async def get_change_id_user(self, user_id: int) -> Optional[int]: + """ + Get the change_id for the restricted_data of an user. + + This is the change_id where the restricted_data was last calculated. + """ + redis = await self.get_connection() + return await redis.hget(self.get_restricted_data_cache_key(user_id), '_config:change_id') + + async def update_restricted_data(self, user_id: int, data: Dict[str, str]) -> None: + """ + Updates the restricted_data for an user. + + data has to be a dict where the key is an element_id and the value the (json-) encoded + element. + """ + redis = await self.get_connection() + await redis.hmset_dict(self.get_restricted_data_cache_key(user_id), data) + + async def get_current_change_id(self) -> List[Tuple[str, int]]: + """ + Get the highest change_id from redis. + """ + redis = await self.get_connection() + return await redis.zrevrangebyscore( + self.get_change_id_cache_key(), + withscores=True, + count=1, + offset=0) + + async def get_lowest_change_id(self) -> Optional[int]: + """ + Get the lowest change_id from redis. + + Returns None if lowest score does not exist. + """ + redis = await self.get_connection() + return await redis.zscore( + self.get_change_id_cache_key(), + '_config:lowest_change_id') + + +class MemmoryCacheProvider(BaseCacheProvider): + """ + CacheProvider for the ElementCache that uses only the memory. + + See the RedisCacheProvider for a description of the methods. + + This provider supports only one process. It saves the data into the memory. + When you use different processes they will use diffrent data. + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + self.clear_cache() + + def clear_cache(self) -> None: + self.full_data = {} # type: Dict[str, str] + self.restricted_data = {} # type: Dict[int, Dict[str, str]] + self.change_id_data = {} # type: Dict[int, Set[str]] + + async def reset_full_cache(self, data: Dict[str, str]) -> None: + self.full_data = data + + async def data_exists(self, user_id: Optional[int] = None) -> bool: + if user_id is None: + cache_dict = self.full_data + else: + cache_dict = self.restricted_data.get(user_id, {}) + + return bool(cache_dict) + + async def add_elements(self, elements: List[str]) -> None: + if len(elements) % 2: + raise ValueError("The argument elements of add_elements has to be a list with an even number of elements.") + + for i in range(0, len(elements), 2): + self.full_data[elements[i]] = elements[i+1] + + async def del_elements(self, elements: List[str], user_id: Optional[int] = None) -> None: + if user_id is None: + cache_dict = self.full_data + else: + cache_dict = self.restricted_data.get(user_id, {}) + + for element in elements: + try: + del cache_dict[element] + except KeyError: + pass + + async def add_changed_elements(self, change_id: int, element_ids: Iterable[str]) -> None: + element_ids = list(element_ids) + + for element_id in element_ids: + if change_id in self.change_id_data: + self.change_id_data[change_id].add(element_id) + else: + self.change_id_data[change_id] = {element_id} + + async def get_all_data(self, user_id: Optional[int] = None) -> Dict[bytes, bytes]: + if user_id is None: + cache_dict = self.full_data + else: + cache_dict = self.restricted_data.get(user_id, {}) + + return str_dict_to_bytes(cache_dict) + + async def get_element(self, element_id: str) -> Optional[bytes]: + value = self.full_data.get(element_id, None) + return value.encode() if value is not None else None + + async def get_data_since( + self, change_id: int, user_id: Optional[int] = None) -> Tuple[Dict[str, List[bytes]], List[str]]: + changed_elements = defaultdict(list) # type: Dict[str, List[bytes]] + deleted_elements = [] # type: List[str] + if user_id is None: + cache_dict = self.full_data + else: + cache_dict = self.restricted_data.get(user_id, {}) + + for data_change_id, element_ids in self.change_id_data.items(): + if data_change_id < change_id: + continue + for element_id in element_ids: + element_json = cache_dict.get(element_id, None) + if element_json is None: + deleted_elements.append(element_id) + else: + collection_string, id = split_element_id(element_id) + changed_elements[collection_string].append(element_json.encode()) + return changed_elements, deleted_elements + + async def del_restricted_data(self, user_id: int) -> None: + try: + del self.restricted_data[user_id] + except KeyError: + pass + + async def set_lock_restricted_data(self, user_id: int) -> bool: + data = self.restricted_data.setdefault(user_id, {}) + if self.lock_key in data: + return False + data[self.lock_key] = "1" + return True + + async def get_lock_restricted_data(self, user_id: int) -> bool: + data = self.restricted_data.get(user_id, {}) + return self.lock_key in data + + async def del_lock_restricted_data(self, user_id: int) -> None: + data = self.restricted_data.get(user_id, {}) + try: + del data[self.lock_key] + except KeyError: + pass + + async def get_change_id_user(self, user_id: int) -> Optional[int]: + data = self.restricted_data.get(user_id, {}) + change_id = data.get('_config:change_id', None) + return int(change_id) if change_id is not None else None + + async def update_restricted_data(self, user_id: int, data: Dict[str, str]) -> None: + redis_data = self.restricted_data.setdefault(user_id, {}) + redis_data.update(data) + + async def get_current_change_id(self) -> List[Tuple[str, int]]: + change_data = self.change_id_data + if change_data: + return [('no_usefull_value', max(change_data.keys()))] + return [] + + async def get_lowest_change_id(self) -> Optional[int]: + change_data = self.change_id_data + if change_data: + return min(change_data.keys()) + return None + + +class Cachable: + """ + A Cachable is an object that returns elements that can be cached. + + It needs at least the methods defined here. + """ + + def get_collection_string(self) -> str: + """ + Returns the string representing the name of the cachable. + """ + raise NotImplementedError("Cachable has to implement the method get_collection_string().") + + def get_elements(self) -> List[Dict[str, Any]]: + """ + Returns all elements of the cachable. + """ + raise NotImplementedError("Cachable has to implement the method get_collection_string().") + + def restrict_elements( + self, + user: Optional['CollectionElement'], + elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Converts full_data to restricted_data. + + elements can be an empty list, a list with some elements of the cachable or with all + elements of the cachable. + + The default implementation returns the full_data. + """ + return elements + + +def get_all_cachables() -> List[Cachable]: + """ + Returns all element of OpenSlides. + """ + out = [] # type: List[Cachable] + for app in apps.get_app_configs(): + try: + # Get the method get_startup_elements() from an app. + # This method has to return an iterable of Collection objects. + get_startup_elements = app.get_startup_elements + except AttributeError: + # Skip apps that do not implement get_startup_elements. + continue + out.extend(get_startup_elements()) + return out diff --git a/openslides/utils/collection.py b/openslides/utils/collection.py index 90d75bdd2..fc43430f6 100644 --- a/openslides/utils/collection.py +++ b/openslides/utils/collection.py @@ -10,11 +10,15 @@ from typing import ( cast, ) +from asgiref.sync import async_to_sync from django.apps import apps +from django.conf import settings from django.db.models import Model from mypy_extensions import TypedDict -from .cache import full_data_cache +from .cache import element_cache +from .cache_providers import Cachable + if TYPE_CHECKING: from .access_permissions import BaseAccessPermissions # noqa @@ -74,19 +78,12 @@ class CollectionElement: 'CollectionElement.from_values() but not CollectionElement() ' 'directly.') - if self.is_deleted(): - # Delete the element from the cache, if self.is_deleted() is True: - full_data_cache.del_element(self.collection_string, self.id) - else: - # The call to get_full_data() has some sideeffects. When the object - # was created with from_instance() or the object is not in the cache - # then get_full_data() will save the object into the cache. - # This will also raise a DoesNotExist error, if the object does - # neither exist in the cache nor in the database. - self.get_full_data() + if not self.deleted: + self.get_full_data() # This raises DoesNotExist, if the element does not exist. @classmethod - def from_instance(cls, instance: Model, deleted: bool = False, information: Dict[str, Any] = None) -> 'CollectionElement': + def from_instance( + cls, instance: Model, deleted: bool = False, information: Dict[str, Any] = None) -> 'CollectionElement': """ Returns a collection element from a database instance. @@ -175,6 +172,20 @@ class CollectionElement: """ return self.get_model().get_access_permissions() + def get_element_from_db(self) -> Optional[Dict[str, Any]]: + # Hack for django 2.0 and channels 2.1 to stay in the same thread. + # This is needed for the tests. + try: + query = self.get_model().objects.get_full_queryset() + except AttributeError: + # If the model des not have to method get_full_queryset(), then use + # the default queryset from django. + query = self.get_model().objects + try: + return self.get_access_permissions().get_full_data(query.get(pk=self.id)) + except self.get_model().DoesNotExist: + return None + def get_full_data(self) -> Dict[str, Any]: """ Returns the full_data of this collection_element from with all other @@ -188,14 +199,20 @@ class CollectionElement: # else: use the cache. if self.full_data is None: if self.instance is None: - # Make sure the cache exists - if not full_data_cache.exists_for_collection(self.collection_string): - # Build the cache if it does not exists. - full_data_cache.build_for_collection(self.collection_string) - self.full_data = full_data_cache.get_element(self.collection_string, self.id) + # The type of data has to be set for mypy + data = None # type: Optional[Dict[str, Any]] + if getattr(settings, 'SKIP_CACHE', False): + # Hack for django 2.0 and channels 2.1 to stay in the same thread. + # This is needed for the tests. + data = self.get_element_from_db() + else: + data = async_to_sync(element_cache.get_element_full_data)(self.collection_string, self.id) + if data is None: + raise self.get_model().DoesNotExist( + "Collection {} with id {} does not exist".format(self.collection_string, self.id)) + self.full_data = data else: self.full_data = self.get_access_permissions().get_full_data(self.instance) - full_data_cache.add_element(self.collection_string, self.id, self.full_data) return self.full_data def is_deleted(self) -> bool: @@ -205,7 +222,7 @@ class CollectionElement: return self.deleted -class Collection: +class Collection(Cachable): """ Represents all elements of one collection. """ @@ -242,17 +259,32 @@ class Collection: full_data['id'], full_data=full_data) + def get_elements_from_db(self) ->Dict[str, List[Dict[str, Any]]]: + # Hack for django 2.0 and channels 2.1 to stay in the same thread. + # This is needed for the tests. + try: + query = self.get_model().objects.get_full_queryset() + except AttributeError: + # If the model des not have to method get_full_queryset(), then use + # the default queryset from django. + query = self.get_model().objects + return {self.collection_string: [self.get_model().get_access_permissions().get_full_data(instance) for instance in query.all()]} + def get_full_data(self) -> List[Dict[str, Any]]: """ Returns a list of dictionaries with full_data of all collection elements. """ if self.full_data is None: - # Build the cache, if it does not exists. - if not full_data_cache.exists_for_collection(self.collection_string): - full_data_cache.build_for_collection(self.collection_string) - - self.full_data = full_data_cache.get_data(self.collection_string) + # The type of all_full_data has to be set for mypy + all_full_data = {} # type: Dict[str, List[Dict[str, Any]]] + if getattr(settings, 'SKIP_CACHE', False): + # Hack for django 2.0 and channels 2.1 to stay in the same thread. + # This is needed for the tests. + all_full_data = self.get_elements_from_db() + else: + all_full_data = async_to_sync(element_cache.get_all_full_data)() + self.full_data = all_full_data[self.collection_string] return self.full_data def as_list_for_user(self, user: Optional[CollectionElement]) -> List[Dict[str, Any]]: @@ -262,6 +294,27 @@ class Collection: """ return self.get_access_permissions().get_restricted_data(self.get_full_data(), user) + def get_collection_string(self) -> str: + """ + Returns the collection_string. + """ + return self.collection_string + + def get_elements(self) -> List[Dict[str, Any]]: + """ + Returns all elements of the Collection as full_data. + """ + return self.get_full_data() + + def restrict_elements( + self, + user: Optional['CollectionElement'], + elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Converts the full_data to restricted data. + """ + return self.get_model().get_access_permissions().get_restricted_data(user, elements) + _models_to_collection_string = {} # type: Dict[str, Type[Model]] @@ -295,7 +348,8 @@ def get_model_from_collection_string(collection_string: str) -> Type[Model]: return model -def format_for_autoupdate(collection_string: str, id: int, action: str, data: Dict[str, Any] = None) -> AutoupdateFormat: +def format_for_autoupdate( + collection_string: str, id: int, action: str, data: Dict[str, Any] = None) -> AutoupdateFormat: """ Returns a dict that can be used for autoupdate. """ diff --git a/openslides/utils/consumers.py b/openslides/utils/consumers.py new file mode 100644 index 000000000..820b1ad36 --- /dev/null +++ b/openslides/utils/consumers.py @@ -0,0 +1,306 @@ +from typing import Any, Dict, List, Optional + +from asgiref.sync import sync_to_async +from channels.db import database_sync_to_async +from channels.generic.websocket import AsyncJsonWebsocketConsumer + +from ..core.config import config +from ..core.models import Projector +from .auth import async_anonymous_is_enabled, has_perm +from .cache import element_cache, split_element_id +from .collection import AutoupdateFormat # noqa +from .collection import ( + Collection, + CollectionElement, + format_for_autoupdate, + from_channel_message, +) + + +class SiteConsumer(AsyncJsonWebsocketConsumer): + """ + Websocket Consumer for the site. + """ + groups = ['site'] + + async def connect(self) -> None: + """ + A user connects to the site. + + If it is an anonymous user and anonymous is disabled, the connection is closed. + + Sends the startup data to the user. + """ + # TODO: add a way to ask for the data since a change_id and send only data that is newer + if not await async_anonymous_is_enabled() and self.scope['user'].id is None: + await self.close() + else: + await self.accept() + data = await startup_data(self.scope['user']) + await self.send_json(data) + + async def receive_json(self, content: Any) -> None: + """ + If we recieve something from the client we currently just interpret this + as a notify message. + + The server adds the sender's user id (0 for anonymous) and reply + channel name so that a receiver client may reply to the sender or to all + sender's instances. + """ + if notify_message_is_valid(content): + await self.channel_layer.group_send( + "projector", + { + "type": "send_notify", + "incomming": content, + "senderReplyChannelName": self.channel_name, + "senderUserId": self.scope['user'].id or 0, + }, + ) + await self.channel_layer.group_send( + "site", + { + "type": "send_notify", + "incomming": content, + "senderReplyChannelName": self.channel_name, + "senderUserId": self.scope['user'].id or 0, + }, + ) + else: + await self.send_json({'error': 'invalid message'}) + + async def send_notify(self, event: Dict[str, Any]) -> None: + """ + Send a notify message to the user. + """ + user_id = self.scope['user'].id or 0 + + out = [] + for item in event['incomming']: + users = item.get('users') + reply_channels = item.get('replyChannels') + projectors = item.get('projectors') + if ((isinstance(users, list) and user_id in users) + or (isinstance(reply_channels, list) and self.channel_name in reply_channels) + or (users is None and reply_channels is None and projectors is None)): + item['senderReplyChannelName'] = event.get('senderReplyChannelName') + item['senderUserId'] = event.get('senderUserId') + item['senderProjectorId'] = event.get('senderProjectorId') + out.append(item) + + if out: + await self.send_json(out) + + async def send_data(self, event: Dict[str, Any]) -> None: + """ + Send changed or deleted elements to the user. + """ + change_id = event['change_id'] + output = [] + changed_elements, deleted_elements = await element_cache.get_restricted_data(self.scope['user'], change_id) + for collection_string, elements in changed_elements.items(): + for element in elements: + output.append(format_for_autoupdate( + collection_string=collection_string, + id=element['id'], + action='changed', + data=element)) + for element_id in deleted_elements: + collection_string, id = split_element_id(element_id) + output.append(format_for_autoupdate( + collection_string=collection_string, + id=id, + action='deleted')) + await self.send_json(output) + + +class ProjectorConsumer(AsyncJsonWebsocketConsumer): + """ + Websocket Consumer for the projector. + """ + + groups = ['projector'] + + async def connect(self) -> None: + """ + Adds the websocket connection to a group specific to the projector with the given id. + Also sends all data that are shown on the projector. + """ + user = self.scope['user'] + projector_id = self.scope["url_route"]["kwargs"]["projector_id"] + await self.accept() + + if not await database_sync_to_async(has_perm)(user, 'core.can_see_projector'): + await self.send_json({'text': 'No permissions to see this projector.'}) + # TODO: Shouldend we just close the websocket connection with an error message? + # self.close(code=4403) + else: + out = await sync_to_async(projector_startup_data)(projector_id) + await self.send_json(out) + + async def receive_json(self, content: Any) -> None: + """ + If we recieve something from the client we currently just interpret this + as a notify message. + + The server adds the sender's user id (0 for anonymous) and reply + channel name so that a receiver client may reply to the sender or to all + sender's instances. + """ + projector_id = self.scope["url_route"]["kwargs"]["projector_id"] + await self.channel_layer.group_send( + "projector", + { + "type": "send_notify", + "incomming": content, + "senderReplyChannelName": self.channel_name, + "senderProjectorId": projector_id, + }, + ) + await self.channel_layer.group_send( + "site", + { + "type": "send_notify", + "incomming": content, + "senderReplyChannelName": self.channel_name, + "senderProjectorId": projector_id, + }, + ) + + async def send_notify(self, event: Dict[str, Any]) -> None: + """ + Send a notify message to the projector. + """ + projector_id = self.scope["url_route"]["kwargs"]["projector_id"] + + out = [] + for item in event['incomming']: + users = item.get('users') + reply_channels = item.get('replyChannels') + projectors = item.get('projectors') + if ((isinstance(projectors, list) and projector_id in projectors) + or (isinstance(reply_channels, list) and self.channel_name in reply_channels) + or (users is None and reply_channels is None and projectors is None)): + item['senderReplyChannelName'] = event.get('senderReplyChannelName') + item['senderUserId'] = event.get('senderUserId') + item['senderProjectorId'] = event.get('senderProjectorId') + out.append(item) + + if out: + await self.send_json(out) + + async def send_data(self, event: Dict[str, Any]) -> None: + """ + Informs all projector clients about changed data. + """ + projector_id = self.scope["url_route"]["kwargs"]["projector_id"] + collection_elements = from_channel_message(event['message']) + + output = await projector_sync_send_data(projector_id, collection_elements) + if output: + await self.send_json(output) + + +async def startup_data(user: Optional[CollectionElement], change_id: int = 0) -> List[Any]: + """ + Returns all data for startup. + """ + # TODO: use the change_id argument + output = [] + restricted_data = await element_cache.get_all_restricted_data(user) + for collection_string, elements in restricted_data.items(): + for element in elements: + formatted_data = format_for_autoupdate( + collection_string=collection_string, + id=element['id'], + action='changed', + data=element) + + output.append(formatted_data) + return output + + +def projector_startup_data(projector_id: int) -> Any: + """ + Generate the startup data for a projector. + """ + try: + projector = Projector.objects.get(pk=projector_id) + except Projector.DoesNotExist: + return {'text': 'The projector {} does not exist.'.format(projector_id)} + else: + # Now check whether broadcast is active at the moment. If yes, + # change the local projector variable. + if config['projector_broadcast'] > 0: + projector = Projector.objects.get(pk=config['projector_broadcast']) + + # Collect all elements that are on the projector. + output = [] # type: List[AutoupdateFormat] + for requirement in projector.get_all_requirements(): + required_collection_element = CollectionElement.from_instance(requirement) + output.append(required_collection_element.as_autoupdate_for_projector()) + + # Collect all config elements. + config_collection = Collection(config.get_collection_string()) + projector_data = (config_collection.get_access_permissions() + .get_projector_data(config_collection.get_full_data())) + for data in projector_data: + output.append(format_for_autoupdate( + config_collection.collection_string, + data['id'], + 'changed', + data)) + + # Collect the projector instance. + collection_element = CollectionElement.from_instance(projector) + output.append(collection_element.as_autoupdate_for_projector()) + + # Send all the data that were only collected before. + return output + + +@sync_to_async +def projector_sync_send_data(projector_id: int, collection_elements: List[CollectionElement]) -> List[Any]: + """ + sync function that generates the elements for an projector. + """ + # Load the projector object. If broadcast is on, use the broadcast projector + # instead. + if config['projector_broadcast'] > 0: + projector_id = config['projector_broadcast'] + + projector = Projector.objects.get(pk=projector_id) + + # TODO: This runs once for every open projector tab. Either use + # caching or something else, so this is only called once + output = [] + for collection_element in collection_elements: + if collection_element.is_deleted(): + output.append(collection_element.as_autoupdate_for_projector()) + else: + for element in projector.get_collection_elements_required_for_this(collection_element): + output.append(element.as_autoupdate_for_projector()) + return output + + +def notify_message_is_valid(message: object) -> bool: + """ + Returns True, when the message is a valid notify_message. + """ + if not isinstance(message, list): + # message has to be a list + return False + + if not message: + # message must contain at least one element + return False + + for element in message: + if not isinstance(element, dict): + # All elements have to be a dict + return False + # TODO: There could be more checks. For example 'users' has to be a list of int + # Check could be done with json-schema: + # https://pypi.org/project/jsonschema/ + return True diff --git a/openslides/utils/main.py b/openslides/utils/main.py index 6d20240f9..43503a6c4 100644 --- a/openslides/utils/main.py +++ b/openslides/utils/main.py @@ -13,6 +13,7 @@ from django.core.exceptions import ImproperlyConfigured from django.utils.crypto import get_random_string from mypy_extensions import NoReturn + DEVELOPMENT_VERSION = 'Development Version' UNIX_VERSION = 'Unix Version' WINDOWS_VERSION = 'Windows Version' @@ -327,16 +328,6 @@ def is_local_installation() -> bool: return True if '--local-installation' in sys.argv or 'manage.py' in sys.argv[0] else False -def get_geiss_path() -> str: - """ - Returns the path and file to the Geiss binary. - """ - from django.conf import settings - download_dir = getattr(settings, 'OPENSLIDES_USER_DATA_PATH', '') - bin_name = 'geiss.exe' if is_windows() else 'geiss' - return os.path.join(download_dir, bin_name) - - def is_windows() -> bool: """ Returns True if the current system is Windows. Returns False otherwise. diff --git a/openslides/utils/middleware.py b/openslides/utils/middleware.py new file mode 100644 index 000000000..152ffae40 --- /dev/null +++ b/openslides/utils/middleware.py @@ -0,0 +1,63 @@ +from typing import Any, Dict, Union + +from channels.auth import ( + AuthMiddleware, + CookieMiddleware, + SessionMiddleware, + _get_user_session_key, +) +from django.conf import settings +from django.contrib.auth import BACKEND_SESSION_KEY, HASH_SESSION_KEY +from django.contrib.auth.models import AnonymousUser +from django.utils.crypto import constant_time_compare + +from .cache import element_cache +from .collection import CollectionElement + + +class CollectionAuthMiddleware(AuthMiddleware): + """ + Like the channels AuthMiddleware but returns a CollectionElement instead of + a django Model as user. + """ + + async def resolve_scope(self, scope: Dict[str, Any]) -> None: + scope["user"]._wrapped = await get_user(scope) + + +async def get_user(scope: Dict[str, Any]) -> Union[CollectionElement, AnonymousUser]: + """ + Returns a User-CollectionElement from a channels-scope-session. + + If no user is retrieved, return AnonymousUser. + """ + # This can not return None because a LazyObject can not become None + + # This code is basicly from channels.auth: + # https://github.com/django/channels/blob/d5e81a78e96770127da79248349808b6ee6ec2a7/channels/auth.py#L16 + if "session" not in scope: + raise ValueError("Cannot find session in scope. You should wrap your consumer in SessionMiddleware.") + session = scope["session"] + user = None + try: + user_id = _get_user_session_key(session) + backend_path = session[BACKEND_SESSION_KEY] + except KeyError: + pass + else: + if backend_path in settings.AUTHENTICATION_BACKENDS: + user = await element_cache.get_element_full_data("users/user", user_id) + if user is not None: + # Verify the session + session_hash = session.get(HASH_SESSION_KEY) + session_hash_verified = session_hash and constant_time_compare( + session_hash, + user['session_auth_hash']) + if not session_hash_verified: + session.flush() + user = None + return CollectionElement.from_values("users/user", user_id, full_data=user) if user else AnonymousUser() + + +# Handy shortcut for applying all three layers at once +AuthMiddlewareStack = lambda inner: CookieMiddleware(SessionMiddleware(CollectionAuthMiddleware(inner))) # noqa diff --git a/openslides/utils/migrations.py b/openslides/utils/migrations.py index 371265db7..303e6baed 100644 --- a/openslides/utils/migrations.py +++ b/openslides/utils/migrations.py @@ -1,4 +1,4 @@ -from typing import Any, Callable # noqa +from typing import Any, Callable from django.contrib.auth.models import Permission from django.contrib.contenttypes.models import ContentType diff --git a/openslides/utils/models.py b/openslides/utils/models.py index 725825f74..08bc033fb 100644 --- a/openslides/utils/models.py +++ b/openslides/utils/models.py @@ -1,12 +1,17 @@ -from typing import Any, Dict +from typing import TYPE_CHECKING, Any, Dict, List, Optional from django.core.exceptions import ImproperlyConfigured from django.db import models -from .access_permissions import BaseAccessPermissions # noqa +from .access_permissions import BaseAccessPermissions from .utils import convert_camel_case_to_pseudo_snake_case +if TYPE_CHECKING: + # Dummy import Collection for mypy, can be fixed with python 3.7 + from .collection import Collection, CollectionElement # noqa + + class MinMaxIntegerField(models.IntegerField): """ IntegerField with options to set a min- and a max-value. @@ -117,3 +122,29 @@ class RESTModelMixin: else: inform_deleted_data([(self.get_collection_string(), instance_pk)], information=information) return return_value + + @classmethod + def get_elements(cls) -> List[Dict[str, Any]]: + """ + Returns all elements as full_data. + """ + # Get the query to receive all data from the database. + try: + query = cls.objects.get_full_queryset() # type: ignore + except AttributeError: + # If the model des not have to method get_full_queryset(), then use + # the default queryset from django. + query = cls.objects # type: ignore + + # Build a dict from the instance id to the full_data + return [cls.get_access_permissions().get_full_data(instance) for instance in query.all()] + + @classmethod + def restrict_elements( + cls, + user: Optional['CollectionElement'], + elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Converts a list of elements from full_data to restricted_data. + """ + return cls.get_access_permissions().get_restricted_data(elements, user) diff --git a/openslides/utils/rest_api.py b/openslides/utils/rest_api.py index bd44c5fe9..e8c40733a 100644 --- a/openslides/utils/rest_api.py +++ b/openslides/utils/rest_api.py @@ -5,17 +5,16 @@ from django.http import Http404 from rest_framework import status # noqa from rest_framework.decorators import detail_route, list_route # noqa from rest_framework.metadata import SimpleMetadata # noqa -from rest_framework.mixins import ListModelMixin as _ListModelMixin -from rest_framework.mixins import RetrieveModelMixin as _RetrieveModelMixin from rest_framework.mixins import ( # noqa CreateModelMixin, DestroyModelMixin, + ListModelMixin as _ListModelMixin, + RetrieveModelMixin as _RetrieveModelMixin, UpdateModelMixin, ) from rest_framework.relations import MANY_RELATION_KWARGS from rest_framework.response import Response from rest_framework.routers import DefaultRouter -from rest_framework.serializers import ModelSerializer as _ModelSerializer from rest_framework.serializers import ( # noqa CharField, DictField, @@ -26,20 +25,24 @@ from rest_framework.serializers import ( # noqa ListField, ListSerializer, ManyRelatedField, + ModelSerializer as _ModelSerializer, PrimaryKeyRelatedField, RelatedField, Serializer, SerializerMethodField, ValidationError, ) -from rest_framework.viewsets import GenericViewSet as _GenericViewSet # noqa -from rest_framework.viewsets import ModelViewSet as _ModelViewSet # noqa -from rest_framework.viewsets import ViewSet as _ViewSet # noqa +from rest_framework.viewsets import ( # noqa + GenericViewSet as _GenericViewSet, + ModelViewSet as _ModelViewSet, + ViewSet as _ViewSet, +) from .access_permissions import BaseAccessPermissions from .auth import user_to_collection_user from .collection import Collection, CollectionElement + router = DefaultRouter() diff --git a/openslides/utils/settings.py.tpl b/openslides/utils/settings.py.tpl index c4d2252d5..6dec3787b 100644 --- a/openslides/utils/settings.py.tpl +++ b/openslides/utils/settings.py.tpl @@ -79,59 +79,38 @@ DATABASES = { use_redis = False if use_redis: - # Redis configuration for django-redis-session. Keep this synchronized to - # the caching settings + # Django Channels + # https://channels.readthedocs.io/en/latest/topics/channel_layers.html#configuration + + CHANNEL_LAYERS['default']['BACKEND'] = 'channels_redis.core.RedisChannelLayer' + CHANNEL_LAYERS['default']['CONFIG'] = {"capacity": 100000} + + # Collection Cache + + # Can be: + # a Redis URI — "redis://host:6379/0?encoding=utf-8"; + # a (host, port) tuple — ('localhost', 6379); + # or a unix domain socket path string — "/path/to/redis.sock". + REDIS_ADDRESS = "redis://127.0.0.1" + + # When use_redis is True, the restricted data cache caches the data individuel + # for each user. This requires a lot of memory if there are a lot of active + # users. + RESTRICTED_DATA_CACHE = True + + # Session backend + + # Redis configuration for django-redis-sessions. + # https://github.com/martinrusev/django-redis-sessions + + SESSION_ENGINE = 'redis_sessions.session' SESSION_REDIS = { 'host': '127.0.0.1', 'post': 6379, 'db': 0, } - # Django Channels - - # Unless you have only a small assembly uncomment the following lines to - # activate Redis as backend for Django Channels and Cache. You have to install - # a Redis server and the python packages asgi_redis and django-redis. - - # https://channels.readthedocs.io/en/latest/backends.html#redis - - CHANNEL_LAYERS['default']['BACKEND'] = 'asgi_redis.RedisChannelLayer' - CHANNEL_LAYERS['default']['CONFIG']['prefix'] = 'asgi:' - - - # Caching - - # Django uses a inmemory cache at default. This supports only one thread. If - # you use more then one thread another caching backend is required. We recommand - # django-redis: https://niwinz.github.io/django-redis/latest/#_user_guide - - CACHES = { - "default": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://127.0.0.1:6379/0", - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - }, - "KEY_PREFIX": "openslides-cache", - } - } - - # Session backend - - # Per default django uses the database as session backend. This can be slow. - # One possibility is to use the cache session backend with redis as cache backend - # Another possibility is to use a native redis session backend. For example: - # https://github.com/martinrusev/django-redis-sessions - - # SESSION_ENGINE = 'django.contrib.sessions.backends.cache' - SESSION_ENGINE = 'redis_sessions.session' - - -# When use_redis is True, the restricted data cache caches the data individuel -# for each user. This requires a lot of memory if there are a lot of active -# users. If use_redis is False, this setting has no effect. -DISABLE_USER_CACHE = False # Internationalization # https://docs.djangoproject.com/en/1.10/topics/i18n/ diff --git a/openslides/utils/test.py b/openslides/utils/test.py index 5c714b611..abc47ca12 100644 --- a/openslides/utils/test.py +++ b/openslides/utils/test.py @@ -1,37 +1,12 @@ from django.test import TestCase as _TestCase -from django.test.runner import DiscoverRunner from ..core.config import config -class OpenSlidesDiscoverRunner(DiscoverRunner): - def run_tests(self, test_labels, extra_tests=None, **kwargs): # type: ignore - """ - Test Runner which does not create a database, if only unittest are run. - """ - if len(test_labels) == 1 and test_labels[0].startswith('tests.unit'): - # Do not create a test database, if only unittests are tested - create_database = False - else: - create_database = True - - self.setup_test_environment() - suite = self.build_suite(test_labels, extra_tests) - if create_database: - old_config = self.setup_databases() - result = self.run_suite(suite) - if create_database: - self.teardown_databases(old_config) - self.teardown_test_environment() - return self.suite_result(suite, result) - - class TestCase(_TestCase): """ Resets the config object after each test. """ def tearDown(self) -> None: - from django_redis import get_redis_connection - config.key_to_id = {} - get_redis_connection("default").flushall() + config.save_default_values() diff --git a/openslides/utils/utils.py b/openslides/utils/utils.py index 884e74e55..8fc4a6dd9 100644 --- a/openslides/utils/utils.py +++ b/openslides/utils/utils.py @@ -1,7 +1,13 @@ import re +from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union import roman + +if TYPE_CHECKING: + # Dummy import Collection for mypy, can be fixed with python 3.7 + from .collection import Collection, CollectionElement # noqa + CAMEL_CASE_TO_PSEUDO_SNAKE_CASE_CONVERSION_REGEX_1 = re.compile('(.)([A-Z][a-z]+)') CAMEL_CASE_TO_PSEUDO_SNAKE_CASE_CONVERSION_REGEX_2 = re.compile('([a-z0-9])([A-Z])') @@ -29,3 +35,43 @@ def to_roman(number: int) -> str: return roman.toRoman(number) except (roman.NotIntegerError, roman.OutOfRangeError): return str(number) + + +def get_element_id(collection_string: str, id: int) -> str: + """ + Returns a combined string from the collection_string and an id. + """ + return "{}:{}".format(collection_string, id) + + +def split_element_id(element_id: Union[str, bytes]) -> Tuple[str, int]: + """ + Splits a combined element_id into the collection_string and the id. + """ + if isinstance(element_id, bytes): + element_id = element_id.decode() + collection_str, id = element_id.rsplit(":", 1) + return (collection_str, int(id)) + + +def get_user_id(user: Optional['CollectionElement']) -> int: + """ + Returns the user id for an CollectionElement user. + + Returns 0 for anonymous. + """ + if user is None: + user_id = 0 + else: + user_id = user.id + return user_id + + +def str_dict_to_bytes(str_dict: Dict[str, str]) -> Dict[bytes, bytes]: + """ + Converts the key and the value of a dict from str to bytes. + """ + out = {} + for key, value in str_dict.items(): + out[key.encode()] = value.encode() + return out diff --git a/openslides/utils/validate.py b/openslides/utils/validate.py index e0e5bbb0e..5d3620720 100644 --- a/openslides/utils/validate.py +++ b/openslides/utils/validate.py @@ -1,5 +1,6 @@ import bleach + allowed_tags = [ 'a', 'img', # links and images 'br', 'p', 'span', 'blockquote', # text layout diff --git a/requirements.txt b/requirements.txt index 0d17f1f97..cc82a1270 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,8 @@ coverage #flake8 # Use master of flake8 until flake8 3.6 is released that supports python3.7 git+https://gitlab.com/pycqa/flake8.git -isort==4.2.5 +isort mypy -fakeredis +pytest-django +pytest-asyncio +pytest-cov diff --git a/requirements_big_mode.txt b/requirements_big_mode.txt index bc4ea3d4d..b12a46e91 100644 --- a/requirements_big_mode.txt +++ b/requirements_big_mode.txt @@ -2,8 +2,7 @@ -r requirements_production.txt # Requirements for Redis and PostgreSQL support -asgi-redis>=1.3,<1.5 -django-redis>=4.7.0,<4.10 +channels-redis>=2.2,<2.3 django-redis-sessions>=0.6.1,<0.7 -psycopg2-binary>=2.7,<2.8 -txredisapi==1.4.4 +psycopg2-binary>=2.7.3.2,<2.8 +aioredis>=1.1.0,<1.2 diff --git a/requirements_production.txt b/requirements_production.txt index edd944dc5..624f06fe7 100644 --- a/requirements_production.txt +++ b/requirements_production.txt @@ -1,8 +1,8 @@ # Requirements for OpenSlides in production in alphabetical order bleach>=1.5.0,<2.2 -channels>=1.1,<1.2 -daphne<2 -Django>=1.10.4,<2.2 +channels>=2.1.2,<2.2 +daphne>=2.2,<2.3 +Django>=1.11,<2.2 djangorestframework>=3.4,<3.9 jsonfield2>=3.0,<3.1 mypy_extensions>=0.3,<0.4 diff --git a/setup.cfg b/setup.cfg index 86bcf93e6..a54329814 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,6 +13,8 @@ max_line_length = 150 [isort] include_trailing_comma = true multi_line_output = 3 +lines_after_imports = 2 +combine_as_imports = true [mypy] ignore_missing_imports = true @@ -25,5 +27,6 @@ disallow_untyped_defs = true [mypy-openslides.core.config] disallow_untyped_defs = true -[mypy-tests.*] -ignore_errors = true +[tool:pytest] +DJANGO_SETTINGS_MODULE = tests.settings +testpaths = tests/ diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..d288c1458 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,40 @@ +from django.test import TestCase, TransactionTestCase +from pytest_django.plugin import validate_django_db + + +def pytest_collection_modifyitems(items): + """ + Helper until https://github.com/pytest-dev/pytest-django/issues/214 is fixed. + """ + def get_marker_transaction(test): + marker = test.get_marker('django_db') + if marker: + validate_django_db(marker) + return marker.kwargs['transaction'] + + return None + + def has_fixture(test, fixture): + funcargnames = getattr(test, 'funcargnames', None) + return funcargnames and fixture in funcargnames + + def weight_test_case(test): + """ + Key function for ordering test cases like the Django test runner. + """ + is_test_case_subclass = test.cls and issubclass(test.cls, TestCase) + is_transaction_test_case_subclass = test.cls and issubclass(test.cls, TransactionTestCase) + + if is_test_case_subclass or get_marker_transaction(test) is False: + return 0 + elif has_fixture(test, 'db'): + return 0 + + if is_transaction_test_case_subclass or get_marker_transaction(test) is True: + return 1 + elif has_fixture(test, 'transactional_db'): + return 1 + + return 0 + + items.sort(key=weight_test_case) diff --git a/tests/example_data_generator/management/commands/create-example-data.py b/tests/example_data_generator/management/commands/create-example-data.py index a7755cc42..060b765cd 100644 --- a/tests/example_data_generator/management/commands/create-example-data.py +++ b/tests/example_data_generator/management/commands/create-example-data.py @@ -12,6 +12,7 @@ from openslides.motions.models import Motion from openslides.topics.models import Topic from openslides.users.models import Group, User + MOTION_NUMBER_OF_PARAGRAPHS = 4 LOREM_IPSUM = [ diff --git a/tests/integration/agenda/test_viewset.py b/tests/integration/agenda/test_viewset.py index 0e4792309..6c810a0fc 100644 --- a/tests/integration/agenda/test_viewset.py +++ b/tests/integration/agenda/test_viewset.py @@ -1,8 +1,8 @@ +import pytest from django.contrib.auth import get_user_model from django.contrib.auth.models import Permission from django.urls import reverse from django.utils.translation import ugettext -from django_redis import get_redis_connection from rest_framework import status from rest_framework.test import APIClient @@ -12,10 +12,11 @@ from openslides.core.config import config from openslides.core.models import Countdown from openslides.motions.models import Motion from openslides.topics.models import Topic -from openslides.users.models import User from openslides.utils.collection import CollectionElement from openslides.utils.test import TestCase +from ..helpers import count_queries + class RetrieveItem(TestCase): """ @@ -89,63 +90,29 @@ class RetrieveItem(TestCase): self.assertTrue(response.data.get('comment') is None) -class TestDBQueries(TestCase): +@pytest.mark.django_db(transaction=False) +def test_agenda_item_db_queries(): """ - Tests that receiving elements only need the required db queries. + Tests that only the following db queries are done: + * 1 requests to get the list of all agenda items, + * 1 request to get all speakers, + * 3 requests to get the assignments, motions and topics and - Therefore in setup some agenda items are created and received with different - user accounts. + * 1 request to get an agenda item (why?) + * 2 requests for the motionsversions. + TODO: The last three request are a bug. """ + for index in range(10): + Topic.objects.create(title='topic{}'.format(index)) + parent = Topic.objects.create(title='parent').agenda_item + child = Topic.objects.create(title='child').agenda_item + child.parent = parent + child.save() + Motion.objects.create(title='motion1') + Motion.objects.create(title='motion2') + Assignment.objects.create(title='assignment', open_posts=5) - def setUp(self): - self.client = APIClient() - config['general_system_enable_anonymous'] = True - for index in range(10): - Topic.objects.create(title='topic{}'.format(index)) - parent = Topic.objects.create(title='parent').agenda_item - child = Topic.objects.create(title='child').agenda_item - child.parent = parent - child.save() - Motion.objects.create(title='motion1') - Motion.objects.create(title='motion2') - Assignment.objects.create(title='assignment', open_posts=5) - - def test_admin(self): - """ - Tests that only the following db queries are done: - * 7 requests to get the session an the request user with its permissions, - * 1 requests to get the list of all agenda items, - * 1 request to get all speakers, - * 3 requests to get the assignments, motions and topics and - - * 1 request to get an agenda item (why?) - - * 2 requests for the motionsversions. - - TODO: The last two request for the motionsversions are a bug. - """ - self.client.force_login(User.objects.get(pk=1)) - get_redis_connection("default").flushall() - with self.assertNumQueries(15): - self.client.get(reverse('item-list')) - - def test_anonymous(self): - """ - Tests that only the following db queries are done: - * 3 requests to get the permission for anonymous, - * 1 requests to get the list of all agenda items, - * 1 request to get all speakers, - * 3 requests to get the assignments, motions and topics and - - * 1 request to get an agenda item (why?) - - * 2 requests for the motionsversions. - - TODO: The last two request for the motionsversions are a bug. - """ - get_redis_connection("default").flushall() - with self.assertNumQueries(11): - self.client.get(reverse('item-list')) + assert count_queries(Item.get_elements) == 8 class ManageSpeaker(TestCase): diff --git a/tests/integration/assignments/test_viewset.py b/tests/integration/assignments/test_viewset.py index dad256b6a..7b490a423 100644 --- a/tests/integration/assignments/test_viewset.py +++ b/tests/integration/assignments/test_viewset.py @@ -1,66 +1,33 @@ +import pytest from django.contrib.auth import get_user_model from django.urls import reverse -from django_redis import get_redis_connection from rest_framework import status from rest_framework.test import APIClient from openslides.assignments.models import Assignment -from openslides.core.config import config -from openslides.users.models import User from openslides.utils.test import TestCase +from ..helpers import count_queries -class TestDBQueries(TestCase): + +@pytest.mark.django_db(transaction=False) +def test_assignment_db_queries(): """ - Tests that receiving elements only need the required db queries. + Tests that only the following db queries are done: + * 1 requests to get the list of all assignments, + * 1 request to get all related users, + * 1 request to get the agenda item, + * 1 request to get the polls, + * 1 request to get the tags and - Therefore in setup some assignments are created and received with different - user accounts. + * 10 request to fetch each related user again. + + TODO: The last request are a bug. """ + for index in range(10): + Assignment.objects.create(title='assignment{}'.format(index), open_posts=1) - def setUp(self): - self.client = APIClient() - config['general_system_enable_anonymous'] = True - config.save_default_values() - for index in range(10): - Assignment.objects.create(title='motion{}'.format(index), open_posts=1) - - def test_admin(self): - """ - Tests that only the following db queries are done: - * 7 requests to get the session an the request user with its permissions, - * 1 requests to get the list of all assignments, - * 1 request to get all related users, - * 1 request to get the agenda item, - * 1 request to get the polls, - * 1 request to get the tags and - - * 10 request to fetch each related user again. - - TODO: The last request are a bug. - """ - self.client.force_login(User.objects.get(pk=1)) - get_redis_connection("default").flushall() - with self.assertNumQueries(22): - self.client.get(reverse('assignment-list')) - - def test_anonymous(self): - """ - Tests that only the following db queries are done: - * 3 requests to get the permission for anonymous, - * 1 requests to get the list of all assignments, - * 1 request to get all related users, - * 1 request to get the agenda item, - * 1 request to get the polls, - * 1 request to get the tags and - - * 10 request to fetch each related user again. - - TODO: The last 10 requests are an bug. - """ - get_redis_connection("default").flushall() - with self.assertNumQueries(18): - self.client.get(reverse('assignment-list')) + assert count_queries(Assignment.get_elements) == 15 class CanidatureSelf(TestCase): @@ -110,7 +77,6 @@ class CanidatureSelf(TestCase): group_delegates = type(group_admin).objects.get(name='Delegates') admin.groups.add(group_delegates) admin.groups.remove(group_admin) - get_redis_connection('default').flushall() response = self.client.post(reverse('assignment-candidature-self', args=[self.assignment.pk])) @@ -157,7 +123,6 @@ class CanidatureSelf(TestCase): group_delegates = type(group_admin).objects.get(name='Delegates') admin.groups.add(group_delegates) admin.groups.remove(group_admin) - get_redis_connection('default').flushall() response = self.client.delete(reverse('assignment-candidature-self', args=[self.assignment.pk])) @@ -238,7 +203,6 @@ class CandidatureOther(TestCase): group_delegates = type(group_admin).objects.get(name='Delegates') admin.groups.add(group_delegates) admin.groups.remove(group_admin) - get_redis_connection('default').flushall() response = self.client.post( reverse('assignment-candidature-other', args=[self.assignment.pk]), @@ -294,7 +258,6 @@ class CandidatureOther(TestCase): group_delegates = type(group_admin).objects.get(name='Delegates') admin.groups.add(group_delegates) admin.groups.remove(group_admin) - get_redis_connection('default').flushall() response = self.client.delete( reverse('assignment-candidature-other', args=[self.assignment.pk]), diff --git a/tests/integration/core/test_views.py b/tests/integration/core/test_views.py index 944e1f092..e88098175 100644 --- a/tests/integration/core/test_views.py +++ b/tests/integration/core/test_views.py @@ -5,9 +5,11 @@ from django.urls import reverse from rest_framework import status from rest_framework.test import APIClient -from openslides import __license__ as license -from openslides import __url__ as url -from openslides import __version__ as version +from openslides import ( + __license__ as license, + __url__ as url, + __version__ as version, +) from openslides.core.config import ConfigVariable, config from openslides.core.models import Projector from openslides.topics.models import Topic @@ -114,7 +116,7 @@ class ConfigViewSet(TestCase): # Save the old value of the config object and add the test values # TODO: Can be changed to setUpClass when Django 1.8 is no longer supported self._config_values = config.config_variables.copy() - config.key_to_id = {} + config.key_to_id = None config.update_config_variables(set_simple_config_view_integration_config_test()) config.save_default_values() diff --git a/tests/integration/core/test_viewset.py b/tests/integration/core/test_viewset.py index 47e046ac3..874b50746 100644 --- a/tests/integration/core/test_viewset.py +++ b/tests/integration/core/test_viewset.py @@ -1,150 +1,62 @@ +import pytest from django.urls import reverse -from django_redis import get_redis_connection from rest_framework import status -from rest_framework.test import APIClient from openslides.core.config import config from openslides.core.models import ChatMessage, Projector, Tag from openslides.users.models import User from openslides.utils.test import TestCase +from ..helpers import count_queries -class TestProjectorDBQueries(TestCase): + +@pytest.mark.django_db(transaction=False) +def test_projector_db_queries(): """ - Tests that receiving elements only need the required db queries. - - Therefore in setup some objects are created and received with different - user accounts. + Tests that only the following db queries are done: + * 1 requests to get the list of all projectors, + * 1 request to get the list of the projector defaults. """ + for index in range(10): + Projector.objects.create(name="Projector{}".format(index)) - def setUp(self): - self.client = APIClient() - config['general_system_enable_anonymous'] = True - config.save_default_values() - for index in range(10): - Projector.objects.create(name="Projector{}".format(index)) - - def test_admin(self): - """ - Tests that only the following db queries are done: - * 7 requests to get the session an the request user with its permissions, - * 1 requests to get the list of all projectors, - * 1 request to get the list of the projector defaults. - """ - self.client.force_login(User.objects.get(pk=1)) - get_redis_connection("default").flushall() - with self.assertNumQueries(9): - self.client.get(reverse('projector-list')) - - def test_anonymous(self): - """ - Tests that only the following db queries are done: - * 3 requests to get the permission for anonymous, - * 1 requests to get the list of all projectors, - * 1 request to get the list of the projector defaults and - """ - get_redis_connection("default").flushall() - with self.assertNumQueries(5): - self.client.get(reverse('projector-list')) + assert count_queries(Projector.get_elements) == 2 -class TestCharmessageDBQueries(TestCase): +@pytest.mark.django_db(transaction=False) +def test_chat_message_db_queries(): """ - Tests that receiving elements only need the required db queries. - - Therefore in setup some objects are created and received with different - user accounts. + Tests that only the following db queries are done: + * 1 requests to get the list of all chatmessages. """ + user = User.objects.get(username='admin') + for index in range(10): + ChatMessage.objects.create(user=user) - def setUp(self): - self.client = APIClient() - config['general_system_enable_anonymous'] = True - config.save_default_values() - user = User.objects.get(pk=1) - for index in range(10): - ChatMessage.objects.create(user=user) - - def test_admin(self): - """ - Tests that only the following db queries are done: - * 7 requests to get the session an the request user with its permissions, - * 1 requests to get the list of all chatmessages, - """ - self.client.force_login(User.objects.get(pk=1)) - get_redis_connection("default").flushall() - with self.assertNumQueries(8): - self.client.get(reverse('chatmessage-list')) + assert count_queries(ChatMessage.get_elements) == 1 -class TestTagDBQueries(TestCase): +@pytest.mark.django_db(transaction=False) +def test_tag_db_queries(): """ - Tests that receiving elements only need the required db queries. - - Therefore in setup some objects are created and received with different - user accounts. + Tests that only the following db queries are done: + * 1 requests to get the list of all tags. """ + for index in range(10): + Tag.objects.create(name='tag{}'.format(index)) - def setUp(self): - self.client = APIClient() - config['general_system_enable_anonymous'] = True - config.save_default_values() - for index in range(10): - Tag.objects.create(name='tag{}'.format(index)) - - def test_admin(self): - """ - Tests that only the following db queries are done: - * 5 requests to get the session an the request user with its permissions, - * 1 requests to get the list of all tags, - """ - self.client.force_login(User.objects.get(pk=1)) - get_redis_connection("default").flushall() - with self.assertNumQueries(6): - self.client.get(reverse('tag-list')) - - def test_anonymous(self): - """ - Tests that only the following db queries are done: - * 1 requests to see if anonyomus is enabled - * 1 requests to get the list of all projectors, - """ - get_redis_connection("default").flushall() - with self.assertNumQueries(2): - self.client.get(reverse('tag-list')) + assert count_queries(Tag.get_elements) == 1 -class TestConfigDBQueries(TestCase): +@pytest.mark.django_db(transaction=False) +def test_config_db_queries(): """ - Tests that receiving elements only need the required db queries. - - Therefore in setup some objects are created and received with different - user accounts. + Tests that only the following db queries are done: + * 1 requests to get the list of all config values """ + config.save_default_values() - def setUp(self): - self.client = APIClient() - config['general_system_enable_anonymous'] = True - config.save_default_values() - - def test_admin(self): - """ - Tests that only the following db queries are done: - * 5 requests to get the session an the request user with its permissions and - * 1 requests to get the list of all config values - """ - self.client.force_login(User.objects.get(pk=1)) - get_redis_connection("default").flushall() - with self.assertNumQueries(6): - self.client.get(reverse('config-list')) - - def test_anonymous(self): - """ - Tests that only the following db queries are done: - * 1 requests to see if anonymous is enabled and get all config values - """ - get_redis_connection("default").flushall() - with self.assertNumQueries(1): - self.client.get(reverse('config-list')) + assert count_queries(Tag.get_elements) == 1 class ChatMessageViewSet(TestCase): @@ -152,7 +64,7 @@ class ChatMessageViewSet(TestCase): Tests requests to deal with chat messages. """ def setUp(self): - admin = User.objects.get(pk=1) + admin = User.objects.get(username='admin') self.client.force_login(admin) ChatMessage.objects.create(message='test_message_peechiel8IeZoohaem9e', user=admin) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py new file mode 100644 index 000000000..b7449bae9 --- /dev/null +++ b/tests/integration/helpers.py @@ -0,0 +1,73 @@ +from typing import Any, Dict, List + +from asgiref.sync import sync_to_async +from django.db import DEFAULT_DB_ALIAS, connections +from django.test.utils import CaptureQueriesContext + +from openslides.core.config import config +from openslides.users.models import User +from openslides.utils.autoupdate import inform_data_collection_element_list +from openslides.utils.cache import element_cache, get_element_id +from openslides.utils.cache_providers import Cachable +from openslides.utils.collection import CollectionElement + + +class TConfig(Cachable): + """ + Cachable, that fills the cache with the default values of the config variables. + """ + + def get_collection_string(self) -> str: + return config.get_collection_string() + + def get_elements(self) -> List[Dict[str, Any]]: + elements = [] + config.key_to_id = {} + for id, item in enumerate(config.config_variables.values()): + elements.append({'id': id+1, 'key': item.name, 'value': item.default_value}) + config.key_to_id[item.name] = id+1 + return elements + + +class TUser(Cachable): + """ + Cachable, that fills the cache with the default values of the config variables. + """ + + def get_collection_string(self) -> str: + return User.get_collection_string() + + def get_elements(self) -> List[Dict[str, Any]]: + return [ + {'id': 1, 'username': 'admin', 'title': '', 'first_name': '', + 'last_name': 'Administrator', 'structure_level': '', 'number': '', 'about_me': '', + 'groups_id': [4], 'is_present': False, 'is_committee': False, 'email': '', + 'last_email_send': None, 'comment': '', 'is_active': True, 'default_password': 'admin', + 'session_auth_hash': '362d4f2de1463293cb3aaba7727c967c35de43ee'}] + + +async def set_config(key, value): + """ + Set a config variable in the element_cache without hitting the database. + """ + if not await element_cache.exists_full_data(): + # Encure that the cache exists and the default values of the config are in it. + await element_cache.build_full_data() + collection_string = config.get_collection_string() + config_id = config.key_to_id[key] # type: ignore + full_data = {'id': config_id, 'key': key, 'value': value} + await element_cache.change_elements({get_element_id(collection_string, config_id): full_data}) + await sync_to_async(inform_data_collection_element_list)([ + CollectionElement.from_values(collection_string, config_id, full_data=full_data)]) + + +def count_queries(func, *args, **kwargs) -> int: + context = CaptureQueriesContext(connections[DEFAULT_DB_ALIAS]) + with context: + func(*args, **kwargs) + + print("%d queries executed\nCaptured queries were:\n%s" % ( + len(context), + '\n'.join( + '%d. %s' % (i, query['sql']) for i, query in enumerate(context.captured_queries, start=1)))) + return len(context) diff --git a/tests/integration/mediafiles/test_viewset.py b/tests/integration/mediafiles/test_viewset.py index 922c49498..47b662c80 100644 --- a/tests/integration/mediafiles/test_viewset.py +++ b/tests/integration/mediafiles/test_viewset.py @@ -1,50 +1,22 @@ +import pytest from django.core.files.uploadedfile import SimpleUploadedFile -from django.urls import reverse -from django_redis import get_redis_connection -from rest_framework.test import APIClient -from openslides.core.config import config from openslides.mediafiles.models import Mediafile -from openslides.users.models import User -from openslides.utils.test import TestCase + +from ..helpers import count_queries -class TestDBQueries(TestCase): +@pytest.mark.django_db(transaction=False) +def test_mediafiles_db_queries(): """ - Tests that receiving elements only need the required db queries. - - Therefore in setup some objects are created and received with different - user accounts. + Tests that only the following db queries are done: + * 1 requests to get the list of all files. """ + for index in range(10): + Mediafile.objects.create( + title='some_file{}'.format(index), + mediafile=SimpleUploadedFile( + 'some_file{}'.format(index), + b'some content.')) - def setUp(self): - self.client = APIClient() - config['general_system_enable_anonymous'] = True - config.save_default_values() - for index in range(10): - Mediafile.objects.create( - title='some_file{}'.format(index), - mediafile=SimpleUploadedFile( - 'some_file{}'.format(index), - b'some content.')) - - def test_admin(self): - """ - Tests that only the following db queries are done: - * 7 requests to get the session an the request user with its permissions and - * 1 requests to get the list of all files. - """ - self.client.force_login(User.objects.get(pk=1)) - get_redis_connection('default').flushall() - with self.assertNumQueries(8): - self.client.get(reverse('mediafile-list')) - - def test_anonymous(self): - """ - Tests that only the following db queries are done: - * 3 requests to get the permission for anonymous and - * 1 requests to get the list of all projectors. - """ - get_redis_connection('default').flushall() - with self.assertNumQueries(4): - self.client.get(reverse('mediafile-list')) + assert count_queries(Mediafile.get_elements) == 1 diff --git a/tests/integration/motions/test_viewset.py b/tests/integration/motions/test_viewset.py index ea8a38cfc..9767cae0a 100644 --- a/tests/integration/motions/test_viewset.py +++ b/tests/integration/motions/test_viewset.py @@ -1,9 +1,9 @@ import json +import pytest from django.contrib.auth import get_user_model from django.contrib.auth.models import Permission from django.urls import reverse -from django_redis import get_redis_connection from rest_framework import status from rest_framework.test import APIClient @@ -22,134 +22,54 @@ from openslides.users.models import Group from openslides.utils.collection import CollectionElement from openslides.utils.test import TestCase +from ..helpers import count_queries -class TestMotionDBQueries(TestCase): + +@pytest.mark.django_db(transaction=False) +def test_motion_db_queries(): """ - Tests that receiving elements only need the required db queries. + Tests that only the following db queries are done: + * 1 requests to get the list of all motions, + * 1 request to get the motion versions, + * 1 request to get the agenda item, + * 1 request to get the motion log, + * 1 request to get the polls, + * 1 request to get the attachments, + * 1 request to get the tags, + * 2 requests to get the submitters and supporters. + """ + for index in range(10): + Motion.objects.create(title='motion{}'.format(index)) + get_user_model().objects.create_user( + username='user_{}'.format(index), + password='password') + # TODO: Create some polls etc. - Therefore in setup some objects are created and received with different - user accounts. + assert count_queries(Motion.get_elements) == 9 + + +@pytest.mark.django_db(transaction=False) +def test_category_db_queries(): + """ + Tests that only the following db queries are done: + * 1 requests to get the list of all categories. + """ + for index in range(10): + Category.objects.create(name='category{}'.format(index)) + + assert count_queries(Category.get_elements) == 1 + + +@pytest.mark.django_db(transaction=False) +def test_workflow_db_queries(): + """ + Tests that only the following db queries are done: + * 1 requests to get the list of all workflows, + * 1 request to get all states and + * 1 request to get the next states of all states. """ - def setUp(self): - self.client = APIClient() - config['general_system_enable_anonymous'] = True - config.save_default_values() - for index in range(10): - Motion.objects.create(title='motion{}'.format(index)) - get_user_model().objects.create_user( - username='user_{}'.format(index), - password='password') - # TODO: Create some polls etc. - - def test_admin(self): - """ - Tests that only the following db queries are done: - * 7 requests to get the session an the request user with its permissions, - * 1 requests to get the list of all motions, - * 1 request to get the motion versions, - * 1 request to get the agenda item, - * 1 request to get the motion log, - * 1 request to get the polls, - * 1 request to get the attachments, - * 1 request to get the tags, - * 2 requests to get the submitters and supporters. - """ - self.client.force_login(get_user_model().objects.get(pk=1)) - get_redis_connection('default').flushall() - with self.assertNumQueries(16): - self.client.get(reverse('motion-list')) - - def test_anonymous(self): - """ - Tests that only the following db queries are done: - * 3 requests to get the permission for anonymous, - * 1 requests to get the list of all motions, - * 1 request to get the motion versions, - * 1 request to get the agenda item, - * 1 request to get the motion log, - * 1 request to get the polls, - * 1 request to get the attachments, - * 1 request to get the tags, - * 2 requests to get the submitters and supporters. - """ - get_redis_connection('default').flushall() - with self.assertNumQueries(12): - self.client.get(reverse('motion-list')) - - -class TestCategoryDBQueries(TestCase): - """ - Tests that receiving elements only need the required db queries. - - Therefore in setup some objects are created and received with different - user accounts. - """ - - def setUp(self): - self.client = APIClient() - config.save_default_values() - config['general_system_enable_anonymous'] = True - for index in range(10): - Category.objects.create(name='category{}'.format(index)) - - def test_admin(self): - """ - Tests that only the following db queries are done: - * 7 requests to get the session an the request user with its permissions and - * 1 requests to get the list of all categories. - """ - self.client.force_login(get_user_model().objects.get(pk=1)) - get_redis_connection('default').flushall() - with self.assertNumQueries(8): - self.client.get(reverse('category-list')) - - def test_anonymous(self): - """ - Tests that only the following db queries are done: - * 3 requests to get the permission for anonymous (config and permissions) - * 1 requests to get the list of all motions and - """ - get_redis_connection('default').flushall() - with self.assertNumQueries(4): - self.client.get(reverse('category-list')) - - -class TestWorkflowDBQueries(TestCase): - """ - Tests that receiving elements only need the required db queries. - """ - - def setUp(self): - self.client = APIClient() - config.save_default_values() - config['general_system_enable_anonymous'] = True - # There do not need to be more workflows - - def test_admin(self): - """ - Tests that only the following db queries are done: - * 7 requests to get the session an the request user with its permissions, - * 1 requests to get the list of all workflows, - * 1 request to get all states and - * 1 request to get the next states of all states. - """ - self.client.force_login(get_user_model().objects.get(pk=1)) - get_redis_connection('default').flushall() - with self.assertNumQueries(10): - self.client.get(reverse('workflow-list')) - - def test_anonymous(self): - """ - Tests that only the following db queries are done: - * 3 requests to get the permission for anonymous, - * 1 requests to get the list of all workflows, - * 1 request to get all states and - * 1 request to get the next states of all states. - """ - get_redis_connection('default').flushall() - with self.assertNumQueries(6): - self.client.get(reverse('workflow-list')) + assert count_queries(Workflow.get_elements) == 3 class CreateMotion(TestCase): @@ -328,6 +248,10 @@ class CreateMotion(TestCase): content_type__app_label='motions', codename='can_manage_comments', )) + group_delegate.permissions.add(Permission.objects.get( + content_type__app_label='motions', + codename='can_see_comments', + )) response = self.client.post( reverse('motion-list'), @@ -383,7 +307,6 @@ class CreateMotion(TestCase): self.admin = get_user_model().objects.get(username='admin') self.admin.groups.add(2) self.admin.groups.remove(4) - get_redis_connection('default').flushall() response = self.client.post( reverse('motion-list'), @@ -424,24 +347,6 @@ class RetrieveMotion(TestCase): username='user_{}'.format(index), password='password') - def test_number_of_queries(self): - """ - Tests that only the following db queries are done: - * 7 requests to get the session and the request user with its permissions (3 of them are possibly a bug) - * 1 request to get the motion, - * 1 request to get the version, - * 1 request to get the agenda item, - * 1 request to get the log, - * 3 request to get the polls (1 of them is possibly a bug), - * 1 request to get the attachments, - * 1 request to get the tags, - * 2 requests to get the submitters and supporters. - TODO: Fix all bugs. - """ - get_redis_connection('default').flushall() - with self.assertNumQueries(18): - self.client.get(reverse('motion-detail', args=[self.motion.pk])) - def test_guest_state_with_required_permission_to_see(self): config['general_system_enable_anonymous'] = True guest_client = APIClient() @@ -450,7 +355,7 @@ class RetrieveMotion(TestCase): state.save() # The cache has to be cleared, see: # https://github.com/OpenSlides/OpenSlides/issues/3396 - get_redis_connection('default').flushall() + response = guest_client.get(reverse('motion-detail', args=[self.motion.pk])) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) @@ -484,7 +389,6 @@ class RetrieveMotion(TestCase): group.permissions.remove(permission) config['general_system_enable_anonymous'] = True guest_client = APIClient() - get_redis_connection('default').flushall() response_1 = guest_client.get(reverse('motion-detail', args=[self.motion.pk])) self.assertEqual(response_1.status_code, status.HTTP_200_OK) @@ -495,7 +399,7 @@ class RetrieveMotion(TestCase): extra_user = get_user_model().objects.create_user( username='username_wequePhieFoom0hai3wa', password='password_ooth7taechai5Oocieya') - get_redis_connection('default').flushall() + response_3 = guest_client.get(reverse('user-detail', args=[extra_user.pk])) self.assertEqual(response_3.status_code, status.HTTP_403_FORBIDDEN) @@ -576,7 +480,6 @@ class UpdateMotion(TestCase): self.motion.supporters.add(supporter) config['motions_remove_supporters'] = True self.assertEqual(self.motion.supporters.count(), 1) - get_redis_connection('default').flushall() response = self.client.patch( reverse('motion-detail', args=[self.motion.pk]), @@ -939,7 +842,7 @@ class SupportMotion(TestCase): def test_support(self): config['motions_min_supporters'] = 1 - get_redis_connection('default').flushall() + response = self.client.post(reverse('motion-support', args=[self.motion.pk])) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data, {'detail': 'You have supported this motion successfully.'}) diff --git a/tests/integration/topics/test_viewset.py b/tests/integration/topics/test_viewset.py index 5acab19c2..fc0bd1406 100644 --- a/tests/integration/topics/test_viewset.py +++ b/tests/integration/topics/test_viewset.py @@ -1,54 +1,26 @@ -from django.contrib.auth import get_user_model +import pytest from django.urls import reverse -from django_redis import get_redis_connection from rest_framework import status -from rest_framework.test import APIClient from openslides.agenda.models import Item -from openslides.core.config import config from openslides.topics.models import Topic from openslides.utils.test import TestCase +from ..helpers import count_queries -class TestDBQueries(TestCase): + +@pytest.mark.django_db(transaction=False) +def test_topic_item_db_queries(): """ - Tests that receiving elements only need the required db queries. - - Therefore in setup some topics are created and received with different - user accounts. + Tests that only the following db queries are done: + * 1 requests to get the list of all topics, + * 1 request to get attachments, + * 1 request to get the agenda item """ + for index in range(10): + Topic.objects.create(title='topic-{}'.format(index)) - def setUp(self): - self.client = APIClient() - config['general_system_enable_anonymous'] = True - config.save_default_values() - for index in range(10): - Topic.objects.create(title='topic-{}'.format(index)) - - def test_admin(self): - """ - Tests that only the following db queries are done: - * 7 requests to get the session an the request user with its permissions, - * 1 requests to get the list of all topics, - * 1 request to get attachments, - * 1 request to get the agenda item - """ - self.client.force_login(get_user_model().objects.get(pk=1)) - get_redis_connection('default').flushall() - with self.assertNumQueries(10): - self.client.get(reverse('topic-list')) - - def test_anonymous(self): - """ - Tests that only the following db queries are done: - * 3 requests to get the permission for anonymous, - * 1 requests to get the list of all topics, - * 1 request to get attachments, - * 1 request to get the agenda item, - """ - get_redis_connection('default').flushall() - with self.assertNumQueries(6): - self.client.get(reverse('topic-list')) + assert count_queries(Topic.get_elements) == 3 class TopicCreate(TestCase): diff --git a/tests/integration/users/test_viewset.py b/tests/integration/users/test_viewset.py index 3d151786e..f02c57846 100644 --- a/tests/integration/users/test_viewset.py +++ b/tests/integration/users/test_viewset.py @@ -1,6 +1,6 @@ +import pytest from django.core import mail from django.urls import reverse -from django_redis import get_redis_connection from rest_framework import status from rest_framework.test import APIClient @@ -9,84 +9,33 @@ from openslides.users.models import Group, PersonalNote, User from openslides.users.serializers import UserFullSerializer from openslides.utils.test import TestCase +from ..helpers import count_queries -class TestUserDBQueries(TestCase): + +@pytest.mark.django_db(transaction=False) +def test_user_db_queries(): """ - Tests that receiving elements only need the required db queries. - - Therefore in setup some objects are created and received with different - user accounts. + Tests that only the following db queries are done: + * 2 requests to get the list of all users and + * 1 requests to get the list of all groups. """ + for index in range(10): + User.objects.create(username='user{}'.format(index)) - def setUp(self): - self.client = APIClient() - config['general_system_enable_anonymous'] = True - config.save_default_values() - for index in range(10): - User.objects.create(username='user{}'.format(index)) - - def test_admin(self): - """ - Tests that only the following db queries are done: - * 2 requests to get the session and the request user with its permissions, - * 2 requests to get the list of all users and - * 1 requests to get the list of all groups. - """ - self.client.force_login(User.objects.get(pk=1)) - get_redis_connection('default').flushall() - with self.assertNumQueries(7): - self.client.get(reverse('user-list')) - - def test_anonymous(self): - """ - Tests that only the following db queries are done: - * 3 requests to get the permission for anonymous, - * 1 requests to get the list of all users and - * 2 request to get all groups (needed by the user serializer). - """ - get_redis_connection('default').flushall() - with self.assertNumQueries(6): - self.client.get(reverse('user-list')) + assert count_queries(User.get_elements) == 3 -class TestGroupDBQueries(TestCase): +@pytest.mark.django_db(transaction=False) +def test_group_db_queries(): """ - Tests that receiving elements only need the required db queries. - - Therefore in setup some objects are created and received with different - user accounts. + Tests that only the following db queries are done: + * 1 request to get the list of all groups. + * 1 request to get the permissions """ + for index in range(10): + Group.objects.create(name='group{}'.format(index)) - def setUp(self): - self.client = APIClient() - config['general_system_enable_anonymous'] = True - config.save_default_values() - for index in range(10): - Group.objects.create(name='group{}'.format(index)) - - def test_admin(self): - """ - Tests that only the following db queries are done: - * 6 requests to get the session an the request user with its permissions and - * 1 request to get the list of all groups. - - The data of the groups where loaded when the admin was authenticated. So - only the list of all groups has be fetched from the db. - """ - self.client.force_login(User.objects.get(pk=1)) - get_redis_connection('default').flushall() - with self.assertNumQueries(7): - self.client.get(reverse('group-list')) - - def test_anonymous(self): - """ - Tests that only the following db queries are done: - * 1 requests to find out if anonymous is enabled - * 2 request to get the list of all groups and - """ - get_redis_connection('default').flushall() - with self.assertNumQueries(3): - self.client.get(reverse('group-list')) + assert count_queries(Group.get_elements) == 2 class UserGetTest(TestCase): @@ -98,7 +47,7 @@ class UserGetTest(TestCase): It is invalid, that a user is in the group with the pk 1. But if the database is invalid, the user should nevertheless be received. """ - admin = User.objects.get(pk=1) + admin = User.objects.get(username='admin') group1 = Group.objects.get(pk=1) admin.groups.add(group1) self.client.login(username='admin', password='admin') @@ -178,7 +127,7 @@ class UserUpdate(TestCase): admin_client = APIClient() admin_client.login(username='admin', password='admin') # This is the builtin user 'Administrator' with username 'admin'. The pk is valid. - user_pk = 1 + user_pk = User.objects.get(username='admin').pk response = admin_client.patch( reverse('user-detail', args=[user_pk]), @@ -198,14 +147,14 @@ class UserUpdate(TestCase): admin_client = APIClient() admin_client.login(username='admin', password='admin') # This is the builtin user 'Administrator'. The pk is valid. - user_pk = 1 + user_pk = User.objects.get(username='admin').pk response = admin_client.put( reverse('user-detail', args=[user_pk]), {'last_name': 'New name Ohy4eeyei5'}) self.assertEqual(response.status_code, 200) - self.assertEqual(User.objects.get(pk=1).username, 'New name Ohy4eeyei5') + self.assertEqual(User.objects.get(pk=user_pk).username, 'New name Ohy4eeyei5') def test_update_deactivate_yourselfself(self): """ @@ -214,7 +163,7 @@ class UserUpdate(TestCase): admin_client = APIClient() admin_client.login(username='admin', password='admin') # This is the builtin user 'Administrator'. The pk is valid. - user_pk = 1 + user_pk = User.objects.get(username='admin').pk response = admin_client.patch( reverse('user-detail', args=[user_pk]), @@ -581,7 +530,7 @@ class PersonalNoteTest(TestCase): Tests for PersonalNote model. """ def test_anonymous_without_personal_notes(self): - admin = User.objects.get(pk=1) + admin = User.objects.get(username='admin') personal_note = PersonalNote.objects.create(user=admin, notes='["admin_personal_note_OoGh8choro0oosh0roob"]') config['general_system_enable_anonymous'] = True guest_client = APIClient() diff --git a/tests/integration/utils/test_collection.py b/tests/integration/utils/test_collection.py index 202c9a3c1..f1ef43a0e 100644 --- a/tests/integration/utils/test_collection.py +++ b/tests/integration/utils/test_collection.py @@ -1,17 +1,17 @@ -from channels.tests import ChannelTestCase as TestCase -from django_redis import get_redis_connection +from unittest import skip from openslides.topics.models import Topic from openslides.utils import collection +from openslides.utils.test import TestCase class TestCollectionElementCache(TestCase): + @skip("Does not work as long as caching does not work in the tests") def test_clean_cache(self): """ Tests that the data is retrieved from the database. """ topic = Topic.objects.create(title='test topic') - get_redis_connection("default").flushall() with self.assertNumQueries(3): collection_element = collection.CollectionElement.from_values('topics/topic', 1) @@ -19,6 +19,7 @@ class TestCollectionElementCache(TestCase): self.assertEqual(topic.title, instance['title']) + @skip("Does not work as long as caching does not work in the tests") def test_with_cache(self): """ Tests that no db query is used when the valie is in the cache. @@ -43,6 +44,7 @@ class TestCollectionElementCache(TestCase): collection.CollectionElement.from_values('topics/topic', 999) +@skip("Does not work as long as caching does not work in the tests") class TestCollectionCache(TestCase): def test_clean_cache(self): """ @@ -52,7 +54,6 @@ class TestCollectionCache(TestCase): Topic.objects.create(title='test topic2') Topic.objects.create(title='test topic3') topic_collection = collection.Collection('topics/topic') - get_redis_connection("default").flushall() with self.assertNumQueries(3): instance_list = list(topic_collection.get_full_data()) @@ -62,7 +63,6 @@ class TestCollectionCache(TestCase): """ Tests that no db query is used when the list is received twice. """ - get_redis_connection("default").flushall() Topic.objects.create(title='test topic1') Topic.objects.create(title='test topic2') Topic.objects.create(title='test topic3') diff --git a/tests/integration/utils/test_consumers.py b/tests/integration/utils/test_consumers.py new file mode 100644 index 000000000..c510f79ab --- /dev/null +++ b/tests/integration/utils/test_consumers.py @@ -0,0 +1,185 @@ +from importlib import import_module + +import pytest +from asgiref.sync import sync_to_async +from channels.testing import WebsocketCommunicator +from django.conf import settings +from django.contrib.auth import ( + BACKEND_SESSION_KEY, + HASH_SESSION_KEY, + SESSION_KEY, +) + +from openslides.asgi import application +from openslides.core.config import config +from openslides.utils.autoupdate import inform_deleted_data +from openslides.utils.cache import element_cache + +from ...unit.utils.cache_provider import ( + Collection1, + Collection2, + get_cachable_provider, +) +from ..helpers import TConfig, TUser, set_config + + +@pytest.fixture(autouse=True) +def prepare_element_cache(settings): + """ + Resets the element cache. + + Uses a cacheable_provider for tests with example data. + """ + settings.SKIP_CACHE = False + element_cache.cache_provider.clear_cache() + orig_cachable_provider = element_cache.cachable_provider + element_cache.cachable_provider = get_cachable_provider([Collection1(), Collection2(), TConfig(), TUser()]) + element_cache._cachables = None + yield + # Reset the cachable_provider + element_cache.cachable_provider = orig_cachable_provider + element_cache._cachables = None + element_cache.cache_provider.clear_cache() + + +@pytest.fixture +def communicator(request, event_loop): + communicator = WebsocketCommunicator(application, "/ws/site/") + + # This style is needed for python 3.5. Use the generaor style when 3.5 ist dropped + def fin(): + async def afin(): + await communicator.disconnect() + event_loop.run_until_complete(afin()) + + request.addfinalizer(fin) + return communicator + + +@pytest.mark.asyncio +async def test_normal_connection(communicator): + await set_config('general_system_enable_anonymous', True) + await communicator.connect() + + response = await communicator.receive_json_from() + + # Test, that there is a lot of startup data. + assert len(response) > 5 + + +@pytest.mark.asyncio +async def test_receive_changed_data(communicator): + await set_config('general_system_enable_anonymous', True) + await communicator.connect() + await communicator.receive_json_from() + + # Change a config value after the startup data has been received + await set_config('general_event_name', 'Test Event') + response = await communicator.receive_json_from() + + id = config.get_key_to_id()['general_event_name'] + assert response == [ + {'action': 'changed', + 'collection': 'core/config', + 'data': {'id': id, 'key': 'general_event_name', 'value': 'Test Event'}, + 'id': id}] + + +@pytest.mark.asyncio +async def test_anonymous_disabled(communicator): + connected, __ = await communicator.connect() + + assert not connected + + +@pytest.mark.asyncio +async def test_with_user(): + # login user with id 1 + engine = import_module(settings.SESSION_ENGINE) + session = engine.SessionStore() # type: ignore + session[SESSION_KEY] = '1' + session[HASH_SESSION_KEY] = '362d4f2de1463293cb3aaba7727c967c35de43ee' # see helpers.TUser + session[BACKEND_SESSION_KEY] = 'django.contrib.auth.backends.ModelBackend' + session.save() + scn = settings.SESSION_COOKIE_NAME + cookies = (b'cookie', '{}={}'.format(scn, session.session_key).encode()) + communicator = WebsocketCommunicator(application, "/ws/site/", headers=[cookies]) + + connected, __ = await communicator.connect() + + assert connected + + await communicator.disconnect() + + +@pytest.mark.asyncio +async def test_receive_deleted_data(communicator): + await set_config('general_system_enable_anonymous', True) + await communicator.connect() + await communicator.receive_json_from() + + # Delete test element + await sync_to_async(inform_deleted_data)([(Collection1().get_collection_string(), 1)]) + response = await communicator.receive_json_from() + + assert response == [{'action': 'deleted', 'collection': Collection1().get_collection_string(), 'id': 1}] + + +@pytest.mark.asyncio +async def test_send_invalid_notify_not_a_list(communicator): + await set_config('general_system_enable_anonymous', True) + await communicator.connect() + # Await the startup data + await communicator.receive_json_from() + + await communicator.send_json_to({'testmessage': 'foobar, what else.'}) + + response = await communicator.receive_json_from() + + assert response == {'error': 'invalid message'} + + +@pytest.mark.asyncio +async def test_send_invalid_notify_no_elements(communicator): + await set_config('general_system_enable_anonymous', True) + await communicator.connect() + # Await the startup data + await communicator.receive_json_from() + + await communicator.send_json_to([]) + + response = await communicator.receive_json_from() + + assert response == {'error': 'invalid message'} + + +@pytest.mark.asyncio +async def test_send_invalid_notify_str_in_list(communicator): + await set_config('general_system_enable_anonymous', True) + await communicator.connect() + # Await the startup data + await communicator.receive_json_from() + + await communicator.send_json_to([{}, 'testmessage']) + + response = await communicator.receive_json_from() + + assert response == {'error': 'invalid message'} + + +@pytest.mark.asyncio +async def test_send_valid_notify(communicator): + await set_config('general_system_enable_anonymous', True) + await communicator.connect() + # Await the startup data + await communicator.receive_json_from() + + await communicator.send_json_to([{'testmessage': 'foobar, what else.'}]) + + response = await communicator.receive_json_from() + + assert isinstance(response, list) + assert len(response) == 1 + assert response[0]['testmessage'] == 'foobar, what else.' + assert 'senderReplyChannelName' in response[0] + assert response[0]['senderUserId'] == 0 diff --git a/tests/old/config/test_config.py b/tests/old/config/test_config.py index c7d7e0301..9b9bc06b5 100644 --- a/tests/old/config/test_config.py +++ b/tests/old/config/test_config.py @@ -1,11 +1,11 @@ -from django_redis import get_redis_connection + from openslides.core.config import ConfigVariable, config from openslides.core.exceptions import ConfigError, ConfigNotFound from openslides.utils.test import TestCase -class TestConfigException(Exception): +class TTestConfigException(Exception): pass @@ -57,7 +57,6 @@ class HandleConfigTest(TestCase): def test_change_config_value(self): self.assertEqual(config['string_var'], 'default_string_rien4ooCZieng6ah') config['string_var'] = 'other_special_unique_string dauTex9eAiy7jeen' - get_redis_connection('default').flushall() self.assertEqual(config['string_var'], 'other_special_unique_string dauTex9eAiy7jeen') def test_missing_cache_(self): @@ -79,7 +78,7 @@ class HandleConfigTest(TestCase): message. """ with self.assertRaisesMessage( - TestConfigException, + TTestConfigException, 'Change callback dhcnfg34dlg06kdg successfully called.'): self.set_config_var( key='var_with_callback_ghvnfjd5768gdfkwg0hm2', @@ -155,7 +154,7 @@ def set_simple_config_collection_disabled_view(): def set_simple_config_collection_with_callback(): def callback(): - raise TestConfigException('Change callback dhcnfg34dlg06kdg successfully called.') + raise TTestConfigException('Change callback dhcnfg34dlg06kdg successfully called.') yield ConfigVariable( name='var_with_callback_ghvnfjd5768gdfkwg0hm2', default_value='', diff --git a/tests/old/motions/test_models.py b/tests/old/motions/test_models.py index 0869c6cdd..18ab6f696 100644 --- a/tests/old/motions/test_models.py +++ b/tests/old/motions/test_models.py @@ -1,5 +1,3 @@ -from django_redis import get_redis_connection - from openslides.core.config import config from openslides.motions.exceptions import WorkflowError from openslides.motions.models import Motion, State, Workflow @@ -132,7 +130,6 @@ class ModelTest(TestCase): def test_is_amendment(self): config['motions_amendments_enabled'] = True - get_redis_connection('default').flushall() amendment = Motion.objects.create(title='amendment', parent=self.motion) self.assertTrue(amendment.is_amendment()) @@ -153,7 +150,6 @@ class ModelTest(TestCase): If the config is set to manually, the method does nothing. """ config['motions_identifier'] = 'manually' - get_redis_connection("default").flushall() motion = Motion() motion.set_identifier() @@ -169,7 +165,6 @@ class ModelTest(TestCase): config['motions_amendments_enabled'] = True self.motion.identifier = 'Parent identifier' self.motion.save() - get_redis_connection("default").flushall() motion = Motion(parent=self.motion) motion.set_identifier() @@ -184,7 +179,6 @@ class ModelTest(TestCase): config['motions_amendments_enabled'] = True self.motion.identifier = 'Parent identifier' self.motion.save() - get_redis_connection("default").flushall() Motion.objects.create(title='Amendment1', parent=self.motion) motion = Motion(parent=self.motion) diff --git a/tests/settings.py b/tests/settings.py index 92bfecd69..5d9b1fe05 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -6,6 +6,7 @@ import os from openslides.global_settings import * # noqa + # Path to the directory for user specific data files OPENSLIDES_USER_DATA_PATH = os.path.realpath(os.path.dirname(os.path.abspath(__file__))) @@ -42,6 +43,16 @@ DATABASES = { } } +# Configure session in the cache + +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + } +} + +SESSION_ENGINE = "django.contrib.sessions.backends.cache" + # When use_redis is True, the restricted data cache caches the data individuel # for each user. This requires a lot of memory if there are a lot of active # users. If use_redis is False, this setting has no effect. @@ -72,20 +83,12 @@ MOTION_IDENTIFIER_MIN_DIGITS = 1 # Special settings only for testing -TEST_RUNNER = 'openslides.utils.test.OpenSlidesDiscoverRunner' - # Use a faster password hasher. PASSWORD_HASHERS = [ 'django.contrib.auth.hashers.MD5PasswordHasher', ] -CACHES = { - "default": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://127.0.0.1:6379/0", - "OPTIONS": { - "REDIS_CLIENT_CLASS": "fakeredis.FakeStrictRedis", - } - } -} +# At least in Django 2.1 and Channels 2.1 the django transactions can not be shared between +# threads. So we have to skip the asyncio-cache. +SKIP_CACHE = True diff --git a/tests/unit/utils/cache_provider.py b/tests/unit/utils/cache_provider.py new file mode 100644 index 000000000..3db52363b --- /dev/null +++ b/tests/unit/utils/cache_provider.py @@ -0,0 +1,82 @@ +import asyncio # noqa +from typing import Any, Callable, Dict, List, Optional + +from openslides.utils.cache_providers import Cachable, MemmoryCacheProvider +from openslides.utils.collection import CollectionElement # noqa + + +def restrict_elements( + user: Optional['CollectionElement'], + elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Adds the prefix 'restricted_' to all values except id. + """ + out = [] + for element in elements: + restricted_element = {} + for key, value in element.items(): + if key == 'id': + restricted_element[key] = value + else: + restricted_element[key] = 'restricted_{}'.format(value) + out.append(restricted_element) + return out + + +class Collection1(Cachable): + def get_collection_string(self) -> str: + return 'app/collection1' + + def get_elements(self) -> List[Dict[str, Any]]: + return [ + {'id': 1, 'value': 'value1'}, + {'id': 2, 'value': 'value2'}] + + def restrict_elements(self, user: Optional['CollectionElement'], elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + return restrict_elements(user, elements) + + +class Collection2(Cachable): + def get_collection_string(self) -> str: + return 'app/collection2' + + def get_elements(self) -> List[Dict[str, Any]]: + return [ + {'id': 1, 'key': 'value1'}, + {'id': 2, 'key': 'value2'}] + + def restrict_elements(self, user: Optional['CollectionElement'], elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + return restrict_elements(user, elements) + + +def get_cachable_provider(cachables: List[Cachable] = [Collection1(), Collection2()]) -> Callable[[], List[Cachable]]: + """ + Returns a cachable_provider. + """ + return lambda: cachables + + +def example_data(): + return { + 'app/collection1': [ + {'id': 1, 'value': 'value1'}, + {'id': 2, 'value': 'value2'}], + 'app/collection2': [ + {'id': 1, 'key': 'value1'}, + {'id': 2, 'key': 'value2'}]} + + +class TTestCacheProvider(MemmoryCacheProvider): + """ + CacheProvider simular to the MemmoryCacheProvider with special methods for + testing. + """ + + async def del_lock_restricted_data_after_wait(self, user_id: int, future: asyncio.Future = None) -> None: + if future is None: + asyncio.ensure_future(self.del_lock_restricted_data(user_id)) + else: + async def set_future() -> None: + await self.del_lock_restricted_data(user_id) + future.set_result(1) # type: ignore + asyncio.ensure_future(set_future()) diff --git a/tests/unit/utils/test_cache.py b/tests/unit/utils/test_cache.py new file mode 100644 index 000000000..aa1a902f1 --- /dev/null +++ b/tests/unit/utils/test_cache.py @@ -0,0 +1,483 @@ +import asyncio +import json +from typing import Any, Dict, List + +import pytest + +from openslides.utils.cache import ElementCache + +from .cache_provider import ( + TTestCacheProvider, + example_data, + get_cachable_provider, +) + + +def decode_dict(encoded_dict: Dict[str, str]) -> Dict[str, Any]: + """ + Helper function that loads the json values of a dict. + """ + return {key: json.loads(value) for key, value in encoded_dict.items()} + + +def sort_dict(encoded_dict: Dict[str, List[Dict[str, Any]]]) -> Dict[str, List[Dict[str, Any]]]: + """ + Helper function that sorts the value of a dict. + """ + return {key: sorted(value, key=lambda x: x['id']) for key, value in encoded_dict.items()} + + +@pytest.fixture +def element_cache(): + return ElementCache( + 'test_redis', + cache_provider_class=TTestCacheProvider, + cachable_provider=get_cachable_provider(), + start_time=0) + + +@pytest.mark.asyncio +async def test_save_full_data(element_cache): + input_data = { + 'app/collection1': [ + {'id': 1, 'value': 'value1'}, + {'id': 2, 'value': 'value2'}], + 'app/collection2': [ + {'id': 1, 'key': 'value1'}, + {'id': 2, 'key': 'value2'}]} + calculated_data = { + 'app/collection1:1': '{"id": 1, "value": "value1"}', + 'app/collection1:2': '{"id": 2, "value": "value2"}', + 'app/collection2:1': '{"id": 1, "key": "value1"}', + 'app/collection2:2': '{"id": 2, "key": "value2"}'} + + await element_cache.save_full_data(input_data) + + assert decode_dict(element_cache.cache_provider.full_data) == decode_dict(calculated_data) + + +@pytest.mark.asyncio +async def test_build_full_data(element_cache): + result = await element_cache.build_full_data() + + assert result == example_data() + assert decode_dict(element_cache.cache_provider.full_data) == decode_dict({ + 'app/collection1:1': '{"id": 1, "value": "value1"}', + 'app/collection1:2': '{"id": 2, "value": "value2"}', + 'app/collection2:1': '{"id": 1, "key": "value1"}', + 'app/collection2:2': '{"id": 2, "key": "value2"}'}) + + +@pytest.mark.asyncio +async def test_exists_full_data(element_cache): + """ + Test that the return value of exists_full_data is the the same as from the + cache_provider. + """ + element_cache.cache_provider.full_data = 'test_value' + assert await element_cache.exists_full_data() + + +@pytest.mark.asyncio +async def test_change_elements(element_cache): + input_data = { + 'app/collection1:1': {"id": 1, "value": "updated"}, + 'app/collection1:2': {"id": 2, "value": "new"}, + 'app/collection2:1': {"id": 1, "key": "updated"}, + 'app/collection2:2': None} + + element_cache.cache_provider.full_data = { + 'app/collection1:1': '{"id": 1, "value": "old"}', + 'app/collection2:1': '{"id": 1, "key": "old"}', + 'app/collection2:2': '{"id": 2, "key": "old"}'} + + result = await element_cache.change_elements(input_data) + + assert result == 1 # first change_id + assert decode_dict(element_cache.cache_provider.full_data) == decode_dict({ + 'app/collection1:1': '{"id": 1, "value": "updated"}', + 'app/collection1:2': '{"id": 2, "value": "new"}', + 'app/collection2:1': '{"id": 1, "key": "updated"}'}) + assert element_cache.cache_provider.change_id_data == { + 1: { + 'app/collection1:1', + 'app/collection1:2', + 'app/collection2:1', + 'app/collection2:2'}} + + +@pytest.mark.asyncio +async def test_change_elements_with_no_data_in_redis(element_cache): + input_data = { + 'app/collection1:1': {"id": 1, "value": "updated"}, + 'app/collection1:2': {"id": 2, "value": "new"}, + 'app/collection2:1': {"id": 1, "key": "updated"}, + 'app/collection2:2': None} + + result = await element_cache.change_elements(input_data) + + assert result == 1 # first change_id + assert decode_dict(element_cache.cache_provider.full_data) == decode_dict({ + 'app/collection1:1': '{"id": 1, "value": "updated"}', + 'app/collection1:2': '{"id": 2, "value": "new"}', + 'app/collection2:1': '{"id": 1, "key": "updated"}'}) + assert element_cache.cache_provider.change_id_data == { + 1: { + 'app/collection1:1', + 'app/collection1:2', + 'app/collection2:1', + 'app/collection2:2'}} + + +@pytest.mark.asyncio +async def test_get_all_full_data_from_db(element_cache): + result = await element_cache.get_all_full_data() + + assert result == example_data() + # Test that elements are written to redis + assert decode_dict(element_cache.cache_provider.full_data) == decode_dict({ + 'app/collection1:1': '{"id": 1, "value": "value1"}', + 'app/collection1:2': '{"id": 2, "value": "value2"}', + 'app/collection2:1': '{"id": 1, "key": "value1"}', + 'app/collection2:2': '{"id": 2, "key": "value2"}'}) + + +@pytest.mark.asyncio +async def test_get_all_full_data_from_redis(element_cache): + element_cache.cache_provider.full_data = { + 'app/collection1:1': '{"id": 1, "value": "value1"}', + 'app/collection1:2': '{"id": 2, "value": "value2"}', + 'app/collection2:1': '{"id": 1, "key": "value1"}', + 'app/collection2:2': '{"id": 2, "key": "value2"}'} + + result = await element_cache.get_all_full_data() + + # The output from redis has to be the same then the db_data + assert sort_dict(result) == sort_dict(example_data()) + + +@pytest.mark.asyncio +async def test_get_full_data_change_id_0(element_cache): + element_cache.cache_provider.full_data = { + 'app/collection1:1': '{"id": 1, "value": "value1"}', + 'app/collection1:2': '{"id": 2, "value": "value2"}', + 'app/collection2:1': '{"id": 1, "key": "value1"}', + 'app/collection2:2': '{"id": 2, "key": "value2"}'} + + result = await element_cache.get_full_data(0) + + assert sort_dict(result[0]) == sort_dict(example_data()) + + +@pytest.mark.asyncio +async def test_get_full_data_change_id_lower_then_in_redis(element_cache): + element_cache.cache_provider.full_data = { + 'app/collection1:1': '{"id": 1, "value": "value1"}', + 'app/collection1:2': '{"id": 2, "value": "value2"}', + 'app/collection2:1': '{"id": 1, "key": "value1"}', + 'app/collection2:2': '{"id": 2, "key": "value2"}'} + element_cache.cache_provider.change_id_data = { + 2: {'app/collection1:1'}} + with pytest.raises(RuntimeError): + await element_cache.get_full_data(1) + + +@pytest.mark.asyncio +async def test_get_full_data_change_id_data_in_redis(element_cache): + element_cache.cache_provider.full_data = { + 'app/collection1:1': '{"id": 1, "value": "value1"}', + 'app/collection1:2': '{"id": 2, "value": "value2"}', + 'app/collection2:1': '{"id": 1, "key": "value1"}', + 'app/collection2:2': '{"id": 2, "key": "value2"}'} + element_cache.cache_provider.change_id_data = { + 1: {'app/collection1:1', 'app/collection1:3'}} + + result = await element_cache.get_full_data(1) + + assert result == ( + {'app/collection1': [{"id": 1, "value": "value1"}]}, + ['app/collection1:3']) + + +@pytest.mark.asyncio +async def test_get_full_data_change_id_data_in_db(element_cache): + element_cache.cache_provider.change_id_data = { + 1: {'app/collection1:1', 'app/collection1:3'}} + + result = await element_cache.get_full_data(1) + + assert result == ( + {'app/collection1': [{"id": 1, "value": "value1"}]}, + ['app/collection1:3']) + + +@pytest.mark.asyncio +async def test_get_full_data_change_id_data_in_db_empty_change_id(element_cache): + with pytest.raises(RuntimeError): + await element_cache.get_full_data(1) + + +@pytest.mark.asyncio +async def test_get_element_full_data_empty_redis(element_cache): + result = await element_cache.get_element_full_data('app/collection1', 1) + + assert result == {'id': 1, 'value': 'value1'} + + +@pytest.mark.asyncio +async def test_get_element_full_data_empty_redis_does_not_exist(element_cache): + result = await element_cache.get_element_full_data('app/collection1', 3) + + assert result is None + + +@pytest.mark.asyncio +async def test_get_element_full_data_full_redis(element_cache): + element_cache.cache_provider.full_data = { + 'app/collection1:1': '{"id": 1, "value": "value1"}', + 'app/collection1:2': '{"id": 2, "value": "value2"}', + 'app/collection2:1': '{"id": 1, "key": "value1"}', + 'app/collection2:2': '{"id": 2, "key": "value2"}'} + + result = await element_cache.get_element_full_data('app/collection1', 1) + + assert result == {'id': 1, 'value': 'value1'} + + +@pytest.mark.asyncio +async def test_exist_restricted_data(element_cache): + element_cache.use_restricted_data_cache = True + element_cache.cache_provider.restricted_data = {0: { + 'app/collection1:1': '{"id": 1, "value": "value1"}', + 'app/collection1:2': '{"id": 2, "value": "value2"}', + 'app/collection2:1': '{"id": 1, "key": "value1"}', + 'app/collection2:2': '{"id": 2, "key": "value2"}'}} + + result = await element_cache.exists_restricted_data(None) + + assert result + + +@pytest.mark.asyncio +async def test_exist_restricted_data_do_not_use_restricted_data(element_cache): + element_cache.use_restricted_data_cache = False + element_cache.cache_provider.restricted_data = {0: { + 'app/collection1:1': '{"id": 1, "value": "value1"}', + 'app/collection1:2': '{"id": 2, "value": "value2"}', + 'app/collection2:1': '{"id": 1, "key": "value1"}', + 'app/collection2:2': '{"id": 2, "key": "value2"}'}} + + result = await element_cache.exists_restricted_data(None) + + assert not result + + +@pytest.mark.asyncio +async def test_del_user(element_cache): + element_cache.use_restricted_data_cache = True + element_cache.cache_provider.restricted_data = {0: { + 'app/collection1:1': '{"id": 1, "value": "value1"}', + 'app/collection1:2': '{"id": 2, "value": "value2"}', + 'app/collection2:1': '{"id": 1, "key": "value1"}', + 'app/collection2:2': '{"id": 2, "key": "value2"}'}} + + await element_cache.del_user(None) + + assert not element_cache.cache_provider.restricted_data + + +@pytest.mark.asyncio +async def test_del_user_for_empty_user(element_cache): + element_cache.use_restricted_data_cache = True + + await element_cache.del_user(None) + + assert not element_cache.cache_provider.restricted_data + + +@pytest.mark.asyncio +async def test_update_restricted_data(element_cache): + element_cache.use_restricted_data_cache = True + + await element_cache.update_restricted_data(None) + + assert decode_dict(element_cache.cache_provider.restricted_data[0]) == decode_dict({ + 'app/collection1:1': '{"id": 1, "value": "restricted_value1"}', + 'app/collection1:2': '{"id": 2, "value": "restricted_value2"}', + 'app/collection2:1': '{"id": 1, "key": "restricted_value1"}', + 'app/collection2:2': '{"id": 2, "key": "restricted_value2"}', + '_config:change_id': '0'}) + # Make sure the lock is deleted + assert not await element_cache.cache_provider.get_lock_restricted_data(0) + # And the future is done + assert element_cache.restricted_data_cache_updater[0].done() + + +@pytest.mark.asyncio +async def test_update_restricted_data_disabled_restricted_data(element_cache): + element_cache.use_restricted_data_cache = False + + await element_cache.update_restricted_data(None) + + assert not element_cache.cache_provider.restricted_data + + +@pytest.mark.asyncio +async def test_update_restricted_data_to_low_change_id(element_cache): + element_cache.use_restricted_data_cache = True + element_cache.cache_provider.restricted_data[0] = { + '_config:change_id': '1'} + element_cache.cache_provider.change_id_data = { + 3: {'app/collection1:1'}} + + await element_cache.update_restricted_data(None) + + assert decode_dict(element_cache.cache_provider.restricted_data[0]) == decode_dict({ + 'app/collection1:1': '{"id": 1, "value": "restricted_value1"}', + 'app/collection1:2': '{"id": 2, "value": "restricted_value2"}', + 'app/collection2:1': '{"id": 1, "key": "restricted_value1"}', + 'app/collection2:2': '{"id": 2, "key": "restricted_value2"}', + '_config:change_id': '3'}) + + +@pytest.mark.asyncio +async def test_update_restricted_data_with_same_id(element_cache): + element_cache.use_restricted_data_cache = True + element_cache.cache_provider.restricted_data[0] = { + '_config:change_id': '1'} + element_cache.cache_provider.change_id_data = { + 1: {'app/collection1:1'}} + + await element_cache.update_restricted_data(None) + + # Same id means, there is nothing to do + assert element_cache.cache_provider.restricted_data[0] == { + '_config:change_id': '1'} + + +@pytest.mark.asyncio +async def test_update_restricted_data_with_deleted_elements(element_cache): + element_cache.use_restricted_data_cache = True + element_cache.cache_provider.restricted_data[0] = { + 'app/collection1:3': '{"id": 1, "value": "restricted_value1"}', + '_config:change_id': '1'} + element_cache.cache_provider.change_id_data = { + 2: {'app/collection1:3'}} + + await element_cache.update_restricted_data(None) + + assert element_cache.cache_provider.restricted_data[0] == { + '_config:change_id': '2'} + + +@pytest.mark.asyncio +async def test_update_restricted_data_second_worker_on_different_server(element_cache): + """ + Test, that if another worker is updating the data, noting is done. + + This tests makes use of the redis key as it would on different daphne servers. + """ + element_cache.use_restricted_data_cache = True + element_cache.cache_provider.restricted_data = {0: {}} + await element_cache.cache_provider.set_lock_restricted_data(0) + await element_cache.cache_provider.del_lock_restricted_data_after_wait(0) + + await element_cache.update_restricted_data(None) + + # Restricted_data_should not be set on second worker + assert element_cache.cache_provider.restricted_data == {0: {}} + + +@pytest.mark.asyncio +async def test_update_restricted_data_second_worker_on_same_server(element_cache): + """ + Test, that if another worker is updating the data, noting is done. + + This tests makes use of the future as it would on the same daphne server. + """ + element_cache.use_restricted_data_cache = True + element_cache.cache_provider.restricted_data = {0: {}} + future = asyncio.Future() # type: asyncio.Future + element_cache.restricted_data_cache_updater[0] = future + await element_cache.cache_provider.set_lock_restricted_data(0) + await element_cache.cache_provider.del_lock_restricted_data_after_wait(0, future) + + await element_cache.update_restricted_data(None) + + # Restricted_data_should not be set on second worker + assert element_cache.cache_provider.restricted_data == {0: {}} + + +@pytest.mark.asyncio +async def test_get_all_restricted_data(element_cache): + element_cache.use_restricted_data_cache = True + + result = await element_cache.get_all_restricted_data(None) + + assert sort_dict(result) == sort_dict({ + 'app/collection1': [{"id": 1, "value": "restricted_value1"}, {"id": 2, "value": "restricted_value2"}], + 'app/collection2': [{"id": 1, "key": "restricted_value1"}, {"id": 2, "key": "restricted_value2"}]}) + + +@pytest.mark.asyncio +async def test_get_all_restricted_data_disabled_restricted_data_cache(element_cache): + element_cache.use_restricted_data_cache = False + result = await element_cache.get_all_restricted_data(None) + + assert sort_dict(result) == sort_dict({ + 'app/collection1': [{"id": 1, "value": "restricted_value1"}, {"id": 2, "value": "restricted_value2"}], + 'app/collection2': [{"id": 1, "key": "restricted_value1"}, {"id": 2, "key": "restricted_value2"}]}) + + +@pytest.mark.asyncio +async def test_get_restricted_data_change_id_0(element_cache): + element_cache.use_restricted_data_cache = True + + result = await element_cache.get_restricted_data(None, 0) + + assert sort_dict(result[0]) == sort_dict({ + 'app/collection1': [{"id": 1, "value": "restricted_value1"}, {"id": 2, "value": "restricted_value2"}], + 'app/collection2': [{"id": 1, "key": "restricted_value1"}, {"id": 2, "key": "restricted_value2"}]}) + + +@pytest.mark.asyncio +async def test_get_restricted_data_disabled_restricted_data_cache(element_cache): + element_cache.use_restricted_data_cache = False + element_cache.cache_provider.change_id_data = {1: {'app/collection1:1', 'app/collection1:3'}} + + result = await element_cache.get_restricted_data(None, 1) + + assert result == ( + {'app/collection1': [{"id": 1, "value": "restricted_value1"}]}, + ['app/collection1:3']) + + +@pytest.mark.asyncio +async def test_get_restricted_data_change_id_lower_then_in_redis(element_cache): + element_cache.use_restricted_data_cache = True + element_cache.cache_provider.change_id_data = {2: {'app/collection1:1'}} + + with pytest.raises(RuntimeError): + await element_cache.get_restricted_data(None, 1) + + +@pytest.mark.asyncio +async def test_get_restricted_data_change_with_id(element_cache): + element_cache.use_restricted_data_cache = True + element_cache.cache_provider.change_id_data = {2: {'app/collection1:1'}} + + result = await element_cache.get_restricted_data(None, 2) + + assert result == ({'app/collection1': [{"id": 1, "value": "restricted_value1"}]}, []) + + +@pytest.mark.asyncio +async def test_lowest_change_id_after_updating_lowest_element(element_cache): + await element_cache.change_elements({'app/collection1:1': {"id": 1, "value": "updated1"}}) + first_lowest_change_id = await element_cache.get_lowest_change_id() + # Alter same element again + await element_cache.change_elements({'app/collection1:1': {"id": 1, "value": "updated2"}}) + second_lowest_change_id = await element_cache.get_lowest_change_id() + + assert first_lowest_change_id == 1 + assert second_lowest_change_id == 1 # The lowest_change_id should not change