From 060856628bac7cdf47e34dbd2675f941d1419462 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norman=20J=C3=A4ckel?= Date: Sun, 4 Nov 2018 14:02:30 +0100 Subject: [PATCH] OpenSlides history mode. Also containing auth check and viewpoint to clear history. --- CHANGELOG.rst | 1 + openslides/core/access_permissions.py | 22 +++ openslides/core/apps.py | 7 +- .../migrations/0009_auto_20181118_2126.py | 54 ++++++ openslides/core/models.py | 103 ++++++++++- openslides/core/serializers.py | 12 ++ openslides/core/urls.py | 4 + openslides/core/views.py | 93 +++++++++- openslides/motions/signals.py | 166 ++++++++++-------- openslides/motions/views.py | 20 ++- openslides/users/models.py | 16 +- openslides/users/signals.py | 29 ++- openslides/users/views.py | 8 +- openslides/utils/autoupdate.py | 81 +++++++-- openslides/utils/projector.py | 4 +- tests/integration/helpers.py | 13 -- tests/integration/utils/test_consumers.py | 72 +++++--- tests/unit/motions/test_views.py | 3 +- tests/unit/users/test_models.py | 13 +- 19 files changed, 557 insertions(+), 164 deletions(-) create mode 100644 openslides/core/migrations/0009_auto_20181118_2126.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 38443a582..c10d3e073 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,7 @@ Core: - Add a change-id system to get only new elements [#3938]. - Switch from Yarn back to npm [#3964]. - Added password reset link (password reset via email) [#3914]. + - Added global history mode [#3977]. Agenda: - Added viewpoint to assign multiple items to a new parent item [#4037]. diff --git a/openslides/core/access_permissions.py b/openslides/core/access_permissions.py index 2e2276465..1a8ce04f5 100644 --- a/openslides/core/access_permissions.py +++ b/openslides/core/access_permissions.py @@ -1,4 +1,5 @@ from ..utils.access_permissions import BaseAccessPermissions +from ..utils.auth import GROUP_ADMIN_PK, async_in_some_groups class ProjectorAccessPermissions(BaseAccessPermissions): @@ -88,3 +89,24 @@ class ConfigAccessPermissions(BaseAccessPermissions): from .serializers import ConfigSerializer return ConfigSerializer + + +class HistoryAccessPermissions(BaseAccessPermissions): + """ + Access permissions container for the Histroy. + """ + + async def async_check_permissions(self, user_id: int) -> bool: + """ + Returns True if the user is in admin group and has read access to + model instances. + """ + return await async_in_some_groups(user_id, [GROUP_ADMIN_PK]) + + def get_serializer_class(self, user=None): + """ + Returns serializer class. + """ + from .serializers import HistorySerializer + + return HistorySerializer diff --git a/openslides/core/apps.py b/openslides/core/apps.py index a4cf3c05e..5fc3ac3e3 100644 --- a/openslides/core/apps.py +++ b/openslides/core/apps.py @@ -32,6 +32,7 @@ class CoreAppConfig(AppConfig): ChatMessageViewSet, ConfigViewSet, CountdownViewSet, + HistoryViewSet, ProjectorMessageViewSet, ProjectorViewSet, TagViewSet, @@ -81,10 +82,12 @@ class CoreAppConfig(AppConfig): router.register(self.get_model('ConfigStore').get_collection_string(), ConfigViewSet, 'config') router.register(self.get_model('ProjectorMessage').get_collection_string(), ProjectorMessageViewSet) router.register(self.get_model('Countdown').get_collection_string(), CountdownViewSet) + router.register(self.get_model('History').get_collection_string(), HistoryViewSet) - # Sets the cache + # Sets the cache and builds the startup history if is_normal_server_start: element_cache.ensure_cache() + self.get_model('History').objects.build_history() # Register client messages register_client_message(NotifyWebsocketClientMessage()) @@ -104,7 +107,7 @@ class CoreAppConfig(AppConfig): Yields all Cachables required on startup i. e. opening the websocket connection. """ - for model_name in ('Projector', 'ChatMessage', 'Tag', 'ProjectorMessage', 'Countdown', 'ConfigStore'): + for model_name in ('Projector', 'ChatMessage', 'Tag', 'ProjectorMessage', 'Countdown', 'ConfigStore', 'History'): yield self.get_model(model_name) def get_angular_constants(self): diff --git a/openslides/core/migrations/0009_auto_20181118_2126.py b/openslides/core/migrations/0009_auto_20181118_2126.py new file mode 100644 index 000000000..3c7f2ff3f --- /dev/null +++ b/openslides/core/migrations/0009_auto_20181118_2126.py @@ -0,0 +1,54 @@ +# Generated by Django 2.1.3 on 2018-11-18 20:26 + +import django.db.models.deletion +import jsonfield.encoder +import jsonfield.fields +from django.conf import settings +from django.db import migrations, models + +import openslides.utils.models + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('core', '0008_changed_logo_fields'), + ] + + operations = [ + migrations.CreateModel( + name='History', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('element_id', models.CharField(max_length=255)), + ('now', models.DateTimeField(auto_now_add=True)), + ('information', models.CharField(max_length=255)), + ], + options={ + 'default_permissions': (), + }, + bases=(openslides.utils.models.RESTModelMixin, models.Model), + ), + migrations.CreateModel( + name='HistoryData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('full_data', jsonfield.fields.JSONField( + dump_kwargs={'cls': jsonfield.encoder.JSONEncoder, 'separators': (',', ':')}, load_kwargs={})), + ], + options={ + 'default_permissions': (), + }, + ), + migrations.AddField( + model_name='history', + name='full_data', + field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='core.HistoryData'), + ), + migrations.AddField( + model_name='history', + name='user', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/openslides/core/models.py b/openslides/core/models.py index 0c5af53f1..394dc241d 100644 --- a/openslides/core/models.py +++ b/openslides/core/models.py @@ -1,14 +1,18 @@ +from asgiref.sync import async_to_sync from django.conf import settings -from django.db import models +from django.db import models, transaction from django.utils.timezone import now from jsonfield import JSONField +from ..utils.autoupdate import Element +from ..utils.cache import element_cache, get_element_id from ..utils.models import RESTModelMixin from ..utils.projector import get_all_projector_elements from .access_permissions import ( ChatMessageAccessPermissions, ConfigAccessPermissions, CountdownAccessPermissions, + HistoryAccessPermissions, ProjectorAccessPermissions, ProjectorMessageAccessPermissions, TagAccessPermissions, @@ -324,3 +328,100 @@ class Countdown(RESTModelMixin, models.Model): self.running = False self.countdown_time = self.default_time self.save(skip_autoupdate=skip_autoupdate) + + +class HistoryData(models.Model): + """ + Django model to save the history of OpenSlides. + + This is not a RESTModel. It is not cachable and can only be reached by a + special viewset. + """ + full_data = JSONField() + + class Meta: + default_permissions = () + + +class HistoryManager(models.Manager): + """ + Customized model manager for the history model. + """ + def add_elements(self, elements): + """ + Method to add elements to the history. This does not trigger autoupdate. + """ + with transaction.atomic(): + instances = [] + for element in elements: + if element['disable_history'] or element['collection_string'] == self.model.get_collection_string(): + # Do not update history for history elements itself or if history is disabled. + continue + # HistoryData is not a root rest element so there is no autoupdate and not history saving here. + data = HistoryData.objects.create(full_data=element['full_data']) + instance = self.model( + element_id=get_element_id(element['collection_string'], element['id']), + information=element['information'], + user_id=element['user_id'], + full_data=data, + ) + instance.save(skip_autoupdate=True) # Skip autoupdate and of course history saving. + instances.append(instance) + return instances + + def build_history(self): + """ + Method to add all cachables to the history. + """ + # TODO: Add lock to prevent multiple history builds at once. See #4039. + instances = None + if self.all().count() == 0: + elements = [] + all_full_data = async_to_sync(element_cache.get_all_full_data)() + for collection_string, data in all_full_data.items(): + for full_data in data: + elements.append(Element( + id=full_data['id'], + collection_string=collection_string, + full_data=full_data, + information='', + user_id=None, + disable_history=False, + )) + instances = self.add_elements(elements) + return instances + + +class History(RESTModelMixin, models.Model): + """ + Django model to save the history of OpenSlides. + + This model itself is not part of the history. This means that if you + delete a user you may lose the information of the user field here. + """ + access_permissions = HistoryAccessPermissions() + + objects = HistoryManager() + + element_id = models.CharField( + max_length=255, + ) + + now = models.DateTimeField(auto_now_add=True) + + information = models.CharField( + max_length=255, + ) + + user = models.ForeignKey( + settings.AUTH_USER_MODEL, + null=True, + on_delete=models.SET_NULL) + + full_data = models.OneToOneField( + HistoryData, + on_delete=models.CASCADE, + ) + + class Meta: + default_permissions = () diff --git a/openslides/core/serializers.py b/openslides/core/serializers.py index 6693a3cc4..555056fcf 100644 --- a/openslides/core/serializers.py +++ b/openslides/core/serializers.py @@ -5,6 +5,7 @@ from .models import ( ChatMessage, ConfigStore, Countdown, + History, ProjectionDefault, Projector, ProjectorMessage, @@ -110,3 +111,14 @@ class CountdownSerializer(ModelSerializer): class Meta: model = Countdown fields = ('id', 'description', 'default_time', 'countdown_time', 'running', ) + + +class HistorySerializer(ModelSerializer): + """ + Serializer for core.models.Countdown objects. + + Does not contain full data of history object. + """ + class Meta: + model = History + fields = ('id', 'element_id', 'now', 'information', 'user', ) diff --git a/openslides/core/urls.py b/openslides/core/urls.py index b087a5bbf..cc6d6e236 100644 --- a/openslides/core/urls.py +++ b/openslides/core/urls.py @@ -11,4 +11,8 @@ urlpatterns = [ url(r'^version/$', views.VersionView.as_view(), name='core_version'), + + url(r'^history/$', + views.HistoryView.as_view(), + name='core_history'), ] diff --git a/openslides/core/views.py b/openslides/core/views.py index 383962a55..f491ed230 100644 --- a/openslides/core/views.py +++ b/openslides/core/views.py @@ -1,3 +1,4 @@ +import datetime import os import uuid from typing import Any, Dict, List @@ -16,7 +17,12 @@ from mypy_extensions import TypedDict from .. import __license__ as license, __url__ as url, __version__ as version from ..utils import views as utils_views from ..utils.arguments import arguments -from ..utils.auth import anonymous_is_enabled, has_perm +from ..utils.auth import ( + GROUP_ADMIN_PK, + anonymous_is_enabled, + has_perm, + in_some_groups, +) from ..utils.autoupdate import inform_changed_data, inform_deleted_data from ..utils.plugins import ( get_plugin_description, @@ -26,8 +32,11 @@ from ..utils.plugins import ( get_plugin_version, ) from ..utils.rest_api import ( + GenericViewSet, + ListModelMixin, ModelViewSet, Response, + RetrieveModelMixin, ValidationError, detail_route, list_route, @@ -36,6 +45,7 @@ from .access_permissions import ( ChatMessageAccessPermissions, ConfigAccessPermissions, CountdownAccessPermissions, + HistoryAccessPermissions, ProjectorAccessPermissions, ProjectorMessageAccessPermissions, TagAccessPermissions, @@ -46,6 +56,8 @@ from .models import ( ChatMessage, ConfigStore, Countdown, + History, + HistoryData, ProjectionDefault, Projector, ProjectorMessage, @@ -716,6 +728,50 @@ class CountdownViewSet(ModelViewSet): return result +class HistoryViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): + """ + API endpoint for History. + + There are the following views: list, retrieve, clear_history. + """ + access_permissions = HistoryAccessPermissions() + queryset = History.objects.all() + + def check_view_permissions(self): + """ + Returns True if the user has required permissions. + """ + if self.action in ('list', 'retrieve', 'clear_history'): + result = self.get_access_permissions().check_permissions(self.request.user) + else: + result = False + return result + + @list_route(methods=['post']) + def clear_history(self, request): + """ + Deletes and rebuilds the history. + """ + # Collect all history objects with their collection_string and id. + args = [] + for history_obj in History.objects.all(): + args.append((history_obj.get_collection_string(), history_obj.pk)) + + # Delete history data and history (via CASCADE) + HistoryData.objects.all().delete() + + # Trigger autoupdate. + if len(args) > 0: + inform_deleted_data(args) + + # Rebuild history. + history_instances = History.objects.build_history() + inform_changed_data(history_instances) + + # Setup response. + return Response({'detail': _('History was deleted successfully.')}) + + # Special API views class ServerTime(utils_views.APIView): @@ -755,3 +811,38 @@ class VersionView(utils_views.APIView): 'license': get_plugin_license(plugin), 'url': get_plugin_url(plugin)}) return result + + +class HistoryView(utils_views.APIView): + """ + View to retrieve the history data of OpenSlides. + + Use query paramter timestamp (UNIX timestamp) to get all elements from begin + until (including) this timestamp. + """ + http_method_names = ['get'] + + def get_context_data(self, **context): + """ + Checks if user is in admin group. If yes all history data until + (including) timestamp are added to the response data. + """ + if not in_some_groups(self.request.user.pk or 0, [GROUP_ADMIN_PK]): + self.permission_denied(self.request) + try: + timestamp = int(self.request.query_params.get('timestamp', 0)) + except (ValueError): + raise ValidationError({'detail': 'Invalid input. Timestamp should be an integer.'}) + data = [] + queryset = History.objects.select_related('full_data') + if timestamp: + queryset = queryset.filter(now__lte=datetime.datetime.fromtimestamp(timestamp)) + for instance in queryset: + data.append({ + 'full_data': instance.full_data.full_data, + 'element_id': instance.element_id, + 'timestamp': instance.now.timestamp(), + 'information': instance.information, + 'user_id': instance.user.pk if instance.user else None, + }) + return data diff --git a/openslides/motions/signals.py b/openslides/motions/signals.py index 1d6a9f85d..cb4911519 100644 --- a/openslides/motions/signals.py +++ b/openslides/motions/signals.py @@ -14,87 +14,103 @@ def create_builtin_workflows(sender, **kwargs): # If there is at least one workflow, then do nothing. return - workflow_1 = Workflow.objects.create(name='Simple Workflow') - state_1_1 = State.objects.create(name=ugettext_noop('submitted'), - workflow=workflow_1, - allow_create_poll=True, - allow_support=True, - allow_submitter_edit=True) - state_1_2 = State.objects.create(name=ugettext_noop('accepted'), - workflow=workflow_1, - action_word='Accept', - recommendation_label='Acceptance', - css_class='success', - merge_amendment_into_final=True) - state_1_3 = State.objects.create(name=ugettext_noop('rejected'), - workflow=workflow_1, - action_word='Reject', - recommendation_label='Rejection', - css_class='danger') - state_1_4 = State.objects.create(name=ugettext_noop('not decided'), - workflow=workflow_1, - action_word='Do not decide', - recommendation_label='No decision', - css_class='default') + workflow_1 = Workflow(name='Simple Workflow') + workflow_1.save(skip_autoupdate=True) + state_1_1 = State(name=ugettext_noop('submitted'), + workflow=workflow_1, + allow_create_poll=True, + allow_support=True, + allow_submitter_edit=True) + state_1_1.save(skip_autoupdate=True) + state_1_2 = State(name=ugettext_noop('accepted'), + workflow=workflow_1, + action_word='Accept', + recommendation_label='Acceptance', + css_class='success'), + merge_amendment_into_final=True) + state_1_2.save(skip_autoupdate=True) + state_1_3 = State(name=ugettext_noop('rejected'), + workflow=workflow_1, + action_word='Reject', + recommendation_label='Rejection', + css_class='danger') + state_1_3.save(skip_autoupdate=True) + state_1_4 = State(name=ugettext_noop('not decided'), + workflow=workflow_1, + action_word='Do not decide', + recommendation_label='No decision', + css_class='default') + state_1_4.save(skip_autoupdate=True) state_1_1.next_states.add(state_1_2, state_1_3, state_1_4) workflow_1.first_state = state_1_1 - workflow_1.save() + workflow_1.save(skip_autoupdate=True) - workflow_2 = Workflow.objects.create(name='Complex Workflow') - state_2_1 = State.objects.create(name=ugettext_noop('published'), - workflow=workflow_2, - allow_support=True, - allow_submitter_edit=True, - dont_set_identifier=True) - state_2_2 = State.objects.create(name=ugettext_noop('permitted'), - workflow=workflow_2, - action_word='Permit', - recommendation_label='Permission', - allow_create_poll=True, - allow_submitter_edit=True) - state_2_3 = State.objects.create(name=ugettext_noop('accepted'), - workflow=workflow_2, - action_word='Accept', - recommendation_label='Acceptance', - css_class='success', - merge_amendment_into_final=True) - state_2_4 = State.objects.create(name=ugettext_noop('rejected'), - workflow=workflow_2, - action_word='Reject', - recommendation_label='Rejection', - css_class='danger') - state_2_5 = State.objects.create(name=ugettext_noop('withdrawed'), - workflow=workflow_2, - action_word='Withdraw', - css_class='default') - state_2_6 = State.objects.create(name=ugettext_noop('adjourned'), - workflow=workflow_2, - action_word='Adjourn', - recommendation_label='Adjournment', - css_class='default') - state_2_7 = State.objects.create(name=ugettext_noop('not concerned'), - workflow=workflow_2, - action_word='Do not concern', - recommendation_label='No concernment', - css_class='default') - state_2_8 = State.objects.create(name=ugettext_noop('refered to committee'), - workflow=workflow_2, - action_word='Refer to committee', - recommendation_label='Referral to committee', - css_class='default') - state_2_9 = State.objects.create(name=ugettext_noop('needs review'), - workflow=workflow_2, - action_word='Needs review', - css_class='default') - state_2_10 = State.objects.create(name=ugettext_noop('rejected (not authorized)'), - workflow=workflow_2, - action_word='Reject (not authorized)', - recommendation_label='Rejection (not authorized)', - css_class='default') + workflow_2 = Workflow(name='Complex Workflow') + workflow_2.save(skip_autoupdate=True) + state_2_1 = State(name=ugettext_noop('published'), + workflow=workflow_2, + allow_support=True, + allow_submitter_edit=True, + dont_set_identifier=True) + state_2_1.save(skip_autoupdate=True) + state_2_2 = State(name=ugettext_noop('permitted'), + workflow=workflow_2, + action_word='Permit', + recommendation_label='Permission', + allow_create_poll=True, + allow_submitter_edit=True) + state_2_2.save(skip_autoupdate=True) + state_2_3 = State(name=ugettext_noop('accepted'), + workflow=workflow_2, + action_word='Accept', + recommendation_label='Acceptance', + css_class='success'), + merge_amendment_into_final=True) + state_2_3.save(skip_autoupdate=True) + state_2_4 = State(name=ugettext_noop('rejected'), + workflow=workflow_2, + action_word='Reject', + recommendation_label='Rejection', + css_class='danger') + state_2_4.save(skip_autoupdate=True) + state_2_5 = State(name=ugettext_noop('withdrawed'), + workflow=workflow_2, + action_word='Withdraw', + css_class='default') + state_2_5.save(skip_autoupdate=True) + state_2_6 = State(name=ugettext_noop('adjourned'), + workflow=workflow_2, + action_word='Adjourn', + recommendation_label='Adjournment', + css_class='default') + state_2_6.save(skip_autoupdate=True) + state_2_7 = State(name=ugettext_noop('not concerned'), + workflow=workflow_2, + action_word='Do not concern', + recommendation_label='No concernment', + css_class='default') + state_2_7.save(skip_autoupdate=True) + state_2_8 = State(name=ugettext_noop('refered to committee'), + workflow=workflow_2, + action_word='Refer to committee', + recommendation_label='Referral to committee', + css_class='default') + state_2_8.save(skip_autoupdate=True) + state_2_9 = State(name=ugettext_noop('needs review'), + workflow=workflow_2, + action_word='Needs review', + css_class='default') + state_2_9.save(skip_autoupdate=True) + state_2_10 = State(name=ugettext_noop('rejected (not authorized)'), + workflow=workflow_2, + action_word='Reject (not authorized)', + recommendation_label='Rejection (not authorized)', + css_class='default') + state_2_10.save(skip_autoupdate=True) state_2_1.next_states.add(state_2_2, state_2_5, state_2_10) state_2_2.next_states.add(state_2_3, state_2_4, state_2_5, state_2_6, state_2_7, state_2_8, state_2_9) workflow_2.first_state = state_2_1 - workflow_2.save() + workflow_2.save(skip_autoupdate=True) def get_permission_change_data(sender, permissions, **kwargs): diff --git a/openslides/motions/views.py b/openslides/motions/views.py index 400a53527..53a4cbf9f 100644 --- a/openslides/motions/views.py +++ b/openslides/motions/views.py @@ -14,7 +14,7 @@ from rest_framework import status from ..core.config import config from ..core.models import Tag from ..utils.auth import has_perm, in_some_groups -from ..utils.autoupdate import inform_changed_data +from ..utils.autoupdate import inform_changed_data, inform_deleted_data from ..utils.exceptions import OpenSlidesError from ..utils.rest_api import ( CreateModelMixin, @@ -107,7 +107,15 @@ class MotionViewSet(ModelViewSet): motion.is_submitter(request.user) and motion.state.allow_submitter_edit)): self.permission_denied(request) - return super().destroy(request, *args, **kwargs) + result = super().destroy(request, *args, **kwargs) + + # Fire autoupdate again to save information to OpenSlides history. + inform_deleted_data( + [(motion.get_collection_string(), motion.pk)], + information='Motion deleted', + user_id=request.user.pk) + + return result def create(self, request, *args, **kwargs): """ @@ -279,6 +287,12 @@ class MotionViewSet(ModelViewSet): new_users = list(updated_motion.supporters.all()) inform_changed_data(new_users) + # Fire autoupdate again to save information to OpenSlides history. + inform_changed_data( + updated_motion, + information='Motion updated', + user_id=request.user.pk) + # We do not add serializer.data to response so nobody gets unrestricted data here. return Response() @@ -630,7 +644,7 @@ class MotionViewSet(ModelViewSet): message_list=[ugettext_noop('State set to'), ' ', motion.state.name], person=request.user, skip_autoupdate=True) - inform_changed_data(motion) + inform_changed_data(motion, information='State set to {}.'.format(motion.state.name)) return Response({'detail': message}) @detail_route(methods=['put']) diff --git a/openslides/users/models.py b/openslides/users/models.py index 311364bbd..b6ca42b10 100644 --- a/openslides/users/models.py +++ b/openslides/users/models.py @@ -11,7 +11,7 @@ from django.contrib.auth.models import ( PermissionsMixin, ) from django.core import mail -from django.core.exceptions import ValidationError +from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.db import models from django.db.models import Prefetch from django.utils import timezone @@ -62,12 +62,18 @@ class UserManager(BaseUserManager): exists, resets it. The password is (re)set to 'admin'. The user becomes member of the group 'Admin'. """ - admin, created = self.get_or_create( - username='admin', - defaults={'last_name': 'Administrator'}) + created = False + try: + admin = self.get(username='admin') + except ObjectDoesNotExist: + admin = self.model( + username='admin', + last_name='Administrator', + ) + created = True admin.default_password = 'admin' admin.password = make_password(admin.default_password) - admin.save() + admin.save(skip_autoupdate=True) admin.groups.add(GROUP_ADMIN_PK) return created diff --git a/openslides/users/signals.py b/openslides/users/signals.py index e7d021a3e..6a79f9fe7 100644 --- a/openslides/users/signals.py +++ b/openslides/users/signals.py @@ -3,7 +3,6 @@ from django.contrib.auth.models import Permission from django.db.models import Q from ..utils.auth import GROUP_ADMIN_PK, GROUP_DEFAULT_PK -from ..utils.autoupdate import inform_changed_data from .models import Group, User @@ -81,11 +80,13 @@ def create_builtin_groups_and_admin(**kwargs): permission_dict['mediafiles.can_see'], permission_dict['motions.can_see'], permission_dict['users.can_see_name'], ) - group_default = Group.objects.create(pk=GROUP_DEFAULT_PK, name='Default') + group_default = Group(pk=GROUP_DEFAULT_PK, name='Default') + group_default.save(skip_autoupdate=True) group_default.permissions.add(*base_permissions) # Admin (pk 2 == GROUP_ADMIN_PK) - group_admin = Group.objects.create(pk=GROUP_ADMIN_PK, name='Admin') + group_admin = Group(pk=GROUP_ADMIN_PK, name='Admin') + group_admin.save(skip_autoupdate=True) # Delegates (pk 3) delegates_permissions = ( @@ -102,7 +103,8 @@ def create_builtin_groups_and_admin(**kwargs): permission_dict['motions.can_create'], permission_dict['motions.can_support'], permission_dict['users.can_see_name'], ) - group_delegates = Group.objects.create(pk=3, name='Delegates') + group_delegates = Group(pk=3, name='Delegates') + group_delegates.save(skip_autoupdate=True) group_delegates.permissions.add(*delegates_permissions) # Staff (pk 4) @@ -132,17 +134,10 @@ def create_builtin_groups_and_admin(**kwargs): permission_dict['users.can_manage'], permission_dict['users.can_see_extra_data'], permission_dict['mediafiles.can_see_hidden'],) - group_staff = Group.objects.create(pk=4, name='Staff') + group_staff = Group(pk=4, name='Staff') + group_staff.save(skip_autoupdate=True) group_staff.permissions.add(*staff_permissions) - # Add users.can_see_name permission to staff/admin - # group to ensure proper management possibilities - # TODO: Remove this redundancy after cleanup of the permission system. - group_staff.permissions.add( - permission_dict['users.can_see_name']) - group_admin.permissions.add( - permission_dict['users.can_see_name']) - # Committees (pk 5) committees_permissions = ( permission_dict['agenda.can_see'], @@ -155,13 +150,13 @@ def create_builtin_groups_and_admin(**kwargs): permission_dict['motions.can_create'], permission_dict['motions.can_support'], permission_dict['users.can_see_name'], ) - group_committee = Group.objects.create(pk=5, name='Committees') + group_committee = Group(pk=5, name='Committees') + group_committee.save(skip_autoupdate=True) group_committee.permissions.add(*committees_permissions) # Create or reset admin user User.objects.create_or_reset_admin_user() # After each group was created, the permissions (many to many fields) where - # added to the group. So we have to update the cache by calling - # inform_changed_data(). - inform_changed_data((group_default, group_admin, group_delegates, group_staff, group_committee)) + # added to the group. But we do not have to update the cache by calling + # inform_changed_data() because the cache is updated on server start. diff --git a/openslides/users/views.py b/openslides/users/views.py index e0e8d0a81..36e1f7700 100644 --- a/openslides/users/views.py +++ b/openslides/users/views.py @@ -336,7 +336,13 @@ class GroupViewSet(ModelViewSet): for receiver, signal_collections in signal_results: for cachable in signal_collections: for full_data in all_full_data.get(cachable.get_collection_string(), {}): - elements.append(Element(id=full_data['id'], collection_string=cachable.get_collection_string(), full_data=full_data)) + elements.append(Element( + id=full_data['id'], + collection_string=cachable.get_collection_string(), + full_data=full_data, + information='', + user_id=None, + disable_history=True)) inform_changed_elements(elements) # TODO: Some permissions are deleted. diff --git a/openslides/utils/autoupdate.py b/openslides/utils/autoupdate.py index cfd359014..8514b7c1d 100644 --- a/openslides/utils/autoupdate.py +++ b/openslides/utils/autoupdate.py @@ -1,3 +1,4 @@ +import itertools import threading from typing import Any, Dict, Iterable, List, Optional, Tuple, Union @@ -15,6 +16,9 @@ Element = TypedDict( 'id': int, 'collection_string': str, 'full_data': Optional[Dict[str, Any]], + 'information': str, + 'user_id': Optional[int], + 'disable_history': bool, } ) @@ -30,12 +34,17 @@ AutoupdateFormat = TypedDict( ) -def inform_changed_data(instances: Union[Iterable[Model], Model]) -> None: +def inform_changed_data( + instances: Union[Iterable[Model], Model], + information: str = '', + user_id: Optional[int] = None) -> None: """ Informs the autoupdate system and the caching system about the creation or update of an element. The argument instances can be one instance or an iterable over instances. + + History creation is enabled. """ root_instances = set() if not isinstance(instances, Iterable): @@ -54,7 +63,11 @@ def inform_changed_data(instances: Union[Iterable[Model], Model]) -> None: elements[key] = Element( id=root_instance.get_rest_pk(), collection_string=root_instance.get_collection_string(), - full_data=root_instance.get_full_data()) + full_data=root_instance.get_full_data(), + information=information, + user_id=user_id, + disable_history=False, + ) bundle = autoupdate_bundle.get(threading.get_ident()) if bundle is not None: @@ -65,15 +78,27 @@ def inform_changed_data(instances: Union[Iterable[Model], Model]) -> None: handle_changed_elements(elements.values()) -def inform_deleted_data(deleted_elements: Iterable[Tuple[str, int]]) -> None: +def inform_deleted_data( + deleted_elements: Iterable[Tuple[str, int]], + information: str = '', + user_id: Optional[int] = None) -> None: """ Informs the autoupdate system and the caching system about the deletion of elements. + + History creation is enabled. """ elements: Dict[str, Element] = {} for deleted_element in deleted_elements: key = deleted_element[0] + str(deleted_element[1]) - elements[key] = Element(id=deleted_element[1], collection_string=deleted_element[0], full_data=None) + elements[key] = Element( + id=deleted_element[1], + collection_string=deleted_element[0], + full_data=None, + information=information, + user_id=user_id, + disable_history=False, + ) bundle = autoupdate_bundle.get(threading.get_ident()) if bundle is not None: @@ -86,8 +111,11 @@ def inform_deleted_data(deleted_elements: Iterable[Tuple[str, int]]) -> None: def inform_changed_elements(changed_elements: Iterable[Element]) -> None: """ - Informs the autoupdate system about some collection elements. This is - used just to send some data to all users. + Informs the autoupdate system about some elements. This is used just to send + some data to all users. + + If you want to save history information, user id or disable history you + have to put information or flag inside the elements. """ elements = {} for changed_element in changed_elements: @@ -135,7 +163,7 @@ def handle_changed_elements(elements: Iterable[Element]) -> None: Does nothing if elements is empty. """ - async def update_cache() -> int: + async def update_cache(elements: Iterable[Element]) -> int: """ Async helper function to update the cache. @@ -147,12 +175,12 @@ def handle_changed_elements(elements: Iterable[Element]) -> None: cache_elements[element_id] = element['full_data'] return await element_cache.change_elements(cache_elements) - async def async_handle_collection_elements() -> None: + async def async_handle_collection_elements(elements: Iterable[Element]) -> None: """ Async helper function to update cache and send autoupdate. """ # Update cache - change_id = await update_cache() + change_id = await update_cache(elements) # Send autoupdate channel_layer = get_channel_layer() @@ -165,7 +193,36 @@ def handle_changed_elements(elements: Iterable[Element]) -> None: ) if elements: - # TODO: Save histroy here using sync code + # Save histroy here using sync code. + history_instances = save_history(elements) - # Update cache and send autoupdate - async_to_sync(async_handle_collection_elements)() + # Convert history instances to Elements. + history_elements: List[Element] = [] + for history_instance in history_instances: + history_elements.append(Element( + id=history_instance.get_rest_pk(), + collection_string=history_instance.get_collection_string(), + full_data=history_instance.get_full_data(), + information='', + user_id=None, + disable_history=True, # This does not matter because history elements can never be part of the history itself. + )) + + # Chain elements and history elements. + itertools.chain(elements, history_elements) + + # Update cache and send autoupdate using async code. + async_to_sync(async_handle_collection_elements)( + itertools.chain(elements, history_elements) + ) + + +def save_history(elements: Iterable[Element]) -> Iterable: # TODO: Try to write Iterable[History] here + """ + Thin wrapper around the call of history saving manager method. + + This is separated to patch it during tests. + """ + from ..core.models import History + + return History.objects.add_elements(elements) diff --git a/openslides/utils/projector.py b/openslides/utils/projector.py index 8975adecf..1cbc10275 100644 --- a/openslides/utils/projector.py +++ b/openslides/utils/projector.py @@ -53,8 +53,8 @@ def register_projector_elements(elements: Generator[Type[ProjectorElement], None Has to be called in the app.ready method. """ - for Element in elements: - element = Element() + for AppProjectorElement in elements: + element = AppProjectorElement() projector_elements[element.name] = element # type: ignore diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 5083cd195..70e5903de 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -1,12 +1,10 @@ from typing import Any, Dict, List -from asgiref.sync import sync_to_async from django.db import DEFAULT_DB_ALIAS, connections from django.test.utils import CaptureQueriesContext from openslides.core.config import config from openslides.users.models import User -from openslides.utils.autoupdate import Element, inform_changed_elements class TConfig: @@ -55,17 +53,6 @@ class TUser: return elements -async def set_config(key, value): - """ - Set a config variable in the element_cache without hitting the database. - """ - collection_string = config.get_collection_string() - config_id = config.key_to_id[key] # type: ignore - full_data = {'id': config_id, 'key': key, 'value': value} - await sync_to_async(inform_changed_elements)([ - Element(id=config_id, collection_string=collection_string, full_data=full_data)]) - - def count_queries(func, *args, **kwargs) -> int: context = CaptureQueriesContext(connections[DEFAULT_DB_ALIAS]) with context: diff --git a/tests/integration/utils/test_consumers.py b/tests/integration/utils/test_consumers.py index 642823305..a1d41d51d 100644 --- a/tests/integration/utils/test_consumers.py +++ b/tests/integration/utils/test_consumers.py @@ -1,5 +1,6 @@ import asyncio from importlib import import_module +from unittest.mock import patch import pytest from asgiref.sync import sync_to_async @@ -13,7 +14,11 @@ from django.contrib.auth import ( from openslides.asgi import application from openslides.core.config import config -from openslides.utils.autoupdate import inform_deleted_data +from openslides.utils.autoupdate import ( + Element, + inform_changed_elements, + inform_deleted_data, +) from openslides.utils.cache import element_cache from ...unit.utils.cache_provider import ( @@ -21,7 +26,7 @@ from ...unit.utils.cache_provider import ( Collection2, get_cachable_provider, ) -from ..helpers import TConfig, TUser, set_config +from ..helpers import TConfig, TUser @pytest.fixture(autouse=True) @@ -64,15 +69,31 @@ async def communicator(get_communicator): yield get_communicator() +@pytest.fixture +async def set_config(): + """ + Set a config variable in the element_cache without hitting the database. + """ + async def _set_config(key, value): + with patch('openslides.utils.autoupdate.save_history'): + collection_string = config.get_collection_string() + config_id = config.key_to_id[key] # type: ignore + full_data = {'id': config_id, 'key': key, 'value': value} + await sync_to_async(inform_changed_elements)([ + Element(id=config_id, collection_string=collection_string, full_data=full_data, information='', user_id=None, disable_history=True)]) + + return _set_config + + @pytest.mark.asyncio -async def test_normal_connection(get_communicator): +async def test_normal_connection(get_communicator, set_config): await set_config('general_system_enable_anonymous', True) connected, __ = await get_communicator().connect() assert connected @pytest.mark.asyncio -async def test_connection_with_change_id(get_communicator): +async def test_connection_with_change_id(get_communicator, set_config): await set_config('general_system_enable_anonymous', True) communicator = get_communicator('change_id=0') await communicator.connect() @@ -93,7 +114,7 @@ async def test_connection_with_change_id(get_communicator): @pytest.mark.asyncio -async def test_connection_with_change_id_get_restricted_data_with_restricted_data_cache(get_communicator): +async def test_connection_with_change_id_get_restricted_data_with_restricted_data_cache(get_communicator, set_config): """ Test, that the returned data is the restricted_data when restricted_data_cache is activated """ @@ -116,7 +137,7 @@ async def test_connection_with_change_id_get_restricted_data_with_restricted_dat @pytest.mark.asyncio -async def test_connection_with_invalid_change_id(get_communicator): +async def test_connection_with_invalid_change_id(get_communicator, set_config): await set_config('general_system_enable_anonymous', True) communicator = get_communicator('change_id=invalid') connected, __ = await communicator.connect() @@ -125,7 +146,7 @@ async def test_connection_with_invalid_change_id(get_communicator): @pytest.mark.asyncio -async def test_connection_with_to_big_change_id(get_communicator): +async def test_connection_with_to_big_change_id(get_communicator, set_config): await set_config('general_system_enable_anonymous', True) communicator = get_communicator('change_id=100') @@ -136,7 +157,7 @@ async def test_connection_with_to_big_change_id(get_communicator): @pytest.mark.asyncio -async def test_changed_data_autoupdate_off(communicator): +async def test_changed_data_autoupdate_off(communicator, set_config): await set_config('general_system_enable_anonymous', True) await communicator.connect() @@ -146,7 +167,7 @@ async def test_changed_data_autoupdate_off(communicator): @pytest.mark.asyncio -async def test_changed_data_autoupdate_on(get_communicator): +async def test_changed_data_autoupdate_on(get_communicator, set_config): await set_config('general_system_enable_anonymous', True) communicator = get_communicator('autoupdate=on') await communicator.connect() @@ -191,13 +212,14 @@ async def test_with_user(): @pytest.mark.asyncio -async def test_receive_deleted_data(get_communicator): +async def test_receive_deleted_data(get_communicator, set_config): await set_config('general_system_enable_anonymous', True) communicator = get_communicator('autoupdate=on') await communicator.connect() # Delete test element - await sync_to_async(inform_deleted_data)([(Collection1().get_collection_string(), 1)]) + with patch('openslides.utils.autoupdate.save_history'): + await sync_to_async(inform_deleted_data)([(Collection1().get_collection_string(), 1)]) response = await communicator.receive_json_from() type = response.get('type') @@ -207,7 +229,7 @@ async def test_receive_deleted_data(get_communicator): @pytest.mark.asyncio -async def test_send_notify(communicator): +async def test_send_notify(communicator, set_config): await set_config('general_system_enable_anonymous', True) await communicator.connect() @@ -223,7 +245,7 @@ async def test_send_notify(communicator): @pytest.mark.asyncio -async def test_invalid_websocket_message_type(communicator): +async def test_invalid_websocket_message_type(communicator, set_config): await set_config('general_system_enable_anonymous', True) await communicator.connect() @@ -234,7 +256,7 @@ async def test_invalid_websocket_message_type(communicator): @pytest.mark.asyncio -async def test_invalid_websocket_message_no_id(communicator): +async def test_invalid_websocket_message_no_id(communicator, set_config): await set_config('general_system_enable_anonymous', True) await communicator.connect() @@ -245,7 +267,7 @@ async def test_invalid_websocket_message_no_id(communicator): @pytest.mark.asyncio -async def test_send_unknown_type(communicator): +async def test_send_unknown_type(communicator, set_config): await set_config('general_system_enable_anonymous', True) await communicator.connect() @@ -257,7 +279,7 @@ async def test_send_unknown_type(communicator): @pytest.mark.asyncio -async def test_request_constants(communicator, settings): +async def test_request_constants(communicator, settings, set_config): await set_config('general_system_enable_anonymous', True) await communicator.connect() @@ -270,7 +292,7 @@ async def test_request_constants(communicator, settings): @pytest.mark.asyncio -async def test_send_get_elements(communicator): +async def test_send_get_elements(communicator, set_config): await set_config('general_system_enable_anonymous', True) await communicator.connect() @@ -291,7 +313,7 @@ async def test_send_get_elements(communicator): @pytest.mark.asyncio -async def test_send_get_elements_to_big_change_id(communicator): +async def test_send_get_elements_to_big_change_id(communicator, set_config): await set_config('general_system_enable_anonymous', True) await communicator.connect() @@ -304,7 +326,7 @@ async def test_send_get_elements_to_big_change_id(communicator): @pytest.mark.asyncio -async def test_send_get_elements_to_small_change_id(communicator): +async def test_send_get_elements_to_small_change_id(communicator, set_config): await set_config('general_system_enable_anonymous', True) await communicator.connect() @@ -318,7 +340,7 @@ async def test_send_get_elements_to_small_change_id(communicator): @pytest.mark.asyncio -async def test_send_connect_twice_with_clear_change_id_cache(communicator): +async def test_send_connect_twice_with_clear_change_id_cache(communicator, set_config): """ Test, that a second request with change_id+1 from the first request, returns an error. @@ -338,7 +360,7 @@ async def test_send_connect_twice_with_clear_change_id_cache(communicator): @pytest.mark.asyncio -async def test_send_connect_twice_with_clear_change_id_cache_same_change_id_then_first_request(communicator): +async def test_send_connect_twice_with_clear_change_id_cache_same_change_id_then_first_request(communicator, set_config): """ Test, that a second request with the change_id from the first request, returns all data. @@ -360,7 +382,7 @@ async def test_send_connect_twice_with_clear_change_id_cache_same_change_id_then @pytest.mark.asyncio -async def test_request_changed_elements_no_douple_elements(communicator): +async def test_request_changed_elements_no_douple_elements(communicator, set_config): """ Test, that when an elements is changed twice, it is only returned onces when ask a range of change ids. @@ -386,7 +408,7 @@ async def test_request_changed_elements_no_douple_elements(communicator): @pytest.mark.asyncio -async def test_send_invalid_get_elements(communicator): +async def test_send_invalid_get_elements(communicator, set_config): await set_config('general_system_enable_anonymous', True) await communicator.connect() @@ -399,7 +421,7 @@ async def test_send_invalid_get_elements(communicator): @pytest.mark.asyncio -async def test_turn_on_autoupdate(communicator): +async def test_turn_on_autoupdate(communicator, set_config): await set_config('general_system_enable_anonymous', True) await communicator.connect() @@ -418,7 +440,7 @@ async def test_turn_on_autoupdate(communicator): @pytest.mark.asyncio -async def test_turn_off_autoupdate(get_communicator): +async def test_turn_off_autoupdate(get_communicator, set_config): await set_config('general_system_enable_anonymous', True) communicator = get_communicator('autoupdate=on') await communicator.connect() diff --git a/tests/unit/motions/test_views.py b/tests/unit/motions/test_views.py index e50734d7d..f42638c82 100644 --- a/tests/unit/motions/test_views.py +++ b/tests/unit/motions/test_views.py @@ -21,7 +21,8 @@ class MotionViewSetUpdate(TestCase): @patch('openslides.motions.views.has_perm') @patch('openslides.motions.views.config') def test_simple_update(self, mock_config, mock_has_perm, mock_icd): - self.request.user = 1 + self.request.user = MagicMock() + self.request.user.pk = 1 self.request.data.get.return_value = MagicMock() mock_has_perm.return_value = True diff --git a/tests/unit/users/test_models.py b/tests/unit/users/test_models.py index 5f20a1872..59ab834df 100644 --- a/tests/unit/users/test_models.py +++ b/tests/unit/users/test_models.py @@ -1,6 +1,8 @@ from unittest import TestCase from unittest.mock import MagicMock, call, patch +from django.core.exceptions import ObjectDoesNotExist + from openslides.users.models import UserManager @@ -127,7 +129,7 @@ class UserManagerCreateOrResetAdminUser(TestCase): """ admin_user = MagicMock() manager = UserManager() - manager.get_or_create = MagicMock(return_value=(admin_user, False)) + manager.get = MagicMock(return_value=(admin_user)) manager.create_or_reset_admin_user() @@ -139,7 +141,7 @@ class UserManagerCreateOrResetAdminUser(TestCase): """ admin_user = MagicMock() manager = UserManager() - manager.get_or_create = MagicMock(return_value=(admin_user, False)) + manager.get = MagicMock(return_value=(admin_user)) staff_group = MagicMock(name="Staff") mock_group.objects.get_or_create = MagicMock(return_value=(staff_group, True)) @@ -150,16 +152,15 @@ class UserManagerCreateOrResetAdminUser(TestCase): self.assertEqual( admin_user.default_password, 'admin') - admin_user.save.assert_called_once_with() + admin_user.save.assert_called_once_with(skip_autoupdate=True) @patch('openslides.users.models.User') def test_return_value(self, mock_user, mock_group, mock_permission): """ Tests that the method returns True when a user is created. """ - admin_user = MagicMock() manager = UserManager() - manager.get_or_create = MagicMock(return_value=(admin_user, True)) + manager.get = MagicMock(side_effect=ObjectDoesNotExist) manager.model = mock_user staff_group = MagicMock(name="Staff") @@ -179,7 +180,7 @@ class UserManagerCreateOrResetAdminUser(TestCase): """ admin_user = MagicMock(username='admin', last_name='Administrator') manager = UserManager() - manager.get_or_create = MagicMock(return_value=(admin_user, True)) + manager.get = MagicMock(side_effect=ObjectDoesNotExist) manager.model = mock_user staff_group = MagicMock(name="Staff")