ag-admin/OpenSlides
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

OpenSlides history mode.

Browse Source 
Also containing auth check and viewpoint to clear history.
This commit is contained in:
Norman Jäckel 2018-11-04 14:02:30 +01:00 committed by Sean Engelhardt
parent 0a823877c2
commit 060856628b
19 changed files with 557 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.rst
View File

@ -20,6 +20,7 @@ Core:
- Add a change-id system to get only new elements [#3938].
- Switch from Yarn back to npm [#3964].
- Added password reset link (password reset via email) [#3914].
- Added global history mode [#3977].
Agenda:
- Added viewpoint to assign multiple items to a new parent item [#4037].

22
openslides/core/access_permissions.py
View File

@ -1,4 +1,5 @@
from ..utils.access_permissions import BaseAccessPermissions
from ..utils.auth import GROUP_ADMIN_PK, async_in_some_groups
class ProjectorAccessPermissions(BaseAccessPermissions):
@ -88,3 +89,24 @@ class ConfigAccessPermissions(BaseAccessPermissions):
from .serializers import ConfigSerializer
return ConfigSerializer
class HistoryAccessPermissions(BaseAccessPermissions):
"""
Access permissions container for the Histroy.
"""
async def async_check_permissions(self, user_id: int) -> bool:
"""
Returns True if the user is in admin group and has read access to
model instances.
"""
return await async_in_some_groups(user_id, [GROUP_ADMIN_PK])
def get_serializer_class(self, user=None):
"""
Returns serializer class.
"""
from .serializers import HistorySerializer
return HistorySerializer

7
openslides/core/apps.py
View File

@ -32,6 +32,7 @@ class CoreAppConfig(AppConfig):
ChatMessageViewSet,
ConfigViewSet,
CountdownViewSet,
HistoryViewSet,
ProjectorMessageViewSet,
ProjectorViewSet,
TagViewSet,
@ -81,10 +82,12 @@ class CoreAppConfig(AppConfig):
router.register(self.get_model('ConfigStore').get_collection_string(), ConfigViewSet, 'config')
router.register(self.get_model('ProjectorMessage').get_collection_string(), ProjectorMessageViewSet)
router.register(self.get_model('Countdown').get_collection_string(), CountdownViewSet)
router.register(self.get_model('History').get_collection_string(), HistoryViewSet)
# Sets the cache
# Sets the cache and builds the startup history
if is_normal_server_start:
element_cache.ensure_cache()
self.get_model('History').objects.build_history()
# Register client messages
register_client_message(NotifyWebsocketClientMessage())
@ -104,7 +107,7 @@ class CoreAppConfig(AppConfig):
Yields all Cachables required on startup i. e. opening the websocket
connection.
"""
for model_name in ('Projector', 'ChatMessage', 'Tag', 'ProjectorMessage', 'Countdown', 'ConfigStore'):
for model_name in ('Projector', 'ChatMessage', 'Tag', 'ProjectorMessage', 'Countdown', 'ConfigStore', 'History'):
yield self.get_model(model_name)
def get_angular_constants(self):

54
openslides/core/migrations/0009_auto_20181118_2126.py Normal file
View File

@ -0,0 +1,54 @@
# Generated by Django 2.1.3 on 2018-11-18 20:26
import django.db.models.deletion
import jsonfield.encoder
import jsonfield.fields
from django.conf import settings
from django.db import migrations, models
import openslides.utils.models
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('core', '0008_changed_logo_fields'),
]
operations = [
migrations.CreateModel(
name='History',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('element_id', models.CharField(max_length=255)),
('now', models.DateTimeField(auto_now_add=True)),
('information', models.CharField(max_length=255)),
],
options={
'default_permissions': (),
},
bases=(openslides.utils.models.RESTModelMixin, models.Model),
),
migrations.CreateModel(
name='HistoryData',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('full_data', jsonfield.fields.JSONField(
dump_kwargs={'cls': jsonfield.encoder.JSONEncoder, 'separators': (',', ':')}, load_kwargs={})),
],
options={
'default_permissions': (),
},
),
migrations.AddField(
model_name='history',
name='full_data',
field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='core.HistoryData'),
),
migrations.AddField(
model_name='history',
name='user',
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL),
),
]

103
openslides/core/models.py
View File

@ -1,14 +1,18 @@
from asgiref.sync import async_to_sync
from django.conf import settings
from django.db import models
from django.db import models, transaction
from django.utils.timezone import now
from jsonfield import JSONField
from ..utils.autoupdate import Element
from ..utils.cache import element_cache, get_element_id
from ..utils.models import RESTModelMixin
from ..utils.projector import get_all_projector_elements
from .access_permissions import (
ChatMessageAccessPermissions,
ConfigAccessPermissions,
CountdownAccessPermissions,
HistoryAccessPermissions,
ProjectorAccessPermissions,
ProjectorMessageAccessPermissions,
TagAccessPermissions,
@ -324,3 +328,100 @@ class Countdown(RESTModelMixin, models.Model):
self.running = False
self.countdown_time = self.default_time
self.save(skip_autoupdate=skip_autoupdate)
class HistoryData(models.Model):
"""
Django model to save the history of OpenSlides.
This is not a RESTModel. It is not cachable and can only be reached by a
special viewset.
"""
full_data = JSONField()
class Meta:
default_permissions = ()
class HistoryManager(models.Manager):
"""
Customized model manager for the history model.
"""
def add_elements(self, elements):
"""
Method to add elements to the history. This does not trigger autoupdate.
"""
with transaction.atomic():
instances = []
for element in elements:
if element['disable_history'] or element['collection_string'] == self.model.get_collection_string():
# Do not update history for history elements itself or if history is disabled.
continue
# HistoryData is not a root rest element so there is no autoupdate and not history saving here.
data = HistoryData.objects.create(full_data=element['full_data'])
instance = self.model(
element_id=get_element_id(element['collection_string'], element['id']),
information=element['information'],
user_id=element['user_id'],
full_data=data,
)
instance.save(skip_autoupdate=True) # Skip autoupdate and of course history saving.
instances.append(instance)
return instances
def build_history(self):
"""
Method to add all cachables to the history.
"""
# TODO: Add lock to prevent multiple history builds at once. See #4039.
instances = None
if self.all().count() == 0:
elements = []
all_full_data = async_to_sync(element_cache.get_all_full_data)()
for collection_string, data in all_full_data.items():
for full_data in data:
elements.append(Element(
id=full_data['id'],
collection_string=collection_string,
full_data=full_data,
information='',
user_id=None,
disable_history=False,
))
instances = self.add_elements(elements)
return instances
class History(RESTModelMixin, models.Model):
"""
Django model to save the history of OpenSlides.
This model itself is not part of the history. This means that if you
delete a user you may lose the information of the user field here.
"""
access_permissions = HistoryAccessPermissions()
objects = HistoryManager()
element_id = models.CharField(
max_length=255,
)
now = models.DateTimeField(auto_now_add=True)
information = models.CharField(
max_length=255,
)
user = models.ForeignKey(
settings.AUTH_USER_MODEL,
null=True,
on_delete=models.SET_NULL)
full_data = models.OneToOneField(
HistoryData,
on_delete=models.CASCADE,
)
class Meta:
default_permissions = ()

12
openslides/core/serializers.py
View File

@ -5,6 +5,7 @@ from .models import (
ChatMessage,
ConfigStore,
Countdown,
History,
ProjectionDefault,
Projector,
ProjectorMessage,
@ -110,3 +111,14 @@ class CountdownSerializer(ModelSerializer):
class Meta:
model = Countdown
fields = ('id', 'description', 'default_time', 'countdown_time', 'running', )
class HistorySerializer(ModelSerializer):
"""
Serializer for core.models.Countdown objects.
Does not contain full data of history object.
"""
class Meta:
model = History
fields = ('id', 'element_id', 'now', 'information', 'user', )

4
openslides/core/urls.py
View File

@ -11,4 +11,8 @@ urlpatterns = [
url(r'^version/$',
views.VersionView.as_view(),
name='core_version'),
url(r'^history/$',
views.HistoryView.as_view(),
name='core_history'),
]

93
openslides/core/views.py
View File

@ -1,3 +1,4 @@
import datetime
import os
import uuid
from typing import Any, Dict, List
@ -16,7 +17,12 @@ from mypy_extensions import TypedDict
from .. import __license__ as license, __url__ as url, __version__ as version
from ..utils import views as utils_views
from ..utils.arguments import arguments
from ..utils.auth import anonymous_is_enabled, has_perm
from ..utils.auth import (
GROUP_ADMIN_PK,
anonymous_is_enabled,
has_perm,
in_some_groups,
)
from ..utils.autoupdate import inform_changed_data, inform_deleted_data
from ..utils.plugins import (
get_plugin_description,
@ -26,8 +32,11 @@ from ..utils.plugins import (
get_plugin_version,
)
from ..utils.rest_api import (
GenericViewSet,
ListModelMixin,
ModelViewSet,
Response,
RetrieveModelMixin,
ValidationError,
detail_route,
list_route,
@ -36,6 +45,7 @@ from .access_permissions import (
ChatMessageAccessPermissions,
ConfigAccessPermissions,
CountdownAccessPermissions,
HistoryAccessPermissions,
ProjectorAccessPermissions,
ProjectorMessageAccessPermissions,
TagAccessPermissions,
@ -46,6 +56,8 @@ from .models import (
ChatMessage,
ConfigStore,
Countdown,
History,
HistoryData,
ProjectionDefault,
Projector,
ProjectorMessage,
@ -716,6 +728,50 @@ class CountdownViewSet(ModelViewSet):
return result
class HistoryViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet):
"""
API endpoint for History.
There are the following views: list, retrieve, clear_history.
"""
access_permissions = HistoryAccessPermissions()
queryset = History.objects.all()
def check_view_permissions(self):
"""
Returns True if the user has required permissions.
"""
if self.action in ('list', 'retrieve', 'clear_history'):
result = self.get_access_permissions().check_permissions(self.request.user)
else:
result = False
return result
@list_route(methods=['post'])
def clear_history(self, request):
"""
Deletes and rebuilds the history.
"""
# Collect all history objects with their collection_string and id.
args = []
for history_obj in History.objects.all():
args.append((history_obj.get_collection_string(), history_obj.pk))
# Delete history data and history (via CASCADE)
HistoryData.objects.all().delete()
# Trigger autoupdate.
if len(args) > 0:
inform_deleted_data(args)
# Rebuild history.
history_instances = History.objects.build_history()
inform_changed_data(history_instances)
# Setup response.
return Response({'detail': _('History was deleted successfully.')})
# Special API views
class ServerTime(utils_views.APIView):
@ -755,3 +811,38 @@ class VersionView(utils_views.APIView):
'license': get_plugin_license(plugin),
'url': get_plugin_url(plugin)})
return result
class HistoryView(utils_views.APIView):
"""
View to retrieve the history data of OpenSlides.
Use query paramter timestamp (UNIX timestamp) to get all elements from begin
until (including) this timestamp.
"""
http_method_names = ['get']
def get_context_data(self, **context):
"""
Checks if user is in admin group. If yes all history data until
(including) timestamp are added to the response data.
"""
if not in_some_groups(self.request.user.pk or 0, [GROUP_ADMIN_PK]):
self.permission_denied(self.request)
try:
timestamp = int(self.request.query_params.get('timestamp', 0))
except (ValueError):
raise ValidationError({'detail': 'Invalid input. Timestamp should be an integer.'})
data = []
queryset = History.objects.select_related('full_data')
if timestamp:
queryset = queryset.filter(now__lte=datetime.datetime.fromtimestamp(timestamp))
for instance in queryset:
data.append({
'full_data': instance.full_data.full_data,
'element_id': instance.element_id,
'timestamp': instance.now.timestamp(),
'information': instance.information,
'user_id': instance.user.pk if instance.user else None,
})
return data

166
openslides/motions/signals.py
View File

@ -14,87 +14,103 @@ def create_builtin_workflows(sender, **kwargs):
# If there is at least one workflow, then do nothing.
return
workflow_1 = Workflow.objects.create(name='Simple Workflow')
state_1_1 = State.objects.create(name=ugettext_noop('submitted'),
workflow=workflow_1,
allow_create_poll=True,
allow_support=True,
allow_submitter_edit=True)
state_1_2 = State.objects.create(name=ugettext_noop('accepted'),
workflow=workflow_1,
action_word='Accept',
recommendation_label='Acceptance',
css_class='success',
merge_amendment_into_final=True)
state_1_3 = State.objects.create(name=ugettext_noop('rejected'),
workflow=workflow_1,
action_word='Reject',
recommendation_label='Rejection',
css_class='danger')
state_1_4 = State.objects.create(name=ugettext_noop('not decided'),
workflow=workflow_1,
action_word='Do not decide',
recommendation_label='No decision',
css_class='default')
workflow_1 = Workflow(name='Simple Workflow')
workflow_1.save(skip_autoupdate=True)
state_1_1 = State(name=ugettext_noop('submitted'),
workflow=workflow_1,
allow_create_poll=True,
allow_support=True,
allow_submitter_edit=True)
state_1_1.save(skip_autoupdate=True)
state_1_2 = State(name=ugettext_noop('accepted'),
workflow=workflow_1,
action_word='Accept',
recommendation_label='Acceptance',
css_class='success'),
merge_amendment_into_final=True)
state_1_2.save(skip_autoupdate=True)
state_1_3 = State(name=ugettext_noop('rejected'),
workflow=workflow_1,
action_word='Reject',
recommendation_label='Rejection',
css_class='danger')
state_1_3.save(skip_autoupdate=True)
state_1_4 = State(name=ugettext_noop('not decided'),
workflow=workflow_1,
action_word='Do not decide',
recommendation_label='No decision',
css_class='default')
state_1_4.save(skip_autoupdate=True)
state_1_1.next_states.add(state_1_2, state_1_3, state_1_4)
workflow_1.first_state = state_1_1
workflow_1.save()
workflow_1.save(skip_autoupdate=True)
workflow_2 = Workflow.objects.create(name='Complex Workflow')
state_2_1 = State.objects.create(name=ugettext_noop('published'),
workflow=workflow_2,
allow_support=True,
allow_submitter_edit=True,
dont_set_identifier=True)
state_2_2 = State.objects.create(name=ugettext_noop('permitted'),
workflow=workflow_2,
action_word='Permit',
recommendation_label='Permission',
allow_create_poll=True,
allow_submitter_edit=True)
state_2_3 = State.objects.create(name=ugettext_noop('accepted'),
workflow=workflow_2,
action_word='Accept',
recommendation_label='Acceptance',
css_class='success',
merge_amendment_into_final=True)
state_2_4 = State.objects.create(name=ugettext_noop('rejected'),
workflow=workflow_2,
action_word='Reject',
recommendation_label='Rejection',
css_class='danger')
state_2_5 = State.objects.create(name=ugettext_noop('withdrawed'),
workflow=workflow_2,
action_word='Withdraw',
css_class='default')
state_2_6 = State.objects.create(name=ugettext_noop('adjourned'),
workflow=workflow_2,
action_word='Adjourn',
recommendation_label='Adjournment',
css_class='default')
state_2_7 = State.objects.create(name=ugettext_noop('not concerned'),
workflow=workflow_2,
action_word='Do not concern',
recommendation_label='No concernment',
css_class='default')
state_2_8 = State.objects.create(name=ugettext_noop('refered to committee'),
workflow=workflow_2,
action_word='Refer to committee',
recommendation_label='Referral to committee',
css_class='default')
state_2_9 = State.objects.create(name=ugettext_noop('needs review'),
workflow=workflow_2,
action_word='Needs review',
css_class='default')
state_2_10 = State.objects.create(name=ugettext_noop('rejected (not authorized)'),
workflow=workflow_2,
action_word='Reject (not authorized)',
recommendation_label='Rejection (not authorized)',
css_class='default')
workflow_2 = Workflow(name='Complex Workflow')
workflow_2.save(skip_autoupdate=True)
state_2_1 = State(name=ugettext_noop('published'),
workflow=workflow_2,
allow_support=True,
allow_submitter_edit=True,
dont_set_identifier=True)
state_2_1.save(skip_autoupdate=True)
state_2_2 = State(name=ugettext_noop('permitted'),
workflow=workflow_2,
action_word='Permit',
recommendation_label='Permission',
allow_create_poll=True,
allow_submitter_edit=True)
state_2_2.save(skip_autoupdate=True)
state_2_3 = State(name=ugettext_noop('accepted'),
workflow=workflow_2,
action_word='Accept',
recommendation_label='Acceptance',
css_class='success'),
merge_amendment_into_final=True)
state_2_3.save(skip_autoupdate=True)
state_2_4 = State(name=ugettext_noop('rejected'),
workflow=workflow_2,
action_word='Reject',
recommendation_label='Rejection',
css_class='danger')
state_2_4.save(skip_autoupdate=True)
state_2_5 = State(name=ugettext_noop('withdrawed'),
workflow=workflow_2,
action_word='Withdraw',
css_class='default')
state_2_5.save(skip_autoupdate=True)
state_2_6 = State(name=ugettext_noop('adjourned'),
workflow=workflow_2,
action_word='Adjourn',
recommendation_label='Adjournment',
css_class='default')
state_2_6.save(skip_autoupdate=True)
state_2_7 = State(name=ugettext_noop('not concerned'),
workflow=workflow_2,
action_word='Do not concern',
recommendation_label='No concernment',
css_class='default')
state_2_7.save(skip_autoupdate=True)
state_2_8 = State(name=ugettext_noop('refered to committee'),
workflow=workflow_2,
action_word='Refer to committee',
recommendation_label='Referral to committee',
css_class='default')
state_2_8.save(skip_autoupdate=True)
state_2_9 = State(name=ugettext_noop('needs review'),
workflow=workflow_2,
action_word='Needs review',
css_class='default')
state_2_9.save(skip_autoupdate=True)
state_2_10 = State(name=ugettext_noop('rejected (not authorized)'),
workflow=workflow_2,
action_word='Reject (not authorized)',
recommendation_label='Rejection (not authorized)',
css_class='default')
state_2_10.save(skip_autoupdate=True)
state_2_1.next_states.add(state_2_2, state_2_5, state_2_10)
state_2_2.next_states.add(state_2_3, state_2_4, state_2_5, state_2_6, state_2_7, state_2_8, state_2_9)
workflow_2.first_state = state_2_1
workflow_2.save()
workflow_2.save(skip_autoupdate=True)
def get_permission_change_data(sender, permissions, **kwargs):

20
openslides/motions/views.py
View File

@ -14,7 +14,7 @@ from rest_framework import status
from ..core.config import config
from ..core.models import Tag
from ..utils.auth import has_perm, in_some_groups
from ..utils.autoupdate import inform_changed_data
from ..utils.autoupdate import inform_changed_data, inform_deleted_data
from ..utils.exceptions import OpenSlidesError
from ..utils.rest_api import (
CreateModelMixin,
@ -107,7 +107,15 @@ class MotionViewSet(ModelViewSet):
motion.is_submitter(request.user) and motion.state.allow_submitter_edit)):
self.permission_denied(request)
return super().destroy(request, *args, **kwargs)
result = super().destroy(request, *args, **kwargs)
# Fire autoupdate again to save information to OpenSlides history.
inform_deleted_data(
[(motion.get_collection_string(), motion.pk)],
information='Motion deleted',
user_id=request.user.pk)
return result
def create(self, request, *args, **kwargs):
"""
@ -279,6 +287,12 @@ class MotionViewSet(ModelViewSet):
new_users = list(updated_motion.supporters.all())
inform_changed_data(new_users)
# Fire autoupdate again to save information to OpenSlides history.
inform_changed_data(
updated_motion,
information='Motion updated',
user_id=request.user.pk)
# We do not add serializer.data to response so nobody gets unrestricted data here.
return Response()
@ -630,7 +644,7 @@ class MotionViewSet(ModelViewSet):
message_list=[ugettext_noop('State set to'), ' ', motion.state.name],
person=request.user,
skip_autoupdate=True)
inform_changed_data(motion)
inform_changed_data(motion, information='State set to {}.'.format(motion.state.name))
return Response({'detail': message})
@detail_route(methods=['put'])

16
openslides/users/models.py
View File

@ -11,7 +11,7 @@ from django.contrib.auth.models import (
PermissionsMixin,
)
from django.core import mail
from django.core.exceptions import ValidationError
from django.core.exceptions import ObjectDoesNotExist, ValidationError
from django.db import models
from django.db.models import Prefetch
from django.utils import timezone
@ -62,12 +62,18 @@ class UserManager(BaseUserManager):
exists, resets it. The password is (re)set to 'admin'. The user
becomes member of the group 'Admin'.
"""
admin, created = self.get_or_create(
username='admin',
defaults={'last_name': 'Administrator'})
created = False
try:
admin = self.get(username='admin')
except ObjectDoesNotExist:
admin = self.model(
username='admin',
last_name='Administrator',
)
created = True
admin.default_password = 'admin'
admin.password = make_password(admin.default_password)
admin.save()
admin.save(skip_autoupdate=True)
admin.groups.add(GROUP_ADMIN_PK)
return created

29
openslides/users/signals.py
View File

@ -3,7 +3,6 @@ from django.contrib.auth.models import Permission
from django.db.models import Q
from ..utils.auth import GROUP_ADMIN_PK, GROUP_DEFAULT_PK
from ..utils.autoupdate import inform_changed_data
from .models import Group, User
@ -81,11 +80,13 @@ def create_builtin_groups_and_admin(**kwargs):
permission_dict['mediafiles.can_see'],
permission_dict['motions.can_see'],
permission_dict['users.can_see_name'], )
group_default = Group.objects.create(pk=GROUP_DEFAULT_PK, name='Default')
group_default = Group(pk=GROUP_DEFAULT_PK, name='Default')
group_default.save(skip_autoupdate=True)
group_default.permissions.add(*base_permissions)
# Admin (pk 2 == GROUP_ADMIN_PK)
group_admin = Group.objects.create(pk=GROUP_ADMIN_PK, name='Admin')
group_admin = Group(pk=GROUP_ADMIN_PK, name='Admin')
group_admin.save(skip_autoupdate=True)
# Delegates (pk 3)
delegates_permissions = (
@ -102,7 +103,8 @@ def create_builtin_groups_and_admin(**kwargs):
permission_dict['motions.can_create'],
permission_dict['motions.can_support'],
permission_dict['users.can_see_name'], )
group_delegates = Group.objects.create(pk=3, name='Delegates')
group_delegates = Group(pk=3, name='Delegates')
group_delegates.save(skip_autoupdate=True)
group_delegates.permissions.add(*delegates_permissions)
# Staff (pk 4)
@ -132,17 +134,10 @@ def create_builtin_groups_and_admin(**kwargs):
permission_dict['users.can_manage'],
permission_dict['users.can_see_extra_data'],
permission_dict['mediafiles.can_see_hidden'],)
group_staff = Group.objects.create(pk=4, name='Staff')
group_staff = Group(pk=4, name='Staff')
group_staff.save(skip_autoupdate=True)
group_staff.permissions.add(*staff_permissions)
# Add users.can_see_name permission to staff/admin
# group to ensure proper management possibilities
# TODO: Remove this redundancy after cleanup of the permission system.
group_staff.permissions.add(
permission_dict['users.can_see_name'])
group_admin.permissions.add(
permission_dict['users.can_see_name'])
# Committees (pk 5)
committees_permissions = (
permission_dict['agenda.can_see'],
@ -155,13 +150,13 @@ def create_builtin_groups_and_admin(**kwargs):
permission_dict['motions.can_create'],
permission_dict['motions.can_support'],
permission_dict['users.can_see_name'], )
group_committee = Group.objects.create(pk=5, name='Committees')
group_committee = Group(pk=5, name='Committees')
group_committee.save(skip_autoupdate=True)
group_committee.permissions.add(*committees_permissions)
# Create or reset admin user
User.objects.create_or_reset_admin_user()
# After each group was created, the permissions (many to many fields) where
# added to the group. So we have to update the cache by calling
# inform_changed_data().
inform_changed_data((group_default, group_admin, group_delegates, group_staff, group_committee))
# added to the group. But we do not have to update the cache by calling
# inform_changed_data() because the cache is updated on server start.

8
openslides/users/views.py
View File

@ -336,7 +336,13 @@ class GroupViewSet(ModelViewSet):
for receiver, signal_collections in signal_results:
for cachable in signal_collections:
for full_data in all_full_data.get(cachable.get_collection_string(), {}):
elements.append(Element(id=full_data['id'], collection_string=cachable.get_collection_string(), full_data=full_data))
elements.append(Element(
id=full_data['id'],
collection_string=cachable.get_collection_string(),
full_data=full_data,
information='',
user_id=None,
disable_history=True))
inform_changed_elements(elements)
# TODO: Some permissions are deleted.

81
openslides/utils/autoupdate.py
View File

@ -1,3 +1,4 @@
import itertools
import threading
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
@ -15,6 +16,9 @@ Element = TypedDict(
'id': int,
'collection_string': str,
'full_data': Optional[Dict[str, Any]],
'information': str,
'user_id': Optional[int],
'disable_history': bool,
}
)
@ -30,12 +34,17 @@ AutoupdateFormat = TypedDict(
)
def inform_changed_data(instances: Union[Iterable[Model], Model]) -> None:
def inform_changed_data(
instances: Union[Iterable[Model], Model],
information: str = '',
user_id: Optional[int] = None) -> None:
"""
Informs the autoupdate system and the caching system about the creation or
update of an element.
The argument instances can be one instance or an iterable over instances.
History creation is enabled.
"""
root_instances = set()
if not isinstance(instances, Iterable):
@ -54,7 +63,11 @@ def inform_changed_data(instances: Union[Iterable[Model], Model]) -> None:
elements[key] = Element(
id=root_instance.get_rest_pk(),
collection_string=root_instance.get_collection_string(),
full_data=root_instance.get_full_data())
full_data=root_instance.get_full_data(),
information=information,
user_id=user_id,
disable_history=False,
)
bundle = autoupdate_bundle.get(threading.get_ident())
if bundle is not None:
@ -65,15 +78,27 @@ def inform_changed_data(instances: Union[Iterable[Model], Model]) -> None:
handle_changed_elements(elements.values())
def inform_deleted_data(deleted_elements: Iterable[Tuple[str, int]]) -> None:
def inform_deleted_data(
deleted_elements: Iterable[Tuple[str, int]],
information: str = '',
user_id: Optional[int] = None) -> None:
"""
Informs the autoupdate system and the caching system about the deletion of
elements.
History creation is enabled.
"""
elements: Dict[str, Element] = {}
for deleted_element in deleted_elements:
key = deleted_element[0] + str(deleted_element[1])
elements[key] = Element(id=deleted_element[1], collection_string=deleted_element[0], full_data=None)
elements[key] = Element(
id=deleted_element[1],
collection_string=deleted_element[0],
full_data=None,
information=information,
user_id=user_id,
disable_history=False,
)
bundle = autoupdate_bundle.get(threading.get_ident())
if bundle is not None:
@ -86,8 +111,11 @@ def inform_deleted_data(deleted_elements: Iterable[Tuple[str, int]]) -> None:
def inform_changed_elements(changed_elements: Iterable[Element]) -> None:
"""
Informs the autoupdate system about some collection elements. This is
used just to send some data to all users.
Informs the autoupdate system about some elements. This is used just to send
some data to all users.
If you want to save history information, user id or disable history you
have to put information or flag inside the elements.
"""
elements = {}
for changed_element in changed_elements:
@ -135,7 +163,7 @@ def handle_changed_elements(elements: Iterable[Element]) -> None:
Does nothing if elements is empty.
"""
async def update_cache() -> int:
async def update_cache(elements: Iterable[Element]) -> int:
"""
Async helper function to update the cache.
@ -147,12 +175,12 @@ def handle_changed_elements(elements: Iterable[Element]) -> None:
cache_elements[element_id] = element['full_data']
return await element_cache.change_elements(cache_elements)
async def async_handle_collection_elements() -> None:
async def async_handle_collection_elements(elements: Iterable[Element]) -> None:
"""
Async helper function to update cache and send autoupdate.
"""
# Update cache
change_id = await update_cache()
change_id = await update_cache(elements)
# Send autoupdate
channel_layer = get_channel_layer()
@ -165,7 +193,36 @@ def handle_changed_elements(elements: Iterable[Element]) -> None:
)
if elements:
# TODO: Save histroy here using sync code
# Save histroy here using sync code.
history_instances = save_history(elements)
# Update cache and send autoupdate
async_to_sync(async_handle_collection_elements)()
# Convert history instances to Elements.
history_elements: List[Element] = []
for history_instance in history_instances:
history_elements.append(Element(
id=history_instance.get_rest_pk(),
collection_string=history_instance.get_collection_string(),
full_data=history_instance.get_full_data(),
information='',
user_id=None,
disable_history=True, # This does not matter because history elements can never be part of the history itself.
))
# Chain elements and history elements.
itertools.chain(elements, history_elements)
# Update cache and send autoupdate using async code.
async_to_sync(async_handle_collection_elements)(
itertools.chain(elements, history_elements)
)
def save_history(elements: Iterable[Element]) -> Iterable: # TODO: Try to write Iterable[History] here
"""
Thin wrapper around the call of history saving manager method.
This is separated to patch it during tests.
"""
from ..core.models import History
return History.objects.add_elements(elements)

4
openslides/utils/projector.py
View File

@ -53,8 +53,8 @@ def register_projector_elements(elements: Generator[Type[ProjectorElement], None
Has to be called in the app.ready method.
"""
for Element in elements:
element = Element()
for AppProjectorElement in elements:
element = AppProjectorElement()
projector_elements[element.name] = element # type: ignore

13
tests/integration/helpers.py
View File

@ -1,12 +1,10 @@
from typing import Any, Dict, List
from asgiref.sync import sync_to_async
from django.db import DEFAULT_DB_ALIAS, connections
from django.test.utils import CaptureQueriesContext
from openslides.core.config import config
from openslides.users.models import User
from openslides.utils.autoupdate import Element, inform_changed_elements
class TConfig:
@ -55,17 +53,6 @@ class TUser:
return elements
async def set_config(key, value):
"""
Set a config variable in the element_cache without hitting the database.
"""
collection_string = config.get_collection_string()
config_id = config.key_to_id[key] # type: ignore
full_data = {'id': config_id, 'key': key, 'value': value}
await sync_to_async(inform_changed_elements)([
Element(id=config_id, collection_string=collection_string, full_data=full_data)])
def count_queries(func, *args, **kwargs) -> int:
context = CaptureQueriesContext(connections[DEFAULT_DB_ALIAS])
with context:

72
tests/integration/utils/test_consumers.py
View File

@ -1,5 +1,6 @@
import asyncio
from importlib import import_module
from unittest.mock import patch
import pytest
from asgiref.sync import sync_to_async
@ -13,7 +14,11 @@ from django.contrib.auth import (
from openslides.asgi import application
from openslides.core.config import config
from openslides.utils.autoupdate import inform_deleted_data
from openslides.utils.autoupdate import (
Element,
inform_changed_elements,
inform_deleted_data,
)
from openslides.utils.cache import element_cache
from ...unit.utils.cache_provider import (
@ -21,7 +26,7 @@ from ...unit.utils.cache_provider import (
Collection2,
get_cachable_provider,
)
from ..helpers import TConfig, TUser, set_config
from ..helpers import TConfig, TUser
@pytest.fixture(autouse=True)
@ -64,15 +69,31 @@ async def communicator(get_communicator):
yield get_communicator()
@pytest.fixture
async def set_config():
"""
Set a config variable in the element_cache without hitting the database.
"""
async def _set_config(key, value):
with patch('openslides.utils.autoupdate.save_history'):
collection_string = config.get_collection_string()
config_id = config.key_to_id[key] # type: ignore
full_data = {'id': config_id, 'key': key, 'value': value}
await sync_to_async(inform_changed_elements)([
Element(id=config_id, collection_string=collection_string, full_data=full_data, information='', user_id=None, disable_history=True)])
return _set_config
@pytest.mark.asyncio
async def test_normal_connection(get_communicator):
async def test_normal_connection(get_communicator, set_config):
await set_config('general_system_enable_anonymous', True)
connected, __ = await get_communicator().connect()
assert connected
@pytest.mark.asyncio
async def test_connection_with_change_id(get_communicator):
async def test_connection_with_change_id(get_communicator, set_config):
await set_config('general_system_enable_anonymous', True)
communicator = get_communicator('change_id=0')
await communicator.connect()
@ -93,7 +114,7 @@ async def test_connection_with_change_id(get_communicator):
@pytest.mark.asyncio
async def test_connection_with_change_id_get_restricted_data_with_restricted_data_cache(get_communicator):
async def test_connection_with_change_id_get_restricted_data_with_restricted_data_cache(get_communicator, set_config):
"""
Test, that the returned data is the restricted_data when restricted_data_cache is activated
"""
@ -116,7 +137,7 @@ async def test_connection_with_change_id_get_restricted_data_with_restricted_dat
@pytest.mark.asyncio
async def test_connection_with_invalid_change_id(get_communicator):
async def test_connection_with_invalid_change_id(get_communicator, set_config):
await set_config('general_system_enable_anonymous', True)
communicator = get_communicator('change_id=invalid')
connected, __ = await communicator.connect()
@ -125,7 +146,7 @@ async def test_connection_with_invalid_change_id(get_communicator):
@pytest.mark.asyncio
async def test_connection_with_to_big_change_id(get_communicator):
async def test_connection_with_to_big_change_id(get_communicator, set_config):
await set_config('general_system_enable_anonymous', True)
communicator = get_communicator('change_id=100')
@ -136,7 +157,7 @@ async def test_connection_with_to_big_change_id(get_communicator):
@pytest.mark.asyncio
async def test_changed_data_autoupdate_off(communicator):
async def test_changed_data_autoupdate_off(communicator, set_config):
await set_config('general_system_enable_anonymous', True)
await communicator.connect()
@ -146,7 +167,7 @@ async def test_changed_data_autoupdate_off(communicator):
@pytest.mark.asyncio
async def test_changed_data_autoupdate_on(get_communicator):
async def test_changed_data_autoupdate_on(get_communicator, set_config):
await set_config('general_system_enable_anonymous', True)
communicator = get_communicator('autoupdate=on')
await communicator.connect()
@ -191,13 +212,14 @@ async def test_with_user():
@pytest.mark.asyncio
async def test_receive_deleted_data(get_communicator):
async def test_receive_deleted_data(get_communicator, set_config):
await set_config('general_system_enable_anonymous', True)
communicator = get_communicator('autoupdate=on')
await communicator.connect()
# Delete test element
await sync_to_async(inform_deleted_data)([(Collection1().get_collection_string(), 1)])
with patch('openslides.utils.autoupdate.save_history'):
await sync_to_async(inform_deleted_data)([(Collection1().get_collection_string(), 1)])
response = await communicator.receive_json_from()
type = response.get('type')
@ -207,7 +229,7 @@ async def test_receive_deleted_data(get_communicator):
@pytest.mark.asyncio
async def test_send_notify(communicator):
async def test_send_notify(communicator, set_config):
await set_config('general_system_enable_anonymous', True)
await communicator.connect()
@ -223,7 +245,7 @@ async def test_send_notify(communicator):
@pytest.mark.asyncio
async def test_invalid_websocket_message_type(communicator):
async def test_invalid_websocket_message_type(communicator, set_config):
await set_config('general_system_enable_anonymous', True)
await communicator.connect()
@ -234,7 +256,7 @@ async def test_invalid_websocket_message_type(communicator):
@pytest.mark.asyncio
async def test_invalid_websocket_message_no_id(communicator):
async def test_invalid_websocket_message_no_id(communicator, set_config):
await set_config('general_system_enable_anonymous', True)
await communicator.connect()
@ -245,7 +267,7 @@ async def test_invalid_websocket_message_no_id(communicator):
@pytest.mark.asyncio
async def test_send_unknown_type(communicator):
async def test_send_unknown_type(communicator, set_config):
await set_config('general_system_enable_anonymous', True)
await communicator.connect()
@ -257,7 +279,7 @@ async def test_send_unknown_type(communicator):
@pytest.mark.asyncio
async def test_request_constants(communicator, settings):
async def test_request_constants(communicator, settings, set_config):
await set_config('general_system_enable_anonymous', True)
await communicator.connect()
@ -270,7 +292,7 @@ async def test_request_constants(communicator, settings):
@pytest.mark.asyncio
async def test_send_get_elements(communicator):
async def test_send_get_elements(communicator, set_config):
await set_config('general_system_enable_anonymous', True)
await communicator.connect()
@ -291,7 +313,7 @@ async def test_send_get_elements(communicator):
@pytest.mark.asyncio
async def test_send_get_elements_to_big_change_id(communicator):
async def test_send_get_elements_to_big_change_id(communicator, set_config):
await set_config('general_system_enable_anonymous', True)
await communicator.connect()
@ -304,7 +326,7 @@ async def test_send_get_elements_to_big_change_id(communicator):
@pytest.mark.asyncio
async def test_send_get_elements_to_small_change_id(communicator):
async def test_send_get_elements_to_small_change_id(communicator, set_config):
await set_config('general_system_enable_anonymous', True)
await communicator.connect()
@ -318,7 +340,7 @@ async def test_send_get_elements_to_small_change_id(communicator):
@pytest.mark.asyncio
async def test_send_connect_twice_with_clear_change_id_cache(communicator):
async def test_send_connect_twice_with_clear_change_id_cache(communicator, set_config):
"""
Test, that a second request with change_id+1 from the first request, returns
an error.
@ -338,7 +360,7 @@ async def test_send_connect_twice_with_clear_change_id_cache(communicator):
@pytest.mark.asyncio
async def test_send_connect_twice_with_clear_change_id_cache_same_change_id_then_first_request(communicator):
async def test_send_connect_twice_with_clear_change_id_cache_same_change_id_then_first_request(communicator, set_config):
"""
Test, that a second request with the change_id from the first request, returns
all data.
@ -360,7 +382,7 @@ async def test_send_connect_twice_with_clear_change_id_cache_same_change_id_then
@pytest.mark.asyncio
async def test_request_changed_elements_no_douple_elements(communicator):
async def test_request_changed_elements_no_douple_elements(communicator, set_config):
"""
Test, that when an elements is changed twice, it is only returned
onces when ask a range of change ids.
@ -386,7 +408,7 @@ async def test_request_changed_elements_no_douple_elements(communicator):
@pytest.mark.asyncio
async def test_send_invalid_get_elements(communicator):
async def test_send_invalid_get_elements(communicator, set_config):
await set_config('general_system_enable_anonymous', True)
await communicator.connect()
@ -399,7 +421,7 @@ async def test_send_invalid_get_elements(communicator):
@pytest.mark.asyncio
async def test_turn_on_autoupdate(communicator):
async def test_turn_on_autoupdate(communicator, set_config):
await set_config('general_system_enable_anonymous', True)
await communicator.connect()
@ -418,7 +440,7 @@ async def test_turn_on_autoupdate(communicator):
@pytest.mark.asyncio
async def test_turn_off_autoupdate(get_communicator):
async def test_turn_off_autoupdate(get_communicator, set_config):
await set_config('general_system_enable_anonymous', True)
communicator = get_communicator('autoupdate=on')
await communicator.connect()

3
tests/unit/motions/test_views.py
View File

@ -21,7 +21,8 @@ class MotionViewSetUpdate(TestCase):
@patch('openslides.motions.views.has_perm')
@patch('openslides.motions.views.config')
def test_simple_update(self, mock_config, mock_has_perm, mock_icd):
self.request.user = 1
self.request.user = MagicMock()
self.request.user.pk = 1
self.request.data.get.return_value = MagicMock()
mock_has_perm.return_value = True

13
tests/unit/users/test_models.py
View File

@ -1,6 +1,8 @@
from unittest import TestCase
from unittest.mock import MagicMock, call, patch
from django.core.exceptions import ObjectDoesNotExist
from openslides.users.models import UserManager
@ -127,7 +129,7 @@ class UserManagerCreateOrResetAdminUser(TestCase):
"""
admin_user = MagicMock()
manager = UserManager()
manager.get_or_create = MagicMock(return_value=(admin_user, False))
manager.get = MagicMock(return_value=(admin_user))
manager.create_or_reset_admin_user()
@ -139,7 +141,7 @@ class UserManagerCreateOrResetAdminUser(TestCase):
"""
admin_user = MagicMock()
manager = UserManager()
manager.get_or_create = MagicMock(return_value=(admin_user, False))
manager.get = MagicMock(return_value=(admin_user))
staff_group = MagicMock(name="Staff")
mock_group.objects.get_or_create = MagicMock(return_value=(staff_group, True))
@ -150,16 +152,15 @@ class UserManagerCreateOrResetAdminUser(TestCase):
self.assertEqual(
admin_user.default_password,
'admin')
admin_user.save.assert_called_once_with()
admin_user.save.assert_called_once_with(skip_autoupdate=True)
@patch('openslides.users.models.User')
def test_return_value(self, mock_user, mock_group, mock_permission):
"""
Tests that the method returns True when a user is created.
"""
admin_user = MagicMock()
manager = UserManager()
manager.get_or_create = MagicMock(return_value=(admin_user, True))
manager.get = MagicMock(side_effect=ObjectDoesNotExist)
manager.model = mock_user
staff_group = MagicMock(name="Staff")
@ -179,7 +180,7 @@ class UserManagerCreateOrResetAdminUser(TestCase):
"""
admin_user = MagicMock(username='admin', last_name='Administrator')
manager = UserManager()
manager.get_or_create = MagicMock(return_value=(admin_user, True))
manager.get = MagicMock(side_effect=ObjectDoesNotExist)
manager.model = mock_user
staff_group = MagicMock(name="Staff")