ag-admin
/
OpenSlides
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #2404 from ostcar/cache2 

Added caching system.
This commit is contained in:
Norman Jäckel 2016-10-01 09:44:57 +02:00 committed by GitHub
parent 2a5bd6d94b 98bab9b358
commit ab164e4e88
44 changed files with 1763 additions and 545 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -26,3 +26,4 @@ dist/*
# Unit test and coverage reports
.coverage
tests/file/*

1
CHANGELOG
View File

@ -42,6 +42,7 @@ Other:
- Fixed bug, that the last change of a config value was not send via autoupdate.
- Added template hooks for plugins (in item detail view and motion poll form).
- Used Django Channels instead of Tornado. Refactoring of the autoupdate process.
- Added new caching system with support for Redis.
Version 2.0 (2016-04-18)

28
DEVELOPMENT.rst
View File

@ -109,3 +109,31 @@ a. Running Angular.js test cases
''''''''''''''''''''''''''''''''
$ node_modules/.bin/karma start tests/karma/karma.conf.js
Installation Openslides in big mode
===================================
1. Install PostgreSQL und redis:
apt-get install postgresql redis-server libpg-dev
TODO: Configure postgresql
2. Install python dependencies
pip install django-redis asgi-redis psycopg2
3. Change settings.py
(See comments in the settings)
The relevant settings are: DATABASES, CHANNEL_LAYERS, CACHES
4. Start one or more workers:
python manage.py runworker
5. Start daphne. Set the DJANGO_SETTINGS_MODULE and the PYTHONPATH
DJANGO_SETTINGS_MODULE=settings PYTHONPATH=personal_data/var/ daphne openslides.asgi:channel_layer

22
openslides/agenda/models.py
View File

@ -23,6 +23,14 @@ class ItemManager(models.Manager):
Customized model manager with special methods for agenda tree and
numbering.
"""
def get_full_queryset(self):
"""
Returns the normal queryset with all items. In the background all
speakers and related items (topics, motions, assignments) are
prefetched from the database.
"""
return self.get_queryset().prefetch_related('speakers', 'content_object')
def get_only_agenda_items(self):
"""
Generator, which yields only agenda items. Skips hidden items.
@ -276,20 +284,6 @@ class Item(RESTModelMixin, models.Model):
def __str__(self):
return self.title
def delete(self, with_children=False):
"""
Delete the Item.
If with_children is True, all children of the item will be deleted as
well. If with_children is False, all children will be children of the
parent of the item.
"""
if not with_children:
for child in self.children.all():
child.parent = self.parent
child.save()
super().delete()
@property
def title(self):
"""

12
openslides/agenda/projector.py
View File

@ -1,5 +1,6 @@
from ..core.config import config
from ..core.exceptions import ProjectorException
from ..utils.collection import CollectionElement
from ..utils.projector import ProjectorElement
from .models import Item
@ -60,10 +61,13 @@ class ListOfSpeakersSlide(ProjectorElement):
# Yield last speakers
yield speaker.user
def need_full_update_for_this(self, collection_element):
# Full update if item changes because then we may have new speakers
# and therefor need new users.
return collection_element.collection_string == Item.get_collection_string()
def get_collection_elements_required_for_this(self, collection_element, config_entry):
output = super().get_collection_elements_required_for_this(collection_element, config_entry)
# Full update if item changes because then we may have new
# candidates and therefor need new users.
if collection_element == CollectionElement.from_values(Item.get_collection_string(), config_entry.get('id')):
output.extend(self.get_requirements_as_collection_elements(config_entry))
return output
class CurrentListOfSpeakersSlide(ProjectorElement):

45
openslides/agenda/views.py
View File

@ -58,26 +58,6 @@ class ItemViewSet(ListModelMixin, RetrieveModelMixin, UpdateModelMixin, GenericV
result = False
return result
def check_object_permissions(self, request, obj):
"""
Checks if the requesting user has permission to see also an
organizational item if it is one.
"""
# TODO: Move this logic to access_permissions.ItemAccessPermissions.
if obj.is_hidden() and not request.user.has_perm('agenda.can_see_hidden_items'):
self.permission_denied(request)
def get_queryset(self):
"""
Filters organizational items if the user has no permission to see them.
"""
# TODO: Move this logic to access_permissions.ItemAccessPermissions.
queryset = super().get_queryset()
if not self.request.user.has_perm('agenda.can_see_hidden_items'):
pk_list = [item.pk for item in Item.objects.get_only_agenda_items()]
queryset = queryset.filter(pk__in=pk_list)
return queryset
@detail_route(methods=['POST', 'DELETE'])
def manage_speaker(self, request, pk=None):
"""
@ -243,31 +223,6 @@ class ItemViewSet(ListModelMixin, RetrieveModelMixin, UpdateModelMixin, GenericV
# Initiate response.
return Response({'detail': _('List of speakers successfully sorted.')})
@list_route(methods=['get', 'put'])
def tree(self, request):
"""
Returns or sets the agenda tree.
"""
if request.method == 'PUT':
if not (request.user.has_perm('agenda.can_manage') and
request.user.has_perm('agenda.can_see_hidden_items')):
self.permission_denied(request)
try:
tree = request.data['tree']
except KeyError as error:
response = Response({'detail': 'Agenda tree is missing.'}, status=400)
else:
try:
Item.objects.set_tree(tree)
except ValueError as error:
response = Response({'detail': str(error)}, status=400)
else:
response = Response({'detail': 'Agenda tree successfully updated.'})
else:
# request.method == 'GET'
response = Response(Item.objects.get_tree())
return response
@list_route(methods=['post'])
def numbering(self, request):
"""

31
openslides/assignments/models.py
View File

@ -1,7 +1,7 @@
from collections import OrderedDict
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericRelation
from django.db import models
from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_noop
@ -16,6 +16,7 @@ from openslides.poll.models import (
CollectDefaultVotesMixin,
PublishPollMixin,
)
from openslides.utils.autoupdate import inform_changed_data
from openslides.utils.exceptions import OpenSlidesError
from openslides.utils.models import RESTModelMixin
from openslides.utils.search import user_name_helper
@ -50,12 +51,30 @@ class AssignmentRelatedUser(RESTModelMixin, models.Model):
return self.assignment
class AssignmentManager(models.Manager):
"""
Customized model manager to support our get_full_queryset method.
"""
def get_full_queryset(self):
"""
Returns the normal queryset with all assignments. In the background
all related users (candidates), the related agenda item and all
polls are prefetched from the database.
"""
return self.get_queryset().prefetch_related(
'related_users',
'agenda_items',
'polls')
class Assignment(RESTModelMixin, models.Model):
"""
Model for assignments.
"""
access_permissions = AssignmentAccessPermissions()
objects = AssignmentManager()
PHASE_SEARCH = 0
PHASE_VOTING = 1
PHASE_FINISHED = 2
@ -111,6 +130,10 @@ class Assignment(RESTModelMixin, models.Model):
Tags for the assignment.
"""
# In theory there could be one then more agenda_item. But we support only
# one. See the property agenda_item.
agenda_items = GenericRelation(Item, related_name='assignments')
class Meta:
default_permissions = ()
permissions = (
@ -187,6 +210,7 @@ class Assignment(RESTModelMixin, models.Model):
Delete the connection from the assignment to the user.
"""
self.assignment_related_users.filter(user=user).delete()
inform_changed_data(self)
def set_phase(self, phase):
"""
@ -290,8 +314,9 @@ class Assignment(RESTModelMixin, models.Model):
"""
Returns the related agenda item.
"""
content_type = ContentType.objects.get_for_model(self)
return Item.objects.get(object_id=self.pk, content_type=content_type)
# We support only one agenda item so just return the first element of
# the queryset.
return self.agenda_items.all()[0]
@property
def agenda_item_id(self):

8
openslides/assignments/projector.py
View File

@ -1,4 +1,5 @@
from ..core.exceptions import ProjectorException
from ..utils.collection import CollectionElement
from ..utils.projector import ProjectorElement
from .models import Assignment, AssignmentPoll
@ -46,7 +47,10 @@ class AssignmentSlide(ProjectorElement):
for option in poll.options.all():
yield option.candidate
def need_full_update_for_this(self, collection_element):
def get_collection_elements_required_for_this(self, collection_element, config_entry):
output = super().get_collection_elements_required_for_this(collection_element, config_entry)
# Full update if assignment changes because then we may have new
# candidates and therefor need new users.
return collection_element.collection_string == Assignment.get_collection_string()
if collection_element == CollectionElement.from_values(Assignment.get_collection_string(), config_entry.get('id')):
output.extend(self.get_requirements_as_collection_elements(config_entry))
return output

8
openslides/core/access_permissions.py
View File

@ -84,7 +84,13 @@ class ConfigAccessPermissions(BaseAccessPermissions):
Returns the serlialized config data.
"""
from .config import config
from .models import ConfigStore
# Attention: The format of this response has to be the same as in
# the retrieve method of ConfigViewSet.
return {'key': instance.key, 'value': config[instance.key]}
if isinstance(instance, ConfigStore):
result = {'key': instance.key, 'value': config[instance.key]}
else:
# It is possible, that the caching system already sends the correct data as "instance".
result = instance
return result

10
openslides/core/apps.py
View File

@ -16,7 +16,6 @@ class CoreAppConfig(AppConfig):
from django.db.models import signals
from openslides.core.config import config
from openslides.core.signals import post_permission_creation
from openslides.utils.autoupdate import inform_changed_data_receiver, inform_deleted_data_receiver
from openslides.utils.rest_api import router
from openslides.utils.search import index_add_instance, index_del_instance
from .config_variables import get_config_variables
@ -45,15 +44,6 @@ class CoreAppConfig(AppConfig):
router.register(self.get_model('Tag').get_collection_string(), TagViewSet)
router.register(self.get_model('ConfigStore').get_collection_string(), ConfigViewSet, 'config')
# Update data when any model of any installed app is saved or deleted.
# TODO: Test if the m2m_changed signal is also needed.
signals.post_save.connect(
inform_changed_data_receiver,
dispatch_uid='inform_changed_data_receiver')
signals.post_delete.connect(
inform_deleted_data_receiver,
dispatch_uid='inform_deleted_data_receiver')
# Update the search when a model is saved or deleted
signals.post_save.connect(
index_add_instance,

19
openslides/core/migrations/0005_auto_20160918_2104.py
View File

@ -2,19 +2,10 @@
# Generated by Django 1.10.1 on 2016-09-18 19:04
from __future__ import unicode_literals
from django.db import migrations, models
from openslides.utils.autoupdate import (
inform_changed_data_receiver,
inform_deleted_data_receiver,
)
from django.db import migrations
def move_custom_slides_to_topics(apps, schema_editor):
# Disconnect autoupdate. We do not want to trigger it here.
models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver')
models.signals.post_save.disconnect(dispatch_uid='inform_deleted_data_receiver')
# We get the model from the versioned app registry;
# if we directly import it, it will be the wrong version.
ContentType = apps.get_model('contenttypes', 'ContentType')
@ -39,14 +30,6 @@ def move_custom_slides_to_topics(apps, schema_editor):
CustomSlide.objects.all().delete()
content_type_custom_slide.delete()
# Reconnect autoupdate.
models.signals.post_save.connect(
inform_changed_data_receiver,
dispatch_uid='inform_changed_data_receiver')
models.signals.post_delete.connect(
inform_deleted_data_receiver,
dispatch_uid='inform_deleted_data_receiver')
class Migration(migrations.Migration):

82
openslides/core/models.py
View File

@ -15,6 +15,19 @@ from .access_permissions import (
from .exceptions import ProjectorException
class ProjectorManager(models.Manager):
"""
Customized model manager to support our get_full_queryset method.
"""
def get_full_queryset(self):
"""
Returns the normal queryset with all projectors. In the background
projector defaults are prefetched from the database.
"""
return self.get_queryset().prefetch_related(
'projectiondefaults')
class Projector(RESTModelMixin, models.Model):
"""
Model for all projectors. At the moment we support only one projector,
@ -57,6 +70,8 @@ class Projector(RESTModelMixin, models.Model):
"""
access_permissions = ProjectorAccessPermissions()
objects = ProjectorManager()
config = JSONField()
scale = models.IntegerField(default=0)
@ -131,50 +146,35 @@ class Projector(RESTModelMixin, models.Model):
if element is not None:
yield from element.get_requirements(value)
def collection_element_is_shown(self, collection_element):
def get_collection_elements_required_for_this(self, collection_element):
"""
Returns True if this collection element is shown on this projector.
Returns an iterable of CollectionElements that have to be sent to this
projector according to the given collection_element and information.
"""
for requirement in self.get_all_requirements():
if (requirement.get_collection_string() == collection_element.collection_string and
requirement.pk == collection_element.id):
result = True
break
from .config import config
output = []
changed_fields = collection_element.information.get('changed_fields', [])
if (collection_element.collection_string == self.get_collection_string() and
changed_fields and
'config' not in changed_fields):
# Projector model changed without changeing the projector config. So we just send this data.
output.append(collection_element)
else:
result = False
return result
@classmethod
def get_projectors_that_show_this(cls, collection_element):
"""
Returns a list of the projectors that show this collection element.
"""
result = []
for projector in cls.objects.all():
if projector.collection_element_is_shown(collection_element):
result.append(projector)
return result
def need_full_update_for_this(self, collection_element):
"""
Returns True if this projector needs to be updated with all
instances as defined in get_all_requirements() because one active
projector element requires this.
"""
# Get all elements from all apps.
elements = {}
for element in ProjectorElement.get_all():
elements[element.name] = element
for key, value in self.config.items():
element = elements.get(value['name'])
if element is not None and element.need_full_update_for_this(collection_element):
result = True
break
else:
result = False
return result
# It is necessary to parse all active projector elements to check whether they require some data.
this_projector = collection_element.collection_string == self.get_collection_string() and collection_element.id == self.pk
collection_element.information['this_projector'] = this_projector
elements = {}
for element in ProjectorElement.get_all():
elements[element.name] = element
for key, value in self.config.items():
element = elements.get(value['name'])
if element is not None:
output.extend(element.get_collection_elements_required_for_this(collection_element, value))
# If config changed, send also this to the projector.
if collection_element.collection_string == config.get_collection_string():
output.append(collection_element)
return output
class ProjectionDefault(RESTModelMixin, models.Model):

3
openslides/core/signals.py
View File

@ -20,8 +20,7 @@ def delete_django_app_permissions(sender, **kwargs):
Q(app_label='auth') |
Q(app_label='contenttypes') |
Q(app_label='sessions'))
for permission in Permission.objects.filter(content_type__in=contenttypes):
permission.delete()
Permission.objects.filter(content_type__in=contenttypes).delete()
def create_builtin_projection_defaults(**kwargs):

23
openslides/core/views.py
View File

@ -18,6 +18,7 @@ from django.utils.timezone import now
from openslides import __version__ as version
from openslides.utils import views as utils_views
from openslides.utils.collection import Collection, CollectionElement
from openslides.utils.plugins import (
get_plugin_description,
get_plugin_verbose_name,
@ -41,7 +42,7 @@ from .access_permissions import (
)
from .config import config
from .exceptions import ConfigError, ConfigNotFound
from .models import ChatMessage, ProjectionDefault, Projector, Tag
from .models import ChatMessage, ConfigStore, ProjectionDefault, Projector, Tag
# Special Django views
@ -606,22 +607,26 @@ class ConfigViewSet(ViewSet):
def list(self, request):
"""
Lists all config variables. Everybody can see them.
Lists all config variables.
"""
return Response([{'key': key, 'value': value} for key, value in config.items()])
collection = Collection(config.get_collection_string())
return Response(collection.as_list_for_user(request.user))
def retrieve(self, request, *args, **kwargs):
"""
Retrieves a config variable. Everybody can see it.
Retrieves a config variable.
"""
key = kwargs['pk']
collection_element = CollectionElement.from_values(config.get_collection_string(), key)
try:
value = config[key]
except ConfigNotFound:
content = collection_element.as_dict_for_user(request.user)
except ConfigStore.DoesNotExist:
raise Http404
# Attention: The format of this response has to be the same as in
# the get_full_data method of ConfigAccessPermissions.
return Response({'key': key, 'value': value})
if content is None:
# If content is None, the user has no permissions to see the item.
# See ConfigAccessPermissions or rather its parent class.
self.permission_denied()
return Response(content)
def update(self, request, *args, **kwargs):
"""

7
openslides/mediafiles/views.py
View File

@ -15,13 +15,6 @@ class MediafileViewSet(ModelViewSet):
access_permissions = MediafileAccessPermissions()
queryset = Mediafile.objects.all()
def get_queryset(self):
queryset = super().get_queryset()
user = self.request.user
if not user.has_perm('mediafiles.can_see_private'):
queryset = queryset.filter(private=False)
return queryset
def check_view_permissions(self):
"""
Returns True if the user has required permissions.

13
openslides/motions/migrations/0004_auto_20160907_2343.py
View File

@ -5,16 +5,11 @@ from __future__ import unicode_literals
import django.db.models.deletion
from django.db import migrations, models
from openslides.utils.autoupdate import inform_changed_data_receiver
def change_label_of_state(apps, schema_editor):
"""
Changes the label of former state "commited a bill" to "refered to committee".
"""
# Disconnect autoupdate. We do not want to trigger it here.
models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver')
# We get the model from the versioned app registry;
# if we directly import it, it will be the wrong version.
State = apps.get_model('motions', 'State')
@ -29,19 +24,11 @@ def change_label_of_state(apps, schema_editor):
state.action_word = 'Refer to committee'
state.save()
# Reconnect autoupdate.
models.signals.post_save.connect(
inform_changed_data_receiver,
dispatch_uid='inform_changed_data_receiver')
def add_recommendation_labels(apps, schema_editor):
"""
Adds recommendation labels to some of the built-in states.
"""
# Disconnect autoupdate. We do not want to trigger it here.
models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver')
# We get the model from the versioned app registry;
# if we directly import it, it will be the wrong version.
State = apps.get_model('motions', 'State')

54
openslides/motions/models.py
View File

@ -1,5 +1,5 @@
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericRelation
from django.db import models
from django.db.models import Max
from django.utils import formats
@ -28,14 +28,38 @@ from .access_permissions import (
from .exceptions import WorkflowError
class MotionManager(models.Manager):
"""
Customized model manager to support our get_full_queryset method.
"""
def get_full_queryset(self):
"""
Returns the normal queryset with all motions. In the background we
join and prefetch all related models.
"""
return (self.get_queryset()
.select_related('active_version')
.prefetch_related(
'versions',
'agenda_items',
'log_messages',
'polls',
'attachments',
'tags',
'submitters',
'supporters'))
class Motion(RESTModelMixin, models.Model):
"""
The Motion Class.
Model for motions.
This class is the main entry point to all other classes related to a motion.
"""
access_permissions = MotionAccessPermissions()
objects = MotionManager()
active_version = models.ForeignKey(
'MotionVersion',
on_delete=models.SET_NULL,
@ -134,6 +158,10 @@ class Motion(RESTModelMixin, models.Model):
Configurable fields for comments. Contains a list of strings.
"""
# In theory there could be one then more agenda_item. But we support only
# one. See the property agenda_item.
agenda_items = GenericRelation(Item, related_name='motions')
class Meta:
default_permissions = ()
permissions = (
@ -519,8 +547,9 @@ class Motion(RESTModelMixin, models.Model):
"""
Returns the related agenda item.
"""
content_type = ContentType.objects.get_for_model(self)
return Item.objects.get(object_id=self.pk, content_type=content_type)
# We support only one agenda item so just return the first element of
# the queryset.
return self.agenda_items.all()[0]
@property
def agenda_item_id(self):
@ -940,12 +969,29 @@ class State(RESTModelMixin, models.Model):
return self.workflow
class WorkflowManager(models.Manager):
"""
Customized model manager to support our get_full_queryset method.
"""
def get_full_queryset(self):
"""
Returns the normal queryset with all workflows. In the background
the first state is joined and all states and next states are
prefetched from the database.
"""
return (self.get_queryset()
.select_related('first_state')
.prefetch_related('states', 'states__next_states'))
class Workflow(RESTModelMixin, models.Model):
"""
Defines a workflow for a motion.
"""
access_permissions = WorkflowAccessPermissions()
objects = WorkflowManager()
name = models.CharField(max_length=255)
"""A string representing the workflow."""

8
openslides/motions/projector.py
View File

@ -1,4 +1,5 @@
from ..core.exceptions import ProjectorException
from ..utils.collection import CollectionElement
from ..utils.projector import ProjectorElement
from .models import Motion
@ -26,10 +27,13 @@ class MotionSlide(ProjectorElement):
yield from motion.submitters.all()
yield from motion.supporters.all()
def need_full_update_for_this(self, collection_element):
def get_collection_elements_required_for_this(self, collection_element, config_entry):
output = super().get_collection_elements_required_for_this(collection_element, config_entry)
# Full update if motion changes because then we may have new
# submitters or supporters and therefor need new users.
#
# Add some logic here if we support live changing of workflows later.
#
return collection_element.collection_string == Motion.get_collection_string()
if collection_element == CollectionElement.from_values(Motion.get_collection_string(), config_entry.get('id')):
output.extend(self.get_requirements_as_collection_elements(config_entry))
return output

21
openslides/motions/views.py
View File

@ -77,27 +77,6 @@ class MotionViewSet(ModelViewSet):
result = False
return result
def list(self, request, *args, **kwargs):
"""
Customized view endpoint to list all motions.
Hides non public comment fields for some users.
"""
response = super().list(request, *args, **kwargs)
for i, motion in enumerate(response.data):
response.data[i] = self.get_access_permissions().get_restricted_data(motion, self.request.user)
return response
def retrieve(self, request, *args, **kwargs):
"""
Customized view endpoint to retrieve a motion.
Hides non public comment fields for some users.
"""
response = super().retrieve(request, *args, **kwargs)
response.data = self.get_access_permissions().get_restricted_data(response.data, self.request.user)
return response
def create(self, request, *args, **kwargs):
"""
Customized view endpoint to create a new motion.

26
openslides/topics/models.py
View File

@ -1,4 +1,4 @@
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericRelation
from django.db import models
from ..agenda.models import Item
@ -7,16 +7,35 @@ from ..utils.models import RESTModelMixin
from .access_permissions import TopicAccessPermissions
class TopicManager(models.Manager):
"""
Customized model manager to support our get_full_queryset method.
"""
def get_full_queryset(self):
"""
Returns the normal queryset with all topics. In the background all
attachments and the related agenda item are prefetched from the
database.
"""
return self.get_queryset().prefetch_related('attachments', 'agenda_items')
class Topic(RESTModelMixin, models.Model):
"""
Model for slides with custom content. Used to be called custom slide.
"""
access_permissions = TopicAccessPermissions()
objects = TopicManager()
title = models.CharField(max_length=256)
text = models.TextField(blank=True)
attachments = models.ManyToManyField(Mediafile, blank=True)
# In theory there could be one then more agenda_item. But we support only
# one. See the property agenda_item.
agenda_items = GenericRelation(Item, related_name='topics')
class Meta:
default_permissions = ()
@ -28,8 +47,9 @@ class Topic(RESTModelMixin, models.Model):
"""
Returns the related agenda item.
"""
content_type = ContentType.objects.get_for_model(self)
return Item.objects.get(object_id=self.pk, content_type=content_type)
# We support only one agenda item so just return the first element of
# the queryset.
return self.agenda_items.all()[0]
@property
def agenda_item_id(self):

10
openslides/users/access_permissions.py
View File

@ -15,15 +15,9 @@ class UserAccessPermissions(BaseAccessPermissions):
"""
Returns different serializer classes with respect user's permissions.
"""
from .serializers import UserCanSeeSerializer, UserCanSeeExtraSerializer, UserFullSerializer
from .serializers import UserFullSerializer
if (user is None or (user.has_perm('users.can_see_extra_data') and user.has_perm('users.can_manage'))):
serializer_class = UserFullSerializer
elif user.has_perm('users.can_see_extra_data'):
serializer_class = UserCanSeeExtraSerializer
else:
serializer_class = UserCanSeeSerializer
return serializer_class
return UserFullSerializer
def get_restricted_data(self, full_data, user):
"""

12
openslides/users/migrations/0004_groups.py
View File

@ -2,9 +2,7 @@
# Generated by Django 1.9.7 on 2016-08-01 14:54
from __future__ import unicode_literals
from django.db import migrations, models
from openslides.utils.autoupdate import inform_changed_data_receiver
from django.db import migrations
def migrate_groups_and_user_permissions(apps, schema_editor):
@ -20,9 +18,6 @@ def migrate_groups_and_user_permissions(apps, schema_editor):
- the name of the first group is 'Guests',
- the name of the second group is 'Registered users'.
"""
# Disconnect autoupdate. We do not want to trigger it here.
models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver')
User = apps.get_model('users', 'User')
Group = apps.get_model('auth', 'Group')
@ -52,11 +47,6 @@ def migrate_groups_and_user_permissions(apps, schema_editor):
for permission in group_registered.permissions.all():
group.permissions.add(permission)
# Reconnect autoupdate.
models.signals.post_save.connect(
inform_changed_data_receiver,
dispatch_uid='inform_changed_data_receiver')
class Migration(migrations.Migration):

9
openslides/users/models.py
View File

@ -21,8 +21,15 @@ from .access_permissions import UserAccessPermissions
class UserManager(BaseUserManager):
"""
Customized manager that creates new users only with a password and a
username.
username. It also supports our get_full_queryset method.
"""
def get_full_queryset(self):
"""
Returns the normal queryset with all users. In the background all
groups are prefetched from the database.
"""
return self.get_queryset().prefetch_related('groups')
def create_user(self, username, password, **kwargs):
"""
Creates a new user only with a password and a username.

87
openslides/users/serializers.py
View File

@ -12,79 +12,24 @@ from ..utils.rest_api import (
from .models import Group, User
USERCANSEESERIALIZER_FIELDS = (
'id',
'username',
'title',
'first_name',
'last_name',
'structure_level',
'number',
'about_me',
'groups',
'is_committee',
)
'id',
'username',
'title',
'first_name',
'last_name',
'structure_level',
'number',
'about_me',
'groups',
'is_present',
'is_committee',
)
class UserCanSeeSerializer(ModelSerializer):
"""
Serializer for users.models.User objects to be used by users who have
only the permission to see users and to change some date of theirselfs.
Attention: Viewset has to ensure that a user can update only himself.
"""
class Meta:
model = User
fields = USERCANSEESERIALIZER_FIELDS
read_only_fields = (
'number',
'groups',
'is_comittee',
)
USERCANSEEEXTRASERIALIZER_FIELDS = (
'id',
'is_present',
'username',
'title',
'first_name',
'last_name',
'number',
'structure_level',
'about_me',
'comment',
'groups',
'is_active',
'is_committee',
)
class UserCanSeeExtraSerializer(ModelSerializer):
"""
Serializer for users.models.User objects to be used by users who have
the permission to see users with extra data and to change some date of
theirselfs.
Attention: Viewset has to ensure that a user can update only himself.
"""
groups = IdPrimaryKeyRelatedField(
many=True,
queryset=Group.objects.exclude(pk=1),
help_text=ugettext_lazy('The groups this user belongs to. A user will '
'get all permissions granted to each of '
'his/her groups.'))
class Meta:
model = User
fields = USERCANSEEEXTRASERIALIZER_FIELDS
read_only_fields = (
'is_present',
'number',
'comment',
'groups',
'is_comittee',
'is_active',
)
USERCANSEEEXTRASERIALIZER_FIELDS = USERCANSEESERIALIZER_FIELDS + (
'comment',
'is_active',
)
class UserFullSerializer(ModelSerializer):

17
openslides/users/views.py
View File

@ -18,7 +18,7 @@ from ..utils.views import APIView, PDFView
from .access_permissions import UserAccessPermissions
from .models import Group, User
from .pdf import users_passwords_to_pdf, users_to_pdf
from .serializers import GroupSerializer, UserFullSerializer
from .serializers import GroupSerializer
# Viewsets for the REST API
@ -49,19 +49,6 @@ class UserViewSet(ModelViewSet):
result = False
return result
def get_serializer_class(self):
"""
Returns different serializer classes with respect to action.
"""
if self.action in ('create', 'partial_update', 'update'):
# Return the UserFullSerializer for edit requests.
serializer_class = UserFullSerializer
else:
# Return different serializers according to user permsissions via
# access permissions class.
serializer_class = super().get_serializer_class()
return serializer_class
def update(self, request, *args, **kwargs):
"""
Customized view endpoint to update an user.
@ -138,7 +125,7 @@ class GroupViewSet(ModelViewSet):
partial_update, update and destroy.
"""
metadata_class = GroupViewSetMetadata
queryset = Group.objects.all()
queryset = Group.objects.prefetch_related('permissions', 'permissions__content_type')
serializer_class = GroupSerializer
def check_view_permissions(self):

8
openslides/utils/access_permissions.py
View File

@ -68,7 +68,7 @@ class BaseAccessPermissions(object, metaclass=SignalConnectMetaClass):
Hint: You should override this method if your get_serializer_class()
method returns different serializers for different users or if you
have access restrictions in your view or viewset in methods like
retrieve(), list() or check_object_permissions().
retrieve() or list().
"""
if self.check_permissions(user):
data = full_data
@ -78,8 +78,8 @@ class BaseAccessPermissions(object, metaclass=SignalConnectMetaClass):
def get_projector_data(self, full_data):
"""
Returns the serialized data for the projector. Returns None if has no
access to this specific data. Returns reduced data if the user has
limited access. Default: Returns full data.
Returns the serialized data for the projector. Returns None if the
user has no access to this specific data. Returns reduced data if
the user has limited access. Default: Returns full data.
"""
return full_data

135
openslides/utils/autoupdate.py
View File

@ -11,7 +11,7 @@ from ..core.config import config
from ..core.models import Projector
from ..users.auth import AnonymousUser
from ..users.models import User
from .collection import CollectionElement
from .collection import Collection, CollectionElement
def get_logged_in_users():
@ -23,21 +23,6 @@ def get_logged_in_users():
return User.objects.exclude(session=None).filter(session__expire_date__gte=timezone.now()).distinct()
def get_projector_element_data(projector):
"""
Returns a list of dicts that are required for a specific projector.
The argument projector has to be a projector instance.
"""
output = []
for requirement in projector.get_all_requirements():
required_collection_element = CollectionElement.from_instance(requirement)
element_dict = required_collection_element.as_autoupdate_for_projector()
if element_dict is not None:
output.append(element_dict)
return output
@channel_session_user_from_http
def ws_add_site(message):
"""
@ -80,15 +65,14 @@ def ws_add_projector(message, projector_id):
Group('projector-{}'.format(projector_id)).add(message.reply_channel)
# Send all elements that are on the projector.
output = get_projector_element_data(projector)
output = []
for requirement in projector.get_all_requirements():
required_collection_element = CollectionElement.from_instance(requirement)
output.append(required_collection_element.as_autoupdate_for_projector())
# Send all config elements.
for key, value in config.items():
output.append({
'collection': config.get_collection_string(),
'id': key,
'action': 'changed',
'data': {'key': key, 'value': value}})
collection = Collection(config.get_collection_string())
output.extend(collection.as_autoupdate_for_projector())
# Send the projector instance.
collection_element = CollectionElement.from_instance(projector)
@ -107,64 +91,33 @@ def ws_disconnect_projector(message, projector_id):
def send_data(message):
"""
Informs all users about changed data.
Informs all site users and projector clients about changed data.
"""
collection_element = CollectionElement.from_values(**message)
# Loop over all logged in users and the anonymous user.
# Loop over all logged in site users and the anonymous user and send changed data.
for user in itertools.chain(get_logged_in_users(), [AnonymousUser()]):
channel = Group('user-{}'.format(user.id))
output = collection_element.as_autoupdate_for_user(user)
if output is None:
# There are no data for the user so he can't see the object. Skip him.
continue
channel.send({'text': json.dumps([output])})
output = [collection_element.as_autoupdate_for_user(user)]
channel.send({'text': json.dumps(output)})
# Get the projector elements where data have to be sent and if whole projector
# has to be updated.
if collection_element.collection_string == config.get_collection_string():
# Config elements are always send to each projector
projectors = Projector.objects.all()
send_all = None # The decission is done later
elif collection_element.collection_string == Projector.get_collection_string():
# Update a projector, when the projector element is updated.
projectors = [collection_element.get_instance()]
send_all = True
elif collection_element.is_deleted():
projectors = Projector.objects.all()
send_all = False
else:
# Other elements are only send to the projector they are currently shown
projectors = Projector.get_projectors_that_show_this(collection_element)
send_all = None # The decission is done later
broadcast_id = config['projector_broadcast']
if broadcast_id > 0:
projectors = Projector.objects.all() # Also the broadcasted projector should get his data
send_all = True
broadcast_projector_data = get_projector_element_data(Projector.objects.get(pk=broadcast_id))
broadcast_projector_data.append(CollectionElement.from_values(
collection_string=Projector.get_collection_string(), id=broadcast_id).as_autoupdate_for_projector())
else:
broadcast_projector_data = None
for projector in projectors:
if send_all is None:
send_all = projector.need_full_update_for_this(collection_element)
if send_all:
if broadcast_projector_data is None:
output = get_projector_element_data(projector)
else:
output = broadcast_projector_data
# Loop over all projectors and send data that they need.
for projector in Projector.objects.all():
if collection_element.is_deleted():
output = [collection_element.as_autoupdate_for_projector()]
else:
output = []
output.append(collection_element.as_autoupdate_for_projector())
collection_elements = projector.get_collection_elements_required_for_this(collection_element)
output = [collection_element.as_autoupdate_for_projector() for collection_element in collection_elements]
if output:
Group('projector-{}'.format(projector.pk)).send(
{'text': json.dumps(output)})
def inform_changed_data(instance, is_deleted=False):
def inform_changed_data(instance, information=None):
"""
Informs the autoupdate system and the caching system about the creation or
update of an element.
"""
try:
root_instance = instance.get_root_rest_element()
except AttributeError:
@ -173,29 +126,37 @@ def inform_changed_data(instance, is_deleted=False):
else:
collection_element = CollectionElement.from_instance(
root_instance,
is_deleted=is_deleted and instance == root_instance)
# If currently there is an open database transaction, then the following
# function is only called, when the transaction is commited. If there
# is currently no transaction, then the function is called immediately.
def send_autoupdate():
try:
Channel('autoupdate.send_data').send(collection_element.as_channels_message())
except ChannelLayer.ChannelFull:
pass
transaction.on_commit(send_autoupdate)
information=information)
# If currently there is an open database transaction, then the
# send_autoupdate function is only called, when the transaction is
# commited. If there is currently no transaction, then the function
# is called immediately.
transaction.on_commit(lambda: send_autoupdate(collection_element))
def inform_changed_data_receiver(sender, instance, **kwargs):
def inform_deleted_data(collection_string, id, information=None):
"""
Receiver for the inform_changed_data function to use in a signal.
Informs the autoupdate system and the caching system about the deletion of
an element.
"""
inform_changed_data(instance)
collection_element = CollectionElement.from_values(
collection_string=collection_string,
id=id,
deleted=True,
information=information)
# If currently there is an open database transaction, then the
# send_autoupdate function is only called, when the transaction is
# commited. If there is currently no transaction, then the function
# is called immediately.
transaction.on_commit(lambda: send_autoupdate(collection_element))
def inform_deleted_data_receiver(sender, instance, **kwargs):
def send_autoupdate(collection_element):
"""
Receiver for the inform_changed_data function to use in a signal.
Helper function, that sends a collection_element through a channel to the
autoupdate system.
"""
inform_changed_data(instance, is_deleted=True)
try:
Channel('autoupdate.send_data').send(collection_element.as_channels_message())
except ChannelLayer.ChannelFull:
pass

418
openslides/utils/collection.py
View File

@ -1,25 +1,41 @@
from django.apps import apps
from django.core.cache import cache, caches
class CollectionElement:
@classmethod
def from_instance(cls, instance, is_deleted=False):
def from_instance(cls, instance, deleted=False, information=None):
"""
Returns a collection element from a database instance.
This will also update the instance in the cache.
If deleted is set to True, the element is deleted from the cache.
"""
return cls(instance=instance, is_deleted=is_deleted)
return cls(instance=instance, deleted=deleted, information=information)
@classmethod
def from_values(cls, collection_string, id, is_deleted=False):
def from_values(cls, collection_string, id, deleted=False, full_data=None, information=None):
"""
Returns a collection element from a collection_string and an id.
"""
return cls(collection_string=collection_string, id=id, is_deleted=is_deleted)
def __init__(self, instance=None, is_deleted=False, collection_string=None, id=None):
If deleted is set to True, the element is deleted from the cache.
With the argument full_data, the content of the CollectionElement can be set.
It has to be a dict in the format that is used be access_permission.get_full_data().
"""
return cls(collection_string=collection_string, id=id, deleted=deleted,
full_data=full_data, information=information)
def __init__(self, instance=None, deleted=False, collection_string=None, id=None,
full_data=None, information=None):
"""
Do not use this. Use the methods from_instance() or from_values().
"""
self.instance = instance
self.deleted = deleted
self.full_data = full_data
self.information = information or {}
if instance is not None:
self.collection_string = instance.get_collection_string()
self.id = instance.pk
@ -31,62 +47,83 @@ class CollectionElement:
'Invalid state. Use CollectionElement.from_instance() or '
'CollectionElement.from_values() but not CollectionElement() '
'directly.')
self.instance = instance
self.deleted = is_deleted
if self.is_deleted():
# Delete the element from the cache, if self.is_deleted() is True:
self.delete_from_cache()
elif instance is not None:
# If this element is created via instance and the instance is not deleted
# then update the cache.
self.save_to_cache()
def __eq__(self, collection_element):
"""
Compares two collection_elements.
Two collection elements are equal, if they have the same collection_string
and id.
"""
return (self.collection_string == collection_element.collection_string and
self.id == collection_element.id)
def as_channels_message(self):
"""
Returns a dictonary that can be used to send the object through the
channels system.
"""
return {
channel_message = {
'collection_string': self.collection_string,
'id': self.id,
'is_deleted': self.is_deleted()}
'deleted': self.is_deleted()}
if self.information is not None:
channel_message['information'] = self.information
return channel_message
def as_autoupdate(self, method, *args):
"""
Only for internal use. Do not use it directly. Use as_autoupdate_for_user()
or as_autoupdate_for_projector().
"""
output = {
'collection': self.collection_string,
'id': self.id,
'action': 'deleted' if self.is_deleted() else 'changed',
}
if not self.is_deleted():
data = getattr(self.get_access_permissions(), method)(
self.get_full_data(),
*args)
if data is None:
# The user is not allowed to see this element. Set action to deleted.
output['action'] = 'deleted'
else:
output['data'] = data
return output
def as_autoupdate_for_user(self, user):
"""
Returns a dict that can be sent through the autoupdate system for a site
user.
Returns None if the user can not see the element.
"""
output = {
'collection': self.collection_string,
'id': self.id,
'action': 'deleted' if self.is_deleted() else 'changed',
}
if not self.is_deleted():
data = self.get_access_permissions().get_restricted_data(
self.get_full_data(), user)
if data is None:
# The user is not allowed to see this element. Reset output to None.
output = None
else:
output['data'] = data
return output
return self.as_autoupdate(
'get_restricted_data',
user)
def as_autoupdate_for_projector(self):
"""
Returns a dict that can be sent through the autoupdate system for the
projector.
Returns None if the projector can not see the element.
"""
output = {
'collection': self.collection_string,
'id': self.id,
'action': 'deleted' if self.is_deleted() else 'changed',
}
if not self.is_deleted():
data = self.get_access_permissions().get_projector_data(
self.get_full_data())
if data is None:
# The user is not allowed to see this element. Reset output to None.
output = None
else:
output['data'] = data
return output
return self.as_autoupdate(
'get_projector_data')
def as_dict_for_user(self, user):
"""
Returns a dict with the data for a user. Can be used for the rest api.
"""
return self.get_access_permissions().get_restricted_data(
self.get_full_data(),
user)
def get_model(self):
"""
@ -102,8 +139,21 @@ class CollectionElement:
"""
if self.is_deleted():
raise RuntimeError("The collection element is deleted.")
if self.instance is None:
self.instance = self.get_model().objects.get(pk=self.id)
# The config instance has to be get from the config element, because
# some config values are not in the db.
from openslides.core.config import config
if (self.collection_string == config.get_collection_string() and
isinstance(self.id, str)):
self.instance = {'key': self.id, 'value': config[self.id]}
else:
model = self.get_model()
try:
query = model.objects.get_full_queryset()
except AttributeError:
query = model.objects
self.instance = query.get(pk=self.id)
return self.instance
def get_access_permissions(self):
@ -117,7 +167,20 @@ class CollectionElement:
Returns the full_data of this collection_element from with all other
dics can be generated.
"""
return self.get_access_permissions().get_full_data(self.get_instance())
# If the full_data is already loaded, return it
# If there is a db_instance, use it to get the full_data
# else: try to use the cache.
# If there is no value in the cach, get the content from the db and save
# it to the cache.
if self.full_data is None and self.instance is None:
# Use the cache version if self.instance is not set.
# After this line full_data can be None, if the element is not in the cache.
self.full_data = cache.get(self.get_cache_key())
if self.full_data is None:
self.full_data = self.get_access_permissions().get_full_data(self.get_instance())
self.save_to_cache()
return self.full_data
def is_deleted(self):
"""
@ -125,6 +188,217 @@ class CollectionElement:
"""
return self.deleted
def get_cache_key(self):
"""
Returns a string that is used as cache key for a single instance.
"""
return get_single_element_cache_key(self.collection_string, self.id)
def delete_from_cache(self):
"""
Delets an element from the cache.
Does nothing if the element is not in the cache.
"""
# Deletes the element from the cache.
cache.delete(self.get_cache_key())
# Delete the id of the instance of the instance list
Collection(self.collection_string).delete_id_from_cache(self.id)
def save_to_cache(self):
"""
Add or update the element to the cache.
"""
# Set the element to the cache.
cache.set(self.get_cache_key(), self.get_full_data())
# Add the id of the element to the collection
Collection(self.collection_string).add_id_to_cache(self.id)
class Collection:
"""
Represents all elements of one collection.
"""
def __init__(self, collection_string):
self.collection_string = collection_string
def get_cache_key(self, raw=False):
"""
Returns a string that is used as cache key for a collection.
Django adds a prefix to the cache key when using the django cache api.
In other cases use raw=True to add the same cache key.
"""
key = get_element_list_cache_key(self.collection_string)
if raw:
key = cache.make_key(key)
return key
def get_model(self):
"""
Returns the django model that is used for this collection.
"""
return get_model_from_collection_string(self.collection_string)
def element_generator(self):
"""
Generator that yields all collection_elements of this collection.
"""
# Get all cache keys.
ids = self.get_all_ids()
cache_keys = [
get_single_element_cache_key(self.collection_string, id)
for id in ids]
cached_full_data_dict = cache.get_many(cache_keys)
# Get all ids that are missing.
missing_ids = set()
for cache_key in cache_keys:
if cache_key not in cached_full_data_dict:
missing_ids.add(get_collection_id_from_cache_key(cache_key)[1])
# Generate collection elements that where in the cache.
for cache_key, cached_full_data in cached_full_data_dict.items():
yield CollectionElement.from_values(
*get_collection_id_from_cache_key(cache_key),
full_data=cached_full_data)
# Generate collection element that where not in the cache.
if missing_ids:
from openslides.core.config import config
if self.collection_string == config.get_collection_string():
# If config elements are not in the cache, they have to be read from
# the config object.
for key, value in config.items():
if key in missing_ids:
collection_element = CollectionElement.from_values(
config.get_collection_string(),
key,
full_data={'key': key, 'value': value})
# We can not use .from_instance therefore the config value
# is not saved to the cache. We have to do it manualy.
collection_element.save_to_cache()
yield collection_element
else:
model = self.get_model()
try:
query = model.objects.get_full_queryset()
except AttributeError:
query = model.objects
for instance in query.filter(pk__in=missing_ids):
yield CollectionElement.from_instance(instance)
def as_autoupdate_for_projector(self):
"""
Returns a list of dictonaries to send them to the projector.
"""
output = []
for collection_element in self.element_generator():
content = collection_element.as_autoupdate_for_projector()
# Content can not be None. If the projector can not see an element,
# then it is marked as deleted.
output.append(content)
return output
def as_list_for_user(self, user):
"""
Returns a list of dictonaries to send them to a user, for example over
the rest api.
"""
output = []
for collection_element in self.element_generator():
content = collection_element.as_dict_for_user(user)
if content is not None:
output.append(content)
return output
def get_all_ids(self):
"""
Returns a set of all ids of instances in this collection.
"""
from openslides.core.config import config
if self.collection_string == config.get_collection_string():
ids = config.config_variables.keys()
elif use_redis_cache():
ids = self.get_all_ids_redis()
else:
ids = self.get_all_ids_other()
return ids
def get_all_ids_redis(self):
redis = get_redis_connection()
ids = redis.smembers(self.get_cache_key(raw=True))
if not ids:
ids = set(self.get_model().objects.values_list('pk', flat=True))
if ids:
redis.sadd(self.get_cache_key(raw=True), *ids)
# Redis returns the ids as string.
ids = set(int(id) for id in ids)
return ids
def get_all_ids_other(self):
ids = cache.get(self.get_cache_key())
if ids is None:
# If it is not in the cache then get it from the database.
ids = set(self.get_model().objects.values_list('pk', flat=True))
cache.set(self.get_cache_key(), ids)
return ids
def delete_id_from_cache(self, id):
"""
Delets a id from the cache.
"""
if use_redis_cache():
self.delete_id_from_cache_redis(id)
else:
self.delete_id_from_cache_other(id)
def delete_id_from_cache_redis(self, id):
redis = get_redis_connection()
redis.srem(self.get_cache_key(raw=True), id)
def delete_id_from_cache_other(self, id):
ids = cache.get(self.get_cache_key())
if ids is not None:
ids = set(ids)
try:
ids.remove(id)
except KeyError:
# The id is not part of id list
pass
else:
if ids:
cache.set(self.get_cache_key(), ids)
else:
# Delete the key, if there are not ids left
cache.delete(self.get_cache_key())
def add_id_to_cache(self, id):
"""
Adds a collection id to the list of collection ids in the cache.
"""
if use_redis_cache():
self.add_id_to_cache_redis(id)
else:
self.add_id_to_cache_other(id)
def add_id_to_cache_redis(self, id):
redis = get_redis_connection()
if redis.exists(self.get_cache_key(raw=True)):
# Only add the value if it is in the cache.
redis.sadd(self.get_cache_key(raw=True), id)
def add_id_to_cache_other(self, id):
ids = cache.get(self.get_cache_key())
if ids is not None:
# Only change the value if it is in the cache.
ids = set(ids)
ids.add(id)
cache.set(self.get_cache_key(), ids)
def get_model_from_collection_string(collection_string):
"""
@ -152,3 +426,59 @@ def get_model_from_collection_string(collection_string):
# No model was found in all apps.
raise ValueError('Invalid message. A valid collection_string is missing.')
return model
def get_single_element_cache_key(collection_string, id):
"""
Returns a string that is used as cache key for a single instance.
"""
return "{prefix}{id}".format(
prefix=get_single_element_cache_key_prefix(collection_string),
id=id)
def get_single_element_cache_key_prefix(collection_string):
"""
Returns the first part of the cache key for single elements, which is the
same for all cache keys of the same collection.
"""
return "{collection}:".format(collection=collection_string)
def get_element_list_cache_key(collection_string):
"""
Returns a string that is used as cache key for a collection.
"""
return "{collection}".format(collection=collection_string)
def get_collection_id_from_cache_key(cache_key):
"""
Returns a tuble of the collection string and the id from a cache_key
created with get_instance_cache_key.
The returned id can be an integer or an string.
"""
collection_string, id = cache_key.rsplit(':', 1)
try:
id = int(id)
except ValueError:
# The id is no integer. This can happen on config elements
pass
return (collection_string, id)
def use_redis_cache():
"""
Returns True if Redis is used als caching backend.
"""
try:
from django_redis.cache import RedisCache
except ImportError:
return False
return isinstance(caches['default'], RedisCache)
def get_redis_connection():
from django_redis import get_redis_connection
return get_redis_connection("default")

51
openslides/utils/models.py
View File

@ -54,3 +54,54 @@ class RESTModelMixin:
the database pk.
"""
return self.pk
def save(self, skip_autoupdate=False, information=None, *args, **kwargs):
"""
Calls Django's save() method and afterwards hits the autoupdate system.
If skip_autoupdate is set to True, then the autoupdate system is not
informed about the model changed. This also means, that the model cache
is not updated. You have to do this manually, by creating a collection
element from the instance:
CollectionElement.from_instance(instance)
The optional argument information can be a dictionary that is given to
the autoupdate system.
"""
# We don't know how to fix this circular import
from .autoupdate import inform_changed_data
return_value = super().save(*args, **kwargs)
if not skip_autoupdate:
inform_changed_data(self.get_root_rest_element(), information=information)
return return_value
def delete(self, skip_autoupdate=False, information=None, *args, **kwargs):
"""
Calls Django's delete() method and afterwards hits the autoupdate system.
If skip_autoupdate is set to True, then the autoupdate system is not
informed about the model changed. This also means, that the model cache
is not updated. You have to do this manually, by creating a collection
element from the instance:
CollectionElement.from_instance(instance, deleted=True)
or
CollectionElement.from_values(collection_string, id, deleted=True)
The optional argument information can be a dictionary that is given to
the autoupdate system.
"""
# We don't know how to fix this circular import
from .autoupdate import inform_changed_data, inform_deleted_data
instance_pk = self.pk
return_value = super().delete(*args, **kwargs)
if not skip_autoupdate:
if self != self.get_root_rest_element():
# The deletion of a included element is a change of the root element.
inform_changed_data(self.get_root_rest_element(), information=information)
else:
inform_deleted_data(self.get_collection_string(), instance_pk, information=information)
return return_value

33
openslides/utils/projector.py
View File

@ -1,5 +1,6 @@
from django.dispatch import Signal
from .collection import CollectionElement
from .dispatch import SignalConnectMetaClass
@ -73,11 +74,31 @@ class ProjectorElement(object, metaclass=SignalConnectMetaClass):
"""
return ()
def need_full_update_for_this(self, collection_element):
def get_requirements_as_collection_elements(self, config_entry):
"""
Returns True if this projector element needs to be updated with all
instances as defined in get_requirements(). The given
collection_element contains information about the changed instance.
Default is False.
Returns an iterable of collection elements that are required for this
projector element. The config_entry has to be given.
"""
return False
return (CollectionElement.from_instance(instance) for instance in self.get_requirements(config_entry))
def get_collection_elements_required_for_this(self, collection_element, config_entry):
"""
Returns a list of CollectionElements that have to be sent to every
projector that shows this projector element according to the given
collection_element and information.
Default: Returns only the collection_element if it belongs to the
requirements but return all requirements if the projector changes.
"""
requirements_as_collection_elements = list(self.get_requirements_as_collection_elements(config_entry))
for requirement in requirements_as_collection_elements:
if collection_element == requirement:
output = [collection_element]
break
else:
if collection_element.information.get('this_projector'):
output = [collection_element]
output.extend(requirements_as_collection_elements)
else:
output = []
return output

68
openslides/utils/rest_api.py
View File

@ -1,15 +1,13 @@
from collections import OrderedDict
from django.http import Http404
from rest_framework import status # noqa
from rest_framework.decorators import detail_route, list_route # noqa
from rest_framework.metadata import SimpleMetadata # noqa
from rest_framework.mixins import ( # noqa
DestroyModelMixin,
ListModelMixin,
RetrieveModelMixin,
UpdateModelMixin,
)
from rest_framework.response import Response # noqa
from rest_framework.mixins import ListModelMixin as _ListModelMixin
from rest_framework.mixins import RetrieveModelMixin as _RetrieveModelMixin
from rest_framework.mixins import DestroyModelMixin, UpdateModelMixin # noqa
from rest_framework.response import Response
from rest_framework.routers import DefaultRouter
from rest_framework.serializers import ModelSerializer as _ModelSerializer
from rest_framework.serializers import ( # noqa
@ -31,6 +29,8 @@ from rest_framework.viewsets import GenericViewSet as _GenericViewSet # noqa
from rest_framework.viewsets import ModelViewSet as _ModelViewSet # noqa
from rest_framework.viewsets import ViewSet as _ViewSet # noqa
from .collection import Collection, CollectionElement
router = DefaultRouter()
@ -164,11 +164,63 @@ class ModelSerializer(_ModelSerializer):
return fields
class ListModelMixin(_ListModelMixin):
"""
Mixin to add the caching system to list requests.
It is not allowed to use the method get_queryset() in derivated classes.
The attribute queryset has to be used in the following form:
queryset = Model.objects.all()
"""
def list(self, request, *args, **kwargs):
model = self.get_queryset().model
try:
collection_string = model.get_collection_string()
except AttributeError:
# The corresponding queryset does not support caching.
response = super().list(request, *args, **kwargs)
else:
collection = Collection(collection_string)
response = Response(collection.as_list_for_user(request.user))
return response
class RetrieveModelMixin(_RetrieveModelMixin):
"""
Mixin to add the caching system to retrieve requests.
It is not allowed to use the method get_queryset() in derivated classes.
The attribute queryset has to be used in the following form:
queryset = Model.objects.all()
"""
def retrieve(self, request, *args, **kwargs):
model = self.get_queryset().model
try:
collection_string = model.get_collection_string()
except AttributeError:
# The corresponding queryset does not support caching.
response = super().retrieve(request, *args, **kwargs)
else:
lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
collection_element = CollectionElement.from_values(
collection_string, self.kwargs[lookup_url_kwarg])
try:
content = collection_element.as_dict_for_user(request.user)
except collection_element.get_model().DoesNotExist:
raise Http404
if content is None:
self.permission_denied(request)
response = Response(content)
return response
class GenericViewSet(PermissionMixin, _GenericViewSet):
pass
class ModelViewSet(PermissionMixin, _ModelViewSet):
class ModelViewSet(PermissionMixin, ListModelMixin, RetrieveModelMixin, _ModelViewSet):
pass

20
openslides/utils/settings.py.tpl
View File

@ -46,6 +46,17 @@ DEBUG = %(debug)s
# Change this setting to use e. g. PostgreSQL or MySQL.
# DATABASES = {
# 'default': {
# 'ENGINE': 'django.db.backends.postgresql',
# 'NAME': 'mydatabase',
# 'USER': 'mydatabaseuser',
# 'PASSWORD': 'mypassword',
# 'HOST': '127.0.0.1',
# 'PORT': '5432',
# }
# }
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
@ -64,12 +75,17 @@ DATABASES = {
# CHANNEL_LAYERS['default']['BACKEND'] = 'asgi_redis.RedisChannelLayer'
# https://niwinz.github.io/django-redis/latest/#_user_guide
# Caching
# Django uses a inmemory cache at default. This supports only one thread. If
# you use more then one thread another caching backend is required. We recommand
# django-redis: https://niwinz.github.io/django-redis/latest/#_user_guide
# CACHES = {
# "default": {
# "BACKEND": "django_redis.cache.RedisCache",
# "LOCATION": "redis://127.0.0.1:6379/1",
# "LOCATION": "redis://127.0.0.1:6379/0",
# "OPTIONS": {
# "CLIENT_CLASS": "django_redis.client.DefaultClient",
# }

83
tests/integration/agenda/test_views.py
View File

@ -1,90 +1,7 @@
import json
from rest_framework.test import APIClient
from openslides.agenda.models import Item
from openslides.topics.models import Topic
from openslides.utils.test import TestCase
class AgendaTreeTest(TestCase):
def setUp(self):
Topic.objects.create(title='item1')
item2 = Topic.objects.create(title='item2').agenda_item
item3 = Topic.objects.create(title='item2a').agenda_item
item3.parent = item2
item3.save()
self.client = APIClient()
self.client.login(username='admin', password='admin')
def test_get(self):
response = self.client.get('/rest/agenda/item/tree/')
self.assertEqual(json.loads(response.content.decode()),
[{'children': [], 'id': 1},
{'children': [{'children': [], 'id': 3}], 'id': 2}])
def test_set(self):
tree = [{'id': 3},
{'children': [{'id': 1}], 'id': 2}]
response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json')
self.assertEqual(response.status_code, 200)
item1 = Item.objects.get(pk=1)
item2 = Item.objects.get(pk=2)
item3 = Item.objects.get(pk=3)
self.assertEqual(item1.parent_id, 2)
self.assertEqual(item1.weight, 0)
self.assertEqual(item2.parent_id, None)
self.assertEqual(item2.weight, 1)
self.assertEqual(item3.parent_id, None)
self.assertEqual(item3.weight, 0)
def test_set_without_perm(self):
self.client = APIClient()
response = self.client.put('/rest/agenda/item/tree/', {'tree': []}, format='json')
self.assertEqual(response.status_code, 403)
def test_tree_with_double_item(self):
"""
Test to send a tree that has an item-pk more then once in it.
It is expected, that the responsecode 400 is returned with a specific
content
"""
tree = [{'id': 1}, {'id': 1}]
response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json')
self.assertEqual(response.status_code, 400)
self.assertEqual(response.data, {'detail': "Item 1 is more then once in the tree."})
def test_tree_with_empty_children(self):
"""
Test that the chrildren element is not required in the tree
"""
tree = [{'id': 1}]
response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json')
self.assertEqual(response.status_code, 200)
def test_tree_with_unknown_item(self):
"""
Tests that unknown items lead to an error.
"""
tree = [{'id': 500}]
response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json')
self.assertEqual(response.status_code, 400)
self.assertEqual(response.data, {'detail': "Item 500 is not in the database."})
class TestAgendaPDF(TestCase):
def test_get(self):
"""

59
tests/integration/agenda/test_viewsets.py
View File

@ -4,9 +4,12 @@ from rest_framework import status
from rest_framework.test import APIClient
from openslides.agenda.models import Item, Speaker
from openslides.assignments.models import Assignment
from openslides.core.config import config
from openslides.core.models import Projector
from openslides.motions.models import Motion
from openslides.topics.models import Topic
from openslides.users.models import User
from openslides.utils.test import TestCase
@ -37,7 +40,61 @@ class RetrieveItem(TestCase):
permission = group.permissions.get(content_type__app_label=app_label, codename=codename)
group.permissions.remove(permission)
response = self.client.get(reverse('item-detail', args=[self.item.pk]))
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
class TestDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some agenda items are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
Topic.objects.create(title='topic{}'.format(index))
parent = Topic.objects.create(title='parent').agenda_item
child = Topic.objects.create(title='child').agenda_item
child.parent = parent
child.save()
Motion.objects.create(title='motion1')
Motion.objects.create(title='motion2')
Assignment.objects.create(title='assignment', open_posts=5)
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions,
* 2 requests to get the list of all agenda items,
* 1 request to get all speakers,
* 3 requests to get the assignments, motions and topics and
* 2 requests for the motionsversions.
TODO: There could be less requests to get the session and the request user.
The last two request for the motionsversions are a bug.
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(13):
self.client.get(reverse('item-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions)
* 2 requests to get the list of all agenda items,
* 1 request to get all speakers,
* 3 requests to get the assignments, motions and topics and
* 32 requests for the motionsversions.
TODO: The last 32 requests are a bug.
"""
with self.assertNumQueries(40):
self.client.get(reverse('item-list'))
class ManageSpeaker(TestCase):

52
tests/integration/assignments/test_viewset.py
View File

@ -4,9 +4,61 @@ from rest_framework import status
from rest_framework.test import APIClient
from openslides.assignments.models import Assignment
from openslides.core.config import config
from openslides.users.models import User
from openslides.utils.test import TestCase
class TestDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some assignments are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
Assignment.objects.create(title='motion{}'.format(index), open_posts=1)
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions,
* 2 requests to get the list of all assignments,
* 1 request to get all related users,
* 1 request to get the agenda item,
* 1 request to get the polls,
* 10 request to featch each related user again.
TODO: There could be less requests to get the session and the request user.
The eleven request are a bug.
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(30):
self.client.get(reverse('assignment-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions)
* 2 requests to get the list of all assignments,
* 1 request to get all related users,
* 1 request to get the agenda item,
* 1 request to get the polls,
* 1 request to get the tags,
* lots of permissions requests.
TODO: The last requests are a bug.
"""
with self.assertNumQueries(57):
self.client.get(reverse('assignment-list'))
class CanidatureSelf(TestCase):
"""
Tests self candidation view.

147
tests/integration/core/test_viewset.py Normal file
View File

@ -0,0 +1,147 @@
from django.core.urlresolvers import reverse
from rest_framework.test import APIClient
from openslides.core.config import config
from openslides.core.models import ChatMessage, Projector, Tag
from openslides.users.models import User
from openslides.utils.test import TestCase
class TestProjectorDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
Projector.objects.create(name="Projector{}".format(index))
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions,
* 2 requests to get the list of all projectors,
* 1 request to get the list of the projector defaults.
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(8):
self.client.get(reverse('projector-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions)
* 2 requests to get the list of all projectors,
* 1 request to get the list of the projector defaults and
* 11 requests for permissions.
TODO: The last 11 requests are a bug.
"""
with self.assertNumQueries(16):
self.client.get(reverse('projector-list'))
class TestCharmessageDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
user = User.objects.get(pk=1)
for index in range(10):
ChatMessage.objects.create(user=user)
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions,
* 2 requests to get the list of all chatmessages,
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(7):
self.client.get(reverse('chatmessage-list'))
class TestTagDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
Tag.objects.create(name='tag{}'.format(index))
def test_admin(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the session an the request user with its permissions,
* 2 requests to get the list of all tags,
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(4):
self.client.get(reverse('tag-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions)
* 2 requests to get the list of all projectors,
* 10 requests for to config
The last 10 requests are a bug.
"""
with self.assertNumQueries(14):
self.client.get(reverse('tag-list'))
class TestConfigDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
def test_admin(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the session an the request user with its permissions and
* 1 requests to get the list of all config values
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(3):
self.client.get(reverse('config-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions),
* 1 to get all config value and
* 57 requests to find out if anonymous is enabled.
TODO: The last 57 requests are a bug.
"""
with self.assertNumQueries(60):
self.client.get(reverse('config-list'))

0
tests/integration/mediafiles/__init__.py Normal file
View File

46
tests/integration/mediafiles/test_viewset.py Normal file
View File

@ -0,0 +1,46 @@
from django.core.files.uploadedfile import SimpleUploadedFile
from django.core.urlresolvers import reverse
from rest_framework.test import APIClient
from openslides.core.config import config
from openslides.mediafiles.models import Mediafile
from openslides.users.models import User
from openslides.utils.test import TestCase
class TestDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
Mediafile.objects.create(
title='some_file{}'.format(index),
mediafile=SimpleUploadedFile(
'some_file{}'.format(index),
b'some content.'))
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions and
* 2 requests to get the list of all files.
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(7):
self.client.get(reverse('mediafile-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions) and
* 2 requests to get the list of all projectors.
"""
with self.assertNumQueries(4):
self.client.get(reverse('mediafile-list'))

130
tests/integration/motions/test_viewset.py
View File

@ -8,9 +8,139 @@ from rest_framework.test import APIClient
from openslides.core.config import config
from openslides.core.models import Tag
from openslides.motions.models import Category, Motion, State
from openslides.users.models import User
from openslides.utils.test import TestCase
class TestMotionDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
Motion.objects.create(title='motion{}'.format(index))
# TODO: Create some polls etc.
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions,
* 2 requests to get the list of all motions,
* 1 request to get the motion versions,
* 1 request to get the agenda item,
* 1 request to get the motion log,
* 1 request to get the polls,
* 1 request to get the attachments,
* 1 request to get the tags,
* 2 requests to get the submitters and supporters
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(15):
self.client.get(reverse('motion-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions)
* 2 requests to get the list of all motions,
* 1 request to get the motion versions,
* 1 request to get the agenda item,
* 1 request to get the motion log,
* 1 request to get the polls,
* 1 request to get the attachments,
* 1 request to get the tags,
* 2 requests to get the submitters and supporters
* 10 requests for permissions.
TODO: The last 10 requests are a bug.
"""
with self.assertNumQueries(22):
self.client.get(reverse('motion-list'))
class TestCategoryDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
Category.objects.create(name='category{}'.format(index))
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions and
* 2 requests to get the list of all categories.
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(7):
self.client.get(reverse('category-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions)
* 2 requests to get the list of all motions and
* 10 requests for permissions.
TODO: The last 10 requests are a bug.
"""
with self.assertNumQueries(14):
self.client.get(reverse('category-list'))
class TestWorkflowDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
# There do not need to be more workflows
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions,
* 2 requests to get the list of all workflows,
* 1 request to get all states and
* 1 request to get the next states of all states.
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(9):
self.client.get(reverse('workflow-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions),
* 2 requests to get the list of all workflows,
* 1 request to get all states and
* 1 request to get the next states of all states.
* 2 requests for permissions.
TODO: The last 2 requests are a bug.
"""
with self.assertNumQueries(8):
self.client.get(reverse('workflow-list'))
class CreateMotion(TestCase):
"""
Tests motion creation.

80
tests/integration/users/test_viewset.py
View File

@ -8,6 +8,86 @@ from openslides.users.serializers import UserFullSerializer
from openslides.utils.test import TestCase
class TestUserDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
User.objects.create(username='user{}'.format(index))
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions,
* 2 requests to get the list of all assignments and
* 1 request to get all groups.
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(8):
self.client.get(reverse('user-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions)
* 2 requests to get the list of all users,
* 1 request to get all groups and
* lots of permissions requests.
TODO: The last requests are a bug.
"""
with self.assertNumQueries(27):
self.client.get(reverse('user-list'))
class TestGroupDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
Group.objects.create(name='group{}'.format(index))
def test_admin(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the session an the request user with its permissions,
* 1 request to get the list of all groups,
* 1 request to get the permissions and
* 1 request to get the content_object for the permissions.
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(5):
self.client.get(reverse('group-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to find out if anonymous is enabled
* 1 request to get the list of all groups,
* 1 request to get the permissions and
* 1 request to get the content_object for the permissions.
TODO: There should be only one request to find out if anonymous is enabled.
"""
with self.assertNumQueries(5):
self.client.get(reverse('group-list'))
class UserGetTest(TestCase):
"""
Tests to receive a users via REST API.

25
tests/integration/utils/autoupdate.py → tests/integration/utils/test_autoupdate.py
View File

@ -24,13 +24,22 @@ class TestGetLoggedInUsers(TestCase):
User.objects.create(username='user3')
# Create a session with a user, that expires in 5 hours
Session.objects.create(user=user1, expire_data=timezone.now() + timedelta(hours=5))
Session.objects.create(
user=user1,
expire_date=timezone.now() + timedelta(hours=5),
session_key='1')
# Create a session with a user, that is expired before 5 hours
Session.objects.create(user=user2, expire_data=timezone.now() + timedelta(hours=-5))
Session.objects.create(
user=user2,
expire_date=timezone.now() + timedelta(hours=-5),
session_key='2')
# Create a session with an anonymous user, that expires in 5 hours
Session.objects.create(user=None, expire_data=timezone.now() + timedelta(hours=5))
Session.objects.create(
user=None,
expire_date=timezone.now() + timedelta(hours=5),
session_key='3')
self.assertEqual(list(get_logged_in_users()), [user1])
@ -40,7 +49,13 @@ class TestGetLoggedInUsers(TestCase):
The user should be returned only once.
"""
user1 = User.objects.create(username='user1')
Session.objects.create(user=user1, expire_data=timezone.now() + timedelta(hours=1))
Session.objects.create(user=user1, expire_data=timezone.now() + timedelta(hours=2))
Session.objects.create(
user=user1,
expire_date=timezone.now() + timedelta(hours=1),
session_key='1')
Session.objects.create(
user=user1,
expire_date=timezone.now() + timedelta(hours=2),
session_key='2')
self.assertEqual(list(get_logged_in_users()), [user1])

134
tests/integration/utils/test_collection.py Normal file
View File

@ -0,0 +1,134 @@
from unittest.mock import patch
from channels.tests import ChannelTestCase
from django.core.cache import caches
from openslides.topics.models import Topic
from openslides.utils import collection
class TestCase(ChannelTestCase):
"""
Testcase that uses the local mem cache and clears the cache after each test.
"""
def setUp(self):
cache = caches['locmem']
cache.clear()
self.patch = patch('openslides.utils.collection.cache', cache)
self.patch.start()
def tearDown(self):
self.patch.stop()
class TestCollectionElementCache(TestCase):
def test_clean_cache(self):
"""
Tests that the data is retrieved from the database.
"""
topic = Topic.objects.create(title='test topic')
collection_element = collection.CollectionElement.from_values('topics/topic', 1)
caches['locmem'].clear()
with self.assertNumQueries(3):
instance = collection_element.get_full_data()
self.assertEqual(topic.title, instance['title'])
def test_with_cache(self):
"""
Tests that no db query is used when the valie is in the cache.
The value is added to the test when .create(...) is called. This hits
the autoupdate system, which fills the cache.
"""
topic = Topic.objects.create(title='test topic')
collection_element = collection.CollectionElement.from_values('topics/topic', 1)
with self.assertNumQueries(0):
collection_element = collection.CollectionElement.from_values('topics/topic', 1)
instance = collection_element.get_full_data()
self.assertEqual(topic.title, instance['title'])
def test_non_existing_instance(self):
collection_element = collection.CollectionElement.from_values('topics/topic', 1)
with self.assertRaises(Topic.DoesNotExist):
collection_element.get_full_data()
class TestCollectionCache(TestCase):
def test_clean_cache(self):
"""
Tests that the instances are retrieved from the database.
Currently there are 10 queries needed. This can change in the future,
but it has to be more then zero.
"""
Topic.objects.create(title='test topic1')
Topic.objects.create(title='test topic2')
Topic.objects.create(title='test topic3')
topic_collection = collection.Collection('topics/topic')
caches['locmem'].clear()
with self.assertNumQueries(4):
instance_list = list(topic_collection.as_autoupdate_for_projector())
self.assertEqual(len(instance_list), 3)
def test_with_cache(self):
"""
Tests that no db query is used when the list is received twice.
"""
Topic.objects.create(title='test topic1')
Topic.objects.create(title='test topic2')
Topic.objects.create(title='test topic3')
topic_collection = collection.Collection('topics/topic')
list(topic_collection.as_autoupdate_for_projector())
with self.assertNumQueries(0):
instance_list = list(topic_collection.as_autoupdate_for_projector())
self.assertEqual(len(instance_list), 3)
def test_with_some_objects_in_the_cache(self):
"""
One element (topic3) is in the cache and two are not.
"""
Topic.objects.create(title='test topic1')
Topic.objects.create(title='test topic2')
caches['locmem'].clear()
Topic.objects.create(title='test topic3')
topic_collection = collection.Collection('topics/topic')
with self.assertNumQueries(4):
instance_list = list(topic_collection.as_autoupdate_for_projector())
self.assertEqual(len(instance_list), 3)
def test_deletion(self):
"""
When an element is deleted, the cache should be updated automaticly via
the autoupdate system. So there should be no db queries.
"""
Topic.objects.create(title='test topic1')
Topic.objects.create(title='test topic2')
topic3 = Topic.objects.create(title='test topic3')
topic_collection = collection.Collection('topics/topic')
list(topic_collection.as_autoupdate_for_projector())
topic3.delete()
with self.assertNumQueries(0):
instance_list = list(topic_collection.as_autoupdate_for_projector())
self.assertEqual(len(instance_list), 2)
def test_config_elements_without_cache(self):
topic_collection = collection.Collection('core/config')
caches['locmem'].clear()
with self.assertNumQueries(1):
list(topic_collection.as_autoupdate_for_projector())
def test_config_elements_with_cache(self):
topic_collection = collection.Collection('core/config')
list(topic_collection.as_autoupdate_for_projector())
with self.assertNumQueries(0):
list(topic_collection.as_autoupdate_for_projector())

11
tests/settings.py
View File

@ -75,3 +75,14 @@ TEST_RUNNER = 'openslides.utils.test.OpenSlidesDiscoverRunner'
PASSWORD_HASHERS = [
'django.contrib.auth.hashers.MD5PasswordHasher',
]
# Use the dummy cache that does not cache anything
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.dummy.DummyCache'
},
'locmem': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'
}
}

251
tests/unit/utils/test_collection.py Normal file
View File

@ -0,0 +1,251 @@
from unittest import TestCase
from unittest.mock import MagicMock, patch
from openslides.core.models import Projector
from openslides.utils import collection
class TestCacheKeys(TestCase):
def test_get_collection_id_from_cache_key(self):
"""
Test that get_collection_id_from_cache_key works together with
get_single_element_cache_key.
"""
element = ('some/testkey', 42)
self.assertEqual(
element,
collection.get_collection_id_from_cache_key(
collection.get_single_element_cache_key(*element)))
def test_get_collection_id_from_cache_key_for_strings(self):
"""
Test get_collection_id_from_cache_key for strings
"""
element = ('some/testkey', 'my_config_value')
self.assertEqual(
element,
collection.get_collection_id_from_cache_key(
collection.get_single_element_cache_key(*element)))
def test_get_single_element_cache_key_prefix(self):
"""
Tests that the cache prefix is realy a prefix.
"""
element = ('some/testkey', 42)
cache_key = collection.get_single_element_cache_key(*element)
prefix = collection.get_single_element_cache_key_prefix(element[0])
self.assertTrue(cache_key.startswith(prefix))
def test_prefix_different_then_list(self):
"""
Test that the return value of get_single_element_cache_key_prefix is
something different then get_element_list_cache_key.
"""
collection_string = 'some/testkey'
prefix = collection.get_single_element_cache_key_prefix(collection_string)
list_cache_key = collection.get_element_list_cache_key(collection_string)
self.assertNotEqual(prefix, list_cache_key)
class TestGetModelFromCollectionString(TestCase):
def test_known_app(self):
projector_model = collection.get_model_from_collection_string('core/projector')
self.assertEqual(projector_model, Projector)
def test_unknown_app(self):
with self.assertRaises(ValueError):
collection.get_model_from_collection_string('invalid/model')
class TestCollectionElement(TestCase):
def test_from_values(self):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
self.assertEqual(collection_element.collection_string, 'testmodule/model')
self.assertEqual(collection_element.id, 42)
@patch('openslides.utils.collection.Collection')
@patch('openslides.utils.collection.cache')
def test_from_values_deleted(self, mock_cache, mock_collection):
"""
Tests that when createing a CollectionElement with deleted=True the element
is deleted from the cache.
"""
collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True)
self.assertTrue(collection_element.is_deleted())
mock_cache.delete.assert_called_with('testmodule/model:42')
mock_collection.assert_called_with('testmodule/model')
mock_collection().delete_id_from_cache.assert_called_with(42)
def test_as_channel_message(self):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
self.assertEqual(
collection_element.as_channels_message(),
{'collection_string': 'testmodule/model',
'id': 42,
'deleted': False,
'information': {}})
def test_as_autoupdate_for_user(self):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
fake_user = MagicMock()
collection_element.get_access_permissions = MagicMock()
collection_element.get_access_permissions().get_restricted_data.return_value = 'restricted_data'
collection_element.get_full_data = MagicMock()
self.assertEqual(
collection_element.as_autoupdate_for_user(fake_user),
{'collection': 'testmodule/model',
'id': 42,
'action': 'changed',
'data': 'restricted_data'})
collection_element.get_full_data.assert_called_once_with()
def test_as_autoupdate_for_user_no_permission(self):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
fake_user = MagicMock()
collection_element.get_access_permissions = MagicMock()
collection_element.get_access_permissions().get_restricted_data.return_value = None
collection_element.get_full_data = MagicMock()
self.assertEqual(
collection_element.as_autoupdate_for_user(fake_user),
{'collection': 'testmodule/model',
'id': 42,
'action': 'deleted'})
collection_element.get_full_data.assert_called_once_with()
def test_as_autoupdate_for_user_deleted(self):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True)
fake_user = MagicMock()
self.assertEqual(
collection_element.as_autoupdate_for_user(fake_user),
{'collection': 'testmodule/model',
'id': 42,
'action': 'deleted'})
def test_get_instance_deleted(self):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True)
with self.assertRaises(RuntimeError):
collection_element.get_instance()
@patch('openslides.core.config.config')
def test_get_instance_config_str(self, mock_config):
mock_config.get_collection_string.return_value = 'core/config'
mock_config.__getitem__.return_value = 'config_value'
collection_element = collection.CollectionElement.from_values('core/config', 'my_config_value')
instance = collection_element.get_instance()
self.assertEqual(
instance,
{'key': 'my_config_value',
'value': 'config_value'})
def test_get_instance(self):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
collection_element.get_model = MagicMock()
collection_element.get_instance()
collection_element.get_model().objects.get_full_queryset().get.assert_called_once_with(pk=42)
@patch('openslides.utils.collection.cache')
def test_get_full_data_already_loaded(self, mock_cache):
"""
Test that the cache and the self.get_instance() is not hit, when the
instance is already loaded.
"""
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
collection_element.full_data = 'my_full_data'
collection_element.get_instance = MagicMock()
collection_element.get_full_data()
mock_cache.get.assert_not_called()
collection_element.get_instance.assert_not_called()
@patch('openslides.utils.collection.cache')
def test_get_full_data_from_cache(self, mock_cache):
"""
Test that the value from the cache is used not get_instance is not
called.
"""
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
collection_element.get_instance = MagicMock()
mock_cache.get.return_value = 'cache_value'
instance = collection_element.get_full_data()
self.assertEqual(instance, 'cache_value')
mock_cache.get.assert_called_once_with('testmodule/model:42')
collection_element.get_instance.assert_not_called
@patch('openslides.utils.collection.Collection')
@patch('openslides.utils.collection.cache')
def test_get_full_data_from_get_instance(self, mock_cache, mock_Collection):
"""
Test that the value from get_instance is used and saved to the cache
"""
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
collection_element.get_instance = MagicMock()
collection_element.get_access_permissions = MagicMock()
collection_element.get_access_permissions().get_full_data.return_value = 'get_instance_value'
mock_cache.get.return_value = None
instance = collection_element.get_full_data()
self.assertEqual(instance, 'get_instance_value')
mock_cache.get.assert_called_once_with('testmodule/model:42')
collection_element.get_instance.assert_called_once_with()
mock_cache.set.assert_called_once_with('testmodule/model:42', 'get_instance_value')
mock_Collection.assert_called_once_with('testmodule/model')
mock_Collection().add_id_to_cache.assert_called_once_with(42)
def test_equal(self):
self.assertEqual(
collection.CollectionElement.from_values('testmodule/model', 1),
collection.CollectionElement.from_values('testmodule/model', 1))
self.assertEqual(
collection.CollectionElement.from_values('testmodule/model', 1),
collection.CollectionElement.from_values('testmodule/model', 1, deleted=True))
self.assertNotEqual(
collection.CollectionElement.from_values('testmodule/model', 1),
collection.CollectionElement.from_values('testmodule/model', 2))
self.assertNotEqual(
collection.CollectionElement.from_values('testmodule/model', 1),
collection.CollectionElement.from_values('testmodule/other_model', 1))
class TestCollection(TestCase):
@patch('openslides.utils.collection.CollectionElement')
@patch('openslides.utils.collection.cache')
def test_element_generator(self, mock_cache, mock_CollectionElement):
"""
Test with the following scenario: The collection has three elements. Two
are in the cache and one is not.
"""
test_collection = collection.Collection('testmodule/model')
test_collection.get_all_ids = MagicMock(return_value=set([1, 2, 3]))
test_collection.get_model = MagicMock()
test_collection.get_model().objects.get_full_queryset().filter.return_value = ['my_instance']
mock_cache.get_many.return_value = {
'testmodule/model:1': 'element1',
'testmodule/model:2': 'element2'}
list(test_collection.element_generator())
mock_cache.get_many.assert_called_once_with(
['testmodule/model:1', 'testmodule/model:2', 'testmodule/model:3'])
test_collection.get_model().objects.get_full_queryset().filter.assert_called_once_with(pk__in={3})
self.assertEqual(mock_CollectionElement.from_values.call_count, 2)
self.assertEqual(mock_CollectionElement.from_instance.call_count, 1)