Adds a cache system to the CollectionElement and add

a Collection class that can be used to call a collection
used this for the list and receive rest api.
This commit is contained in:
Oskar Hahn 2016-09-18 16:00:31 +02:00
parent 2a5bd6d94b
commit 368873e738
36 changed files with 1525 additions and 414 deletions

1
.gitignore vendored
View File

@ -26,3 +26,4 @@ dist/*
# Unit test and coverage reports
.coverage
tests/file/*

View File

@ -23,6 +23,9 @@ class ItemManager(models.Manager):
Customized model manager with special methods for agenda tree and
numbering.
"""
def get_full_queryset(self):
return self.get_queryset().prefetch_related('speakers', 'content_object')
def get_only_agenda_items(self):
"""
Generator, which yields only agenda items. Skips hidden items.

View File

@ -58,26 +58,6 @@ class ItemViewSet(ListModelMixin, RetrieveModelMixin, UpdateModelMixin, GenericV
result = False
return result
def check_object_permissions(self, request, obj):
"""
Checks if the requesting user has permission to see also an
organizational item if it is one.
"""
# TODO: Move this logic to access_permissions.ItemAccessPermissions.
if obj.is_hidden() and not request.user.has_perm('agenda.can_see_hidden_items'):
self.permission_denied(request)
def get_queryset(self):
"""
Filters organizational items if the user has no permission to see them.
"""
# TODO: Move this logic to access_permissions.ItemAccessPermissions.
queryset = super().get_queryset()
if not self.request.user.has_perm('agenda.can_see_hidden_items'):
pk_list = [item.pk for item in Item.objects.get_only_agenda_items()]
queryset = queryset.filter(pk__in=pk_list)
return queryset
@detail_route(methods=['POST', 'DELETE'])
def manage_speaker(self, request, pk=None):
"""
@ -243,31 +223,6 @@ class ItemViewSet(ListModelMixin, RetrieveModelMixin, UpdateModelMixin, GenericV
# Initiate response.
return Response({'detail': _('List of speakers successfully sorted.')})
@list_route(methods=['get', 'put'])
def tree(self, request):
"""
Returns or sets the agenda tree.
"""
if request.method == 'PUT':
if not (request.user.has_perm('agenda.can_manage') and
request.user.has_perm('agenda.can_see_hidden_items')):
self.permission_denied(request)
try:
tree = request.data['tree']
except KeyError as error:
response = Response({'detail': 'Agenda tree is missing.'}, status=400)
else:
try:
Item.objects.set_tree(tree)
except ValueError as error:
response = Response({'detail': str(error)}, status=400)
else:
response = Response({'detail': 'Agenda tree successfully updated.'})
else:
# request.method == 'GET'
response = Response(Item.objects.get_tree())
return response
@list_route(methods=['post'])
def numbering(self, request):
"""

View File

@ -1,7 +1,7 @@
from collections import OrderedDict
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericRelation
from django.db import models
from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_noop
@ -50,12 +50,22 @@ class AssignmentRelatedUser(RESTModelMixin, models.Model):
return self.assignment
class AssignmentManager(models.Manager):
def get_full_queryset(self):
return self.get_queryset().prefetch_related(
'related_users',
'agenda_items',
'polls')
class Assignment(RESTModelMixin, models.Model):
"""
Model for assignments.
"""
access_permissions = AssignmentAccessPermissions()
objects = AssignmentManager()
PHASE_SEARCH = 0
PHASE_VOTING = 1
PHASE_FINISHED = 2
@ -111,6 +121,10 @@ class Assignment(RESTModelMixin, models.Model):
Tags for the assignment.
"""
# In theory there could be one then more agenda_item. But support only one.
# See the property agenda_item.
agenda_items = GenericRelation(Item, related_name='assignments')
class Meta:
default_permissions = ()
permissions = (
@ -290,8 +304,7 @@ class Assignment(RESTModelMixin, models.Model):
"""
Returns the related agenda item.
"""
content_type = ContentType.objects.get_for_model(self)
return Item.objects.get(object_id=self.pk, content_type=content_type)
return self.agenda_items.all()[0]
@property
def agenda_item_id(self):

View File

@ -87,4 +87,8 @@ class ConfigAccessPermissions(BaseAccessPermissions):
# Attention: The format of this response has to be the same as in
# the retrieve method of ConfigViewSet.
if isinstance(instance, dict):
# It happens, that the caching system already sends the correct dict
# as instance.
return instance
return {'key': instance.key, 'value': config[instance.key]}

View File

@ -16,7 +16,6 @@ class CoreAppConfig(AppConfig):
from django.db.models import signals
from openslides.core.config import config
from openslides.core.signals import post_permission_creation
from openslides.utils.autoupdate import inform_changed_data_receiver, inform_deleted_data_receiver
from openslides.utils.rest_api import router
from openslides.utils.search import index_add_instance, index_del_instance
from .config_variables import get_config_variables
@ -45,15 +44,6 @@ class CoreAppConfig(AppConfig):
router.register(self.get_model('Tag').get_collection_string(), TagViewSet)
router.register(self.get_model('ConfigStore').get_collection_string(), ConfigViewSet, 'config')
# Update data when any model of any installed app is saved or deleted.
# TODO: Test if the m2m_changed signal is also needed.
signals.post_save.connect(
inform_changed_data_receiver,
dispatch_uid='inform_changed_data_receiver')
signals.post_delete.connect(
inform_deleted_data_receiver,
dispatch_uid='inform_deleted_data_receiver')
# Update the search when a model is saved or deleted
signals.post_save.connect(
index_add_instance,

View File

@ -2,19 +2,10 @@
# Generated by Django 1.10.1 on 2016-09-18 19:04
from __future__ import unicode_literals
from django.db import migrations, models
from openslides.utils.autoupdate import (
inform_changed_data_receiver,
inform_deleted_data_receiver,
)
from django.db import migrations
def move_custom_slides_to_topics(apps, schema_editor):
# Disconnect autoupdate. We do not want to trigger it here.
models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver')
models.signals.post_save.disconnect(dispatch_uid='inform_deleted_data_receiver')
# We get the model from the versioned app registry;
# if we directly import it, it will be the wrong version.
ContentType = apps.get_model('contenttypes', 'ContentType')
@ -39,14 +30,6 @@ def move_custom_slides_to_topics(apps, schema_editor):
CustomSlide.objects.all().delete()
content_type_custom_slide.delete()
# Reconnect autoupdate.
models.signals.post_save.connect(
inform_changed_data_receiver,
dispatch_uid='inform_changed_data_receiver')
models.signals.post_delete.connect(
inform_deleted_data_receiver,
dispatch_uid='inform_deleted_data_receiver')
class Migration(migrations.Migration):

View File

@ -116,7 +116,7 @@ class Projector(RESTModelMixin, models.Model):
result[key]['error'] = str(e)
return result
def get_all_requirements(self):
def get_all_requirements(self, on_slide=None):
"""
Generator which returns all instances that are shown on this projector.
"""

View File

@ -18,6 +18,7 @@ from django.utils.timezone import now
from openslides import __version__ as version
from openslides.utils import views as utils_views
from openslides.utils.collection import Collection, CollectionElement
from openslides.utils.plugins import (
get_plugin_description,
get_plugin_verbose_name,
@ -41,7 +42,7 @@ from .access_permissions import (
)
from .config import config
from .exceptions import ConfigError, ConfigNotFound
from .models import ChatMessage, ProjectionDefault, Projector, Tag
from .models import ChatMessage, ConfigStore, ProjectionDefault, Projector, Tag
# Special Django views
@ -606,22 +607,25 @@ class ConfigViewSet(ViewSet):
def list(self, request):
"""
Lists all config variables. Everybody can see them.
Lists all config variables.
"""
return Response([{'key': key, 'value': value} for key, value in config.items()])
collection = Collection(config.get_collection_string())
return Response(collection.as_list_for_user(request.user))
def retrieve(self, request, *args, **kwargs):
"""
Retrieves a config variable. Everybody can see it.
Retrieves a config variable.
"""
key = kwargs['pk']
collection_element = CollectionElement.from_values(config.get_collection_string(), key)
try:
value = config[key]
except ConfigNotFound:
content = collection_element.as_dict_for_user(request.user)
except ConfigStore.DoesNotExist:
raise Http404
# Attention: The format of this response has to be the same as in
# the get_full_data method of ConfigAccessPermissions.
return Response({'key': key, 'value': value})
if content is None:
# If content is None, the user has no permissions to see the item.
self.permission_denied()
return Response(content)
def update(self, request, *args, **kwargs):
"""

View File

@ -15,13 +15,6 @@ class MediafileViewSet(ModelViewSet):
access_permissions = MediafileAccessPermissions()
queryset = Mediafile.objects.all()
def get_queryset(self):
queryset = super().get_queryset()
user = self.request.user
if not user.has_perm('mediafiles.can_see_private'):
queryset = queryset.filter(private=False)
return queryset
def check_view_permissions(self):
"""
Returns True if the user has required permissions.

View File

@ -5,16 +5,11 @@ from __future__ import unicode_literals
import django.db.models.deletion
from django.db import migrations, models
from openslides.utils.autoupdate import inform_changed_data_receiver
def change_label_of_state(apps, schema_editor):
"""
Changes the label of former state "commited a bill" to "refered to committee".
"""
# Disconnect autoupdate. We do not want to trigger it here.
models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver')
# We get the model from the versioned app registry;
# if we directly import it, it will be the wrong version.
State = apps.get_model('motions', 'State')
@ -29,19 +24,11 @@ def change_label_of_state(apps, schema_editor):
state.action_word = 'Refer to committee'
state.save()
# Reconnect autoupdate.
models.signals.post_save.connect(
inform_changed_data_receiver,
dispatch_uid='inform_changed_data_receiver')
def add_recommendation_labels(apps, schema_editor):
"""
Adds recommendation labels to some of the built-in states.
"""
# Disconnect autoupdate. We do not want to trigger it here.
models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver')
# We get the model from the versioned app registry;
# if we directly import it, it will be the wrong version.
State = apps.get_model('motions', 'State')

View File

@ -1,5 +1,5 @@
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericRelation
from django.db import models
from django.db.models import Max
from django.utils import formats
@ -28,6 +28,21 @@ from .access_permissions import (
from .exceptions import WorkflowError
class MotionManager(models.Manager):
def get_full_queryset(self):
return (super().get_queryset()
.select_related('active_version')
.prefetch_related(
'versions',
'agenda_items',
'log_messages',
'polls',
'attachments',
'tags',
'submitters',
'supporters'))
class Motion(RESTModelMixin, models.Model):
"""
The Motion Class.
@ -36,6 +51,8 @@ class Motion(RESTModelMixin, models.Model):
"""
access_permissions = MotionAccessPermissions()
objects = MotionManager()
active_version = models.ForeignKey(
'MotionVersion',
on_delete=models.SET_NULL,
@ -134,6 +151,10 @@ class Motion(RESTModelMixin, models.Model):
Configurable fields for comments. Contains a list of strings.
"""
# In theory there could be one then more agenda_item. But support only one.
# See the property agenda_item.
agenda_items = GenericRelation(Item, related_name='motions')
class Meta:
default_permissions = ()
permissions = (
@ -519,8 +540,7 @@ class Motion(RESTModelMixin, models.Model):
"""
Returns the related agenda item.
"""
content_type = ContentType.objects.get_for_model(self)
return Item.objects.get(object_id=self.pk, content_type=content_type)
return self.agenda_items.all()[0]
@property
def agenda_item_id(self):
@ -940,12 +960,21 @@ class State(RESTModelMixin, models.Model):
return self.workflow
class WorkflowManager(models.Manager):
def get_full_queryset(self):
return (self.get_queryset()
.select_related('first_state')
.prefetch_related('states', 'states__next_states'))
class Workflow(RESTModelMixin, models.Model):
"""
Defines a workflow for a motion.
"""
access_permissions = WorkflowAccessPermissions()
objects = WorkflowManager()
name = models.CharField(max_length=255)
"""A string representing the workflow."""

View File

@ -77,27 +77,6 @@ class MotionViewSet(ModelViewSet):
result = False
return result
def list(self, request, *args, **kwargs):
"""
Customized view endpoint to list all motions.
Hides non public comment fields for some users.
"""
response = super().list(request, *args, **kwargs)
for i, motion in enumerate(response.data):
response.data[i] = self.get_access_permissions().get_restricted_data(motion, self.request.user)
return response
def retrieve(self, request, *args, **kwargs):
"""
Customized view endpoint to retrieve a motion.
Hides non public comment fields for some users.
"""
response = super().retrieve(request, *args, **kwargs)
response.data = self.get_access_permissions().get_restricted_data(response.data, self.request.user)
return response
def create(self, request, *args, **kwargs):
"""
Customized view endpoint to create a new motion.

View File

@ -1,4 +1,4 @@
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericRelation
from django.db import models
from ..agenda.models import Item
@ -7,16 +7,27 @@ from ..utils.models import RESTModelMixin
from .access_permissions import TopicAccessPermissions
class TopicManager(models.Manager):
def get_queryset(self):
query = super().get_queryset().prefetch_related('attachments', 'agenda_items')
return query
class Topic(RESTModelMixin, models.Model):
"""
Model for slides with custom content. Used to be called custom slide.
"""
access_permissions = TopicAccessPermissions()
objects = TopicManager()
title = models.CharField(max_length=256)
text = models.TextField(blank=True)
attachments = models.ManyToManyField(Mediafile, blank=True)
# In theory there could be one then more agenda_item. But support only one.
# See the property agenda_item.
agenda_items = GenericRelation(Item, related_name='topics')
class Meta:
default_permissions = ()
@ -28,8 +39,7 @@ class Topic(RESTModelMixin, models.Model):
"""
Returns the related agenda item.
"""
content_type = ContentType.objects.get_for_model(self)
return Item.objects.get(object_id=self.pk, content_type=content_type)
return self.agenda_items.all()[0]
@property
def agenda_item_id(self):

View File

@ -15,15 +15,9 @@ class UserAccessPermissions(BaseAccessPermissions):
"""
Returns different serializer classes with respect user's permissions.
"""
from .serializers import UserCanSeeSerializer, UserCanSeeExtraSerializer, UserFullSerializer
from .serializers import UserFullSerializer
if (user is None or (user.has_perm('users.can_see_extra_data') and user.has_perm('users.can_manage'))):
serializer_class = UserFullSerializer
elif user.has_perm('users.can_see_extra_data'):
serializer_class = UserCanSeeExtraSerializer
else:
serializer_class = UserCanSeeSerializer
return serializer_class
return UserFullSerializer
def get_restricted_data(self, full_data, user):
"""

View File

@ -2,9 +2,7 @@
# Generated by Django 1.9.7 on 2016-08-01 14:54
from __future__ import unicode_literals
from django.db import migrations, models
from openslides.utils.autoupdate import inform_changed_data_receiver
from django.db import migrations
def migrate_groups_and_user_permissions(apps, schema_editor):
@ -20,9 +18,6 @@ def migrate_groups_and_user_permissions(apps, schema_editor):
- the name of the first group is 'Guests',
- the name of the second group is 'Registered users'.
"""
# Disconnect autoupdate. We do not want to trigger it here.
models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver')
User = apps.get_model('users', 'User')
Group = apps.get_model('auth', 'Group')
@ -52,11 +47,6 @@ def migrate_groups_and_user_permissions(apps, schema_editor):
for permission in group_registered.permissions.all():
group.permissions.add(permission)
# Reconnect autoupdate.
models.signals.post_save.connect(
inform_changed_data_receiver,
dispatch_uid='inform_changed_data_receiver')
class Migration(migrations.Migration):

View File

@ -23,6 +23,10 @@ class UserManager(BaseUserManager):
Customized manager that creates new users only with a password and a
username.
"""
def get_full_queryset(self):
return self.get_queryset().prefetch_related('groups')
def create_user(self, username, password, **kwargs):
"""
Creates a new user only with a password and a username.

View File

@ -12,79 +12,24 @@ from ..utils.rest_api import (
from .models import Group, User
USERCANSEESERIALIZER_FIELDS = (
'id',
'username',
'title',
'first_name',
'last_name',
'structure_level',
'number',
'about_me',
'groups',
'is_committee',
)
'id',
'username',
'title',
'first_name',
'last_name',
'structure_level',
'number',
'about_me',
'groups',
'is_committee',
)
class UserCanSeeSerializer(ModelSerializer):
"""
Serializer for users.models.User objects to be used by users who have
only the permission to see users and to change some date of theirselfs.
Attention: Viewset has to ensure that a user can update only himself.
"""
class Meta:
model = User
fields = USERCANSEESERIALIZER_FIELDS
read_only_fields = (
'number',
'groups',
'is_comittee',
)
USERCANSEEEXTRASERIALIZER_FIELDS = (
'id',
'is_present',
'username',
'title',
'first_name',
'last_name',
'number',
'structure_level',
'about_me',
'comment',
'groups',
'is_active',
'is_committee',
)
class UserCanSeeExtraSerializer(ModelSerializer):
"""
Serializer for users.models.User objects to be used by users who have
the permission to see users with extra data and to change some date of
theirselfs.
Attention: Viewset has to ensure that a user can update only himself.
"""
groups = IdPrimaryKeyRelatedField(
many=True,
queryset=Group.objects.exclude(pk=1),
help_text=ugettext_lazy('The groups this user belongs to. A user will '
'get all permissions granted to each of '
'his/her groups.'))
class Meta:
model = User
fields = USERCANSEEEXTRASERIALIZER_FIELDS
read_only_fields = (
'is_present',
'number',
'comment',
'groups',
'is_comittee',
'is_active',
)
USERCANSEEEXTRASERIALIZER_FIELDS = USERCANSEESERIALIZER_FIELDS + (
'is_present',
'comment',
'is_active',
)
class UserFullSerializer(ModelSerializer):

View File

@ -18,7 +18,7 @@ from ..utils.views import APIView, PDFView
from .access_permissions import UserAccessPermissions
from .models import Group, User
from .pdf import users_passwords_to_pdf, users_to_pdf
from .serializers import GroupSerializer, UserFullSerializer
from .serializers import GroupSerializer
# Viewsets for the REST API
@ -49,19 +49,6 @@ class UserViewSet(ModelViewSet):
result = False
return result
def get_serializer_class(self):
"""
Returns different serializer classes with respect to action.
"""
if self.action in ('create', 'partial_update', 'update'):
# Return the UserFullSerializer for edit requests.
serializer_class = UserFullSerializer
else:
# Return different serializers according to user permsissions via
# access permissions class.
serializer_class = super().get_serializer_class()
return serializer_class
def update(self, request, *args, **kwargs):
"""
Customized view endpoint to update an user.
@ -138,7 +125,7 @@ class GroupViewSet(ModelViewSet):
partial_update, update and destroy.
"""
metadata_class = GroupViewSetMetadata
queryset = Group.objects.all()
queryset = Group.objects.prefetch_related('permissions', 'permissions__content_type')
serializer_class = GroupSerializer
def check_view_permissions(self):

View File

@ -11,7 +11,7 @@ from ..core.config import config
from ..core.models import Projector
from ..users.auth import AnonymousUser
from ..users.models import User
from .collection import CollectionElement
from .collection import Collection, CollectionElement
def get_logged_in_users():
@ -23,18 +23,19 @@ def get_logged_in_users():
return User.objects.exclude(session=None).filter(session__expire_date__gte=timezone.now()).distinct()
def get_projector_element_data(projector):
def get_projector_element_data(projector, on_slide=None):
"""
Returns a list of dicts that are required for a specific projector.
The argument projector has to be a projector instance.
If on_slide is a string that matches an slide on the projector, then only
elements on this slide are returned.
"""
output = []
for requirement in projector.get_all_requirements():
for requirement in projector.get_all_requirements(on_slide):
required_collection_element = CollectionElement.from_instance(requirement)
element_dict = required_collection_element.as_autoupdate_for_projector()
if element_dict is not None:
output.append(element_dict)
output.append(required_collection_element.as_autoupdate_for_projector())
return output
@ -83,12 +84,8 @@ def ws_add_projector(message, projector_id):
output = get_projector_element_data(projector)
# Send all config elements.
for key, value in config.items():
output.append({
'collection': config.get_collection_string(),
'id': key,
'action': 'changed',
'data': {'key': key, 'value': value}})
collection = Collection(config.get_collection_string())
output.extend(collection.as_autoupdate_for_projector())
# Send the projector instance.
collection_element = CollectionElement.from_instance(projector)
@ -115,9 +112,6 @@ def send_data(message):
for user in itertools.chain(get_logged_in_users(), [AnonymousUser()]):
channel = Group('user-{}'.format(user.id))
output = collection_element.as_autoupdate_for_user(user)
if output is None:
# There are no data for the user so he can't see the object. Skip him.
continue
channel.send({'text': json.dumps([output])})
# Get the projector elements where data have to be sent and if whole projector
@ -157,14 +151,20 @@ def send_data(message):
else:
output = broadcast_projector_data
else:
# The list will be filled in the next lines.
output = []
output.append(collection_element.as_autoupdate_for_projector())
if output:
Group('projector-{}'.format(projector.pk)).send(
{'text': json.dumps(output)})
def inform_changed_data(instance, is_deleted=False):
def inform_changed_data(instance, information=None):
"""
Informs the autoupdate system and the caching system about the creation or
update of an element.
"""
try:
root_instance = instance.get_root_rest_element()
except AttributeError:
@ -173,29 +173,39 @@ def inform_changed_data(instance, is_deleted=False):
else:
collection_element = CollectionElement.from_instance(
root_instance,
is_deleted=is_deleted and instance == root_instance)
information=information)
# If currently there is an open database transaction, then the following
# function is only called, when the transaction is commited. If there
# is currently no transaction, then the function is called immediately.
def send_autoupdate():
try:
Channel('autoupdate.send_data').send(collection_element.as_channels_message())
except ChannelLayer.ChannelFull:
pass
transaction.on_commit(lambda: send_autoupdate(collection_element))
transaction.on_commit(send_autoupdate)
def inform_changed_data_receiver(sender, instance, **kwargs):
def inform_deleted_data(collection_string, id, information=None):
"""
Receiver for the inform_changed_data function to use in a signal.
Informs the autoupdate system and the caching system about the deletion of
an element.
"""
inform_changed_data(instance)
collection_element = CollectionElement.from_values(
collection_string=collection_string,
id=id,
deleted=True,
information=information)
# If currently there is an open database transaction, then the following
# function is only called, when the transaction is commited. If there
# is currently no transaction, then the function is called immediately.
def send_autoupdate():
try:
Channel('autoupdate.send_data').send(collection_element.as_channels_message())
except ChannelLayer.ChannelFull:
pass
transaction.on_commit(lambda: send_autoupdate(collection_element))
def inform_deleted_data_receiver(sender, instance, **kwargs):
def send_autoupdate(collection_element):
"""
Receiver for the inform_changed_data function to use in a signal.
Helper function, that sends a collection_element through a channel to the
autoupdate system.
"""
inform_changed_data(instance, is_deleted=True)
try:
Channel('autoupdate.send_data').send(collection_element.as_channels_message())
except ChannelLayer.ChannelFull:
pass

View File

@ -1,25 +1,41 @@
from django.apps import apps
from django.core.cache import cache, caches
class CollectionElement:
@classmethod
def from_instance(cls, instance, is_deleted=False):
def from_instance(cls, instance, deleted=False, information=None):
"""
Returns a collection element from a database instance.
This will also update the instance in the cache.
If deleted is set to True, the element is deleted from the cache.
"""
return cls(instance=instance, is_deleted=is_deleted)
return cls(instance=instance, deleted=deleted, information=information)
@classmethod
def from_values(cls, collection_string, id, is_deleted=False):
def from_values(cls, collection_string, id, deleted=False, full_data=None, information=None):
"""
Returns a collection element from a collection_string and an id.
"""
return cls(collection_string=collection_string, id=id, is_deleted=is_deleted)
def __init__(self, instance=None, is_deleted=False, collection_string=None, id=None):
If deleted is set to True, the element is deleted from the cache.
With the argument full_data, the content of the CollectionElement can be set.
It has to be a dict in the format that is used be access_permission.get_full_data().
"""
return cls(collection_string=collection_string, id=id, deleted=deleted,
full_data=full_data, information=information)
def __init__(self, instance=None, deleted=False, collection_string=None, id=None,
full_data=None, information=None):
"""
Do not use this. Use the methods from_instance() or from_values().
"""
self.instance = instance
self.deleted = deleted
self.full_data = full_data
self.information = information
if instance is not None:
self.collection_string = instance.get_collection_string()
self.id = instance.pk
@ -31,62 +47,73 @@ class CollectionElement:
'Invalid state. Use CollectionElement.from_instance() or '
'CollectionElement.from_values() but not CollectionElement() '
'directly.')
self.instance = instance
self.deleted = is_deleted
if self.is_deleted():
# Delete the element from the cache, if self.is_deleted() is True:
self.delete_from_cache()
elif instance is not None:
# If this element is created via instance and the instance is not deleted
# then update the cache.
self.save_to_cache()
def as_channels_message(self):
"""
Returns a dictonary that can be used to send the object through the
channels system.
"""
return {
channel_message = {
'collection_string': self.collection_string,
'id': self.id,
'is_deleted': self.is_deleted()}
'deleted': self.is_deleted()}
if self.information is not None:
channel_message['information'] = self.information
return channel_message
def as_autoupdate(self, method, *args):
"""
Only for internal use. Do not use it directly. Use as_autoupdate_for_user()
or as_autoupdate_for_projector().
"""
output = {
'collection': self.collection_string,
'id': self.id,
'action': 'deleted' if self.is_deleted() else 'changed',
}
if not self.is_deleted():
data = getattr(self.get_access_permissions(), method)(
self.get_full_data(),
*args)
if data is None:
# The user is not allowed to see this element. Set action to deleted.
output['action'] = 'deleted'
else:
output['data'] = data
return output
def as_autoupdate_for_user(self, user):
"""
Returns a dict that can be sent through the autoupdate system for a site
user.
Returns None if the user can not see the element.
"""
output = {
'collection': self.collection_string,
'id': self.id,
'action': 'deleted' if self.is_deleted() else 'changed',
}
if not self.is_deleted():
data = self.get_access_permissions().get_restricted_data(
self.get_full_data(), user)
if data is None:
# The user is not allowed to see this element. Reset output to None.
output = None
else:
output['data'] = data
return output
return self.as_autoupdate(
'get_restricted_data',
user)
def as_autoupdate_for_projector(self):
"""
Returns a dict that can be sent through the autoupdate system for the
projector.
Returns None if the projector can not see the element.
"""
output = {
'collection': self.collection_string,
'id': self.id,
'action': 'deleted' if self.is_deleted() else 'changed',
}
if not self.is_deleted():
data = self.get_access_permissions().get_projector_data(
self.get_full_data())
if data is None:
# The user is not allowed to see this element. Reset output to None.
output = None
else:
output['data'] = data
return output
return self.as_autoupdate(
'get_projector_data')
def as_dict_for_user(self, user):
"""
Returns a dict with the data for a user. Can be used for the rest api.
"""
return self.get_access_permissions().get_restricted_data(
self.get_full_data(),
user)
def get_model(self):
"""
@ -102,8 +129,21 @@ class CollectionElement:
"""
if self.is_deleted():
raise RuntimeError("The collection element is deleted.")
if self.instance is None:
self.instance = self.get_model().objects.get(pk=self.id)
# The config instance has to be get from the config element, because
# some config values are not in the db.
from openslides.core.config import config
if (self.collection_string == config.get_collection_string() and
isinstance(self.id, str)):
self.instance = {'key': self.id, 'value': config[self.id]}
else:
model = self.get_model()
try:
query = model.objects.get_full_queryset()
except AttributeError:
query = model.objects
self.instance = query.get(pk=self.id)
return self.instance
def get_access_permissions(self):
@ -117,7 +157,20 @@ class CollectionElement:
Returns the full_data of this collection_element from with all other
dics can be generated.
"""
return self.get_access_permissions().get_full_data(self.get_instance())
# If the full_data is already loaded, return it
# If there is a db_instance, use it to get the full_data
# else: try to use the cache.
# If there is no value in the cach, get the content from the db and save
# it to the cache.
if self.full_data is None and self.instance is None:
# Use the cache version if self.instance is not set.
# After this line full_data can be None, if the element is not in the cache.
self.full_data = cache.get(self.get_cache_key())
if self.full_data is None:
self.full_data = self.get_access_permissions().get_full_data(self.get_instance())
self.save_to_cache()
return self.full_data
def is_deleted(self):
"""
@ -125,6 +178,217 @@ class CollectionElement:
"""
return self.deleted
def get_cache_key(self):
"""
Returns a string that is used as cache key for a single instance.
"""
return get_single_element_cache_key(self.collection_string, self.id)
def delete_from_cache(self):
"""
Delets an element from the cache.
Does nothing if the element is not in the cache.
"""
# Deletes the element from the cache.
cache.delete(self.get_cache_key())
# Delete the id of the instance of the instance list
Collection(self.collection_string).delete_id_from_cache(self.id)
def save_to_cache(self):
"""
Add or update the element to the cache.
"""
# Set the element to the cache.
cache.set(self.get_cache_key(), self.get_full_data())
# Add the id of the element to the collection
Collection(self.collection_string).add_id_to_cache(self.id)
class Collection:
"""
Represents all elements of one collection.
"""
def __init__(self, collection_string):
self.collection_string = collection_string
def get_cache_key(self, raw=False):
"""
Returns a string that is used as cache key for a collection.
Django adds a prefix to the cache key when using the django cache api.
In other cases use raw=True to add the same cache key.
"""
key = get_element_list_cache_key(self.collection_string)
if raw:
key = self.make_key(key)
return key
def get_model(self):
"""
Returns the django model that is used for this collection.
"""
return get_model_from_collection_string(self.collection_string)
def element_generator(self):
"""
Generator that yields all collection_elements of this collection.
"""
# Get all cache keys.
ids = self.get_all_ids()
cache_keys = [
get_single_element_cache_key(self.collection_string, id)
for id in ids]
cached_full_data_dict = cache.get_many(cache_keys)
# Get all ids that are missing.
missing_ids = set()
for cache_key in cache_keys:
if cache_key not in cached_full_data_dict:
missing_ids.add(get_collection_id_from_cache_key(cache_key)[1])
# Generate collection elements that where in the cache.
for cache_key, cached_full_data in cached_full_data_dict.items():
yield CollectionElement.from_values(
*get_collection_id_from_cache_key(cache_key),
full_data=cached_full_data)
# Generate collection element that where not in the cache.
if missing_ids:
from openslides.core.config import config
if self.collection_string == config.get_collection_string():
# If config elements are not in the cache, they have to be read from
# the config object.
for key, value in config.items():
if key in missing_ids:
collection_element = CollectionElement.from_values(
config.get_collection_string(),
key,
full_data={'key': key, 'value': value})
# We can not use .from_instance therefore the config value
# is not saved to the cache. We have to do it manualy.
collection_element.save_to_cache()
yield collection_element
else:
model = self.get_model()
try:
query = model.objects.get_full_queryset()
except AttributeError:
query = model.objects
for instance in query.filter(pk__in=missing_ids):
yield CollectionElement.from_instance(instance)
def as_autoupdate_for_projector(self):
"""
Returns a list of dictonaries to send them to the projector.
"""
output = []
for collection_element in self.element_generator():
content = collection_element.as_autoupdate_for_projector()
# Content can not be None. If the projector can not see an element,
# then it is marked as deleted.
output.append(content)
return output
def as_list_for_user(self, user):
"""
Returns a list of dictonaries to send them to a user, for example over
the rest api.
"""
output = []
for collection_element in self.element_generator():
content = collection_element.as_dict_for_user(user)
if content is not None:
output.append(content)
return output
def get_all_ids(self):
"""
Returns a set of all ids of instances in this collection.
"""
from openslides.core.config import config
if self.collection_string == config.get_collection_string():
ids = config.config_variables.keys()
elif use_redis_cache():
ids = self.get_all_ids_redis()
else:
ids = self.get_all_ids_other()
return ids
def get_all_ids_redis(self):
redis = get_redis_connection()
ids = redis.smembers(self.get_cache_key(raw=True))
if not ids:
ids = set(self.get_model().objects.values_list('pk', flat=True))
if ids:
redis.sadd(self.get_cache_key(raw=True), *ids)
# Redis returns the ids as string.
ids = set(int(id) for id in ids)
return ids
def get_all_ids_other(self):
ids = cache.get(self.get_cache_key())
if ids is None:
# If it is not in the cache then get it from the database.
ids = set(self.get_model().objects.values_list('pk', flat=True))
cache.set(self.get_cache_key(), ids)
return ids
def delete_id_from_cache(self, id):
"""
Delets a id from the cache.
"""
if use_redis_cache():
self.delete_id_from_cache_redis(id)
else:
self.delete_id_from_cache_other(id)
def delete_id_from_cache_redis(self, id):
redis = get_redis_connection()
redis.srem(self.get_cache_key(raw=True), id)
def delete_id_from_cache_other(self, id):
ids = cache.get(self.get_cache_key())
if ids is not None:
ids = set(ids)
try:
ids.remove(id)
except KeyError:
# The id is not part of id list
pass
else:
if ids:
cache.set(self.get_cache_key(), ids)
else:
# Delete the key, if there are not ids left
cache.delete(self.get_cache_key())
def add_id_to_cache(self, id):
"""
Adds a collection id to the list of collection ids in the cache.
"""
if use_redis_cache():
self.add_id_to_cache_redis(id)
else:
self.add_id_to_cache_other(id)
def add_id_to_cache_redis(self, id):
redis = get_redis_connection()
if redis.exists(self.get_cache_key(raw=True)):
# Only add the value if it is in the cache.
redis.sadd(self.get_cache_key(raw=True), id)
def add_id_to_cache_other(self, id):
ids = cache.get(self.get_cache_key())
if ids is not None:
# Only change the value if it is in the cache.
ids = set(ids)
ids.add(id)
cache.set(self.get_cache_key(), ids)
def get_model_from_collection_string(collection_string):
"""
@ -152,3 +416,56 @@ def get_model_from_collection_string(collection_string):
# No model was found in all apps.
raise ValueError('Invalid message. A valid collection_string is missing.')
return model
def get_single_element_cache_key(collection_string, id):
"""
Returns a string that is used as cache key for a single instance.
"""
return "{prefix}{id}".format(
prefix=get_single_element_cache_key_prefix(collection_string),
id=id)
def get_single_element_cache_key_prefix(collection_string):
"""
Returns the first part of the cache key for single elements, which is the
same for all cache keys of the same collection.
"""
return "{collection}:".format(collection=collection_string)
def get_element_list_cache_key(collection_string):
"""
Returns a string that is used as cache key for a collection.
"""
return "{collection}".format(collection=collection_string)
def get_collection_id_from_cache_key(cache_key):
"""
Returns a tuble of the collection string and the id from a cache_key
created with get_instance_cache_key.
The returned id can be an integer or an string.
"""
collection_string, id = cache_key.rsplit(':', 1)
try:
id = int(id)
except ValueError:
# The id is no integer. This can happen on config elements
pass
return (collection_string, id)
def use_redis_cache():
try:
from django_redis.cache import RedisCache
except ImportError:
return False
return isinstance(caches['default'], RedisCache)
def get_redis_connection():
from django_redis import get_redis_connection
return get_redis_connection("default")

View File

@ -54,3 +54,41 @@ class RESTModelMixin:
the database pk.
"""
return self.pk
def save(self, skip_autoupdate=False, information=None, *args, **kwargs):
"""
Calls the django save-method and afterwards hits the autoupdate system.
If skip_autoupdate is set to True, then the autoupdate system is not
informed about the model changed. This also means, that the model cache
is not updated.
The optional argument information can be an object that is given to the
autoupdate system. It should be a dict.
"""
# TODO: Fix circular imports
from .autoupdate import inform_changed_data
return_value = super().save(*args, **kwargs)
inform_changed_data(self.get_root_rest_element(), information=information)
return return_value
def delete(self, skip_autoupdate=False, information=None, *args, **kwargs):
"""
Calls the django delete-method and afterwards hits the autoupdate system.
See the save method above.
"""
# TODO: Fix circular imports
from .autoupdate import inform_changed_data, inform_deleted_data
# Django sets the pk of the instance to None after deleting it. But
# we need the pk to tell the autoupdate system which element was deleted.
instance_pk = self.pk
return_value = super().delete(*args, **kwargs)
if self != self.get_root_rest_element():
# The deletion of a included element is a change of the master
# element.
# TODO: Does this work in any case with self.pk = None?
inform_changed_data(self.get_root_rest_element(), information=information)
else:
inform_deleted_data(self, information=information)
return return_value

View File

@ -1,15 +1,13 @@
from collections import OrderedDict
from django.http import Http404
from rest_framework import status # noqa
from rest_framework.decorators import detail_route, list_route # noqa
from rest_framework.metadata import SimpleMetadata # noqa
from rest_framework.mixins import ( # noqa
DestroyModelMixin,
ListModelMixin,
RetrieveModelMixin,
UpdateModelMixin,
)
from rest_framework.response import Response # noqa
from rest_framework.mixins import ListModelMixin as _ListModelMixin
from rest_framework.mixins import RetrieveModelMixin as _RetrieveModelMixin
from rest_framework.mixins import DestroyModelMixin, UpdateModelMixin # noqa
from rest_framework.response import Response
from rest_framework.routers import DefaultRouter
from rest_framework.serializers import ModelSerializer as _ModelSerializer
from rest_framework.serializers import ( # noqa
@ -31,6 +29,8 @@ from rest_framework.viewsets import GenericViewSet as _GenericViewSet # noqa
from rest_framework.viewsets import ModelViewSet as _ModelViewSet # noqa
from rest_framework.viewsets import ViewSet as _ViewSet # noqa
from .collection import Collection, CollectionElement
router = DefaultRouter()
@ -164,11 +164,63 @@ class ModelSerializer(_ModelSerializer):
return fields
class ListModelMixin(_ListModelMixin):
"""
Mixin to add the caching system to list requests.
It is not allowed to use the method get_queryset() in derivated classes.
The attribute queryset has to be used in the following form:
queryset = Model.objects.all()
"""
def list(self, request, *args, **kwargs):
model = self.get_queryset().model
try:
collection_string = model.get_collection_string()
except AttributeError:
# The corresponding queryset does not support caching.
response = super().list(request, *args, **kwargs)
else:
collection = Collection(collection_string)
response = Response(collection.as_list_for_user(request.user))
return response
class RetrieveModelMixin(_RetrieveModelMixin):
"""
Mixin to add the caching system to retrieve requests.
It is not allowed to use the method get_queryset() in derivated classes.
The attribute queryset has to be used in the following form:
queryset = Model.objects.all()
"""
def retrieve(self, request, *args, **kwargs):
model = self.get_queryset().model
try:
collection_string = model.get_collection_string()
except AttributeError:
# The corresponding queryset does not support caching.
response = super().retrieve(request, *args, **kwargs)
else:
lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
collection_element = CollectionElement.from_values(
collection_string, self.kwargs[lookup_url_kwarg])
try:
content = collection_element.as_dict_for_user(request.user)
except collection_element.get_model().DoesNotExist:
raise Http404
if content is None:
self.permission_denied(request)
response = Response(content)
return response
class GenericViewSet(PermissionMixin, _GenericViewSet):
pass
class ModelViewSet(PermissionMixin, _ModelViewSet):
class ModelViewSet(PermissionMixin, ListModelMixin, RetrieveModelMixin, _ModelViewSet):
pass

View File

@ -69,7 +69,7 @@ DATABASES = {
# CACHES = {
# "default": {
# "BACKEND": "django_redis.cache.RedisCache",
# "LOCATION": "redis://127.0.0.1:6379/1",
# "LOCATION": "redis://127.0.0.1:6379/0",
# "OPTIONS": {
# "CLIENT_CLASS": "django_redis.client.DefaultClient",
# }

View File

@ -1,90 +1,7 @@
import json
from rest_framework.test import APIClient
from openslides.agenda.models import Item
from openslides.topics.models import Topic
from openslides.utils.test import TestCase
class AgendaTreeTest(TestCase):
def setUp(self):
Topic.objects.create(title='item1')
item2 = Topic.objects.create(title='item2').agenda_item
item3 = Topic.objects.create(title='item2a').agenda_item
item3.parent = item2
item3.save()
self.client = APIClient()
self.client.login(username='admin', password='admin')
def test_get(self):
response = self.client.get('/rest/agenda/item/tree/')
self.assertEqual(json.loads(response.content.decode()),
[{'children': [], 'id': 1},
{'children': [{'children': [], 'id': 3}], 'id': 2}])
def test_set(self):
tree = [{'id': 3},
{'children': [{'id': 1}], 'id': 2}]
response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json')
self.assertEqual(response.status_code, 200)
item1 = Item.objects.get(pk=1)
item2 = Item.objects.get(pk=2)
item3 = Item.objects.get(pk=3)
self.assertEqual(item1.parent_id, 2)
self.assertEqual(item1.weight, 0)
self.assertEqual(item2.parent_id, None)
self.assertEqual(item2.weight, 1)
self.assertEqual(item3.parent_id, None)
self.assertEqual(item3.weight, 0)
def test_set_without_perm(self):
self.client = APIClient()
response = self.client.put('/rest/agenda/item/tree/', {'tree': []}, format='json')
self.assertEqual(response.status_code, 403)
def test_tree_with_double_item(self):
"""
Test to send a tree that has an item-pk more then once in it.
It is expected, that the responsecode 400 is returned with a specific
content
"""
tree = [{'id': 1}, {'id': 1}]
response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json')
self.assertEqual(response.status_code, 400)
self.assertEqual(response.data, {'detail': "Item 1 is more then once in the tree."})
def test_tree_with_empty_children(self):
"""
Test that the chrildren element is not required in the tree
"""
tree = [{'id': 1}]
response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json')
self.assertEqual(response.status_code, 200)
def test_tree_with_unknown_item(self):
"""
Tests that unknown items lead to an error.
"""
tree = [{'id': 500}]
response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json')
self.assertEqual(response.status_code, 400)
self.assertEqual(response.data, {'detail': "Item 500 is not in the database."})
class TestAgendaPDF(TestCase):
def test_get(self):
"""

View File

@ -4,9 +4,12 @@ from rest_framework import status
from rest_framework.test import APIClient
from openslides.agenda.models import Item, Speaker
from openslides.assignments.models import Assignment
from openslides.core.config import config
from openslides.core.models import Projector
from openslides.motions.models import Motion
from openslides.topics.models import Topic
from openslides.users.models import User
from openslides.utils.test import TestCase
@ -37,7 +40,61 @@ class RetrieveItem(TestCase):
permission = group.permissions.get(content_type__app_label=app_label, codename=codename)
group.permissions.remove(permission)
response = self.client.get(reverse('item-detail', args=[self.item.pk]))
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
self.assertEqual(response.status_code, 403)
class TestDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some agenda items are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
Topic.objects.create(title='topic{}'.format(index))
parent = Topic.objects.create(title='parent').agenda_item
child = Topic.objects.create(title='child').agenda_item
child.parent = parent
child.save()
Motion.objects.create(title='motion1')
Motion.objects.create(title='motion2')
Assignment.objects.create(title='assignment', open_posts=5)
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions,
* 2 requests to get the list of all agenda items,
* 1 request to get all speakers,
* 3 requests to get the assignments, motions and topics and
* 2 requests for the motionsversions.
TODO: There could be less requests to get the session and the request user.
The last two request for the motionsversions are a bug.
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(13):
self.client.get(reverse('item-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions)
* 2 requests to get the list of all agenda items,
* 1 request to get all speakers,
* 3 requests to get the assignments, motions and topics and
* 32 requests for the motionsversions.
TODO: The last 32 requests are a bug.
"""
with self.assertNumQueries(40):
self.client.get(reverse('item-list'))
class ManageSpeaker(TestCase):

View File

@ -4,9 +4,61 @@ from rest_framework import status
from rest_framework.test import APIClient
from openslides.assignments.models import Assignment
from openslides.core.config import config
from openslides.users.models import User
from openslides.utils.test import TestCase
class TestDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some assignments are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
Assignment.objects.create(title='motion{}'.format(index), open_posts=1)
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions,
* 2 requests to get the list of all assignments,
* 1 request to get all related users,
* 1 request to get the agenda item,
* 1 request to get the polls,
* 10 request to featch each related user again.
TODO: There could be less requests to get the session and the request user.
The eleven request are a bug.
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(30):
self.client.get(reverse('assignment-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions)
* 2 requests to get the list of all assignments,
* 1 request to get all related users,
* 1 request to get the agenda item,
* 1 request to get the polls,
* 1 request to get the tags,
* lots of permissions requests.
TODO: The last requests are a bug.
"""
with self.assertNumQueries(57):
self.client.get(reverse('assignment-list'))
class CanidatureSelf(TestCase):
"""
Tests self candidation view.

View File

@ -0,0 +1,145 @@
from django.core.urlresolvers import reverse
from rest_framework.test import APIClient
from openslides.core.config import config
from openslides.core.models import ChatMessage, Projector, Tag
from openslides.users.models import User
from openslides.utils.test import TestCase
class TestProjectorDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
Projector.objects.create()
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions,
* 2 requests to get the list of all projectors,
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(7):
self.client.get(reverse('projector-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions)
* 2 requests to get the list of all projectors,
* 11 requests for permissions.
TODO: The last 11 requests are a bug.
"""
with self.assertNumQueries(15):
self.client.get(reverse('projector-list'))
class TestCharmessageDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
user = User.objects.get(pk=1)
for index in range(10):
ChatMessage.objects.create(user=user)
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions,
* 2 requests to get the list of all chatmessages,
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(7):
self.client.get(reverse('chatmessage-list'))
class TestTagDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
Tag.objects.create(name='tag{}'.format(index))
def test_admin(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the session an the request user with its permissions,
* 2 requests to get the list of all tags,
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(4):
self.client.get(reverse('tag-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions)
* 2 requests to get the list of all projectors,
* 10 requests for to config
The last 10 requests are a bug.
"""
with self.assertNumQueries(14):
self.client.get(reverse('tag-list'))
class TestConfigDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
def test_admin(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the session an the request user with its permissions and
* 1 requests to get the list of all config values
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(3):
self.client.get(reverse('config-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions),
* 1 to get all config value and
* 55 requests to find out if anonymous is enabled.
TODO: The last 55 requests are a bug.
"""
with self.assertNumQueries(58):
self.client.get(reverse('config-list'))

View File

View File

@ -0,0 +1,46 @@
from django.core.files.uploadedfile import SimpleUploadedFile
from django.core.urlresolvers import reverse
from rest_framework.test import APIClient
from openslides.core.config import config
from openslides.mediafiles.models import Mediafile
from openslides.users.models import User
from openslides.utils.test import TestCase
class TestDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
Mediafile.objects.create(
title='some_file{}'.format(index),
mediafile=SimpleUploadedFile(
'some_file{}'.format(index),
b'some content.'))
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions and
* 2 requests to get the list of all files.
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(7):
self.client.get(reverse('mediafile-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions) and
* 2 requests to get the list of all projectors.
"""
with self.assertNumQueries(4):
self.client.get(reverse('mediafile-list'))

View File

@ -8,9 +8,139 @@ from rest_framework.test import APIClient
from openslides.core.config import config
from openslides.core.models import Tag
from openslides.motions.models import Category, Motion, State
from openslides.users.models import User
from openslides.utils.test import TestCase
class TestMotionDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
Motion.objects.create(title='motion{}'.format(index))
# TODO: Create some polls etc.
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions,
* 2 requests to get the list of all motions,
* 1 request to get the motion versions,
* 1 request to get the agenda item,
* 1 request to get the motion log,
* 1 request to get the polls,
* 1 request to get the attachments,
* 1 request to get the tags,
* 2 requests to get the submitters and supporters
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(15):
self.client.get(reverse('motion-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions)
* 2 requests to get the list of all motions,
* 1 request to get the motion versions,
* 1 request to get the agenda item,
* 1 request to get the motion log,
* 1 request to get the polls,
* 1 request to get the attachments,
* 1 request to get the tags,
* 2 requests to get the submitters and supporters
* 10 requests for permissions.
TODO: The last 10 requests are a bug.
"""
with self.assertNumQueries(22):
self.client.get(reverse('motion-list'))
class TestCategoryDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
Category.objects.create(name='category{}'.format(index))
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions and
* 2 requests to get the list of all categories.
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(7):
self.client.get(reverse('category-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions)
* 2 requests to get the list of all motions and
* 10 requests for permissions.
TODO: The last 10 requests are a bug.
"""
with self.assertNumQueries(14):
self.client.get(reverse('category-list'))
class TestWorkflowDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
# There do not need to be more workflows
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions,
* 2 requests to get the list of all workflows,
* 1 request to get all states and
* 1 request to get the next states of all states.
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(9):
self.client.get(reverse('workflow-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions),
* 2 requests to get the list of all workflows,
* 1 request to get all states and
* 1 request to get the next states of all states.
* 2 requests for permissions.
TODO: The last 2 requests are a bug.
"""
with self.assertNumQueries(8):
self.client.get(reverse('workflow-list'))
class CreateMotion(TestCase):
"""
Tests motion creation.

View File

@ -8,6 +8,86 @@ from openslides.users.serializers import UserFullSerializer
from openslides.utils.test import TestCase
class TestUserDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
User.objects.create(username='user{}'.format(index))
def test_admin(self):
"""
Tests that only the following db queries are done:
* 5 requests to get the session an the request user with its permissions,
* 2 requests to get the list of all assignments and
* 1 request to get all groups.
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(8):
self.client.get(reverse('user-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the permission for anonymous (config and permissions)
* 2 requests to get the list of all users,
* 1 request to get all groups and
* lots of permissions requests.
TODO: The last requests are a bug.
"""
with self.assertNumQueries(27):
self.client.get(reverse('user-list'))
class TestGroupDBQueries(TestCase):
"""
Tests that receiving elements only need the required db queries.
Therefore in setup some objects are created and received with different
user accounts.
"""
def setUp(self):
self.client = APIClient()
config['general_system_enable_anonymous'] = True
for index in range(10):
Group.objects.create(name='group{}'.format(index))
def test_admin(self):
"""
Tests that only the following db queries are done:
* 2 requests to get the session an the request user with its permissions,
* 1 request to get the list of all groups,
* 1 request to get the permissions and
* 1 request to get the content_object for the permissions.
"""
self.client.force_login(User.objects.get(pk=1))
with self.assertNumQueries(5):
self.client.get(reverse('group-list'))
def test_anonymous(self):
"""
Tests that only the following db queries are done:
* 2 requests to find out if anonymous is enabled
* 1 request to get the list of all groups,
* 1 request to get the permissions and
* 1 request to get the content_object for the permissions.
TODO: There should be only one request to find out if anonymous is enabled.
"""
with self.assertNumQueries(5):
self.client.get(reverse('group-list'))
class UserGetTest(TestCase):
"""
Tests to receive a users via REST API.

View File

@ -24,13 +24,22 @@ class TestGetLoggedInUsers(TestCase):
User.objects.create(username='user3')
# Create a session with a user, that expires in 5 hours
Session.objects.create(user=user1, expire_data=timezone.now() + timedelta(hours=5))
Session.objects.create(
user=user1,
expire_date=timezone.now() + timedelta(hours=5),
session_key='1')
# Create a session with a user, that is expired before 5 hours
Session.objects.create(user=user2, expire_data=timezone.now() + timedelta(hours=-5))
Session.objects.create(
user=user2,
expire_date=timezone.now() + timedelta(hours=-5),
session_key='2')
# Create a session with an anonymous user, that expires in 5 hours
Session.objects.create(user=None, expire_data=timezone.now() + timedelta(hours=5))
Session.objects.create(
user=None,
expire_date=timezone.now() + timedelta(hours=5),
session_key='3')
self.assertEqual(list(get_logged_in_users()), [user1])
@ -40,7 +49,13 @@ class TestGetLoggedInUsers(TestCase):
The user should be returned only once.
"""
user1 = User.objects.create(username='user1')
Session.objects.create(user=user1, expire_data=timezone.now() + timedelta(hours=1))
Session.objects.create(user=user1, expire_data=timezone.now() + timedelta(hours=2))
Session.objects.create(
user=user1,
expire_date=timezone.now() + timedelta(hours=1),
session_key='1')
Session.objects.create(
user=user1,
expire_date=timezone.now() + timedelta(hours=2),
session_key='2')
self.assertEqual(list(get_logged_in_users()), [user1])

View File

@ -0,0 +1,134 @@
from unittest.mock import patch
from channels.tests import ChannelTestCase
from django.core.cache import caches
from openslides.topics.models import Topic
from openslides.utils import collection
class TestCase(ChannelTestCase):
"""
Testcase that uses the local mem cache and clears the cache after each test.
"""
def setUp(self):
cache = caches['locmem']
cache.clear()
self.patch = patch('openslides.utils.collection.cache', cache)
self.patch.start()
def tearDown(self):
self.patch.stop()
class TestCollectionElementCache(TestCase):
def test_clean_cache(self):
"""
Tests that the data is retrieved from the database.
"""
topic = Topic.objects.create(title='test topic')
collection_element = collection.CollectionElement.from_values('topics/topic', 1)
caches['locmem'].clear()
with self.assertNumQueries(3):
instance = collection_element.get_full_data()
self.assertEqual(topic.title, instance['title'])
def test_with_cache(self):
"""
Tests that no db query is used when the valie is in the cache.
The value is added to the test when .create(...) is called. This hits
the autoupdate system, which fills the cache.
"""
topic = Topic.objects.create(title='test topic')
collection_element = collection.CollectionElement.from_values('topics/topic', 1)
with self.assertNumQueries(0):
collection_element = collection.CollectionElement.from_values('topics/topic', 1)
instance = collection_element.get_full_data()
self.assertEqual(topic.title, instance['title'])
def test_non_existing_instance(self):
collection_element = collection.CollectionElement.from_values('topics/topic', 1)
with self.assertRaises(Topic.DoesNotExist):
collection_element.get_full_data()
class TestCollectionCache(TestCase):
def test_clean_cache(self):
"""
Tests that the instances are retrieved from the database.
Currently there are 10 queries needed. This can change in the future,
but it has to be more then zero.
"""
Topic.objects.create(title='test topic1')
Topic.objects.create(title='test topic2')
Topic.objects.create(title='test topic3')
topic_collection = collection.Collection('topics/topic')
caches['locmem'].clear()
with self.assertNumQueries(4):
instance_list = list(topic_collection.as_autoupdate_for_projector())
self.assertEqual(len(instance_list), 3)
def test_with_cache(self):
"""
Tests that no db query is used when the list is received twice.
"""
Topic.objects.create(title='test topic1')
Topic.objects.create(title='test topic2')
Topic.objects.create(title='test topic3')
topic_collection = collection.Collection('topics/topic')
list(topic_collection.as_autoupdate_for_projector())
with self.assertNumQueries(0):
instance_list = list(topic_collection.as_autoupdate_for_projector())
self.assertEqual(len(instance_list), 3)
def test_with_some_objects_in_the_cache(self):
"""
One element (topic3) is in the cache and two are not.
"""
Topic.objects.create(title='test topic1')
Topic.objects.create(title='test topic2')
caches['locmem'].clear()
Topic.objects.create(title='test topic3')
topic_collection = collection.Collection('topics/topic')
with self.assertNumQueries(4):
instance_list = list(topic_collection.as_autoupdate_for_projector())
self.assertEqual(len(instance_list), 3)
def test_deletion(self):
"""
When an element is deleted, the cache should be updated automaticly via
the autoupdate system. So there should be no db queries.
"""
Topic.objects.create(title='test topic1')
Topic.objects.create(title='test topic2')
topic3 = Topic.objects.create(title='test topic3')
topic_collection = collection.Collection('topics/topic')
list(topic_collection.as_autoupdate_for_projector())
topic3.delete()
with self.assertNumQueries(0):
instance_list = list(topic_collection.as_autoupdate_for_projector())
self.assertEqual(len(instance_list), 2)
def test_config_elements_without_cache(self):
topic_collection = collection.Collection('core/config')
caches['locmem'].clear()
with self.assertNumQueries(1):
list(topic_collection.as_autoupdate_for_projector())
def test_config_elements_with_cache(self):
topic_collection = collection.Collection('core/config')
list(topic_collection.as_autoupdate_for_projector())
with self.assertNumQueries(0):
list(topic_collection.as_autoupdate_for_projector())

View File

@ -75,3 +75,14 @@ TEST_RUNNER = 'openslides.utils.test.OpenSlidesDiscoverRunner'
PASSWORD_HASHERS = [
'django.contrib.auth.hashers.MD5PasswordHasher',
]
# Use the dummy cache that does not cache anything
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.dummy.DummyCache'
},
'locmem': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'
}
}

View File

@ -0,0 +1,236 @@
from unittest import TestCase
from unittest.mock import MagicMock, patch
from openslides.core.models import Projector
from openslides.utils import collection
class TestCacheKeys(TestCase):
def test_get_collection_id_from_cache_key(self):
"""
Test that get_collection_id_from_cache_key works together with
get_single_element_cache_key.
"""
element = ('some/testkey', 42)
self.assertEqual(
element,
collection.get_collection_id_from_cache_key(
collection.get_single_element_cache_key(*element)))
def test_get_collection_id_from_cache_key_for_strings(self):
"""
Test get_collection_id_from_cache_key for strings
"""
element = ('some/testkey', 'my_config_value')
self.assertEqual(
element,
collection.get_collection_id_from_cache_key(
collection.get_single_element_cache_key(*element)))
def test_get_single_element_cache_key_prefix(self):
"""
Tests that the cache prefix is realy a prefix.
"""
element = ('some/testkey', 42)
cache_key = collection.get_single_element_cache_key(*element)
prefix = collection.get_single_element_cache_key_prefix(element[0])
self.assertTrue(cache_key.startswith(prefix))
def test_prefix_different_then_list(self):
"""
Test that the return value of get_single_element_cache_key_prefix is
something different then get_element_list_cache_key.
"""
collection_string = 'some/testkey'
prefix = collection.get_single_element_cache_key_prefix(collection_string)
list_cache_key = collection.get_element_list_cache_key(collection_string)
self.assertNotEqual(prefix, list_cache_key)
class TestGetModelFromCollectionString(TestCase):
def test_known_app(self):
projector_model = collection.get_model_from_collection_string('core/projector')
self.assertEqual(projector_model, Projector)
def test_unknown_app(self):
with self.assertRaises(ValueError):
collection.get_model_from_collection_string('invalid/model')
class TestCollectionElement(TestCase):
def test_from_values(self):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
self.assertEqual(collection_element.collection_string, 'testmodule/model')
self.assertEqual(collection_element.id, 42)
@patch('openslides.utils.collection.Collection')
@patch('openslides.utils.collection.cache')
def test_from_values_deleted(self, mock_cache, mock_collection):
"""
Tests that when createing a CollectionElement with deleted=True the element
is deleted from the cache.
"""
collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True)
self.assertTrue(collection_element.is_deleted())
mock_cache.delete.assert_called_with('testmodule/model:42')
mock_collection.assert_called_with('testmodule/model')
mock_collection().delete_id_from_cache.assert_called_with(42)
def test_as_channel_message(self):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
self.assertEqual(
collection_element.as_channels_message(),
{'collection_string': 'testmodule/model',
'id': 42,
'deleted': False})
def test_as_autoupdate_for_user(self):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
fake_user = MagicMock()
collection_element.get_access_permissions = MagicMock()
collection_element.get_access_permissions().get_restricted_data.return_value = 'restricted_data'
collection_element.get_full_data = MagicMock()
self.assertEqual(
collection_element.as_autoupdate_for_user(fake_user),
{'collection': 'testmodule/model',
'id': 42,
'action': 'changed',
'data': 'restricted_data'})
collection_element.get_full_data.assert_called_once_with()
def test_as_autoupdate_for_user_no_permission(self):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
fake_user = MagicMock()
collection_element.get_access_permissions = MagicMock()
collection_element.get_access_permissions().get_restricted_data.return_value = None
collection_element.get_full_data = MagicMock()
self.assertEqual(
collection_element.as_autoupdate_for_user(fake_user),
{'collection': 'testmodule/model',
'id': 42,
'action': 'deleted'})
collection_element.get_full_data.assert_called_once_with()
def test_as_autoupdate_for_user_deleted(self):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True)
fake_user = MagicMock()
self.assertEqual(
collection_element.as_autoupdate_for_user(fake_user),
{'collection': 'testmodule/model',
'id': 42,
'action': 'deleted'})
def test_get_instance_deleted(self):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True)
with self.assertRaises(RuntimeError):
collection_element.get_instance()
@patch('openslides.core.config.config')
def test_get_instance_config_str(self, mock_config):
mock_config.get_collection_string.return_value = 'core/config'
mock_config.__getitem__.return_value = 'config_value'
collection_element = collection.CollectionElement.from_values('core/config', 'my_config_value')
instance = collection_element.get_instance()
self.assertEqual(
instance,
{'key': 'my_config_value',
'value': 'config_value'})
def test_get_instance(self):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
collection_element.get_model = MagicMock()
collection_element.get_instance()
collection_element.get_model().objects.get_full_queryset().get.assert_called_once_with(pk=42)
@patch('openslides.utils.collection.cache')
def test_get_full_data_already_loaded(self, mock_cache):
"""
Test that the cache and the self.get_instance() is not hit, when the
instance is already loaded.
"""
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
collection_element.full_data = 'my_full_data'
collection_element.get_instance = MagicMock()
collection_element.get_full_data()
mock_cache.get.assert_not_called()
collection_element.get_instance.assert_not_called()
@patch('openslides.utils.collection.cache')
def test_get_full_data_from_cache(self, mock_cache):
"""
Test that the value from the cache is used not get_instance is not
called.
"""
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
collection_element.get_instance = MagicMock()
mock_cache.get.return_value = 'cache_value'
instance = collection_element.get_full_data()
self.assertEqual(instance, 'cache_value')
mock_cache.get.assert_called_once_with('testmodule/model:42')
collection_element.get_instance.assert_not_called
@patch('openslides.utils.collection.Collection')
@patch('openslides.utils.collection.cache')
def test_get_full_data_from_get_instance(self, mock_cache, mock_Collection):
"""
Test that the value from get_instance is used and saved to the cache
"""
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
collection_element.get_instance = MagicMock()
collection_element.get_access_permissions = MagicMock()
collection_element.get_access_permissions().get_full_data.return_value = 'get_instance_value'
mock_cache.get.return_value = None
instance = collection_element.get_full_data()
self.assertEqual(instance, 'get_instance_value')
mock_cache.get.assert_called_once_with('testmodule/model:42')
collection_element.get_instance.assert_called_once_with()
mock_cache.set.assert_called_once_with('testmodule/model:42', 'get_instance_value')
mock_Collection.assert_called_once_with('testmodule/model')
mock_Collection().add_id_to_cache.assert_called_once_with(42)
class TestCollection(TestCase):
@patch('openslides.utils.collection.CollectionElement')
@patch('openslides.utils.collection.cache')
def test_element_generator(self, mock_cache, mock_CollectionElement):
"""
Test with the following scenario: The collection has three elements. Two
are in the cache and one is not.
"""
test_collection = collection.Collection('testmodule/model')
test_collection.get_all_ids = MagicMock(return_value=set([1, 2, 3]))
test_collection.get_model = MagicMock()
test_collection.get_model().objects.get_full_queryset().filter.return_value = ['my_instance']
mock_cache.get_many.return_value = {
'testmodule/model:1': 'element1',
'testmodule/model:2': 'element2'}
list(test_collection.element_generator())
mock_cache.get_many.assert_called_once_with(
['testmodule/model:1', 'testmodule/model:2', 'testmodule/model:3'])
test_collection.get_model().objects.get_full_queryset().filter.assert_called_once_with(pk__in={3})
self.assertEqual(mock_CollectionElement.from_values.call_count, 2)
self.assertEqual(mock_CollectionElement.from_instance.call_count, 1)