Merge pull request #2404 from ostcar/cache2

Added caching system.

.gitignore

@@ -26,3 +26,4 @@ dist/*
 
 # Unit test and coverage reports
 .coverage
+tests/file/*

CHANGELOG

@@ -42,6 +42,7 @@ Other:
 - Fixed bug, that the last change of a config value was not send via autoupdate.
 - Added template hooks for plugins (in item detail view and motion poll form).
 - Used Django Channels instead of Tornado. Refactoring of the autoupdate process.
+- Added new caching system with support for Redis.
 
 
 Version 2.0 (2016-04-18)

DEVELOPMENT.rst

@@ -109,3 +109,31 @@ a. Running Angular.js test cases
 ''''''''''''''''''''''''''''''''
 
     $ node_modules/.bin/karma start tests/karma/karma.conf.js
+
+
+Installation Openslides in big mode
+===================================
+
+1. Install PostgreSQL und redis:
+
+apt-get install postgresql redis-server libpg-dev
+
+TODO: Configure postgresql
+
+2. Install python dependencies
+
+pip install django-redis asgi-redis psycopg2
+
+3. Change settings.py
+
+(See comments in the settings)
+
+The relevant settings are: DATABASES, CHANNEL_LAYERS, CACHES
+
+4. Start one or more workers:
+
+python manage.py runworker
+
+5. Start daphne. Set the DJANGO_SETTINGS_MODULE and the PYTHONPATH
+
+DJANGO_SETTINGS_MODULE=settings PYTHONPATH=personal_data/var/ daphne openslides.asgi:channel_layer

openslides/agenda/models.py

@@ -23,6 +23,14 @@ class ItemManager(models.Manager):
     Customized model manager with special methods for agenda tree and
     numbering.
     """
+    def get_full_queryset(self):
+        """
+        Returns the normal queryset with all items. In the background all
+        speakers and related items (topics, motions, assignments) are
+        prefetched from the database.
+        """
+        return self.get_queryset().prefetch_related('speakers', 'content_object')
+
     def get_only_agenda_items(self):
         """
         Generator, which yields only agenda items. Skips hidden items.
@@ -276,20 +284,6 @@ class Item(RESTModelMixin, models.Model):
     def __str__(self):
         return self.title
 
-    def delete(self, with_children=False):
-        """
-        Delete the Item.
-
-        If with_children is True, all children of the item will be deleted as
-        well. If with_children is False, all children will be children of the
-        parent of the item.
-        """
-        if not with_children:
-            for child in self.children.all():
-                child.parent = self.parent
-                child.save()
-        super().delete()
-
     @property
     def title(self):
         """

openslides/agenda/projector.py

@@ -1,5 +1,6 @@
 from ..core.config import config
 from ..core.exceptions import ProjectorException
+from ..utils.collection import CollectionElement
 from ..utils.projector import ProjectorElement
 from .models import Item
 
@@ -60,10 +61,13 @@ class ListOfSpeakersSlide(ProjectorElement):
                     # Yield last speakers
                     yield speaker.user
 
-    def need_full_update_for_this(self, collection_element):
+    def get_collection_elements_required_for_this(self, collection_element, config_entry):
-        # Full update if item changes because then we may have new speakers
+        output = super().get_collection_elements_required_for_this(collection_element, config_entry)
-        # and therefor need new users.
+        # Full update if item changes because then we may have new
-        return collection_element.collection_string == Item.get_collection_string()
+        # candidates and therefor need new users.
+        if collection_element == CollectionElement.from_values(Item.get_collection_string(), config_entry.get('id')):
+            output.extend(self.get_requirements_as_collection_elements(config_entry))
+        return output
 
 
 class CurrentListOfSpeakersSlide(ProjectorElement):

openslides/agenda/views.py

@@ -58,26 +58,6 @@ class ItemViewSet(ListModelMixin, RetrieveModelMixin, UpdateModelMixin, GenericV
             result = False
         return result
 
-    def check_object_permissions(self, request, obj):
-        """
-        Checks if the requesting user has permission to see also an
-        organizational item if it is one.
-        """
-        # TODO: Move this logic to access_permissions.ItemAccessPermissions.
-        if obj.is_hidden() and not request.user.has_perm('agenda.can_see_hidden_items'):
-            self.permission_denied(request)
-
-    def get_queryset(self):
-        """
-        Filters organizational items if the user has no permission to see them.
-        """
-        # TODO: Move this logic to access_permissions.ItemAccessPermissions.
-        queryset = super().get_queryset()
-        if not self.request.user.has_perm('agenda.can_see_hidden_items'):
-            pk_list = [item.pk for item in Item.objects.get_only_agenda_items()]
-            queryset = queryset.filter(pk__in=pk_list)
-        return queryset
-
     @detail_route(methods=['POST', 'DELETE'])
     def manage_speaker(self, request, pk=None):
         """
@@ -243,31 +223,6 @@ class ItemViewSet(ListModelMixin, RetrieveModelMixin, UpdateModelMixin, GenericV
         # Initiate response.
         return Response({'detail': _('List of speakers successfully sorted.')})
 
-    @list_route(methods=['get', 'put'])
-    def tree(self, request):
-        """
-        Returns or sets the agenda tree.
-        """
-        if request.method == 'PUT':
-            if not (request.user.has_perm('agenda.can_manage') and
-                    request.user.has_perm('agenda.can_see_hidden_items')):
-                self.permission_denied(request)
-            try:
-                tree = request.data['tree']
-            except KeyError as error:
-                response = Response({'detail': 'Agenda tree is missing.'}, status=400)
-            else:
-                try:
-                    Item.objects.set_tree(tree)
-                except ValueError as error:
-                    response = Response({'detail': str(error)}, status=400)
-                else:
-                    response = Response({'detail': 'Agenda tree successfully updated.'})
-        else:
-            # request.method == 'GET'
-            response = Response(Item.objects.get_tree())
-        return response
-
     @list_route(methods=['post'])
     def numbering(self, request):
         """

openslides/assignments/models.py

@@ -1,7 +1,7 @@
 from collections import OrderedDict
 
 from django.conf import settings
-from django.contrib.contenttypes.models import ContentType
+from django.contrib.contenttypes.fields import GenericRelation
 from django.db import models
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext_noop
@@ -16,6 +16,7 @@ from openslides.poll.models import (
     CollectDefaultVotesMixin,
     PublishPollMixin,
 )
+from openslides.utils.autoupdate import inform_changed_data
 from openslides.utils.exceptions import OpenSlidesError
 from openslides.utils.models import RESTModelMixin
 from openslides.utils.search import user_name_helper
@@ -50,12 +51,30 @@ class AssignmentRelatedUser(RESTModelMixin, models.Model):
         return self.assignment
 
 
+class AssignmentManager(models.Manager):
+    """
+    Customized model manager to support our get_full_queryset method.
+    """
+    def get_full_queryset(self):
+        """
+        Returns the normal queryset with all assignments. In the background
+        all related users (candidates), the related agenda item and all
+        polls are prefetched from the database.
+        """
+        return self.get_queryset().prefetch_related(
+            'related_users',
+            'agenda_items',
+            'polls')
+
+
 class Assignment(RESTModelMixin, models.Model):
     """
     Model for assignments.
     """
     access_permissions = AssignmentAccessPermissions()
 
+    objects = AssignmentManager()
+
     PHASE_SEARCH = 0
     PHASE_VOTING = 1
     PHASE_FINISHED = 2
@@ -111,6 +130,10 @@ class Assignment(RESTModelMixin, models.Model):
     Tags for the assignment.
     """
 
+    # In theory there could be one then more agenda_item. But we support only
+    # one. See the property agenda_item.
+    agenda_items = GenericRelation(Item, related_name='assignments')
+
     class Meta:
         default_permissions = ()
         permissions = (
@@ -187,6 +210,7 @@ class Assignment(RESTModelMixin, models.Model):
         Delete the connection from the assignment to the user.
         """
         self.assignment_related_users.filter(user=user).delete()
+        inform_changed_data(self)
 
     def set_phase(self, phase):
         """
@@ -290,8 +314,9 @@ class Assignment(RESTModelMixin, models.Model):
         """
         Returns the related agenda item.
         """
-        content_type = ContentType.objects.get_for_model(self)
+        # We support only one agenda item so just return the first element of
-        return Item.objects.get(object_id=self.pk, content_type=content_type)
+        # the queryset.
+        return self.agenda_items.all()[0]
 
     @property
     def agenda_item_id(self):

openslides/assignments/projector.py

@@ -1,4 +1,5 @@
 from ..core.exceptions import ProjectorException
+from ..utils.collection import CollectionElement
 from ..utils.projector import ProjectorElement
 from .models import Assignment, AssignmentPoll
 
@@ -46,7 +47,10 @@ class AssignmentSlide(ProjectorElement):
                     for option in poll.options.all():
                         yield option.candidate
 
-    def need_full_update_for_this(self, collection_element):
+    def get_collection_elements_required_for_this(self, collection_element, config_entry):
+        output = super().get_collection_elements_required_for_this(collection_element, config_entry)
         # Full update if assignment changes because then we may have new
         # candidates and therefor need new users.
-        return collection_element.collection_string == Assignment.get_collection_string()
+        if collection_element == CollectionElement.from_values(Assignment.get_collection_string(), config_entry.get('id')):
+            output.extend(self.get_requirements_as_collection_elements(config_entry))
+        return output

openslides/core/access_permissions.py

@@ -84,7 +84,13 @@ class ConfigAccessPermissions(BaseAccessPermissions):
         Returns the serlialized config data.
         """
         from .config import config
+        from .models import ConfigStore
 
         # Attention: The format of this response has to be the same as in
         # the retrieve method of ConfigViewSet.
-        return {'key': instance.key, 'value': config[instance.key]}
+        if isinstance(instance, ConfigStore):
+            result = {'key': instance.key, 'value': config[instance.key]}
+        else:
+            # It is possible, that the caching system already sends the correct data as "instance".
+            result = instance
+        return result

openslides/core/apps.py

@@ -16,7 +16,6 @@ class CoreAppConfig(AppConfig):
         from django.db.models import signals
         from openslides.core.config import config
         from openslides.core.signals import post_permission_creation
-        from openslides.utils.autoupdate import inform_changed_data_receiver, inform_deleted_data_receiver
         from openslides.utils.rest_api import router
         from openslides.utils.search import index_add_instance, index_del_instance
         from .config_variables import get_config_variables
@@ -45,15 +44,6 @@ class CoreAppConfig(AppConfig):
         router.register(self.get_model('Tag').get_collection_string(), TagViewSet)
         router.register(self.get_model('ConfigStore').get_collection_string(), ConfigViewSet, 'config')
 
-        # Update data when any model of any installed app is saved or deleted.
-        # TODO: Test if the m2m_changed signal is also needed.
-        signals.post_save.connect(
-            inform_changed_data_receiver,
-            dispatch_uid='inform_changed_data_receiver')
-        signals.post_delete.connect(
-            inform_deleted_data_receiver,
-            dispatch_uid='inform_deleted_data_receiver')
-
         # Update the search when a model is saved or deleted
         signals.post_save.connect(
             index_add_instance,

openslides/core/migrations/0005_auto_20160918_2104.py

@@ -2,19 +2,10 @@
 # Generated by Django 1.10.1 on 2016-09-18 19:04
 from __future__ import unicode_literals
 
-from django.db import migrations, models
+from django.db import migrations
-
-from openslides.utils.autoupdate import (
-    inform_changed_data_receiver,
-    inform_deleted_data_receiver,
-)
 
 
 def move_custom_slides_to_topics(apps, schema_editor):
-    # Disconnect autoupdate. We do not want to trigger it here.
-    models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver')
-    models.signals.post_save.disconnect(dispatch_uid='inform_deleted_data_receiver')
-
     # We get the model from the versioned app registry;
     # if we directly import it, it will be the wrong version.
     ContentType = apps.get_model('contenttypes', 'ContentType')
@@ -39,14 +30,6 @@ def move_custom_slides_to_topics(apps, schema_editor):
     CustomSlide.objects.all().delete()
     content_type_custom_slide.delete()
 
-    # Reconnect autoupdate.
-    models.signals.post_save.connect(
-        inform_changed_data_receiver,
-        dispatch_uid='inform_changed_data_receiver')
-    models.signals.post_delete.connect(
-        inform_deleted_data_receiver,
-        dispatch_uid='inform_deleted_data_receiver')
-
 
 class Migration(migrations.Migration):
 

openslides/core/models.py

@@ -15,6 +15,19 @@ from .access_permissions import (
 from .exceptions import ProjectorException
 
 
+class ProjectorManager(models.Manager):
+    """
+    Customized model manager to support our get_full_queryset method.
+    """
+    def get_full_queryset(self):
+        """
+        Returns the normal queryset with all projectors. In the background
+        projector defaults are prefetched from the database.
+        """
+        return self.get_queryset().prefetch_related(
+            'projectiondefaults')
+
+
 class Projector(RESTModelMixin, models.Model):
     """
     Model for all projectors. At the moment we support only one projector,
@@ -57,6 +70,8 @@ class Projector(RESTModelMixin, models.Model):
     """
     access_permissions = ProjectorAccessPermissions()
 
+    objects = ProjectorManager()
+
     config = JSONField()
 
     scale = models.IntegerField(default=0)
@@ -131,50 +146,35 @@ class Projector(RESTModelMixin, models.Model):
             if element is not None:
                 yield from element.get_requirements(value)
 
-    def collection_element_is_shown(self, collection_element):
+    def get_collection_elements_required_for_this(self, collection_element):
         """
-        Returns True if this collection element is shown on this projector.
+        Returns an iterable of CollectionElements that have to be sent to this
+        projector according to the given collection_element and information.
         """
-        for requirement in self.get_all_requirements():
+        from .config import config
-            if (requirement.get_collection_string() == collection_element.collection_string and
+
-                    requirement.pk == collection_element.id):
+        output = []
-                result = True
+        changed_fields = collection_element.information.get('changed_fields', [])
-                break
+        if (collection_element.collection_string == self.get_collection_string() and
+                changed_fields and
+                'config' not in changed_fields):
+            # Projector model changed without changeing the projector config. So we just send this data.
+            output.append(collection_element)
         else:
-            result = False
+            # It is necessary to parse all active projector elements to check whether they require some data.
-        return result
+            this_projector = collection_element.collection_string == self.get_collection_string() and collection_element.id == self.pk
-
+            collection_element.information['this_projector'] = this_projector
-    @classmethod
-    def get_projectors_that_show_this(cls, collection_element):
-        """
-        Returns a list of the projectors that show this collection element.
-        """
-        result = []
-        for projector in cls.objects.all():
-            if projector.collection_element_is_shown(collection_element):
-                result.append(projector)
-        return result
-
-    def need_full_update_for_this(self, collection_element):
-        """
-        Returns True if this projector needs to be updated with all
-        instances as defined in get_all_requirements() because one active
-        projector element requires this.
-        """
-        # Get all elements from all apps.
             elements = {}
             for element in ProjectorElement.get_all():
                 elements[element.name] = element
-
             for key, value in self.config.items():
                 element = elements.get(value['name'])
-            if element is not None and element.need_full_update_for_this(collection_element):
+                if element is not None:
-                result = True
+                    output.extend(element.get_collection_elements_required_for_this(collection_element, value))
-                break
+            # If config changed, send also this to the projector.
-        else:
+            if collection_element.collection_string == config.get_collection_string():
-            result = False
+                output.append(collection_element)
-
+        return output
-        return result
 
 
 class ProjectionDefault(RESTModelMixin, models.Model):

openslides/core/signals.py

@@ -20,8 +20,7 @@ def delete_django_app_permissions(sender, **kwargs):
         Q(app_label='auth') |
         Q(app_label='contenttypes') |
         Q(app_label='sessions'))
-    for permission in Permission.objects.filter(content_type__in=contenttypes):
+    Permission.objects.filter(content_type__in=contenttypes).delete()
-        permission.delete()
 
 
 def create_builtin_projection_defaults(**kwargs):

openslides/core/views.py

@@ -18,6 +18,7 @@ from django.utils.timezone import now
 
 from openslides import __version__ as version
 from openslides.utils import views as utils_views
+from openslides.utils.collection import Collection, CollectionElement
 from openslides.utils.plugins import (
     get_plugin_description,
     get_plugin_verbose_name,
@@ -41,7 +42,7 @@ from .access_permissions import (
 )
 from .config import config
 from .exceptions import ConfigError, ConfigNotFound
-from .models import ChatMessage, ProjectionDefault, Projector, Tag
+from .models import ChatMessage, ConfigStore, ProjectionDefault, Projector, Tag
 
 
 # Special Django views
@@ -606,22 +607,26 @@ class ConfigViewSet(ViewSet):
 
     def list(self, request):
         """
-        Lists all config variables. Everybody can see them.
+        Lists all config variables.
         """
-        return Response([{'key': key, 'value': value} for key, value in config.items()])
+        collection = Collection(config.get_collection_string())
+        return Response(collection.as_list_for_user(request.user))
 
     def retrieve(self, request, *args, **kwargs):
         """
-        Retrieves a config variable. Everybody can see it.
+        Retrieves a config variable.
         """
         key = kwargs['pk']
+        collection_element = CollectionElement.from_values(config.get_collection_string(), key)
         try:
-            value = config[key]
+            content = collection_element.as_dict_for_user(request.user)
-        except ConfigNotFound:
+        except ConfigStore.DoesNotExist:
             raise Http404
-        # Attention: The format of this response has to be the same as in
+        if content is None:
-        # the get_full_data method of ConfigAccessPermissions.
+            # If content is None, the user has no permissions to see the item.
-        return Response({'key': key, 'value': value})
+            # See ConfigAccessPermissions or rather its parent class.
+            self.permission_denied()
+        return Response(content)
 
     def update(self, request, *args, **kwargs):
         """

openslides/mediafiles/views.py

@@ -15,13 +15,6 @@ class MediafileViewSet(ModelViewSet):
     access_permissions = MediafileAccessPermissions()
     queryset = Mediafile.objects.all()
 
-    def get_queryset(self):
-        queryset = super().get_queryset()
-        user = self.request.user
-        if not user.has_perm('mediafiles.can_see_private'):
-            queryset = queryset.filter(private=False)
-        return queryset
-
     def check_view_permissions(self):
         """
         Returns True if the user has required permissions.

openslides/motions/migrations/0004_auto_20160907_2343.py

@@ -5,16 +5,11 @@ from __future__ import unicode_literals
 import django.db.models.deletion
 from django.db import migrations, models
 
-from openslides.utils.autoupdate import inform_changed_data_receiver
-
 
 def change_label_of_state(apps, schema_editor):
     """
     Changes the label of former state "commited a bill" to "refered to committee".
     """
-    # Disconnect autoupdate. We do not want to trigger it here.
-    models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver')
-
     # We get the model from the versioned app registry;
     # if we directly import it, it will be the wrong version.
     State = apps.get_model('motions', 'State')
@@ -29,19 +24,11 @@ def change_label_of_state(apps, schema_editor):
         state.action_word = 'Refer to committee'
         state.save()
 
-    # Reconnect autoupdate.
-    models.signals.post_save.connect(
-        inform_changed_data_receiver,
-        dispatch_uid='inform_changed_data_receiver')
-
 
 def add_recommendation_labels(apps, schema_editor):
     """
     Adds recommendation labels to some of the built-in states.
     """
-    # Disconnect autoupdate. We do not want to trigger it here.
-    models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver')
-
     # We get the model from the versioned app registry;
     # if we directly import it, it will be the wrong version.
     State = apps.get_model('motions', 'State')

openslides/motions/models.py

@@ -1,5 +1,5 @@
 from django.conf import settings
-from django.contrib.contenttypes.models import ContentType
+from django.contrib.contenttypes.fields import GenericRelation
 from django.db import models
 from django.db.models import Max
 from django.utils import formats
@@ -28,14 +28,38 @@ from .access_permissions import (
 from .exceptions import WorkflowError
 
 
+class MotionManager(models.Manager):
+    """
+    Customized model manager to support our get_full_queryset method.
+    """
+    def get_full_queryset(self):
+        """
+        Returns the normal queryset with all motions. In the background we
+        join and prefetch all related models.
+        """
+        return (self.get_queryset()
+                .select_related('active_version')
+                .prefetch_related(
+                    'versions',
+                    'agenda_items',
+                    'log_messages',
+                    'polls',
+                    'attachments',
+                    'tags',
+                    'submitters',
+                    'supporters'))
+
+
 class Motion(RESTModelMixin, models.Model):
     """
-    The Motion Class.
+    Model for motions.
 
     This class is the main entry point to all other classes related to a motion.
     """
     access_permissions = MotionAccessPermissions()
 
+    objects = MotionManager()
+
     active_version = models.ForeignKey(
         'MotionVersion',
         on_delete=models.SET_NULL,
@@ -134,6 +158,10 @@ class Motion(RESTModelMixin, models.Model):
     Configurable fields for comments. Contains a list of strings.
     """
 
+    # In theory there could be one then more agenda_item. But we support only
+    # one. See the property agenda_item.
+    agenda_items = GenericRelation(Item, related_name='motions')
+
     class Meta:
         default_permissions = ()
         permissions = (
@@ -519,8 +547,9 @@ class Motion(RESTModelMixin, models.Model):
         """
         Returns the related agenda item.
         """
-        content_type = ContentType.objects.get_for_model(self)
+        # We support only one agenda item so just return the first element of
-        return Item.objects.get(object_id=self.pk, content_type=content_type)
+        # the queryset.
+        return self.agenda_items.all()[0]
 
     @property
     def agenda_item_id(self):
@@ -940,12 +969,29 @@ class State(RESTModelMixin, models.Model):
         return self.workflow
 
 
+class WorkflowManager(models.Manager):
+    """
+    Customized model manager to support our get_full_queryset method.
+    """
+    def get_full_queryset(self):
+        """
+        Returns the normal queryset with all workflows. In the background
+        the first state is joined and all states and next states are
+        prefetched from the database.
+        """
+        return (self.get_queryset()
+                .select_related('first_state')
+                .prefetch_related('states', 'states__next_states'))
+
+
 class Workflow(RESTModelMixin, models.Model):
     """
     Defines a workflow for a motion.
     """
     access_permissions = WorkflowAccessPermissions()
 
+    objects = WorkflowManager()
+
     name = models.CharField(max_length=255)
     """A string representing the workflow."""
 

openslides/motions/projector.py

@@ -1,4 +1,5 @@
 from ..core.exceptions import ProjectorException
+from ..utils.collection import CollectionElement
 from ..utils.projector import ProjectorElement
 from .models import Motion
 
@@ -26,10 +27,13 @@ class MotionSlide(ProjectorElement):
             yield from motion.submitters.all()
             yield from motion.supporters.all()
 
-    def need_full_update_for_this(self, collection_element):
+    def get_collection_elements_required_for_this(self, collection_element, config_entry):
+        output = super().get_collection_elements_required_for_this(collection_element, config_entry)
         # Full update if motion changes because then we may have new
         # submitters or supporters and therefor need new users.
         #
         # Add some logic here if we support live changing of workflows later.
         #
-        return collection_element.collection_string == Motion.get_collection_string()
+        if collection_element == CollectionElement.from_values(Motion.get_collection_string(), config_entry.get('id')):
+            output.extend(self.get_requirements_as_collection_elements(config_entry))
+        return output

openslides/motions/views.py

@@ -77,27 +77,6 @@ class MotionViewSet(ModelViewSet):
             result = False
         return result
 
-    def list(self, request, *args, **kwargs):
-        """
-        Customized view endpoint to list all motions.
-
-        Hides non public comment fields for some users.
-        """
-        response = super().list(request, *args, **kwargs)
-        for i, motion in enumerate(response.data):
-            response.data[i] = self.get_access_permissions().get_restricted_data(motion, self.request.user)
-        return response
-
-    def retrieve(self, request, *args, **kwargs):
-        """
-        Customized view endpoint to retrieve a motion.
-
-        Hides non public comment fields for some users.
-        """
-        response = super().retrieve(request, *args, **kwargs)
-        response.data = self.get_access_permissions().get_restricted_data(response.data, self.request.user)
-        return response
-
     def create(self, request, *args, **kwargs):
         """
         Customized view endpoint to create a new motion.

openslides/topics/models.py

@@ -1,4 +1,4 @@
-from django.contrib.contenttypes.models import ContentType
+from django.contrib.contenttypes.fields import GenericRelation
 from django.db import models
 
 from ..agenda.models import Item
@@ -7,16 +7,35 @@ from ..utils.models import RESTModelMixin
 from .access_permissions import TopicAccessPermissions
 
 
+class TopicManager(models.Manager):
+    """
+    Customized model manager to support our get_full_queryset method.
+    """
+    def get_full_queryset(self):
+        """
+        Returns the normal queryset with all topics. In the background all
+        attachments and the related agenda item are prefetched from the
+        database.
+        """
+        return self.get_queryset().prefetch_related('attachments', 'agenda_items')
+
+
 class Topic(RESTModelMixin, models.Model):
     """
     Model for slides with custom content. Used to be called custom slide.
     """
     access_permissions = TopicAccessPermissions()
 
+    objects = TopicManager()
+
     title = models.CharField(max_length=256)
     text = models.TextField(blank=True)
     attachments = models.ManyToManyField(Mediafile, blank=True)
 
+    # In theory there could be one then more agenda_item. But we support only
+    # one. See the property agenda_item.
+    agenda_items = GenericRelation(Item, related_name='topics')
+
     class Meta:
         default_permissions = ()
 
@@ -28,8 +47,9 @@ class Topic(RESTModelMixin, models.Model):
         """
         Returns the related agenda item.
         """
-        content_type = ContentType.objects.get_for_model(self)
+        # We support only one agenda item so just return the first element of
-        return Item.objects.get(object_id=self.pk, content_type=content_type)
+        # the queryset.
+        return self.agenda_items.all()[0]
 
     @property
     def agenda_item_id(self):

openslides/users/access_permissions.py

@@ -15,15 +15,9 @@ class UserAccessPermissions(BaseAccessPermissions):
         """
         Returns different serializer classes with respect user's permissions.
         """
-        from .serializers import UserCanSeeSerializer, UserCanSeeExtraSerializer, UserFullSerializer
+        from .serializers import UserFullSerializer
 
-        if (user is None or (user.has_perm('users.can_see_extra_data') and user.has_perm('users.can_manage'))):
+        return UserFullSerializer
-            serializer_class = UserFullSerializer
-        elif user.has_perm('users.can_see_extra_data'):
-            serializer_class = UserCanSeeExtraSerializer
-        else:
-            serializer_class = UserCanSeeSerializer
-        return serializer_class
 
     def get_restricted_data(self, full_data, user):
         """

openslides/users/migrations/0004_groups.py

@@ -2,9 +2,7 @@
 # Generated by Django 1.9.7 on 2016-08-01 14:54
 from __future__ import unicode_literals
 
-from django.db import migrations, models
+from django.db import migrations
-
-from openslides.utils.autoupdate import inform_changed_data_receiver
 
 
 def migrate_groups_and_user_permissions(apps, schema_editor):
@@ -20,9 +18,6 @@ def migrate_groups_and_user_permissions(apps, schema_editor):
     - the name of the first group is 'Guests',
     - the name of the second group is 'Registered users'.
     """
-    # Disconnect autoupdate. We do not want to trigger it here.
-    models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver')
-
     User = apps.get_model('users', 'User')
     Group = apps.get_model('auth', 'Group')
 
@@ -52,11 +47,6 @@ def migrate_groups_and_user_permissions(apps, schema_editor):
                 for permission in group_registered.permissions.all():
                     group.permissions.add(permission)
 
-    # Reconnect autoupdate.
-    models.signals.post_save.connect(
-        inform_changed_data_receiver,
-        dispatch_uid='inform_changed_data_receiver')
-
 
 class Migration(migrations.Migration):
 

openslides/users/models.py

@@ -21,8 +21,15 @@ from .access_permissions import UserAccessPermissions
 class UserManager(BaseUserManager):
     """
     Customized manager that creates new users only with a password and a
-    username.
+    username. It also supports our get_full_queryset method.
     """
+    def get_full_queryset(self):
+        """
+        Returns the normal queryset with all users. In the background all
+        groups are prefetched from the database.
+        """
+        return self.get_queryset().prefetch_related('groups')
+
     def create_user(self, username, password, **kwargs):
         """
         Creates a new user only with a password and a username.

openslides/users/serializers.py

@@ -21,70 +21,15 @@ USERCANSEESERIALIZER_FIELDS = (
     'number',
     'about_me',
     'groups',
-            'is_committee',
-        )
-
-
-class UserCanSeeSerializer(ModelSerializer):
-    """
-    Serializer for users.models.User objects to be used by users who have
-    only the permission to see users and to change some date of theirselfs.
-
-    Attention: Viewset has to ensure that a user can update only himself.
-    """
-    class Meta:
-        model = User
-        fields = USERCANSEESERIALIZER_FIELDS
-        read_only_fields = (
-            'number',
-            'groups',
-            'is_comittee',
-        )
-
-
-USERCANSEEEXTRASERIALIZER_FIELDS = (
-            'id',
     'is_present',
-            'username',
-            'title',
-            'first_name',
-            'last_name',
-            'number',
-            'structure_level',
-            'about_me',
-            'comment',
-            'groups',
-            'is_active',
     'is_committee',
-        )
+)
 
 
-class UserCanSeeExtraSerializer(ModelSerializer):
+USERCANSEEEXTRASERIALIZER_FIELDS = USERCANSEESERIALIZER_FIELDS + (
-    """
-    Serializer for users.models.User objects to be used by users who have
-    the permission to see users with extra data and to change some date of
-    theirselfs.
-
-    Attention: Viewset has to ensure that a user can update only himself.
-    """
-    groups = IdPrimaryKeyRelatedField(
-        many=True,
-        queryset=Group.objects.exclude(pk=1),
-        help_text=ugettext_lazy('The groups this user belongs to. A user will '
-                                'get all permissions granted to each of '
-                                'his/her groups.'))
-
-    class Meta:
-        model = User
-        fields = USERCANSEEEXTRASERIALIZER_FIELDS
-        read_only_fields = (
-            'is_present',
-            'number',
     'comment',
-            'groups',
-            'is_comittee',
     'is_active',
-        )
+)
 
 
 class UserFullSerializer(ModelSerializer):

openslides/users/views.py

@@ -18,7 +18,7 @@ from ..utils.views import APIView, PDFView
 from .access_permissions import UserAccessPermissions
 from .models import Group, User
 from .pdf import users_passwords_to_pdf, users_to_pdf
-from .serializers import GroupSerializer, UserFullSerializer
+from .serializers import GroupSerializer
 
 
 # Viewsets for the REST API
@@ -49,19 +49,6 @@ class UserViewSet(ModelViewSet):
             result = False
         return result
 
-    def get_serializer_class(self):
-        """
-        Returns different serializer classes with respect to action.
-        """
-        if self.action in ('create', 'partial_update', 'update'):
-            # Return the UserFullSerializer for edit requests.
-            serializer_class = UserFullSerializer
-        else:
-            # Return different serializers according to user permsissions via
-            # access permissions class.
-            serializer_class = super().get_serializer_class()
-        return serializer_class
-
     def update(self, request, *args, **kwargs):
         """
         Customized view endpoint to update an user.
@@ -138,7 +125,7 @@ class GroupViewSet(ModelViewSet):
     partial_update, update and destroy.
     """
     metadata_class = GroupViewSetMetadata
-    queryset = Group.objects.all()
+    queryset = Group.objects.prefetch_related('permissions', 'permissions__content_type')
     serializer_class = GroupSerializer
 
     def check_view_permissions(self):

openslides/utils/access_permissions.py

@@ -68,7 +68,7 @@ class BaseAccessPermissions(object, metaclass=SignalConnectMetaClass):
         Hint: You should override this method if your get_serializer_class()
         method returns different serializers for different users or if you
         have access restrictions in your view or viewset in methods like
-        retrieve(), list() or check_object_permissions().
+        retrieve() or list().
         """
         if self.check_permissions(user):
             data = full_data
@@ -78,8 +78,8 @@ class BaseAccessPermissions(object, metaclass=SignalConnectMetaClass):
 
     def get_projector_data(self, full_data):
         """
-        Returns the serialized data for the projector. Returns None if has no
+        Returns the serialized data for the projector. Returns None if the
-        access to this specific data. Returns reduced data if the user has
+        user has no access to this specific data. Returns reduced data if
-        limited access. Default: Returns full data.
+        the user has limited access. Default: Returns full data.
         """
         return full_data

openslides/utils/autoupdate.py

@@ -11,7 +11,7 @@ from ..core.config import config
 from ..core.models import Projector
 from ..users.auth import AnonymousUser
 from ..users.models import User
-from .collection import CollectionElement
+from .collection import Collection, CollectionElement
 
 
 def get_logged_in_users():
@@ -23,21 +23,6 @@ def get_logged_in_users():
     return User.objects.exclude(session=None).filter(session__expire_date__gte=timezone.now()).distinct()
 
 
-def get_projector_element_data(projector):
-    """
-    Returns a list of dicts that are required for a specific projector.
-
-    The argument projector has to be a projector instance.
-    """
-    output = []
-    for requirement in projector.get_all_requirements():
-        required_collection_element = CollectionElement.from_instance(requirement)
-        element_dict = required_collection_element.as_autoupdate_for_projector()
-        if element_dict is not None:
-            output.append(element_dict)
-    return output
-
-
 @channel_session_user_from_http
 def ws_add_site(message):
     """
@@ -80,15 +65,14 @@ def ws_add_projector(message, projector_id):
             Group('projector-{}'.format(projector_id)).add(message.reply_channel)
 
             # Send all elements that are on the projector.
-            output = get_projector_element_data(projector)
+            output = []
+            for requirement in projector.get_all_requirements():
+                required_collection_element = CollectionElement.from_instance(requirement)
+                output.append(required_collection_element.as_autoupdate_for_projector())
 
             # Send all config elements.
-            for key, value in config.items():
+            collection = Collection(config.get_collection_string())
-                output.append({
+            output.extend(collection.as_autoupdate_for_projector())
-                    'collection': config.get_collection_string(),
-                    'id': key,
-                    'action': 'changed',
-                    'data': {'key': key, 'value': value}})
 
             # Send the projector instance.
             collection_element = CollectionElement.from_instance(projector)
@@ -107,64 +91,33 @@ def ws_disconnect_projector(message, projector_id):
 
 def send_data(message):
     """
-    Informs all users about changed data.
+    Informs all site users and projector clients about changed data.
     """
     collection_element = CollectionElement.from_values(**message)
 
-    # Loop over all logged in users and the anonymous user.
+    # Loop over all logged in site users and the anonymous user and send changed data.
     for user in itertools.chain(get_logged_in_users(), [AnonymousUser()]):
         channel = Group('user-{}'.format(user.id))
-        output = collection_element.as_autoupdate_for_user(user)
+        output = [collection_element.as_autoupdate_for_user(user)]
-        if output is None:
+        channel.send({'text': json.dumps(output)})
-            # There are no data for the user so he can't see the object. Skip him.
-            continue
-        channel.send({'text': json.dumps([output])})
 
-    # Get the projector elements where data have to be sent and if whole projector
+    # Loop over all projectors and send data that they need.
-    # has to be updated.
+    for projector in Projector.objects.all():
-    if collection_element.collection_string == config.get_collection_string():
+        if collection_element.is_deleted():
-        # Config elements are always send to each projector
+            output = [collection_element.as_autoupdate_for_projector()]
-        projectors = Projector.objects.all()
-        send_all = None  # The decission is done later
-    elif collection_element.collection_string == Projector.get_collection_string():
-        # Update a projector, when the projector element is updated.
-        projectors = [collection_element.get_instance()]
-        send_all = True
-    elif collection_element.is_deleted():
-        projectors = Projector.objects.all()
-        send_all = False
         else:
-        # Other elements are only send to the projector they are currently shown
+            collection_elements = projector.get_collection_elements_required_for_this(collection_element)
-        projectors = Projector.get_projectors_that_show_this(collection_element)
+            output = [collection_element.as_autoupdate_for_projector() for collection_element in collection_elements]
-        send_all = None  # The decission is done later
-
-    broadcast_id = config['projector_broadcast']
-    if broadcast_id > 0:
-        projectors = Projector.objects.all()  # Also the broadcasted projector should get his data
-        send_all = True
-        broadcast_projector_data = get_projector_element_data(Projector.objects.get(pk=broadcast_id))
-        broadcast_projector_data.append(CollectionElement.from_values(
-            collection_string=Projector.get_collection_string(), id=broadcast_id).as_autoupdate_for_projector())
-    else:
-        broadcast_projector_data = None
-
-    for projector in projectors:
-        if send_all is None:
-            send_all = projector.need_full_update_for_this(collection_element)
-        if send_all:
-            if broadcast_projector_data is None:
-                output = get_projector_element_data(projector)
-            else:
-                output = broadcast_projector_data
-        else:
-            output = []
-        output.append(collection_element.as_autoupdate_for_projector())
         if output:
             Group('projector-{}'.format(projector.pk)).send(
                 {'text': json.dumps(output)})
 
 
-def inform_changed_data(instance, is_deleted=False):
+def inform_changed_data(instance, information=None):
+    """
+    Informs the autoupdate system and the caching system about the creation or
+    update of an element.
+    """
     try:
         root_instance = instance.get_root_rest_element()
     except AttributeError:
@@ -173,29 +126,37 @@ def inform_changed_data(instance, is_deleted=False):
     else:
         collection_element = CollectionElement.from_instance(
             root_instance,
-            is_deleted=is_deleted and instance == root_instance)
+            information=information)
+        # If currently there is an open database transaction, then the
+        # send_autoupdate function is only called, when the transaction is
+        # commited. If there is currently no transaction, then the function
+        # is called immediately.
+        transaction.on_commit(lambda: send_autoupdate(collection_element))
 
-        # If currently there is an open database transaction, then the following
+
-        # function is only called, when the transaction is commited. If there
+def inform_deleted_data(collection_string, id, information=None):
-        # is currently no transaction, then the function is called immediately.
+    """
-        def send_autoupdate():
+    Informs the autoupdate system and the caching system about the deletion of
+    an element.
+    """
+    collection_element = CollectionElement.from_values(
+        collection_string=collection_string,
+        id=id,
+        deleted=True,
+        information=information)
+    # If currently there is an open database transaction, then the
+    # send_autoupdate function is only called, when the transaction is
+    # commited. If there is currently no transaction, then the function
+    # is called immediately.
+    transaction.on_commit(lambda: send_autoupdate(collection_element))
+
+
+def send_autoupdate(collection_element):
+    """
+    Helper function, that sends a collection_element through a channel to the
+    autoupdate system.
+    """
     try:
         Channel('autoupdate.send_data').send(collection_element.as_channels_message())
     except ChannelLayer.ChannelFull:
         pass
-
-        transaction.on_commit(send_autoupdate)
-
-
-def inform_changed_data_receiver(sender, instance, **kwargs):
-    """
-    Receiver for the inform_changed_data function to use in a signal.
-    """
-    inform_changed_data(instance)
-
-
-def inform_deleted_data_receiver(sender, instance, **kwargs):
-    """
-    Receiver for the inform_changed_data function to use in a signal.
-    """
-    inform_changed_data(instance, is_deleted=True)

openslides/utils/collection.py

@@ -1,25 +1,41 @@
 from django.apps import apps
+from django.core.cache import cache, caches
 
 
 class CollectionElement:
     @classmethod
-    def from_instance(cls, instance, is_deleted=False):
+    def from_instance(cls, instance, deleted=False, information=None):
         """
         Returns a collection element from a database instance.
+
+        This will also update the instance in the cache.
+
+        If deleted is set to True, the element is deleted from the cache.
         """
-        return cls(instance=instance, is_deleted=is_deleted)
+        return cls(instance=instance, deleted=deleted, information=information)
 
     @classmethod
-    def from_values(cls, collection_string, id, is_deleted=False):
+    def from_values(cls, collection_string, id, deleted=False, full_data=None, information=None):
         """
         Returns a collection element from a collection_string and an id.
-        """
-        return cls(collection_string=collection_string, id=id, is_deleted=is_deleted)
 
-    def __init__(self, instance=None, is_deleted=False, collection_string=None, id=None):
+        If deleted is set to True, the element is deleted from the cache.
+
+        With the argument full_data, the content of the CollectionElement can be set.
+        It has to be a dict in the format that is used be access_permission.get_full_data().
+        """
+        return cls(collection_string=collection_string, id=id, deleted=deleted,
+                   full_data=full_data, information=information)
+
+    def __init__(self, instance=None, deleted=False, collection_string=None, id=None,
+                 full_data=None, information=None):
         """
         Do not use this. Use the methods from_instance() or from_values().
         """
+        self.instance = instance
+        self.deleted = deleted
+        self.full_data = full_data
+        self.information = information or {}
         if instance is not None:
             self.collection_string = instance.get_collection_string()
             self.id = instance.pk
@@ -31,62 +47,83 @@ class CollectionElement:
                 'Invalid state. Use CollectionElement.from_instance() or '
                 'CollectionElement.from_values() but not CollectionElement() '
                 'directly.')
-        self.instance = instance
+
-        self.deleted = is_deleted
+        if self.is_deleted():
+            # Delete the element from the cache, if self.is_deleted() is True:
+            self.delete_from_cache()
+        elif instance is not None:
+            # If this element is created via instance and the instance is not deleted
+            # then update the cache.
+            self.save_to_cache()
+
+    def __eq__(self, collection_element):
+        """
+        Compares two collection_elements.
+
+        Two collection elements are equal, if they have the same collection_string
+        and id.
+        """
+        return (self.collection_string == collection_element.collection_string and
+                self.id == collection_element.id)
 
     def as_channels_message(self):
         """
         Returns a dictonary that can be used to send the object through the
         channels system.
         """
-        return {
+        channel_message = {
             'collection_string': self.collection_string,
             'id': self.id,
-            'is_deleted': self.is_deleted()}
+            'deleted': self.is_deleted()}
+        if self.information is not None:
+            channel_message['information'] = self.information
+        return channel_message
+
+    def as_autoupdate(self, method, *args):
+        """
+        Only for internal use. Do not use it directly. Use as_autoupdate_for_user()
+        or as_autoupdate_for_projector().
+        """
+        output = {
+            'collection': self.collection_string,
+            'id': self.id,
+            'action': 'deleted' if self.is_deleted() else 'changed',
+        }
+        if not self.is_deleted():
+            data = getattr(self.get_access_permissions(), method)(
+                self.get_full_data(),
+                *args)
+            if data is None:
+                # The user is not allowed to see this element. Set action to deleted.
+                output['action'] = 'deleted'
+            else:
+                output['data'] = data
+        return output
 
     def as_autoupdate_for_user(self, user):
         """
         Returns a dict that can be sent through the autoupdate system for a site
         user.
-
-        Returns None if the user can not see the element.
         """
-        output = {
+        return self.as_autoupdate(
-            'collection': self.collection_string,
+            'get_restricted_data',
-            'id': self.id,
+            user)
-            'action': 'deleted' if self.is_deleted() else 'changed',
-        }
-        if not self.is_deleted():
-            data = self.get_access_permissions().get_restricted_data(
-                self.get_full_data(), user)
-            if data is None:
-                # The user is not allowed to see this element. Reset output to None.
-                output = None
-            else:
-                output['data'] = data
-        return output
 
     def as_autoupdate_for_projector(self):
         """
         Returns a dict that can be sent through the autoupdate system for the
         projector.
-
-        Returns None if the projector can not see the element.
         """
-        output = {
+        return self.as_autoupdate(
-            'collection': self.collection_string,
+            'get_projector_data')
-            'id': self.id,
+
-            'action': 'deleted' if self.is_deleted() else 'changed',
+    def as_dict_for_user(self, user):
-        }
+        """
-        if not self.is_deleted():
+        Returns a dict with the data for a user. Can be used for the rest api.
-            data = self.get_access_permissions().get_projector_data(
+        """
-                self.get_full_data())
+        return self.get_access_permissions().get_restricted_data(
-            if data is None:
+            self.get_full_data(),
-                # The user is not allowed to see this element. Reset output to None.
+            user)
-                output = None
-            else:
-                output['data'] = data
-        return output
 
     def get_model(self):
         """
@@ -102,8 +139,21 @@ class CollectionElement:
         """
         if self.is_deleted():
             raise RuntimeError("The collection element is deleted.")
+
         if self.instance is None:
-            self.instance = self.get_model().objects.get(pk=self.id)
+            # The config instance has to be get from the config element, because
+            # some config values are not in the db.
+            from openslides.core.config import config
+            if (self.collection_string == config.get_collection_string() and
+                    isinstance(self.id, str)):
+                self.instance = {'key': self.id, 'value': config[self.id]}
+            else:
+                model = self.get_model()
+                try:
+                    query = model.objects.get_full_queryset()
+                except AttributeError:
+                    query = model.objects
+                self.instance = query.get(pk=self.id)
         return self.instance
 
     def get_access_permissions(self):
@@ -117,7 +167,20 @@ class CollectionElement:
         Returns the full_data of this collection_element from with all other
         dics can be generated.
         """
-        return self.get_access_permissions().get_full_data(self.get_instance())
+        # If the full_data is already loaded, return it
+        # If there is a db_instance, use it to get the full_data
+        # else: try to use the cache.
+        # If there is no value in the cach, get the content from the db and save
+        # it to the cache.
+        if self.full_data is None and self.instance is None:
+            # Use the cache version if self.instance is not set.
+            # After this line full_data can be None, if the element is not in the cache.
+            self.full_data = cache.get(self.get_cache_key())
+
+        if self.full_data is None:
+            self.full_data = self.get_access_permissions().get_full_data(self.get_instance())
+            self.save_to_cache()
+        return self.full_data
 
     def is_deleted(self):
         """
@@ -125,6 +188,217 @@ class CollectionElement:
         """
         return self.deleted
 
+    def get_cache_key(self):
+        """
+        Returns a string that is used as cache key for a single instance.
+        """
+        return get_single_element_cache_key(self.collection_string, self.id)
+
+    def delete_from_cache(self):
+        """
+        Delets an element from the cache.
+
+        Does nothing if the element is not in the cache.
+        """
+        # Deletes the element from the cache.
+        cache.delete(self.get_cache_key())
+
+        # Delete the id of the instance of the instance list
+        Collection(self.collection_string).delete_id_from_cache(self.id)
+
+    def save_to_cache(self):
+        """
+        Add or update the element to the cache.
+        """
+        # Set the element to the cache.
+        cache.set(self.get_cache_key(), self.get_full_data())
+
+        # Add the id of the element to the collection
+        Collection(self.collection_string).add_id_to_cache(self.id)
+
+
+class Collection:
+    """
+    Represents all elements of one collection.
+    """
+
+    def __init__(self, collection_string):
+        self.collection_string = collection_string
+
+    def get_cache_key(self, raw=False):
+        """
+        Returns a string that is used as cache key for a collection.
+
+        Django adds a prefix to the cache key when using the django cache api.
+        In other cases use raw=True to add the same cache key.
+        """
+        key = get_element_list_cache_key(self.collection_string)
+        if raw:
+            key = cache.make_key(key)
+        return key
+
+    def get_model(self):
+        """
+        Returns the django model that is used for this collection.
+        """
+        return get_model_from_collection_string(self.collection_string)
+
+    def element_generator(self):
+        """
+        Generator that yields all collection_elements of this collection.
+        """
+        # Get all cache keys.
+        ids = self.get_all_ids()
+        cache_keys = [
+            get_single_element_cache_key(self.collection_string, id)
+            for id in ids]
+        cached_full_data_dict = cache.get_many(cache_keys)
+
+        # Get all ids that are missing.
+        missing_ids = set()
+        for cache_key in cache_keys:
+            if cache_key not in cached_full_data_dict:
+                missing_ids.add(get_collection_id_from_cache_key(cache_key)[1])
+
+        # Generate collection elements that where in the cache.
+        for cache_key, cached_full_data in cached_full_data_dict.items():
+            yield CollectionElement.from_values(
+                *get_collection_id_from_cache_key(cache_key),
+                full_data=cached_full_data)
+
+        # Generate collection element that where not in the cache.
+        if missing_ids:
+            from openslides.core.config import config
+            if self.collection_string == config.get_collection_string():
+                # If config elements are not in the cache, they have to be read from
+                # the config object.
+                for key, value in config.items():
+                    if key in missing_ids:
+                        collection_element = CollectionElement.from_values(
+                            config.get_collection_string(),
+                            key,
+                            full_data={'key': key, 'value': value})
+                        # We can not use .from_instance therefore the config value
+                        # is not saved to the cache. We have to do it manualy.
+                        collection_element.save_to_cache()
+                        yield collection_element
+            else:
+                model = self.get_model()
+                try:
+                    query = model.objects.get_full_queryset()
+                except AttributeError:
+                    query = model.objects
+                for instance in query.filter(pk__in=missing_ids):
+                    yield CollectionElement.from_instance(instance)
+
+    def as_autoupdate_for_projector(self):
+        """
+        Returns a list of dictonaries to send them to the projector.
+        """
+        output = []
+        for collection_element in self.element_generator():
+            content = collection_element.as_autoupdate_for_projector()
+            # Content can not be None. If the projector can not see an element,
+            # then it is marked as deleted.
+            output.append(content)
+        return output
+
+    def as_list_for_user(self, user):
+        """
+        Returns a list of dictonaries to send them to a user, for example over
+        the rest api.
+        """
+        output = []
+        for collection_element in self.element_generator():
+            content = collection_element.as_dict_for_user(user)
+            if content is not None:
+                output.append(content)
+        return output
+
+    def get_all_ids(self):
+        """
+        Returns a set of all ids of instances in this collection.
+        """
+        from openslides.core.config import config
+        if self.collection_string == config.get_collection_string():
+            ids = config.config_variables.keys()
+        elif use_redis_cache():
+            ids = self.get_all_ids_redis()
+        else:
+            ids = self.get_all_ids_other()
+        return ids
+
+    def get_all_ids_redis(self):
+        redis = get_redis_connection()
+        ids = redis.smembers(self.get_cache_key(raw=True))
+        if not ids:
+            ids = set(self.get_model().objects.values_list('pk', flat=True))
+            if ids:
+                redis.sadd(self.get_cache_key(raw=True), *ids)
+        # Redis returns the ids as string.
+        ids = set(int(id) for id in ids)
+        return ids
+
+    def get_all_ids_other(self):
+        ids = cache.get(self.get_cache_key())
+        if ids is None:
+            # If it is not in the cache then get it from the database.
+            ids = set(self.get_model().objects.values_list('pk', flat=True))
+            cache.set(self.get_cache_key(), ids)
+        return ids
+
+    def delete_id_from_cache(self, id):
+        """
+        Delets a id from the cache.
+        """
+        if use_redis_cache():
+            self.delete_id_from_cache_redis(id)
+        else:
+            self.delete_id_from_cache_other(id)
+
+    def delete_id_from_cache_redis(self, id):
+        redis = get_redis_connection()
+        redis.srem(self.get_cache_key(raw=True), id)
+
+    def delete_id_from_cache_other(self, id):
+        ids = cache.get(self.get_cache_key())
+        if ids is not None:
+            ids = set(ids)
+            try:
+                ids.remove(id)
+            except KeyError:
+                # The id is not part of id list
+                pass
+            else:
+                if ids:
+                    cache.set(self.get_cache_key(), ids)
+                else:
+                    # Delete the key, if there are not ids left
+                    cache.delete(self.get_cache_key())
+
+    def add_id_to_cache(self, id):
+        """
+        Adds a collection id to the list of collection ids in the cache.
+        """
+        if use_redis_cache():
+            self.add_id_to_cache_redis(id)
+        else:
+            self.add_id_to_cache_other(id)
+
+    def add_id_to_cache_redis(self, id):
+        redis = get_redis_connection()
+        if redis.exists(self.get_cache_key(raw=True)):
+            # Only add the value if it is in the cache.
+            redis.sadd(self.get_cache_key(raw=True), id)
+
+    def add_id_to_cache_other(self, id):
+        ids = cache.get(self.get_cache_key())
+        if ids is not None:
+            # Only change the value if it is in the cache.
+            ids = set(ids)
+            ids.add(id)
+            cache.set(self.get_cache_key(), ids)
+
 
 def get_model_from_collection_string(collection_string):
     """
@@ -152,3 +426,59 @@ def get_model_from_collection_string(collection_string):
         # No model was found in all apps.
         raise ValueError('Invalid message. A valid collection_string is missing.')
     return model
+
+
+def get_single_element_cache_key(collection_string, id):
+    """
+    Returns a string that is used as cache key for a single instance.
+    """
+    return "{prefix}{id}".format(
+        prefix=get_single_element_cache_key_prefix(collection_string),
+        id=id)
+
+
+def get_single_element_cache_key_prefix(collection_string):
+    """
+    Returns the first part of the cache key for single elements, which is the
+    same for all cache keys of the same collection.
+    """
+    return "{collection}:".format(collection=collection_string)
+
+
+def get_element_list_cache_key(collection_string):
+    """
+    Returns a string that is used as cache key for a collection.
+    """
+    return "{collection}".format(collection=collection_string)
+
+
+def get_collection_id_from_cache_key(cache_key):
+    """
+    Returns a tuble of the collection string and the id from a cache_key
+    created with get_instance_cache_key.
+
+    The returned id can be an integer or an string.
+    """
+    collection_string, id = cache_key.rsplit(':', 1)
+    try:
+        id = int(id)
+    except ValueError:
+        # The id is no integer. This can happen on config elements
+        pass
+    return (collection_string, id)
+
+
+def use_redis_cache():
+    """
+    Returns True if Redis is used als caching backend.
+    """
+    try:
+        from django_redis.cache import RedisCache
+    except ImportError:
+        return False
+    return isinstance(caches['default'], RedisCache)
+
+
+def get_redis_connection():
+    from django_redis import get_redis_connection
+    return get_redis_connection("default")

openslides/utils/models.py

@@ -54,3 +54,54 @@ class RESTModelMixin:
         the database pk.
         """
         return self.pk
+
+    def save(self, skip_autoupdate=False, information=None, *args, **kwargs):
+        """
+        Calls Django's save() method and afterwards hits the autoupdate system.
+
+        If skip_autoupdate is set to True, then the autoupdate system is not
+        informed about the model changed. This also means, that the model cache
+        is not updated. You have to do this manually, by creating a collection
+        element from the instance:
+
+        CollectionElement.from_instance(instance)
+
+        The optional argument information can be a dictionary that is given to
+        the autoupdate system.
+        """
+        # We don't know how to fix this circular import
+        from .autoupdate import inform_changed_data
+        return_value = super().save(*args, **kwargs)
+        if not skip_autoupdate:
+            inform_changed_data(self.get_root_rest_element(), information=information)
+        return return_value
+
+    def delete(self, skip_autoupdate=False, information=None, *args, **kwargs):
+        """
+        Calls Django's delete() method and afterwards hits the autoupdate system.
+
+        If skip_autoupdate is set to True, then the autoupdate system is not
+        informed about the model changed. This also means, that the model cache
+        is not updated. You have to do this manually, by creating a collection
+        element from the instance:
+
+        CollectionElement.from_instance(instance, deleted=True)
+
+        or
+
+        CollectionElement.from_values(collection_string, id, deleted=True)
+
+        The optional argument information can be a dictionary that is given to
+        the autoupdate system.
+        """
+        # We don't know how to fix this circular import
+        from .autoupdate import inform_changed_data, inform_deleted_data
+        instance_pk = self.pk
+        return_value = super().delete(*args, **kwargs)
+        if not skip_autoupdate:
+            if self != self.get_root_rest_element():
+                # The deletion of a included element is a change of the root element.
+                inform_changed_data(self.get_root_rest_element(), information=information)
+            else:
+                inform_deleted_data(self.get_collection_string(), instance_pk, information=information)
+        return return_value

openslides/utils/projector.py

@@ -1,5 +1,6 @@
 from django.dispatch import Signal
 
+from .collection import CollectionElement
 from .dispatch import SignalConnectMetaClass
 
 
@@ -73,11 +74,31 @@ class ProjectorElement(object, metaclass=SignalConnectMetaClass):
         """
         return ()
 
-    def need_full_update_for_this(self, collection_element):
+    def get_requirements_as_collection_elements(self, config_entry):
         """
-        Returns True if this projector element needs to be updated with all
+        Returns an iterable of collection elements that are required for this
-        instances as defined in get_requirements(). The given
+        projector element. The config_entry has to be given.
-        collection_element contains information about the changed instance.
-        Default is False.
         """
-        return False
+        return (CollectionElement.from_instance(instance) for instance in self.get_requirements(config_entry))
+
+    def get_collection_elements_required_for_this(self, collection_element, config_entry):
+        """
+        Returns a list of CollectionElements that have to be sent to every
+        projector that shows this projector element according to the given
+        collection_element and information.
+
+        Default: Returns only the collection_element if it belongs to the
+        requirements but return all requirements if the projector changes.
+        """
+        requirements_as_collection_elements = list(self.get_requirements_as_collection_elements(config_entry))
+        for requirement in requirements_as_collection_elements:
+            if collection_element == requirement:
+                output = [collection_element]
+                break
+        else:
+            if collection_element.information.get('this_projector'):
+                output = [collection_element]
+                output.extend(requirements_as_collection_elements)
+            else:
+                output = []
+        return output

openslides/utils/rest_api.py

@@ -1,15 +1,13 @@
 from collections import OrderedDict
 
+from django.http import Http404
 from rest_framework import status  # noqa
 from rest_framework.decorators import detail_route, list_route  # noqa
 from rest_framework.metadata import SimpleMetadata  # noqa
-from rest_framework.mixins import (  # noqa
+from rest_framework.mixins import ListModelMixin as _ListModelMixin
-    DestroyModelMixin,
+from rest_framework.mixins import RetrieveModelMixin as _RetrieveModelMixin
-    ListModelMixin,
+from rest_framework.mixins import DestroyModelMixin, UpdateModelMixin  # noqa
-    RetrieveModelMixin,
+from rest_framework.response import Response
-    UpdateModelMixin,
-)
-from rest_framework.response import Response  # noqa
 from rest_framework.routers import DefaultRouter
 from rest_framework.serializers import ModelSerializer as _ModelSerializer
 from rest_framework.serializers import (  # noqa
@@ -31,6 +29,8 @@ from rest_framework.viewsets import GenericViewSet as _GenericViewSet  # noqa
 from rest_framework.viewsets import ModelViewSet as _ModelViewSet  # noqa
 from rest_framework.viewsets import ViewSet as _ViewSet  # noqa
 
+from .collection import Collection, CollectionElement
+
 router = DefaultRouter()
 
 
@@ -164,11 +164,63 @@ class ModelSerializer(_ModelSerializer):
         return fields
 
 
+class ListModelMixin(_ListModelMixin):
+    """
+    Mixin to add the caching system to list requests.
+
+    It is not allowed to use the method get_queryset() in derivated classes.
+    The attribute queryset has to be used in the following form:
+
+    queryset = Model.objects.all()
+    """
+    def list(self, request, *args, **kwargs):
+        model = self.get_queryset().model
+        try:
+            collection_string = model.get_collection_string()
+        except AttributeError:
+            # The corresponding queryset does not support caching.
+            response = super().list(request, *args, **kwargs)
+        else:
+            collection = Collection(collection_string)
+            response = Response(collection.as_list_for_user(request.user))
+        return response
+
+
+class RetrieveModelMixin(_RetrieveModelMixin):
+    """
+    Mixin to add the caching system to retrieve requests.
+
+    It is not allowed to use the method get_queryset() in derivated classes.
+    The attribute queryset has to be used in the following form:
+
+    queryset = Model.objects.all()
+    """
+    def retrieve(self, request, *args, **kwargs):
+        model = self.get_queryset().model
+        try:
+            collection_string = model.get_collection_string()
+        except AttributeError:
+            # The corresponding queryset does not support caching.
+            response = super().retrieve(request, *args, **kwargs)
+        else:
+            lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
+            collection_element = CollectionElement.from_values(
+                collection_string, self.kwargs[lookup_url_kwarg])
+            try:
+                content = collection_element.as_dict_for_user(request.user)
+            except collection_element.get_model().DoesNotExist:
+                raise Http404
+            if content is None:
+                self.permission_denied(request)
+            response = Response(content)
+        return response
+
+
 class GenericViewSet(PermissionMixin, _GenericViewSet):
     pass
 
 
-class ModelViewSet(PermissionMixin, _ModelViewSet):
+class ModelViewSet(PermissionMixin, ListModelMixin, RetrieveModelMixin, _ModelViewSet):
     pass
 
 

openslides/utils/settings.py.tpl

@@ -46,6 +46,17 @@ DEBUG = %(debug)s
 
 # Change this setting to use e. g. PostgreSQL or MySQL.
 
+# DATABASES = {
+#     'default': {
+#         'ENGINE': 'django.db.backends.postgresql',
+#         'NAME': 'mydatabase',
+#         'USER': 'mydatabaseuser',
+#         'PASSWORD': 'mypassword',
+#         'HOST': '127.0.0.1',
+#         'PORT': '5432',
+#     }
+# }
+
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
@@ -64,12 +75,17 @@ DATABASES = {
 
 # CHANNEL_LAYERS['default']['BACKEND'] = 'asgi_redis.RedisChannelLayer'
 
-# https://niwinz.github.io/django-redis/latest/#_user_guide
+
+# Caching
+
+# Django uses a inmemory cache at default. This supports only one thread. If
+# you use more then one thread another caching backend is required. We recommand
+# django-redis: https://niwinz.github.io/django-redis/latest/#_user_guide
 
 # CACHES = {
 #     "default": {
 #         "BACKEND": "django_redis.cache.RedisCache",
-#         "LOCATION": "redis://127.0.0.1:6379/1",
+#         "LOCATION": "redis://127.0.0.1:6379/0",
 #         "OPTIONS": {
 #             "CLIENT_CLASS": "django_redis.client.DefaultClient",
 #         }

tests/integration/agenda/test_views.py

@@ -1,90 +1,7 @@
-import json
-
-from rest_framework.test import APIClient
-
-from openslides.agenda.models import Item
 from openslides.topics.models import Topic
 from openslides.utils.test import TestCase
 
 
-class AgendaTreeTest(TestCase):
-    def setUp(self):
-        Topic.objects.create(title='item1')
-        item2 = Topic.objects.create(title='item2').agenda_item
-        item3 = Topic.objects.create(title='item2a').agenda_item
-        item3.parent = item2
-        item3.save()
-        self.client = APIClient()
-        self.client.login(username='admin', password='admin')
-
-    def test_get(self):
-        response = self.client.get('/rest/agenda/item/tree/')
-
-        self.assertEqual(json.loads(response.content.decode()),
-                         [{'children': [], 'id': 1},
-                          {'children': [{'children': [], 'id': 3}], 'id': 2}])
-
-    def test_set(self):
-        tree = [{'id': 3},
-                {'children': [{'id': 1}], 'id': 2}]
-
-        response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json')
-
-        self.assertEqual(response.status_code, 200)
-
-        item1 = Item.objects.get(pk=1)
-        item2 = Item.objects.get(pk=2)
-        item3 = Item.objects.get(pk=3)
-        self.assertEqual(item1.parent_id, 2)
-        self.assertEqual(item1.weight, 0)
-        self.assertEqual(item2.parent_id, None)
-        self.assertEqual(item2.weight, 1)
-        self.assertEqual(item3.parent_id, None)
-        self.assertEqual(item3.weight, 0)
-
-    def test_set_without_perm(self):
-        self.client = APIClient()
-
-        response = self.client.put('/rest/agenda/item/tree/', {'tree': []}, format='json')
-
-        self.assertEqual(response.status_code, 403)
-
-    def test_tree_with_double_item(self):
-        """
-        Test to send a tree that has an item-pk more then once in it.
-
-        It is expected, that the responsecode 400 is returned with a specific
-        content
-        """
-        tree = [{'id': 1}, {'id': 1}]
-
-        response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json')
-
-        self.assertEqual(response.status_code, 400)
-        self.assertEqual(response.data, {'detail': "Item 1 is more then once in the tree."})
-
-    def test_tree_with_empty_children(self):
-        """
-        Test that the chrildren element is not required in the tree
-        """
-        tree = [{'id': 1}]
-
-        response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json')
-
-        self.assertEqual(response.status_code, 200)
-
-    def test_tree_with_unknown_item(self):
-        """
-        Tests that unknown items lead to an error.
-        """
-        tree = [{'id': 500}]
-
-        response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json')
-
-        self.assertEqual(response.status_code, 400)
-        self.assertEqual(response.data, {'detail': "Item 500 is not in the database."})
-
-
 class TestAgendaPDF(TestCase):
     def test_get(self):
         """

tests/integration/agenda/test_viewsets.py

@@ -4,9 +4,12 @@ from rest_framework import status
 from rest_framework.test import APIClient
 
 from openslides.agenda.models import Item, Speaker
+from openslides.assignments.models import Assignment
 from openslides.core.config import config
 from openslides.core.models import Projector
+from openslides.motions.models import Motion
 from openslides.topics.models import Topic
+from openslides.users.models import User
 from openslides.utils.test import TestCase
 
 
@@ -37,7 +40,61 @@ class RetrieveItem(TestCase):
         permission = group.permissions.get(content_type__app_label=app_label, codename=codename)
         group.permissions.remove(permission)
         response = self.client.get(reverse('item-detail', args=[self.item.pk]))
-        self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
+        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+
+
+class TestDBQueries(TestCase):
+    """
+    Tests that receiving elements only need the required db queries.
+
+    Therefore in setup some agenda items are created and received with different
+    user accounts.
+    """
+
+    def setUp(self):
+        self.client = APIClient()
+        config['general_system_enable_anonymous'] = True
+        for index in range(10):
+            Topic.objects.create(title='topic{}'.format(index))
+        parent = Topic.objects.create(title='parent').agenda_item
+        child = Topic.objects.create(title='child').agenda_item
+        child.parent = parent
+        child.save()
+        Motion.objects.create(title='motion1')
+        Motion.objects.create(title='motion2')
+        Assignment.objects.create(title='assignment', open_posts=5)
+
+    def test_admin(self):
+        """
+        Tests that only the following db queries are done:
+        * 5 requests to get the session an the request user with its permissions,
+        * 2 requests to get the list of all agenda items,
+        * 1 request to get all speakers,
+        * 3 requests to get the assignments, motions and topics and
+
+        * 2 requests for the motionsversions.
+
+        TODO: There could be less requests to get the session and the request user.
+        The last two request for the motionsversions are a bug.
+        """
+        self.client.force_login(User.objects.get(pk=1))
+        with self.assertNumQueries(13):
+            self.client.get(reverse('item-list'))
+
+    def test_anonymous(self):
+        """
+        Tests that only the following db queries are done:
+        * 2 requests to get the permission for anonymous (config and permissions)
+        * 2 requests to get the list of all agenda items,
+        * 1 request to get all speakers,
+        * 3 requests to get the assignments, motions and topics and
+
+        * 32 requests for the motionsversions.
+
+        TODO: The last 32 requests are a bug.
+        """
+        with self.assertNumQueries(40):
+            self.client.get(reverse('item-list'))
 
 
 class ManageSpeaker(TestCase):

tests/integration/assignments/test_viewset.py

@@ -4,9 +4,61 @@ from rest_framework import status
 from rest_framework.test import APIClient
 
 from openslides.assignments.models import Assignment
+from openslides.core.config import config
+from openslides.users.models import User
 from openslides.utils.test import TestCase
 
 
+class TestDBQueries(TestCase):
+    """
+    Tests that receiving elements only need the required db queries.
+
+    Therefore in setup some assignments are created and received with different
+    user accounts.
+    """
+
+    def setUp(self):
+        self.client = APIClient()
+        config['general_system_enable_anonymous'] = True
+        for index in range(10):
+            Assignment.objects.create(title='motion{}'.format(index), open_posts=1)
+
+    def test_admin(self):
+        """
+        Tests that only the following db queries are done:
+        * 5 requests to get the session an the request user with its permissions,
+        * 2 requests to get the list of all assignments,
+        * 1 request to get all related users,
+        * 1 request to get the agenda item,
+        * 1 request to get the polls,
+
+        * 10 request to featch each related user again.
+
+        TODO: There could be less requests to get the session and the request user.
+        The eleven request are a bug.
+        """
+        self.client.force_login(User.objects.get(pk=1))
+        with self.assertNumQueries(30):
+            self.client.get(reverse('assignment-list'))
+
+    def test_anonymous(self):
+        """
+        Tests that only the following db queries are done:
+        * 2 requests to get the permission for anonymous (config and permissions)
+        * 2 requests to get the list of all assignments,
+        * 1 request to get all related users,
+        * 1 request to get the agenda item,
+        * 1 request to get the polls,
+        * 1 request to get the tags,
+
+        * lots of permissions requests.
+
+        TODO: The last requests are a bug.
+        """
+        with self.assertNumQueries(57):
+            self.client.get(reverse('assignment-list'))
+
+
 class CanidatureSelf(TestCase):
     """
     Tests self candidation view.

tests/integration/core/test_viewset.py

@@ -0,0 +1,147 @@
+from django.core.urlresolvers import reverse
+from rest_framework.test import APIClient
+
+from openslides.core.config import config
+from openslides.core.models import ChatMessage, Projector, Tag
+from openslides.users.models import User
+from openslides.utils.test import TestCase
+
+
+class TestProjectorDBQueries(TestCase):
+    """
+    Tests that receiving elements only need the required db queries.
+
+    Therefore in setup some objects are created and received with different
+    user accounts.
+    """
+
+    def setUp(self):
+        self.client = APIClient()
+        config['general_system_enable_anonymous'] = True
+        for index in range(10):
+            Projector.objects.create(name="Projector{}".format(index))
+
+    def test_admin(self):
+        """
+        Tests that only the following db queries are done:
+        * 5 requests to get the session an the request user with its permissions,
+        * 2 requests to get the list of all projectors,
+        * 1 request to get the list of the projector defaults.
+        """
+        self.client.force_login(User.objects.get(pk=1))
+        with self.assertNumQueries(8):
+            self.client.get(reverse('projector-list'))
+
+    def test_anonymous(self):
+        """
+        Tests that only the following db queries are done:
+        * 2 requests to get the permission for anonymous (config and permissions)
+        * 2 requests to get the list of all projectors,
+        * 1 request to get the list of the projector defaults and
+
+        * 11 requests for permissions.
+
+        TODO: The last 11 requests are a bug.
+        """
+        with self.assertNumQueries(16):
+            self.client.get(reverse('projector-list'))
+
+
+class TestCharmessageDBQueries(TestCase):
+    """
+    Tests that receiving elements only need the required db queries.
+
+    Therefore in setup some objects are created and received with different
+    user accounts.
+    """
+
+    def setUp(self):
+        self.client = APIClient()
+        config['general_system_enable_anonymous'] = True
+        user = User.objects.get(pk=1)
+        for index in range(10):
+            ChatMessage.objects.create(user=user)
+
+    def test_admin(self):
+        """
+        Tests that only the following db queries are done:
+        * 5 requests to get the session an the request user with its permissions,
+        * 2 requests to get the list of all chatmessages,
+        """
+        self.client.force_login(User.objects.get(pk=1))
+        with self.assertNumQueries(7):
+            self.client.get(reverse('chatmessage-list'))
+
+
+class TestTagDBQueries(TestCase):
+    """
+    Tests that receiving elements only need the required db queries.
+
+    Therefore in setup some objects are created and received with different
+    user accounts.
+    """
+
+    def setUp(self):
+        self.client = APIClient()
+        config['general_system_enable_anonymous'] = True
+        for index in range(10):
+            Tag.objects.create(name='tag{}'.format(index))
+
+    def test_admin(self):
+        """
+        Tests that only the following db queries are done:
+        * 2 requests to get the session an the request user with its permissions,
+        * 2 requests to get the list of all tags,
+        """
+        self.client.force_login(User.objects.get(pk=1))
+        with self.assertNumQueries(4):
+            self.client.get(reverse('tag-list'))
+
+    def test_anonymous(self):
+        """
+        Tests that only the following db queries are done:
+        * 2 requests to get the permission for anonymous (config and permissions)
+        * 2 requests to get the list of all projectors,
+
+        * 10 requests for to config
+
+        The last 10 requests are a bug.
+        """
+        with self.assertNumQueries(14):
+            self.client.get(reverse('tag-list'))
+
+
+class TestConfigDBQueries(TestCase):
+    """
+    Tests that receiving elements only need the required db queries.
+
+    Therefore in setup some objects are created and received with different
+    user accounts.
+    """
+
+    def setUp(self):
+        self.client = APIClient()
+        config['general_system_enable_anonymous'] = True
+
+    def test_admin(self):
+        """
+        Tests that only the following db queries are done:
+        * 2 requests to get the session an the request user with its permissions and
+        * 1 requests to get the list of all config values
+        """
+        self.client.force_login(User.objects.get(pk=1))
+        with self.assertNumQueries(3):
+            self.client.get(reverse('config-list'))
+
+    def test_anonymous(self):
+        """
+        Tests that only the following db queries are done:
+        * 2 requests to get the permission for anonymous (config and permissions),
+        * 1 to get all config value and
+
+        * 57 requests to find out if anonymous is enabled.
+
+        TODO: The last 57 requests are a bug.
+        """
+        with self.assertNumQueries(60):
+            self.client.get(reverse('config-list'))

tests/integration/mediafiles/test_viewset.py

@@ -0,0 +1,46 @@
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.core.urlresolvers import reverse
+from rest_framework.test import APIClient
+
+from openslides.core.config import config
+from openslides.mediafiles.models import Mediafile
+from openslides.users.models import User
+from openslides.utils.test import TestCase
+
+
+class TestDBQueries(TestCase):
+    """
+    Tests that receiving elements only need the required db queries.
+
+    Therefore in setup some objects are created and received with different
+    user accounts.
+    """
+
+    def setUp(self):
+        self.client = APIClient()
+        config['general_system_enable_anonymous'] = True
+        for index in range(10):
+            Mediafile.objects.create(
+                title='some_file{}'.format(index),
+                mediafile=SimpleUploadedFile(
+                    'some_file{}'.format(index),
+                    b'some content.'))
+
+    def test_admin(self):
+        """
+        Tests that only the following db queries are done:
+        * 5 requests to get the session an the request user with its permissions and
+        * 2 requests to get the list of all files.
+        """
+        self.client.force_login(User.objects.get(pk=1))
+        with self.assertNumQueries(7):
+            self.client.get(reverse('mediafile-list'))
+
+    def test_anonymous(self):
+        """
+        Tests that only the following db queries are done:
+        * 2 requests to get the permission for anonymous (config and permissions) and
+        * 2 requests to get the list of all projectors.
+        """
+        with self.assertNumQueries(4):
+            self.client.get(reverse('mediafile-list'))

tests/integration/motions/test_viewset.py

@@ -8,9 +8,139 @@ from rest_framework.test import APIClient
 from openslides.core.config import config
 from openslides.core.models import Tag
 from openslides.motions.models import Category, Motion, State
+from openslides.users.models import User
 from openslides.utils.test import TestCase
 
 
+class TestMotionDBQueries(TestCase):
+    """
+    Tests that receiving elements only need the required db queries.
+
+    Therefore in setup some objects are created and received with different
+    user accounts.
+    """
+
+    def setUp(self):
+        self.client = APIClient()
+        config['general_system_enable_anonymous'] = True
+        for index in range(10):
+            Motion.objects.create(title='motion{}'.format(index))
+        # TODO: Create some polls etc.
+
+    def test_admin(self):
+        """
+        Tests that only the following db queries are done:
+        * 5 requests to get the session an the request user with its permissions,
+        * 2 requests to get the list of all motions,
+        * 1 request to get the motion versions,
+        * 1 request to get the agenda item,
+        * 1 request to get the motion log,
+        * 1 request to get the polls,
+        * 1 request to get the attachments,
+        * 1 request to get the tags,
+        * 2 requests to get the submitters and supporters
+        """
+        self.client.force_login(User.objects.get(pk=1))
+        with self.assertNumQueries(15):
+            self.client.get(reverse('motion-list'))
+
+    def test_anonymous(self):
+        """
+        Tests that only the following db queries are done:
+        * 2 requests to get the permission for anonymous (config and permissions)
+        * 2 requests to get the list of all motions,
+        * 1 request to get the motion versions,
+        * 1 request to get the agenda item,
+        * 1 request to get the motion log,
+        * 1 request to get the polls,
+        * 1 request to get the attachments,
+        * 1 request to get the tags,
+        * 2 requests to get the submitters and supporters
+
+        * 10 requests for permissions.
+
+        TODO: The last 10 requests are a bug.
+        """
+        with self.assertNumQueries(22):
+            self.client.get(reverse('motion-list'))
+
+
+class TestCategoryDBQueries(TestCase):
+    """
+    Tests that receiving elements only need the required db queries.
+
+    Therefore in setup some objects are created and received with different
+    user accounts.
+    """
+
+    def setUp(self):
+        self.client = APIClient()
+        config['general_system_enable_anonymous'] = True
+        for index in range(10):
+            Category.objects.create(name='category{}'.format(index))
+
+    def test_admin(self):
+        """
+        Tests that only the following db queries are done:
+        * 5 requests to get the session an the request user with its permissions and
+        * 2 requests to get the list of all categories.
+        """
+        self.client.force_login(User.objects.get(pk=1))
+        with self.assertNumQueries(7):
+            self.client.get(reverse('category-list'))
+
+    def test_anonymous(self):
+        """
+        Tests that only the following db queries are done:
+        * 2 requests to get the permission for anonymous (config and permissions)
+        * 2 requests to get the list of all motions and
+
+        * 10 requests for permissions.
+
+        TODO: The last 10 requests are a bug.
+        """
+        with self.assertNumQueries(14):
+            self.client.get(reverse('category-list'))
+
+
+class TestWorkflowDBQueries(TestCase):
+    """
+    Tests that receiving elements only need the required db queries.
+    """
+
+    def setUp(self):
+        self.client = APIClient()
+        config['general_system_enable_anonymous'] = True
+        # There do not need to be more workflows
+
+    def test_admin(self):
+        """
+        Tests that only the following db queries are done:
+        * 5 requests to get the session an the request user with its permissions,
+        * 2 requests to get the list of all workflows,
+        * 1 request to get all states and
+        * 1 request to get the next states of all states.
+        """
+        self.client.force_login(User.objects.get(pk=1))
+        with self.assertNumQueries(9):
+            self.client.get(reverse('workflow-list'))
+
+    def test_anonymous(self):
+        """
+        Tests that only the following db queries are done:
+        * 2 requests to get the permission for anonymous (config and permissions),
+        * 2 requests to get the list of all workflows,
+        * 1 request to get all states and
+        * 1 request to get the next states of all states.
+
+        * 2 requests for permissions.
+
+        TODO: The last 2 requests are a bug.
+        """
+        with self.assertNumQueries(8):
+            self.client.get(reverse('workflow-list'))
+
+
 class CreateMotion(TestCase):
     """
     Tests motion creation.

tests/integration/users/test_viewset.py

@@ -8,6 +8,86 @@ from openslides.users.serializers import UserFullSerializer
 from openslides.utils.test import TestCase
 
 
+class TestUserDBQueries(TestCase):
+    """
+    Tests that receiving elements only need the required db queries.
+
+    Therefore in setup some objects are created and received with different
+    user accounts.
+    """
+
+    def setUp(self):
+        self.client = APIClient()
+        config['general_system_enable_anonymous'] = True
+        for index in range(10):
+            User.objects.create(username='user{}'.format(index))
+
+    def test_admin(self):
+        """
+        Tests that only the following db queries are done:
+        * 5 requests to get the session an the request user with its permissions,
+        * 2 requests to get the list of all assignments and
+        * 1 request to get all groups.
+        """
+        self.client.force_login(User.objects.get(pk=1))
+        with self.assertNumQueries(8):
+            self.client.get(reverse('user-list'))
+
+    def test_anonymous(self):
+        """
+        Tests that only the following db queries are done:
+        * 2 requests to get the permission for anonymous (config and permissions)
+        * 2 requests to get the list of all users,
+        * 1 request to get all groups and
+
+        * lots of permissions requests.
+
+        TODO: The last requests are a bug.
+        """
+        with self.assertNumQueries(27):
+            self.client.get(reverse('user-list'))
+
+
+class TestGroupDBQueries(TestCase):
+    """
+    Tests that receiving elements only need the required db queries.
+
+    Therefore in setup some objects are created and received with different
+    user accounts.
+    """
+
+    def setUp(self):
+        self.client = APIClient()
+        config['general_system_enable_anonymous'] = True
+        for index in range(10):
+            Group.objects.create(name='group{}'.format(index))
+
+    def test_admin(self):
+        """
+        Tests that only the following db queries are done:
+        * 2 requests to get the session an the request user with its permissions,
+        * 1 request to get the list of all groups,
+        * 1 request to get the permissions and
+        * 1 request to get the content_object for the permissions.
+        """
+        self.client.force_login(User.objects.get(pk=1))
+        with self.assertNumQueries(5):
+            self.client.get(reverse('group-list'))
+
+    def test_anonymous(self):
+        """
+        Tests that only the following db queries are done:
+        * 2 requests to find out if anonymous is enabled
+        * 1 request to get the list of all groups,
+        * 1 request to get the permissions and
+        * 1 request to get the content_object for the permissions.
+
+        TODO: There should be only one request to find out if anonymous is enabled.
+        """
+        with self.assertNumQueries(5):
+            self.client.get(reverse('group-list'))
+
+
 class UserGetTest(TestCase):
     """
     Tests to receive a users via REST API.

tests/integration/utils/test_autoupdate.py

@@ -24,13 +24,22 @@ class TestGetLoggedInUsers(TestCase):
         User.objects.create(username='user3')
 
         # Create a session with a user, that expires in 5 hours
-        Session.objects.create(user=user1, expire_data=timezone.now() + timedelta(hours=5))
+        Session.objects.create(
+            user=user1,
+            expire_date=timezone.now() + timedelta(hours=5),
+            session_key='1')
 
         # Create a session with a user, that is expired before 5 hours
-        Session.objects.create(user=user2, expire_data=timezone.now() + timedelta(hours=-5))
+        Session.objects.create(
+            user=user2,
+            expire_date=timezone.now() + timedelta(hours=-5),
+            session_key='2')
 
         # Create a session with an anonymous user, that expires in 5 hours
-        Session.objects.create(user=None, expire_data=timezone.now() + timedelta(hours=5))
+        Session.objects.create(
+            user=None,
+            expire_date=timezone.now() + timedelta(hours=5),
+            session_key='3')
 
         self.assertEqual(list(get_logged_in_users()), [user1])
 
@@ -40,7 +49,13 @@ class TestGetLoggedInUsers(TestCase):
         The user should be returned only once.
         """
         user1 = User.objects.create(username='user1')
-        Session.objects.create(user=user1, expire_data=timezone.now() + timedelta(hours=1))
+        Session.objects.create(
-        Session.objects.create(user=user1, expire_data=timezone.now() + timedelta(hours=2))
+            user=user1,
+            expire_date=timezone.now() + timedelta(hours=1),
+            session_key='1')
+        Session.objects.create(
+            user=user1,
+            expire_date=timezone.now() + timedelta(hours=2),
+            session_key='2')
 
         self.assertEqual(list(get_logged_in_users()), [user1])

tests/integration/utils/test_collection.py

@@ -0,0 +1,134 @@
+from unittest.mock import patch
+
+from channels.tests import ChannelTestCase
+from django.core.cache import caches
+
+from openslides.topics.models import Topic
+from openslides.utils import collection
+
+
+class TestCase(ChannelTestCase):
+    """
+    Testcase that uses the local mem cache and clears the cache after each test.
+    """
+    def setUp(self):
+        cache = caches['locmem']
+        cache.clear()
+        self.patch = patch('openslides.utils.collection.cache', cache)
+        self.patch.start()
+
+    def tearDown(self):
+        self.patch.stop()
+
+
+class TestCollectionElementCache(TestCase):
+    def test_clean_cache(self):
+        """
+        Tests that the data is retrieved from the database.
+        """
+        topic = Topic.objects.create(title='test topic')
+        collection_element = collection.CollectionElement.from_values('topics/topic', 1)
+        caches['locmem'].clear()
+
+        with self.assertNumQueries(3):
+            instance = collection_element.get_full_data()
+        self.assertEqual(topic.title, instance['title'])
+
+    def test_with_cache(self):
+        """
+        Tests that no db query is used when the valie is in the cache.
+
+        The value is added to the test when .create(...) is called. This hits
+        the autoupdate system, which fills the cache.
+        """
+        topic = Topic.objects.create(title='test topic')
+        collection_element = collection.CollectionElement.from_values('topics/topic', 1)
+
+        with self.assertNumQueries(0):
+            collection_element = collection.CollectionElement.from_values('topics/topic', 1)
+            instance = collection_element.get_full_data()
+        self.assertEqual(topic.title, instance['title'])
+
+    def test_non_existing_instance(self):
+        collection_element = collection.CollectionElement.from_values('topics/topic', 1)
+
+        with self.assertRaises(Topic.DoesNotExist):
+            collection_element.get_full_data()
+
+
+class TestCollectionCache(TestCase):
+    def test_clean_cache(self):
+        """
+        Tests that the instances are retrieved from the database.
+
+        Currently there are 10 queries needed. This can change in the future,
+        but it has to be more then zero.
+        """
+        Topic.objects.create(title='test topic1')
+        Topic.objects.create(title='test topic2')
+        Topic.objects.create(title='test topic3')
+        topic_collection = collection.Collection('topics/topic')
+        caches['locmem'].clear()
+
+        with self.assertNumQueries(4):
+            instance_list = list(topic_collection.as_autoupdate_for_projector())
+        self.assertEqual(len(instance_list), 3)
+
+    def test_with_cache(self):
+        """
+        Tests that no db query is used when the list is received twice.
+        """
+        Topic.objects.create(title='test topic1')
+        Topic.objects.create(title='test topic2')
+        Topic.objects.create(title='test topic3')
+        topic_collection = collection.Collection('topics/topic')
+        list(topic_collection.as_autoupdate_for_projector())
+
+        with self.assertNumQueries(0):
+            instance_list = list(topic_collection.as_autoupdate_for_projector())
+        self.assertEqual(len(instance_list), 3)
+
+    def test_with_some_objects_in_the_cache(self):
+        """
+        One element (topic3) is in the cache and two are not.
+        """
+        Topic.objects.create(title='test topic1')
+        Topic.objects.create(title='test topic2')
+        caches['locmem'].clear()
+        Topic.objects.create(title='test topic3')
+        topic_collection = collection.Collection('topics/topic')
+
+        with self.assertNumQueries(4):
+            instance_list = list(topic_collection.as_autoupdate_for_projector())
+        self.assertEqual(len(instance_list), 3)
+
+    def test_deletion(self):
+        """
+        When an element is deleted, the cache should be updated automaticly via
+        the autoupdate system. So there should be no db queries.
+        """
+        Topic.objects.create(title='test topic1')
+        Topic.objects.create(title='test topic2')
+        topic3 = Topic.objects.create(title='test topic3')
+        topic_collection = collection.Collection('topics/topic')
+        list(topic_collection.as_autoupdate_for_projector())
+
+        topic3.delete()
+
+        with self.assertNumQueries(0):
+            instance_list = list(topic_collection.as_autoupdate_for_projector())
+        self.assertEqual(len(instance_list), 2)
+
+    def test_config_elements_without_cache(self):
+        topic_collection = collection.Collection('core/config')
+        caches['locmem'].clear()
+
+        with self.assertNumQueries(1):
+            list(topic_collection.as_autoupdate_for_projector())
+
+    def test_config_elements_with_cache(self):
+        topic_collection = collection.Collection('core/config')
+        list(topic_collection.as_autoupdate_for_projector())
+
+        with self.assertNumQueries(0):
+            list(topic_collection.as_autoupdate_for_projector())

tests/settings.py

@@ -75,3 +75,14 @@ TEST_RUNNER = 'openslides.utils.test.OpenSlidesDiscoverRunner'
 PASSWORD_HASHERS = [
     'django.contrib.auth.hashers.MD5PasswordHasher',
 ]
+
+
+# Use the dummy cache that does not cache anything
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.dummy.DummyCache'
+    },
+    'locmem': {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'
+    }
+}

tests/unit/utils/test_collection.py

@@ -0,0 +1,251 @@
+from unittest import TestCase
+from unittest.mock import MagicMock, patch
+
+from openslides.core.models import Projector
+from openslides.utils import collection
+
+
+class TestCacheKeys(TestCase):
+    def test_get_collection_id_from_cache_key(self):
+        """
+        Test that get_collection_id_from_cache_key works together with
+        get_single_element_cache_key.
+        """
+        element = ('some/testkey', 42)
+        self.assertEqual(
+            element,
+            collection.get_collection_id_from_cache_key(
+                collection.get_single_element_cache_key(*element)))
+
+    def test_get_collection_id_from_cache_key_for_strings(self):
+        """
+        Test get_collection_id_from_cache_key for strings
+        """
+        element = ('some/testkey', 'my_config_value')
+        self.assertEqual(
+            element,
+            collection.get_collection_id_from_cache_key(
+                collection.get_single_element_cache_key(*element)))
+
+    def test_get_single_element_cache_key_prefix(self):
+        """
+        Tests that the cache prefix is realy a prefix.
+        """
+        element = ('some/testkey', 42)
+
+        cache_key = collection.get_single_element_cache_key(*element)
+        prefix = collection.get_single_element_cache_key_prefix(element[0])
+
+        self.assertTrue(cache_key.startswith(prefix))
+
+    def test_prefix_different_then_list(self):
+        """
+        Test that the return value of get_single_element_cache_key_prefix is
+        something different then get_element_list_cache_key.
+        """
+        collection_string = 'some/testkey'
+
+        prefix = collection.get_single_element_cache_key_prefix(collection_string)
+        list_cache_key = collection.get_element_list_cache_key(collection_string)
+
+        self.assertNotEqual(prefix, list_cache_key)
+
+
+class TestGetModelFromCollectionString(TestCase):
+    def test_known_app(self):
+        projector_model = collection.get_model_from_collection_string('core/projector')
+
+        self.assertEqual(projector_model, Projector)
+
+    def test_unknown_app(self):
+        with self.assertRaises(ValueError):
+            collection.get_model_from_collection_string('invalid/model')
+
+
+class TestCollectionElement(TestCase):
+    def test_from_values(self):
+        collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
+
+        self.assertEqual(collection_element.collection_string, 'testmodule/model')
+        self.assertEqual(collection_element.id, 42)
+
+    @patch('openslides.utils.collection.Collection')
+    @patch('openslides.utils.collection.cache')
+    def test_from_values_deleted(self, mock_cache, mock_collection):
+        """
+        Tests that when createing a CollectionElement with deleted=True the element
+        is deleted from the cache.
+        """
+        collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True)
+
+        self.assertTrue(collection_element.is_deleted())
+        mock_cache.delete.assert_called_with('testmodule/model:42')
+        mock_collection.assert_called_with('testmodule/model')
+        mock_collection().delete_id_from_cache.assert_called_with(42)
+
+    def test_as_channel_message(self):
+        collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
+
+        self.assertEqual(
+            collection_element.as_channels_message(),
+            {'collection_string': 'testmodule/model',
+             'id': 42,
+             'deleted': False,
+             'information': {}})
+
+    def test_as_autoupdate_for_user(self):
+        collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
+        fake_user = MagicMock()
+        collection_element.get_access_permissions = MagicMock()
+        collection_element.get_access_permissions().get_restricted_data.return_value = 'restricted_data'
+        collection_element.get_full_data = MagicMock()
+
+        self.assertEqual(
+            collection_element.as_autoupdate_for_user(fake_user),
+            {'collection': 'testmodule/model',
+             'id': 42,
+             'action': 'changed',
+             'data': 'restricted_data'})
+        collection_element.get_full_data.assert_called_once_with()
+
+    def test_as_autoupdate_for_user_no_permission(self):
+        collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
+        fake_user = MagicMock()
+        collection_element.get_access_permissions = MagicMock()
+        collection_element.get_access_permissions().get_restricted_data.return_value = None
+        collection_element.get_full_data = MagicMock()
+
+        self.assertEqual(
+            collection_element.as_autoupdate_for_user(fake_user),
+            {'collection': 'testmodule/model',
+             'id': 42,
+             'action': 'deleted'})
+        collection_element.get_full_data.assert_called_once_with()
+
+    def test_as_autoupdate_for_user_deleted(self):
+        collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True)
+        fake_user = MagicMock()
+
+        self.assertEqual(
+            collection_element.as_autoupdate_for_user(fake_user),
+            {'collection': 'testmodule/model',
+             'id': 42,
+             'action': 'deleted'})
+
+    def test_get_instance_deleted(self):
+        collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True)
+
+        with self.assertRaises(RuntimeError):
+            collection_element.get_instance()
+
+    @patch('openslides.core.config.config')
+    def test_get_instance_config_str(self, mock_config):
+        mock_config.get_collection_string.return_value = 'core/config'
+        mock_config.__getitem__.return_value = 'config_value'
+        collection_element = collection.CollectionElement.from_values('core/config', 'my_config_value')
+
+        instance = collection_element.get_instance()
+
+        self.assertEqual(
+            instance,
+            {'key': 'my_config_value',
+             'value': 'config_value'})
+
+    def test_get_instance(self):
+        collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
+        collection_element.get_model = MagicMock()
+
+        collection_element.get_instance()
+
+        collection_element.get_model().objects.get_full_queryset().get.assert_called_once_with(pk=42)
+
+    @patch('openslides.utils.collection.cache')
+    def test_get_full_data_already_loaded(self, mock_cache):
+        """
+        Test that the cache and the self.get_instance() is not hit, when the
+        instance is already loaded.
+        """
+        collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
+        collection_element.full_data = 'my_full_data'
+        collection_element.get_instance = MagicMock()
+
+        collection_element.get_full_data()
+
+        mock_cache.get.assert_not_called()
+        collection_element.get_instance.assert_not_called()
+
+    @patch('openslides.utils.collection.cache')
+    def test_get_full_data_from_cache(self, mock_cache):
+        """
+        Test that the value from the cache is used not get_instance is not
+        called.
+        """
+        collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
+        collection_element.get_instance = MagicMock()
+        mock_cache.get.return_value = 'cache_value'
+
+        instance = collection_element.get_full_data()
+
+        self.assertEqual(instance, 'cache_value')
+        mock_cache.get.assert_called_once_with('testmodule/model:42')
+        collection_element.get_instance.assert_not_called
+
+    @patch('openslides.utils.collection.Collection')
+    @patch('openslides.utils.collection.cache')
+    def test_get_full_data_from_get_instance(self, mock_cache, mock_Collection):
+        """
+        Test that the value from get_instance is used and saved to the cache
+        """
+        collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
+        collection_element.get_instance = MagicMock()
+        collection_element.get_access_permissions = MagicMock()
+        collection_element.get_access_permissions().get_full_data.return_value = 'get_instance_value'
+        mock_cache.get.return_value = None
+
+        instance = collection_element.get_full_data()
+
+        self.assertEqual(instance, 'get_instance_value')
+        mock_cache.get.assert_called_once_with('testmodule/model:42')
+        collection_element.get_instance.assert_called_once_with()
+        mock_cache.set.assert_called_once_with('testmodule/model:42', 'get_instance_value')
+        mock_Collection.assert_called_once_with('testmodule/model')
+        mock_Collection().add_id_to_cache.assert_called_once_with(42)
+
+    def test_equal(self):
+        self.assertEqual(
+            collection.CollectionElement.from_values('testmodule/model', 1),
+            collection.CollectionElement.from_values('testmodule/model', 1))
+        self.assertEqual(
+            collection.CollectionElement.from_values('testmodule/model', 1),
+            collection.CollectionElement.from_values('testmodule/model', 1, deleted=True))
+        self.assertNotEqual(
+            collection.CollectionElement.from_values('testmodule/model', 1),
+            collection.CollectionElement.from_values('testmodule/model', 2))
+        self.assertNotEqual(
+            collection.CollectionElement.from_values('testmodule/model', 1),
+            collection.CollectionElement.from_values('testmodule/other_model', 1))
+
+
+class TestCollection(TestCase):
+    @patch('openslides.utils.collection.CollectionElement')
+    @patch('openslides.utils.collection.cache')
+    def test_element_generator(self, mock_cache, mock_CollectionElement):
+        """
+        Test with the following scenario: The collection has three elements. Two
+        are in the cache and one is not.
+        """
+        test_collection = collection.Collection('testmodule/model')
+        test_collection.get_all_ids = MagicMock(return_value=set([1, 2, 3]))
+        test_collection.get_model = MagicMock()
+        test_collection.get_model().objects.get_full_queryset().filter.return_value = ['my_instance']
+        mock_cache.get_many.return_value = {
+            'testmodule/model:1': 'element1',
+            'testmodule/model:2': 'element2'}
+
+        list(test_collection.element_generator())
+
+        mock_cache.get_many.assert_called_once_with(
+            ['testmodule/model:1', 'testmodule/model:2', 'testmodule/model:3'])
+        test_collection.get_model().objects.get_full_queryset().filter.assert_called_once_with(pk__in={3})
+        self.assertEqual(mock_CollectionElement.from_values.call_count, 2)
+        self.assertEqual(mock_CollectionElement.from_instance.call_count, 1)