diff --git a/.gitignore b/.gitignore index c49dc0900..335be371b 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ dist/* # Unit test and coverage reports .coverage +tests/file/* diff --git a/openslides/agenda/models.py b/openslides/agenda/models.py index a97d9701a..27e17645e 100644 --- a/openslides/agenda/models.py +++ b/openslides/agenda/models.py @@ -23,6 +23,9 @@ class ItemManager(models.Manager): Customized model manager with special methods for agenda tree and numbering. """ + def get_full_queryset(self): + return self.get_queryset().prefetch_related('speakers', 'content_object') + def get_only_agenda_items(self): """ Generator, which yields only agenda items. Skips hidden items. diff --git a/openslides/agenda/views.py b/openslides/agenda/views.py index d4c91d0c2..bfe066c75 100644 --- a/openslides/agenda/views.py +++ b/openslides/agenda/views.py @@ -58,26 +58,6 @@ class ItemViewSet(ListModelMixin, RetrieveModelMixin, UpdateModelMixin, GenericV result = False return result - def check_object_permissions(self, request, obj): - """ - Checks if the requesting user has permission to see also an - organizational item if it is one. - """ - # TODO: Move this logic to access_permissions.ItemAccessPermissions. - if obj.is_hidden() and not request.user.has_perm('agenda.can_see_hidden_items'): - self.permission_denied(request) - - def get_queryset(self): - """ - Filters organizational items if the user has no permission to see them. - """ - # TODO: Move this logic to access_permissions.ItemAccessPermissions. - queryset = super().get_queryset() - if not self.request.user.has_perm('agenda.can_see_hidden_items'): - pk_list = [item.pk for item in Item.objects.get_only_agenda_items()] - queryset = queryset.filter(pk__in=pk_list) - return queryset - @detail_route(methods=['POST', 'DELETE']) def manage_speaker(self, request, pk=None): """ @@ -243,31 +223,6 @@ class ItemViewSet(ListModelMixin, RetrieveModelMixin, UpdateModelMixin, GenericV # Initiate response. return Response({'detail': _('List of speakers successfully sorted.')}) - @list_route(methods=['get', 'put']) - def tree(self, request): - """ - Returns or sets the agenda tree. - """ - if request.method == 'PUT': - if not (request.user.has_perm('agenda.can_manage') and - request.user.has_perm('agenda.can_see_hidden_items')): - self.permission_denied(request) - try: - tree = request.data['tree'] - except KeyError as error: - response = Response({'detail': 'Agenda tree is missing.'}, status=400) - else: - try: - Item.objects.set_tree(tree) - except ValueError as error: - response = Response({'detail': str(error)}, status=400) - else: - response = Response({'detail': 'Agenda tree successfully updated.'}) - else: - # request.method == 'GET' - response = Response(Item.objects.get_tree()) - return response - @list_route(methods=['post']) def numbering(self, request): """ diff --git a/openslides/assignments/models.py b/openslides/assignments/models.py index b997c33b8..5986c4c8a 100644 --- a/openslides/assignments/models.py +++ b/openslides/assignments/models.py @@ -1,7 +1,7 @@ from collections import OrderedDict from django.conf import settings -from django.contrib.contenttypes.models import ContentType +from django.contrib.contenttypes.fields import GenericRelation from django.db import models from django.utils.translation import ugettext as _ from django.utils.translation import ugettext_noop @@ -50,12 +50,22 @@ class AssignmentRelatedUser(RESTModelMixin, models.Model): return self.assignment +class AssignmentManager(models.Manager): + def get_full_queryset(self): + return self.get_queryset().prefetch_related( + 'related_users', + 'agenda_items', + 'polls') + + class Assignment(RESTModelMixin, models.Model): """ Model for assignments. """ access_permissions = AssignmentAccessPermissions() + objects = AssignmentManager() + PHASE_SEARCH = 0 PHASE_VOTING = 1 PHASE_FINISHED = 2 @@ -111,6 +121,10 @@ class Assignment(RESTModelMixin, models.Model): Tags for the assignment. """ + # In theory there could be one then more agenda_item. But support only one. + # See the property agenda_item. + agenda_items = GenericRelation(Item, related_name='assignments') + class Meta: default_permissions = () permissions = ( @@ -290,8 +304,7 @@ class Assignment(RESTModelMixin, models.Model): """ Returns the related agenda item. """ - content_type = ContentType.objects.get_for_model(self) - return Item.objects.get(object_id=self.pk, content_type=content_type) + return self.agenda_items.all()[0] @property def agenda_item_id(self): diff --git a/openslides/core/access_permissions.py b/openslides/core/access_permissions.py index e2b553661..623e6af80 100644 --- a/openslides/core/access_permissions.py +++ b/openslides/core/access_permissions.py @@ -87,4 +87,8 @@ class ConfigAccessPermissions(BaseAccessPermissions): # Attention: The format of this response has to be the same as in # the retrieve method of ConfigViewSet. + if isinstance(instance, dict): + # It happens, that the caching system already sends the correct dict + # as instance. + return instance return {'key': instance.key, 'value': config[instance.key]} diff --git a/openslides/core/apps.py b/openslides/core/apps.py index 81d25365c..c0f68c1f5 100644 --- a/openslides/core/apps.py +++ b/openslides/core/apps.py @@ -16,7 +16,6 @@ class CoreAppConfig(AppConfig): from django.db.models import signals from openslides.core.config import config from openslides.core.signals import post_permission_creation - from openslides.utils.autoupdate import inform_changed_data_receiver, inform_deleted_data_receiver from openslides.utils.rest_api import router from openslides.utils.search import index_add_instance, index_del_instance from .config_variables import get_config_variables @@ -45,15 +44,6 @@ class CoreAppConfig(AppConfig): router.register(self.get_model('Tag').get_collection_string(), TagViewSet) router.register(self.get_model('ConfigStore').get_collection_string(), ConfigViewSet, 'config') - # Update data when any model of any installed app is saved or deleted. - # TODO: Test if the m2m_changed signal is also needed. - signals.post_save.connect( - inform_changed_data_receiver, - dispatch_uid='inform_changed_data_receiver') - signals.post_delete.connect( - inform_deleted_data_receiver, - dispatch_uid='inform_deleted_data_receiver') - # Update the search when a model is saved or deleted signals.post_save.connect( index_add_instance, diff --git a/openslides/core/migrations/0005_auto_20160918_2104.py b/openslides/core/migrations/0005_auto_20160918_2104.py index c35ab6b41..377e106df 100644 --- a/openslides/core/migrations/0005_auto_20160918_2104.py +++ b/openslides/core/migrations/0005_auto_20160918_2104.py @@ -2,19 +2,10 @@ # Generated by Django 1.10.1 on 2016-09-18 19:04 from __future__ import unicode_literals -from django.db import migrations, models - -from openslides.utils.autoupdate import ( - inform_changed_data_receiver, - inform_deleted_data_receiver, -) +from django.db import migrations def move_custom_slides_to_topics(apps, schema_editor): - # Disconnect autoupdate. We do not want to trigger it here. - models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver') - models.signals.post_save.disconnect(dispatch_uid='inform_deleted_data_receiver') - # We get the model from the versioned app registry; # if we directly import it, it will be the wrong version. ContentType = apps.get_model('contenttypes', 'ContentType') @@ -39,14 +30,6 @@ def move_custom_slides_to_topics(apps, schema_editor): CustomSlide.objects.all().delete() content_type_custom_slide.delete() - # Reconnect autoupdate. - models.signals.post_save.connect( - inform_changed_data_receiver, - dispatch_uid='inform_changed_data_receiver') - models.signals.post_delete.connect( - inform_deleted_data_receiver, - dispatch_uid='inform_deleted_data_receiver') - class Migration(migrations.Migration): diff --git a/openslides/core/models.py b/openslides/core/models.py index 953496b42..4937b7bfa 100644 --- a/openslides/core/models.py +++ b/openslides/core/models.py @@ -116,7 +116,7 @@ class Projector(RESTModelMixin, models.Model): result[key]['error'] = str(e) return result - def get_all_requirements(self): + def get_all_requirements(self, on_slide=None): """ Generator which returns all instances that are shown on this projector. """ diff --git a/openslides/core/views.py b/openslides/core/views.py index e015f90dd..83e396419 100644 --- a/openslides/core/views.py +++ b/openslides/core/views.py @@ -18,6 +18,7 @@ from django.utils.timezone import now from openslides import __version__ as version from openslides.utils import views as utils_views +from openslides.utils.collection import Collection, CollectionElement from openslides.utils.plugins import ( get_plugin_description, get_plugin_verbose_name, @@ -41,7 +42,7 @@ from .access_permissions import ( ) from .config import config from .exceptions import ConfigError, ConfigNotFound -from .models import ChatMessage, ProjectionDefault, Projector, Tag +from .models import ChatMessage, ConfigStore, ProjectionDefault, Projector, Tag # Special Django views @@ -606,22 +607,25 @@ class ConfigViewSet(ViewSet): def list(self, request): """ - Lists all config variables. Everybody can see them. + Lists all config variables. """ - return Response([{'key': key, 'value': value} for key, value in config.items()]) + collection = Collection(config.get_collection_string()) + return Response(collection.as_list_for_user(request.user)) def retrieve(self, request, *args, **kwargs): """ - Retrieves a config variable. Everybody can see it. + Retrieves a config variable. """ key = kwargs['pk'] + collection_element = CollectionElement.from_values(config.get_collection_string(), key) try: - value = config[key] - except ConfigNotFound: + content = collection_element.as_dict_for_user(request.user) + except ConfigStore.DoesNotExist: raise Http404 - # Attention: The format of this response has to be the same as in - # the get_full_data method of ConfigAccessPermissions. - return Response({'key': key, 'value': value}) + if content is None: + # If content is None, the user has no permissions to see the item. + self.permission_denied() + return Response(content) def update(self, request, *args, **kwargs): """ diff --git a/openslides/mediafiles/views.py b/openslides/mediafiles/views.py index 91ff41ace..92170899f 100644 --- a/openslides/mediafiles/views.py +++ b/openslides/mediafiles/views.py @@ -15,13 +15,6 @@ class MediafileViewSet(ModelViewSet): access_permissions = MediafileAccessPermissions() queryset = Mediafile.objects.all() - def get_queryset(self): - queryset = super().get_queryset() - user = self.request.user - if not user.has_perm('mediafiles.can_see_private'): - queryset = queryset.filter(private=False) - return queryset - def check_view_permissions(self): """ Returns True if the user has required permissions. diff --git a/openslides/motions/migrations/0004_auto_20160907_2343.py b/openslides/motions/migrations/0004_auto_20160907_2343.py index 34b3269bb..88dde5d08 100644 --- a/openslides/motions/migrations/0004_auto_20160907_2343.py +++ b/openslides/motions/migrations/0004_auto_20160907_2343.py @@ -5,16 +5,11 @@ from __future__ import unicode_literals import django.db.models.deletion from django.db import migrations, models -from openslides.utils.autoupdate import inform_changed_data_receiver - def change_label_of_state(apps, schema_editor): """ Changes the label of former state "commited a bill" to "refered to committee". """ - # Disconnect autoupdate. We do not want to trigger it here. - models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver') - # We get the model from the versioned app registry; # if we directly import it, it will be the wrong version. State = apps.get_model('motions', 'State') @@ -29,19 +24,11 @@ def change_label_of_state(apps, schema_editor): state.action_word = 'Refer to committee' state.save() - # Reconnect autoupdate. - models.signals.post_save.connect( - inform_changed_data_receiver, - dispatch_uid='inform_changed_data_receiver') - def add_recommendation_labels(apps, schema_editor): """ Adds recommendation labels to some of the built-in states. """ - # Disconnect autoupdate. We do not want to trigger it here. - models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver') - # We get the model from the versioned app registry; # if we directly import it, it will be the wrong version. State = apps.get_model('motions', 'State') diff --git a/openslides/motions/models.py b/openslides/motions/models.py index f46163b4e..5d4788eda 100644 --- a/openslides/motions/models.py +++ b/openslides/motions/models.py @@ -1,5 +1,5 @@ from django.conf import settings -from django.contrib.contenttypes.models import ContentType +from django.contrib.contenttypes.fields import GenericRelation from django.db import models from django.db.models import Max from django.utils import formats @@ -28,6 +28,21 @@ from .access_permissions import ( from .exceptions import WorkflowError +class MotionManager(models.Manager): + def get_full_queryset(self): + return (super().get_queryset() + .select_related('active_version') + .prefetch_related( + 'versions', + 'agenda_items', + 'log_messages', + 'polls', + 'attachments', + 'tags', + 'submitters', + 'supporters')) + + class Motion(RESTModelMixin, models.Model): """ The Motion Class. @@ -36,6 +51,8 @@ class Motion(RESTModelMixin, models.Model): """ access_permissions = MotionAccessPermissions() + objects = MotionManager() + active_version = models.ForeignKey( 'MotionVersion', on_delete=models.SET_NULL, @@ -134,6 +151,10 @@ class Motion(RESTModelMixin, models.Model): Configurable fields for comments. Contains a list of strings. """ + # In theory there could be one then more agenda_item. But support only one. + # See the property agenda_item. + agenda_items = GenericRelation(Item, related_name='motions') + class Meta: default_permissions = () permissions = ( @@ -519,8 +540,7 @@ class Motion(RESTModelMixin, models.Model): """ Returns the related agenda item. """ - content_type = ContentType.objects.get_for_model(self) - return Item.objects.get(object_id=self.pk, content_type=content_type) + return self.agenda_items.all()[0] @property def agenda_item_id(self): @@ -940,12 +960,21 @@ class State(RESTModelMixin, models.Model): return self.workflow +class WorkflowManager(models.Manager): + def get_full_queryset(self): + return (self.get_queryset() + .select_related('first_state') + .prefetch_related('states', 'states__next_states')) + + class Workflow(RESTModelMixin, models.Model): """ Defines a workflow for a motion. """ access_permissions = WorkflowAccessPermissions() + objects = WorkflowManager() + name = models.CharField(max_length=255) """A string representing the workflow.""" diff --git a/openslides/motions/views.py b/openslides/motions/views.py index 7678ee99d..ff5e913eb 100644 --- a/openslides/motions/views.py +++ b/openslides/motions/views.py @@ -77,27 +77,6 @@ class MotionViewSet(ModelViewSet): result = False return result - def list(self, request, *args, **kwargs): - """ - Customized view endpoint to list all motions. - - Hides non public comment fields for some users. - """ - response = super().list(request, *args, **kwargs) - for i, motion in enumerate(response.data): - response.data[i] = self.get_access_permissions().get_restricted_data(motion, self.request.user) - return response - - def retrieve(self, request, *args, **kwargs): - """ - Customized view endpoint to retrieve a motion. - - Hides non public comment fields for some users. - """ - response = super().retrieve(request, *args, **kwargs) - response.data = self.get_access_permissions().get_restricted_data(response.data, self.request.user) - return response - def create(self, request, *args, **kwargs): """ Customized view endpoint to create a new motion. diff --git a/openslides/topics/models.py b/openslides/topics/models.py index 25a3b967e..51d65968e 100644 --- a/openslides/topics/models.py +++ b/openslides/topics/models.py @@ -1,4 +1,4 @@ -from django.contrib.contenttypes.models import ContentType +from django.contrib.contenttypes.fields import GenericRelation from django.db import models from ..agenda.models import Item @@ -7,16 +7,27 @@ from ..utils.models import RESTModelMixin from .access_permissions import TopicAccessPermissions +class TopicManager(models.Manager): + def get_queryset(self): + query = super().get_queryset().prefetch_related('attachments', 'agenda_items') + return query + + class Topic(RESTModelMixin, models.Model): """ Model for slides with custom content. Used to be called custom slide. """ access_permissions = TopicAccessPermissions() + objects = TopicManager() title = models.CharField(max_length=256) text = models.TextField(blank=True) attachments = models.ManyToManyField(Mediafile, blank=True) + # In theory there could be one then more agenda_item. But support only one. + # See the property agenda_item. + agenda_items = GenericRelation(Item, related_name='topics') + class Meta: default_permissions = () @@ -28,8 +39,7 @@ class Topic(RESTModelMixin, models.Model): """ Returns the related agenda item. """ - content_type = ContentType.objects.get_for_model(self) - return Item.objects.get(object_id=self.pk, content_type=content_type) + return self.agenda_items.all()[0] @property def agenda_item_id(self): diff --git a/openslides/users/access_permissions.py b/openslides/users/access_permissions.py index 0be9d0291..eed1020b7 100644 --- a/openslides/users/access_permissions.py +++ b/openslides/users/access_permissions.py @@ -15,15 +15,9 @@ class UserAccessPermissions(BaseAccessPermissions): """ Returns different serializer classes with respect user's permissions. """ - from .serializers import UserCanSeeSerializer, UserCanSeeExtraSerializer, UserFullSerializer + from .serializers import UserFullSerializer - if (user is None or (user.has_perm('users.can_see_extra_data') and user.has_perm('users.can_manage'))): - serializer_class = UserFullSerializer - elif user.has_perm('users.can_see_extra_data'): - serializer_class = UserCanSeeExtraSerializer - else: - serializer_class = UserCanSeeSerializer - return serializer_class + return UserFullSerializer def get_restricted_data(self, full_data, user): """ diff --git a/openslides/users/migrations/0004_groups.py b/openslides/users/migrations/0004_groups.py index 66ff1bb5a..1adc3428c 100644 --- a/openslides/users/migrations/0004_groups.py +++ b/openslides/users/migrations/0004_groups.py @@ -2,9 +2,7 @@ # Generated by Django 1.9.7 on 2016-08-01 14:54 from __future__ import unicode_literals -from django.db import migrations, models - -from openslides.utils.autoupdate import inform_changed_data_receiver +from django.db import migrations def migrate_groups_and_user_permissions(apps, schema_editor): @@ -20,9 +18,6 @@ def migrate_groups_and_user_permissions(apps, schema_editor): - the name of the first group is 'Guests', - the name of the second group is 'Registered users'. """ - # Disconnect autoupdate. We do not want to trigger it here. - models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver') - User = apps.get_model('users', 'User') Group = apps.get_model('auth', 'Group') @@ -52,11 +47,6 @@ def migrate_groups_and_user_permissions(apps, schema_editor): for permission in group_registered.permissions.all(): group.permissions.add(permission) - # Reconnect autoupdate. - models.signals.post_save.connect( - inform_changed_data_receiver, - dispatch_uid='inform_changed_data_receiver') - class Migration(migrations.Migration): diff --git a/openslides/users/models.py b/openslides/users/models.py index 2037f7af4..76b7e775e 100644 --- a/openslides/users/models.py +++ b/openslides/users/models.py @@ -23,6 +23,10 @@ class UserManager(BaseUserManager): Customized manager that creates new users only with a password and a username. """ + + def get_full_queryset(self): + return self.get_queryset().prefetch_related('groups') + def create_user(self, username, password, **kwargs): """ Creates a new user only with a password and a username. diff --git a/openslides/users/serializers.py b/openslides/users/serializers.py index 35ea95d52..e023fd087 100644 --- a/openslides/users/serializers.py +++ b/openslides/users/serializers.py @@ -12,79 +12,24 @@ from ..utils.rest_api import ( from .models import Group, User USERCANSEESERIALIZER_FIELDS = ( - 'id', - 'username', - 'title', - 'first_name', - 'last_name', - 'structure_level', - 'number', - 'about_me', - 'groups', - 'is_committee', - ) + 'id', + 'username', + 'title', + 'first_name', + 'last_name', + 'structure_level', + 'number', + 'about_me', + 'groups', + 'is_committee', +) -class UserCanSeeSerializer(ModelSerializer): - """ - Serializer for users.models.User objects to be used by users who have - only the permission to see users and to change some date of theirselfs. - - Attention: Viewset has to ensure that a user can update only himself. - """ - class Meta: - model = User - fields = USERCANSEESERIALIZER_FIELDS - read_only_fields = ( - 'number', - 'groups', - 'is_comittee', - ) - - -USERCANSEEEXTRASERIALIZER_FIELDS = ( - 'id', - 'is_present', - 'username', - 'title', - 'first_name', - 'last_name', - 'number', - 'structure_level', - 'about_me', - 'comment', - 'groups', - 'is_active', - 'is_committee', - ) - - -class UserCanSeeExtraSerializer(ModelSerializer): - """ - Serializer for users.models.User objects to be used by users who have - the permission to see users with extra data and to change some date of - theirselfs. - - Attention: Viewset has to ensure that a user can update only himself. - """ - groups = IdPrimaryKeyRelatedField( - many=True, - queryset=Group.objects.exclude(pk=1), - help_text=ugettext_lazy('The groups this user belongs to. A user will ' - 'get all permissions granted to each of ' - 'his/her groups.')) - - class Meta: - model = User - fields = USERCANSEEEXTRASERIALIZER_FIELDS - read_only_fields = ( - 'is_present', - 'number', - 'comment', - 'groups', - 'is_comittee', - 'is_active', - ) +USERCANSEEEXTRASERIALIZER_FIELDS = USERCANSEESERIALIZER_FIELDS + ( + 'is_present', + 'comment', + 'is_active', +) class UserFullSerializer(ModelSerializer): diff --git a/openslides/users/views.py b/openslides/users/views.py index 52e89f3f6..e46f6c245 100644 --- a/openslides/users/views.py +++ b/openslides/users/views.py @@ -18,7 +18,7 @@ from ..utils.views import APIView, PDFView from .access_permissions import UserAccessPermissions from .models import Group, User from .pdf import users_passwords_to_pdf, users_to_pdf -from .serializers import GroupSerializer, UserFullSerializer +from .serializers import GroupSerializer # Viewsets for the REST API @@ -49,19 +49,6 @@ class UserViewSet(ModelViewSet): result = False return result - def get_serializer_class(self): - """ - Returns different serializer classes with respect to action. - """ - if self.action in ('create', 'partial_update', 'update'): - # Return the UserFullSerializer for edit requests. - serializer_class = UserFullSerializer - else: - # Return different serializers according to user permsissions via - # access permissions class. - serializer_class = super().get_serializer_class() - return serializer_class - def update(self, request, *args, **kwargs): """ Customized view endpoint to update an user. @@ -138,7 +125,7 @@ class GroupViewSet(ModelViewSet): partial_update, update and destroy. """ metadata_class = GroupViewSetMetadata - queryset = Group.objects.all() + queryset = Group.objects.prefetch_related('permissions', 'permissions__content_type') serializer_class = GroupSerializer def check_view_permissions(self): diff --git a/openslides/utils/autoupdate.py b/openslides/utils/autoupdate.py index 60d132203..b9039ef4f 100644 --- a/openslides/utils/autoupdate.py +++ b/openslides/utils/autoupdate.py @@ -11,7 +11,7 @@ from ..core.config import config from ..core.models import Projector from ..users.auth import AnonymousUser from ..users.models import User -from .collection import CollectionElement +from .collection import Collection, CollectionElement def get_logged_in_users(): @@ -23,18 +23,19 @@ def get_logged_in_users(): return User.objects.exclude(session=None).filter(session__expire_date__gte=timezone.now()).distinct() -def get_projector_element_data(projector): +def get_projector_element_data(projector, on_slide=None): """ Returns a list of dicts that are required for a specific projector. The argument projector has to be a projector instance. + + If on_slide is a string that matches an slide on the projector, then only + elements on this slide are returned. """ output = [] - for requirement in projector.get_all_requirements(): + for requirement in projector.get_all_requirements(on_slide): required_collection_element = CollectionElement.from_instance(requirement) - element_dict = required_collection_element.as_autoupdate_for_projector() - if element_dict is not None: - output.append(element_dict) + output.append(required_collection_element.as_autoupdate_for_projector()) return output @@ -83,12 +84,8 @@ def ws_add_projector(message, projector_id): output = get_projector_element_data(projector) # Send all config elements. - for key, value in config.items(): - output.append({ - 'collection': config.get_collection_string(), - 'id': key, - 'action': 'changed', - 'data': {'key': key, 'value': value}}) + collection = Collection(config.get_collection_string()) + output.extend(collection.as_autoupdate_for_projector()) # Send the projector instance. collection_element = CollectionElement.from_instance(projector) @@ -115,9 +112,6 @@ def send_data(message): for user in itertools.chain(get_logged_in_users(), [AnonymousUser()]): channel = Group('user-{}'.format(user.id)) output = collection_element.as_autoupdate_for_user(user) - if output is None: - # There are no data for the user so he can't see the object. Skip him. - continue channel.send({'text': json.dumps([output])}) # Get the projector elements where data have to be sent and if whole projector @@ -157,14 +151,20 @@ def send_data(message): else: output = broadcast_projector_data else: + # The list will be filled in the next lines. output = [] + output.append(collection_element.as_autoupdate_for_projector()) if output: Group('projector-{}'.format(projector.pk)).send( {'text': json.dumps(output)}) -def inform_changed_data(instance, is_deleted=False): +def inform_changed_data(instance, information=None): + """ + Informs the autoupdate system and the caching system about the creation or + update of an element. + """ try: root_instance = instance.get_root_rest_element() except AttributeError: @@ -173,29 +173,39 @@ def inform_changed_data(instance, is_deleted=False): else: collection_element = CollectionElement.from_instance( root_instance, - is_deleted=is_deleted and instance == root_instance) + information=information) - # If currently there is an open database transaction, then the following - # function is only called, when the transaction is commited. If there - # is currently no transaction, then the function is called immediately. - def send_autoupdate(): - try: - Channel('autoupdate.send_data').send(collection_element.as_channels_message()) - except ChannelLayer.ChannelFull: - pass + transaction.on_commit(lambda: send_autoupdate(collection_element)) - transaction.on_commit(send_autoupdate) - - -def inform_changed_data_receiver(sender, instance, **kwargs): +def inform_deleted_data(collection_string, id, information=None): """ - Receiver for the inform_changed_data function to use in a signal. + Informs the autoupdate system and the caching system about the deletion of + an element. """ - inform_changed_data(instance) + collection_element = CollectionElement.from_values( + collection_string=collection_string, + id=id, + deleted=True, + information=information) + + # If currently there is an open database transaction, then the following + # function is only called, when the transaction is commited. If there + # is currently no transaction, then the function is called immediately. + def send_autoupdate(): + try: + Channel('autoupdate.send_data').send(collection_element.as_channels_message()) + except ChannelLayer.ChannelFull: + pass + + transaction.on_commit(lambda: send_autoupdate(collection_element)) -def inform_deleted_data_receiver(sender, instance, **kwargs): +def send_autoupdate(collection_element): """ - Receiver for the inform_changed_data function to use in a signal. + Helper function, that sends a collection_element through a channel to the + autoupdate system. """ - inform_changed_data(instance, is_deleted=True) + try: + Channel('autoupdate.send_data').send(collection_element.as_channels_message()) + except ChannelLayer.ChannelFull: + pass diff --git a/openslides/utils/collection.py b/openslides/utils/collection.py index 30293b9b1..7a0413700 100644 --- a/openslides/utils/collection.py +++ b/openslides/utils/collection.py @@ -1,25 +1,41 @@ from django.apps import apps +from django.core.cache import cache, caches class CollectionElement: @classmethod - def from_instance(cls, instance, is_deleted=False): + def from_instance(cls, instance, deleted=False, information=None): """ Returns a collection element from a database instance. + + This will also update the instance in the cache. + + If deleted is set to True, the element is deleted from the cache. """ - return cls(instance=instance, is_deleted=is_deleted) + return cls(instance=instance, deleted=deleted, information=information) @classmethod - def from_values(cls, collection_string, id, is_deleted=False): + def from_values(cls, collection_string, id, deleted=False, full_data=None, information=None): """ Returns a collection element from a collection_string and an id. - """ - return cls(collection_string=collection_string, id=id, is_deleted=is_deleted) - def __init__(self, instance=None, is_deleted=False, collection_string=None, id=None): + If deleted is set to True, the element is deleted from the cache. + + With the argument full_data, the content of the CollectionElement can be set. + It has to be a dict in the format that is used be access_permission.get_full_data(). + """ + return cls(collection_string=collection_string, id=id, deleted=deleted, + full_data=full_data, information=information) + + def __init__(self, instance=None, deleted=False, collection_string=None, id=None, + full_data=None, information=None): """ Do not use this. Use the methods from_instance() or from_values(). """ + self.instance = instance + self.deleted = deleted + self.full_data = full_data + self.information = information if instance is not None: self.collection_string = instance.get_collection_string() self.id = instance.pk @@ -31,62 +47,73 @@ class CollectionElement: 'Invalid state. Use CollectionElement.from_instance() or ' 'CollectionElement.from_values() but not CollectionElement() ' 'directly.') - self.instance = instance - self.deleted = is_deleted + + if self.is_deleted(): + # Delete the element from the cache, if self.is_deleted() is True: + self.delete_from_cache() + elif instance is not None: + # If this element is created via instance and the instance is not deleted + # then update the cache. + self.save_to_cache() def as_channels_message(self): """ Returns a dictonary that can be used to send the object through the channels system. """ - return { + channel_message = { 'collection_string': self.collection_string, 'id': self.id, - 'is_deleted': self.is_deleted()} + 'deleted': self.is_deleted()} + if self.information is not None: + channel_message['information'] = self.information + return channel_message + + def as_autoupdate(self, method, *args): + """ + Only for internal use. Do not use it directly. Use as_autoupdate_for_user() + or as_autoupdate_for_projector(). + """ + output = { + 'collection': self.collection_string, + 'id': self.id, + 'action': 'deleted' if self.is_deleted() else 'changed', + } + if not self.is_deleted(): + data = getattr(self.get_access_permissions(), method)( + self.get_full_data(), + *args) + if data is None: + # The user is not allowed to see this element. Set action to deleted. + output['action'] = 'deleted' + else: + output['data'] = data + return output def as_autoupdate_for_user(self, user): """ Returns a dict that can be sent through the autoupdate system for a site user. - - Returns None if the user can not see the element. """ - output = { - 'collection': self.collection_string, - 'id': self.id, - 'action': 'deleted' if self.is_deleted() else 'changed', - } - if not self.is_deleted(): - data = self.get_access_permissions().get_restricted_data( - self.get_full_data(), user) - if data is None: - # The user is not allowed to see this element. Reset output to None. - output = None - else: - output['data'] = data - return output + return self.as_autoupdate( + 'get_restricted_data', + user) def as_autoupdate_for_projector(self): """ Returns a dict that can be sent through the autoupdate system for the projector. - - Returns None if the projector can not see the element. """ - output = { - 'collection': self.collection_string, - 'id': self.id, - 'action': 'deleted' if self.is_deleted() else 'changed', - } - if not self.is_deleted(): - data = self.get_access_permissions().get_projector_data( - self.get_full_data()) - if data is None: - # The user is not allowed to see this element. Reset output to None. - output = None - else: - output['data'] = data - return output + return self.as_autoupdate( + 'get_projector_data') + + def as_dict_for_user(self, user): + """ + Returns a dict with the data for a user. Can be used for the rest api. + """ + return self.get_access_permissions().get_restricted_data( + self.get_full_data(), + user) def get_model(self): """ @@ -102,8 +129,21 @@ class CollectionElement: """ if self.is_deleted(): raise RuntimeError("The collection element is deleted.") + if self.instance is None: - self.instance = self.get_model().objects.get(pk=self.id) + # The config instance has to be get from the config element, because + # some config values are not in the db. + from openslides.core.config import config + if (self.collection_string == config.get_collection_string() and + isinstance(self.id, str)): + self.instance = {'key': self.id, 'value': config[self.id]} + else: + model = self.get_model() + try: + query = model.objects.get_full_queryset() + except AttributeError: + query = model.objects + self.instance = query.get(pk=self.id) return self.instance def get_access_permissions(self): @@ -117,7 +157,20 @@ class CollectionElement: Returns the full_data of this collection_element from with all other dics can be generated. """ - return self.get_access_permissions().get_full_data(self.get_instance()) + # If the full_data is already loaded, return it + # If there is a db_instance, use it to get the full_data + # else: try to use the cache. + # If there is no value in the cach, get the content from the db and save + # it to the cache. + if self.full_data is None and self.instance is None: + # Use the cache version if self.instance is not set. + # After this line full_data can be None, if the element is not in the cache. + self.full_data = cache.get(self.get_cache_key()) + + if self.full_data is None: + self.full_data = self.get_access_permissions().get_full_data(self.get_instance()) + self.save_to_cache() + return self.full_data def is_deleted(self): """ @@ -125,6 +178,217 @@ class CollectionElement: """ return self.deleted + def get_cache_key(self): + """ + Returns a string that is used as cache key for a single instance. + """ + return get_single_element_cache_key(self.collection_string, self.id) + + def delete_from_cache(self): + """ + Delets an element from the cache. + + Does nothing if the element is not in the cache. + """ + # Deletes the element from the cache. + cache.delete(self.get_cache_key()) + + # Delete the id of the instance of the instance list + Collection(self.collection_string).delete_id_from_cache(self.id) + + def save_to_cache(self): + """ + Add or update the element to the cache. + """ + # Set the element to the cache. + cache.set(self.get_cache_key(), self.get_full_data()) + + # Add the id of the element to the collection + Collection(self.collection_string).add_id_to_cache(self.id) + + +class Collection: + """ + Represents all elements of one collection. + """ + + def __init__(self, collection_string): + self.collection_string = collection_string + + def get_cache_key(self, raw=False): + """ + Returns a string that is used as cache key for a collection. + + Django adds a prefix to the cache key when using the django cache api. + In other cases use raw=True to add the same cache key. + """ + key = get_element_list_cache_key(self.collection_string) + if raw: + key = self.make_key(key) + return key + + def get_model(self): + """ + Returns the django model that is used for this collection. + """ + return get_model_from_collection_string(self.collection_string) + + def element_generator(self): + """ + Generator that yields all collection_elements of this collection. + """ + # Get all cache keys. + ids = self.get_all_ids() + cache_keys = [ + get_single_element_cache_key(self.collection_string, id) + for id in ids] + cached_full_data_dict = cache.get_many(cache_keys) + + # Get all ids that are missing. + missing_ids = set() + for cache_key in cache_keys: + if cache_key not in cached_full_data_dict: + missing_ids.add(get_collection_id_from_cache_key(cache_key)[1]) + + # Generate collection elements that where in the cache. + for cache_key, cached_full_data in cached_full_data_dict.items(): + yield CollectionElement.from_values( + *get_collection_id_from_cache_key(cache_key), + full_data=cached_full_data) + + # Generate collection element that where not in the cache. + if missing_ids: + from openslides.core.config import config + if self.collection_string == config.get_collection_string(): + # If config elements are not in the cache, they have to be read from + # the config object. + for key, value in config.items(): + if key in missing_ids: + collection_element = CollectionElement.from_values( + config.get_collection_string(), + key, + full_data={'key': key, 'value': value}) + # We can not use .from_instance therefore the config value + # is not saved to the cache. We have to do it manualy. + collection_element.save_to_cache() + yield collection_element + else: + model = self.get_model() + try: + query = model.objects.get_full_queryset() + except AttributeError: + query = model.objects + for instance in query.filter(pk__in=missing_ids): + yield CollectionElement.from_instance(instance) + + def as_autoupdate_for_projector(self): + """ + Returns a list of dictonaries to send them to the projector. + """ + output = [] + for collection_element in self.element_generator(): + content = collection_element.as_autoupdate_for_projector() + # Content can not be None. If the projector can not see an element, + # then it is marked as deleted. + output.append(content) + return output + + def as_list_for_user(self, user): + """ + Returns a list of dictonaries to send them to a user, for example over + the rest api. + """ + output = [] + for collection_element in self.element_generator(): + content = collection_element.as_dict_for_user(user) + if content is not None: + output.append(content) + return output + + def get_all_ids(self): + """ + Returns a set of all ids of instances in this collection. + """ + from openslides.core.config import config + if self.collection_string == config.get_collection_string(): + ids = config.config_variables.keys() + elif use_redis_cache(): + ids = self.get_all_ids_redis() + else: + ids = self.get_all_ids_other() + return ids + + def get_all_ids_redis(self): + redis = get_redis_connection() + ids = redis.smembers(self.get_cache_key(raw=True)) + if not ids: + ids = set(self.get_model().objects.values_list('pk', flat=True)) + if ids: + redis.sadd(self.get_cache_key(raw=True), *ids) + # Redis returns the ids as string. + ids = set(int(id) for id in ids) + return ids + + def get_all_ids_other(self): + ids = cache.get(self.get_cache_key()) + if ids is None: + # If it is not in the cache then get it from the database. + ids = set(self.get_model().objects.values_list('pk', flat=True)) + cache.set(self.get_cache_key(), ids) + return ids + + def delete_id_from_cache(self, id): + """ + Delets a id from the cache. + """ + if use_redis_cache(): + self.delete_id_from_cache_redis(id) + else: + self.delete_id_from_cache_other(id) + + def delete_id_from_cache_redis(self, id): + redis = get_redis_connection() + redis.srem(self.get_cache_key(raw=True), id) + + def delete_id_from_cache_other(self, id): + ids = cache.get(self.get_cache_key()) + if ids is not None: + ids = set(ids) + try: + ids.remove(id) + except KeyError: + # The id is not part of id list + pass + else: + if ids: + cache.set(self.get_cache_key(), ids) + else: + # Delete the key, if there are not ids left + cache.delete(self.get_cache_key()) + + def add_id_to_cache(self, id): + """ + Adds a collection id to the list of collection ids in the cache. + """ + if use_redis_cache(): + self.add_id_to_cache_redis(id) + else: + self.add_id_to_cache_other(id) + + def add_id_to_cache_redis(self, id): + redis = get_redis_connection() + if redis.exists(self.get_cache_key(raw=True)): + # Only add the value if it is in the cache. + redis.sadd(self.get_cache_key(raw=True), id) + + def add_id_to_cache_other(self, id): + ids = cache.get(self.get_cache_key()) + if ids is not None: + # Only change the value if it is in the cache. + ids = set(ids) + ids.add(id) + cache.set(self.get_cache_key(), ids) + def get_model_from_collection_string(collection_string): """ @@ -152,3 +416,56 @@ def get_model_from_collection_string(collection_string): # No model was found in all apps. raise ValueError('Invalid message. A valid collection_string is missing.') return model + + +def get_single_element_cache_key(collection_string, id): + """ + Returns a string that is used as cache key for a single instance. + """ + return "{prefix}{id}".format( + prefix=get_single_element_cache_key_prefix(collection_string), + id=id) + + +def get_single_element_cache_key_prefix(collection_string): + """ + Returns the first part of the cache key for single elements, which is the + same for all cache keys of the same collection. + """ + return "{collection}:".format(collection=collection_string) + + +def get_element_list_cache_key(collection_string): + """ + Returns a string that is used as cache key for a collection. + """ + return "{collection}".format(collection=collection_string) + + +def get_collection_id_from_cache_key(cache_key): + """ + Returns a tuble of the collection string and the id from a cache_key + created with get_instance_cache_key. + + The returned id can be an integer or an string. + """ + collection_string, id = cache_key.rsplit(':', 1) + try: + id = int(id) + except ValueError: + # The id is no integer. This can happen on config elements + pass + return (collection_string, id) + + +def use_redis_cache(): + try: + from django_redis.cache import RedisCache + except ImportError: + return False + return isinstance(caches['default'], RedisCache) + + +def get_redis_connection(): + from django_redis import get_redis_connection + return get_redis_connection("default") diff --git a/openslides/utils/models.py b/openslides/utils/models.py index ee868cb6a..c2f1547fb 100644 --- a/openslides/utils/models.py +++ b/openslides/utils/models.py @@ -54,3 +54,41 @@ class RESTModelMixin: the database pk. """ return self.pk + + def save(self, skip_autoupdate=False, information=None, *args, **kwargs): + """ + Calls the django save-method and afterwards hits the autoupdate system. + + If skip_autoupdate is set to True, then the autoupdate system is not + informed about the model changed. This also means, that the model cache + is not updated. + + The optional argument information can be an object that is given to the + autoupdate system. It should be a dict. + """ + # TODO: Fix circular imports + from .autoupdate import inform_changed_data + return_value = super().save(*args, **kwargs) + inform_changed_data(self.get_root_rest_element(), information=information) + return return_value + + def delete(self, skip_autoupdate=False, information=None, *args, **kwargs): + """ + Calls the django delete-method and afterwards hits the autoupdate system. + + See the save method above. + """ + # TODO: Fix circular imports + from .autoupdate import inform_changed_data, inform_deleted_data + # Django sets the pk of the instance to None after deleting it. But + # we need the pk to tell the autoupdate system which element was deleted. + instance_pk = self.pk + return_value = super().delete(*args, **kwargs) + if self != self.get_root_rest_element(): + # The deletion of a included element is a change of the master + # element. + # TODO: Does this work in any case with self.pk = None? + inform_changed_data(self.get_root_rest_element(), information=information) + else: + inform_deleted_data(self, information=information) + return return_value diff --git a/openslides/utils/rest_api.py b/openslides/utils/rest_api.py index 68272cd46..66a49ea13 100644 --- a/openslides/utils/rest_api.py +++ b/openslides/utils/rest_api.py @@ -1,15 +1,13 @@ from collections import OrderedDict +from django.http import Http404 from rest_framework import status # noqa from rest_framework.decorators import detail_route, list_route # noqa from rest_framework.metadata import SimpleMetadata # noqa -from rest_framework.mixins import ( # noqa - DestroyModelMixin, - ListModelMixin, - RetrieveModelMixin, - UpdateModelMixin, -) -from rest_framework.response import Response # noqa +from rest_framework.mixins import ListModelMixin as _ListModelMixin +from rest_framework.mixins import RetrieveModelMixin as _RetrieveModelMixin +from rest_framework.mixins import DestroyModelMixin, UpdateModelMixin # noqa +from rest_framework.response import Response from rest_framework.routers import DefaultRouter from rest_framework.serializers import ModelSerializer as _ModelSerializer from rest_framework.serializers import ( # noqa @@ -31,6 +29,8 @@ from rest_framework.viewsets import GenericViewSet as _GenericViewSet # noqa from rest_framework.viewsets import ModelViewSet as _ModelViewSet # noqa from rest_framework.viewsets import ViewSet as _ViewSet # noqa +from .collection import Collection, CollectionElement + router = DefaultRouter() @@ -164,11 +164,63 @@ class ModelSerializer(_ModelSerializer): return fields +class ListModelMixin(_ListModelMixin): + """ + Mixin to add the caching system to list requests. + + It is not allowed to use the method get_queryset() in derivated classes. + The attribute queryset has to be used in the following form: + + queryset = Model.objects.all() + """ + def list(self, request, *args, **kwargs): + model = self.get_queryset().model + try: + collection_string = model.get_collection_string() + except AttributeError: + # The corresponding queryset does not support caching. + response = super().list(request, *args, **kwargs) + else: + collection = Collection(collection_string) + response = Response(collection.as_list_for_user(request.user)) + return response + + +class RetrieveModelMixin(_RetrieveModelMixin): + """ + Mixin to add the caching system to retrieve requests. + + It is not allowed to use the method get_queryset() in derivated classes. + The attribute queryset has to be used in the following form: + + queryset = Model.objects.all() + """ + def retrieve(self, request, *args, **kwargs): + model = self.get_queryset().model + try: + collection_string = model.get_collection_string() + except AttributeError: + # The corresponding queryset does not support caching. + response = super().retrieve(request, *args, **kwargs) + else: + lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field + collection_element = CollectionElement.from_values( + collection_string, self.kwargs[lookup_url_kwarg]) + try: + content = collection_element.as_dict_for_user(request.user) + except collection_element.get_model().DoesNotExist: + raise Http404 + if content is None: + self.permission_denied(request) + response = Response(content) + return response + + class GenericViewSet(PermissionMixin, _GenericViewSet): pass -class ModelViewSet(PermissionMixin, _ModelViewSet): +class ModelViewSet(PermissionMixin, ListModelMixin, RetrieveModelMixin, _ModelViewSet): pass diff --git a/openslides/utils/settings.py.tpl b/openslides/utils/settings.py.tpl index eb8da2300..756d31f7d 100644 --- a/openslides/utils/settings.py.tpl +++ b/openslides/utils/settings.py.tpl @@ -69,7 +69,7 @@ DATABASES = { # CACHES = { # "default": { # "BACKEND": "django_redis.cache.RedisCache", -# "LOCATION": "redis://127.0.0.1:6379/1", +# "LOCATION": "redis://127.0.0.1:6379/0", # "OPTIONS": { # "CLIENT_CLASS": "django_redis.client.DefaultClient", # } diff --git a/tests/integration/agenda/test_views.py b/tests/integration/agenda/test_views.py index 1ddb7ce42..b0e9d0acd 100644 --- a/tests/integration/agenda/test_views.py +++ b/tests/integration/agenda/test_views.py @@ -1,90 +1,7 @@ -import json - -from rest_framework.test import APIClient - -from openslides.agenda.models import Item from openslides.topics.models import Topic from openslides.utils.test import TestCase -class AgendaTreeTest(TestCase): - def setUp(self): - Topic.objects.create(title='item1') - item2 = Topic.objects.create(title='item2').agenda_item - item3 = Topic.objects.create(title='item2a').agenda_item - item3.parent = item2 - item3.save() - self.client = APIClient() - self.client.login(username='admin', password='admin') - - def test_get(self): - response = self.client.get('/rest/agenda/item/tree/') - - self.assertEqual(json.loads(response.content.decode()), - [{'children': [], 'id': 1}, - {'children': [{'children': [], 'id': 3}], 'id': 2}]) - - def test_set(self): - tree = [{'id': 3}, - {'children': [{'id': 1}], 'id': 2}] - - response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json') - - self.assertEqual(response.status_code, 200) - - item1 = Item.objects.get(pk=1) - item2 = Item.objects.get(pk=2) - item3 = Item.objects.get(pk=3) - self.assertEqual(item1.parent_id, 2) - self.assertEqual(item1.weight, 0) - self.assertEqual(item2.parent_id, None) - self.assertEqual(item2.weight, 1) - self.assertEqual(item3.parent_id, None) - self.assertEqual(item3.weight, 0) - - def test_set_without_perm(self): - self.client = APIClient() - - response = self.client.put('/rest/agenda/item/tree/', {'tree': []}, format='json') - - self.assertEqual(response.status_code, 403) - - def test_tree_with_double_item(self): - """ - Test to send a tree that has an item-pk more then once in it. - - It is expected, that the responsecode 400 is returned with a specific - content - """ - tree = [{'id': 1}, {'id': 1}] - - response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json') - - self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, {'detail': "Item 1 is more then once in the tree."}) - - def test_tree_with_empty_children(self): - """ - Test that the chrildren element is not required in the tree - """ - tree = [{'id': 1}] - - response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json') - - self.assertEqual(response.status_code, 200) - - def test_tree_with_unknown_item(self): - """ - Tests that unknown items lead to an error. - """ - tree = [{'id': 500}] - - response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json') - - self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, {'detail': "Item 500 is not in the database."}) - - class TestAgendaPDF(TestCase): def test_get(self): """ diff --git a/tests/integration/agenda/test_viewsets.py b/tests/integration/agenda/test_viewsets.py index c31eb95b1..4d581dae3 100644 --- a/tests/integration/agenda/test_viewsets.py +++ b/tests/integration/agenda/test_viewsets.py @@ -4,9 +4,12 @@ from rest_framework import status from rest_framework.test import APIClient from openslides.agenda.models import Item, Speaker +from openslides.assignments.models import Assignment from openslides.core.config import config from openslides.core.models import Projector +from openslides.motions.models import Motion from openslides.topics.models import Topic +from openslides.users.models import User from openslides.utils.test import TestCase @@ -37,7 +40,61 @@ class RetrieveItem(TestCase): permission = group.permissions.get(content_type__app_label=app_label, codename=codename) group.permissions.remove(permission) response = self.client.get(reverse('item-detail', args=[self.item.pk])) - self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + self.assertEqual(response.status_code, 403) + + +class TestDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some agenda items are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + Topic.objects.create(title='topic{}'.format(index)) + parent = Topic.objects.create(title='parent').agenda_item + child = Topic.objects.create(title='child').agenda_item + child.parent = parent + child.save() + Motion.objects.create(title='motion1') + Motion.objects.create(title='motion2') + Assignment.objects.create(title='assignment', open_posts=5) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions, + * 2 requests to get the list of all agenda items, + * 1 request to get all speakers, + * 3 requests to get the assignments, motions and topics and + + * 2 requests for the motionsversions. + + TODO: There could be less requests to get the session and the request user. + The last two request for the motionsversions are a bug. + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(13): + self.client.get(reverse('item-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions) + * 2 requests to get the list of all agenda items, + * 1 request to get all speakers, + * 3 requests to get the assignments, motions and topics and + + * 32 requests for the motionsversions. + + TODO: The last 32 requests are a bug. + """ + with self.assertNumQueries(40): + self.client.get(reverse('item-list')) class ManageSpeaker(TestCase): diff --git a/tests/integration/assignments/test_viewset.py b/tests/integration/assignments/test_viewset.py index e209a23fe..aebbb8f7c 100644 --- a/tests/integration/assignments/test_viewset.py +++ b/tests/integration/assignments/test_viewset.py @@ -4,9 +4,61 @@ from rest_framework import status from rest_framework.test import APIClient from openslides.assignments.models import Assignment +from openslides.core.config import config +from openslides.users.models import User from openslides.utils.test import TestCase +class TestDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some assignments are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + Assignment.objects.create(title='motion{}'.format(index), open_posts=1) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions, + * 2 requests to get the list of all assignments, + * 1 request to get all related users, + * 1 request to get the agenda item, + * 1 request to get the polls, + + * 10 request to featch each related user again. + + TODO: There could be less requests to get the session and the request user. + The eleven request are a bug. + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(30): + self.client.get(reverse('assignment-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions) + * 2 requests to get the list of all assignments, + * 1 request to get all related users, + * 1 request to get the agenda item, + * 1 request to get the polls, + * 1 request to get the tags, + + * lots of permissions requests. + + TODO: The last requests are a bug. + """ + with self.assertNumQueries(57): + self.client.get(reverse('assignment-list')) + + class CanidatureSelf(TestCase): """ Tests self candidation view. diff --git a/tests/integration/core/test_viewset.py b/tests/integration/core/test_viewset.py new file mode 100644 index 000000000..402c7fc61 --- /dev/null +++ b/tests/integration/core/test_viewset.py @@ -0,0 +1,145 @@ +from django.core.urlresolvers import reverse +from rest_framework.test import APIClient + +from openslides.core.config import config +from openslides.core.models import ChatMessage, Projector, Tag +from openslides.users.models import User +from openslides.utils.test import TestCase + + +class TestProjectorDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + Projector.objects.create() + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions, + * 2 requests to get the list of all projectors, + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(7): + self.client.get(reverse('projector-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions) + * 2 requests to get the list of all projectors, + + * 11 requests for permissions. + + TODO: The last 11 requests are a bug. + """ + with self.assertNumQueries(15): + self.client.get(reverse('projector-list')) + + +class TestCharmessageDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + user = User.objects.get(pk=1) + for index in range(10): + ChatMessage.objects.create(user=user) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions, + * 2 requests to get the list of all chatmessages, + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(7): + self.client.get(reverse('chatmessage-list')) + + +class TestTagDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + Tag.objects.create(name='tag{}'.format(index)) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the session an the request user with its permissions, + * 2 requests to get the list of all tags, + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(4): + self.client.get(reverse('tag-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions) + * 2 requests to get the list of all projectors, + + * 10 requests for to config + + The last 10 requests are a bug. + """ + with self.assertNumQueries(14): + self.client.get(reverse('tag-list')) + + +class TestConfigDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the session an the request user with its permissions and + * 1 requests to get the list of all config values + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(3): + self.client.get(reverse('config-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions), + * 1 to get all config value and + + * 55 requests to find out if anonymous is enabled. + + TODO: The last 55 requests are a bug. + """ + with self.assertNumQueries(58): + self.client.get(reverse('config-list')) diff --git a/tests/integration/mediafiles/__init__.py b/tests/integration/mediafiles/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/mediafiles/test_viewset.py b/tests/integration/mediafiles/test_viewset.py new file mode 100644 index 000000000..9048f34e7 --- /dev/null +++ b/tests/integration/mediafiles/test_viewset.py @@ -0,0 +1,46 @@ +from django.core.files.uploadedfile import SimpleUploadedFile +from django.core.urlresolvers import reverse +from rest_framework.test import APIClient + +from openslides.core.config import config +from openslides.mediafiles.models import Mediafile +from openslides.users.models import User +from openslides.utils.test import TestCase + + +class TestDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + Mediafile.objects.create( + title='some_file{}'.format(index), + mediafile=SimpleUploadedFile( + 'some_file{}'.format(index), + b'some content.')) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions and + * 2 requests to get the list of all files. + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(7): + self.client.get(reverse('mediafile-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions) and + * 2 requests to get the list of all projectors. + """ + with self.assertNumQueries(4): + self.client.get(reverse('mediafile-list')) diff --git a/tests/integration/motions/test_viewset.py b/tests/integration/motions/test_viewset.py index 2c3b72ab9..d60c8dd92 100644 --- a/tests/integration/motions/test_viewset.py +++ b/tests/integration/motions/test_viewset.py @@ -8,9 +8,139 @@ from rest_framework.test import APIClient from openslides.core.config import config from openslides.core.models import Tag from openslides.motions.models import Category, Motion, State +from openslides.users.models import User from openslides.utils.test import TestCase +class TestMotionDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + Motion.objects.create(title='motion{}'.format(index)) + # TODO: Create some polls etc. + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions, + * 2 requests to get the list of all motions, + * 1 request to get the motion versions, + * 1 request to get the agenda item, + * 1 request to get the motion log, + * 1 request to get the polls, + * 1 request to get the attachments, + * 1 request to get the tags, + * 2 requests to get the submitters and supporters + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(15): + self.client.get(reverse('motion-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions) + * 2 requests to get the list of all motions, + * 1 request to get the motion versions, + * 1 request to get the agenda item, + * 1 request to get the motion log, + * 1 request to get the polls, + * 1 request to get the attachments, + * 1 request to get the tags, + * 2 requests to get the submitters and supporters + + * 10 requests for permissions. + + TODO: The last 10 requests are a bug. + """ + with self.assertNumQueries(22): + self.client.get(reverse('motion-list')) + + +class TestCategoryDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + Category.objects.create(name='category{}'.format(index)) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions and + * 2 requests to get the list of all categories. + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(7): + self.client.get(reverse('category-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions) + * 2 requests to get the list of all motions and + + * 10 requests for permissions. + + TODO: The last 10 requests are a bug. + """ + with self.assertNumQueries(14): + self.client.get(reverse('category-list')) + + +class TestWorkflowDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + # There do not need to be more workflows + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions, + * 2 requests to get the list of all workflows, + * 1 request to get all states and + * 1 request to get the next states of all states. + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(9): + self.client.get(reverse('workflow-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions), + * 2 requests to get the list of all workflows, + * 1 request to get all states and + * 1 request to get the next states of all states. + + * 2 requests for permissions. + + TODO: The last 2 requests are a bug. + """ + with self.assertNumQueries(8): + self.client.get(reverse('workflow-list')) + + class CreateMotion(TestCase): """ Tests motion creation. diff --git a/tests/integration/users/test_viewset.py b/tests/integration/users/test_viewset.py index 6d3692d26..c53679074 100644 --- a/tests/integration/users/test_viewset.py +++ b/tests/integration/users/test_viewset.py @@ -8,6 +8,86 @@ from openslides.users.serializers import UserFullSerializer from openslides.utils.test import TestCase +class TestUserDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + User.objects.create(username='user{}'.format(index)) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions, + * 2 requests to get the list of all assignments and + * 1 request to get all groups. + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(8): + self.client.get(reverse('user-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions) + * 2 requests to get the list of all users, + * 1 request to get all groups and + + * lots of permissions requests. + + TODO: The last requests are a bug. + """ + with self.assertNumQueries(27): + self.client.get(reverse('user-list')) + + +class TestGroupDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + Group.objects.create(name='group{}'.format(index)) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the session an the request user with its permissions, + * 1 request to get the list of all groups, + * 1 request to get the permissions and + * 1 request to get the content_object for the permissions. + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(5): + self.client.get(reverse('group-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to find out if anonymous is enabled + * 1 request to get the list of all groups, + * 1 request to get the permissions and + * 1 request to get the content_object for the permissions. + + TODO: There should be only one request to find out if anonymous is enabled. + """ + with self.assertNumQueries(5): + self.client.get(reverse('group-list')) + + class UserGetTest(TestCase): """ Tests to receive a users via REST API. diff --git a/tests/integration/utils/autoupdate.py b/tests/integration/utils/test_autoupdate.py similarity index 65% rename from tests/integration/utils/autoupdate.py rename to tests/integration/utils/test_autoupdate.py index 1aa0cef5e..4644db167 100644 --- a/tests/integration/utils/autoupdate.py +++ b/tests/integration/utils/test_autoupdate.py @@ -24,13 +24,22 @@ class TestGetLoggedInUsers(TestCase): User.objects.create(username='user3') # Create a session with a user, that expires in 5 hours - Session.objects.create(user=user1, expire_data=timezone.now() + timedelta(hours=5)) + Session.objects.create( + user=user1, + expire_date=timezone.now() + timedelta(hours=5), + session_key='1') # Create a session with a user, that is expired before 5 hours - Session.objects.create(user=user2, expire_data=timezone.now() + timedelta(hours=-5)) + Session.objects.create( + user=user2, + expire_date=timezone.now() + timedelta(hours=-5), + session_key='2') # Create a session with an anonymous user, that expires in 5 hours - Session.objects.create(user=None, expire_data=timezone.now() + timedelta(hours=5)) + Session.objects.create( + user=None, + expire_date=timezone.now() + timedelta(hours=5), + session_key='3') self.assertEqual(list(get_logged_in_users()), [user1]) @@ -40,7 +49,13 @@ class TestGetLoggedInUsers(TestCase): The user should be returned only once. """ user1 = User.objects.create(username='user1') - Session.objects.create(user=user1, expire_data=timezone.now() + timedelta(hours=1)) - Session.objects.create(user=user1, expire_data=timezone.now() + timedelta(hours=2)) + Session.objects.create( + user=user1, + expire_date=timezone.now() + timedelta(hours=1), + session_key='1') + Session.objects.create( + user=user1, + expire_date=timezone.now() + timedelta(hours=2), + session_key='2') self.assertEqual(list(get_logged_in_users()), [user1]) diff --git a/tests/integration/utils/test_collection.py b/tests/integration/utils/test_collection.py new file mode 100644 index 000000000..f5cf7c0ed --- /dev/null +++ b/tests/integration/utils/test_collection.py @@ -0,0 +1,134 @@ +from unittest.mock import patch + +from channels.tests import ChannelTestCase +from django.core.cache import caches + +from openslides.topics.models import Topic +from openslides.utils import collection + + +class TestCase(ChannelTestCase): + """ + Testcase that uses the local mem cache and clears the cache after each test. + """ + def setUp(self): + cache = caches['locmem'] + cache.clear() + self.patch = patch('openslides.utils.collection.cache', cache) + self.patch.start() + + def tearDown(self): + self.patch.stop() + + +class TestCollectionElementCache(TestCase): + def test_clean_cache(self): + """ + Tests that the data is retrieved from the database. + """ + topic = Topic.objects.create(title='test topic') + collection_element = collection.CollectionElement.from_values('topics/topic', 1) + caches['locmem'].clear() + + with self.assertNumQueries(3): + instance = collection_element.get_full_data() + self.assertEqual(topic.title, instance['title']) + + def test_with_cache(self): + """ + Tests that no db query is used when the valie is in the cache. + + The value is added to the test when .create(...) is called. This hits + the autoupdate system, which fills the cache. + """ + topic = Topic.objects.create(title='test topic') + collection_element = collection.CollectionElement.from_values('topics/topic', 1) + + with self.assertNumQueries(0): + collection_element = collection.CollectionElement.from_values('topics/topic', 1) + instance = collection_element.get_full_data() + self.assertEqual(topic.title, instance['title']) + + def test_non_existing_instance(self): + collection_element = collection.CollectionElement.from_values('topics/topic', 1) + + with self.assertRaises(Topic.DoesNotExist): + collection_element.get_full_data() + + +class TestCollectionCache(TestCase): + def test_clean_cache(self): + """ + Tests that the instances are retrieved from the database. + + Currently there are 10 queries needed. This can change in the future, + but it has to be more then zero. + """ + Topic.objects.create(title='test topic1') + Topic.objects.create(title='test topic2') + Topic.objects.create(title='test topic3') + topic_collection = collection.Collection('topics/topic') + caches['locmem'].clear() + + with self.assertNumQueries(4): + instance_list = list(topic_collection.as_autoupdate_for_projector()) + self.assertEqual(len(instance_list), 3) + + def test_with_cache(self): + """ + Tests that no db query is used when the list is received twice. + """ + Topic.objects.create(title='test topic1') + Topic.objects.create(title='test topic2') + Topic.objects.create(title='test topic3') + topic_collection = collection.Collection('topics/topic') + list(topic_collection.as_autoupdate_for_projector()) + + with self.assertNumQueries(0): + instance_list = list(topic_collection.as_autoupdate_for_projector()) + self.assertEqual(len(instance_list), 3) + + def test_with_some_objects_in_the_cache(self): + """ + One element (topic3) is in the cache and two are not. + """ + Topic.objects.create(title='test topic1') + Topic.objects.create(title='test topic2') + caches['locmem'].clear() + Topic.objects.create(title='test topic3') + topic_collection = collection.Collection('topics/topic') + + with self.assertNumQueries(4): + instance_list = list(topic_collection.as_autoupdate_for_projector()) + self.assertEqual(len(instance_list), 3) + + def test_deletion(self): + """ + When an element is deleted, the cache should be updated automaticly via + the autoupdate system. So there should be no db queries. + """ + Topic.objects.create(title='test topic1') + Topic.objects.create(title='test topic2') + topic3 = Topic.objects.create(title='test topic3') + topic_collection = collection.Collection('topics/topic') + list(topic_collection.as_autoupdate_for_projector()) + + topic3.delete() + + with self.assertNumQueries(0): + instance_list = list(topic_collection.as_autoupdate_for_projector()) + self.assertEqual(len(instance_list), 2) + + def test_config_elements_without_cache(self): + topic_collection = collection.Collection('core/config') + caches['locmem'].clear() + + with self.assertNumQueries(1): + list(topic_collection.as_autoupdate_for_projector()) + + def test_config_elements_with_cache(self): + topic_collection = collection.Collection('core/config') + list(topic_collection.as_autoupdate_for_projector()) + + with self.assertNumQueries(0): + list(topic_collection.as_autoupdate_for_projector()) diff --git a/tests/settings.py b/tests/settings.py index 0db70f417..e3483c5cc 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -75,3 +75,14 @@ TEST_RUNNER = 'openslides.utils.test.OpenSlidesDiscoverRunner' PASSWORD_HASHERS = [ 'django.contrib.auth.hashers.MD5PasswordHasher', ] + + +# Use the dummy cache that does not cache anything +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.dummy.DummyCache' + }, + 'locmem': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache' + } +} diff --git a/tests/unit/utils/test_collection.py b/tests/unit/utils/test_collection.py new file mode 100644 index 000000000..83b1c2555 --- /dev/null +++ b/tests/unit/utils/test_collection.py @@ -0,0 +1,236 @@ +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from openslides.core.models import Projector +from openslides.utils import collection + + +class TestCacheKeys(TestCase): + def test_get_collection_id_from_cache_key(self): + """ + Test that get_collection_id_from_cache_key works together with + get_single_element_cache_key. + """ + element = ('some/testkey', 42) + self.assertEqual( + element, + collection.get_collection_id_from_cache_key( + collection.get_single_element_cache_key(*element))) + + def test_get_collection_id_from_cache_key_for_strings(self): + """ + Test get_collection_id_from_cache_key for strings + """ + element = ('some/testkey', 'my_config_value') + self.assertEqual( + element, + collection.get_collection_id_from_cache_key( + collection.get_single_element_cache_key(*element))) + + def test_get_single_element_cache_key_prefix(self): + """ + Tests that the cache prefix is realy a prefix. + """ + element = ('some/testkey', 42) + + cache_key = collection.get_single_element_cache_key(*element) + prefix = collection.get_single_element_cache_key_prefix(element[0]) + + self.assertTrue(cache_key.startswith(prefix)) + + def test_prefix_different_then_list(self): + """ + Test that the return value of get_single_element_cache_key_prefix is + something different then get_element_list_cache_key. + """ + collection_string = 'some/testkey' + + prefix = collection.get_single_element_cache_key_prefix(collection_string) + list_cache_key = collection.get_element_list_cache_key(collection_string) + + self.assertNotEqual(prefix, list_cache_key) + + +class TestGetModelFromCollectionString(TestCase): + def test_known_app(self): + projector_model = collection.get_model_from_collection_string('core/projector') + + self.assertEqual(projector_model, Projector) + + def test_unknown_app(self): + with self.assertRaises(ValueError): + collection.get_model_from_collection_string('invalid/model') + + +class TestCollectionElement(TestCase): + def test_from_values(self): + collection_element = collection.CollectionElement.from_values('testmodule/model', 42) + + self.assertEqual(collection_element.collection_string, 'testmodule/model') + self.assertEqual(collection_element.id, 42) + + @patch('openslides.utils.collection.Collection') + @patch('openslides.utils.collection.cache') + def test_from_values_deleted(self, mock_cache, mock_collection): + """ + Tests that when createing a CollectionElement with deleted=True the element + is deleted from the cache. + """ + collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True) + + self.assertTrue(collection_element.is_deleted()) + mock_cache.delete.assert_called_with('testmodule/model:42') + mock_collection.assert_called_with('testmodule/model') + mock_collection().delete_id_from_cache.assert_called_with(42) + + def test_as_channel_message(self): + collection_element = collection.CollectionElement.from_values('testmodule/model', 42) + + self.assertEqual( + collection_element.as_channels_message(), + {'collection_string': 'testmodule/model', + 'id': 42, + 'deleted': False}) + + def test_as_autoupdate_for_user(self): + collection_element = collection.CollectionElement.from_values('testmodule/model', 42) + fake_user = MagicMock() + collection_element.get_access_permissions = MagicMock() + collection_element.get_access_permissions().get_restricted_data.return_value = 'restricted_data' + collection_element.get_full_data = MagicMock() + + self.assertEqual( + collection_element.as_autoupdate_for_user(fake_user), + {'collection': 'testmodule/model', + 'id': 42, + 'action': 'changed', + 'data': 'restricted_data'}) + collection_element.get_full_data.assert_called_once_with() + + def test_as_autoupdate_for_user_no_permission(self): + collection_element = collection.CollectionElement.from_values('testmodule/model', 42) + fake_user = MagicMock() + collection_element.get_access_permissions = MagicMock() + collection_element.get_access_permissions().get_restricted_data.return_value = None + collection_element.get_full_data = MagicMock() + + self.assertEqual( + collection_element.as_autoupdate_for_user(fake_user), + {'collection': 'testmodule/model', + 'id': 42, + 'action': 'deleted'}) + collection_element.get_full_data.assert_called_once_with() + + def test_as_autoupdate_for_user_deleted(self): + collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True) + fake_user = MagicMock() + + self.assertEqual( + collection_element.as_autoupdate_for_user(fake_user), + {'collection': 'testmodule/model', + 'id': 42, + 'action': 'deleted'}) + + def test_get_instance_deleted(self): + collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True) + + with self.assertRaises(RuntimeError): + collection_element.get_instance() + + @patch('openslides.core.config.config') + def test_get_instance_config_str(self, mock_config): + mock_config.get_collection_string.return_value = 'core/config' + mock_config.__getitem__.return_value = 'config_value' + collection_element = collection.CollectionElement.from_values('core/config', 'my_config_value') + + instance = collection_element.get_instance() + + self.assertEqual( + instance, + {'key': 'my_config_value', + 'value': 'config_value'}) + + def test_get_instance(self): + collection_element = collection.CollectionElement.from_values('testmodule/model', 42) + collection_element.get_model = MagicMock() + + collection_element.get_instance() + + collection_element.get_model().objects.get_full_queryset().get.assert_called_once_with(pk=42) + + @patch('openslides.utils.collection.cache') + def test_get_full_data_already_loaded(self, mock_cache): + """ + Test that the cache and the self.get_instance() is not hit, when the + instance is already loaded. + """ + collection_element = collection.CollectionElement.from_values('testmodule/model', 42) + collection_element.full_data = 'my_full_data' + collection_element.get_instance = MagicMock() + + collection_element.get_full_data() + + mock_cache.get.assert_not_called() + collection_element.get_instance.assert_not_called() + + @patch('openslides.utils.collection.cache') + def test_get_full_data_from_cache(self, mock_cache): + """ + Test that the value from the cache is used not get_instance is not + called. + """ + collection_element = collection.CollectionElement.from_values('testmodule/model', 42) + collection_element.get_instance = MagicMock() + mock_cache.get.return_value = 'cache_value' + + instance = collection_element.get_full_data() + + self.assertEqual(instance, 'cache_value') + mock_cache.get.assert_called_once_with('testmodule/model:42') + collection_element.get_instance.assert_not_called + + @patch('openslides.utils.collection.Collection') + @patch('openslides.utils.collection.cache') + def test_get_full_data_from_get_instance(self, mock_cache, mock_Collection): + """ + Test that the value from get_instance is used and saved to the cache + """ + collection_element = collection.CollectionElement.from_values('testmodule/model', 42) + collection_element.get_instance = MagicMock() + collection_element.get_access_permissions = MagicMock() + collection_element.get_access_permissions().get_full_data.return_value = 'get_instance_value' + mock_cache.get.return_value = None + + instance = collection_element.get_full_data() + + self.assertEqual(instance, 'get_instance_value') + mock_cache.get.assert_called_once_with('testmodule/model:42') + collection_element.get_instance.assert_called_once_with() + mock_cache.set.assert_called_once_with('testmodule/model:42', 'get_instance_value') + mock_Collection.assert_called_once_with('testmodule/model') + mock_Collection().add_id_to_cache.assert_called_once_with(42) + + +class TestCollection(TestCase): + @patch('openslides.utils.collection.CollectionElement') + @patch('openslides.utils.collection.cache') + def test_element_generator(self, mock_cache, mock_CollectionElement): + """ + Test with the following scenario: The collection has three elements. Two + are in the cache and one is not. + """ + test_collection = collection.Collection('testmodule/model') + test_collection.get_all_ids = MagicMock(return_value=set([1, 2, 3])) + test_collection.get_model = MagicMock() + test_collection.get_model().objects.get_full_queryset().filter.return_value = ['my_instance'] + mock_cache.get_many.return_value = { + 'testmodule/model:1': 'element1', + 'testmodule/model:2': 'element2'} + + list(test_collection.element_generator()) + + mock_cache.get_many.assert_called_once_with( + ['testmodule/model:1', 'testmodule/model:2', 'testmodule/model:3']) + test_collection.get_model().objects.get_full_queryset().filter.assert_called_once_with(pk__in={3}) + self.assertEqual(mock_CollectionElement.from_values.call_count, 2) + self.assertEqual(mock_CollectionElement.from_instance.call_count, 1)