diff --git a/.gitignore b/.gitignore index c49dc0900..335be371b 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ dist/* # Unit test and coverage reports .coverage +tests/file/* diff --git a/CHANGELOG b/CHANGELOG index 17146e2e6..3f14ccdc0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -42,6 +42,7 @@ Other: - Fixed bug, that the last change of a config value was not send via autoupdate. - Added template hooks for plugins (in item detail view and motion poll form). - Used Django Channels instead of Tornado. Refactoring of the autoupdate process. +- Added new caching system with support for Redis. Version 2.0 (2016-04-18) diff --git a/DEVELOPMENT.rst b/DEVELOPMENT.rst index 97628c1fe..047fb03ad 100644 --- a/DEVELOPMENT.rst +++ b/DEVELOPMENT.rst @@ -109,3 +109,31 @@ a. Running Angular.js test cases '''''''''''''''''''''''''''''''' $ node_modules/.bin/karma start tests/karma/karma.conf.js + + +Installation Openslides in big mode +=================================== + +1. Install PostgreSQL und redis: + +apt-get install postgresql redis-server libpg-dev + +TODO: Configure postgresql + +2. Install python dependencies + +pip install django-redis asgi-redis psycopg2 + +3. Change settings.py + +(See comments in the settings) + +The relevant settings are: DATABASES, CHANNEL_LAYERS, CACHES + +4. Start one or more workers: + +python manage.py runworker + +5. Start daphne. Set the DJANGO_SETTINGS_MODULE and the PYTHONPATH + +DJANGO_SETTINGS_MODULE=settings PYTHONPATH=personal_data/var/ daphne openslides.asgi:channel_layer diff --git a/openslides/agenda/models.py b/openslides/agenda/models.py index a97d9701a..12b921a67 100644 --- a/openslides/agenda/models.py +++ b/openslides/agenda/models.py @@ -23,6 +23,14 @@ class ItemManager(models.Manager): Customized model manager with special methods for agenda tree and numbering. """ + def get_full_queryset(self): + """ + Returns the normal queryset with all items. In the background all + speakers and related items (topics, motions, assignments) are + prefetched from the database. + """ + return self.get_queryset().prefetch_related('speakers', 'content_object') + def get_only_agenda_items(self): """ Generator, which yields only agenda items. Skips hidden items. @@ -276,20 +284,6 @@ class Item(RESTModelMixin, models.Model): def __str__(self): return self.title - def delete(self, with_children=False): - """ - Delete the Item. - - If with_children is True, all children of the item will be deleted as - well. If with_children is False, all children will be children of the - parent of the item. - """ - if not with_children: - for child in self.children.all(): - child.parent = self.parent - child.save() - super().delete() - @property def title(self): """ diff --git a/openslides/agenda/projector.py b/openslides/agenda/projector.py index c93e70124..e8ce31f5d 100644 --- a/openslides/agenda/projector.py +++ b/openslides/agenda/projector.py @@ -1,5 +1,6 @@ from ..core.config import config from ..core.exceptions import ProjectorException +from ..utils.collection import CollectionElement from ..utils.projector import ProjectorElement from .models import Item @@ -60,10 +61,13 @@ class ListOfSpeakersSlide(ProjectorElement): # Yield last speakers yield speaker.user - def need_full_update_for_this(self, collection_element): - # Full update if item changes because then we may have new speakers - # and therefor need new users. - return collection_element.collection_string == Item.get_collection_string() + def get_collection_elements_required_for_this(self, collection_element, config_entry): + output = super().get_collection_elements_required_for_this(collection_element, config_entry) + # Full update if item changes because then we may have new + # candidates and therefor need new users. + if collection_element == CollectionElement.from_values(Item.get_collection_string(), config_entry.get('id')): + output.extend(self.get_requirements_as_collection_elements(config_entry)) + return output class CurrentListOfSpeakersSlide(ProjectorElement): diff --git a/openslides/agenda/views.py b/openslides/agenda/views.py index d4c91d0c2..bfe066c75 100644 --- a/openslides/agenda/views.py +++ b/openslides/agenda/views.py @@ -58,26 +58,6 @@ class ItemViewSet(ListModelMixin, RetrieveModelMixin, UpdateModelMixin, GenericV result = False return result - def check_object_permissions(self, request, obj): - """ - Checks if the requesting user has permission to see also an - organizational item if it is one. - """ - # TODO: Move this logic to access_permissions.ItemAccessPermissions. - if obj.is_hidden() and not request.user.has_perm('agenda.can_see_hidden_items'): - self.permission_denied(request) - - def get_queryset(self): - """ - Filters organizational items if the user has no permission to see them. - """ - # TODO: Move this logic to access_permissions.ItemAccessPermissions. - queryset = super().get_queryset() - if not self.request.user.has_perm('agenda.can_see_hidden_items'): - pk_list = [item.pk for item in Item.objects.get_only_agenda_items()] - queryset = queryset.filter(pk__in=pk_list) - return queryset - @detail_route(methods=['POST', 'DELETE']) def manage_speaker(self, request, pk=None): """ @@ -243,31 +223,6 @@ class ItemViewSet(ListModelMixin, RetrieveModelMixin, UpdateModelMixin, GenericV # Initiate response. return Response({'detail': _('List of speakers successfully sorted.')}) - @list_route(methods=['get', 'put']) - def tree(self, request): - """ - Returns or sets the agenda tree. - """ - if request.method == 'PUT': - if not (request.user.has_perm('agenda.can_manage') and - request.user.has_perm('agenda.can_see_hidden_items')): - self.permission_denied(request) - try: - tree = request.data['tree'] - except KeyError as error: - response = Response({'detail': 'Agenda tree is missing.'}, status=400) - else: - try: - Item.objects.set_tree(tree) - except ValueError as error: - response = Response({'detail': str(error)}, status=400) - else: - response = Response({'detail': 'Agenda tree successfully updated.'}) - else: - # request.method == 'GET' - response = Response(Item.objects.get_tree()) - return response - @list_route(methods=['post']) def numbering(self, request): """ diff --git a/openslides/assignments/models.py b/openslides/assignments/models.py index b997c33b8..40fe85a00 100644 --- a/openslides/assignments/models.py +++ b/openslides/assignments/models.py @@ -1,7 +1,7 @@ from collections import OrderedDict from django.conf import settings -from django.contrib.contenttypes.models import ContentType +from django.contrib.contenttypes.fields import GenericRelation from django.db import models from django.utils.translation import ugettext as _ from django.utils.translation import ugettext_noop @@ -16,6 +16,7 @@ from openslides.poll.models import ( CollectDefaultVotesMixin, PublishPollMixin, ) +from openslides.utils.autoupdate import inform_changed_data from openslides.utils.exceptions import OpenSlidesError from openslides.utils.models import RESTModelMixin from openslides.utils.search import user_name_helper @@ -50,12 +51,30 @@ class AssignmentRelatedUser(RESTModelMixin, models.Model): return self.assignment +class AssignmentManager(models.Manager): + """ + Customized model manager to support our get_full_queryset method. + """ + def get_full_queryset(self): + """ + Returns the normal queryset with all assignments. In the background + all related users (candidates), the related agenda item and all + polls are prefetched from the database. + """ + return self.get_queryset().prefetch_related( + 'related_users', + 'agenda_items', + 'polls') + + class Assignment(RESTModelMixin, models.Model): """ Model for assignments. """ access_permissions = AssignmentAccessPermissions() + objects = AssignmentManager() + PHASE_SEARCH = 0 PHASE_VOTING = 1 PHASE_FINISHED = 2 @@ -111,6 +130,10 @@ class Assignment(RESTModelMixin, models.Model): Tags for the assignment. """ + # In theory there could be one then more agenda_item. But we support only + # one. See the property agenda_item. + agenda_items = GenericRelation(Item, related_name='assignments') + class Meta: default_permissions = () permissions = ( @@ -187,6 +210,7 @@ class Assignment(RESTModelMixin, models.Model): Delete the connection from the assignment to the user. """ self.assignment_related_users.filter(user=user).delete() + inform_changed_data(self) def set_phase(self, phase): """ @@ -290,8 +314,9 @@ class Assignment(RESTModelMixin, models.Model): """ Returns the related agenda item. """ - content_type = ContentType.objects.get_for_model(self) - return Item.objects.get(object_id=self.pk, content_type=content_type) + # We support only one agenda item so just return the first element of + # the queryset. + return self.agenda_items.all()[0] @property def agenda_item_id(self): diff --git a/openslides/assignments/projector.py b/openslides/assignments/projector.py index 6e61fb513..fac925372 100644 --- a/openslides/assignments/projector.py +++ b/openslides/assignments/projector.py @@ -1,4 +1,5 @@ from ..core.exceptions import ProjectorException +from ..utils.collection import CollectionElement from ..utils.projector import ProjectorElement from .models import Assignment, AssignmentPoll @@ -46,7 +47,10 @@ class AssignmentSlide(ProjectorElement): for option in poll.options.all(): yield option.candidate - def need_full_update_for_this(self, collection_element): + def get_collection_elements_required_for_this(self, collection_element, config_entry): + output = super().get_collection_elements_required_for_this(collection_element, config_entry) # Full update if assignment changes because then we may have new # candidates and therefor need new users. - return collection_element.collection_string == Assignment.get_collection_string() + if collection_element == CollectionElement.from_values(Assignment.get_collection_string(), config_entry.get('id')): + output.extend(self.get_requirements_as_collection_elements(config_entry)) + return output diff --git a/openslides/core/access_permissions.py b/openslides/core/access_permissions.py index e2b553661..0ffb49cba 100644 --- a/openslides/core/access_permissions.py +++ b/openslides/core/access_permissions.py @@ -84,7 +84,13 @@ class ConfigAccessPermissions(BaseAccessPermissions): Returns the serlialized config data. """ from .config import config + from .models import ConfigStore # Attention: The format of this response has to be the same as in # the retrieve method of ConfigViewSet. - return {'key': instance.key, 'value': config[instance.key]} + if isinstance(instance, ConfigStore): + result = {'key': instance.key, 'value': config[instance.key]} + else: + # It is possible, that the caching system already sends the correct data as "instance". + result = instance + return result diff --git a/openslides/core/apps.py b/openslides/core/apps.py index 81d25365c..c0f68c1f5 100644 --- a/openslides/core/apps.py +++ b/openslides/core/apps.py @@ -16,7 +16,6 @@ class CoreAppConfig(AppConfig): from django.db.models import signals from openslides.core.config import config from openslides.core.signals import post_permission_creation - from openslides.utils.autoupdate import inform_changed_data_receiver, inform_deleted_data_receiver from openslides.utils.rest_api import router from openslides.utils.search import index_add_instance, index_del_instance from .config_variables import get_config_variables @@ -45,15 +44,6 @@ class CoreAppConfig(AppConfig): router.register(self.get_model('Tag').get_collection_string(), TagViewSet) router.register(self.get_model('ConfigStore').get_collection_string(), ConfigViewSet, 'config') - # Update data when any model of any installed app is saved or deleted. - # TODO: Test if the m2m_changed signal is also needed. - signals.post_save.connect( - inform_changed_data_receiver, - dispatch_uid='inform_changed_data_receiver') - signals.post_delete.connect( - inform_deleted_data_receiver, - dispatch_uid='inform_deleted_data_receiver') - # Update the search when a model is saved or deleted signals.post_save.connect( index_add_instance, diff --git a/openslides/core/migrations/0005_auto_20160918_2104.py b/openslides/core/migrations/0005_auto_20160918_2104.py index c35ab6b41..377e106df 100644 --- a/openslides/core/migrations/0005_auto_20160918_2104.py +++ b/openslides/core/migrations/0005_auto_20160918_2104.py @@ -2,19 +2,10 @@ # Generated by Django 1.10.1 on 2016-09-18 19:04 from __future__ import unicode_literals -from django.db import migrations, models - -from openslides.utils.autoupdate import ( - inform_changed_data_receiver, - inform_deleted_data_receiver, -) +from django.db import migrations def move_custom_slides_to_topics(apps, schema_editor): - # Disconnect autoupdate. We do not want to trigger it here. - models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver') - models.signals.post_save.disconnect(dispatch_uid='inform_deleted_data_receiver') - # We get the model from the versioned app registry; # if we directly import it, it will be the wrong version. ContentType = apps.get_model('contenttypes', 'ContentType') @@ -39,14 +30,6 @@ def move_custom_slides_to_topics(apps, schema_editor): CustomSlide.objects.all().delete() content_type_custom_slide.delete() - # Reconnect autoupdate. - models.signals.post_save.connect( - inform_changed_data_receiver, - dispatch_uid='inform_changed_data_receiver') - models.signals.post_delete.connect( - inform_deleted_data_receiver, - dispatch_uid='inform_deleted_data_receiver') - class Migration(migrations.Migration): diff --git a/openslides/core/models.py b/openslides/core/models.py index 953496b42..bdb6d0c40 100644 --- a/openslides/core/models.py +++ b/openslides/core/models.py @@ -15,6 +15,19 @@ from .access_permissions import ( from .exceptions import ProjectorException +class ProjectorManager(models.Manager): + """ + Customized model manager to support our get_full_queryset method. + """ + def get_full_queryset(self): + """ + Returns the normal queryset with all projectors. In the background + projector defaults are prefetched from the database. + """ + return self.get_queryset().prefetch_related( + 'projectiondefaults') + + class Projector(RESTModelMixin, models.Model): """ Model for all projectors. At the moment we support only one projector, @@ -57,6 +70,8 @@ class Projector(RESTModelMixin, models.Model): """ access_permissions = ProjectorAccessPermissions() + objects = ProjectorManager() + config = JSONField() scale = models.IntegerField(default=0) @@ -131,50 +146,35 @@ class Projector(RESTModelMixin, models.Model): if element is not None: yield from element.get_requirements(value) - def collection_element_is_shown(self, collection_element): + def get_collection_elements_required_for_this(self, collection_element): """ - Returns True if this collection element is shown on this projector. + Returns an iterable of CollectionElements that have to be sent to this + projector according to the given collection_element and information. """ - for requirement in self.get_all_requirements(): - if (requirement.get_collection_string() == collection_element.collection_string and - requirement.pk == collection_element.id): - result = True - break + from .config import config + + output = [] + changed_fields = collection_element.information.get('changed_fields', []) + if (collection_element.collection_string == self.get_collection_string() and + changed_fields and + 'config' not in changed_fields): + # Projector model changed without changeing the projector config. So we just send this data. + output.append(collection_element) else: - result = False - return result - - @classmethod - def get_projectors_that_show_this(cls, collection_element): - """ - Returns a list of the projectors that show this collection element. - """ - result = [] - for projector in cls.objects.all(): - if projector.collection_element_is_shown(collection_element): - result.append(projector) - return result - - def need_full_update_for_this(self, collection_element): - """ - Returns True if this projector needs to be updated with all - instances as defined in get_all_requirements() because one active - projector element requires this. - """ - # Get all elements from all apps. - elements = {} - for element in ProjectorElement.get_all(): - elements[element.name] = element - - for key, value in self.config.items(): - element = elements.get(value['name']) - if element is not None and element.need_full_update_for_this(collection_element): - result = True - break - else: - result = False - - return result + # It is necessary to parse all active projector elements to check whether they require some data. + this_projector = collection_element.collection_string == self.get_collection_string() and collection_element.id == self.pk + collection_element.information['this_projector'] = this_projector + elements = {} + for element in ProjectorElement.get_all(): + elements[element.name] = element + for key, value in self.config.items(): + element = elements.get(value['name']) + if element is not None: + output.extend(element.get_collection_elements_required_for_this(collection_element, value)) + # If config changed, send also this to the projector. + if collection_element.collection_string == config.get_collection_string(): + output.append(collection_element) + return output class ProjectionDefault(RESTModelMixin, models.Model): diff --git a/openslides/core/signals.py b/openslides/core/signals.py index 92609c323..f2667ca21 100644 --- a/openslides/core/signals.py +++ b/openslides/core/signals.py @@ -20,8 +20,7 @@ def delete_django_app_permissions(sender, **kwargs): Q(app_label='auth') | Q(app_label='contenttypes') | Q(app_label='sessions')) - for permission in Permission.objects.filter(content_type__in=contenttypes): - permission.delete() + Permission.objects.filter(content_type__in=contenttypes).delete() def create_builtin_projection_defaults(**kwargs): diff --git a/openslides/core/views.py b/openslides/core/views.py index e015f90dd..9982c5652 100644 --- a/openslides/core/views.py +++ b/openslides/core/views.py @@ -18,6 +18,7 @@ from django.utils.timezone import now from openslides import __version__ as version from openslides.utils import views as utils_views +from openslides.utils.collection import Collection, CollectionElement from openslides.utils.plugins import ( get_plugin_description, get_plugin_verbose_name, @@ -41,7 +42,7 @@ from .access_permissions import ( ) from .config import config from .exceptions import ConfigError, ConfigNotFound -from .models import ChatMessage, ProjectionDefault, Projector, Tag +from .models import ChatMessage, ConfigStore, ProjectionDefault, Projector, Tag # Special Django views @@ -606,22 +607,26 @@ class ConfigViewSet(ViewSet): def list(self, request): """ - Lists all config variables. Everybody can see them. + Lists all config variables. """ - return Response([{'key': key, 'value': value} for key, value in config.items()]) + collection = Collection(config.get_collection_string()) + return Response(collection.as_list_for_user(request.user)) def retrieve(self, request, *args, **kwargs): """ - Retrieves a config variable. Everybody can see it. + Retrieves a config variable. """ key = kwargs['pk'] + collection_element = CollectionElement.from_values(config.get_collection_string(), key) try: - value = config[key] - except ConfigNotFound: + content = collection_element.as_dict_for_user(request.user) + except ConfigStore.DoesNotExist: raise Http404 - # Attention: The format of this response has to be the same as in - # the get_full_data method of ConfigAccessPermissions. - return Response({'key': key, 'value': value}) + if content is None: + # If content is None, the user has no permissions to see the item. + # See ConfigAccessPermissions or rather its parent class. + self.permission_denied() + return Response(content) def update(self, request, *args, **kwargs): """ diff --git a/openslides/mediafiles/views.py b/openslides/mediafiles/views.py index 91ff41ace..92170899f 100644 --- a/openslides/mediafiles/views.py +++ b/openslides/mediafiles/views.py @@ -15,13 +15,6 @@ class MediafileViewSet(ModelViewSet): access_permissions = MediafileAccessPermissions() queryset = Mediafile.objects.all() - def get_queryset(self): - queryset = super().get_queryset() - user = self.request.user - if not user.has_perm('mediafiles.can_see_private'): - queryset = queryset.filter(private=False) - return queryset - def check_view_permissions(self): """ Returns True if the user has required permissions. diff --git a/openslides/motions/migrations/0004_auto_20160907_2343.py b/openslides/motions/migrations/0004_auto_20160907_2343.py index 34b3269bb..88dde5d08 100644 --- a/openslides/motions/migrations/0004_auto_20160907_2343.py +++ b/openslides/motions/migrations/0004_auto_20160907_2343.py @@ -5,16 +5,11 @@ from __future__ import unicode_literals import django.db.models.deletion from django.db import migrations, models -from openslides.utils.autoupdate import inform_changed_data_receiver - def change_label_of_state(apps, schema_editor): """ Changes the label of former state "commited a bill" to "refered to committee". """ - # Disconnect autoupdate. We do not want to trigger it here. - models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver') - # We get the model from the versioned app registry; # if we directly import it, it will be the wrong version. State = apps.get_model('motions', 'State') @@ -29,19 +24,11 @@ def change_label_of_state(apps, schema_editor): state.action_word = 'Refer to committee' state.save() - # Reconnect autoupdate. - models.signals.post_save.connect( - inform_changed_data_receiver, - dispatch_uid='inform_changed_data_receiver') - def add_recommendation_labels(apps, schema_editor): """ Adds recommendation labels to some of the built-in states. """ - # Disconnect autoupdate. We do not want to trigger it here. - models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver') - # We get the model from the versioned app registry; # if we directly import it, it will be the wrong version. State = apps.get_model('motions', 'State') diff --git a/openslides/motions/models.py b/openslides/motions/models.py index f46163b4e..dae32ca75 100644 --- a/openslides/motions/models.py +++ b/openslides/motions/models.py @@ -1,5 +1,5 @@ from django.conf import settings -from django.contrib.contenttypes.models import ContentType +from django.contrib.contenttypes.fields import GenericRelation from django.db import models from django.db.models import Max from django.utils import formats @@ -28,14 +28,38 @@ from .access_permissions import ( from .exceptions import WorkflowError +class MotionManager(models.Manager): + """ + Customized model manager to support our get_full_queryset method. + """ + def get_full_queryset(self): + """ + Returns the normal queryset with all motions. In the background we + join and prefetch all related models. + """ + return (self.get_queryset() + .select_related('active_version') + .prefetch_related( + 'versions', + 'agenda_items', + 'log_messages', + 'polls', + 'attachments', + 'tags', + 'submitters', + 'supporters')) + + class Motion(RESTModelMixin, models.Model): """ - The Motion Class. + Model for motions. This class is the main entry point to all other classes related to a motion. """ access_permissions = MotionAccessPermissions() + objects = MotionManager() + active_version = models.ForeignKey( 'MotionVersion', on_delete=models.SET_NULL, @@ -134,6 +158,10 @@ class Motion(RESTModelMixin, models.Model): Configurable fields for comments. Contains a list of strings. """ + # In theory there could be one then more agenda_item. But we support only + # one. See the property agenda_item. + agenda_items = GenericRelation(Item, related_name='motions') + class Meta: default_permissions = () permissions = ( @@ -519,8 +547,9 @@ class Motion(RESTModelMixin, models.Model): """ Returns the related agenda item. """ - content_type = ContentType.objects.get_for_model(self) - return Item.objects.get(object_id=self.pk, content_type=content_type) + # We support only one agenda item so just return the first element of + # the queryset. + return self.agenda_items.all()[0] @property def agenda_item_id(self): @@ -940,12 +969,29 @@ class State(RESTModelMixin, models.Model): return self.workflow +class WorkflowManager(models.Manager): + """ + Customized model manager to support our get_full_queryset method. + """ + def get_full_queryset(self): + """ + Returns the normal queryset with all workflows. In the background + the first state is joined and all states and next states are + prefetched from the database. + """ + return (self.get_queryset() + .select_related('first_state') + .prefetch_related('states', 'states__next_states')) + + class Workflow(RESTModelMixin, models.Model): """ Defines a workflow for a motion. """ access_permissions = WorkflowAccessPermissions() + objects = WorkflowManager() + name = models.CharField(max_length=255) """A string representing the workflow.""" diff --git a/openslides/motions/projector.py b/openslides/motions/projector.py index efee87177..07e006bfa 100644 --- a/openslides/motions/projector.py +++ b/openslides/motions/projector.py @@ -1,4 +1,5 @@ from ..core.exceptions import ProjectorException +from ..utils.collection import CollectionElement from ..utils.projector import ProjectorElement from .models import Motion @@ -26,10 +27,13 @@ class MotionSlide(ProjectorElement): yield from motion.submitters.all() yield from motion.supporters.all() - def need_full_update_for_this(self, collection_element): + def get_collection_elements_required_for_this(self, collection_element, config_entry): + output = super().get_collection_elements_required_for_this(collection_element, config_entry) # Full update if motion changes because then we may have new # submitters or supporters and therefor need new users. # # Add some logic here if we support live changing of workflows later. # - return collection_element.collection_string == Motion.get_collection_string() + if collection_element == CollectionElement.from_values(Motion.get_collection_string(), config_entry.get('id')): + output.extend(self.get_requirements_as_collection_elements(config_entry)) + return output diff --git a/openslides/motions/views.py b/openslides/motions/views.py index 7678ee99d..ff5e913eb 100644 --- a/openslides/motions/views.py +++ b/openslides/motions/views.py @@ -77,27 +77,6 @@ class MotionViewSet(ModelViewSet): result = False return result - def list(self, request, *args, **kwargs): - """ - Customized view endpoint to list all motions. - - Hides non public comment fields for some users. - """ - response = super().list(request, *args, **kwargs) - for i, motion in enumerate(response.data): - response.data[i] = self.get_access_permissions().get_restricted_data(motion, self.request.user) - return response - - def retrieve(self, request, *args, **kwargs): - """ - Customized view endpoint to retrieve a motion. - - Hides non public comment fields for some users. - """ - response = super().retrieve(request, *args, **kwargs) - response.data = self.get_access_permissions().get_restricted_data(response.data, self.request.user) - return response - def create(self, request, *args, **kwargs): """ Customized view endpoint to create a new motion. diff --git a/openslides/topics/models.py b/openslides/topics/models.py index 25a3b967e..d60e162e7 100644 --- a/openslides/topics/models.py +++ b/openslides/topics/models.py @@ -1,4 +1,4 @@ -from django.contrib.contenttypes.models import ContentType +from django.contrib.contenttypes.fields import GenericRelation from django.db import models from ..agenda.models import Item @@ -7,16 +7,35 @@ from ..utils.models import RESTModelMixin from .access_permissions import TopicAccessPermissions +class TopicManager(models.Manager): + """ + Customized model manager to support our get_full_queryset method. + """ + def get_full_queryset(self): + """ + Returns the normal queryset with all topics. In the background all + attachments and the related agenda item are prefetched from the + database. + """ + return self.get_queryset().prefetch_related('attachments', 'agenda_items') + + class Topic(RESTModelMixin, models.Model): """ Model for slides with custom content. Used to be called custom slide. """ access_permissions = TopicAccessPermissions() + objects = TopicManager() + title = models.CharField(max_length=256) text = models.TextField(blank=True) attachments = models.ManyToManyField(Mediafile, blank=True) + # In theory there could be one then more agenda_item. But we support only + # one. See the property agenda_item. + agenda_items = GenericRelation(Item, related_name='topics') + class Meta: default_permissions = () @@ -28,8 +47,9 @@ class Topic(RESTModelMixin, models.Model): """ Returns the related agenda item. """ - content_type = ContentType.objects.get_for_model(self) - return Item.objects.get(object_id=self.pk, content_type=content_type) + # We support only one agenda item so just return the first element of + # the queryset. + return self.agenda_items.all()[0] @property def agenda_item_id(self): diff --git a/openslides/users/access_permissions.py b/openslides/users/access_permissions.py index 0be9d0291..eed1020b7 100644 --- a/openslides/users/access_permissions.py +++ b/openslides/users/access_permissions.py @@ -15,15 +15,9 @@ class UserAccessPermissions(BaseAccessPermissions): """ Returns different serializer classes with respect user's permissions. """ - from .serializers import UserCanSeeSerializer, UserCanSeeExtraSerializer, UserFullSerializer + from .serializers import UserFullSerializer - if (user is None or (user.has_perm('users.can_see_extra_data') and user.has_perm('users.can_manage'))): - serializer_class = UserFullSerializer - elif user.has_perm('users.can_see_extra_data'): - serializer_class = UserCanSeeExtraSerializer - else: - serializer_class = UserCanSeeSerializer - return serializer_class + return UserFullSerializer def get_restricted_data(self, full_data, user): """ diff --git a/openslides/users/migrations/0004_groups.py b/openslides/users/migrations/0004_groups.py index 66ff1bb5a..1adc3428c 100644 --- a/openslides/users/migrations/0004_groups.py +++ b/openslides/users/migrations/0004_groups.py @@ -2,9 +2,7 @@ # Generated by Django 1.9.7 on 2016-08-01 14:54 from __future__ import unicode_literals -from django.db import migrations, models - -from openslides.utils.autoupdate import inform_changed_data_receiver +from django.db import migrations def migrate_groups_and_user_permissions(apps, schema_editor): @@ -20,9 +18,6 @@ def migrate_groups_and_user_permissions(apps, schema_editor): - the name of the first group is 'Guests', - the name of the second group is 'Registered users'. """ - # Disconnect autoupdate. We do not want to trigger it here. - models.signals.post_save.disconnect(dispatch_uid='inform_changed_data_receiver') - User = apps.get_model('users', 'User') Group = apps.get_model('auth', 'Group') @@ -52,11 +47,6 @@ def migrate_groups_and_user_permissions(apps, schema_editor): for permission in group_registered.permissions.all(): group.permissions.add(permission) - # Reconnect autoupdate. - models.signals.post_save.connect( - inform_changed_data_receiver, - dispatch_uid='inform_changed_data_receiver') - class Migration(migrations.Migration): diff --git a/openslides/users/models.py b/openslides/users/models.py index 2037f7af4..e7a8fcf03 100644 --- a/openslides/users/models.py +++ b/openslides/users/models.py @@ -21,8 +21,15 @@ from .access_permissions import UserAccessPermissions class UserManager(BaseUserManager): """ Customized manager that creates new users only with a password and a - username. + username. It also supports our get_full_queryset method. """ + def get_full_queryset(self): + """ + Returns the normal queryset with all users. In the background all + groups are prefetched from the database. + """ + return self.get_queryset().prefetch_related('groups') + def create_user(self, username, password, **kwargs): """ Creates a new user only with a password and a username. diff --git a/openslides/users/serializers.py b/openslides/users/serializers.py index 35ea95d52..fb5088543 100644 --- a/openslides/users/serializers.py +++ b/openslides/users/serializers.py @@ -12,79 +12,24 @@ from ..utils.rest_api import ( from .models import Group, User USERCANSEESERIALIZER_FIELDS = ( - 'id', - 'username', - 'title', - 'first_name', - 'last_name', - 'structure_level', - 'number', - 'about_me', - 'groups', - 'is_committee', - ) + 'id', + 'username', + 'title', + 'first_name', + 'last_name', + 'structure_level', + 'number', + 'about_me', + 'groups', + 'is_present', + 'is_committee', +) -class UserCanSeeSerializer(ModelSerializer): - """ - Serializer for users.models.User objects to be used by users who have - only the permission to see users and to change some date of theirselfs. - - Attention: Viewset has to ensure that a user can update only himself. - """ - class Meta: - model = User - fields = USERCANSEESERIALIZER_FIELDS - read_only_fields = ( - 'number', - 'groups', - 'is_comittee', - ) - - -USERCANSEEEXTRASERIALIZER_FIELDS = ( - 'id', - 'is_present', - 'username', - 'title', - 'first_name', - 'last_name', - 'number', - 'structure_level', - 'about_me', - 'comment', - 'groups', - 'is_active', - 'is_committee', - ) - - -class UserCanSeeExtraSerializer(ModelSerializer): - """ - Serializer for users.models.User objects to be used by users who have - the permission to see users with extra data and to change some date of - theirselfs. - - Attention: Viewset has to ensure that a user can update only himself. - """ - groups = IdPrimaryKeyRelatedField( - many=True, - queryset=Group.objects.exclude(pk=1), - help_text=ugettext_lazy('The groups this user belongs to. A user will ' - 'get all permissions granted to each of ' - 'his/her groups.')) - - class Meta: - model = User - fields = USERCANSEEEXTRASERIALIZER_FIELDS - read_only_fields = ( - 'is_present', - 'number', - 'comment', - 'groups', - 'is_comittee', - 'is_active', - ) +USERCANSEEEXTRASERIALIZER_FIELDS = USERCANSEESERIALIZER_FIELDS + ( + 'comment', + 'is_active', +) class UserFullSerializer(ModelSerializer): diff --git a/openslides/users/views.py b/openslides/users/views.py index 52e89f3f6..e46f6c245 100644 --- a/openslides/users/views.py +++ b/openslides/users/views.py @@ -18,7 +18,7 @@ from ..utils.views import APIView, PDFView from .access_permissions import UserAccessPermissions from .models import Group, User from .pdf import users_passwords_to_pdf, users_to_pdf -from .serializers import GroupSerializer, UserFullSerializer +from .serializers import GroupSerializer # Viewsets for the REST API @@ -49,19 +49,6 @@ class UserViewSet(ModelViewSet): result = False return result - def get_serializer_class(self): - """ - Returns different serializer classes with respect to action. - """ - if self.action in ('create', 'partial_update', 'update'): - # Return the UserFullSerializer for edit requests. - serializer_class = UserFullSerializer - else: - # Return different serializers according to user permsissions via - # access permissions class. - serializer_class = super().get_serializer_class() - return serializer_class - def update(self, request, *args, **kwargs): """ Customized view endpoint to update an user. @@ -138,7 +125,7 @@ class GroupViewSet(ModelViewSet): partial_update, update and destroy. """ metadata_class = GroupViewSetMetadata - queryset = Group.objects.all() + queryset = Group.objects.prefetch_related('permissions', 'permissions__content_type') serializer_class = GroupSerializer def check_view_permissions(self): diff --git a/openslides/utils/access_permissions.py b/openslides/utils/access_permissions.py index 1586fb6ae..3a1c4d6c7 100644 --- a/openslides/utils/access_permissions.py +++ b/openslides/utils/access_permissions.py @@ -68,7 +68,7 @@ class BaseAccessPermissions(object, metaclass=SignalConnectMetaClass): Hint: You should override this method if your get_serializer_class() method returns different serializers for different users or if you have access restrictions in your view or viewset in methods like - retrieve(), list() or check_object_permissions(). + retrieve() or list(). """ if self.check_permissions(user): data = full_data @@ -78,8 +78,8 @@ class BaseAccessPermissions(object, metaclass=SignalConnectMetaClass): def get_projector_data(self, full_data): """ - Returns the serialized data for the projector. Returns None if has no - access to this specific data. Returns reduced data if the user has - limited access. Default: Returns full data. + Returns the serialized data for the projector. Returns None if the + user has no access to this specific data. Returns reduced data if + the user has limited access. Default: Returns full data. """ return full_data diff --git a/openslides/utils/autoupdate.py b/openslides/utils/autoupdate.py index 60d132203..18ec21e9b 100644 --- a/openslides/utils/autoupdate.py +++ b/openslides/utils/autoupdate.py @@ -11,7 +11,7 @@ from ..core.config import config from ..core.models import Projector from ..users.auth import AnonymousUser from ..users.models import User -from .collection import CollectionElement +from .collection import Collection, CollectionElement def get_logged_in_users(): @@ -23,21 +23,6 @@ def get_logged_in_users(): return User.objects.exclude(session=None).filter(session__expire_date__gte=timezone.now()).distinct() -def get_projector_element_data(projector): - """ - Returns a list of dicts that are required for a specific projector. - - The argument projector has to be a projector instance. - """ - output = [] - for requirement in projector.get_all_requirements(): - required_collection_element = CollectionElement.from_instance(requirement) - element_dict = required_collection_element.as_autoupdate_for_projector() - if element_dict is not None: - output.append(element_dict) - return output - - @channel_session_user_from_http def ws_add_site(message): """ @@ -80,15 +65,14 @@ def ws_add_projector(message, projector_id): Group('projector-{}'.format(projector_id)).add(message.reply_channel) # Send all elements that are on the projector. - output = get_projector_element_data(projector) + output = [] + for requirement in projector.get_all_requirements(): + required_collection_element = CollectionElement.from_instance(requirement) + output.append(required_collection_element.as_autoupdate_for_projector()) # Send all config elements. - for key, value in config.items(): - output.append({ - 'collection': config.get_collection_string(), - 'id': key, - 'action': 'changed', - 'data': {'key': key, 'value': value}}) + collection = Collection(config.get_collection_string()) + output.extend(collection.as_autoupdate_for_projector()) # Send the projector instance. collection_element = CollectionElement.from_instance(projector) @@ -107,64 +91,33 @@ def ws_disconnect_projector(message, projector_id): def send_data(message): """ - Informs all users about changed data. + Informs all site users and projector clients about changed data. """ collection_element = CollectionElement.from_values(**message) - # Loop over all logged in users and the anonymous user. + # Loop over all logged in site users and the anonymous user and send changed data. for user in itertools.chain(get_logged_in_users(), [AnonymousUser()]): channel = Group('user-{}'.format(user.id)) - output = collection_element.as_autoupdate_for_user(user) - if output is None: - # There are no data for the user so he can't see the object. Skip him. - continue - channel.send({'text': json.dumps([output])}) + output = [collection_element.as_autoupdate_for_user(user)] + channel.send({'text': json.dumps(output)}) - # Get the projector elements where data have to be sent and if whole projector - # has to be updated. - if collection_element.collection_string == config.get_collection_string(): - # Config elements are always send to each projector - projectors = Projector.objects.all() - send_all = None # The decission is done later - elif collection_element.collection_string == Projector.get_collection_string(): - # Update a projector, when the projector element is updated. - projectors = [collection_element.get_instance()] - send_all = True - elif collection_element.is_deleted(): - projectors = Projector.objects.all() - send_all = False - else: - # Other elements are only send to the projector they are currently shown - projectors = Projector.get_projectors_that_show_this(collection_element) - send_all = None # The decission is done later - - broadcast_id = config['projector_broadcast'] - if broadcast_id > 0: - projectors = Projector.objects.all() # Also the broadcasted projector should get his data - send_all = True - broadcast_projector_data = get_projector_element_data(Projector.objects.get(pk=broadcast_id)) - broadcast_projector_data.append(CollectionElement.from_values( - collection_string=Projector.get_collection_string(), id=broadcast_id).as_autoupdate_for_projector()) - else: - broadcast_projector_data = None - - for projector in projectors: - if send_all is None: - send_all = projector.need_full_update_for_this(collection_element) - if send_all: - if broadcast_projector_data is None: - output = get_projector_element_data(projector) - else: - output = broadcast_projector_data + # Loop over all projectors and send data that they need. + for projector in Projector.objects.all(): + if collection_element.is_deleted(): + output = [collection_element.as_autoupdate_for_projector()] else: - output = [] - output.append(collection_element.as_autoupdate_for_projector()) + collection_elements = projector.get_collection_elements_required_for_this(collection_element) + output = [collection_element.as_autoupdate_for_projector() for collection_element in collection_elements] if output: Group('projector-{}'.format(projector.pk)).send( {'text': json.dumps(output)}) -def inform_changed_data(instance, is_deleted=False): +def inform_changed_data(instance, information=None): + """ + Informs the autoupdate system and the caching system about the creation or + update of an element. + """ try: root_instance = instance.get_root_rest_element() except AttributeError: @@ -173,29 +126,37 @@ def inform_changed_data(instance, is_deleted=False): else: collection_element = CollectionElement.from_instance( root_instance, - is_deleted=is_deleted and instance == root_instance) - - # If currently there is an open database transaction, then the following - # function is only called, when the transaction is commited. If there - # is currently no transaction, then the function is called immediately. - def send_autoupdate(): - try: - Channel('autoupdate.send_data').send(collection_element.as_channels_message()) - except ChannelLayer.ChannelFull: - pass - - transaction.on_commit(send_autoupdate) + information=information) + # If currently there is an open database transaction, then the + # send_autoupdate function is only called, when the transaction is + # commited. If there is currently no transaction, then the function + # is called immediately. + transaction.on_commit(lambda: send_autoupdate(collection_element)) -def inform_changed_data_receiver(sender, instance, **kwargs): +def inform_deleted_data(collection_string, id, information=None): """ - Receiver for the inform_changed_data function to use in a signal. + Informs the autoupdate system and the caching system about the deletion of + an element. """ - inform_changed_data(instance) + collection_element = CollectionElement.from_values( + collection_string=collection_string, + id=id, + deleted=True, + information=information) + # If currently there is an open database transaction, then the + # send_autoupdate function is only called, when the transaction is + # commited. If there is currently no transaction, then the function + # is called immediately. + transaction.on_commit(lambda: send_autoupdate(collection_element)) -def inform_deleted_data_receiver(sender, instance, **kwargs): +def send_autoupdate(collection_element): """ - Receiver for the inform_changed_data function to use in a signal. + Helper function, that sends a collection_element through a channel to the + autoupdate system. """ - inform_changed_data(instance, is_deleted=True) + try: + Channel('autoupdate.send_data').send(collection_element.as_channels_message()) + except ChannelLayer.ChannelFull: + pass diff --git a/openslides/utils/collection.py b/openslides/utils/collection.py index 30293b9b1..3cd25aefd 100644 --- a/openslides/utils/collection.py +++ b/openslides/utils/collection.py @@ -1,25 +1,41 @@ from django.apps import apps +from django.core.cache import cache, caches class CollectionElement: @classmethod - def from_instance(cls, instance, is_deleted=False): + def from_instance(cls, instance, deleted=False, information=None): """ Returns a collection element from a database instance. + + This will also update the instance in the cache. + + If deleted is set to True, the element is deleted from the cache. """ - return cls(instance=instance, is_deleted=is_deleted) + return cls(instance=instance, deleted=deleted, information=information) @classmethod - def from_values(cls, collection_string, id, is_deleted=False): + def from_values(cls, collection_string, id, deleted=False, full_data=None, information=None): """ Returns a collection element from a collection_string and an id. - """ - return cls(collection_string=collection_string, id=id, is_deleted=is_deleted) - def __init__(self, instance=None, is_deleted=False, collection_string=None, id=None): + If deleted is set to True, the element is deleted from the cache. + + With the argument full_data, the content of the CollectionElement can be set. + It has to be a dict in the format that is used be access_permission.get_full_data(). + """ + return cls(collection_string=collection_string, id=id, deleted=deleted, + full_data=full_data, information=information) + + def __init__(self, instance=None, deleted=False, collection_string=None, id=None, + full_data=None, information=None): """ Do not use this. Use the methods from_instance() or from_values(). """ + self.instance = instance + self.deleted = deleted + self.full_data = full_data + self.information = information or {} if instance is not None: self.collection_string = instance.get_collection_string() self.id = instance.pk @@ -31,62 +47,83 @@ class CollectionElement: 'Invalid state. Use CollectionElement.from_instance() or ' 'CollectionElement.from_values() but not CollectionElement() ' 'directly.') - self.instance = instance - self.deleted = is_deleted + + if self.is_deleted(): + # Delete the element from the cache, if self.is_deleted() is True: + self.delete_from_cache() + elif instance is not None: + # If this element is created via instance and the instance is not deleted + # then update the cache. + self.save_to_cache() + + def __eq__(self, collection_element): + """ + Compares two collection_elements. + + Two collection elements are equal, if they have the same collection_string + and id. + """ + return (self.collection_string == collection_element.collection_string and + self.id == collection_element.id) def as_channels_message(self): """ Returns a dictonary that can be used to send the object through the channels system. """ - return { + channel_message = { 'collection_string': self.collection_string, 'id': self.id, - 'is_deleted': self.is_deleted()} + 'deleted': self.is_deleted()} + if self.information is not None: + channel_message['information'] = self.information + return channel_message + + def as_autoupdate(self, method, *args): + """ + Only for internal use. Do not use it directly. Use as_autoupdate_for_user() + or as_autoupdate_for_projector(). + """ + output = { + 'collection': self.collection_string, + 'id': self.id, + 'action': 'deleted' if self.is_deleted() else 'changed', + } + if not self.is_deleted(): + data = getattr(self.get_access_permissions(), method)( + self.get_full_data(), + *args) + if data is None: + # The user is not allowed to see this element. Set action to deleted. + output['action'] = 'deleted' + else: + output['data'] = data + return output def as_autoupdate_for_user(self, user): """ Returns a dict that can be sent through the autoupdate system for a site user. - - Returns None if the user can not see the element. """ - output = { - 'collection': self.collection_string, - 'id': self.id, - 'action': 'deleted' if self.is_deleted() else 'changed', - } - if not self.is_deleted(): - data = self.get_access_permissions().get_restricted_data( - self.get_full_data(), user) - if data is None: - # The user is not allowed to see this element. Reset output to None. - output = None - else: - output['data'] = data - return output + return self.as_autoupdate( + 'get_restricted_data', + user) def as_autoupdate_for_projector(self): """ Returns a dict that can be sent through the autoupdate system for the projector. - - Returns None if the projector can not see the element. """ - output = { - 'collection': self.collection_string, - 'id': self.id, - 'action': 'deleted' if self.is_deleted() else 'changed', - } - if not self.is_deleted(): - data = self.get_access_permissions().get_projector_data( - self.get_full_data()) - if data is None: - # The user is not allowed to see this element. Reset output to None. - output = None - else: - output['data'] = data - return output + return self.as_autoupdate( + 'get_projector_data') + + def as_dict_for_user(self, user): + """ + Returns a dict with the data for a user. Can be used for the rest api. + """ + return self.get_access_permissions().get_restricted_data( + self.get_full_data(), + user) def get_model(self): """ @@ -102,8 +139,21 @@ class CollectionElement: """ if self.is_deleted(): raise RuntimeError("The collection element is deleted.") + if self.instance is None: - self.instance = self.get_model().objects.get(pk=self.id) + # The config instance has to be get from the config element, because + # some config values are not in the db. + from openslides.core.config import config + if (self.collection_string == config.get_collection_string() and + isinstance(self.id, str)): + self.instance = {'key': self.id, 'value': config[self.id]} + else: + model = self.get_model() + try: + query = model.objects.get_full_queryset() + except AttributeError: + query = model.objects + self.instance = query.get(pk=self.id) return self.instance def get_access_permissions(self): @@ -117,7 +167,20 @@ class CollectionElement: Returns the full_data of this collection_element from with all other dics can be generated. """ - return self.get_access_permissions().get_full_data(self.get_instance()) + # If the full_data is already loaded, return it + # If there is a db_instance, use it to get the full_data + # else: try to use the cache. + # If there is no value in the cach, get the content from the db and save + # it to the cache. + if self.full_data is None and self.instance is None: + # Use the cache version if self.instance is not set. + # After this line full_data can be None, if the element is not in the cache. + self.full_data = cache.get(self.get_cache_key()) + + if self.full_data is None: + self.full_data = self.get_access_permissions().get_full_data(self.get_instance()) + self.save_to_cache() + return self.full_data def is_deleted(self): """ @@ -125,6 +188,217 @@ class CollectionElement: """ return self.deleted + def get_cache_key(self): + """ + Returns a string that is used as cache key for a single instance. + """ + return get_single_element_cache_key(self.collection_string, self.id) + + def delete_from_cache(self): + """ + Delets an element from the cache. + + Does nothing if the element is not in the cache. + """ + # Deletes the element from the cache. + cache.delete(self.get_cache_key()) + + # Delete the id of the instance of the instance list + Collection(self.collection_string).delete_id_from_cache(self.id) + + def save_to_cache(self): + """ + Add or update the element to the cache. + """ + # Set the element to the cache. + cache.set(self.get_cache_key(), self.get_full_data()) + + # Add the id of the element to the collection + Collection(self.collection_string).add_id_to_cache(self.id) + + +class Collection: + """ + Represents all elements of one collection. + """ + + def __init__(self, collection_string): + self.collection_string = collection_string + + def get_cache_key(self, raw=False): + """ + Returns a string that is used as cache key for a collection. + + Django adds a prefix to the cache key when using the django cache api. + In other cases use raw=True to add the same cache key. + """ + key = get_element_list_cache_key(self.collection_string) + if raw: + key = cache.make_key(key) + return key + + def get_model(self): + """ + Returns the django model that is used for this collection. + """ + return get_model_from_collection_string(self.collection_string) + + def element_generator(self): + """ + Generator that yields all collection_elements of this collection. + """ + # Get all cache keys. + ids = self.get_all_ids() + cache_keys = [ + get_single_element_cache_key(self.collection_string, id) + for id in ids] + cached_full_data_dict = cache.get_many(cache_keys) + + # Get all ids that are missing. + missing_ids = set() + for cache_key in cache_keys: + if cache_key not in cached_full_data_dict: + missing_ids.add(get_collection_id_from_cache_key(cache_key)[1]) + + # Generate collection elements that where in the cache. + for cache_key, cached_full_data in cached_full_data_dict.items(): + yield CollectionElement.from_values( + *get_collection_id_from_cache_key(cache_key), + full_data=cached_full_data) + + # Generate collection element that where not in the cache. + if missing_ids: + from openslides.core.config import config + if self.collection_string == config.get_collection_string(): + # If config elements are not in the cache, they have to be read from + # the config object. + for key, value in config.items(): + if key in missing_ids: + collection_element = CollectionElement.from_values( + config.get_collection_string(), + key, + full_data={'key': key, 'value': value}) + # We can not use .from_instance therefore the config value + # is not saved to the cache. We have to do it manualy. + collection_element.save_to_cache() + yield collection_element + else: + model = self.get_model() + try: + query = model.objects.get_full_queryset() + except AttributeError: + query = model.objects + for instance in query.filter(pk__in=missing_ids): + yield CollectionElement.from_instance(instance) + + def as_autoupdate_for_projector(self): + """ + Returns a list of dictonaries to send them to the projector. + """ + output = [] + for collection_element in self.element_generator(): + content = collection_element.as_autoupdate_for_projector() + # Content can not be None. If the projector can not see an element, + # then it is marked as deleted. + output.append(content) + return output + + def as_list_for_user(self, user): + """ + Returns a list of dictonaries to send them to a user, for example over + the rest api. + """ + output = [] + for collection_element in self.element_generator(): + content = collection_element.as_dict_for_user(user) + if content is not None: + output.append(content) + return output + + def get_all_ids(self): + """ + Returns a set of all ids of instances in this collection. + """ + from openslides.core.config import config + if self.collection_string == config.get_collection_string(): + ids = config.config_variables.keys() + elif use_redis_cache(): + ids = self.get_all_ids_redis() + else: + ids = self.get_all_ids_other() + return ids + + def get_all_ids_redis(self): + redis = get_redis_connection() + ids = redis.smembers(self.get_cache_key(raw=True)) + if not ids: + ids = set(self.get_model().objects.values_list('pk', flat=True)) + if ids: + redis.sadd(self.get_cache_key(raw=True), *ids) + # Redis returns the ids as string. + ids = set(int(id) for id in ids) + return ids + + def get_all_ids_other(self): + ids = cache.get(self.get_cache_key()) + if ids is None: + # If it is not in the cache then get it from the database. + ids = set(self.get_model().objects.values_list('pk', flat=True)) + cache.set(self.get_cache_key(), ids) + return ids + + def delete_id_from_cache(self, id): + """ + Delets a id from the cache. + """ + if use_redis_cache(): + self.delete_id_from_cache_redis(id) + else: + self.delete_id_from_cache_other(id) + + def delete_id_from_cache_redis(self, id): + redis = get_redis_connection() + redis.srem(self.get_cache_key(raw=True), id) + + def delete_id_from_cache_other(self, id): + ids = cache.get(self.get_cache_key()) + if ids is not None: + ids = set(ids) + try: + ids.remove(id) + except KeyError: + # The id is not part of id list + pass + else: + if ids: + cache.set(self.get_cache_key(), ids) + else: + # Delete the key, if there are not ids left + cache.delete(self.get_cache_key()) + + def add_id_to_cache(self, id): + """ + Adds a collection id to the list of collection ids in the cache. + """ + if use_redis_cache(): + self.add_id_to_cache_redis(id) + else: + self.add_id_to_cache_other(id) + + def add_id_to_cache_redis(self, id): + redis = get_redis_connection() + if redis.exists(self.get_cache_key(raw=True)): + # Only add the value if it is in the cache. + redis.sadd(self.get_cache_key(raw=True), id) + + def add_id_to_cache_other(self, id): + ids = cache.get(self.get_cache_key()) + if ids is not None: + # Only change the value if it is in the cache. + ids = set(ids) + ids.add(id) + cache.set(self.get_cache_key(), ids) + def get_model_from_collection_string(collection_string): """ @@ -152,3 +426,59 @@ def get_model_from_collection_string(collection_string): # No model was found in all apps. raise ValueError('Invalid message. A valid collection_string is missing.') return model + + +def get_single_element_cache_key(collection_string, id): + """ + Returns a string that is used as cache key for a single instance. + """ + return "{prefix}{id}".format( + prefix=get_single_element_cache_key_prefix(collection_string), + id=id) + + +def get_single_element_cache_key_prefix(collection_string): + """ + Returns the first part of the cache key for single elements, which is the + same for all cache keys of the same collection. + """ + return "{collection}:".format(collection=collection_string) + + +def get_element_list_cache_key(collection_string): + """ + Returns a string that is used as cache key for a collection. + """ + return "{collection}".format(collection=collection_string) + + +def get_collection_id_from_cache_key(cache_key): + """ + Returns a tuble of the collection string and the id from a cache_key + created with get_instance_cache_key. + + The returned id can be an integer or an string. + """ + collection_string, id = cache_key.rsplit(':', 1) + try: + id = int(id) + except ValueError: + # The id is no integer. This can happen on config elements + pass + return (collection_string, id) + + +def use_redis_cache(): + """ + Returns True if Redis is used als caching backend. + """ + try: + from django_redis.cache import RedisCache + except ImportError: + return False + return isinstance(caches['default'], RedisCache) + + +def get_redis_connection(): + from django_redis import get_redis_connection + return get_redis_connection("default") diff --git a/openslides/utils/models.py b/openslides/utils/models.py index ee868cb6a..eac60b948 100644 --- a/openslides/utils/models.py +++ b/openslides/utils/models.py @@ -54,3 +54,54 @@ class RESTModelMixin: the database pk. """ return self.pk + + def save(self, skip_autoupdate=False, information=None, *args, **kwargs): + """ + Calls Django's save() method and afterwards hits the autoupdate system. + + If skip_autoupdate is set to True, then the autoupdate system is not + informed about the model changed. This also means, that the model cache + is not updated. You have to do this manually, by creating a collection + element from the instance: + + CollectionElement.from_instance(instance) + + The optional argument information can be a dictionary that is given to + the autoupdate system. + """ + # We don't know how to fix this circular import + from .autoupdate import inform_changed_data + return_value = super().save(*args, **kwargs) + if not skip_autoupdate: + inform_changed_data(self.get_root_rest_element(), information=information) + return return_value + + def delete(self, skip_autoupdate=False, information=None, *args, **kwargs): + """ + Calls Django's delete() method and afterwards hits the autoupdate system. + + If skip_autoupdate is set to True, then the autoupdate system is not + informed about the model changed. This also means, that the model cache + is not updated. You have to do this manually, by creating a collection + element from the instance: + + CollectionElement.from_instance(instance, deleted=True) + + or + + CollectionElement.from_values(collection_string, id, deleted=True) + + The optional argument information can be a dictionary that is given to + the autoupdate system. + """ + # We don't know how to fix this circular import + from .autoupdate import inform_changed_data, inform_deleted_data + instance_pk = self.pk + return_value = super().delete(*args, **kwargs) + if not skip_autoupdate: + if self != self.get_root_rest_element(): + # The deletion of a included element is a change of the root element. + inform_changed_data(self.get_root_rest_element(), information=information) + else: + inform_deleted_data(self.get_collection_string(), instance_pk, information=information) + return return_value diff --git a/openslides/utils/projector.py b/openslides/utils/projector.py index 89ae0a6a4..5f3686211 100644 --- a/openslides/utils/projector.py +++ b/openslides/utils/projector.py @@ -1,5 +1,6 @@ from django.dispatch import Signal +from .collection import CollectionElement from .dispatch import SignalConnectMetaClass @@ -73,11 +74,31 @@ class ProjectorElement(object, metaclass=SignalConnectMetaClass): """ return () - def need_full_update_for_this(self, collection_element): + def get_requirements_as_collection_elements(self, config_entry): """ - Returns True if this projector element needs to be updated with all - instances as defined in get_requirements(). The given - collection_element contains information about the changed instance. - Default is False. + Returns an iterable of collection elements that are required for this + projector element. The config_entry has to be given. """ - return False + return (CollectionElement.from_instance(instance) for instance in self.get_requirements(config_entry)) + + def get_collection_elements_required_for_this(self, collection_element, config_entry): + """ + Returns a list of CollectionElements that have to be sent to every + projector that shows this projector element according to the given + collection_element and information. + + Default: Returns only the collection_element if it belongs to the + requirements but return all requirements if the projector changes. + """ + requirements_as_collection_elements = list(self.get_requirements_as_collection_elements(config_entry)) + for requirement in requirements_as_collection_elements: + if collection_element == requirement: + output = [collection_element] + break + else: + if collection_element.information.get('this_projector'): + output = [collection_element] + output.extend(requirements_as_collection_elements) + else: + output = [] + return output diff --git a/openslides/utils/rest_api.py b/openslides/utils/rest_api.py index 68272cd46..66a49ea13 100644 --- a/openslides/utils/rest_api.py +++ b/openslides/utils/rest_api.py @@ -1,15 +1,13 @@ from collections import OrderedDict +from django.http import Http404 from rest_framework import status # noqa from rest_framework.decorators import detail_route, list_route # noqa from rest_framework.metadata import SimpleMetadata # noqa -from rest_framework.mixins import ( # noqa - DestroyModelMixin, - ListModelMixin, - RetrieveModelMixin, - UpdateModelMixin, -) -from rest_framework.response import Response # noqa +from rest_framework.mixins import ListModelMixin as _ListModelMixin +from rest_framework.mixins import RetrieveModelMixin as _RetrieveModelMixin +from rest_framework.mixins import DestroyModelMixin, UpdateModelMixin # noqa +from rest_framework.response import Response from rest_framework.routers import DefaultRouter from rest_framework.serializers import ModelSerializer as _ModelSerializer from rest_framework.serializers import ( # noqa @@ -31,6 +29,8 @@ from rest_framework.viewsets import GenericViewSet as _GenericViewSet # noqa from rest_framework.viewsets import ModelViewSet as _ModelViewSet # noqa from rest_framework.viewsets import ViewSet as _ViewSet # noqa +from .collection import Collection, CollectionElement + router = DefaultRouter() @@ -164,11 +164,63 @@ class ModelSerializer(_ModelSerializer): return fields +class ListModelMixin(_ListModelMixin): + """ + Mixin to add the caching system to list requests. + + It is not allowed to use the method get_queryset() in derivated classes. + The attribute queryset has to be used in the following form: + + queryset = Model.objects.all() + """ + def list(self, request, *args, **kwargs): + model = self.get_queryset().model + try: + collection_string = model.get_collection_string() + except AttributeError: + # The corresponding queryset does not support caching. + response = super().list(request, *args, **kwargs) + else: + collection = Collection(collection_string) + response = Response(collection.as_list_for_user(request.user)) + return response + + +class RetrieveModelMixin(_RetrieveModelMixin): + """ + Mixin to add the caching system to retrieve requests. + + It is not allowed to use the method get_queryset() in derivated classes. + The attribute queryset has to be used in the following form: + + queryset = Model.objects.all() + """ + def retrieve(self, request, *args, **kwargs): + model = self.get_queryset().model + try: + collection_string = model.get_collection_string() + except AttributeError: + # The corresponding queryset does not support caching. + response = super().retrieve(request, *args, **kwargs) + else: + lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field + collection_element = CollectionElement.from_values( + collection_string, self.kwargs[lookup_url_kwarg]) + try: + content = collection_element.as_dict_for_user(request.user) + except collection_element.get_model().DoesNotExist: + raise Http404 + if content is None: + self.permission_denied(request) + response = Response(content) + return response + + class GenericViewSet(PermissionMixin, _GenericViewSet): pass -class ModelViewSet(PermissionMixin, _ModelViewSet): +class ModelViewSet(PermissionMixin, ListModelMixin, RetrieveModelMixin, _ModelViewSet): pass diff --git a/openslides/utils/settings.py.tpl b/openslides/utils/settings.py.tpl index eb8da2300..7ca33e101 100644 --- a/openslides/utils/settings.py.tpl +++ b/openslides/utils/settings.py.tpl @@ -46,6 +46,17 @@ DEBUG = %(debug)s # Change this setting to use e. g. PostgreSQL or MySQL. +# DATABASES = { +# 'default': { +# 'ENGINE': 'django.db.backends.postgresql', +# 'NAME': 'mydatabase', +# 'USER': 'mydatabaseuser', +# 'PASSWORD': 'mypassword', +# 'HOST': '127.0.0.1', +# 'PORT': '5432', +# } +# } + DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', @@ -64,12 +75,17 @@ DATABASES = { # CHANNEL_LAYERS['default']['BACKEND'] = 'asgi_redis.RedisChannelLayer' -# https://niwinz.github.io/django-redis/latest/#_user_guide + +# Caching + +# Django uses a inmemory cache at default. This supports only one thread. If +# you use more then one thread another caching backend is required. We recommand +# django-redis: https://niwinz.github.io/django-redis/latest/#_user_guide # CACHES = { # "default": { # "BACKEND": "django_redis.cache.RedisCache", -# "LOCATION": "redis://127.0.0.1:6379/1", +# "LOCATION": "redis://127.0.0.1:6379/0", # "OPTIONS": { # "CLIENT_CLASS": "django_redis.client.DefaultClient", # } diff --git a/tests/integration/agenda/test_views.py b/tests/integration/agenda/test_views.py index 1ddb7ce42..b0e9d0acd 100644 --- a/tests/integration/agenda/test_views.py +++ b/tests/integration/agenda/test_views.py @@ -1,90 +1,7 @@ -import json - -from rest_framework.test import APIClient - -from openslides.agenda.models import Item from openslides.topics.models import Topic from openslides.utils.test import TestCase -class AgendaTreeTest(TestCase): - def setUp(self): - Topic.objects.create(title='item1') - item2 = Topic.objects.create(title='item2').agenda_item - item3 = Topic.objects.create(title='item2a').agenda_item - item3.parent = item2 - item3.save() - self.client = APIClient() - self.client.login(username='admin', password='admin') - - def test_get(self): - response = self.client.get('/rest/agenda/item/tree/') - - self.assertEqual(json.loads(response.content.decode()), - [{'children': [], 'id': 1}, - {'children': [{'children': [], 'id': 3}], 'id': 2}]) - - def test_set(self): - tree = [{'id': 3}, - {'children': [{'id': 1}], 'id': 2}] - - response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json') - - self.assertEqual(response.status_code, 200) - - item1 = Item.objects.get(pk=1) - item2 = Item.objects.get(pk=2) - item3 = Item.objects.get(pk=3) - self.assertEqual(item1.parent_id, 2) - self.assertEqual(item1.weight, 0) - self.assertEqual(item2.parent_id, None) - self.assertEqual(item2.weight, 1) - self.assertEqual(item3.parent_id, None) - self.assertEqual(item3.weight, 0) - - def test_set_without_perm(self): - self.client = APIClient() - - response = self.client.put('/rest/agenda/item/tree/', {'tree': []}, format='json') - - self.assertEqual(response.status_code, 403) - - def test_tree_with_double_item(self): - """ - Test to send a tree that has an item-pk more then once in it. - - It is expected, that the responsecode 400 is returned with a specific - content - """ - tree = [{'id': 1}, {'id': 1}] - - response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json') - - self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, {'detail': "Item 1 is more then once in the tree."}) - - def test_tree_with_empty_children(self): - """ - Test that the chrildren element is not required in the tree - """ - tree = [{'id': 1}] - - response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json') - - self.assertEqual(response.status_code, 200) - - def test_tree_with_unknown_item(self): - """ - Tests that unknown items lead to an error. - """ - tree = [{'id': 500}] - - response = self.client.put('/rest/agenda/item/tree/', {'tree': tree}, format='json') - - self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, {'detail': "Item 500 is not in the database."}) - - class TestAgendaPDF(TestCase): def test_get(self): """ diff --git a/tests/integration/agenda/test_viewsets.py b/tests/integration/agenda/test_viewsets.py index c31eb95b1..e72d6313b 100644 --- a/tests/integration/agenda/test_viewsets.py +++ b/tests/integration/agenda/test_viewsets.py @@ -4,9 +4,12 @@ from rest_framework import status from rest_framework.test import APIClient from openslides.agenda.models import Item, Speaker +from openslides.assignments.models import Assignment from openslides.core.config import config from openslides.core.models import Projector +from openslides.motions.models import Motion from openslides.topics.models import Topic +from openslides.users.models import User from openslides.utils.test import TestCase @@ -37,7 +40,61 @@ class RetrieveItem(TestCase): permission = group.permissions.get(content_type__app_label=app_label, codename=codename) group.permissions.remove(permission) response = self.client.get(reverse('item-detail', args=[self.item.pk])) - self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + +class TestDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some agenda items are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + Topic.objects.create(title='topic{}'.format(index)) + parent = Topic.objects.create(title='parent').agenda_item + child = Topic.objects.create(title='child').agenda_item + child.parent = parent + child.save() + Motion.objects.create(title='motion1') + Motion.objects.create(title='motion2') + Assignment.objects.create(title='assignment', open_posts=5) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions, + * 2 requests to get the list of all agenda items, + * 1 request to get all speakers, + * 3 requests to get the assignments, motions and topics and + + * 2 requests for the motionsversions. + + TODO: There could be less requests to get the session and the request user. + The last two request for the motionsversions are a bug. + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(13): + self.client.get(reverse('item-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions) + * 2 requests to get the list of all agenda items, + * 1 request to get all speakers, + * 3 requests to get the assignments, motions and topics and + + * 32 requests for the motionsversions. + + TODO: The last 32 requests are a bug. + """ + with self.assertNumQueries(40): + self.client.get(reverse('item-list')) class ManageSpeaker(TestCase): diff --git a/tests/integration/assignments/test_viewset.py b/tests/integration/assignments/test_viewset.py index e209a23fe..aebbb8f7c 100644 --- a/tests/integration/assignments/test_viewset.py +++ b/tests/integration/assignments/test_viewset.py @@ -4,9 +4,61 @@ from rest_framework import status from rest_framework.test import APIClient from openslides.assignments.models import Assignment +from openslides.core.config import config +from openslides.users.models import User from openslides.utils.test import TestCase +class TestDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some assignments are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + Assignment.objects.create(title='motion{}'.format(index), open_posts=1) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions, + * 2 requests to get the list of all assignments, + * 1 request to get all related users, + * 1 request to get the agenda item, + * 1 request to get the polls, + + * 10 request to featch each related user again. + + TODO: There could be less requests to get the session and the request user. + The eleven request are a bug. + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(30): + self.client.get(reverse('assignment-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions) + * 2 requests to get the list of all assignments, + * 1 request to get all related users, + * 1 request to get the agenda item, + * 1 request to get the polls, + * 1 request to get the tags, + + * lots of permissions requests. + + TODO: The last requests are a bug. + """ + with self.assertNumQueries(57): + self.client.get(reverse('assignment-list')) + + class CanidatureSelf(TestCase): """ Tests self candidation view. diff --git a/tests/integration/core/test_viewset.py b/tests/integration/core/test_viewset.py new file mode 100644 index 000000000..c1a803fa6 --- /dev/null +++ b/tests/integration/core/test_viewset.py @@ -0,0 +1,147 @@ +from django.core.urlresolvers import reverse +from rest_framework.test import APIClient + +from openslides.core.config import config +from openslides.core.models import ChatMessage, Projector, Tag +from openslides.users.models import User +from openslides.utils.test import TestCase + + +class TestProjectorDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + Projector.objects.create(name="Projector{}".format(index)) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions, + * 2 requests to get the list of all projectors, + * 1 request to get the list of the projector defaults. + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(8): + self.client.get(reverse('projector-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions) + * 2 requests to get the list of all projectors, + * 1 request to get the list of the projector defaults and + + * 11 requests for permissions. + + TODO: The last 11 requests are a bug. + """ + with self.assertNumQueries(16): + self.client.get(reverse('projector-list')) + + +class TestCharmessageDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + user = User.objects.get(pk=1) + for index in range(10): + ChatMessage.objects.create(user=user) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions, + * 2 requests to get the list of all chatmessages, + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(7): + self.client.get(reverse('chatmessage-list')) + + +class TestTagDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + Tag.objects.create(name='tag{}'.format(index)) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the session an the request user with its permissions, + * 2 requests to get the list of all tags, + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(4): + self.client.get(reverse('tag-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions) + * 2 requests to get the list of all projectors, + + * 10 requests for to config + + The last 10 requests are a bug. + """ + with self.assertNumQueries(14): + self.client.get(reverse('tag-list')) + + +class TestConfigDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the session an the request user with its permissions and + * 1 requests to get the list of all config values + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(3): + self.client.get(reverse('config-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions), + * 1 to get all config value and + + * 57 requests to find out if anonymous is enabled. + + TODO: The last 57 requests are a bug. + """ + with self.assertNumQueries(60): + self.client.get(reverse('config-list')) diff --git a/tests/integration/mediafiles/__init__.py b/tests/integration/mediafiles/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/mediafiles/test_viewset.py b/tests/integration/mediafiles/test_viewset.py new file mode 100644 index 000000000..9048f34e7 --- /dev/null +++ b/tests/integration/mediafiles/test_viewset.py @@ -0,0 +1,46 @@ +from django.core.files.uploadedfile import SimpleUploadedFile +from django.core.urlresolvers import reverse +from rest_framework.test import APIClient + +from openslides.core.config import config +from openslides.mediafiles.models import Mediafile +from openslides.users.models import User +from openslides.utils.test import TestCase + + +class TestDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + Mediafile.objects.create( + title='some_file{}'.format(index), + mediafile=SimpleUploadedFile( + 'some_file{}'.format(index), + b'some content.')) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions and + * 2 requests to get the list of all files. + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(7): + self.client.get(reverse('mediafile-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions) and + * 2 requests to get the list of all projectors. + """ + with self.assertNumQueries(4): + self.client.get(reverse('mediafile-list')) diff --git a/tests/integration/motions/test_viewset.py b/tests/integration/motions/test_viewset.py index 2c3b72ab9..d60c8dd92 100644 --- a/tests/integration/motions/test_viewset.py +++ b/tests/integration/motions/test_viewset.py @@ -8,9 +8,139 @@ from rest_framework.test import APIClient from openslides.core.config import config from openslides.core.models import Tag from openslides.motions.models import Category, Motion, State +from openslides.users.models import User from openslides.utils.test import TestCase +class TestMotionDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + Motion.objects.create(title='motion{}'.format(index)) + # TODO: Create some polls etc. + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions, + * 2 requests to get the list of all motions, + * 1 request to get the motion versions, + * 1 request to get the agenda item, + * 1 request to get the motion log, + * 1 request to get the polls, + * 1 request to get the attachments, + * 1 request to get the tags, + * 2 requests to get the submitters and supporters + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(15): + self.client.get(reverse('motion-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions) + * 2 requests to get the list of all motions, + * 1 request to get the motion versions, + * 1 request to get the agenda item, + * 1 request to get the motion log, + * 1 request to get the polls, + * 1 request to get the attachments, + * 1 request to get the tags, + * 2 requests to get the submitters and supporters + + * 10 requests for permissions. + + TODO: The last 10 requests are a bug. + """ + with self.assertNumQueries(22): + self.client.get(reverse('motion-list')) + + +class TestCategoryDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + Category.objects.create(name='category{}'.format(index)) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions and + * 2 requests to get the list of all categories. + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(7): + self.client.get(reverse('category-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions) + * 2 requests to get the list of all motions and + + * 10 requests for permissions. + + TODO: The last 10 requests are a bug. + """ + with self.assertNumQueries(14): + self.client.get(reverse('category-list')) + + +class TestWorkflowDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + # There do not need to be more workflows + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions, + * 2 requests to get the list of all workflows, + * 1 request to get all states and + * 1 request to get the next states of all states. + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(9): + self.client.get(reverse('workflow-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions), + * 2 requests to get the list of all workflows, + * 1 request to get all states and + * 1 request to get the next states of all states. + + * 2 requests for permissions. + + TODO: The last 2 requests are a bug. + """ + with self.assertNumQueries(8): + self.client.get(reverse('workflow-list')) + + class CreateMotion(TestCase): """ Tests motion creation. diff --git a/tests/integration/users/test_viewset.py b/tests/integration/users/test_viewset.py index 6d3692d26..c53679074 100644 --- a/tests/integration/users/test_viewset.py +++ b/tests/integration/users/test_viewset.py @@ -8,6 +8,86 @@ from openslides.users.serializers import UserFullSerializer from openslides.utils.test import TestCase +class TestUserDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + User.objects.create(username='user{}'.format(index)) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 5 requests to get the session an the request user with its permissions, + * 2 requests to get the list of all assignments and + * 1 request to get all groups. + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(8): + self.client.get(reverse('user-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the permission for anonymous (config and permissions) + * 2 requests to get the list of all users, + * 1 request to get all groups and + + * lots of permissions requests. + + TODO: The last requests are a bug. + """ + with self.assertNumQueries(27): + self.client.get(reverse('user-list')) + + +class TestGroupDBQueries(TestCase): + """ + Tests that receiving elements only need the required db queries. + + Therefore in setup some objects are created and received with different + user accounts. + """ + + def setUp(self): + self.client = APIClient() + config['general_system_enable_anonymous'] = True + for index in range(10): + Group.objects.create(name='group{}'.format(index)) + + def test_admin(self): + """ + Tests that only the following db queries are done: + * 2 requests to get the session an the request user with its permissions, + * 1 request to get the list of all groups, + * 1 request to get the permissions and + * 1 request to get the content_object for the permissions. + """ + self.client.force_login(User.objects.get(pk=1)) + with self.assertNumQueries(5): + self.client.get(reverse('group-list')) + + def test_anonymous(self): + """ + Tests that only the following db queries are done: + * 2 requests to find out if anonymous is enabled + * 1 request to get the list of all groups, + * 1 request to get the permissions and + * 1 request to get the content_object for the permissions. + + TODO: There should be only one request to find out if anonymous is enabled. + """ + with self.assertNumQueries(5): + self.client.get(reverse('group-list')) + + class UserGetTest(TestCase): """ Tests to receive a users via REST API. diff --git a/tests/integration/utils/autoupdate.py b/tests/integration/utils/test_autoupdate.py similarity index 65% rename from tests/integration/utils/autoupdate.py rename to tests/integration/utils/test_autoupdate.py index 1aa0cef5e..4644db167 100644 --- a/tests/integration/utils/autoupdate.py +++ b/tests/integration/utils/test_autoupdate.py @@ -24,13 +24,22 @@ class TestGetLoggedInUsers(TestCase): User.objects.create(username='user3') # Create a session with a user, that expires in 5 hours - Session.objects.create(user=user1, expire_data=timezone.now() + timedelta(hours=5)) + Session.objects.create( + user=user1, + expire_date=timezone.now() + timedelta(hours=5), + session_key='1') # Create a session with a user, that is expired before 5 hours - Session.objects.create(user=user2, expire_data=timezone.now() + timedelta(hours=-5)) + Session.objects.create( + user=user2, + expire_date=timezone.now() + timedelta(hours=-5), + session_key='2') # Create a session with an anonymous user, that expires in 5 hours - Session.objects.create(user=None, expire_data=timezone.now() + timedelta(hours=5)) + Session.objects.create( + user=None, + expire_date=timezone.now() + timedelta(hours=5), + session_key='3') self.assertEqual(list(get_logged_in_users()), [user1]) @@ -40,7 +49,13 @@ class TestGetLoggedInUsers(TestCase): The user should be returned only once. """ user1 = User.objects.create(username='user1') - Session.objects.create(user=user1, expire_data=timezone.now() + timedelta(hours=1)) - Session.objects.create(user=user1, expire_data=timezone.now() + timedelta(hours=2)) + Session.objects.create( + user=user1, + expire_date=timezone.now() + timedelta(hours=1), + session_key='1') + Session.objects.create( + user=user1, + expire_date=timezone.now() + timedelta(hours=2), + session_key='2') self.assertEqual(list(get_logged_in_users()), [user1]) diff --git a/tests/integration/utils/test_collection.py b/tests/integration/utils/test_collection.py new file mode 100644 index 000000000..f5cf7c0ed --- /dev/null +++ b/tests/integration/utils/test_collection.py @@ -0,0 +1,134 @@ +from unittest.mock import patch + +from channels.tests import ChannelTestCase +from django.core.cache import caches + +from openslides.topics.models import Topic +from openslides.utils import collection + + +class TestCase(ChannelTestCase): + """ + Testcase that uses the local mem cache and clears the cache after each test. + """ + def setUp(self): + cache = caches['locmem'] + cache.clear() + self.patch = patch('openslides.utils.collection.cache', cache) + self.patch.start() + + def tearDown(self): + self.patch.stop() + + +class TestCollectionElementCache(TestCase): + def test_clean_cache(self): + """ + Tests that the data is retrieved from the database. + """ + topic = Topic.objects.create(title='test topic') + collection_element = collection.CollectionElement.from_values('topics/topic', 1) + caches['locmem'].clear() + + with self.assertNumQueries(3): + instance = collection_element.get_full_data() + self.assertEqual(topic.title, instance['title']) + + def test_with_cache(self): + """ + Tests that no db query is used when the valie is in the cache. + + The value is added to the test when .create(...) is called. This hits + the autoupdate system, which fills the cache. + """ + topic = Topic.objects.create(title='test topic') + collection_element = collection.CollectionElement.from_values('topics/topic', 1) + + with self.assertNumQueries(0): + collection_element = collection.CollectionElement.from_values('topics/topic', 1) + instance = collection_element.get_full_data() + self.assertEqual(topic.title, instance['title']) + + def test_non_existing_instance(self): + collection_element = collection.CollectionElement.from_values('topics/topic', 1) + + with self.assertRaises(Topic.DoesNotExist): + collection_element.get_full_data() + + +class TestCollectionCache(TestCase): + def test_clean_cache(self): + """ + Tests that the instances are retrieved from the database. + + Currently there are 10 queries needed. This can change in the future, + but it has to be more then zero. + """ + Topic.objects.create(title='test topic1') + Topic.objects.create(title='test topic2') + Topic.objects.create(title='test topic3') + topic_collection = collection.Collection('topics/topic') + caches['locmem'].clear() + + with self.assertNumQueries(4): + instance_list = list(topic_collection.as_autoupdate_for_projector()) + self.assertEqual(len(instance_list), 3) + + def test_with_cache(self): + """ + Tests that no db query is used when the list is received twice. + """ + Topic.objects.create(title='test topic1') + Topic.objects.create(title='test topic2') + Topic.objects.create(title='test topic3') + topic_collection = collection.Collection('topics/topic') + list(topic_collection.as_autoupdate_for_projector()) + + with self.assertNumQueries(0): + instance_list = list(topic_collection.as_autoupdate_for_projector()) + self.assertEqual(len(instance_list), 3) + + def test_with_some_objects_in_the_cache(self): + """ + One element (topic3) is in the cache and two are not. + """ + Topic.objects.create(title='test topic1') + Topic.objects.create(title='test topic2') + caches['locmem'].clear() + Topic.objects.create(title='test topic3') + topic_collection = collection.Collection('topics/topic') + + with self.assertNumQueries(4): + instance_list = list(topic_collection.as_autoupdate_for_projector()) + self.assertEqual(len(instance_list), 3) + + def test_deletion(self): + """ + When an element is deleted, the cache should be updated automaticly via + the autoupdate system. So there should be no db queries. + """ + Topic.objects.create(title='test topic1') + Topic.objects.create(title='test topic2') + topic3 = Topic.objects.create(title='test topic3') + topic_collection = collection.Collection('topics/topic') + list(topic_collection.as_autoupdate_for_projector()) + + topic3.delete() + + with self.assertNumQueries(0): + instance_list = list(topic_collection.as_autoupdate_for_projector()) + self.assertEqual(len(instance_list), 2) + + def test_config_elements_without_cache(self): + topic_collection = collection.Collection('core/config') + caches['locmem'].clear() + + with self.assertNumQueries(1): + list(topic_collection.as_autoupdate_for_projector()) + + def test_config_elements_with_cache(self): + topic_collection = collection.Collection('core/config') + list(topic_collection.as_autoupdate_for_projector()) + + with self.assertNumQueries(0): + list(topic_collection.as_autoupdate_for_projector()) diff --git a/tests/settings.py b/tests/settings.py index 0db70f417..e3483c5cc 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -75,3 +75,14 @@ TEST_RUNNER = 'openslides.utils.test.OpenSlidesDiscoverRunner' PASSWORD_HASHERS = [ 'django.contrib.auth.hashers.MD5PasswordHasher', ] + + +# Use the dummy cache that does not cache anything +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.dummy.DummyCache' + }, + 'locmem': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache' + } +} diff --git a/tests/unit/utils/test_collection.py b/tests/unit/utils/test_collection.py new file mode 100644 index 000000000..9886fdba2 --- /dev/null +++ b/tests/unit/utils/test_collection.py @@ -0,0 +1,251 @@ +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from openslides.core.models import Projector +from openslides.utils import collection + + +class TestCacheKeys(TestCase): + def test_get_collection_id_from_cache_key(self): + """ + Test that get_collection_id_from_cache_key works together with + get_single_element_cache_key. + """ + element = ('some/testkey', 42) + self.assertEqual( + element, + collection.get_collection_id_from_cache_key( + collection.get_single_element_cache_key(*element))) + + def test_get_collection_id_from_cache_key_for_strings(self): + """ + Test get_collection_id_from_cache_key for strings + """ + element = ('some/testkey', 'my_config_value') + self.assertEqual( + element, + collection.get_collection_id_from_cache_key( + collection.get_single_element_cache_key(*element))) + + def test_get_single_element_cache_key_prefix(self): + """ + Tests that the cache prefix is realy a prefix. + """ + element = ('some/testkey', 42) + + cache_key = collection.get_single_element_cache_key(*element) + prefix = collection.get_single_element_cache_key_prefix(element[0]) + + self.assertTrue(cache_key.startswith(prefix)) + + def test_prefix_different_then_list(self): + """ + Test that the return value of get_single_element_cache_key_prefix is + something different then get_element_list_cache_key. + """ + collection_string = 'some/testkey' + + prefix = collection.get_single_element_cache_key_prefix(collection_string) + list_cache_key = collection.get_element_list_cache_key(collection_string) + + self.assertNotEqual(prefix, list_cache_key) + + +class TestGetModelFromCollectionString(TestCase): + def test_known_app(self): + projector_model = collection.get_model_from_collection_string('core/projector') + + self.assertEqual(projector_model, Projector) + + def test_unknown_app(self): + with self.assertRaises(ValueError): + collection.get_model_from_collection_string('invalid/model') + + +class TestCollectionElement(TestCase): + def test_from_values(self): + collection_element = collection.CollectionElement.from_values('testmodule/model', 42) + + self.assertEqual(collection_element.collection_string, 'testmodule/model') + self.assertEqual(collection_element.id, 42) + + @patch('openslides.utils.collection.Collection') + @patch('openslides.utils.collection.cache') + def test_from_values_deleted(self, mock_cache, mock_collection): + """ + Tests that when createing a CollectionElement with deleted=True the element + is deleted from the cache. + """ + collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True) + + self.assertTrue(collection_element.is_deleted()) + mock_cache.delete.assert_called_with('testmodule/model:42') + mock_collection.assert_called_with('testmodule/model') + mock_collection().delete_id_from_cache.assert_called_with(42) + + def test_as_channel_message(self): + collection_element = collection.CollectionElement.from_values('testmodule/model', 42) + + self.assertEqual( + collection_element.as_channels_message(), + {'collection_string': 'testmodule/model', + 'id': 42, + 'deleted': False, + 'information': {}}) + + def test_as_autoupdate_for_user(self): + collection_element = collection.CollectionElement.from_values('testmodule/model', 42) + fake_user = MagicMock() + collection_element.get_access_permissions = MagicMock() + collection_element.get_access_permissions().get_restricted_data.return_value = 'restricted_data' + collection_element.get_full_data = MagicMock() + + self.assertEqual( + collection_element.as_autoupdate_for_user(fake_user), + {'collection': 'testmodule/model', + 'id': 42, + 'action': 'changed', + 'data': 'restricted_data'}) + collection_element.get_full_data.assert_called_once_with() + + def test_as_autoupdate_for_user_no_permission(self): + collection_element = collection.CollectionElement.from_values('testmodule/model', 42) + fake_user = MagicMock() + collection_element.get_access_permissions = MagicMock() + collection_element.get_access_permissions().get_restricted_data.return_value = None + collection_element.get_full_data = MagicMock() + + self.assertEqual( + collection_element.as_autoupdate_for_user(fake_user), + {'collection': 'testmodule/model', + 'id': 42, + 'action': 'deleted'}) + collection_element.get_full_data.assert_called_once_with() + + def test_as_autoupdate_for_user_deleted(self): + collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True) + fake_user = MagicMock() + + self.assertEqual( + collection_element.as_autoupdate_for_user(fake_user), + {'collection': 'testmodule/model', + 'id': 42, + 'action': 'deleted'}) + + def test_get_instance_deleted(self): + collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True) + + with self.assertRaises(RuntimeError): + collection_element.get_instance() + + @patch('openslides.core.config.config') + def test_get_instance_config_str(self, mock_config): + mock_config.get_collection_string.return_value = 'core/config' + mock_config.__getitem__.return_value = 'config_value' + collection_element = collection.CollectionElement.from_values('core/config', 'my_config_value') + + instance = collection_element.get_instance() + + self.assertEqual( + instance, + {'key': 'my_config_value', + 'value': 'config_value'}) + + def test_get_instance(self): + collection_element = collection.CollectionElement.from_values('testmodule/model', 42) + collection_element.get_model = MagicMock() + + collection_element.get_instance() + + collection_element.get_model().objects.get_full_queryset().get.assert_called_once_with(pk=42) + + @patch('openslides.utils.collection.cache') + def test_get_full_data_already_loaded(self, mock_cache): + """ + Test that the cache and the self.get_instance() is not hit, when the + instance is already loaded. + """ + collection_element = collection.CollectionElement.from_values('testmodule/model', 42) + collection_element.full_data = 'my_full_data' + collection_element.get_instance = MagicMock() + + collection_element.get_full_data() + + mock_cache.get.assert_not_called() + collection_element.get_instance.assert_not_called() + + @patch('openslides.utils.collection.cache') + def test_get_full_data_from_cache(self, mock_cache): + """ + Test that the value from the cache is used not get_instance is not + called. + """ + collection_element = collection.CollectionElement.from_values('testmodule/model', 42) + collection_element.get_instance = MagicMock() + mock_cache.get.return_value = 'cache_value' + + instance = collection_element.get_full_data() + + self.assertEqual(instance, 'cache_value') + mock_cache.get.assert_called_once_with('testmodule/model:42') + collection_element.get_instance.assert_not_called + + @patch('openslides.utils.collection.Collection') + @patch('openslides.utils.collection.cache') + def test_get_full_data_from_get_instance(self, mock_cache, mock_Collection): + """ + Test that the value from get_instance is used and saved to the cache + """ + collection_element = collection.CollectionElement.from_values('testmodule/model', 42) + collection_element.get_instance = MagicMock() + collection_element.get_access_permissions = MagicMock() + collection_element.get_access_permissions().get_full_data.return_value = 'get_instance_value' + mock_cache.get.return_value = None + + instance = collection_element.get_full_data() + + self.assertEqual(instance, 'get_instance_value') + mock_cache.get.assert_called_once_with('testmodule/model:42') + collection_element.get_instance.assert_called_once_with() + mock_cache.set.assert_called_once_with('testmodule/model:42', 'get_instance_value') + mock_Collection.assert_called_once_with('testmodule/model') + mock_Collection().add_id_to_cache.assert_called_once_with(42) + + def test_equal(self): + self.assertEqual( + collection.CollectionElement.from_values('testmodule/model', 1), + collection.CollectionElement.from_values('testmodule/model', 1)) + self.assertEqual( + collection.CollectionElement.from_values('testmodule/model', 1), + collection.CollectionElement.from_values('testmodule/model', 1, deleted=True)) + self.assertNotEqual( + collection.CollectionElement.from_values('testmodule/model', 1), + collection.CollectionElement.from_values('testmodule/model', 2)) + self.assertNotEqual( + collection.CollectionElement.from_values('testmodule/model', 1), + collection.CollectionElement.from_values('testmodule/other_model', 1)) + + +class TestCollection(TestCase): + @patch('openslides.utils.collection.CollectionElement') + @patch('openslides.utils.collection.cache') + def test_element_generator(self, mock_cache, mock_CollectionElement): + """ + Test with the following scenario: The collection has three elements. Two + are in the cache and one is not. + """ + test_collection = collection.Collection('testmodule/model') + test_collection.get_all_ids = MagicMock(return_value=set([1, 2, 3])) + test_collection.get_model = MagicMock() + test_collection.get_model().objects.get_full_queryset().filter.return_value = ['my_instance'] + mock_cache.get_many.return_value = { + 'testmodule/model:1': 'element1', + 'testmodule/model:2': 'element2'} + + list(test_collection.element_generator()) + + mock_cache.get_many.assert_called_once_with( + ['testmodule/model:1', 'testmodule/model:2', 'testmodule/model:3']) + test_collection.get_model().objects.get_full_queryset().filter.assert_called_once_with(pk__in={3}) + self.assertEqual(mock_CollectionElement.from_values.call_count, 2) + self.assertEqual(mock_CollectionElement.from_instance.call_count, 1)