Changed restricted data parsing. Cached full data on startup.

openslides/agenda/access_permissions.py

@@ -27,32 +27,42 @@ class ItemAccessPermissions(BaseAccessPermissions):
         Returns the restricted serialized data for the instance prepared
         for the user.
         """
+        def filtered_data(full_data, blocked_keys):
+            """
+            Returns a new dict like full_data but with all blocked_keys removed.
+            """
+            whitelist = full_data.keys() - blocked_keys
+            return {key: full_data[key] for key in whitelist}
+
+        # many_items is True, when there are more then one items in full_data.
+        many_items = not isinstance(full_data, dict)
+        full_data = full_data if many_items else [full_data]
+
         if has_perm(user, 'agenda.can_see'):
-            if full_data['is_hidden'] and not has_perm(user, 'agenda.can_see_hidden_items'):
+            if has_perm(user, 'agenda.can_manage'):
-                # The data is hidden but the user isn't allowed to see it. Jst pass
+                data = full_data
-                # the whitelisted keys so the list of speakers is provided regardless.
+            elif has_perm(user, 'agenda.can_see_hidden_items'):
-                whitelist = (
+                blocked_keys = ('comment',)
+                data = [filtered_data(full, blocked_keys) for full in full_data]
+            else:
+                data = []
+                filtered_blocked_keys = full_data[0].keys() - (
                     'id',
                     'title',
                     'speakers',
                     'speaker_list_closed',
-                    'content_object',)
+                    'content_object')
-                data = {}
+                not_filtered_blocked_keys = ('comment',)
-                for key in full_data.keys():
+                for full in full_data:
-                    if key in whitelist:
+                    if full['is_hidden']:
-                        data[key] = full_data[key]
+                        blocked_keys = filtered_blocked_keys
                     else:
-                if has_perm(user, 'agenda.can_manage'):
+                        blocked_keys = not_filtered_blocked_keys
-                    data = full_data
+                    data.append(filtered_data(full, blocked_keys))
-                else:
-                    # Strip out item comments for unprivileged users.
-                    data = {}
-                    for key in full_data.keys():
-                        if key != 'comment':
-                            data[key] = full_data[key]
         else:
             data = None
-        return data
+
+        return data if many_items else data[0]
 
     def get_projector_data(self, full_data):
         """

openslides/agenda/signals.py

@@ -56,18 +56,12 @@ def get_permission_change_data(sender, permissions, **kwargs):
 
 def is_user_data_required(sender, request_user, user_data, **kwargs):
     """
-    Returns True if request user can see the agenda and user_data is required
+    If request_user can see the agenda, then returns all user ids that are
-    to be displayed as speaker.
+    speakers in some agenda items. Else, it returns an empty set.
     """
-    result = False
+    speakers = set()
     if has_perm(request_user, 'agenda.can_see'):
         for item_collection_element in Collection(Item.get_collection_string()).element_generator():
             full_data = item_collection_element.get_full_data()
-            for speaker in full_data['speakers']:
+            speakers.update(speaker['user_id'] for speaker in full_data['speakers'])
-                if user_data['id'] == speaker['user_id']:
+    return speakers
-                    result = True
-                    break
-            else:
-                continue
-            break
-    return result

openslides/assignments/access_permissions.py

@@ -33,8 +33,14 @@ class AssignmentAccessPermissions(BaseAccessPermissions):
         if has_perm(user, 'assignments.can_see') and has_perm(user, 'assignments.can_manage'):
             data = full_data
         elif has_perm(user, 'assignments.can_see'):
+            many_items = not isinstance(full_data, dict)
+            full_data_list = full_data if many_items else [full_data]
+            out = []
+            for full_data in full_data_list:
                 data = full_data.copy()
-            data['polls'] = [poll for poll in data['polls'] if poll['published']]
+                data['polls'] = [poll for poll in data['polls'] if data['published']]
+                out.append(data)
+            data = out if many_items else out[0]
         else:
             data = None
         return data

openslides/assignments/signals.py

@@ -18,28 +18,14 @@ def get_permission_change_data(sender, permissions=None, **kwargs):
 
 def is_user_data_required(sender, request_user, user_data, **kwargs):
     """
-    Returns True if request user can see assignments and user_data is required
+    If request_user can see assignments, then returns all user ids that are
-    to be displayed as candidates (including poll options).
+    displayed as candidates (including poll options). Else, it returns an empty set.
     """
-    result = False
+    user_ids = set()
     if has_perm(request_user, 'assignments.can_see'):
         for assignment_collection_element in Collection(Assignment.get_collection_string()).element_generator():
             full_data = assignment_collection_element.get_full_data()
-            for related_user in full_data['assignment_related_users']:
+            user_ids.update(related_user['user_id'] for related_user in full_data['assignment_related_users'])
-                if user_data['id'] == related_user['user_id']:
-                    result = True
-                    break
-            else:
             for poll in full_data['polls']:
-                    for option in poll['options']:
+                user_ids.update(option['candidate_id'] for option in poll['options'])
-                        if user_data['id'] == option['candidate_id']:
+    return user_ids
-                            result = True
-                            break
-                    else:
-                        continue
-                    break
-                else:
-                    continue
-                break
-            break
-    return result

openslides/core/signals.py

@@ -54,14 +54,12 @@ def get_permission_change_data(sender, permissions, **kwargs):
 
 def is_user_data_required(sender, request_user, user_data, **kwargs):
     """
-    Returns True if request user can use chat and user_data is required
+    If request_user can use chat, then returns all user ids that are
-    to be displayed as chatter.
+    displayed as chatter. Else, it returns an empty set.
     """
-    result = False
+    user_ids = set()
     if has_perm(request_user, 'core.can_use_chat'):
         for chat_message_collection_element in Collection(ChatMessage.get_collection_string()).element_generator():
             full_data = chat_message_collection_element.get_full_data()
-            if user_data['id'] == full_data['user_id']:
+            user_ids.add(full_data['user_id'])
-                result = True
+    return user_ids
-                break
-    return result

openslides/mediafiles/access_permissions.py

@@ -26,7 +26,13 @@ class MediafileAccessPermissions(BaseAccessPermissions):
         for the user.
         """
         data = None
-        if has_perm(user, 'mediafiles.can_see'):
+        if has_perm(user, 'mediafiles.can_see') and has_perm(user, 'mediafiles.can_see_hidden'):
-            if (not full_data['hidden'] or has_perm(user, 'mediafiles.can_see_hidden')):
             data = full_data
+        elif has_perm(user, 'mediafiles.can_see'):
+            many_items = not isinstance(full_data, dict)
+            full_data_list = full_data if many_items else [full_data]
+            data = [full_data for full_data in full_data_list if not full_data['hidden']]
+            data = data if many_items else data[0]
+        else:
+            data = None
         return data

openslides/mediafiles/signals.py

@@ -20,12 +20,13 @@ def is_user_data_required(sender, request_user, user_data, **kwargs):
     """
     Returns True if request user can see mediafiles and user_data is required
     to be displayed as uploader.
+
+    If request_user can see mediafiles, then returns all user ids that are
+    displayed as uploader. Else, it returns an empty set.
     """
-    result = False
+    user_ids = set()
     if has_perm(request_user, 'mediafiles.can_see'):
         for mediafile_collection_element in Collection(Mediafile.get_collection_string()).element_generator():
             full_data = mediafile_collection_element.get_full_data()
-            if user_data['id'] == full_data['uploader_id']:
+            user_ids.add(full_data['uploader_id'])
-                result = True
+    return user_ids
-                break
-    return result

openslides/motions/access_permissions.py

@@ -1,7 +1,5 @@
 from copy import deepcopy
 
-from django.contrib.auth import get_user_model
-
 from ..core.config import config
 from ..utils.access_permissions import BaseAccessPermissions
 from ..utils.auth import has_perm
@@ -33,13 +31,10 @@ class MotionAccessPermissions(BaseAccessPermissions):
         the motion in this state. Removes non public comment fields for
         some unauthorized users.
         """
-        if isinstance(user, get_user_model()):
+        many_items = not isinstance(full_data, dict)
-            # Converts a user object to a collection element.
+        full_data_list = full_data if many_items else [full_data]
-            # from_instance can not be used because the user serializer loads
+        out = []
-            # the group from the db. So each call to from_instance(user) consts
+        for full_data in full_data_list:
-            # one db query.
-            user = CollectionElement.from_values('users/user', user.id)
-
             if isinstance(user, CollectionElement):
                 is_submitter = user.get_full_data()['id'] in full_data.get('submitters_id', [])
             else:
@@ -72,7 +67,8 @@ class MotionAccessPermissions(BaseAccessPermissions):
                             if personal_note.get('user_id') == user.id:
                                 data['personal_notes'].append(personal_note)
                                 break
-        return data
+            out.append(data)
+        return out if many_items else out[0]
 
     def get_projector_data(self, full_data):
         """

openslides/motions/signals.py

@@ -120,14 +120,13 @@ def get_permission_change_data(sender, permissions, **kwargs):
 
 def is_user_data_required(sender, request_user, user_data, **kwargs):
     """
-    Returns True if request user can see motions and user_data is required
+    If request_user can see motions, then returns all user ids that are
-    to be displayed as motion submitter or supporter.
+    displayed as submitter or supporter. Else, it returns an empty set.
     """
-    result = False
+    user_ids = set()
     if has_perm(request_user, 'motions.can_see'):
         for motion_collection_element in Collection(Motion.get_collection_string()).element_generator():
             full_data = motion_collection_element.get_full_data()
-            if user_data['id'] in full_data['submitters_id'] or user_data['id'] in full_data['supporters_id']:
+            user_ids.update(full_data['submitters_id'])
-                result = True
+            user_ids.update(full_data['supporters_id'])
-                break
+    return user_ids
-    return result

openslides/users/access_permissions.py

@@ -31,57 +31,56 @@ class UserAccessPermissions(BaseAccessPermissions):
         """
         from .serializers import USERCANSEESERIALIZER_FIELDS, USERCANSEEEXTRASERIALIZER_FIELDS
 
-        NO_DATA = 0
+        def filtered_data(full_data, only_keys):
-        LITTLE_DATA = 1
+            """
-        MANY_DATA = 2
+            Returns a new dict like full_data but with all blocked_keys removed.
-        FULL_DATA = 3
+            """
+            return {key: full_data[key] for key in only_keys}
+
+        # many_items is True, when there are more then one items in full_data.
+        many_items = not isinstance(full_data, dict)
+        full_data = full_data if many_items else [full_data]
+
+        many_fields = set(USERCANSEEEXTRASERIALIZER_FIELDS)
+        little_fields = set(USERCANSEESERIALIZER_FIELDS)
+        many_fields.add('groups_id')
+        many_fields.discard('groups')
+        little_fields.add('groups_id')
+        little_fields.discard('groups')
 
         # Check user permissions.
         if has_perm(user, 'users.can_see_name'):
             if has_perm(user, 'users.can_see_extra_data'):
                 if has_perm(user, 'users.can_manage'):
-                    case = FULL_DATA
+                    data = full_data
                 else:
-                    case = MANY_DATA
+                    data = [filtered_data(full, many_fields) for full in full_data]
             else:
-                case = LITTLE_DATA
+                data = [filtered_data(full, little_fields) for full in full_data]
-        elif user is not None and user.id == full_data.get('id'):
-            # An authenticated user without the permission to see users tries
-            # to see himself.
-            case = LITTLE_DATA
         else:
-            # Now check if the user to be sent out is required by any app e. g.
+            # Build a list of users, that can be seen without permissions.
-            # as motion submitter or assignment candidate.
+            no_perm_users = set()
+            if user is not None:
+                no_perm_users.add(user.id)
+
+            # Get a list of all users, that are needed by another app
             receiver_responses = user_data_required.send(
                 sender=self.__class__,
                 request_user=user,
                 user_data=full_data)
             for receiver, response in receiver_responses:
-                if response:
+                no_perm_users.update(response)
-                    case = LITTLE_DATA
-                    break
-            else:
-                case = NO_DATA
 
-        # Setup data.
+            data = [
-        if case == FULL_DATA:
+                filtered_data(full, little_fields)
-            data = full_data
+                for full
-        elif case == NO_DATA:
+                in full_data
-            data = None
+                if full['id'] in no_perm_users]
-        else:
+
-            # case in (LITTLE_DATA, ḾANY_DATA)
+            # Set data to [None] if data is empty
-            if case == MANY_DATA:
+            data = data or [None]
-                fields = USERCANSEEEXTRASERIALIZER_FIELDS
+
-            else:
+        return data if many_items else data[0]
-                # case == LITTLE_DATA
-                fields = USERCANSEESERIALIZER_FIELDS
-            # Let only some fields pass this method.
-            data = {}
-            for base_key in fields:
-                for key in (base_key, base_key + '_id'):
-                    if key in full_data.keys():
-                        data[key] = full_data[key]
-        return data
 
     def get_projector_data(self, full_data):
         """

openslides/users/migrations/0004_personalnote.py

@@ -3,7 +3,6 @@
 from __future__ import unicode_literals
 
 import django.db.models.deletion
-
 from django.conf import settings
 from django.db import migrations, models
 

openslides/utils/access_permissions.py

@@ -61,19 +61,26 @@ class BaseAccessPermissions(object, metaclass=SignalConnectMetaClass):
         Returns the restricted serialized data for the instance prepared
         for the user.
 
-        Returns None if the user has no read access. Returns reduced data
+        Returns None or an empty list if the user has no read access. Returns
-        if the user has limited access. Default: Returns full data if the
+        reduced data if the user has limited access. Default: Returns full data
-        user has read access to model instances.
+        if the user has read access to model instances.
 
         Hint: You should override this method if your get_serializer_class()
         method returns different serializers for different users or if you
         have access restrictions in your view or viewset in methods like
         retrieve() or list().
+
+        full_data can be a list or one single item. If it is a list, then the
+        retun value is also a list of restricted data.
+
+        When the user can not access
         """
         if self.check_permissions(user):
             data = full_data
-        else:
+        elif isinstance(full_data, dict):
             data = None
+        else:
+            data = []
         return data
 
     def get_projector_data(self, full_data):

openslides/utils/autoupdate.py

@@ -6,15 +6,20 @@ from collections import Iterable, defaultdict
 from channels import Channel, Group
 from channels.asgi import get_channel_layer
 from channels.auth import channel_session_user, channel_session_user_from_http
-from django.apps import apps
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import transaction
 
 from ..core.config import config
 from ..core.models import Projector
 from .auth import has_perm, user_to_collection_user
-from .cache import websocket_user_cache
+from .cache import start_up_cache, websocket_user_cache
-from .collection import Collection, CollectionElement, CollectionElementList
+from .collection import (
+    Collection,
+    CollectionElement,
+    CollectionElementList,
+    format_for_autoupdate,
+    get_model_from_collection_string,
+)
 
 
 def send_or_wait(send_func, *args, **kwargs):
@@ -61,18 +66,29 @@ def ws_add_site(message):
 
     # Collect all elements that shoud be send to the client when the websocket
     # connection is established
-    output = []
-    for app in apps.get_app_configs():
-        try:
-            # Get the method get_startup_elements() from an app.
-            # This method has to return an iterable of Collection objects.
-            get_startup_elements = app.get_startup_elements
-        except AttributeError:
-            # Skip apps that do not implement get_startup_elements
-            continue
-        for collection in get_startup_elements():
     user = user_to_collection_user(message.user.id)
-            output.extend(collection.as_autoupdate_for_user(user))
+    output = []
+    for collection_string, full_data in start_up_cache.get_collection_elements().items():
+        access_permission = get_model_from_collection_string(collection_string).get_access_permissions()
+        if collection_string == 'core/config':
+            id_key = 'key'
+        else:
+            id_key = 'id'
+
+        restricted_data = access_permission.get_restricted_data(full_data, user)
+        if restricted_data is None:
+            continue
+
+        for data in restricted_data:
+            if data is None:
+                continue
+
+            output.append(
+                format_for_autoupdate(
+                    collection_string=collection_string,
+                    id=data[id_key],
+                    action='changed',
+                    data=data))
 
     # Send all data. If there is no data, then only accept the connection
     if output:
@@ -341,6 +357,7 @@ def send_autoupdate(collection_elements):
     Does nothing if collection_elements is empty.
     """
     if collection_elements:
+        start_up_cache.clear()
         send_or_wait(
             Channel('autoupdate.send_data').send,
             collection_elements.as_channels_message())

openslides/utils/cache.py

@@ -2,6 +2,7 @@ from collections import defaultdict
 
 from channels import Group
 from channels.sessions import session_for_reply_channel
+from django.apps import apps
 from django.core.cache import cache, caches
 
 
@@ -184,6 +185,65 @@ class DjangoCacheWebsocketUserCache(BaseWebsocketUserCache):
         cache.set(self.get_cache_key(), data)
 
 
+class StartupCache:
+    """
+    Cache of all data that are needed when a clients connects via websocket.
+    """
+
+    cache_key = "full_data_cache"
+
+    def build(self):
+        """
+        Generate the cache by going though all apps.
+        Returns a dict where the key is the collection string and the value a
+        list of the full_data from the collection_elements.
+        """
+        cache_data = {}
+        for app in apps.get_app_configs():
+            try:
+                # Get the method get_startup_elements() from an app.
+                # This method has to return an iterable of Collection objects.
+                get_startup_elements = app.get_startup_elements
+            except AttributeError:
+                # Skip apps that do not implement get_startup_elements
+                continue
+
+            for collection in get_startup_elements():
+                cache_data[collection.collection_string] = [
+                    collection_element.get_full_data()
+                    for collection_element
+                    in collection.element_generator()]
+
+        cache.set(self.cache_key, cache_data, 86400)
+        return cache_data
+
+    def clear(self):
+        """
+        Clears the cache.
+        """
+        cache.delete(self.cache_key)
+
+    def get_collection_elements(self):
+        """
+        Returns a dict of all collection_elements, that are needed at startup.
+
+        The key is the collection_string and the value a list of CollectionElement
+        objects. Each list has at least one value.
+
+        The data is read from the cache if it exists. It builds the cache, if it
+        does not exists.
+        """
+        data = cache.get(self.cache_key)
+        if data is None:
+            # The cache does not exist.
+            data = self.build()
+
+        return data
+
+
+start_up_cache = StartupCache()
+
+
 def use_redis_cache():
     """
     Returns True if Redis is used als caching backend.

openslides/utils/collection.py

@@ -100,21 +100,18 @@ class CollectionElement:
         Only for internal use. Do not use it directly. Use as_autoupdate_for_user()
         or as_autoupdate_for_projector().
         """
-        output = {
-            'collection': self.collection_string,
-            'id': self.id,
-            'action': 'deleted' if self.is_deleted() else 'changed',
-        }
         if not self.is_deleted():
             data = getattr(self.get_access_permissions(), method)(
                 self.get_full_data(),
                 *args)
-            if data is None:
-                # The user is not allowed to see this element. Set action to deleted.
-                output['action'] = 'deleted'
         else:
-                output['data'] = data
+            data = None
-        return output
+
+        return format_for_autoupdate(
+            collection_string=self.collection_string,
+            id=self.id,
+            action='deleted' if self.is_deleted() else 'changed',
+            data=data)
 
     def as_autoupdate_for_user(self, user):
         """
@@ -551,3 +548,25 @@ def get_collection_id_from_cache_key(cache_key):
         # The id is no integer. This can happen on config elements
         pass
     return (collection_string, id)
+
+
+def format_for_autoupdate(collection_string, id, action, data=None):
+    """
+    Returns a dict that can be used for autoupdate.
+    """
+    if not data:
+        # If the data is None or is empty, then the action has to be deleted,
+        # even when it says diffrently. This can happen when the object is not
+        # deleted, but the user has no permission to see it.
+        action = 'deleted'
+
+    output = {
+        'collection': collection_string,
+        'id': id,
+        'action': action,
+    }
+
+    if action != 'deleted':
+        output['data'] = data
+
+    return output