Merge pull request #3223 from ostcar/many_restricted_data

Speed up WS connection
This commit is contained in:
Norman Jäckel 2017-05-03 09:06:55 +02:00 committed by GitHub
commit 933a5ba13f
17 changed files with 432 additions and 208 deletions

View File

@ -42,6 +42,8 @@ Core:
pdf [#3184, #3207, #3208]. pdf [#3184, #3207, #3208].
- Fixing error when clearing empty chat [#3199]. - Fixing error when clearing empty chat [#3199].
- Added notify system [#3212]. - Added notify system [#3212].
- Enhanced performance esp. for server restart and first connection of all
clients by refactorting autoupdate, Collection and AccessPermission [#3223].
General: General:
- Switched from npm to Yarn [#3188]. - Switched from npm to Yarn [#3188].

View File

@ -1,5 +1,6 @@
from ..utils.access_permissions import BaseAccessPermissions from ..utils.access_permissions import BaseAccessPermissions
from ..utils.auth import has_perm from ..utils.auth import has_perm
from ..utils.collection import Collection
class ItemAccessPermissions(BaseAccessPermissions): class ItemAccessPermissions(BaseAccessPermissions):
@ -22,37 +23,71 @@ class ItemAccessPermissions(BaseAccessPermissions):
# TODO: In the following method we use full_data['is_hidden'] but this can be out of date. # TODO: In the following method we use full_data['is_hidden'] but this can be out of date.
def get_restricted_data(self, full_data, user): def get_restricted_data(self, container, user):
""" """
Returns the restricted serialized data for the instance prepared Returns the restricted serialized data for the instance prepared
for the user. for the user.
We remove comments for non admins/managers and a lot of fields of
hidden items for users without permission to see hidden items.
""" """
def filtered_data(full_data, blocked_keys):
"""
Returns a new dict like full_data but with all blocked_keys removed.
"""
whitelist = full_data.keys() - blocked_keys
return {key: full_data[key] for key in whitelist}
# Expand full_data to a list if it is not one.
full_data = container.get_full_data() if isinstance(container, Collection) else [container.get_full_data()]
# Parse data.
if has_perm(user, 'agenda.can_see'): if has_perm(user, 'agenda.can_see'):
if full_data['is_hidden'] and not has_perm(user, 'agenda.can_see_hidden_items'): if has_perm(user, 'agenda.can_manage') and has_perm(user, 'agenda.can_see_hidden_items'):
# The data is hidden but the user isn't allowed to see it. Jst pass # Managers with special permission can see everything.
# the whitelisted keys so the list of speakers is provided regardless. data = full_data
whitelist = ( elif has_perm(user, 'agenda.can_see_hidden_items'):
# Non managers with special permission can see everything but comments.
blocked_keys = ('comment',)
data = [filtered_data(full, blocked_keys) for full in full_data]
else:
# Users without special permissin for hidden items.
# In hidden case managers and non managers see only some fields
# so that list of speakers is provided regardless.
blocked_keys_hidden_case = full_data[0].keys() - (
'id', 'id',
'title', 'title',
'speakers', 'speakers',
'speaker_list_closed', 'speaker_list_closed',
'content_object',) 'content_object')
data = {}
for key in full_data.keys(): # In non hidden case managers see everything and non managers see
if key in whitelist: # everything but comments.
data[key] = full_data[key]
else:
if has_perm(user, 'agenda.can_manage'): if has_perm(user, 'agenda.can_manage'):
data = full_data blocked_keys_non_hidden_case = []
else: else:
# Strip out item comments for unprivileged users. blocked_keys_non_hidden_case = ('comment',)
data = {}
for key in full_data.keys(): data = []
if key != 'comment': for full in full_data:
data[key] = full_data[key] if full['is_hidden']:
data.append(filtered_data(full, blocked_keys_hidden_case))
else: else:
data = None data.append(filtered_data(full, blocked_keys_non_hidden_case))
return data else:
data = []
# Reduce result to a single item or None if it was not a collection at
# the beginning of the method.
if isinstance(container, Collection):
restricted_data = data
elif data:
restricted_data = data[0]
else:
restricted_data = None
return restricted_data
def get_projector_data(self, full_data): def get_projector_data(self, full_data):
""" """

View File

@ -54,20 +54,15 @@ def get_permission_change_data(sender, permissions, **kwargs):
break break
def is_user_data_required(sender, request_user, user_data, **kwargs): def is_user_data_required(sender, request_user, **kwargs):
""" """
Returns True if request user can see the agenda and user_data is required Returns all user ids that are displayed as speakers in any agenda item
to be displayed as speaker. if request_user can see the agenda. This function may return an empty
set.
""" """
result = False speakers = set()
if has_perm(request_user, 'agenda.can_see'): if has_perm(request_user, 'agenda.can_see'):
for item_collection_element in Collection(Item.get_collection_string()).element_generator(): for item_collection_element in Collection(Item.get_collection_string()).element_generator():
full_data = item_collection_element.get_full_data() full_data = item_collection_element.get_full_data()
for speaker in full_data['speakers']: speakers.update(speaker['user_id'] for speaker in full_data['speakers'])
if user_data['id'] == speaker['user_id']: return speakers
result = True
break
else:
continue
break
return result

View File

@ -1,5 +1,6 @@
from ..utils.access_permissions import BaseAccessPermissions from ..utils.access_permissions import BaseAccessPermissions
from ..utils.auth import has_perm from ..utils.auth import has_perm
from ..utils.collection import Collection
class AssignmentAccessPermissions(BaseAccessPermissions): class AssignmentAccessPermissions(BaseAccessPermissions):
@ -24,20 +25,38 @@ class AssignmentAccessPermissions(BaseAccessPermissions):
serializer_class = AssignmentShortSerializer serializer_class = AssignmentShortSerializer
return serializer_class return serializer_class
def get_restricted_data(self, full_data, user): def get_restricted_data(self, container, user):
""" """
Returns the restricted serialized data for the instance prepared Returns the restricted serialized data for the instance prepared
for the user. Removes unpublished polls for non admins so that they for the user. Removes unpublished polls for non admins so that they
only get a result like the AssignmentShortSerializer would give them. only get a result like the AssignmentShortSerializer would give them.
""" """
# Expand full_data to a list if it is not one.
full_data = container.get_full_data() if isinstance(container, Collection) else [container.get_full_data()]
# Parse data.
if has_perm(user, 'assignments.can_see') and has_perm(user, 'assignments.can_manage'): if has_perm(user, 'assignments.can_see') and has_perm(user, 'assignments.can_manage'):
data = full_data data = full_data
elif has_perm(user, 'assignments.can_see'): elif has_perm(user, 'assignments.can_see'):
data = full_data.copy() # Exclude unpublished polls.
data['polls'] = [poll for poll in data['polls'] if poll['published']] data = []
for full in full_data:
full_copy = full.copy()
full_copy['polls'] = [poll for poll in full['polls'] if poll['published']]
data.append(full_copy)
else: else:
data = None data = []
return data
# Reduce result to a single item or None if it was not a collection at
# the beginning of the method.
if isinstance(container, Collection):
restricted_data = data
elif data:
restricted_data = data[0]
else:
restricted_data = None
return restricted_data
def get_projector_data(self, full_data): def get_projector_data(self, full_data):
""" """

View File

@ -16,30 +16,17 @@ def get_permission_change_data(sender, permissions=None, **kwargs):
yield from assignments_app.get_startup_elements() yield from assignments_app.get_startup_elements()
def is_user_data_required(sender, request_user, user_data, **kwargs): def is_user_data_required(sender, request_user, **kwargs):
""" """
Returns True if request user can see assignments and user_data is required Returns all user ids that are displayed as candidates (including poll
to be displayed as candidates (including poll options). options) in any assignment if request_user can see assignments. This
function may return an empty set.
""" """
result = False candidates = set()
if has_perm(request_user, 'assignments.can_see'): if has_perm(request_user, 'assignments.can_see'):
for assignment_collection_element in Collection(Assignment.get_collection_string()).element_generator(): for assignment_collection_element in Collection(Assignment.get_collection_string()).element_generator():
full_data = assignment_collection_element.get_full_data() full_data = assignment_collection_element.get_full_data()
for related_user in full_data['assignment_related_users']: candidates.update(related_user['user_id'] for related_user in full_data['assignment_related_users'])
if user_data['id'] == related_user['user_id']:
result = True
break
else:
for poll in full_data['polls']: for poll in full_data['polls']:
for option in poll['options']: candidates.update(option['candidate_id'] for option in poll['options'])
if user_data['id'] == option['candidate_id']: return candidates
result = True
break
else:
continue
break
else:
continue
break
break
return result

View File

@ -52,16 +52,14 @@ def get_permission_change_data(sender, permissions, **kwargs):
yield Collection(core_app.get_model('ChatMessage').get_collection_string()) yield Collection(core_app.get_model('ChatMessage').get_collection_string())
def is_user_data_required(sender, request_user, user_data, **kwargs): def is_user_data_required(sender, request_user, **kwargs):
""" """
Returns True if request user can use chat and user_data is required Returns all user ids that are displayed as chatters if request_user can
to be displayed as chatter. use the chat. This function may return an empty set.
""" """
result = False chatters = set()
if has_perm(request_user, 'core.can_use_chat'): if has_perm(request_user, 'core.can_use_chat'):
for chat_message_collection_element in Collection(ChatMessage.get_collection_string()).element_generator(): for chat_message_collection_element in Collection(ChatMessage.get_collection_string()).element_generator():
full_data = chat_message_collection_element.get_full_data() full_data = chat_message_collection_element.get_full_data()
if user_data['id'] == full_data['user_id']: chatters.add(full_data['user_id'])
result = True return chatters
break
return result

View File

@ -1,5 +1,6 @@
from ..utils.access_permissions import BaseAccessPermissions from ..utils.access_permissions import BaseAccessPermissions
from ..utils.auth import has_perm from ..utils.auth import has_perm
from ..utils.collection import Collection
class MediafileAccessPermissions(BaseAccessPermissions): class MediafileAccessPermissions(BaseAccessPermissions):
@ -20,13 +21,30 @@ class MediafileAccessPermissions(BaseAccessPermissions):
return MediafileSerializer return MediafileSerializer
def get_restricted_data(self, full_data, user): def get_restricted_data(self, container, user):
""" """
Returns the restricted serialized data for the instance prepared Returns the restricted serialized data for the instance prepared
for the user. for the user. Removes hidden mediafiles for some users.
""" """
data = None # Expand full_data to a list if it is not one.
if has_perm(user, 'mediafiles.can_see'): full_data = container.get_full_data() if isinstance(container, Collection) else [container.get_full_data()]
if (not full_data['hidden'] or has_perm(user, 'mediafiles.can_see_hidden')):
# Parse data.
if has_perm(user, 'mediafiles.can_see') and has_perm(user, 'mediafiles.can_see_hidden'):
data = full_data data = full_data
return data elif has_perm(user, 'mediafiles.can_see'):
# Exclude hidden mediafiles.
data = [full for full in full_data if not full['hidden']]
else:
data = []
# Reduce result to a single item or None if it was not a collection at
# the beginning of the method.
if isinstance(container, Collection):
restricted_data = data
elif data:
restricted_data = data[0]
else:
restricted_data = None
return restricted_data

View File

@ -16,16 +16,15 @@ def get_permission_change_data(sender, permissions=None, **kwargs):
yield from mediafiles_app.get_startup_elements() yield from mediafiles_app.get_startup_elements()
def is_user_data_required(sender, request_user, user_data, **kwargs): def is_user_data_required(sender, request_user, **kwargs):
""" """
Returns True if request user can see mediafiles and user_data is required Returns all user ids that are displayed as uploaders in any mediafile
to be displayed as uploader. if request_user can see mediafiles. This function may return an empty
set.
""" """
result = False uploaders = set()
if has_perm(request_user, 'mediafiles.can_see'): if has_perm(request_user, 'mediafiles.can_see'):
for mediafile_collection_element in Collection(Mediafile.get_collection_string()).element_generator(): for mediafile_collection_element in Collection(Mediafile.get_collection_string()).element_generator():
full_data = mediafile_collection_element.get_full_data() full_data = mediafile_collection_element.get_full_data()
if user_data['id'] == full_data['uploader_id']: uploaders.add(full_data['uploader_id'])
result = True return uploaders
break
return result

View File

@ -1,11 +1,9 @@
from copy import deepcopy from copy import deepcopy
from django.contrib.auth import get_user_model
from ..core.config import config from ..core.config import config
from ..utils.access_permissions import BaseAccessPermissions from ..utils.access_permissions import BaseAccessPermissions
from ..utils.auth import has_perm from ..utils.auth import has_perm
from ..utils.collection import CollectionElement from ..utils.collection import Collection, CollectionElement
class MotionAccessPermissions(BaseAccessPermissions): class MotionAccessPermissions(BaseAccessPermissions):
@ -26,53 +24,77 @@ class MotionAccessPermissions(BaseAccessPermissions):
return MotionSerializer return MotionSerializer
def get_restricted_data(self, full_data, user): def get_restricted_data(self, container, user):
""" """
Returns the restricted serialized data for the instance prepared for Returns the restricted serialized data for the instance prepared for
the user. Removes motion if the user has not the permission to see the user. Removes motion if the user has not the permission to see
the motion in this state. Removes non public comment fields for the motion in this state. Removes non public comment fields for
some unauthorized users. some unauthorized users. Ensures that a user can only see his own
personal notes.
""" """
if isinstance(user, get_user_model()): # Expand full_data to a list if it is not one.
# Converts a user object to a collection element. full_data = container.get_full_data() if isinstance(container, Collection) else [container.get_full_data()]
# from_instance can not be used because the user serializer loads
# the group from the db. So each call to from_instance(user) consts
# one db query.
user = CollectionElement.from_values('users/user', user.id)
# Parse data.
if has_perm(user, 'motions.can_see'):
# TODO: Refactor this after personal_notes system is refactored.
data = []
for full in full_data:
# Check if user is submitter of this motion.
if isinstance(user, CollectionElement): if isinstance(user, CollectionElement):
is_submitter = user.get_full_data()['id'] in full_data.get('submitters_id', []) is_submitter = user.get_full_data()['id'] in full.get('submitters_id', [])
else: else:
# Anonymous users can not be submitters # Anonymous users can not be submitters.
is_submitter = False is_submitter = False
required_permission_to_see = full_data['state_required_permission_to_see'] # Check see permission for this motion.
data = None required_permission_to_see = full['state_required_permission_to_see']
if has_perm(user, 'motions.can_see'): permission = (
if (not required_permission_to_see or not required_permission_to_see or
has_perm(user, required_permission_to_see) or has_perm(user, required_permission_to_see) or
has_perm(user, 'motions.can_manage') or has_perm(user, 'motions.can_manage') or
is_submitter): is_submitter)
if has_perm(user, 'motions.can_see_and_manage_comments') or not full_data.get('comments'):
data = full_data # Parse single motion.
if permission:
if has_perm(user, 'motions.can_see_and_manage_comments') or not full.get('comments'):
# Provide access to all fields.
motion = full
else: else:
data = deepcopy(full_data) # Set private comment fields to None.
full_copy = deepcopy(full)
for i, field in enumerate(config['motions_comments']): for i, field in enumerate(config['motions_comments']):
if not field.get('public'): if not field.get('public'):
try: try:
data['comments'][i] = None full_copy['comments'][i] = None
except IndexError: except IndexError:
# No data in range. Just do nothing. # No data in range. Just do nothing.
pass pass
motion = full_copy
# Now filter personal notes. # Now filter personal notes.
data = data.copy() motion = motion.copy()
data['personal_notes'] = [] motion['personal_notes'] = []
if user is not None: if user is not None:
for personal_note in full_data.get('personal_notes', []): for personal_note in full.get('personal_notes', []):
if personal_note.get('user_id') == user.id: if personal_note.get('user_id') == user.id:
data['personal_notes'].append(personal_note) motion['personal_notes'].append(personal_note)
break break
return data
data.append(motion)
else:
data = []
# Reduce result to a single item or None if it was not a collection at
# the beginning of the method.
if isinstance(container, Collection):
restricted_data = data
elif data:
restricted_data = data[0]
else:
restricted_data = None
return restricted_data
def get_projector_data(self, full_data): def get_projector_data(self, full_data):
""" """

View File

@ -118,16 +118,16 @@ def get_permission_change_data(sender, permissions, **kwargs):
yield from motions_app.get_startup_elements() yield from motions_app.get_startup_elements()
def is_user_data_required(sender, request_user, user_data, **kwargs): def is_user_data_required(sender, request_user, **kwargs):
""" """
Returns True if request user can see motions and user_data is required Returns all user ids that are displayed as as submitter or supporter in
to be displayed as motion submitter or supporter. any motion if request_user can see motions. This function may return an
empty set.
""" """
result = False submitters_supporters = set()
if has_perm(request_user, 'motions.can_see'): if has_perm(request_user, 'motions.can_see'):
for motion_collection_element in Collection(Motion.get_collection_string()).element_generator(): for motion_collection_element in Collection(Motion.get_collection_string()).element_generator():
full_data = motion_collection_element.get_full_data() full_data = motion_collection_element.get_full_data()
if user_data['id'] in full_data['submitters_id'] or user_data['id'] in full_data['supporters_id']: submitters_supporters.update(full_data['submitters_id'])
result = True submitters_supporters.update(full_data['supporters_id'])
break return submitters_supporters
return result

View File

@ -3,6 +3,7 @@ from django.contrib.auth.models import AnonymousUser
from ..core.signals import user_data_required from ..core.signals import user_data_required
from ..utils.access_permissions import BaseAccessPermissions from ..utils.access_permissions import BaseAccessPermissions
from ..utils.auth import anonymous_is_enabled, has_perm from ..utils.auth import anonymous_is_enabled, has_perm
from ..utils.collection import Collection
class UserAccessPermissions(BaseAccessPermissions): class UserAccessPermissions(BaseAccessPermissions):
@ -23,7 +24,7 @@ class UserAccessPermissions(BaseAccessPermissions):
return UserFullSerializer return UserFullSerializer
def get_restricted_data(self, full_data, user): def get_restricted_data(self, container, user):
""" """
Returns the restricted serialized data for the instance prepared Returns the restricted serialized data for the instance prepared
for the user. Removes several fields for non admins so that they do for the user. Removes several fields for non admins so that they do
@ -31,57 +32,75 @@ class UserAccessPermissions(BaseAccessPermissions):
""" """
from .serializers import USERCANSEESERIALIZER_FIELDS, USERCANSEEEXTRASERIALIZER_FIELDS from .serializers import USERCANSEESERIALIZER_FIELDS, USERCANSEEEXTRASERIALIZER_FIELDS
NO_DATA = 0 def filtered_data(full_data, whitelist):
LITTLE_DATA = 1 """
MANY_DATA = 2 Returns a new dict like full_data but only with whitelisted keys.
FULL_DATA = 3 """
return {key: full_data[key] for key in whitelist}
# Expand full_data to a list if it is not one.
full_data = container.get_full_data() if isinstance(container, Collection) else [container.get_full_data()]
# We have four sets of data to be sent:
# * full data i. e. all fields,
# * many data i. e. all fields but not the default password,
# * little data i. e. all fields but not the default password, comments and active status,
# * no data.
# Prepare field set for users with "many" data and with "little" data.
many_data_fields = set(USERCANSEEEXTRASERIALIZER_FIELDS)
many_data_fields.add('groups_id')
many_data_fields.discard('groups')
litte_data_fields = set(USERCANSEESERIALIZER_FIELDS)
litte_data_fields.add('groups_id')
litte_data_fields.discard('groups')
# Check user permissions. # Check user permissions.
if has_perm(user, 'users.can_see_name'): if has_perm(user, 'users.can_see_name'):
if has_perm(user, 'users.can_see_extra_data'): if has_perm(user, 'users.can_see_extra_data'):
if has_perm(user, 'users.can_manage'): if has_perm(user, 'users.can_manage'):
case = FULL_DATA data = full_data
else: else:
case = MANY_DATA data = [filtered_data(full, many_data_fields) for full in full_data]
else: else:
case = LITTLE_DATA data = [filtered_data(full, litte_data_fields) for full in full_data]
elif user is not None and user.id == full_data.get('id'):
# An authenticated user without the permission to see users tries
# to see himself.
case = LITTLE_DATA
else: else:
# Now check if the user to be sent out is required by any app e. g. # Build a list of users, that can be seen without any permissions (with little fields).
# as motion submitter or assignment candidate.
user_ids = set()
# Everybody can see himself. Also everybody can see every user
# that is required e. g. as speaker, motion submitter or
# assignment candidate.
# Add oneself.
if user is not None:
user_ids.add(user.id)
# Get a list of all users, that are required by another app.
receiver_responses = user_data_required.send( receiver_responses = user_data_required.send(
sender=self.__class__, sender=self.__class__,
request_user=user, request_user=user)
user_data=full_data)
for receiver, response in receiver_responses: for receiver, response in receiver_responses:
if response: user_ids.update(response)
case = LITTLE_DATA
break
else:
case = NO_DATA
# Setup data. # Parse data.
if case == FULL_DATA: data = [
data = full_data filtered_data(full, litte_data_fields)
elif case == NO_DATA: for full
data = None in full_data
if full['id'] in user_ids]
# Reduce result to a single item or None if it was not a collection at
# the beginning of the method.
if isinstance(container, Collection):
restricted_data = data
elif data:
restricted_data = data[0]
else: else:
# case in (LITTLE_DATA, ḾANY_DATA) restricted_data = None
if case == MANY_DATA:
fields = USERCANSEEEXTRASERIALIZER_FIELDS return restricted_data
else:
# case == LITTLE_DATA
fields = USERCANSEESERIALIZER_FIELDS
# Let only some fields pass this method.
data = {}
for base_key in fields:
for key in (base_key, base_key + '_id'):
if key in full_data.keys():
data[key] = full_data[key]
return data
def get_projector_data(self, full_data): def get_projector_data(self, full_data):
""" """

View File

@ -3,7 +3,6 @@
from __future__ import unicode_literals from __future__ import unicode_literals
import django.db.models.deletion import django.db.models.deletion
from django.conf import settings from django.conf import settings
from django.db import migrations, models from django.db import migrations, models

View File

@ -1,5 +1,6 @@
from django.dispatch import Signal from django.dispatch import Signal
from .collection import Collection
from .dispatch import SignalConnectMetaClass from .dispatch import SignalConnectMetaClass
@ -56,14 +57,17 @@ class BaseAccessPermissions(object, metaclass=SignalConnectMetaClass):
""" """
return self.get_serializer_class(user=None)(instance).data return self.get_serializer_class(user=None)(instance).data
def get_restricted_data(self, full_data, user): def get_restricted_data(self, container, user):
""" """
Returns the restricted serialized data for the instance prepared Returns the restricted serialized data for the instance prepared
for the user. for the user.
Returns None if the user has no read access. Returns reduced data The argument container should be a CollectionElement or a
if the user has limited access. Default: Returns full data if the Collection. The type of the return value is a dictionary or a list
user has read access to model instances. according to the given type (or None). Returns None or an empty
list if the user has no read access. Returns reduced data if the
user has limited access. Default: Returns full data if the user has
read access to model instances.
Hint: You should override this method if your get_serializer_class() Hint: You should override this method if your get_serializer_class()
method returns different serializers for different users or if you method returns different serializers for different users or if you
@ -71,7 +75,9 @@ class BaseAccessPermissions(object, metaclass=SignalConnectMetaClass):
retrieve() or list(). retrieve() or list().
""" """
if self.check_permissions(user): if self.check_permissions(user):
data = full_data data = container.get_full_data()
elif isinstance(container, Collection):
data = []
else: else:
data = None data = None
return data return data

View File

@ -6,14 +6,13 @@ from collections import Iterable, defaultdict
from channels import Channel, Group from channels import Channel, Group
from channels.asgi import get_channel_layer from channels.asgi import get_channel_layer
from channels.auth import channel_session_user, channel_session_user_from_http from channels.auth import channel_session_user, channel_session_user_from_http
from django.apps import apps
from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ObjectDoesNotExist
from django.db import transaction from django.db import transaction
from ..core.config import config from ..core.config import config
from ..core.models import Projector from ..core.models import Projector
from .auth import has_perm, user_to_collection_user from .auth import has_perm, user_to_collection_user
from .cache import websocket_user_cache from .cache import startup_cache, websocket_user_cache
from .collection import Collection, CollectionElement, CollectionElementList from .collection import Collection, CollectionElement, CollectionElementList
@ -42,6 +41,28 @@ def send_or_wait(send_func, *args, **kwargs):
) )
def format_for_autoupdate(collection_string, id, action, data=None):
"""
Returns a dict that can be used for autoupdate.
"""
if not data:
# If the data is None or is empty, then the action has to be deleted,
# even when it says diffrently. This can happen when the object is not
# deleted, but the user has no permission to see it.
action = 'deleted'
output = {
'collection': collection_string,
'id': id,
'action': action,
}
if action != 'deleted':
output['data'] = data
return output
@channel_session_user_from_http @channel_session_user_from_http
def ws_add_site(message): def ws_add_site(message):
""" """
@ -60,21 +81,31 @@ def ws_add_site(message):
send_or_wait(message.reply_channel.send, {'accept': True}) send_or_wait(message.reply_channel.send, {'accept': True})
# Collect all elements that shoud be send to the client when the websocket # Collect all elements that shoud be send to the client when the websocket
# connection is established # connection is established.
output = []
for app in apps.get_app_configs():
try:
# Get the method get_startup_elements() from an app.
# This method has to return an iterable of Collection objects.
get_startup_elements = app.get_startup_elements
except AttributeError:
# Skip apps that do not implement get_startup_elements
continue
for collection in get_startup_elements():
user = user_to_collection_user(message.user.id) user = user_to_collection_user(message.user.id)
output.extend(collection.as_autoupdate_for_user(user)) output = []
for collection in startup_cache.get_collections():
access_permissions = collection.get_access_permissions()
restricted_data = access_permissions.get_restricted_data(collection, user)
# Send all data. If there is no data, then only accept the connection if collection.collection_string == 'core/config':
id_key = 'key'
else:
id_key = 'id'
for data in restricted_data:
if data is None:
# We do not want to send 'deleted' objects on startup.
# That's why we skip such data.
continue
output.append(
format_for_autoupdate(
collection_string=collection.collection_string,
id=data[id_key],
action='changed',
data=data))
# Send all data.
if output: if output:
send_or_wait(message.reply_channel.send, {'text': json.dumps(output)}) send_or_wait(message.reply_channel.send, {'text': json.dumps(output)})
@ -338,9 +369,13 @@ def send_autoupdate(collection_elements):
Helper function, that sends collection_elements through a channel to the Helper function, that sends collection_elements through a channel to the
autoupdate system. autoupdate system.
Before sending the startup_cache is cleared because it is possibly out of
date.
Does nothing if collection_elements is empty. Does nothing if collection_elements is empty.
""" """
if collection_elements: if collection_elements:
startup_cache.clear()
send_or_wait( send_or_wait(
Channel('autoupdate.send_data').send, Channel('autoupdate.send_data').send,
collection_elements.as_channels_message()) collection_elements.as_channels_message())

View File

@ -2,6 +2,7 @@ from collections import defaultdict
from channels import Group from channels import Group
from channels.sessions import session_for_reply_channel from channels.sessions import session_for_reply_channel
from django.apps import apps
from django.core.cache import cache, caches from django.core.cache import cache, caches
@ -184,6 +185,62 @@ class DjangoCacheWebsocketUserCache(BaseWebsocketUserCache):
cache.set(self.get_cache_key(), data) cache.set(self.get_cache_key(), data)
class StartupCache:
"""
Cache of all data that are required when a client connects via websocket.
"""
cache_key = "full_data_startup_cache"
def build(self):
"""
Generate the cache by going through all apps. Returns a dict where the
key is the collection string and the value a list of the full_data from
the collection elements.
"""
cache_data = {}
for app in apps.get_app_configs():
try:
# Get the method get_startup_elements() from an app.
# This method has to return an iterable of Collection objects.
get_startup_elements = app.get_startup_elements
except AttributeError:
# Skip apps that do not implement get_startup_elements.
continue
for collection in get_startup_elements():
cache_data[collection.collection_string] = [
collection_element.get_full_data()
for collection_element
in collection.element_generator()]
cache.set(self.cache_key, cache_data, 86400)
return cache_data
def clear(self):
"""
Clears the cache.
"""
cache.delete(self.cache_key)
def get_collections(self):
"""
Generator that returns all cached Collections.
The data is read from the cache if it exists. It builds the cache if it
does not exists.
"""
from .collection import Collection
data = cache.get(self.cache_key)
if data is None:
# The cache does not exist.
data = self.build()
for collection_string, full_data in data.items():
yield Collection(collection_string, full_data)
startup_cache = StartupCache()
def use_redis_cache(): def use_redis_cache():
""" """
Returns True if Redis is used als caching backend. Returns True if Redis is used als caching backend.

View File

@ -100,21 +100,27 @@ class CollectionElement:
Only for internal use. Do not use it directly. Use as_autoupdate_for_user() Only for internal use. Do not use it directly. Use as_autoupdate_for_user()
or as_autoupdate_for_projector(). or as_autoupdate_for_projector().
""" """
output = { from .autoupdate import format_for_autoupdate
'collection': self.collection_string,
'id': self.id, # TODO: Revert this after get_projector_data is also enhanced like get_restricted_data.
'action': 'deleted' if self.is_deleted() else 'changed', if method == 'get_restricted_data':
} container = self
else:
container = self.get_full_data()
# End of hack
if not self.is_deleted(): if not self.is_deleted():
data = getattr(self.get_access_permissions(), method)( data = getattr(self.get_access_permissions(), method)(
self.get_full_data(), container,
*args) *args)
if data is None:
# The user is not allowed to see this element. Set action to deleted.
output['action'] = 'deleted'
else: else:
output['data'] = data data = None
return output
return format_for_autoupdate(
collection_string=self.collection_string,
id=self.id,
action='deleted' if self.is_deleted() else 'changed',
data=data)
def as_autoupdate_for_user(self, user): def as_autoupdate_for_user(self, user):
""" """
@ -143,9 +149,7 @@ class CollectionElement:
The argument `user` can be anything, that is allowd as argument for The argument `user` can be anything, that is allowd as argument for
utils.auth.has_perm(). utils.auth.has_perm().
""" """
return self.get_access_permissions().get_restricted_data( return self.get_access_permissions().get_restricted_data(self, user)
self.get_full_data(),
user)
def get_model(self): def get_model(self):
""" """
@ -286,8 +290,15 @@ class Collection:
Represents all elements of one collection. Represents all elements of one collection.
""" """
def __init__(self, collection_string): def __init__(self, collection_string, full_data=None):
"""
Initiates a Collection. A collection_string has to be given. If
full_data (a list of dictionaries) is not given the method
get_full_data() exposes all data by iterating over all
CollectionElements.
"""
self.collection_string = collection_string self.collection_string = collection_string
self.full_data = full_data
def get_cache_key(self, raw=False): def get_cache_key(self, raw=False):
""" """
@ -307,10 +318,18 @@ class Collection:
""" """
return get_model_from_collection_string(self.collection_string) return get_model_from_collection_string(self.collection_string)
def get_access_permissions(self):
"""
Returns the get_access_permissions object for the this collection.
"""
return self.get_model().get_access_permissions()
def element_generator(self): def element_generator(self):
""" """
Generator that yields all collection_elements of this collection. Generator that yields all collection_elements of this collection.
""" """
# TODO: This method should use self.full_data if it already exists.
# Get all cache keys. # Get all cache keys.
ids = self.get_all_ids() ids = self.get_all_ids()
cache_keys = [ cache_keys = [
@ -355,10 +374,23 @@ class Collection:
for instance in query.filter(pk__in=missing_ids): for instance in query.filter(pk__in=missing_ids):
yield CollectionElement.from_instance(instance) yield CollectionElement.from_instance(instance)
def get_full_data(self):
"""
Returns a list of dictionaries with full_data of all collection
elements.
"""
if self.full_data is None:
self.full_data = [
collection_element.get_full_data()
for collection_element
in self.element_generator()]
return self.full_data
def as_autoupdate_for_projector(self): def as_autoupdate_for_projector(self):
""" """
Returns a list of dictonaries to send them to the projector. Returns a list of dictonaries to send them to the projector.
""" """
# TODO: This method is only used in one case. Remove it.
output = [] output = []
for collection_element in self.element_generator(): for collection_element in self.element_generator():
content = collection_element.as_autoupdate_for_projector() content = collection_element.as_autoupdate_for_projector()
@ -374,6 +406,7 @@ class Collection:
The argument `user` can be anything, that is allowd as argument for The argument `user` can be anything, that is allowd as argument for
utils.auth.has_perm(). utils.auth.has_perm().
""" """
# TODO: This method is not used. Remove it.
output = [] output = []
for collection_element in self.element_generator(): for collection_element in self.element_generator():
content = collection_element.as_autoupdate_for_user(user) content = collection_element.as_autoupdate_for_user(user)

View File

@ -128,7 +128,7 @@ class TestCollectionElement(TestCase):
'id': 42, 'id': 42,
'action': 'changed', 'action': 'changed',
'data': 'restricted_data'}) 'data': 'restricted_data'})
collection_element.get_full_data.assert_called_once_with() collection_element.get_full_data.assert_not_called()
def test_as_autoupdate_for_user_no_permission(self): def test_as_autoupdate_for_user_no_permission(self):
with patch.object(collection.CollectionElement, 'get_full_data'): with patch.object(collection.CollectionElement, 'get_full_data'):
@ -143,7 +143,7 @@ class TestCollectionElement(TestCase):
{'collection': 'testmodule/model', {'collection': 'testmodule/model',
'id': 42, 'id': 42,
'action': 'deleted'}) 'action': 'deleted'})
collection_element.get_full_data.assert_called_once_with() collection_element.get_full_data.assert_not_called()
def test_as_autoupdate_for_user_deleted(self): def test_as_autoupdate_for_user_deleted(self):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True) collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True)