ag-admin/OpenSlides
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Changed restricted data parsing. Cached full data on startup.

Browse Source
This commit is contained in:
Oskar Hahn 2017-04-28 00:50:37 +02:00 committed by Norman Jäckel
parent 6559f5508f
commit 4963bfa7bf
15 changed files with 279 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
openslides/agenda/access_permissions.py
View File

@ -27,32 +27,42 @@ class ItemAccessPermissions(BaseAccessPermissions):
Returns the restricted serialized data for the instance prepared
for the user.
"""
def filtered_data(full_data, blocked_keys):
"""
Returns a new dict like full_data but with all blocked_keys removed.
"""
whitelist = full_data.keys() - blocked_keys
return {key: full_data[key] for key in whitelist}
# many_items is True, when there are more then one items in full_data.
many_items = not isinstance(full_data, dict)
full_data = full_data if many_items else [full_data]
if has_perm(user, 'agenda.can_see'):
if full_data['is_hidden'] and not has_perm(user, 'agenda.can_see_hidden_items'):
# The data is hidden but the user isn't allowed to see it. Jst pass
# the whitelisted keys so the list of speakers is provided regardless.
whitelist = (
if has_perm(user, 'agenda.can_manage'):
data = full_data
elif has_perm(user, 'agenda.can_see_hidden_items'):
blocked_keys = ('comment',)
data = [filtered_data(full, blocked_keys) for full in full_data]
else:
data = []
filtered_blocked_keys = full_data[0].keys() - (
'id',
'title',
'speakers',
'speaker_list_closed',
'content_object',)
data = {}
for key in full_data.keys():
if key in whitelist:
data[key] = full_data[key]
'content_object')
not_filtered_blocked_keys = ('comment',)
for full in full_data:
if full['is_hidden']:
blocked_keys = filtered_blocked_keys
else:
if has_perm(user, 'agenda.can_manage'):
data = full_data
else:
# Strip out item comments for unprivileged users.
data = {}
for key in full_data.keys():
if key != 'comment':
data[key] = full_data[key]
blocked_keys = not_filtered_blocked_keys
data.append(filtered_data(full, blocked_keys))
else:
data = None
return data
return data if many_items else data[0]
def get_projector_data(self, full_data):
"""

16
openslides/agenda/signals.py
View File

@ -56,18 +56,12 @@ def get_permission_change_data(sender, permissions, **kwargs):
def is_user_data_required(sender, request_user, user_data, **kwargs):
"""
Returns True if request user can see the agenda and user_data is required
to be displayed as speaker.
If request_user can see the agenda, then returns all user ids that are
speakers in some agenda items. Else, it returns an empty set.
"""
result = False
speakers = set()
if has_perm(request_user, 'agenda.can_see'):
for item_collection_element in Collection(Item.get_collection_string()).element_generator():
full_data = item_collection_element.get_full_data()
for speaker in full_data['speakers']:
if user_data['id'] == speaker['user_id']:
result = True
break
else:
continue
break
return result
speakers.update(speaker['user_id'] for speaker in full_data['speakers'])
return speakers

8
openslides/assignments/access_permissions.py
View File

@ -33,8 +33,14 @@ class AssignmentAccessPermissions(BaseAccessPermissions):
if has_perm(user, 'assignments.can_see') and has_perm(user, 'assignments.can_manage'):
data = full_data
elif has_perm(user, 'assignments.can_see'):
many_items = not isinstance(full_data, dict)
full_data_list = full_data if many_items else [full_data]
out = []
for full_data in full_data_list:
data = full_data.copy()
data['polls'] = [poll for poll in data['polls'] if poll['published']]
data['polls'] = [poll for poll in data['polls'] if data['published']]
out.append(data)
data = out if many_items else out[0]
else:
data = None
return data

26
openslides/assignments/signals.py
View File

@ -18,28 +18,14 @@ def get_permission_change_data(sender, permissions=None, **kwargs):
def is_user_data_required(sender, request_user, user_data, **kwargs):
"""
Returns True if request user can see assignments and user_data is required
to be displayed as candidates (including poll options).
If request_user can see assignments, then returns all user ids that are
displayed as candidates (including poll options). Else, it returns an empty set.
"""
result = False
user_ids = set()
if has_perm(request_user, 'assignments.can_see'):
for assignment_collection_element in Collection(Assignment.get_collection_string()).element_generator():
full_data = assignment_collection_element.get_full_data()
for related_user in full_data['assignment_related_users']:
if user_data['id'] == related_user['user_id']:
result = True
break
else:
user_ids.update(related_user['user_id'] for related_user in full_data['assignment_related_users'])
for poll in full_data['polls']:
for option in poll['options']:
if user_data['id'] == option['candidate_id']:
result = True
break
else:
continue
break
else:
continue
break
break
return result
user_ids.update(option['candidate_id'] for option in poll['options'])
return user_ids

12
openslides/core/signals.py
View File

@ -54,14 +54,12 @@ def get_permission_change_data(sender, permissions, **kwargs):
def is_user_data_required(sender, request_user, user_data, **kwargs):
"""
Returns True if request user can use chat and user_data is required
to be displayed as chatter.
If request_user can use chat, then returns all user ids that are
displayed as chatter. Else, it returns an empty set.
"""
result = False
user_ids = set()
if has_perm(request_user, 'core.can_use_chat'):
for chat_message_collection_element in Collection(ChatMessage.get_collection_string()).element_generator():
full_data = chat_message_collection_element.get_full_data()
if user_data['id'] == full_data['user_id']:
result = True
break
return result
user_ids.add(full_data['user_id'])
return user_ids

10
openslides/mediafiles/access_permissions.py
View File

@ -26,7 +26,13 @@ class MediafileAccessPermissions(BaseAccessPermissions):
for the user.
"""
data = None
if has_perm(user, 'mediafiles.can_see'):
if (not full_data['hidden'] or has_perm(user, 'mediafiles.can_see_hidden')):
if has_perm(user, 'mediafiles.can_see') and has_perm(user, 'mediafiles.can_see_hidden'):
data = full_data
elif has_perm(user, 'mediafiles.can_see'):
many_items = not isinstance(full_data, dict)
full_data_list = full_data if many_items else [full_data]
data = [full_data for full_data in full_data_list if not full_data['hidden']]
data = data if many_items else data[0]
else:
data = None
return data

11
openslides/mediafiles/signals.py
View File

@ -20,12 +20,13 @@ def is_user_data_required(sender, request_user, user_data, **kwargs):
"""
Returns True if request user can see mediafiles and user_data is required
to be displayed as uploader.
If request_user can see mediafiles, then returns all user ids that are
displayed as uploader. Else, it returns an empty set.
"""
result = False
user_ids = set()
if has_perm(request_user, 'mediafiles.can_see'):
for mediafile_collection_element in Collection(Mediafile.get_collection_string()).element_generator():
full_data = mediafile_collection_element.get_full_data()
if user_data['id'] == full_data['uploader_id']:
result = True
break
return result
user_ids.add(full_data['uploader_id'])
return user_ids

16
openslides/motions/access_permissions.py
View File

@ -1,7 +1,5 @@
from copy import deepcopy
from django.contrib.auth import get_user_model
from ..core.config import config
from ..utils.access_permissions import BaseAccessPermissions
from ..utils.auth import has_perm
@ -33,13 +31,10 @@ class MotionAccessPermissions(BaseAccessPermissions):
the motion in this state. Removes non public comment fields for
some unauthorized users.
"""
if isinstance(user, get_user_model()):
# Converts a user object to a collection element.
# from_instance can not be used because the user serializer loads
# the group from the db. So each call to from_instance(user) consts
# one db query.
user = CollectionElement.from_values('users/user', user.id)
many_items = not isinstance(full_data, dict)
full_data_list = full_data if many_items else [full_data]
out = []
for full_data in full_data_list:
if isinstance(user, CollectionElement):
is_submitter = user.get_full_data()['id'] in full_data.get('submitters_id', [])
else:
@ -72,7 +67,8 @@ class MotionAccessPermissions(BaseAccessPermissions):
if personal_note.get('user_id') == user.id:
data['personal_notes'].append(personal_note)
break
return data
out.append(data)
return out if many_items else out[0]
def get_projector_data(self, full_data):
"""

13
openslides/motions/signals.py
View File

@ -120,14 +120,13 @@ def get_permission_change_data(sender, permissions, **kwargs):
def is_user_data_required(sender, request_user, user_data, **kwargs):
"""
Returns True if request user can see motions and user_data is required
to be displayed as motion submitter or supporter.
If request_user can see motions, then returns all user ids that are
displayed as submitter or supporter. Else, it returns an empty set.
"""
result = False
user_ids = set()
if has_perm(request_user, 'motions.can_see'):
for motion_collection_element in Collection(Motion.get_collection_string()).element_generator():
full_data = motion_collection_element.get_full_data()
if user_data['id'] in full_data['submitters_id'] or user_data['id'] in full_data['supporters_id']:
result = True
break
return result
user_ids.update(full_data['submitters_id'])
user_ids.update(full_data['supporters_id'])
return user_ids

73
openslides/users/access_permissions.py
View File

@ -31,57 +31,56 @@ class UserAccessPermissions(BaseAccessPermissions):
"""
from .serializers import USERCANSEESERIALIZER_FIELDS, USERCANSEEEXTRASERIALIZER_FIELDS
NO_DATA = 0
LITTLE_DATA = 1
MANY_DATA = 2
FULL_DATA = 3
def filtered_data(full_data, only_keys):
"""
Returns a new dict like full_data but with all blocked_keys removed.
"""
return {key: full_data[key] for key in only_keys}
# many_items is True, when there are more then one items in full_data.
many_items = not isinstance(full_data, dict)
full_data = full_data if many_items else [full_data]
many_fields = set(USERCANSEEEXTRASERIALIZER_FIELDS)
little_fields = set(USERCANSEESERIALIZER_FIELDS)
many_fields.add('groups_id')
many_fields.discard('groups')
little_fields.add('groups_id')
little_fields.discard('groups')
# Check user permissions.
if has_perm(user, 'users.can_see_name'):
if has_perm(user, 'users.can_see_extra_data'):
if has_perm(user, 'users.can_manage'):
case = FULL_DATA
data = full_data
else:
case = MANY_DATA
data = [filtered_data(full, many_fields) for full in full_data]
else:
case = LITTLE_DATA
elif user is not None and user.id == full_data.get('id'):
# An authenticated user without the permission to see users tries
# to see himself.
case = LITTLE_DATA
data = [filtered_data(full, little_fields) for full in full_data]
else:
# Now check if the user to be sent out is required by any app e. g.
# as motion submitter or assignment candidate.
# Build a list of users, that can be seen without permissions.
no_perm_users = set()
if user is not None:
no_perm_users.add(user.id)
# Get a list of all users, that are needed by another app
receiver_responses = user_data_required.send(
sender=self.__class__,
request_user=user,
user_data=full_data)
for receiver, response in receiver_responses:
if response:
case = LITTLE_DATA
break
else:
case = NO_DATA
no_perm_users.update(response)
# Setup data.
if case == FULL_DATA:
data = full_data
elif case == NO_DATA:
data = None
else:
# case in (LITTLE_DATA, ḾANY_DATA)
if case == MANY_DATA:
fields = USERCANSEEEXTRASERIALIZER_FIELDS
else:
# case == LITTLE_DATA
fields = USERCANSEESERIALIZER_FIELDS
# Let only some fields pass this method.
data = {}
for base_key in fields:
for key in (base_key, base_key + '_id'):
if key in full_data.keys():
data[key] = full_data[key]
return data
data = [
filtered_data(full, little_fields)
for full
in full_data
if full['id'] in no_perm_users]
# Set data to [None] if data is empty
data = data or [None]
return data if many_items else data[0]
def get_projector_data(self, full_data):
"""

1
openslides/users/migrations/0004_personalnote.py
View File

@ -3,7 +3,6 @@
from __future__ import unicode_literals
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models

15
openslides/utils/access_permissions.py
View File

@ -61,19 +61,26 @@ class BaseAccessPermissions(object, metaclass=SignalConnectMetaClass):
Returns the restricted serialized data for the instance prepared
for the user.
Returns None if the user has no read access. Returns reduced data
if the user has limited access. Default: Returns full data if the
user has read access to model instances.
Returns None or an empty list if the user has no read access. Returns
reduced data if the user has limited access. Default: Returns full data
if the user has read access to model instances.
Hint: You should override this method if your get_serializer_class()
method returns different serializers for different users or if you
have access restrictions in your view or viewset in methods like
retrieve() or list().
full_data can be a list or one single item. If it is a list, then the
retun value is also a list of restricted data.
When the user can not access
"""
if self.check_permissions(user):
data = full_data
else:
elif isinstance(full_data, dict):
data = None
else:
data = []
return data
def get_projector_data(self, full_data):

45
openslides/utils/autoupdate.py
View File

@ -6,15 +6,20 @@ from collections import Iterable, defaultdict
from channels import Channel, Group
from channels.asgi import get_channel_layer
from channels.auth import channel_session_user, channel_session_user_from_http
from django.apps import apps
from django.core.exceptions import ObjectDoesNotExist
from django.db import transaction
from ..core.config import config
from ..core.models import Projector
from .auth import has_perm, user_to_collection_user
from .cache import websocket_user_cache
from .collection import Collection, CollectionElement, CollectionElementList
from .cache import start_up_cache, websocket_user_cache
from .collection import (
Collection,
CollectionElement,
CollectionElementList,
format_for_autoupdate,
get_model_from_collection_string,
)
def send_or_wait(send_func, *args, **kwargs):
@ -61,18 +66,29 @@ def ws_add_site(message):
# Collect all elements that shoud be send to the client when the websocket
# connection is established
output = []
for app in apps.get_app_configs():
try:
# Get the method get_startup_elements() from an app.
# This method has to return an iterable of Collection objects.
get_startup_elements = app.get_startup_elements
except AttributeError:
# Skip apps that do not implement get_startup_elements
continue
for collection in get_startup_elements():
user = user_to_collection_user(message.user.id)
output.extend(collection.as_autoupdate_for_user(user))
output = []
for collection_string, full_data in start_up_cache.get_collection_elements().items():
access_permission = get_model_from_collection_string(collection_string).get_access_permissions()
if collection_string == 'core/config':
id_key = 'key'
else:
id_key = 'id'
restricted_data = access_permission.get_restricted_data(full_data, user)
if restricted_data is None:
continue
for data in restricted_data:
if data is None:
continue
output.append(
format_for_autoupdate(
collection_string=collection_string,
id=data[id_key],
action='changed',
data=data))
# Send all data. If there is no data, then only accept the connection
if output:
@ -341,6 +357,7 @@ def send_autoupdate(collection_elements):
Does nothing if collection_elements is empty.
"""
if collection_elements:
start_up_cache.clear()
send_or_wait(
Channel('autoupdate.send_data').send,
collection_elements.as_channels_message())

60
openslides/utils/cache.py
View File

@ -2,6 +2,7 @@ from collections import defaultdict
from channels import Group
from channels.sessions import session_for_reply_channel
from django.apps import apps
from django.core.cache import cache, caches
@ -184,6 +185,65 @@ class DjangoCacheWebsocketUserCache(BaseWebsocketUserCache):
cache.set(self.get_cache_key(), data)
class StartupCache:
"""
Cache of all data that are needed when a clients connects via websocket.
"""
cache_key = "full_data_cache"
def build(self):
"""
Generate the cache by going though all apps.
Returns a dict where the key is the collection string and the value a
list of the full_data from the collection_elements.
"""
cache_data = {}
for app in apps.get_app_configs():
try:
# Get the method get_startup_elements() from an app.
# This method has to return an iterable of Collection objects.
get_startup_elements = app.get_startup_elements
except AttributeError:
# Skip apps that do not implement get_startup_elements
continue
for collection in get_startup_elements():
cache_data[collection.collection_string] = [
collection_element.get_full_data()
for collection_element
in collection.element_generator()]
cache.set(self.cache_key, cache_data, 86400)
return cache_data
def clear(self):
"""
Clears the cache.
"""
cache.delete(self.cache_key)
def get_collection_elements(self):
"""
Returns a dict of all collection_elements, that are needed at startup.
The key is the collection_string and the value a list of CollectionElement
objects. Each list has at least one value.
The data is read from the cache if it exists. It builds the cache, if it
does not exists.
"""
data = cache.get(self.cache_key)
if data is None:
# The cache does not exist.
data = self.build()
return data
start_up_cache = StartupCache()
def use_redis_cache():
"""
Returns True if Redis is used als caching backend.

39
openslides/utils/collection.py
View File

@ -100,21 +100,18 @@ class CollectionElement:
Only for internal use. Do not use it directly. Use as_autoupdate_for_user()
or as_autoupdate_for_projector().
"""
output = {
'collection': self.collection_string,
'id': self.id,
'action': 'deleted' if self.is_deleted() else 'changed',
}
if not self.is_deleted():
data = getattr(self.get_access_permissions(), method)(
self.get_full_data(),
*args)
if data is None:
# The user is not allowed to see this element. Set action to deleted.
output['action'] = 'deleted'
else:
output['data'] = data
return output
data = None
return format_for_autoupdate(
collection_string=self.collection_string,
id=self.id,
action='deleted' if self.is_deleted() else 'changed',
data=data)
def as_autoupdate_for_user(self, user):
"""
@ -551,3 +548,25 @@ def get_collection_id_from_cache_key(cache_key):
# The id is no integer. This can happen on config elements
pass
return (collection_string, id)
def format_for_autoupdate(collection_string, id, action, data=None):
"""
Returns a dict that can be used for autoupdate.
"""
if not data:
# If the data is None or is empty, then the action has to be deleted,
# even when it says diffrently. This can happen when the object is not
# deleted, but the user has no permission to see it.
action = 'deleted'
output = {
'collection': collection_string,
'id': id,
'action': action,
}
if action != 'deleted':
output['data'] = data
return output