97b21427aa
This is probalby faster but in any way it maks debugging a lot easier. To go though a big list in a for-loop is no fun at all.
530 lines
18 KiB
Python
530 lines
18 KiB
Python
from django.apps import apps
|
|
from django.core.cache import cache, caches
|
|
|
|
|
|
class CollectionElement:
|
|
@classmethod
|
|
def from_instance(cls, instance, deleted=False, information=None):
|
|
"""
|
|
Returns a collection element from a database instance.
|
|
|
|
This will also update the instance in the cache.
|
|
|
|
If deleted is set to True, the element is deleted from the cache.
|
|
"""
|
|
return cls(instance=instance, deleted=deleted, information=information)
|
|
|
|
@classmethod
|
|
def from_values(cls, collection_string, id, deleted=False, full_data=None, information=None):
|
|
"""
|
|
Returns a collection element from a collection_string and an id.
|
|
|
|
If deleted is set to True, the element is deleted from the cache.
|
|
|
|
With the argument full_data, the content of the CollectionElement can be set.
|
|
It has to be a dict in the format that is used be access_permission.get_full_data().
|
|
"""
|
|
return cls(collection_string=collection_string, id=id, deleted=deleted,
|
|
full_data=full_data, information=information)
|
|
|
|
def __init__(self, instance=None, deleted=False, collection_string=None, id=None,
|
|
full_data=None, information=None):
|
|
"""
|
|
Do not use this. Use the methods from_instance() or from_values().
|
|
"""
|
|
self.instance = instance
|
|
self.deleted = deleted
|
|
self.full_data = full_data
|
|
self.information = information or {}
|
|
if instance is not None:
|
|
self.collection_string = instance.get_collection_string()
|
|
from openslides.core.config import config
|
|
if self.collection_string == config.get_collection_string():
|
|
# For config objects we do not work with the pk but with the key.
|
|
self.id = instance.key
|
|
else:
|
|
self.id = instance.pk
|
|
elif collection_string is not None and id is not None:
|
|
self.collection_string = collection_string
|
|
self.id = id
|
|
else:
|
|
raise RuntimeError(
|
|
'Invalid state. Use CollectionElement.from_instance() or '
|
|
'CollectionElement.from_values() but not CollectionElement() '
|
|
'directly.')
|
|
|
|
if self.is_deleted():
|
|
# Delete the element from the cache, if self.is_deleted() is True:
|
|
self.delete_from_cache()
|
|
elif instance is not None:
|
|
# If this element is created via instance and the instance is not deleted
|
|
# then update the cache.
|
|
self.save_to_cache()
|
|
|
|
def __eq__(self, collection_element):
|
|
"""
|
|
Compares two collection_elements.
|
|
|
|
Two collection elements are equal, if they have the same collection_string
|
|
and id.
|
|
"""
|
|
return (self.collection_string == collection_element.collection_string and
|
|
self.id == collection_element.id)
|
|
|
|
def as_channels_message(self):
|
|
"""
|
|
Returns a dictonary that can be used to send the object through the
|
|
channels system.
|
|
"""
|
|
channel_message = {
|
|
'collection_string': self.collection_string,
|
|
'id': self.id,
|
|
'deleted': self.is_deleted()}
|
|
if self.information:
|
|
channel_message['information'] = self.information
|
|
if self.full_data:
|
|
# Do not use the method get_full_data but the attribute, so the
|
|
# full_data is not generated.
|
|
channel_message['full_data'] = self.full_data
|
|
return channel_message
|
|
|
|
def as_autoupdate(self, method, *args):
|
|
"""
|
|
Only for internal use. Do not use it directly. Use as_autoupdate_for_user()
|
|
or as_autoupdate_for_projector().
|
|
"""
|
|
output = {
|
|
'collection': self.collection_string,
|
|
'id': self.id,
|
|
'action': 'deleted' if self.is_deleted() else 'changed',
|
|
}
|
|
if not self.is_deleted():
|
|
data = getattr(self.get_access_permissions(), method)(
|
|
self.get_full_data(),
|
|
*args)
|
|
if data is None:
|
|
# The user is not allowed to see this element. Set action to deleted.
|
|
output['action'] = 'deleted'
|
|
else:
|
|
output['data'] = data
|
|
return output
|
|
|
|
def as_autoupdate_for_user(self, user):
|
|
"""
|
|
Returns a dict that can be sent through the autoupdate system for a site
|
|
user.
|
|
"""
|
|
return self.as_autoupdate(
|
|
'get_restricted_data',
|
|
user)
|
|
|
|
def as_autoupdate_for_projector(self):
|
|
"""
|
|
Returns a dict that can be sent through the autoupdate system for the
|
|
projector.
|
|
"""
|
|
return self.as_autoupdate(
|
|
'get_projector_data')
|
|
|
|
def as_dict_for_user(self, user):
|
|
"""
|
|
Returns a dict with the data for a user. Can be used for the rest api.
|
|
"""
|
|
return self.get_access_permissions().get_restricted_data(
|
|
self.get_full_data(),
|
|
user)
|
|
|
|
def get_model(self):
|
|
"""
|
|
Returns the django model that is used for this collection.
|
|
"""
|
|
return get_model_from_collection_string(self.collection_string)
|
|
|
|
def get_instance(self):
|
|
"""
|
|
Returns the instance as django object.
|
|
|
|
May raise a DoesNotExist exception.
|
|
"""
|
|
if self.is_deleted():
|
|
raise RuntimeError("The collection element is deleted.")
|
|
|
|
if self.instance is None:
|
|
# The config instance has to be get from the config element, because
|
|
# some config values are not in the db.
|
|
from openslides.core.config import config
|
|
if self.collection_string == config.get_collection_string():
|
|
self.instance = {'key': self.id, 'value': config[self.id]}
|
|
else:
|
|
model = self.get_model()
|
|
try:
|
|
query = model.objects.get_full_queryset()
|
|
except AttributeError:
|
|
query = model.objects
|
|
self.instance = query.get(pk=self.id)
|
|
return self.instance
|
|
|
|
def get_access_permissions(self):
|
|
"""
|
|
Returns the get_access_permissions object for the this collection element.
|
|
"""
|
|
return self.get_model().get_access_permissions()
|
|
|
|
def get_full_data(self):
|
|
"""
|
|
Returns the full_data of this collection_element from with all other
|
|
dics can be generated.
|
|
"""
|
|
# If the full_data is already loaded, return it
|
|
# If there is a db_instance, use it to get the full_data
|
|
# else: try to use the cache.
|
|
# If there is no value in the cach, get the content from the db and save
|
|
# it to the cache.
|
|
if self.full_data is None and self.instance is None:
|
|
# Use the cache version if self.instance is not set.
|
|
# After this line full_data can be None, if the element is not in the cache.
|
|
self.full_data = cache.get(self.get_cache_key())
|
|
|
|
if self.full_data is None:
|
|
self.full_data = self.get_access_permissions().get_full_data(self.get_instance())
|
|
self.save_to_cache()
|
|
return self.full_data
|
|
|
|
def is_deleted(self):
|
|
"""
|
|
Returns Ture if the item is marked as deleted.
|
|
"""
|
|
return self.deleted
|
|
|
|
def get_cache_key(self):
|
|
"""
|
|
Returns a string that is used as cache key for a single instance.
|
|
"""
|
|
return get_single_element_cache_key(self.collection_string, self.id)
|
|
|
|
def delete_from_cache(self):
|
|
"""
|
|
Delets an element from the cache.
|
|
|
|
Does nothing if the element is not in the cache.
|
|
"""
|
|
# Deletes the element from the cache.
|
|
cache.delete(self.get_cache_key())
|
|
|
|
# Delete the id of the instance of the instance list
|
|
Collection(self.collection_string).delete_id_from_cache(self.id)
|
|
|
|
def save_to_cache(self):
|
|
"""
|
|
Add or update the element to the cache.
|
|
"""
|
|
# Set the element to the cache.
|
|
cache.set(self.get_cache_key(), self.get_full_data())
|
|
|
|
# Add the id of the element to the collection
|
|
Collection(self.collection_string).add_id_to_cache(self.id)
|
|
|
|
|
|
class CollectionElementList(list):
|
|
"""
|
|
List for collection elements that can hold collection elements from
|
|
different collections.
|
|
|
|
It acts like a normal python list but with the following methods.
|
|
"""
|
|
|
|
@classmethod
|
|
def from_channels_message(cls, message):
|
|
"""
|
|
Creates a collection element list from a channel message.
|
|
"""
|
|
self = cls()
|
|
for values in message['elements']:
|
|
self.append(CollectionElement.from_values(**values))
|
|
return self
|
|
|
|
def as_channels_message(self):
|
|
"""
|
|
Returns a list of dicts that can be send through the channel system.
|
|
"""
|
|
message = {'elements': []}
|
|
for element in self:
|
|
message['elements'].append(element.as_channels_message())
|
|
return message
|
|
|
|
def as_autoupdate_for_user(self, user):
|
|
"""
|
|
Returns a list of dicts, that can be send though the websocket to a user.
|
|
"""
|
|
result = []
|
|
for element in self:
|
|
result.append(element.as_autoupdate_for_user(user))
|
|
return result
|
|
|
|
|
|
class Collection:
|
|
"""
|
|
Represents all elements of one collection.
|
|
"""
|
|
|
|
def __init__(self, collection_string):
|
|
self.collection_string = collection_string
|
|
|
|
def get_cache_key(self, raw=False):
|
|
"""
|
|
Returns a string that is used as cache key for a collection.
|
|
|
|
Django adds a prefix to the cache key when using the django cache api.
|
|
In other cases use raw=True to add the same cache key.
|
|
"""
|
|
key = get_element_list_cache_key(self.collection_string)
|
|
if raw:
|
|
key = cache.make_key(key)
|
|
return key
|
|
|
|
def get_model(self):
|
|
"""
|
|
Returns the django model that is used for this collection.
|
|
"""
|
|
return get_model_from_collection_string(self.collection_string)
|
|
|
|
def element_generator(self):
|
|
"""
|
|
Generator that yields all collection_elements of this collection.
|
|
"""
|
|
# Get all cache keys.
|
|
ids = self.get_all_ids()
|
|
cache_keys = [
|
|
get_single_element_cache_key(self.collection_string, id)
|
|
for id in ids]
|
|
cached_full_data_dict = cache.get_many(cache_keys)
|
|
|
|
# Get all ids that are missing.
|
|
missing_cache_keys = set(cache_keys).difference(cached_full_data_dict.keys())
|
|
missing_ids = set(
|
|
get_collection_id_from_cache_key(cache_key)[1]
|
|
for cache_key in missing_cache_keys)
|
|
|
|
# Generate collection elements that where in the cache.
|
|
for cache_key, cached_full_data in cached_full_data_dict.items():
|
|
yield CollectionElement.from_values(
|
|
*get_collection_id_from_cache_key(cache_key),
|
|
full_data=cached_full_data)
|
|
|
|
# Generate collection element that where not in the cache.
|
|
if missing_ids:
|
|
from openslides.core.config import config
|
|
if self.collection_string == config.get_collection_string():
|
|
# If config elements are not in the cache, they have to be read from
|
|
# the config object.
|
|
for key, value in config.items():
|
|
if key in missing_ids:
|
|
collection_element = CollectionElement.from_values(
|
|
config.get_collection_string(),
|
|
key,
|
|
full_data={'key': key, 'value': value})
|
|
# We can not use .from_instance therefore the config value
|
|
# is not saved to the cache. We have to do it manualy.
|
|
collection_element.save_to_cache()
|
|
yield collection_element
|
|
else:
|
|
model = self.get_model()
|
|
try:
|
|
query = model.objects.get_full_queryset()
|
|
except AttributeError:
|
|
query = model.objects
|
|
for instance in query.filter(pk__in=missing_ids):
|
|
yield CollectionElement.from_instance(instance)
|
|
|
|
def as_autoupdate_for_projector(self):
|
|
"""
|
|
Returns a list of dictonaries to send them to the projector.
|
|
"""
|
|
output = []
|
|
for collection_element in self.element_generator():
|
|
content = collection_element.as_autoupdate_for_projector()
|
|
# Content can not be None. If the projector can not see an element,
|
|
# then it is marked as deleted.
|
|
output.append(content)
|
|
return output
|
|
|
|
def as_list_for_user(self, user):
|
|
"""
|
|
Returns a list of dictonaries to send them to a user, for example over
|
|
the rest api.
|
|
"""
|
|
output = []
|
|
for collection_element in self.element_generator():
|
|
content = collection_element.as_dict_for_user(user)
|
|
if content is not None:
|
|
output.append(content)
|
|
return output
|
|
|
|
def get_all_ids(self):
|
|
"""
|
|
Returns a set of all ids of instances in this collection.
|
|
"""
|
|
from openslides.core.config import config
|
|
if self.collection_string == config.get_collection_string():
|
|
ids = config.config_variables.keys()
|
|
elif use_redis_cache():
|
|
ids = self.get_all_ids_redis()
|
|
else:
|
|
ids = self.get_all_ids_other()
|
|
return ids
|
|
|
|
def get_all_ids_redis(self):
|
|
redis = get_redis_connection()
|
|
ids = redis.smembers(self.get_cache_key(raw=True))
|
|
if not ids:
|
|
ids = set(self.get_model().objects.values_list('pk', flat=True))
|
|
if ids:
|
|
redis.sadd(self.get_cache_key(raw=True), *ids)
|
|
# Redis returns the ids as string.
|
|
ids = set(int(id) for id in ids)
|
|
return ids
|
|
|
|
def get_all_ids_other(self):
|
|
ids = cache.get(self.get_cache_key())
|
|
if ids is None:
|
|
# If it is not in the cache then get it from the database.
|
|
ids = set(self.get_model().objects.values_list('pk', flat=True))
|
|
cache.set(self.get_cache_key(), ids)
|
|
return ids
|
|
|
|
def delete_id_from_cache(self, id):
|
|
"""
|
|
Delets a id from the cache.
|
|
"""
|
|
if use_redis_cache():
|
|
self.delete_id_from_cache_redis(id)
|
|
else:
|
|
self.delete_id_from_cache_other(id)
|
|
|
|
def delete_id_from_cache_redis(self, id):
|
|
redis = get_redis_connection()
|
|
redis.srem(self.get_cache_key(raw=True), id)
|
|
|
|
def delete_id_from_cache_other(self, id):
|
|
ids = cache.get(self.get_cache_key())
|
|
if ids is not None:
|
|
ids = set(ids)
|
|
try:
|
|
ids.remove(id)
|
|
except KeyError:
|
|
# The id is not part of id list
|
|
pass
|
|
else:
|
|
if ids:
|
|
cache.set(self.get_cache_key(), ids)
|
|
else:
|
|
# Delete the key, if there are not ids left
|
|
cache.delete(self.get_cache_key())
|
|
|
|
def add_id_to_cache(self, id):
|
|
"""
|
|
Adds a collection id to the list of collection ids in the cache.
|
|
"""
|
|
if use_redis_cache():
|
|
self.add_id_to_cache_redis(id)
|
|
else:
|
|
self.add_id_to_cache_other(id)
|
|
|
|
def add_id_to_cache_redis(self, id):
|
|
redis = get_redis_connection()
|
|
if redis.exists(self.get_cache_key(raw=True)):
|
|
# Only add the value if it is in the cache.
|
|
redis.sadd(self.get_cache_key(raw=True), id)
|
|
|
|
def add_id_to_cache_other(self, id):
|
|
ids = cache.get(self.get_cache_key())
|
|
if ids is not None:
|
|
# Only change the value if it is in the cache.
|
|
ids = set(ids)
|
|
ids.add(id)
|
|
cache.set(self.get_cache_key(), ids)
|
|
|
|
|
|
def get_model_from_collection_string(collection_string):
|
|
"""
|
|
Returns a model class which belongs to the argument collection_string.
|
|
"""
|
|
def model_generator():
|
|
"""
|
|
Yields all models of all apps.
|
|
"""
|
|
for app_config in apps.get_app_configs():
|
|
for model in app_config.get_models():
|
|
yield model
|
|
|
|
for model in model_generator():
|
|
try:
|
|
model_collection_string = model.get_collection_string()
|
|
except AttributeError:
|
|
# Skip models which do not have the method get_collection_string.
|
|
pass
|
|
else:
|
|
if model_collection_string == collection_string:
|
|
# The model was found.
|
|
break
|
|
else:
|
|
# No model was found in all apps.
|
|
raise ValueError('Invalid message. A valid collection_string is missing.')
|
|
return model
|
|
|
|
|
|
def get_single_element_cache_key(collection_string, id):
|
|
"""
|
|
Returns a string that is used as cache key for a single instance.
|
|
"""
|
|
return "{prefix}{id}".format(
|
|
prefix=get_single_element_cache_key_prefix(collection_string),
|
|
id=id)
|
|
|
|
|
|
def get_single_element_cache_key_prefix(collection_string):
|
|
"""
|
|
Returns the first part of the cache key for single elements, which is the
|
|
same for all cache keys of the same collection.
|
|
"""
|
|
return "{collection}:".format(collection=collection_string)
|
|
|
|
|
|
def get_element_list_cache_key(collection_string):
|
|
"""
|
|
Returns a string that is used as cache key for a collection.
|
|
"""
|
|
return "{collection}".format(collection=collection_string)
|
|
|
|
|
|
def get_collection_id_from_cache_key(cache_key):
|
|
"""
|
|
Returns a tuble of the collection string and the id from a cache_key
|
|
created with get_instance_cache_key.
|
|
|
|
The returned id can be an integer or an string.
|
|
"""
|
|
collection_string, id = cache_key.rsplit(':', 1)
|
|
try:
|
|
id = int(id)
|
|
except ValueError:
|
|
# The id is no integer. This can happen on config elements
|
|
pass
|
|
return (collection_string, id)
|
|
|
|
|
|
def use_redis_cache():
|
|
"""
|
|
Returns True if Redis is used als caching backend.
|
|
"""
|
|
try:
|
|
from django_redis.cache import RedisCache
|
|
except ImportError:
|
|
return False
|
|
return isinstance(caches['default'], RedisCache)
|
|
|
|
|
|
def get_redis_connection():
|
|
from django_redis import get_redis_connection
|
|
return get_redis_connection("default")
|