From 9af6bf160650ccf74b21c5834a25cd6c125e28c3 Mon Sep 17 00:00:00 2001 From: Oskar Hahn Date: Sat, 1 Sep 2018 08:00:00 +0200 Subject: [PATCH] ensures test on startup --- openslides/core/apps.py | 8 + openslides/core/config.py | 2 - openslides/utils/autoupdate.py | 8 +- openslides/utils/cache.py | 86 ++++--- openslides/utils/cache_providers.py | 214 ++++++++++-------- openslides/utils/collection.py | 15 +- tests/conftest.py | 12 + tests/integration/agenda/test_viewset.py | 13 +- tests/integration/assignments/test_viewset.py | 5 + tests/integration/helpers.py | 3 - tests/integration/motions/test_viewset.py | 15 +- tests/integration/users/test_viewset.py | 2 + tests/integration/utils/test_collection.py | 30 --- tests/integration/utils/test_consumers.py | 8 +- tests/old/config/test_config.py | 21 +- tests/settings.py | 6 +- tests/unit/utils/cache_provider.py | 6 +- tests/unit/utils/test_cache.py | 60 +---- 18 files changed, 229 insertions(+), 285 deletions(-) diff --git a/openslides/core/apps.py b/openslides/core/apps.py index 6f56fa92d..c643c3074 100644 --- a/openslides/core/apps.py +++ b/openslides/core/apps.py @@ -21,6 +21,7 @@ class CoreAppConfig(AppConfig): # Import all required stuff. from .config import config from ..utils.rest_api import router + from ..utils.cache import element_cache from .projector import get_projector_elements from .signals import ( delete_django_app_permissions, @@ -74,6 +75,13 @@ class CoreAppConfig(AppConfig): router.register(self.get_model('ProjectorMessage').get_collection_string(), ProjectorMessageViewSet) router.register(self.get_model('Countdown').get_collection_string(), CountdownViewSet) + # Sets the cache + try: + element_cache.ensure_cache() + except (ImproperlyConfigured, OperationalError): + # This happens in the tests or in migrations. Do nothing + pass + def get_config_variables(self): from .config_variables import get_config_variables return get_config_variables() diff --git a/openslides/core/config.py b/openslides/core/config.py index f62d2175f..dce49330a 100644 --- a/openslides/core/config.py +++ b/openslides/core/config.py @@ -236,7 +236,6 @@ OnChangeType = Callable[[], None] ConfigVariableDict = TypedDict('ConfigVariableDict', { 'key': str, 'default_value': Any, - 'value': Any, 'input_type': str, 'label': str, 'help_text': str, @@ -303,7 +302,6 @@ class ConfigVariable: return ConfigVariableDict( key=self.name, default_value=self.default_value, - value=config[self.name], input_type=self.input_type, label=self.label, help_text=self.help_text, diff --git a/openslides/utils/autoupdate.py b/openslides/utils/autoupdate.py index e0757ced1..ee5c59ae8 100644 --- a/openslides/utils/autoupdate.py +++ b/openslides/utils/autoupdate.py @@ -4,7 +4,6 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple, Union from asgiref.sync import async_to_sync from channels.layers import get_channel_layer -from django.conf import settings from django.db.models import Model from .cache import element_cache, get_element_id @@ -146,12 +145,7 @@ async def send_autoupdate(collection_elements: Iterable[CollectionElement]) -> N else: cache_elements[element_id] = element.get_full_data() - if not getattr(settings, 'SKIP_CACHE', False): - # Hack for django 2.0 and channels 2.1 to stay in the same thread. - # This is needed for the tests. - change_id = await element_cache.change_elements(cache_elements) - else: - change_id = 1 + change_id = await element_cache.change_elements(cache_elements) channel_layer = get_channel_layer() # TODO: don't await. They can be send in parallel diff --git a/openslides/utils/cache.py b/openslides/utils/cache.py index 178488936..ce25e2e5f 100644 --- a/openslides/utils/cache.py +++ b/openslides/utils/cache.py @@ -2,6 +2,7 @@ import asyncio import json from collections import defaultdict from datetime import datetime +from time import sleep from typing import ( TYPE_CHECKING, Any, @@ -13,8 +14,7 @@ from typing import ( Type, ) -from asgiref.sync import sync_to_async -from channels.db import database_sync_to_async +from asgiref.sync import async_to_sync, sync_to_async from django.conf import settings from .cache_providers import ( @@ -83,6 +83,9 @@ class ElementCache: # Contains Futures to controll, that only one client updates the restricted_data. self.restricted_data_cache_updater: Dict[int, asyncio.Future] = {} + # Tells if self.ensure_cache was called. + self.ensured = False + @property def cachables(self) -> Dict[str, Cachable]: """ @@ -93,33 +96,35 @@ class ElementCache: self._cachables = {cachable.get_collection_string(): cachable for cachable in self.cachable_provider()} return self._cachables - async def save_full_data(self, db_data: Dict[str, List[Dict[str, Any]]]) -> None: + def ensure_cache(self, reset: bool = False) -> None: """ - Saves the full data. - """ - mapping = {} - for collection_string, elements in db_data.items(): - for element in elements: - mapping.update( - {get_element_id(collection_string, element['id']): - json.dumps(element)}) - await self.cache_provider.reset_full_cache(mapping) + Makes sure that the cache exist. - async def build_full_data(self) -> Dict[str, List[Dict[str, Any]]]: - """ - Build or rebuild the full_data cache. - """ - db_data = {} - for collection_string, cachable in self.cachables.items(): - db_data[collection_string] = await database_sync_to_async(cachable.get_elements)() - await self.save_full_data(db_data) - return db_data + Builds the cache if not. If reset is True, it will be reset in any case. - async def exists_full_data(self) -> bool: + This method is sync, so it can be run when OpenSlides starts. """ - Returns True, if the full_data_cache exists. - """ - return await self.cache_provider.data_exists() + cache_exists = async_to_sync(self.cache_provider.data_exists)() + + if reset or not cache_exists: + lock_name = 'ensure_cache' + # Set a lock so only one process builds the cache + if async_to_sync(self.cache_provider.set_lock)(lock_name): + try: + mapping = {} + for collection_string, cachable in self.cachables.items(): + for element in cachable.get_elements(): + mapping.update( + {get_element_id(collection_string, element['id']): + json.dumps(element)}) + async_to_sync(self.cache_provider.reset_full_cache)(mapping) + finally: + async_to_sync(self.cache_provider.del_lock)(lock_name) + else: + while async_to_sync(self.cache_provider.get_lock)(lock_name): + sleep(0.01) + + self.ensured = True async def change_elements( self, elements: Dict[str, Optional[Dict[str, Any]]]) -> int: @@ -131,9 +136,6 @@ class ElementCache: Returns the new generated change_id. """ - if not await self.exists_full_data(): - await self.build_full_data() - deleted_elements = [] changed_elements = [] for element_id, data in elements.items(): @@ -164,14 +166,11 @@ class ElementCache: The returned value is a dict where the key is the collection_string and the value is a list of data. """ - if not await self.exists_full_data(): - out = await self.build_full_data() - else: - out = defaultdict(list) - full_data = await self.cache_provider.get_all_data() - for element_id, data in full_data.items(): - collection_string, __ = split_element_id(element_id) - out[collection_string].append(json.loads(data.decode())) + out: Dict[str, List[Dict[str, Any]]] = defaultdict(list) + full_data = await self.cache_provider.get_all_data() + for element_id, data in full_data.items(): + collection_string, __ = split_element_id(element_id) + out[collection_string].append(json.loads(data.decode())) return dict(out) async def get_full_data( @@ -203,10 +202,6 @@ class ElementCache: "Catch this exception and rerun the method with change_id=0." .format(change_id, lowest_change_id)) - if not await self.exists_full_data(): - # If the cache does not exist, create it. - await self.build_full_data() - raw_changed_elements, deleted_elements = await self.cache_provider.get_data_since(change_id) return ( {collection_string: [json.loads(value.decode()) for value in value_list] @@ -221,9 +216,6 @@ class ElementCache: Returns None if the element does not exist. """ - if not await self.exists_full_data(): - await self.build_full_data() - element = await self.cache_provider.get_element(get_element_id(collection_string, id)) if element is None: @@ -261,7 +253,8 @@ class ElementCache: # Try to write a special key. # If this succeeds, there is noone else currently updating the cache. # TODO: Make a timeout. Else this could block forever - if await self.cache_provider.set_lock_restricted_data(get_user_id(user)): + lock_name = "restricted_data_{}".format(get_user_id(user)) + if await self.cache_provider.set_lock(lock_name): future: asyncio.Future = asyncio.Future() self.restricted_data_cache_updater[get_user_id(user)] = future # Get change_id for this user @@ -293,7 +286,7 @@ class ElementCache: mapping['_config:change_id'] = str(change_id) await self.cache_provider.update_restricted_data(get_user_id(user), mapping) # Unset the lock - await self.cache_provider.del_lock_restricted_data(get_user_id(user)) + await self.cache_provider.del_lock(lock_name) future.set_result(1) else: # Wait until the update if finshed @@ -301,7 +294,7 @@ class ElementCache: # The active worker is on the same asgi server, we can use the future await self.restricted_data_cache_updater[get_user_id(user)] else: - while await self.cache_provider.get_lock_restricted_data(get_user_id(user)): + while await self.cache_provider.get_lock(lock_name): await asyncio.sleep(0.01) async def get_all_restricted_data(self, user: Optional['CollectionElement']) -> Dict[str, List[Dict[str, Any]]]: @@ -412,6 +405,7 @@ def load_element_cache(redis_addr: str = '', restricted_data: bool = True) -> El return ElementCache(redis=redis_addr, use_restricted_data_cache=restricted_data) +# Set the element_cache redis_address = getattr(settings, 'REDIS_ADDRESS', '') use_restricted_data = getattr(settings, 'RESTRICTED_DATA_CACHE', True) element_cache = load_element_cache(redis_addr=redis_address, restricted_data=use_restricted_data) diff --git a/openslides/utils/cache_providers.py b/openslides/utils/cache_providers.py index e2150d4d4..5aba38741 100644 --- a/openslides/utils/cache_providers.py +++ b/openslides/utils/cache_providers.py @@ -52,7 +52,7 @@ class BaseCacheProvider: def get_change_id_cache_key(self) -> str: return self.change_id_cache_key - def clear_cache(self) -> None: + async def clear_cache(self) -> None: raise NotImplementedError("CacheProvider has to implement the method clear_cache().") async def reset_full_cache(self, data: Dict[str, str]) -> None: @@ -82,14 +82,14 @@ class BaseCacheProvider: async def del_restricted_data(self, user_id: int) -> None: raise NotImplementedError("CacheProvider has to implement the method del_restricted_data().") - async def set_lock_restricted_data(self, user_id: int) -> bool: - raise NotImplementedError("CacheProvider has to implement the method set_lock_restricted_data().") + async def set_lock(self, lock_name: str) -> bool: + raise NotImplementedError("CacheProvider has to implement the method set_lock().") - async def get_lock_restricted_data(self, user_id: int) -> bool: - raise NotImplementedError("CacheProvider has to implement the method get_lock_restricted_data().") + async def get_lock(self, lock_name: str) -> bool: + raise NotImplementedError("CacheProvider has to implement the method get_lock().") - async def del_lock_restricted_data(self, user_id: int) -> None: - raise NotImplementedError("CacheProvider has to implement the method del_lock_restricted_data().") + async def del_lock(self, lock_name: str) -> None: + raise NotImplementedError("CacheProvider has to implement the method del_lock().") async def get_change_id_user(self, user_id: int) -> Optional[int]: raise NotImplementedError("CacheProvider has to implement the method get_change_id_user().") @@ -104,6 +104,23 @@ class BaseCacheProvider: raise NotImplementedError("CacheProvider has to implement the method get_lowest_change_id().") +class RedisConnectionContextManager: + """ + Async context manager for connections + """ + # TODO: contextlib.asynccontextmanager can be used in python 3.7 + + def __init__(self, redis_address: str) -> None: + self.redis_address = redis_address + + async def __aenter__(self) -> 'aioredis.RedisConnection': + self.conn = await aioredis.create_redis(self.redis_address) + return self.conn + + async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None: + self.conn.close() + + class RedisCacheProvider(BaseCacheProvider): """ Cache provider that loads and saves the data to redis. @@ -113,22 +130,28 @@ class RedisCacheProvider(BaseCacheProvider): def __init__(self, redis: str) -> None: self.redis_address = redis - async def get_connection(self) -> 'aioredis.RedisConnection': + def get_connection(self) -> RedisConnectionContextManager: """ - Returns a redis connection. + Returns contextmanager for a redis connection. """ - if self.redis_pool is None: - self.redis_pool = await aioredis.create_redis_pool(self.redis_address) - return self.redis_pool + return RedisConnectionContextManager(self.redis_address) + + async def clear_cache(self) -> None: + """ + Deleted all cache entries created with this element cache. + """ + async with self.get_connection() as redis: + # TODO: Fix me. Do only delete keys, that are created with this cache. + await redis.flushall() async def reset_full_cache(self, data: Dict[str, str]) -> None: """ Deletes the cache and write new data in it. """ # TODO: lua or transaction - redis = await self.get_connection() - await redis.delete(self.get_full_data_cache_key()) - await redis.hmset_dict(self.get_full_data_cache_key(), data) + async with self.get_connection() as redis: + await redis.delete(self.get_full_data_cache_key()) + await redis.hmset_dict(self.get_full_data_cache_key(), data) async def data_exists(self, user_id: Optional[int] = None) -> bool: """ @@ -137,12 +160,12 @@ class RedisCacheProvider(BaseCacheProvider): If user_id is None, the method tests for full_data. If user_id is an int, it tests for the restricted_data_cache for the user with the user_id. 0 is for anonymous. """ - redis = await self.get_connection() - if user_id is None: - cache_key = self.get_full_data_cache_key() - else: - cache_key = self.get_restricted_data_cache_key(user_id) - return await redis.exists(cache_key) + async with self.get_connection() as redis: + if user_id is None: + cache_key = self.get_full_data_cache_key() + else: + cache_key = self.get_restricted_data_cache_key(user_id) + return await redis.exists(cache_key) async def add_elements(self, elements: List[str]) -> None: """ @@ -151,10 +174,10 @@ class RedisCacheProvider(BaseCacheProvider): elements is a list with an even len. the odd values are the element_ids and the even values are the elements. The elements have to be encoded, for example with json. """ - redis = await self.get_connection() - await redis.hmset( - self.get_full_data_cache_key(), - *elements) + async with self.get_connection() as redis: + await redis.hmset( + self.get_full_data_cache_key(), + *elements) async def del_elements(self, elements: List[str], user_id: Optional[int] = None) -> None: """ @@ -165,14 +188,14 @@ class RedisCacheProvider(BaseCacheProvider): If user_id is None, the elements are deleted from the full_data cache. If user_id is an int, the elements are deleted one restricted_data_cache. 0 is for anonymous. """ - redis = await self.get_connection() - if user_id is None: - cache_key = self.get_full_data_cache_key() - else: - cache_key = self.get_restricted_data_cache_key(user_id) - await redis.hdel( - cache_key, - *elements) + async with self.get_connection() as redis: + if user_id is None: + cache_key = self.get_full_data_cache_key() + else: + cache_key = self.get_restricted_data_cache_key(user_id) + await redis.hdel( + cache_key, + *elements) async def add_changed_elements(self, change_id: int, element_ids: Iterable[str]) -> None: """ @@ -189,10 +212,10 @@ class RedisCacheProvider(BaseCacheProvider): yield change_id yield element_id - redis = await self.get_connection() - await redis.zadd(self.get_change_id_cache_key(), *zadd_args(change_id)) - # Saves the lowest_change_id if it does not exist - await redis.zadd(self.get_change_id_cache_key(), change_id, '_config:lowest_change_id', exist='ZSET_IF_NOT_EXIST') + async with self.get_connection() as redis: + await redis.zadd(self.get_change_id_cache_key(), *zadd_args(change_id)) + # Saves the lowest_change_id if it does not exist + await redis.zadd(self.get_change_id_cache_key(), change_id, '_config:lowest_change_id', exist='ZSET_IF_NOT_EXIST') async def get_all_data(self, user_id: Optional[int] = None) -> Dict[bytes, bytes]: """ @@ -205,8 +228,8 @@ class RedisCacheProvider(BaseCacheProvider): cache_key = self.get_full_data_cache_key() else: cache_key = self.get_restricted_data_cache_key(user_id) - redis = await self.get_connection() - return await redis.hgetall(cache_key) + async with self.get_connection() as redis: + return await redis.hgetall(cache_key) async def get_element(self, element_id: str) -> Optional[bytes]: """ @@ -214,10 +237,10 @@ class RedisCacheProvider(BaseCacheProvider): Returns None, when the element does not exist. """ - redis = await self.get_connection() - return await redis.hget( - self.get_full_data_cache_key(), - element_id) + async with self.get_connection() as redis: + return await redis.hget( + self.get_full_data_cache_key(), + element_id) async def get_data_since(self, change_id: int, user_id: Optional[int] = None) -> Tuple[Dict[str, List[bytes]], List[str]]: """ @@ -231,53 +254,53 @@ class RedisCacheProvider(BaseCacheProvider): for an user is used. 0 is for the anonymous user. """ # TODO: rewrite with lua to get all elements with one request - redis = await self.get_connection() - changed_elements: Dict[str, List[bytes]] = defaultdict(list) - deleted_elements: List[str] = [] - for element_id in await redis.zrangebyscore(self.get_change_id_cache_key(), min=change_id): - if element_id.startswith(b'_config'): - continue - element_json = await redis.hget(self.get_full_data_cache_key(), element_id) # Optional[bytes] - if element_json is None: - # The element is not in the cache. It has to be deleted. - deleted_elements.append(element_id) - else: - collection_string, id = split_element_id(element_id) - changed_elements[collection_string].append(element_json) - return changed_elements, deleted_elements + async with self.get_connection() as redis: + changed_elements: Dict[str, List[bytes]] = defaultdict(list) + deleted_elements: List[str] = [] + for element_id in await redis.zrangebyscore(self.get_change_id_cache_key(), min=change_id): + if element_id.startswith(b'_config'): + continue + element_json = await redis.hget(self.get_full_data_cache_key(), element_id) # Optional[bytes] + if element_json is None: + # The element is not in the cache. It has to be deleted. + deleted_elements.append(element_id) + else: + collection_string, id = split_element_id(element_id) + changed_elements[collection_string].append(element_json) + return changed_elements, deleted_elements async def del_restricted_data(self, user_id: int) -> None: """ Deletes all restricted_data for an user. 0 is for the anonymous user. """ - redis = await self.get_connection() - await redis.delete(self.get_restricted_data_cache_key(user_id)) + async with self.get_connection() as redis: + await redis.delete(self.get_restricted_data_cache_key(user_id)) - async def set_lock_restricted_data(self, user_id: int) -> bool: + async def set_lock(self, lock_name: str) -> bool: """ - Tries to sets a lock for the restricted_data of an user. + Tries to sets a lock. Returns True when the lock could be set. Returns False when the lock was already set. """ - redis = await self.get_connection() - return await redis.hsetnx(self.get_restricted_data_cache_key(user_id), self.lock_key, 1) + async with self.get_connection() as redis: + return await redis.hsetnx("lock_{}".format(lock_name), self.lock_key, 1) - async def get_lock_restricted_data(self, user_id: int) -> bool: + async def get_lock(self, lock_name: str) -> bool: """ Returns True, when the lock for the restricted_data of an user is set. Else False. """ - redis = await self.get_connection() - return await redis.hget(self.get_restricted_data_cache_key(user_id), self.lock_key) + async with self.get_connection() as redis: + return await redis.hget("lock_{}".format(lock_name), self.lock_key) - async def del_lock_restricted_data(self, user_id: int) -> None: + async def del_lock(self, lock_name: str) -> None: """ Deletes the lock for the restricted_data of an user. Does nothing when the lock is not set. """ - redis = await self.get_connection() - await redis.hdel(self.get_restricted_data_cache_key(user_id), self.lock_key) + async with self.get_connection() as redis: + await redis.hdel("lock_{}".format(lock_name), self.lock_key) async def get_change_id_user(self, user_id: int) -> Optional[int]: """ @@ -285,8 +308,8 @@ class RedisCacheProvider(BaseCacheProvider): This is the change_id where the restricted_data was last calculated. """ - redis = await self.get_connection() - return await redis.hget(self.get_restricted_data_cache_key(user_id), '_config:change_id') + async with self.get_connection() as redis: + return await redis.hget(self.get_restricted_data_cache_key(user_id), '_config:change_id') async def update_restricted_data(self, user_id: int, data: Dict[str, str]) -> None: """ @@ -295,19 +318,19 @@ class RedisCacheProvider(BaseCacheProvider): data has to be a dict where the key is an element_id and the value the (json-) encoded element. """ - redis = await self.get_connection() - await redis.hmset_dict(self.get_restricted_data_cache_key(user_id), data) + async with self.get_connection() as redis: + await redis.hmset_dict(self.get_restricted_data_cache_key(user_id), data) async def get_current_change_id(self) -> List[Tuple[str, int]]: """ Get the highest change_id from redis. """ - redis = await self.get_connection() - return await redis.zrevrangebyscore( - self.get_change_id_cache_key(), - withscores=True, - count=1, - offset=0) + async with self.get_connection() as redis: + return await redis.zrevrangebyscore( + self.get_change_id_cache_key(), + withscores=True, + count=1, + offset=0) async def get_lowest_change_id(self) -> Optional[int]: """ @@ -315,10 +338,10 @@ class RedisCacheProvider(BaseCacheProvider): Returns None if lowest score does not exist. """ - redis = await self.get_connection() - return await redis.zscore( - self.get_change_id_cache_key(), - '_config:lowest_change_id') + async with self.get_connection() as redis: + return await redis.zscore( + self.get_change_id_cache_key(), + '_config:lowest_change_id') class MemmoryCacheProvider(BaseCacheProvider): @@ -332,12 +355,16 @@ class MemmoryCacheProvider(BaseCacheProvider): """ def __init__(self, *args: Any, **kwargs: Any) -> None: - self.clear_cache() + self.set_data_dicts() - def clear_cache(self) -> None: + def set_data_dicts(self) -> None: self.full_data: Dict[str, str] = {} self.restricted_data: Dict[int, Dict[str, str]] = {} self.change_id_data: Dict[int, Set[str]] = {} + self.locks: Dict[str, str] = {} + + async def clear_cache(self) -> None: + self.set_data_dicts() async def reset_full_cache(self, data: Dict[str, str]) -> None: self.full_data = data @@ -417,21 +444,18 @@ class MemmoryCacheProvider(BaseCacheProvider): except KeyError: pass - async def set_lock_restricted_data(self, user_id: int) -> bool: - data = self.restricted_data.setdefault(user_id, {}) - if self.lock_key in data: + async def set_lock(self, lock_name: str) -> bool: + if lock_name in self.locks: return False - data[self.lock_key] = "1" + self.locks[lock_name] = "1" return True - async def get_lock_restricted_data(self, user_id: int) -> bool: - data = self.restricted_data.get(user_id, {}) - return self.lock_key in data + async def get_lock(self, lock_name: str) -> bool: + return lock_name in self.locks - async def del_lock_restricted_data(self, user_id: int) -> None: - data = self.restricted_data.get(user_id, {}) + async def del_lock(self, lock_name: str) -> None: try: - del data[self.lock_key] + del self.locks[lock_name] except KeyError: pass diff --git a/openslides/utils/collection.py b/openslides/utils/collection.py index 6471e4092..0f4817525 100644 --- a/openslides/utils/collection.py +++ b/openslides/utils/collection.py @@ -12,7 +12,6 @@ from typing import ( from asgiref.sync import async_to_sync from django.apps import apps -from django.conf import settings from django.db.models import Model from mypy_extensions import TypedDict @@ -201,12 +200,7 @@ class CollectionElement: if self.instance is None: # The type of data has to be set for mypy data: Optional[Dict[str, Any]] = None - if getattr(settings, 'SKIP_CACHE', False): - # Hack for django 2.0 and channels 2.1 to stay in the same thread. - # This is needed for the tests. - data = self.get_element_from_db() - else: - data = async_to_sync(element_cache.get_element_full_data)(self.collection_string, self.id) + data = async_to_sync(element_cache.get_element_full_data)(self.collection_string, self.id) if data is None: raise self.get_model().DoesNotExist( "Collection {} with id {} does not exist".format(self.collection_string, self.id)) @@ -278,12 +272,7 @@ class Collection(Cachable): if self.full_data is None: # The type of all_full_data has to be set for mypy all_full_data: Dict[str, List[Dict[str, Any]]] = {} - if getattr(settings, 'SKIP_CACHE', False): - # Hack for django 2.0 and channels 2.1 to stay in the same thread. - # This is needed for the tests. - all_full_data = self.get_elements_from_db() - else: - all_full_data = async_to_sync(element_cache.get_all_full_data)() + all_full_data = async_to_sync(element_cache.get_all_full_data)() self.full_data = all_full_data.get(self.collection_string, []) return self.full_data # type: ignore diff --git a/tests/conftest.py b/tests/conftest.py index d06807c9e..cc7364fc2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,8 @@ from django.test import TestCase, TransactionTestCase from pytest_django.django_compat import is_django_unittest from pytest_django.plugin import validate_django_db +from openslides.utils.cache import element_cache + def pytest_collection_modifyitems(items): """ @@ -57,3 +59,13 @@ def constants(request): else: # Else: Use fake constants set_constants({'constant1': 'value1', 'constant2': 'value2'}) + + +@pytest.fixture(autouse=True) +def reset_cache(request): + """ + Resetts the cache for every test + """ + if 'django_db' in request.node.keywords or is_django_unittest(request): + # When the db is created, use the original cachables + element_cache.ensure_cache(reset=True) diff --git a/tests/integration/agenda/test_viewset.py b/tests/integration/agenda/test_viewset.py index 9a44ad069..ec7232d60 100644 --- a/tests/integration/agenda/test_viewset.py +++ b/tests/integration/agenda/test_viewset.py @@ -12,6 +12,8 @@ from openslides.core.config import config from openslides.core.models import Countdown from openslides.motions.models import Motion from openslides.topics.models import Topic +from openslides.users.models import Group +from openslides.utils.autoupdate import inform_changed_data from openslides.utils.collection import CollectionElement from openslides.utils.test import TestCase @@ -43,20 +45,22 @@ class RetrieveItem(TestCase): self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) def test_hidden_by_anonymous_with_manage_perms(self): - group = get_user_model().groups.field.related_model.objects.get(pk=1) # Group with pk 1 is for anonymous users. + group = Group.objects.get(pk=1) # Group with pk 1 is for anonymous users. permission_string = 'agenda.can_manage' app_label, codename = permission_string.split('.') permission = Permission.objects.get(content_type__app_label=app_label, codename=codename) group.permissions.add(permission) + inform_changed_data(group) response = self.client.get(reverse('item-detail', args=[self.item.pk])) self.assertEqual(response.status_code, status.HTTP_200_OK) def test_internal_by_anonymous_without_perm_to_see_internal_items(self): - group = get_user_model().groups.field.related_model.objects.get(pk=1) # Group with pk 1 is for anonymous users. + group = Group.objects.get(pk=1) # Group with pk 1 is for anonymous users. permission_string = 'agenda.can_see_internal_items' app_label, codename = permission_string.split('.') permission = group.permissions.get(content_type__app_label=app_label, codename=codename) group.permissions.remove(permission) + inform_changed_data(group) self.item.type = Item.INTERNAL_ITEM self.item.save() response = self.client.get(reverse('item-detail', args=[self.item.pk])) @@ -194,6 +198,7 @@ class ManageSpeaker(TestCase): group_delegates = type(group_admin).objects.get(name='Delegates') admin.groups.add(group_delegates) admin.groups.remove(group_admin) + inform_changed_data(admin) CollectionElement.from_instance(admin) response = self.client.post( @@ -231,7 +236,7 @@ class ManageSpeaker(TestCase): group_delegates = type(group_admin).objects.get(name='Delegates') admin.groups.add(group_delegates) admin.groups.remove(group_admin) - CollectionElement.from_instance(admin) + inform_changed_data(admin) speaker = Speaker.objects.add(self.user, self.item) response = self.client.delete( @@ -259,7 +264,7 @@ class ManageSpeaker(TestCase): group_delegates = type(group_admin).objects.get(name='Delegates') admin.groups.add(group_delegates) admin.groups.remove(group_admin) - CollectionElement.from_instance(admin) + inform_changed_data(admin) Speaker.objects.add(self.user, self.item) response = self.client.patch( diff --git a/tests/integration/assignments/test_viewset.py b/tests/integration/assignments/test_viewset.py index 7b490a423..3dba1d558 100644 --- a/tests/integration/assignments/test_viewset.py +++ b/tests/integration/assignments/test_viewset.py @@ -5,6 +5,7 @@ from rest_framework import status from rest_framework.test import APIClient from openslides.assignments.models import Assignment +from openslides.utils.autoupdate import inform_changed_data from openslides.utils.test import TestCase from ..helpers import count_queries @@ -77,6 +78,7 @@ class CanidatureSelf(TestCase): group_delegates = type(group_admin).objects.get(name='Delegates') admin.groups.add(group_delegates) admin.groups.remove(group_admin) + inform_changed_data(admin) response = self.client.post(reverse('assignment-candidature-self', args=[self.assignment.pk])) @@ -123,6 +125,7 @@ class CanidatureSelf(TestCase): group_delegates = type(group_admin).objects.get(name='Delegates') admin.groups.add(group_delegates) admin.groups.remove(group_admin) + inform_changed_data(admin) response = self.client.delete(reverse('assignment-candidature-self', args=[self.assignment.pk])) @@ -203,6 +206,7 @@ class CandidatureOther(TestCase): group_delegates = type(group_admin).objects.get(name='Delegates') admin.groups.add(group_delegates) admin.groups.remove(group_admin) + inform_changed_data(admin) response = self.client.post( reverse('assignment-candidature-other', args=[self.assignment.pk]), @@ -258,6 +262,7 @@ class CandidatureOther(TestCase): group_delegates = type(group_admin).objects.get(name='Delegates') admin.groups.add(group_delegates) admin.groups.remove(group_admin) + inform_changed_data(admin) response = self.client.delete( reverse('assignment-candidature-other', args=[self.assignment.pk]), diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index b7449bae9..2edfb4567 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -50,9 +50,6 @@ async def set_config(key, value): """ Set a config variable in the element_cache without hitting the database. """ - if not await element_cache.exists_full_data(): - # Encure that the cache exists and the default values of the config are in it. - await element_cache.build_full_data() collection_string = config.get_collection_string() config_id = config.key_to_id[key] # type: ignore full_data = {'id': config_id, 'key': key, 'value': value} diff --git a/tests/integration/motions/test_viewset.py b/tests/integration/motions/test_viewset.py index edba5ecc8..dbe2e0e37 100644 --- a/tests/integration/motions/test_viewset.py +++ b/tests/integration/motions/test_viewset.py @@ -20,7 +20,7 @@ from openslides.motions.models import ( Workflow, ) from openslides.utils.auth import get_group_model -from openslides.utils.collection import CollectionElement +from openslides.utils.autoupdate import inform_changed_data from openslides.utils.test import TestCase from ..helpers import count_queries @@ -207,6 +207,7 @@ class CreateMotion(TestCase): self.admin = get_user_model().objects.get(username='admin') self.admin.groups.add(2) self.admin.groups.remove(4) + inform_changed_data(self.admin) response = self.client.post( reverse('motion-list'), @@ -258,6 +259,7 @@ class CreateMotion(TestCase): self.admin = get_user_model().objects.get(username='admin') self.admin.groups.add(2) self.admin.groups.remove(4) + inform_changed_data(self.admin) response = self.client.post( reverse('motion-list'), @@ -306,6 +308,7 @@ class RetrieveMotion(TestCase): state.save() # The cache has to be cleared, see: # https://github.com/OpenSlides/OpenSlides/issues/3396 + inform_changed_data(self.motion) response = guest_client.get(reverse('motion-detail', args=[self.motion.pk])) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) @@ -340,6 +343,7 @@ class RetrieveMotion(TestCase): group.permissions.remove(permission) config['general_system_enable_anonymous'] = True guest_client = APIClient() + inform_changed_data(group) response_1 = guest_client.get(reverse('motion-detail', args=[self.motion.pk])) self.assertEqual(response_1.status_code, status.HTTP_200_OK) @@ -431,6 +435,7 @@ class UpdateMotion(TestCase): self.motion.supporters.add(supporter) config['motions_remove_supporters'] = True self.assertEqual(self.motion.supporters.count(), 1) + inform_changed_data((admin, self.motion)) response = self.client.patch( reverse('motion-detail', args=[self.motion.pk]), @@ -467,7 +472,7 @@ class DeleteMotion(TestCase): group_delegates = get_group_model().objects.get(name='Delegates') self.admin.groups.remove(group_admin) self.admin.groups.add(group_delegates) - CollectionElement.from_instance(self.admin) + inform_changed_data(self.admin) def put_motion_in_complex_workflow(self): workflow = Workflow.objects.get(name='Complex Workflow') @@ -551,7 +556,7 @@ class ManageSubmitters(TestCase): group_delegates = type(group_admin).objects.get(name='Delegates') admin.groups.add(group_delegates) admin.groups.remove(group_admin) - CollectionElement.from_instance(admin) + inform_changed_data(admin) response = self.client.post( reverse('motion-manage-submitters', args=[self.motion.pk]), @@ -884,6 +889,7 @@ class TestMotionCommentSection(TestCase): any of the read_groups. """ self.admin.groups.remove(self.group_in) + inform_changed_data(self.admin) section = MotionCommentSection(name='test_name_f3jOF3m8fp. None: + async def del_lock_after_wait(self, lock_name: str, future: asyncio.Future = None) -> None: if future is None: - asyncio.ensure_future(self.del_lock_restricted_data(user_id)) + asyncio.ensure_future(self.del_lock(lock_name)) else: async def set_future() -> None: - await self.del_lock_restricted_data(user_id) + await self.del_lock(lock_name) future.set_result(1) # type: ignore asyncio.ensure_future(set_future()) diff --git a/tests/unit/utils/test_cache.py b/tests/unit/utils/test_cache.py index 9c7218b22..f90ac7623 100644 --- a/tests/unit/utils/test_cache.py +++ b/tests/unit/utils/test_cache.py @@ -29,53 +29,13 @@ def sort_dict(encoded_dict: Dict[str, List[Dict[str, Any]]]) -> Dict[str, List[D @pytest.fixture def element_cache(): - return ElementCache( + element_cache = ElementCache( 'test_redis', cache_provider_class=TTestCacheProvider, cachable_provider=get_cachable_provider(), start_time=0) - - -@pytest.mark.asyncio -async def test_save_full_data(element_cache): - input_data = { - 'app/collection1': [ - {'id': 1, 'value': 'value1'}, - {'id': 2, 'value': 'value2'}], - 'app/collection2': [ - {'id': 1, 'key': 'value1'}, - {'id': 2, 'key': 'value2'}]} - calculated_data = { - 'app/collection1:1': '{"id": 1, "value": "value1"}', - 'app/collection1:2': '{"id": 2, "value": "value2"}', - 'app/collection2:1': '{"id": 1, "key": "value1"}', - 'app/collection2:2': '{"id": 2, "key": "value2"}'} - - await element_cache.save_full_data(input_data) - - assert decode_dict(element_cache.cache_provider.full_data) == decode_dict(calculated_data) - - -@pytest.mark.asyncio -async def test_build_full_data(element_cache): - result = await element_cache.build_full_data() - - assert result == example_data() - assert decode_dict(element_cache.cache_provider.full_data) == decode_dict({ - 'app/collection1:1': '{"id": 1, "value": "value1"}', - 'app/collection1:2': '{"id": 2, "value": "value2"}', - 'app/collection2:1': '{"id": 1, "key": "value1"}', - 'app/collection2:2': '{"id": 2, "key": "value2"}'}) - - -@pytest.mark.asyncio -async def test_exists_full_data(element_cache): - """ - Test that the return value of exists_full_data is the the same as from the - cache_provider. - """ - element_cache.cache_provider.full_data = 'test_value' - assert await element_cache.exists_full_data() + element_cache.ensure_cache() + return element_cache @pytest.mark.asyncio @@ -245,7 +205,7 @@ async def test_get_element_full_data_full_redis(element_cache): @pytest.mark.asyncio -async def test_exist_restricted_data(element_cache): +async def test_exists_restricted_data(element_cache): element_cache.use_restricted_data_cache = True element_cache.cache_provider.restricted_data = {0: { 'app/collection1:1': '{"id": 1, "value": "value1"}', @@ -259,7 +219,7 @@ async def test_exist_restricted_data(element_cache): @pytest.mark.asyncio -async def test_exist_restricted_data_do_not_use_restricted_data(element_cache): +async def test_exists_restricted_data_do_not_use_restricted_data(element_cache): element_cache.use_restricted_data_cache = False element_cache.cache_provider.restricted_data = {0: { 'app/collection1:1': '{"id": 1, "value": "value1"}', @@ -308,7 +268,7 @@ async def test_update_restricted_data(element_cache): 'app/collection2:2': '{"id": 2, "key": "restricted_value2"}', '_config:change_id': '0'}) # Make sure the lock is deleted - assert not await element_cache.cache_provider.get_lock_restricted_data(0) + assert not await element_cache.cache_provider.get_lock("restricted_data_0") # And the future is done assert element_cache.restricted_data_cache_updater[0].done() @@ -379,8 +339,8 @@ async def test_update_restricted_data_second_worker_on_different_server(element_ """ element_cache.use_restricted_data_cache = True element_cache.cache_provider.restricted_data = {0: {}} - await element_cache.cache_provider.set_lock_restricted_data(0) - await element_cache.cache_provider.del_lock_restricted_data_after_wait(0) + await element_cache.cache_provider.set_lock("restricted_data_0") + await element_cache.cache_provider.del_lock_after_wait("restricted_data_0") await element_cache.update_restricted_data(None) @@ -399,8 +359,8 @@ async def test_update_restricted_data_second_worker_on_same_server(element_cache element_cache.cache_provider.restricted_data = {0: {}} future: asyncio.Future = asyncio.Future() element_cache.restricted_data_cache_updater[0] = future - await element_cache.cache_provider.set_lock_restricted_data(0) - await element_cache.cache_provider.del_lock_restricted_data_after_wait(0, future) + await element_cache.cache_provider.set_lock("restricted_data_0") + await element_cache.cache_provider.del_lock_after_wait("restricted_data_0", future) await element_cache.update_restricted_data(None)