import asyncio import json from typing import Any, Dict, List import pytest from openslides.utils.cache import ElementCache from .cache_provider import ( TTestCacheProvider, example_data, get_cachable_provider, ) def decode_dict(encoded_dict: Dict[str, str]) -> Dict[str, Any]: """ Helper function that loads the json values of a dict. """ return {key: json.loads(value) for key, value in encoded_dict.items()} def sort_dict(encoded_dict: Dict[str, List[Dict[str, Any]]]) -> Dict[str, List[Dict[str, Any]]]: """ Helper function that sorts the value of a dict. """ return {key: sorted(value, key=lambda x: x['id']) for key, value in encoded_dict.items()} @pytest.fixture def element_cache(): return ElementCache( 'test_redis', cache_provider_class=TTestCacheProvider, cachable_provider=get_cachable_provider(), start_time=0) @pytest.mark.asyncio async def test_save_full_data(element_cache): input_data = { 'app/collection1': [ {'id': 1, 'value': 'value1'}, {'id': 2, 'value': 'value2'}], 'app/collection2': [ {'id': 1, 'key': 'value1'}, {'id': 2, 'key': 'value2'}]} calculated_data = { 'app/collection1:1': '{"id": 1, "value": "value1"}', 'app/collection1:2': '{"id": 2, "value": "value2"}', 'app/collection2:1': '{"id": 1, "key": "value1"}', 'app/collection2:2': '{"id": 2, "key": "value2"}'} await element_cache.save_full_data(input_data) assert decode_dict(element_cache.cache_provider.full_data) == decode_dict(calculated_data) @pytest.mark.asyncio async def test_build_full_data(element_cache): result = await element_cache.build_full_data() assert result == example_data() assert decode_dict(element_cache.cache_provider.full_data) == decode_dict({ 'app/collection1:1': '{"id": 1, "value": "value1"}', 'app/collection1:2': '{"id": 2, "value": "value2"}', 'app/collection2:1': '{"id": 1, "key": "value1"}', 'app/collection2:2': '{"id": 2, "key": "value2"}'}) @pytest.mark.asyncio async def test_exists_full_data(element_cache): """ Test that the return value of exists_full_data is the the same as from the cache_provider. """ element_cache.cache_provider.full_data = 'test_value' assert await element_cache.exists_full_data() @pytest.mark.asyncio async def test_change_elements(element_cache): input_data = { 'app/collection1:1': {"id": 1, "value": "updated"}, 'app/collection1:2': {"id": 2, "value": "new"}, 'app/collection2:1': {"id": 1, "key": "updated"}, 'app/collection2:2': None} element_cache.cache_provider.full_data = { 'app/collection1:1': '{"id": 1, "value": "old"}', 'app/collection2:1': '{"id": 1, "key": "old"}', 'app/collection2:2': '{"id": 2, "key": "old"}'} result = await element_cache.change_elements(input_data) assert result == 1 # first change_id assert decode_dict(element_cache.cache_provider.full_data) == decode_dict({ 'app/collection1:1': '{"id": 1, "value": "updated"}', 'app/collection1:2': '{"id": 2, "value": "new"}', 'app/collection2:1': '{"id": 1, "key": "updated"}'}) assert element_cache.cache_provider.change_id_data == { 1: { 'app/collection1:1', 'app/collection1:2', 'app/collection2:1', 'app/collection2:2'}} @pytest.mark.asyncio async def test_change_elements_with_no_data_in_redis(element_cache): input_data = { 'app/collection1:1': {"id": 1, "value": "updated"}, 'app/collection1:2': {"id": 2, "value": "new"}, 'app/collection2:1': {"id": 1, "key": "updated"}, 'app/collection2:2': None} result = await element_cache.change_elements(input_data) assert result == 1 # first change_id assert decode_dict(element_cache.cache_provider.full_data) == decode_dict({ 'app/collection1:1': '{"id": 1, "value": "updated"}', 'app/collection1:2': '{"id": 2, "value": "new"}', 'app/collection2:1': '{"id": 1, "key": "updated"}'}) assert element_cache.cache_provider.change_id_data == { 1: { 'app/collection1:1', 'app/collection1:2', 'app/collection2:1', 'app/collection2:2'}} @pytest.mark.asyncio async def test_get_all_full_data_from_db(element_cache): result = await element_cache.get_all_full_data() assert result == example_data() # Test that elements are written to redis assert decode_dict(element_cache.cache_provider.full_data) == decode_dict({ 'app/collection1:1': '{"id": 1, "value": "value1"}', 'app/collection1:2': '{"id": 2, "value": "value2"}', 'app/collection2:1': '{"id": 1, "key": "value1"}', 'app/collection2:2': '{"id": 2, "key": "value2"}'}) @pytest.mark.asyncio async def test_get_all_full_data_from_redis(element_cache): element_cache.cache_provider.full_data = { 'app/collection1:1': '{"id": 1, "value": "value1"}', 'app/collection1:2': '{"id": 2, "value": "value2"}', 'app/collection2:1': '{"id": 1, "key": "value1"}', 'app/collection2:2': '{"id": 2, "key": "value2"}'} result = await element_cache.get_all_full_data() # The output from redis has to be the same then the db_data assert sort_dict(result) == sort_dict(example_data()) @pytest.mark.asyncio async def test_get_full_data_change_id_0(element_cache): element_cache.cache_provider.full_data = { 'app/collection1:1': '{"id": 1, "value": "value1"}', 'app/collection1:2': '{"id": 2, "value": "value2"}', 'app/collection2:1': '{"id": 1, "key": "value1"}', 'app/collection2:2': '{"id": 2, "key": "value2"}'} result = await element_cache.get_full_data(0) assert sort_dict(result[0]) == sort_dict(example_data()) @pytest.mark.asyncio async def test_get_full_data_change_id_lower_then_in_redis(element_cache): element_cache.cache_provider.full_data = { 'app/collection1:1': '{"id": 1, "value": "value1"}', 'app/collection1:2': '{"id": 2, "value": "value2"}', 'app/collection2:1': '{"id": 1, "key": "value1"}', 'app/collection2:2': '{"id": 2, "key": "value2"}'} element_cache.cache_provider.change_id_data = { 2: {'app/collection1:1'}} with pytest.raises(RuntimeError): await element_cache.get_full_data(1) @pytest.mark.asyncio async def test_get_full_data_change_id_data_in_redis(element_cache): element_cache.cache_provider.full_data = { 'app/collection1:1': '{"id": 1, "value": "value1"}', 'app/collection1:2': '{"id": 2, "value": "value2"}', 'app/collection2:1': '{"id": 1, "key": "value1"}', 'app/collection2:2': '{"id": 2, "key": "value2"}'} element_cache.cache_provider.change_id_data = { 1: {'app/collection1:1', 'app/collection1:3'}} result = await element_cache.get_full_data(1) assert result == ( {'app/collection1': [{"id": 1, "value": "value1"}]}, ['app/collection1:3']) @pytest.mark.asyncio async def test_get_full_data_change_id_data_in_db(element_cache): element_cache.cache_provider.change_id_data = { 1: {'app/collection1:1', 'app/collection1:3'}} result = await element_cache.get_full_data(1) assert result == ( {'app/collection1': [{"id": 1, "value": "value1"}]}, ['app/collection1:3']) @pytest.mark.asyncio async def test_get_full_data_change_id_data_in_db_empty_change_id(element_cache): with pytest.raises(RuntimeError): await element_cache.get_full_data(1) @pytest.mark.asyncio async def test_get_element_full_data_empty_redis(element_cache): result = await element_cache.get_element_full_data('app/collection1', 1) assert result == {'id': 1, 'value': 'value1'} @pytest.mark.asyncio async def test_get_element_full_data_empty_redis_does_not_exist(element_cache): result = await element_cache.get_element_full_data('app/collection1', 3) assert result is None @pytest.mark.asyncio async def test_get_element_full_data_full_redis(element_cache): element_cache.cache_provider.full_data = { 'app/collection1:1': '{"id": 1, "value": "value1"}', 'app/collection1:2': '{"id": 2, "value": "value2"}', 'app/collection2:1': '{"id": 1, "key": "value1"}', 'app/collection2:2': '{"id": 2, "key": "value2"}'} result = await element_cache.get_element_full_data('app/collection1', 1) assert result == {'id': 1, 'value': 'value1'} @pytest.mark.asyncio async def test_exist_restricted_data(element_cache): element_cache.use_restricted_data_cache = True element_cache.cache_provider.restricted_data = {0: { 'app/collection1:1': '{"id": 1, "value": "value1"}', 'app/collection1:2': '{"id": 2, "value": "value2"}', 'app/collection2:1': '{"id": 1, "key": "value1"}', 'app/collection2:2': '{"id": 2, "key": "value2"}'}} result = await element_cache.exists_restricted_data(None) assert result @pytest.mark.asyncio async def test_exist_restricted_data_do_not_use_restricted_data(element_cache): element_cache.use_restricted_data_cache = False element_cache.cache_provider.restricted_data = {0: { 'app/collection1:1': '{"id": 1, "value": "value1"}', 'app/collection1:2': '{"id": 2, "value": "value2"}', 'app/collection2:1': '{"id": 1, "key": "value1"}', 'app/collection2:2': '{"id": 2, "key": "value2"}'}} result = await element_cache.exists_restricted_data(None) assert not result @pytest.mark.asyncio async def test_del_user(element_cache): element_cache.use_restricted_data_cache = True element_cache.cache_provider.restricted_data = {0: { 'app/collection1:1': '{"id": 1, "value": "value1"}', 'app/collection1:2': '{"id": 2, "value": "value2"}', 'app/collection2:1': '{"id": 1, "key": "value1"}', 'app/collection2:2': '{"id": 2, "key": "value2"}'}} await element_cache.del_user(None) assert not element_cache.cache_provider.restricted_data @pytest.mark.asyncio async def test_del_user_for_empty_user(element_cache): element_cache.use_restricted_data_cache = True await element_cache.del_user(None) assert not element_cache.cache_provider.restricted_data @pytest.mark.asyncio async def test_update_restricted_data(element_cache): element_cache.use_restricted_data_cache = True await element_cache.update_restricted_data(None) assert decode_dict(element_cache.cache_provider.restricted_data[0]) == decode_dict({ 'app/collection1:1': '{"id": 1, "value": "restricted_value1"}', 'app/collection1:2': '{"id": 2, "value": "restricted_value2"}', 'app/collection2:1': '{"id": 1, "key": "restricted_value1"}', 'app/collection2:2': '{"id": 2, "key": "restricted_value2"}', '_config:change_id': '0'}) # Make sure the lock is deleted assert not await element_cache.cache_provider.get_lock_restricted_data(0) # And the future is done assert element_cache.restricted_data_cache_updater[0].done() @pytest.mark.asyncio async def test_update_restricted_data_disabled_restricted_data(element_cache): element_cache.use_restricted_data_cache = False await element_cache.update_restricted_data(None) assert not element_cache.cache_provider.restricted_data @pytest.mark.asyncio async def test_update_restricted_data_to_low_change_id(element_cache): element_cache.use_restricted_data_cache = True element_cache.cache_provider.restricted_data[0] = { '_config:change_id': '1'} element_cache.cache_provider.change_id_data = { 3: {'app/collection1:1'}} await element_cache.update_restricted_data(None) assert decode_dict(element_cache.cache_provider.restricted_data[0]) == decode_dict({ 'app/collection1:1': '{"id": 1, "value": "restricted_value1"}', 'app/collection1:2': '{"id": 2, "value": "restricted_value2"}', 'app/collection2:1': '{"id": 1, "key": "restricted_value1"}', 'app/collection2:2': '{"id": 2, "key": "restricted_value2"}', '_config:change_id': '3'}) @pytest.mark.asyncio async def test_update_restricted_data_with_same_id(element_cache): element_cache.use_restricted_data_cache = True element_cache.cache_provider.restricted_data[0] = { '_config:change_id': '1'} element_cache.cache_provider.change_id_data = { 1: {'app/collection1:1'}} await element_cache.update_restricted_data(None) # Same id means, there is nothing to do assert element_cache.cache_provider.restricted_data[0] == { '_config:change_id': '1'} @pytest.mark.asyncio async def test_update_restricted_data_with_deleted_elements(element_cache): element_cache.use_restricted_data_cache = True element_cache.cache_provider.restricted_data[0] = { 'app/collection1:3': '{"id": 1, "value": "restricted_value1"}', '_config:change_id': '1'} element_cache.cache_provider.change_id_data = { 2: {'app/collection1:3'}} await element_cache.update_restricted_data(None) assert element_cache.cache_provider.restricted_data[0] == { '_config:change_id': '2'} @pytest.mark.asyncio async def test_update_restricted_data_second_worker_on_different_server(element_cache): """ Test, that if another worker is updating the data, noting is done. This tests makes use of the redis key as it would on different daphne servers. """ element_cache.use_restricted_data_cache = True element_cache.cache_provider.restricted_data = {0: {}} await element_cache.cache_provider.set_lock_restricted_data(0) await element_cache.cache_provider.del_lock_restricted_data_after_wait(0) await element_cache.update_restricted_data(None) # Restricted_data_should not be set on second worker assert element_cache.cache_provider.restricted_data == {0: {}} @pytest.mark.asyncio async def test_update_restricted_data_second_worker_on_same_server(element_cache): """ Test, that if another worker is updating the data, noting is done. This tests makes use of the future as it would on the same daphne server. """ element_cache.use_restricted_data_cache = True element_cache.cache_provider.restricted_data = {0: {}} future: asyncio.Future = asyncio.Future() element_cache.restricted_data_cache_updater[0] = future await element_cache.cache_provider.set_lock_restricted_data(0) await element_cache.cache_provider.del_lock_restricted_data_after_wait(0, future) await element_cache.update_restricted_data(None) # Restricted_data_should not be set on second worker assert element_cache.cache_provider.restricted_data == {0: {}} @pytest.mark.asyncio async def test_get_all_restricted_data(element_cache): element_cache.use_restricted_data_cache = True result = await element_cache.get_all_restricted_data(None) assert sort_dict(result) == sort_dict({ 'app/collection1': [{"id": 1, "value": "restricted_value1"}, {"id": 2, "value": "restricted_value2"}], 'app/collection2': [{"id": 1, "key": "restricted_value1"}, {"id": 2, "key": "restricted_value2"}]}) @pytest.mark.asyncio async def test_get_all_restricted_data_disabled_restricted_data_cache(element_cache): element_cache.use_restricted_data_cache = False result = await element_cache.get_all_restricted_data(None) assert sort_dict(result) == sort_dict({ 'app/collection1': [{"id": 1, "value": "restricted_value1"}, {"id": 2, "value": "restricted_value2"}], 'app/collection2': [{"id": 1, "key": "restricted_value1"}, {"id": 2, "key": "restricted_value2"}]}) @pytest.mark.asyncio async def test_get_restricted_data_change_id_0(element_cache): element_cache.use_restricted_data_cache = True result = await element_cache.get_restricted_data(None, 0) assert sort_dict(result[0]) == sort_dict({ 'app/collection1': [{"id": 1, "value": "restricted_value1"}, {"id": 2, "value": "restricted_value2"}], 'app/collection2': [{"id": 1, "key": "restricted_value1"}, {"id": 2, "key": "restricted_value2"}]}) @pytest.mark.asyncio async def test_get_restricted_data_disabled_restricted_data_cache(element_cache): element_cache.use_restricted_data_cache = False element_cache.cache_provider.change_id_data = {1: {'app/collection1:1', 'app/collection1:3'}} result = await element_cache.get_restricted_data(None, 1) assert result == ( {'app/collection1': [{"id": 1, "value": "restricted_value1"}]}, ['app/collection1:3']) @pytest.mark.asyncio async def test_get_restricted_data_change_id_lower_then_in_redis(element_cache): element_cache.use_restricted_data_cache = True element_cache.cache_provider.change_id_data = {2: {'app/collection1:1'}} with pytest.raises(RuntimeError): await element_cache.get_restricted_data(None, 1) @pytest.mark.asyncio async def test_get_restricted_data_change_with_id(element_cache): element_cache.use_restricted_data_cache = True element_cache.cache_provider.change_id_data = {2: {'app/collection1:1'}} result = await element_cache.get_restricted_data(None, 2) assert result == ({'app/collection1': [{"id": 1, "value": "restricted_value1"}]}, []) @pytest.mark.asyncio async def test_lowest_change_id_after_updating_lowest_element(element_cache): await element_cache.change_elements({'app/collection1:1': {"id": 1, "value": "updated1"}}) first_lowest_change_id = await element_cache.get_lowest_change_id() # Alter same element again await element_cache.change_elements({'app/collection1:1': {"id": 1, "value": "updated2"}}) second_lowest_change_id = await element_cache.get_lowest_change_id() assert first_lowest_change_id == 1 assert second_lowest_change_id == 1 # The lowest_change_id should not change