Rewrite projector code to be cache friendly

This speeds up the requests/seconds by a factor of 100

openslides/agenda/projector.py

@@ -3,9 +3,10 @@ from typing import Any, Dict, List, Union
 
 from ..users.projector import get_user_name
 from ..utils.projector import (
-    AllData,
+    ProjectorAllDataProvider,
     ProjectorElementException,
     get_config,
+    get_model,
     register_projector_slide,
 )
 
@@ -15,20 +16,24 @@ from ..utils.projector import (
 #            side effects.
 
 
-async def get_sorted_agenda_items(all_data: AllData) -> List[Dict[str, Any]]:
+async def get_sorted_agenda_items(
+    agenda_items: Dict[int, Dict[str, Any]]
+) -> List[Dict[str, Any]]:
     """
     Returns all sorted agenda items by id first and then weight, resulting in
     ordered items, if some have the same weight.
     """
     return sorted(
-        sorted(all_data["agenda/item"].values(), key=lambda item: item["id"]),
+        sorted(agenda_items.values(), key=lambda item: item["id"]),
         key=lambda item: item["weight"],
     )
 
 
-async def get_flat_tree(all_data: AllData, parent_id: int = 0) -> List[Dict[str, Any]]:
+async def get_flat_tree(
+    agenda_items: Dict[int, Dict[str, Any]], parent_id: int = 0
+) -> List[Dict[str, Any]]:
     """
-    Build the item tree from all_data.
+    Build the item tree from all_data_provider.
 
     Only build the tree from elements unterneath parent_id.
 
@@ -38,16 +43,16 @@ async def get_flat_tree(all_data: AllData, parent_id: int = 0) -> List[Dict[str,
 
     # Build a dict from an item_id to all its children
     children: Dict[int, List[int]] = defaultdict(list)
-    if "agenda/item" in all_data:
+
-        for item in await get_sorted_agenda_items(all_data):
+    for item in await get_sorted_agenda_items(agenda_items):
         if item["type"] == 1:  # only normal items
             children[item["parent_id"] or 0].append(item["id"])
 
     tree = []
 
-    async def get_children(item_ids: List[int], depth: int) -> None:
+    def build_tree(item_ids: List[int], depth: int) -> None:
         for item_id in item_ids:
-            item = all_data["agenda/item"][item_id]
+            item = agenda_items[item_id]
             title_information = item["title_information"]
             title_information["_agenda_item_number"] = item["item_number"]
             tree.append(
@@ -57,25 +62,29 @@ async def get_flat_tree(all_data: AllData, parent_id: int = 0) -> List[Dict[str,
                     "depth": depth,
                 }
             )
-            await get_children(children[item_id], depth + 1)
+            build_tree(children[item_id], depth + 1)
 
-    await get_children(children[parent_id], 0)
+    build_tree(children[parent_id], 0)
     return tree
 
 
 async def item_list_slide(
-    all_data: AllData, element: Dict[str, Any], projector_id: int
+    all_data_provider: ProjectorAllDataProvider,
+    element: Dict[str, Any],
+    projector_id: int,
 ) -> Dict[str, Any]:
     """
     Item list slide.
 
     Returns all root items or all children of an item.
     """
-    only_main_items = element.get("only_main_items", True)
+    # fetch all items, so they are cached:
+    all_agenda_items = await all_data_provider.get_collection("agenda/item")
 
+    only_main_items = element.get("only_main_items", True)
     if only_main_items:
         agenda_items = []
-        for item in await get_sorted_agenda_items(all_data):
+        for item in await get_sorted_agenda_items(all_agenda_items):
             if item["parent_id"] is None and item["type"] == 1:
                 title_information = item["title_information"]
                 title_information["_agenda_item_number"] = item["item_number"]
@@ -86,13 +95,15 @@ async def item_list_slide(
                     }
                 )
     else:
-        agenda_items = await get_flat_tree(all_data)
+        agenda_items = await get_flat_tree(all_agenda_items)
 
     return {"items": agenda_items}
 
 
 async def list_of_speakers_slide(
-    all_data: AllData, element: Dict[str, Any], projector_id: int
+    all_data_provider: ProjectorAllDataProvider,
+    element: Dict[str, Any],
+    projector_id: int,
 ) -> Dict[str, Any]:
     """
     List of speakers slide.
@@ -104,35 +115,35 @@ async def list_of_speakers_slide(
     if list_of_speakers_id is None:
         raise ProjectorElementException("id is required for list of speakers slide")
 
-    return await get_list_of_speakers_slide_data(all_data, list_of_speakers_id)
+    return await get_list_of_speakers_slide_data(all_data_provider, list_of_speakers_id)
 
 
 async def get_list_of_speakers_slide_data(
-    all_data: AllData, list_of_speakers_id: int
+    all_data_provider: ProjectorAllDataProvider, list_of_speakers_id: int
 ) -> Dict[str, Any]:
-    try:
+    list_of_speakers = await get_model(
-        list_of_speakers = all_data["agenda/list-of-speakers"][list_of_speakers_id]
+        all_data_provider, "agenda/list-of-speakers", list_of_speakers_id
-    except KeyError:
-        raise ProjectorElementException(
-            f"List of speakers {list_of_speakers_id} does not exist"
     )
 
     title_information = list_of_speakers["title_information"]
     # try to get the agenda item for the content object (which must not exist)
-    agenda_item_id = all_data[list_of_speakers["content_object"]["collection"]][
+    content_object = await get_model(
-        list_of_speakers["content_object"]["id"]
+        all_data_provider,
-    ].get("agenda_item_id")
+        list_of_speakers["content_object"]["collection"],
-    if agenda_item_id:
+        list_of_speakers["content_object"]["id"],
-        title_information["_agenda_item_number"] = all_data["agenda/item"][
+    )
-            agenda_item_id
+    agenda_item_id = content_object.get("agenda_item_id")
-        ]["item_number"]
+    if agenda_item_id is not None:
+        agenda_item = await all_data_provider.get("agenda/item", agenda_item_id)
+        if agenda_item is not None:
+            title_information["_agenda_item_number"] = agenda_item["item_number"]
 
     # Partition speaker objects to waiting, current and finished
     speakers_waiting = []
     speakers_finished = []
     current_speaker = None
     for speaker in list_of_speakers["speakers"]:
-        user = await get_user_name(all_data, speaker["user_id"])
+        user = await get_user_name(all_data_provider, speaker["user_id"])
         formatted_speaker = {
             "user": user,
             "marked": speaker["marked"],
@@ -151,8 +162,12 @@ async def get_list_of_speakers_slide_data(
     speakers_waiting = sorted(speakers_waiting, key=lambda s: s["weight"])
     speakers_finished = sorted(speakers_finished, key=lambda s: s["end_time"])
 
-    number_of_last_speakers = await get_config(all_data, "agenda_show_last_speakers")
+    number_of_last_speakers = await get_config(
-    number_of_next_speakers = await get_config(all_data, "agenda_show_next_speakers")
+        all_data_provider, "agenda_show_last_speakers"
+    )
+    number_of_next_speakers = await get_config(
+        all_data_provider, "agenda_show_next_speakers"
+    )
 
     if number_of_last_speakers == 0:
         speakers_finished = []
@@ -174,7 +189,7 @@ async def get_list_of_speakers_slide_data(
 
 
 async def get_current_list_of_speakers_id_for_projector(
-    all_data: AllData, projector: Dict[str, Any]
+    all_data_provider: ProjectorAllDataProvider, projector: Dict[str, Any]
 ) -> Union[int, None]:
     """
     Search for elements, that do have a list of speakers:
@@ -189,94 +204,88 @@ async def get_current_list_of_speakers_id_for_projector(
             continue
         collection = element["name"]
         id = element["id"]
-        if collection not in all_data or id not in all_data[collection]:
+        model = await all_data_provider.get(collection, id)
+        if model is None:
             continue
 
-        model = all_data[collection][id]
         if "list_of_speakers_id" not in model:
             continue
 
-        if not model["list_of_speakers_id"] in all_data["agenda/list-of-speakers"]:
+        list_of_speakers_id = model["list_of_speakers_id"]
+        los_exists = await all_data_provider.exists(
+            "agenda/list-of-speakers", list_of_speakers_id
+        )
+        if not los_exists:
             continue
 
-        list_of_speakers_id = model["list_of_speakers_id"]
         break
 
     return list_of_speakers_id
 
 
 async def get_reference_projector(
-    all_data: AllData, projector_id: int
+    all_data_provider: ProjectorAllDataProvider, projector_id: int
 ) -> Dict[str, Any]:
     """
     Returns the reference projector to the given projector (by id)
     """
-    try:
+    this_projector = await get_model(all_data_provider, "core/projector", projector_id)
-        this_projector = all_data["core/projector"][projector_id]
-    except KeyError:
-        raise ProjectorElementException(f"Projector {projector_id} does not exist")
 
     reference_projector_id = this_projector["reference_projector_id"] or projector_id
-    try:
+    return await get_model(all_data_provider, "core/projector", reference_projector_id)
-        reference_projector = all_data["core/projector"][reference_projector_id]
-    except KeyError:
-        raise ProjectorElementException(
-            f"Projector {reference_projector_id} does not exist"
-        )
-
-    return reference_projector
 
 
 async def current_list_of_speakers_slide(
-    all_data: AllData, element: Dict[str, Any], projector_id: int
+    all_data_provider: ProjectorAllDataProvider,
+    element: Dict[str, Any],
+    projector_id: int,
 ) -> Dict[str, Any]:
     """
     The current list of speakers slide. Creates the data for the given projector.
     """
-    reference_projector = await get_reference_projector(all_data, projector_id)
+    reference_projector = await get_reference_projector(all_data_provider, projector_id)
     list_of_speakers_id = await get_current_list_of_speakers_id_for_projector(
-        all_data, reference_projector
+        all_data_provider, reference_projector
     )
     if list_of_speakers_id is None:  # no element found
         return {}
 
-    return await get_list_of_speakers_slide_data(all_data, list_of_speakers_id)
+    return await get_list_of_speakers_slide_data(all_data_provider, list_of_speakers_id)
 
 
 async def current_speaker_chyron_slide(
-    all_data: AllData, element: Dict[str, Any], projector_id: int
+    all_data_provider: ProjectorAllDataProvider,
+    element: Dict[str, Any],
+    projector_id: int,
 ) -> Dict[str, Any]:
     """
     Returns the username for the current speaker.
     """
     # get projector for color information
-    projector = all_data["core/projector"][projector_id]
+    projector = await get_model(all_data_provider, "core/projector", projector_id)
 
     slide_data = {
         "background_color": projector["chyron_background_color"],
         "font_color": projector["chyron_font_color"],
     }
 
-    reference_projector = await get_reference_projector(all_data, projector_id)
+    reference_projector = await get_reference_projector(all_data_provider, projector_id)
     list_of_speakers_id = await get_current_list_of_speakers_id_for_projector(
-        all_data, reference_projector
+        all_data_provider, reference_projector
     )
     if list_of_speakers_id is None:  # no element found
         return slide_data
 
     # get list of speakers to search current speaker
-    try:
+    list_of_speakers = await get_model(
-        list_of_speakers = all_data["agenda/list-of-speakers"][list_of_speakers_id]
+        all_data_provider, "agenda/list-of-speakers", list_of_speakers_id
-    except KeyError:
-        raise ProjectorElementException(
-            f"List of speakers {list_of_speakers_id} does not exist"
     )
 
     # find current speaker
     current_speaker = None
     for speaker in list_of_speakers["speakers"]:
         if speaker["begin_time"] is not None and speaker["end_time"] is None:
-            current_speaker = await get_user_name(all_data, speaker["user_id"])
+            current_speaker = await get_user_name(all_data_provider, speaker["user_id"])
             break
 
     if current_speaker is not None:

openslides/assignments/projector.py

@@ -1,25 +1,29 @@
 from typing import Any, Dict, List
 
 from ..users.projector import get_user_name
-from ..utils.projector import AllData, get_model, get_models, register_projector_slide
+from ..utils.projector import (
+    ProjectorAllDataProvider,
+    get_model,
+    get_models,
+    register_projector_slide,
+)
 from .models import AssignmentPoll
 
 
-# Important: All functions have to be prune. This means, that thay can only
-#            access the data, that they get as argument and do not have any
-#            side effects.
-
-
 async def assignment_slide(
-    all_data: AllData, element: Dict[str, Any], projector_id: int
+    all_data_provider: ProjectorAllDataProvider,
+    element: Dict[str, Any],
+    projector_id: int,
 ) -> Dict[str, Any]:
     """
     Assignment slide.
     """
-    assignment = get_model(all_data, "assignments/assignment", element.get("id"))
+    assignment = await get_model(
+        all_data_provider, "assignments/assignment", element.get("id")
+    )
 
     assignment_related_users: List[Dict[str, Any]] = [
-        {"user": await get_user_name(all_data, aru["user_id"])}
+        {"user": await get_user_name(all_data_provider, aru["user_id"])}
         for aru in sorted(
             assignment["assignment_related_users"], key=lambda aru: aru["weight"]
         )
@@ -36,13 +40,19 @@ async def assignment_slide(
 
 
 async def assignment_poll_slide(
-    all_data: AllData, element: Dict[str, Any], projector_id: int
+    all_data_provider: ProjectorAllDataProvider,
+    element: Dict[str, Any],
+    projector_id: int,
 ) -> Dict[str, Any]:
     """
     Poll slide.
     """
-    poll = get_model(all_data, "assignments/assignment-poll", element.get("id"))
+    poll = await get_model(
-    assignment = get_model(all_data, "assignments/assignment", poll["assignment_id"])
+        all_data_provider, "assignments/assignment-poll", element.get("id")
+    )
+    assignment = await get_model(
+        all_data_provider, "assignments/assignment", poll["assignment_id"]
+    )
 
     poll_data = {
         key: poll[key]
@@ -60,10 +70,14 @@ async def assignment_poll_slide(
 
     # Add options:
     poll_data["options"] = []
-    options = get_models(all_data, "assignments/assignment-option", poll["options_id"])
+    options = await get_models(
+        all_data_provider, "assignments/assignment-option", poll["options_id"]
+    )
     for option in sorted(options, key=lambda option: option["weight"]):
         option_data: Dict[str, Any] = {
-            "user": {"short_name": await get_user_name(all_data, option["user_id"])}
+            "user": {
+                "short_name": await get_user_name(all_data_provider, option["user_id"])
+            }
         }
         if poll["state"] == AssignmentPoll.STATE_PUBLISHED:
             option_data["yes"] = float(option["yes"])

openslides/core/projector.py

@@ -1,20 +1,17 @@
 from typing import Any, Dict
 
 from ..utils.projector import (
-    AllData,
+    ProjectorAllDataProvider,
-    ProjectorElementException,
     get_config,
+    get_model,
     register_projector_slide,
 )
 
 
-# Important: All functions have to be prune. This means, that thay can only
-#            access the data, that they get as argument and do not have any
-#            side effects.
-
-
 async def countdown_slide(
-    all_data: AllData, element: Dict[str, Any], projector_id: int
+    all_data_provider: ProjectorAllDataProvider,
+    element: Dict[str, Any],
+    projector_id: int,
 ) -> Dict[str, Any]:
     """
     Countdown slide.
@@ -26,23 +23,21 @@ async def countdown_slide(
         id: 5,  # Countdown ID
     }
     """
-    countdown_id = element.get("id") or 1
+    countdown = await get_model(all_data_provider, "core/countdown", element.get("id"))
-
-    try:
-        countdown = all_data["core/countdown"][countdown_id]
-    except KeyError:
-        raise ProjectorElementException(f"Countdown {countdown_id} does not exist")
-
     return {
         "description": countdown["description"],
         "running": countdown["running"],
         "countdown_time": countdown["countdown_time"],
-        "warning_time": await get_config(all_data, "agenda_countdown_warning_time"),
+        "warning_time": await get_config(
+            all_data_provider, "agenda_countdown_warning_time"
+        ),
     }
 
 
 async def message_slide(
-    all_data: AllData, element: Dict[str, Any], projector_id: int
+    all_data_provider: ProjectorAllDataProvider,
+    element: Dict[str, Any],
+    projector_id: int,
 ) -> Dict[str, Any]:
     """
     Message slide.
@@ -54,16 +49,15 @@ async def message_slide(
         id: 5,  # ProjectorMessage ID
     }
     """
-    message_id = element.get("id") or 1
+    return await get_model(
-
+        all_data_provider, "core/projector-message", element.get("id")
-    try:
+    )
-        return all_data["core/projector-message"][message_id]
-    except KeyError:
-        raise ProjectorElementException(f"Message {message_id} does not exist")
 
 
 async def clock_slide(
-    all_data: AllData, element: Dict[str, Any], projector_id: int
+    all_data_provider: ProjectorAllDataProvider,
+    element: Dict[str, Any],
+    projector_id: int,
 ) -> Dict[str, Any]:
     return {}
 

openslides/mediafiles/projector.py

@@ -1,35 +1,23 @@
 from typing import Any, Dict
 
 from ..utils.projector import (
-    AllData,
+    ProjectorAllDataProvider,
-    ProjectorElementException,
+    get_model,
     register_projector_slide,
 )
 
 
-# Important: All functions have to be prune. This means, that thay can only
-#            access the data, that they get as argument and do not have any
-#            side effects.
-
-
 async def mediafile_slide(
-    all_data: AllData, element: Dict[str, Any], projector_id: int
+    all_data_provider: ProjectorAllDataProvider,
+    element: Dict[str, Any],
+    projector_id: int,
 ) -> Dict[str, Any]:
     """
     Slide for Mediafile.
     """
-    mediafile_id = element.get("id")
+    mediafile = await get_model(
-
+        all_data_provider, "mediafiles/mediafile", element.get("id")
-    if mediafile_id is None:
-        raise ProjectorElementException("id is required for mediafile slide")
-
-    try:
-        mediafile = all_data["mediafiles/mediafile"][mediafile_id]
-    except KeyError:
-        raise ProjectorElementException(
-            f"mediafile with id {mediafile_id} does not exist"
     )
-
     return {
         "path": mediafile["path"],
         "mimetype": mediafile["mimetype"],

openslides/motions/projector.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional
 
 from ..users.projector import get_user_name
 from ..utils.projector import (
-    AllData,
+    ProjectorAllDataProvider,
     ProjectorElementException,
     get_config,
     get_model,
@@ -14,33 +14,31 @@ from .models import MotionPoll
 
 motion_placeholder_regex = re.compile(r"\[motion:(\d+)\]")
 
-# Important: All functions have to be prune. This means, that thay can only
-#            access the data, that they get as argument and do not have any
-#            side effects.
-
 
 async def get_state(
-    all_data: AllData, motion: Dict[str, Any], state_id_key: str
+    all_data_provider: ProjectorAllDataProvider,
+    motion: Dict[str, Any],
+    state_id_key: str,
 ) -> Dict[str, Any]:
     """
     Returns a state element from one motion. Raises an error if the state does not exist.
     """
-    state = all_data["motions/state"].get(motion[state_id_key])
+    state = await all_data_provider.get("motions/state", motion[state_id_key])
-    if not state:
+    if state is None:
         raise ProjectorElementException(
             f"motion {motion['id']} can not be on the state with id {motion[state_id_key]}"
         )
     return state
 
 
-async def get_amendment_merge_into_motion_diff(all_data, amendment):
+async def get_amendment_merge_into_motion_diff(all_data_provider, amendment):
     """
     HINT: This implementation should be consistent to showInDiffView() in ViewMotionAmendedParagraph.ts
     """
     if amendment["state_id"] is None:
         return 0
 
-    state = await get_state(all_data, amendment, "state_id")
+    state = await get_state(all_data_provider, amendment, "state_id")
     if state["merge_amendment_into_final"] == -1:
         return 0
     if state["merge_amendment_into_final"] == 1:
@@ -48,36 +46,37 @@ async def get_amendment_merge_into_motion_diff(all_data, amendment):
 
     if amendment["recommendation_id"] is None:
         return 0
-    recommendation = await get_state(all_data, amendment, "recommendation_id")
+    recommendation = await get_state(all_data_provider, amendment, "recommendation_id")
     if recommendation["merge_amendment_into_final"] == 1:
         return 1
 
     return 0
 
 
-async def get_amendment_merge_into_motion_final(all_data, amendment):
+async def get_amendment_merge_into_motion_final(all_data_provider, amendment):
     """
     HINT: This implementation should be consistent to showInFinalView() in ViewMotionAmendedParagraph.ts
     """
     if amendment["state_id"] is None:
         return 0
 
-    state = await get_state(all_data, amendment, "state_id")
+    state = await get_state(all_data_provider, amendment, "state_id")
     if state["merge_amendment_into_final"] == 1:
         return 1
 
     return 0
 
 
-async def get_amendments_for_motion(motion, all_data):
+async def get_amendments_for_motion(motion, all_data_provider):
     amendment_data = []
-    for amendment_id, amendment in all_data["motions/motion"].items():
+    all_motions = await all_data_provider.get_collection("motions/motion")
+    for amendment_id, amendment in all_motions.items():
         if amendment["parent_id"] == motion["id"]:
             merge_amendment_into_final = await get_amendment_merge_into_motion_final(
-                all_data, amendment
+                all_data_provider, amendment
             )
             merge_amendment_into_diff = await get_amendment_merge_into_motion_diff(
-                all_data, amendment
+                all_data_provider, amendment
             )
             amendment_data.append(
                 {
@@ -92,8 +91,10 @@ async def get_amendments_for_motion(motion, all_data):
     return amendment_data
 
 
-async def get_amendment_base_motion(amendment, all_data):
+async def get_amendment_base_motion(amendment, all_data_provider):
-    motion = get_model(all_data, "motions/motion", amendment.get("parent_id"))
+    motion = await get_model(
+        all_data_provider, "motions/motion", amendment.get("parent_id")
+    )
 
     return {
         "identifier": motion["identifier"],
@@ -102,15 +103,17 @@ async def get_amendment_base_motion(amendment, all_data):
     }
 
 
-async def get_amendment_base_statute(amendment, all_data):
+async def get_amendment_base_statute(amendment, all_data_provider):
-    statute = get_model(
+    statute = await get_model(
-        all_data, "motions/statute-paragraph", amendment.get("statute_paragraph_id")
+        all_data_provider,
+        "motions/statute-paragraph",
+        amendment.get("statute_paragraph_id"),
     )
     return {"title": statute["title"], "text": statute["text"]}
 
 
 async def extend_reference_motion_dict(
-    all_data: AllData,
+    all_data_provider: ProjectorAllDataProvider,
     recommendation: Optional[str],
     referenced_motions: Dict[int, Dict[str, str]],
 ) -> None:
@@ -127,15 +130,18 @@ async def extend_reference_motion_dict(
     ]
     for id in referenced_ids:
         # Put every referenced motion into the referenced_motions dict
-        if id not in referenced_motions and id in all_data["motions/motion"]:
+        referenced_motion = await all_data_provider.get("motions/motion", id)
+        if id not in referenced_motions and referenced_motion is not None:
             referenced_motions[id] = {
-                "title": all_data["motions/motion"][id]["title"],
+                "title": referenced_motion["title"],
-                "identifier": all_data["motions/motion"][id]["identifier"],
+                "identifier": referenced_motion["identifier"],
             }
 
 
 async def motion_slide(
-    all_data: AllData, element: Dict[str, Any], projector_id: int
+    all_data_provider: ProjectorAllDataProvider,
+    element: Dict[str, Any],
+    projector_id: int,
 ) -> Dict[str, Any]:
     """
     Motion slide.
@@ -158,13 +164,16 @@ async def motion_slide(
     """
     # Get motion
     mode = element.get(
-        "mode", await get_config(all_data, "motions_recommendation_text_mode")
+        "mode", await get_config(all_data_provider, "motions_recommendation_text_mode")
     )
-    motion = get_model(all_data, "motions/motion", element.get("id"))
+
+    # populate cache:
+
+    motion = await get_model(all_data_provider, "motions/motion", element.get("id"))
 
     # Add submitters
     submitters = [
-        await get_user_name(all_data, submitter["user_id"])
+        await get_user_name(all_data_provider, submitter["user_id"])
         for submitter in sorted(
             motion["submitters"], key=lambda submitter: submitter["weight"]
         )
@@ -172,14 +181,16 @@ async def motion_slide(
 
     # Get some needed config values
     show_meta_box = not await get_config(
-        all_data, "motions_disable_sidebox_on_projector"
+        all_data_provider, "motions_disable_sidebox_on_projector"
     )
     show_referring_motions = not await get_config(
-        all_data, "motions_hide_referring_motions"
+        all_data_provider, "motions_hide_referring_motions"
     )
-    line_length = await get_config(all_data, "motions_line_length")
+    line_length = await get_config(all_data_provider, "motions_line_length")
-    line_numbering_mode = await get_config(all_data, "motions_default_line_numbering")
+    line_numbering_mode = await get_config(
-    motions_preamble = await get_config(all_data, "motions_preamble")
+        all_data_provider, "motions_default_line_numbering"
+    )
+    motions_preamble = await get_config(all_data_provider, "motions_preamble")
 
     # Query all change-recommendation and amendment related things.
     change_recommendations = []  # type: ignore
@@ -187,17 +198,19 @@ async def motion_slide(
     base_motion = None
     base_statute = None
     if motion["statute_paragraph_id"]:
-        base_statute = await get_amendment_base_statute(motion, all_data)
+        base_statute = await get_amendment_base_statute(motion, all_data_provider)
     elif motion["parent_id"] is not None and motion["amendment_paragraphs"]:
-        base_motion = await get_amendment_base_motion(motion, all_data)
+        base_motion = await get_amendment_base_motion(motion, all_data_provider)
     else:
         for change_recommendation_id in motion["change_recommendations_id"]:
-            cr = all_data["motions/motion-change-recommendation"].get(
+            cr = await get_model(
-                change_recommendation_id
+                all_data_provider,
+                "motions/motion-change-recommendation",
+                change_recommendation_id,
             )
             if cr is not None and not cr["internal"]:
                 change_recommendations.append(cr)
-        amendments = await get_amendments_for_motion(motion, all_data)
+        amendments = await get_amendments_for_motion(motion, all_data_provider)
 
     # The base return value. More fields will get added below.
     return_value = {
@@ -217,10 +230,10 @@ async def motion_slide(
         "line_numbering_mode": line_numbering_mode,
     }
 
-    if not await get_config(all_data, "motions_disable_text_on_projector"):
+    if not await get_config(all_data_provider, "motions_disable_text_on_projector"):
         return_value["text"] = motion["text"]
 
-    if not await get_config(all_data, "motions_disable_reason_on_projector"):
+    if not await get_config(all_data_provider, "motions_disable_reason_on_projector"):
         return_value["reason"] = motion["reason"]
 
     if mode == "final":
@@ -228,40 +241,46 @@ async def motion_slide(
 
     # Add recommendation, if enabled in config (and the motion has one)
     if (
-        not await get_config(all_data, "motions_disable_recommendation_on_projector")
+        not await get_config(
+            all_data_provider, "motions_disable_recommendation_on_projector"
+        )
         and motion["recommendation_id"]
     ):
-        recommendation_state = await get_state(all_data, motion, "recommendation_id")
+        recommendation_state = await get_state(
+            all_data_provider, motion, "recommendation_id"
+        )
         return_value["recommendation"] = recommendation_state["recommendation_label"]
         if recommendation_state["show_recommendation_extension_field"]:
             recommendation_extension = motion["recommendation_extension"]
             # All title information for referenced motions in the recommendation
             referenced_motions: Dict[int, Dict[str, str]] = {}
             await extend_reference_motion_dict(
-                all_data, recommendation_extension, referenced_motions
+                all_data_provider, recommendation_extension, referenced_motions
             )
             return_value["recommendation_extension"] = recommendation_extension
             return_value["referenced_motions"] = referenced_motions
         if motion["statute_paragraph_id"]:
             return_value["recommender"] = await get_config(
-                all_data, "motions_statute_recommendations_by"
+                all_data_provider, "motions_statute_recommendations_by"
             )
         else:
             return_value["recommender"] = await get_config(
-                all_data, "motions_recommendations_by"
+                all_data_provider, "motions_recommendations_by"
             )
 
     if show_referring_motions:
         # Add recommendation-referencing motions
         return_value[
             "recommendation_referencing_motions"
-        ] = await get_recommendation_referencing_motions(all_data, motion["id"])
+        ] = await get_recommendation_referencing_motions(
+            all_data_provider, motion["id"]
+        )
 
     return return_value
 
 
 async def get_recommendation_referencing_motions(
-    all_data: AllData, motion_id: int
+    all_data_provider: ProjectorAllDataProvider, motion_id: int
 ) -> Optional[List[Dict[str, Any]]]:
     """
     Returns all title information for motions, that are referencing
@@ -269,14 +288,15 @@ async def get_recommendation_referencing_motions(
     motions, None is returned (instead of []).
     """
     recommendation_referencing_motions = []
-    for motion in all_data["motions/motion"].values():
+    all_motions = await all_data_provider.get_collection("motions/motion")
+    for motion in all_motions.values():
         # Motion must have a recommendation and a recommendaiton extension
         if not motion["recommendation_id"] or not motion["recommendation_extension"]:
             continue
 
         # The recommendation must allow the extension field (there might be left-overs
         # in a motions recommendation extension..)
-        recommendation = await get_state(all_data, motion, "recommendation_id")
+        recommendation = await get_state(all_data_provider, motion, "recommendation_id")
         if not recommendation["show_recommendation_extension_field"]:
             continue
 
@@ -297,12 +317,16 @@ async def get_recommendation_referencing_motions(
 
 
 async def motion_block_slide(
-    all_data: AllData, element: Dict[str, Any], projector_id: int
+    all_data_provider: ProjectorAllDataProvider,
+    element: Dict[str, Any],
+    projector_id: int,
 ) -> Dict[str, Any]:
     """
     Motion block slide.
     """
-    motion_block = get_model(all_data, "motions/motion-block", element.get("id"))
+    motion_block = await get_model(
+        all_data_provider, "motions/motion-block", element.get("id")
+    )
 
     # All motions in this motion block
     motions = []
@@ -311,7 +335,8 @@ async def motion_block_slide(
     referenced_motions: Dict[int, Dict[str, str]] = {}
 
     # Search motions.
-    for motion in all_data["motions/motion"].values():
+    all_motions = await all_data_provider.get_collection("motions/motion")
+    for motion in all_motions.values():
         if motion["motion_block_id"] == motion_block["id"]:
             motion_object = {
                 "title": motion["title"],
@@ -320,7 +345,9 @@ async def motion_block_slide(
 
             recommendation_id = motion["recommendation_id"]
             if recommendation_id is not None:
-                recommendation = await get_state(all_data, motion, "recommendation_id")
+                recommendation = await get_state(
+                    all_data_provider, motion, "recommendation_id"
+                )
                 motion_object["recommendation"] = {
                     "name": recommendation["recommendation_label"],
                     "css_class": recommendation["css_class"],
@@ -328,7 +355,7 @@ async def motion_block_slide(
                 if recommendation["show_recommendation_extension_field"]:
                     recommendation_extension = motion["recommendation_extension"]
                     await extend_reference_motion_dict(
-                        all_data, recommendation_extension, referenced_motions
+                        all_data_provider, recommendation_extension, referenced_motions
                     )
                     motion_object["recommendation_extension"] = recommendation_extension
 
@@ -342,13 +369,15 @@ async def motion_block_slide(
 
 
 async def motion_poll_slide(
-    all_data: AllData, element: Dict[str, Any], projector_id: int
+    all_data_provider: ProjectorAllDataProvider,
+    element: Dict[str, Any],
+    projector_id: int,
 ) -> Dict[str, Any]:
     """
     Poll slide.
     """
-    poll = get_model(all_data, "motions/motion-poll", element.get("id"))
+    poll = await get_model(all_data_provider, "motions/motion-poll", element.get("id"))
-    motion = get_model(all_data, "motions/motion", poll["motion_id"])
+    motion = await get_model(all_data_provider, "motions/motion", poll["motion_id"])
 
     poll_data = {
         key: poll[key]
@@ -363,8 +392,8 @@ async def motion_poll_slide(
     }
 
     if poll["state"] == MotionPoll.STATE_PUBLISHED:
-        option = get_model(
+        option = await get_model(
-            all_data, "motions/motion-option", poll["options_id"][0]
+            all_data_provider, "motions/motion-option", poll["options_id"][0]
         )  # there can only be exactly one option
         poll_data["options"] = [
             {

openslides/topics/projector.py

@@ -1,19 +1,16 @@
 from typing import Any, Dict
 
 from ..utils.projector import (
-    AllData,
+    ProjectorAllDataProvider,
-    ProjectorElementException,
+    get_model,
     register_projector_slide,
 )
 
 
-# Important: All functions have to be prune. This means, that thay can only
-#            access the data, that they get as argument and do not have any
-#            side effects.
-
-
 async def topic_slide(
-    all_data: AllData, element: Dict[str, Any], projector_id: int
+    all_data_provider: ProjectorAllDataProvider,
+    element: Dict[str, Any],
+    projector_id: int,
 ) -> Dict[str, Any]:
     """
     Topic slide.
@@ -22,22 +19,8 @@ async def topic_slide(
     * title
     * text
     """
-    topic_id = element.get("id")
+    topic = await get_model(all_data_provider, "topics/topic", element.get("id"))
-
+    item = await get_model(all_data_provider, "agenda/item", topic["agenda_item_id"])
-    if topic_id is None:
-        raise ProjectorElementException("id is required for topic slide")
-
-    try:
-        topic = all_data["topics/topic"][topic_id]
-    except KeyError:
-        raise ProjectorElementException(f"topic with id {topic_id} does not exist")
-
-    item_id = topic["agenda_item_id"]
-    try:
-        item = all_data["agenda/item"][item_id]
-    except KeyError:
-        raise ProjectorElementException(f"item with id {item_id} does not exist")
-
     return {
         "title": topic["title"],
         "text": topic["text"],

openslides/users/projector.py

@@ -1,19 +1,16 @@
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from ..utils.projector import (
-    AllData,
+    ProjectorAllDataProvider,
-    ProjectorElementException,
+    get_model,
     register_projector_slide,
 )
 
 
-# Important: All functions have to be prune. This means, that thay can only
-#            access the data, that they get as argument and do not have any
-#            side effects.
-
-
 async def user_slide(
-    all_data: AllData, element: Dict[str, Any], projector_id: int
+    all_data_provider: ProjectorAllDataProvider,
+    element: Dict[str, Any],
+    projector_id: int,
 ) -> Dict[str, Any]:
     """
     User slide.
@@ -21,22 +18,16 @@ async def user_slide(
     The returned dict can contain the following fields:
     * user
     """
-    user_id = element.get("id")
+    return {"user": await get_user_name(all_data_provider, element.get("id"))}
-
-    if user_id is None:
-        raise ProjectorElementException("id is required for user slide")
-
-    return {"user": await get_user_name(all_data, user_id)}
 
 
-async def get_user_name(all_data: AllData, user_id: int) -> str:
+async def get_user_name(
+    all_data_provider: ProjectorAllDataProvider, user_id: Optional[int]
+) -> str:
     """
     Returns the short name for an user_id.
     """
-    try:
+    user = await get_model(all_data_provider, "users/user", user_id)
-        user = all_data["users/user"][user_id]
-    except KeyError:
-        raise ProjectorElementException(f"user with id {user_id} does not exist")
 
     name_parts: List[str] = []
     for name_part in ("title", "first_name", "last_name"):

openslides/utils/autoupdate.py

@@ -92,13 +92,13 @@ class AutoupdateBundle:
                     elements[full_data["id"]]["full_data"] = full_data
 
         # Save histroy here using sync code.
-        save_history(self.elements)
+        save_history(self.element_iterator)
 
         # Update cache and send autoupdate using async code.
         async_to_sync(self.async_handle_collection_elements)()
 
     @property
-    def elements(self) -> Iterable[AutoupdateElement]:
+    def element_iterator(self) -> Iterable[AutoupdateElement]:
         """ Iterator for all elements in this bundle """
         for elements in self.autoupdate_elements.values():
             yield from elements.values()
@@ -110,7 +110,7 @@ class AutoupdateBundle:
         Returns the change_id
         """
         cache_elements: Dict[str, Optional[Dict[str, Any]]] = {}
-        for element in self.elements:
+        for element in self.element_iterator:
             element_id = get_element_id(element["collection_string"], element["id"])
             full_data = element.get("full_data")
             if full_data:
@@ -253,7 +253,7 @@ class AutoupdateBundleMiddleware:
         return response
 
 
-def save_history(elements: Iterable[AutoupdateElement]) -> Iterable:
+def save_history(element_iterator: Iterable[AutoupdateElement]) -> Iterable:
     """
     Thin wrapper around the call of history saving manager method.
 
@@ -261,4 +261,4 @@ def save_history(elements: Iterable[AutoupdateElement]) -> Iterable:
     """
     from ..core.models import History
 
-    return History.objects.add_elements(elements)
+    return History.objects.add_elements(element_iterator)

openslides/utils/cache.py

@@ -254,25 +254,6 @@ class ElementCache:
                 all_data[collection] = await restricter(user_id, all_data[collection])
         return dict(all_data)
 
-    async def get_all_data_dict(self) -> Dict[str, Dict[int, Dict[str, Any]]]:
-        """
-        Returns all data with a dict (id <-> element) per collection:
-        {
-            <collection>: {
-                <id>: <element>
-            }
-        }
-        """
-        all_data: Dict[str, Dict[int, Dict[str, Any]]] = defaultdict(dict)
-        for element_id, data in (await self.cache_provider.get_all_data()).items():
-            collection, id = split_element_id(element_id)
-            element = json.loads(data.decode())
-            element.pop(
-                "_no_delete_on_restriction", False
-            )  # remove special field for get_data_since
-            all_data[collection][id] = element
-        return dict(all_data)
-
     async def get_collection_data(self, collection: str) -> Dict[int, Dict[str, Any]]:
         """
         Returns the data for one collection as dict: {id: <element>}

openslides/utils/cache_providers.py

@@ -460,7 +460,8 @@ class RedisCacheProvider:
                 )
                 if reported_amount != read_only_redis_amount_replicas:
                     logger.warn(
-                        f"WAIT reported {reported_amount} replicas of {read_only_redis_amount_replicas} requested after {read_only_redis_wait_timeout} ms!"
+                        f"WAIT reported {reported_amount} replicas of {read_only_redis_amount_replicas} "
+                        + f"requested after {read_only_redis_wait_timeout} ms!"
                     )
             return result
 

openslides/utils/projector.py

@@ -5,16 +5,14 @@ Functions  that handel the registration of projector elements and the rendering
 of the data to present it on the projector.
 """
 
-from typing import Any, Awaitable, Callable, Dict, List
+from collections import defaultdict
+from typing import Any, Awaitable, Callable, Dict, List, Optional
 
+from . import logging
 from .cache import element_cache
 
 
-AllData = Dict[str, Dict[int, Dict[str, Any]]]
+logger = logging.getLogger(__name__)
-ProjectorSlide = Callable[[AllData, Dict[str, Any], int], Awaitable[Dict[str, Any]]]
-
-
-projector_slides: Dict[str, ProjectorSlide] = {}
 
 
 class ProjectorElementException(Exception):
@@ -23,6 +21,44 @@ class ProjectorElementException(Exception):
     """
 
 
+class ProjectorAllDataProvider:
+    NON_EXISTENT_MARKER = object()
+
+    def __init__(self) -> None:
+        self.cache: Any = defaultdict(dict)  # fuu you mypy
+        self.fetched_collection: Dict[str, bool] = {}
+
+    async def get(self, collection: str, id: int) -> Optional[Dict[str, Any]]:
+        cache_data = self.cache[collection].get(id)
+        if cache_data is None:
+            data: Any = await element_cache.get_element_data(collection, id)
+            if data is None:
+                data = ProjectorAllDataProvider.NON_EXISTENT_MARKER
+            self.cache[collection][id] = data
+        elif cache_data == ProjectorAllDataProvider.NON_EXISTENT_MARKER:
+            return None
+        return self.cache[collection][id]
+
+    async def get_collection(self, collection: str) -> Dict[int, Dict[str, Any]]:
+        if not self.fetched_collection.get(collection, False):
+            collection_data = await element_cache.get_collection_data(collection)
+            self.cache[collection] = collection_data
+            self.fetched_collection[collection] = True
+        return self.cache[collection]
+
+    async def exists(self, collection: str, id: int) -> bool:
+        model = await self.get(collection, id)
+        return model is not None
+
+
+ProjectorSlide = Callable[
+    [ProjectorAllDataProvider, Dict[str, Any], int], Awaitable[Dict[str, Any]]
+]
+
+
+projector_slides: Dict[str, ProjectorSlide] = {}
+
+
 def register_projector_slide(name: str, slide: ProjectorSlide) -> None:
     """
     Registers a projector slide.
@@ -67,10 +103,11 @@ async def get_projector_data(
     if projector_ids is None:
         projector_ids = []
 
-    all_data = await element_cache.get_all_data_dict()
     projector_data: Dict[int, List[Dict[str, Any]]] = {}
+    all_data_provider = ProjectorAllDataProvider()
+    projectors = await all_data_provider.get_collection("core/projector")
 
-    for projector_id, projector in all_data.get("core/projector", {}).items():
+    for projector_id, projector in projectors.items():
         if projector_ids and projector_id not in projector_ids:
             # only render the projector in question.
             continue
@@ -83,7 +120,7 @@ async def get_projector_data(
         for element in projector["elements"]:
             projector_slide = projector_slides[element["name"]]
             try:
-                data = await projector_slide(all_data, element, projector_id)
+                data = await projector_slide(all_data_provider, element, projector_id)
             except ProjectorElementException as err:
                 data = {"error": str(err)}
             projector_data[projector_id].append({"data": data, "element": element})
@@ -91,18 +128,23 @@ async def get_projector_data(
     return projector_data
 
 
-async def get_config(all_data: AllData, key: str) -> Any:
+async def get_config(all_data_provider: ProjectorAllDataProvider, key: str) -> Any:
     """
-    Returns a config value from all_data.
+    Returns a config value from all_data_provider.
+    Triggers the cache early: It access `get_colelction` instead of `get`. It
+    allows for all successive queries for configs to be cached.
     """
     from ..core.config import config
 
     config_id = (await config.async_get_key_to_id())[key]
 
-    return all_data[config.get_collection_string()][config_id]["value"]
+    configs = await all_data_provider.get_collection(config.get_collection_string())
+    return configs[config_id]["value"]
 
 
-def get_model(all_data: AllData, collection: str, id: Any) -> Dict[str, Any]:
+async def get_model(
+    all_data_provider: ProjectorAllDataProvider, collection: str, id: Any
+) -> Dict[str, Any]:
     """
     Tries to get the model identified by the collection and id.
     If the id is invalid or the model not found, ProjectorElementExceptions will be raised.
@@ -110,17 +152,19 @@ def get_model(all_data: AllData, collection: str, id: Any) -> Dict[str, Any]:
     if id is None:
         raise ProjectorElementException(f"id is required for {collection} slide")
 
-    try:
+    model = await all_data_provider.get(collection, id)
-        model = all_data[collection][id]
+    if model is None:
-    except KeyError:
         raise ProjectorElementException(f"{collection} with id {id} does not exist")
     return model
 
 
-def get_models(
+async def get_models(
-    all_data: AllData, collection: str, ids: List[Any]
+    all_data_provider: ProjectorAllDataProvider, collection: str, ids: List[Any]
 ) -> List[Dict[str, Any]]:
     """
     Tries to fetch all given models. Models are required to be all of the collection `collection`.
     """
-    return [get_model(all_data, collection, id) for id in ids]
+    logger.info(
+        f"Note: a call to `get_models` with {collection}/{ids}. This might be cache-intensive"
+    )
+    return [await get_model(all_data_provider, collection, id) for id in ids]