diff --git a/openslides/agenda/projector.py b/openslides/agenda/projector.py index 36d0a47d4..eb8b305ca 100644 --- a/openslides/agenda/projector.py +++ b/openslides/agenda/projector.py @@ -3,9 +3,10 @@ from typing import Any, Dict, List, Union from ..users.projector import get_user_name from ..utils.projector import ( - AllData, + ProjectorAllDataProvider, ProjectorElementException, get_config, + get_model, register_projector_slide, ) @@ -15,20 +16,24 @@ from ..utils.projector import ( # side effects. -async def get_sorted_agenda_items(all_data: AllData) -> List[Dict[str, Any]]: +async def get_sorted_agenda_items( + agenda_items: Dict[int, Dict[str, Any]] +) -> List[Dict[str, Any]]: """ Returns all sorted agenda items by id first and then weight, resulting in ordered items, if some have the same weight. """ return sorted( - sorted(all_data["agenda/item"].values(), key=lambda item: item["id"]), + sorted(agenda_items.values(), key=lambda item: item["id"]), key=lambda item: item["weight"], ) -async def get_flat_tree(all_data: AllData, parent_id: int = 0) -> List[Dict[str, Any]]: +async def get_flat_tree( + agenda_items: Dict[int, Dict[str, Any]], parent_id: int = 0 +) -> List[Dict[str, Any]]: """ - Build the item tree from all_data. + Build the item tree from all_data_provider. Only build the tree from elements unterneath parent_id. @@ -38,16 +43,16 @@ async def get_flat_tree(all_data: AllData, parent_id: int = 0) -> List[Dict[str, # Build a dict from an item_id to all its children children: Dict[int, List[int]] = defaultdict(list) - if "agenda/item" in all_data: - for item in await get_sorted_agenda_items(all_data): - if item["type"] == 1: # only normal items - children[item["parent_id"] or 0].append(item["id"]) + + for item in await get_sorted_agenda_items(agenda_items): + if item["type"] == 1: # only normal items + children[item["parent_id"] or 0].append(item["id"]) tree = [] - async def get_children(item_ids: List[int], depth: int) -> None: + def build_tree(item_ids: List[int], depth: int) -> None: for item_id in item_ids: - item = all_data["agenda/item"][item_id] + item = agenda_items[item_id] title_information = item["title_information"] title_information["_agenda_item_number"] = item["item_number"] tree.append( @@ -57,25 +62,29 @@ async def get_flat_tree(all_data: AllData, parent_id: int = 0) -> List[Dict[str, "depth": depth, } ) - await get_children(children[item_id], depth + 1) + build_tree(children[item_id], depth + 1) - await get_children(children[parent_id], 0) + build_tree(children[parent_id], 0) return tree async def item_list_slide( - all_data: AllData, element: Dict[str, Any], projector_id: int + all_data_provider: ProjectorAllDataProvider, + element: Dict[str, Any], + projector_id: int, ) -> Dict[str, Any]: """ Item list slide. Returns all root items or all children of an item. """ - only_main_items = element.get("only_main_items", True) + # fetch all items, so they are cached: + all_agenda_items = await all_data_provider.get_collection("agenda/item") + only_main_items = element.get("only_main_items", True) if only_main_items: agenda_items = [] - for item in await get_sorted_agenda_items(all_data): + for item in await get_sorted_agenda_items(all_agenda_items): if item["parent_id"] is None and item["type"] == 1: title_information = item["title_information"] title_information["_agenda_item_number"] = item["item_number"] @@ -86,13 +95,15 @@ async def item_list_slide( } ) else: - agenda_items = await get_flat_tree(all_data) + agenda_items = await get_flat_tree(all_agenda_items) return {"items": agenda_items} async def list_of_speakers_slide( - all_data: AllData, element: Dict[str, Any], projector_id: int + all_data_provider: ProjectorAllDataProvider, + element: Dict[str, Any], + projector_id: int, ) -> Dict[str, Any]: """ List of speakers slide. @@ -104,35 +115,35 @@ async def list_of_speakers_slide( if list_of_speakers_id is None: raise ProjectorElementException("id is required for list of speakers slide") - return await get_list_of_speakers_slide_data(all_data, list_of_speakers_id) + return await get_list_of_speakers_slide_data(all_data_provider, list_of_speakers_id) async def get_list_of_speakers_slide_data( - all_data: AllData, list_of_speakers_id: int + all_data_provider: ProjectorAllDataProvider, list_of_speakers_id: int ) -> Dict[str, Any]: - try: - list_of_speakers = all_data["agenda/list-of-speakers"][list_of_speakers_id] - except KeyError: - raise ProjectorElementException( - f"List of speakers {list_of_speakers_id} does not exist" - ) + list_of_speakers = await get_model( + all_data_provider, "agenda/list-of-speakers", list_of_speakers_id + ) title_information = list_of_speakers["title_information"] # try to get the agenda item for the content object (which must not exist) - agenda_item_id = all_data[list_of_speakers["content_object"]["collection"]][ - list_of_speakers["content_object"]["id"] - ].get("agenda_item_id") - if agenda_item_id: - title_information["_agenda_item_number"] = all_data["agenda/item"][ - agenda_item_id - ]["item_number"] + content_object = await get_model( + all_data_provider, + list_of_speakers["content_object"]["collection"], + list_of_speakers["content_object"]["id"], + ) + agenda_item_id = content_object.get("agenda_item_id") + if agenda_item_id is not None: + agenda_item = await all_data_provider.get("agenda/item", agenda_item_id) + if agenda_item is not None: + title_information["_agenda_item_number"] = agenda_item["item_number"] # Partition speaker objects to waiting, current and finished speakers_waiting = [] speakers_finished = [] current_speaker = None for speaker in list_of_speakers["speakers"]: - user = await get_user_name(all_data, speaker["user_id"]) + user = await get_user_name(all_data_provider, speaker["user_id"]) formatted_speaker = { "user": user, "marked": speaker["marked"], @@ -151,8 +162,12 @@ async def get_list_of_speakers_slide_data( speakers_waiting = sorted(speakers_waiting, key=lambda s: s["weight"]) speakers_finished = sorted(speakers_finished, key=lambda s: s["end_time"]) - number_of_last_speakers = await get_config(all_data, "agenda_show_last_speakers") - number_of_next_speakers = await get_config(all_data, "agenda_show_next_speakers") + number_of_last_speakers = await get_config( + all_data_provider, "agenda_show_last_speakers" + ) + number_of_next_speakers = await get_config( + all_data_provider, "agenda_show_next_speakers" + ) if number_of_last_speakers == 0: speakers_finished = [] @@ -174,7 +189,7 @@ async def get_list_of_speakers_slide_data( async def get_current_list_of_speakers_id_for_projector( - all_data: AllData, projector: Dict[str, Any] + all_data_provider: ProjectorAllDataProvider, projector: Dict[str, Any] ) -> Union[int, None]: """ Search for elements, that do have a list of speakers: @@ -189,94 +204,88 @@ async def get_current_list_of_speakers_id_for_projector( continue collection = element["name"] id = element["id"] - if collection not in all_data or id not in all_data[collection]: + model = await all_data_provider.get(collection, id) + if model is None: continue - model = all_data[collection][id] if "list_of_speakers_id" not in model: continue - if not model["list_of_speakers_id"] in all_data["agenda/list-of-speakers"]: + list_of_speakers_id = model["list_of_speakers_id"] + los_exists = await all_data_provider.exists( + "agenda/list-of-speakers", list_of_speakers_id + ) + if not los_exists: continue - list_of_speakers_id = model["list_of_speakers_id"] break return list_of_speakers_id async def get_reference_projector( - all_data: AllData, projector_id: int + all_data_provider: ProjectorAllDataProvider, projector_id: int ) -> Dict[str, Any]: """ Returns the reference projector to the given projector (by id) """ - try: - this_projector = all_data["core/projector"][projector_id] - except KeyError: - raise ProjectorElementException(f"Projector {projector_id} does not exist") + this_projector = await get_model(all_data_provider, "core/projector", projector_id) reference_projector_id = this_projector["reference_projector_id"] or projector_id - try: - reference_projector = all_data["core/projector"][reference_projector_id] - except KeyError: - raise ProjectorElementException( - f"Projector {reference_projector_id} does not exist" - ) - - return reference_projector + return await get_model(all_data_provider, "core/projector", reference_projector_id) async def current_list_of_speakers_slide( - all_data: AllData, element: Dict[str, Any], projector_id: int + all_data_provider: ProjectorAllDataProvider, + element: Dict[str, Any], + projector_id: int, ) -> Dict[str, Any]: """ The current list of speakers slide. Creates the data for the given projector. """ - reference_projector = await get_reference_projector(all_data, projector_id) + reference_projector = await get_reference_projector(all_data_provider, projector_id) list_of_speakers_id = await get_current_list_of_speakers_id_for_projector( - all_data, reference_projector + all_data_provider, reference_projector ) if list_of_speakers_id is None: # no element found return {} - return await get_list_of_speakers_slide_data(all_data, list_of_speakers_id) + return await get_list_of_speakers_slide_data(all_data_provider, list_of_speakers_id) async def current_speaker_chyron_slide( - all_data: AllData, element: Dict[str, Any], projector_id: int + all_data_provider: ProjectorAllDataProvider, + element: Dict[str, Any], + projector_id: int, ) -> Dict[str, Any]: """ Returns the username for the current speaker. """ # get projector for color information - projector = all_data["core/projector"][projector_id] + projector = await get_model(all_data_provider, "core/projector", projector_id) slide_data = { "background_color": projector["chyron_background_color"], "font_color": projector["chyron_font_color"], } - reference_projector = await get_reference_projector(all_data, projector_id) + reference_projector = await get_reference_projector(all_data_provider, projector_id) list_of_speakers_id = await get_current_list_of_speakers_id_for_projector( - all_data, reference_projector + all_data_provider, reference_projector ) if list_of_speakers_id is None: # no element found return slide_data # get list of speakers to search current speaker - try: - list_of_speakers = all_data["agenda/list-of-speakers"][list_of_speakers_id] - except KeyError: - raise ProjectorElementException( - f"List of speakers {list_of_speakers_id} does not exist" - ) + list_of_speakers = await get_model( + all_data_provider, "agenda/list-of-speakers", list_of_speakers_id + ) # find current speaker current_speaker = None for speaker in list_of_speakers["speakers"]: if speaker["begin_time"] is not None and speaker["end_time"] is None: - current_speaker = await get_user_name(all_data, speaker["user_id"]) + current_speaker = await get_user_name(all_data_provider, speaker["user_id"]) break if current_speaker is not None: diff --git a/openslides/assignments/projector.py b/openslides/assignments/projector.py index c868ad103..97c0be82a 100644 --- a/openslides/assignments/projector.py +++ b/openslides/assignments/projector.py @@ -1,25 +1,29 @@ from typing import Any, Dict, List from ..users.projector import get_user_name -from ..utils.projector import AllData, get_model, get_models, register_projector_slide +from ..utils.projector import ( + ProjectorAllDataProvider, + get_model, + get_models, + register_projector_slide, +) from .models import AssignmentPoll -# Important: All functions have to be prune. This means, that thay can only -# access the data, that they get as argument and do not have any -# side effects. - - async def assignment_slide( - all_data: AllData, element: Dict[str, Any], projector_id: int + all_data_provider: ProjectorAllDataProvider, + element: Dict[str, Any], + projector_id: int, ) -> Dict[str, Any]: """ Assignment slide. """ - assignment = get_model(all_data, "assignments/assignment", element.get("id")) + assignment = await get_model( + all_data_provider, "assignments/assignment", element.get("id") + ) assignment_related_users: List[Dict[str, Any]] = [ - {"user": await get_user_name(all_data, aru["user_id"])} + {"user": await get_user_name(all_data_provider, aru["user_id"])} for aru in sorted( assignment["assignment_related_users"], key=lambda aru: aru["weight"] ) @@ -36,13 +40,19 @@ async def assignment_slide( async def assignment_poll_slide( - all_data: AllData, element: Dict[str, Any], projector_id: int + all_data_provider: ProjectorAllDataProvider, + element: Dict[str, Any], + projector_id: int, ) -> Dict[str, Any]: """ Poll slide. """ - poll = get_model(all_data, "assignments/assignment-poll", element.get("id")) - assignment = get_model(all_data, "assignments/assignment", poll["assignment_id"]) + poll = await get_model( + all_data_provider, "assignments/assignment-poll", element.get("id") + ) + assignment = await get_model( + all_data_provider, "assignments/assignment", poll["assignment_id"] + ) poll_data = { key: poll[key] @@ -60,10 +70,14 @@ async def assignment_poll_slide( # Add options: poll_data["options"] = [] - options = get_models(all_data, "assignments/assignment-option", poll["options_id"]) + options = await get_models( + all_data_provider, "assignments/assignment-option", poll["options_id"] + ) for option in sorted(options, key=lambda option: option["weight"]): option_data: Dict[str, Any] = { - "user": {"short_name": await get_user_name(all_data, option["user_id"])} + "user": { + "short_name": await get_user_name(all_data_provider, option["user_id"]) + } } if poll["state"] == AssignmentPoll.STATE_PUBLISHED: option_data["yes"] = float(option["yes"]) diff --git a/openslides/core/projector.py b/openslides/core/projector.py index 9bbf75942..3c9f2ed82 100644 --- a/openslides/core/projector.py +++ b/openslides/core/projector.py @@ -1,20 +1,17 @@ from typing import Any, Dict from ..utils.projector import ( - AllData, - ProjectorElementException, + ProjectorAllDataProvider, get_config, + get_model, register_projector_slide, ) -# Important: All functions have to be prune. This means, that thay can only -# access the data, that they get as argument and do not have any -# side effects. - - async def countdown_slide( - all_data: AllData, element: Dict[str, Any], projector_id: int + all_data_provider: ProjectorAllDataProvider, + element: Dict[str, Any], + projector_id: int, ) -> Dict[str, Any]: """ Countdown slide. @@ -26,23 +23,21 @@ async def countdown_slide( id: 5, # Countdown ID } """ - countdown_id = element.get("id") or 1 - - try: - countdown = all_data["core/countdown"][countdown_id] - except KeyError: - raise ProjectorElementException(f"Countdown {countdown_id} does not exist") - + countdown = await get_model(all_data_provider, "core/countdown", element.get("id")) return { "description": countdown["description"], "running": countdown["running"], "countdown_time": countdown["countdown_time"], - "warning_time": await get_config(all_data, "agenda_countdown_warning_time"), + "warning_time": await get_config( + all_data_provider, "agenda_countdown_warning_time" + ), } async def message_slide( - all_data: AllData, element: Dict[str, Any], projector_id: int + all_data_provider: ProjectorAllDataProvider, + element: Dict[str, Any], + projector_id: int, ) -> Dict[str, Any]: """ Message slide. @@ -54,16 +49,15 @@ async def message_slide( id: 5, # ProjectorMessage ID } """ - message_id = element.get("id") or 1 - - try: - return all_data["core/projector-message"][message_id] - except KeyError: - raise ProjectorElementException(f"Message {message_id} does not exist") + return await get_model( + all_data_provider, "core/projector-message", element.get("id") + ) async def clock_slide( - all_data: AllData, element: Dict[str, Any], projector_id: int + all_data_provider: ProjectorAllDataProvider, + element: Dict[str, Any], + projector_id: int, ) -> Dict[str, Any]: return {} diff --git a/openslides/mediafiles/projector.py b/openslides/mediafiles/projector.py index 196d5e8c8..b996c823f 100644 --- a/openslides/mediafiles/projector.py +++ b/openslides/mediafiles/projector.py @@ -1,35 +1,23 @@ from typing import Any, Dict from ..utils.projector import ( - AllData, - ProjectorElementException, + ProjectorAllDataProvider, + get_model, register_projector_slide, ) -# Important: All functions have to be prune. This means, that thay can only -# access the data, that they get as argument and do not have any -# side effects. - - async def mediafile_slide( - all_data: AllData, element: Dict[str, Any], projector_id: int + all_data_provider: ProjectorAllDataProvider, + element: Dict[str, Any], + projector_id: int, ) -> Dict[str, Any]: """ Slide for Mediafile. """ - mediafile_id = element.get("id") - - if mediafile_id is None: - raise ProjectorElementException("id is required for mediafile slide") - - try: - mediafile = all_data["mediafiles/mediafile"][mediafile_id] - except KeyError: - raise ProjectorElementException( - f"mediafile with id {mediafile_id} does not exist" - ) - + mediafile = await get_model( + all_data_provider, "mediafiles/mediafile", element.get("id") + ) return { "path": mediafile["path"], "mimetype": mediafile["mimetype"], diff --git a/openslides/motions/projector.py b/openslides/motions/projector.py index 74a9a399f..c7a39040c 100644 --- a/openslides/motions/projector.py +++ b/openslides/motions/projector.py @@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional from ..users.projector import get_user_name from ..utils.projector import ( - AllData, + ProjectorAllDataProvider, ProjectorElementException, get_config, get_model, @@ -14,33 +14,31 @@ from .models import MotionPoll motion_placeholder_regex = re.compile(r"\[motion:(\d+)\]") -# Important: All functions have to be prune. This means, that thay can only -# access the data, that they get as argument and do not have any -# side effects. - async def get_state( - all_data: AllData, motion: Dict[str, Any], state_id_key: str + all_data_provider: ProjectorAllDataProvider, + motion: Dict[str, Any], + state_id_key: str, ) -> Dict[str, Any]: """ Returns a state element from one motion. Raises an error if the state does not exist. """ - state = all_data["motions/state"].get(motion[state_id_key]) - if not state: + state = await all_data_provider.get("motions/state", motion[state_id_key]) + if state is None: raise ProjectorElementException( f"motion {motion['id']} can not be on the state with id {motion[state_id_key]}" ) return state -async def get_amendment_merge_into_motion_diff(all_data, amendment): +async def get_amendment_merge_into_motion_diff(all_data_provider, amendment): """ HINT: This implementation should be consistent to showInDiffView() in ViewMotionAmendedParagraph.ts """ if amendment["state_id"] is None: return 0 - state = await get_state(all_data, amendment, "state_id") + state = await get_state(all_data_provider, amendment, "state_id") if state["merge_amendment_into_final"] == -1: return 0 if state["merge_amendment_into_final"] == 1: @@ -48,36 +46,37 @@ async def get_amendment_merge_into_motion_diff(all_data, amendment): if amendment["recommendation_id"] is None: return 0 - recommendation = await get_state(all_data, amendment, "recommendation_id") + recommendation = await get_state(all_data_provider, amendment, "recommendation_id") if recommendation["merge_amendment_into_final"] == 1: return 1 return 0 -async def get_amendment_merge_into_motion_final(all_data, amendment): +async def get_amendment_merge_into_motion_final(all_data_provider, amendment): """ HINT: This implementation should be consistent to showInFinalView() in ViewMotionAmendedParagraph.ts """ if amendment["state_id"] is None: return 0 - state = await get_state(all_data, amendment, "state_id") + state = await get_state(all_data_provider, amendment, "state_id") if state["merge_amendment_into_final"] == 1: return 1 return 0 -async def get_amendments_for_motion(motion, all_data): +async def get_amendments_for_motion(motion, all_data_provider): amendment_data = [] - for amendment_id, amendment in all_data["motions/motion"].items(): + all_motions = await all_data_provider.get_collection("motions/motion") + for amendment_id, amendment in all_motions.items(): if amendment["parent_id"] == motion["id"]: merge_amendment_into_final = await get_amendment_merge_into_motion_final( - all_data, amendment + all_data_provider, amendment ) merge_amendment_into_diff = await get_amendment_merge_into_motion_diff( - all_data, amendment + all_data_provider, amendment ) amendment_data.append( { @@ -92,8 +91,10 @@ async def get_amendments_for_motion(motion, all_data): return amendment_data -async def get_amendment_base_motion(amendment, all_data): - motion = get_model(all_data, "motions/motion", amendment.get("parent_id")) +async def get_amendment_base_motion(amendment, all_data_provider): + motion = await get_model( + all_data_provider, "motions/motion", amendment.get("parent_id") + ) return { "identifier": motion["identifier"], @@ -102,15 +103,17 @@ async def get_amendment_base_motion(amendment, all_data): } -async def get_amendment_base_statute(amendment, all_data): - statute = get_model( - all_data, "motions/statute-paragraph", amendment.get("statute_paragraph_id") +async def get_amendment_base_statute(amendment, all_data_provider): + statute = await get_model( + all_data_provider, + "motions/statute-paragraph", + amendment.get("statute_paragraph_id"), ) return {"title": statute["title"], "text": statute["text"]} async def extend_reference_motion_dict( - all_data: AllData, + all_data_provider: ProjectorAllDataProvider, recommendation: Optional[str], referenced_motions: Dict[int, Dict[str, str]], ) -> None: @@ -127,15 +130,18 @@ async def extend_reference_motion_dict( ] for id in referenced_ids: # Put every referenced motion into the referenced_motions dict - if id not in referenced_motions and id in all_data["motions/motion"]: + referenced_motion = await all_data_provider.get("motions/motion", id) + if id not in referenced_motions and referenced_motion is not None: referenced_motions[id] = { - "title": all_data["motions/motion"][id]["title"], - "identifier": all_data["motions/motion"][id]["identifier"], + "title": referenced_motion["title"], + "identifier": referenced_motion["identifier"], } async def motion_slide( - all_data: AllData, element: Dict[str, Any], projector_id: int + all_data_provider: ProjectorAllDataProvider, + element: Dict[str, Any], + projector_id: int, ) -> Dict[str, Any]: """ Motion slide. @@ -158,13 +164,16 @@ async def motion_slide( """ # Get motion mode = element.get( - "mode", await get_config(all_data, "motions_recommendation_text_mode") + "mode", await get_config(all_data_provider, "motions_recommendation_text_mode") ) - motion = get_model(all_data, "motions/motion", element.get("id")) + + # populate cache: + + motion = await get_model(all_data_provider, "motions/motion", element.get("id")) # Add submitters submitters = [ - await get_user_name(all_data, submitter["user_id"]) + await get_user_name(all_data_provider, submitter["user_id"]) for submitter in sorted( motion["submitters"], key=lambda submitter: submitter["weight"] ) @@ -172,14 +181,16 @@ async def motion_slide( # Get some needed config values show_meta_box = not await get_config( - all_data, "motions_disable_sidebox_on_projector" + all_data_provider, "motions_disable_sidebox_on_projector" ) show_referring_motions = not await get_config( - all_data, "motions_hide_referring_motions" + all_data_provider, "motions_hide_referring_motions" ) - line_length = await get_config(all_data, "motions_line_length") - line_numbering_mode = await get_config(all_data, "motions_default_line_numbering") - motions_preamble = await get_config(all_data, "motions_preamble") + line_length = await get_config(all_data_provider, "motions_line_length") + line_numbering_mode = await get_config( + all_data_provider, "motions_default_line_numbering" + ) + motions_preamble = await get_config(all_data_provider, "motions_preamble") # Query all change-recommendation and amendment related things. change_recommendations = [] # type: ignore @@ -187,17 +198,19 @@ async def motion_slide( base_motion = None base_statute = None if motion["statute_paragraph_id"]: - base_statute = await get_amendment_base_statute(motion, all_data) + base_statute = await get_amendment_base_statute(motion, all_data_provider) elif motion["parent_id"] is not None and motion["amendment_paragraphs"]: - base_motion = await get_amendment_base_motion(motion, all_data) + base_motion = await get_amendment_base_motion(motion, all_data_provider) else: for change_recommendation_id in motion["change_recommendations_id"]: - cr = all_data["motions/motion-change-recommendation"].get( - change_recommendation_id + cr = await get_model( + all_data_provider, + "motions/motion-change-recommendation", + change_recommendation_id, ) if cr is not None and not cr["internal"]: change_recommendations.append(cr) - amendments = await get_amendments_for_motion(motion, all_data) + amendments = await get_amendments_for_motion(motion, all_data_provider) # The base return value. More fields will get added below. return_value = { @@ -217,10 +230,10 @@ async def motion_slide( "line_numbering_mode": line_numbering_mode, } - if not await get_config(all_data, "motions_disable_text_on_projector"): + if not await get_config(all_data_provider, "motions_disable_text_on_projector"): return_value["text"] = motion["text"] - if not await get_config(all_data, "motions_disable_reason_on_projector"): + if not await get_config(all_data_provider, "motions_disable_reason_on_projector"): return_value["reason"] = motion["reason"] if mode == "final": @@ -228,40 +241,46 @@ async def motion_slide( # Add recommendation, if enabled in config (and the motion has one) if ( - not await get_config(all_data, "motions_disable_recommendation_on_projector") + not await get_config( + all_data_provider, "motions_disable_recommendation_on_projector" + ) and motion["recommendation_id"] ): - recommendation_state = await get_state(all_data, motion, "recommendation_id") + recommendation_state = await get_state( + all_data_provider, motion, "recommendation_id" + ) return_value["recommendation"] = recommendation_state["recommendation_label"] if recommendation_state["show_recommendation_extension_field"]: recommendation_extension = motion["recommendation_extension"] # All title information for referenced motions in the recommendation referenced_motions: Dict[int, Dict[str, str]] = {} await extend_reference_motion_dict( - all_data, recommendation_extension, referenced_motions + all_data_provider, recommendation_extension, referenced_motions ) return_value["recommendation_extension"] = recommendation_extension return_value["referenced_motions"] = referenced_motions if motion["statute_paragraph_id"]: return_value["recommender"] = await get_config( - all_data, "motions_statute_recommendations_by" + all_data_provider, "motions_statute_recommendations_by" ) else: return_value["recommender"] = await get_config( - all_data, "motions_recommendations_by" + all_data_provider, "motions_recommendations_by" ) if show_referring_motions: # Add recommendation-referencing motions return_value[ "recommendation_referencing_motions" - ] = await get_recommendation_referencing_motions(all_data, motion["id"]) + ] = await get_recommendation_referencing_motions( + all_data_provider, motion["id"] + ) return return_value async def get_recommendation_referencing_motions( - all_data: AllData, motion_id: int + all_data_provider: ProjectorAllDataProvider, motion_id: int ) -> Optional[List[Dict[str, Any]]]: """ Returns all title information for motions, that are referencing @@ -269,14 +288,15 @@ async def get_recommendation_referencing_motions( motions, None is returned (instead of []). """ recommendation_referencing_motions = [] - for motion in all_data["motions/motion"].values(): + all_motions = await all_data_provider.get_collection("motions/motion") + for motion in all_motions.values(): # Motion must have a recommendation and a recommendaiton extension if not motion["recommendation_id"] or not motion["recommendation_extension"]: continue # The recommendation must allow the extension field (there might be left-overs # in a motions recommendation extension..) - recommendation = await get_state(all_data, motion, "recommendation_id") + recommendation = await get_state(all_data_provider, motion, "recommendation_id") if not recommendation["show_recommendation_extension_field"]: continue @@ -297,12 +317,16 @@ async def get_recommendation_referencing_motions( async def motion_block_slide( - all_data: AllData, element: Dict[str, Any], projector_id: int + all_data_provider: ProjectorAllDataProvider, + element: Dict[str, Any], + projector_id: int, ) -> Dict[str, Any]: """ Motion block slide. """ - motion_block = get_model(all_data, "motions/motion-block", element.get("id")) + motion_block = await get_model( + all_data_provider, "motions/motion-block", element.get("id") + ) # All motions in this motion block motions = [] @@ -311,7 +335,8 @@ async def motion_block_slide( referenced_motions: Dict[int, Dict[str, str]] = {} # Search motions. - for motion in all_data["motions/motion"].values(): + all_motions = await all_data_provider.get_collection("motions/motion") + for motion in all_motions.values(): if motion["motion_block_id"] == motion_block["id"]: motion_object = { "title": motion["title"], @@ -320,7 +345,9 @@ async def motion_block_slide( recommendation_id = motion["recommendation_id"] if recommendation_id is not None: - recommendation = await get_state(all_data, motion, "recommendation_id") + recommendation = await get_state( + all_data_provider, motion, "recommendation_id" + ) motion_object["recommendation"] = { "name": recommendation["recommendation_label"], "css_class": recommendation["css_class"], @@ -328,7 +355,7 @@ async def motion_block_slide( if recommendation["show_recommendation_extension_field"]: recommendation_extension = motion["recommendation_extension"] await extend_reference_motion_dict( - all_data, recommendation_extension, referenced_motions + all_data_provider, recommendation_extension, referenced_motions ) motion_object["recommendation_extension"] = recommendation_extension @@ -342,13 +369,15 @@ async def motion_block_slide( async def motion_poll_slide( - all_data: AllData, element: Dict[str, Any], projector_id: int + all_data_provider: ProjectorAllDataProvider, + element: Dict[str, Any], + projector_id: int, ) -> Dict[str, Any]: """ Poll slide. """ - poll = get_model(all_data, "motions/motion-poll", element.get("id")) - motion = get_model(all_data, "motions/motion", poll["motion_id"]) + poll = await get_model(all_data_provider, "motions/motion-poll", element.get("id")) + motion = await get_model(all_data_provider, "motions/motion", poll["motion_id"]) poll_data = { key: poll[key] @@ -363,8 +392,8 @@ async def motion_poll_slide( } if poll["state"] == MotionPoll.STATE_PUBLISHED: - option = get_model( - all_data, "motions/motion-option", poll["options_id"][0] + option = await get_model( + all_data_provider, "motions/motion-option", poll["options_id"][0] ) # there can only be exactly one option poll_data["options"] = [ { diff --git a/openslides/topics/projector.py b/openslides/topics/projector.py index 18306aa81..3687c0f31 100644 --- a/openslides/topics/projector.py +++ b/openslides/topics/projector.py @@ -1,19 +1,16 @@ from typing import Any, Dict from ..utils.projector import ( - AllData, - ProjectorElementException, + ProjectorAllDataProvider, + get_model, register_projector_slide, ) -# Important: All functions have to be prune. This means, that thay can only -# access the data, that they get as argument and do not have any -# side effects. - - async def topic_slide( - all_data: AllData, element: Dict[str, Any], projector_id: int + all_data_provider: ProjectorAllDataProvider, + element: Dict[str, Any], + projector_id: int, ) -> Dict[str, Any]: """ Topic slide. @@ -22,22 +19,8 @@ async def topic_slide( * title * text """ - topic_id = element.get("id") - - if topic_id is None: - raise ProjectorElementException("id is required for topic slide") - - try: - topic = all_data["topics/topic"][topic_id] - except KeyError: - raise ProjectorElementException(f"topic with id {topic_id} does not exist") - - item_id = topic["agenda_item_id"] - try: - item = all_data["agenda/item"][item_id] - except KeyError: - raise ProjectorElementException(f"item with id {item_id} does not exist") - + topic = await get_model(all_data_provider, "topics/topic", element.get("id")) + item = await get_model(all_data_provider, "agenda/item", topic["agenda_item_id"]) return { "title": topic["title"], "text": topic["text"], diff --git a/openslides/users/projector.py b/openslides/users/projector.py index c1b02fc85..902a19a59 100644 --- a/openslides/users/projector.py +++ b/openslides/users/projector.py @@ -1,19 +1,16 @@ -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from ..utils.projector import ( - AllData, - ProjectorElementException, + ProjectorAllDataProvider, + get_model, register_projector_slide, ) -# Important: All functions have to be prune. This means, that thay can only -# access the data, that they get as argument and do not have any -# side effects. - - async def user_slide( - all_data: AllData, element: Dict[str, Any], projector_id: int + all_data_provider: ProjectorAllDataProvider, + element: Dict[str, Any], + projector_id: int, ) -> Dict[str, Any]: """ User slide. @@ -21,22 +18,16 @@ async def user_slide( The returned dict can contain the following fields: * user """ - user_id = element.get("id") - - if user_id is None: - raise ProjectorElementException("id is required for user slide") - - return {"user": await get_user_name(all_data, user_id)} + return {"user": await get_user_name(all_data_provider, element.get("id"))} -async def get_user_name(all_data: AllData, user_id: int) -> str: +async def get_user_name( + all_data_provider: ProjectorAllDataProvider, user_id: Optional[int] +) -> str: """ Returns the short name for an user_id. """ - try: - user = all_data["users/user"][user_id] - except KeyError: - raise ProjectorElementException(f"user with id {user_id} does not exist") + user = await get_model(all_data_provider, "users/user", user_id) name_parts: List[str] = [] for name_part in ("title", "first_name", "last_name"): diff --git a/openslides/utils/autoupdate.py b/openslides/utils/autoupdate.py index 7701b1598..a43c5261a 100644 --- a/openslides/utils/autoupdate.py +++ b/openslides/utils/autoupdate.py @@ -92,13 +92,13 @@ class AutoupdateBundle: elements[full_data["id"]]["full_data"] = full_data # Save histroy here using sync code. - save_history(self.elements) + save_history(self.element_iterator) # Update cache and send autoupdate using async code. async_to_sync(self.async_handle_collection_elements)() @property - def elements(self) -> Iterable[AutoupdateElement]: + def element_iterator(self) -> Iterable[AutoupdateElement]: """ Iterator for all elements in this bundle """ for elements in self.autoupdate_elements.values(): yield from elements.values() @@ -110,7 +110,7 @@ class AutoupdateBundle: Returns the change_id """ cache_elements: Dict[str, Optional[Dict[str, Any]]] = {} - for element in self.elements: + for element in self.element_iterator: element_id = get_element_id(element["collection_string"], element["id"]) full_data = element.get("full_data") if full_data: @@ -253,7 +253,7 @@ class AutoupdateBundleMiddleware: return response -def save_history(elements: Iterable[AutoupdateElement]) -> Iterable: +def save_history(element_iterator: Iterable[AutoupdateElement]) -> Iterable: """ Thin wrapper around the call of history saving manager method. @@ -261,4 +261,4 @@ def save_history(elements: Iterable[AutoupdateElement]) -> Iterable: """ from ..core.models import History - return History.objects.add_elements(elements) + return History.objects.add_elements(element_iterator) diff --git a/openslides/utils/cache.py b/openslides/utils/cache.py index 00054ca74..14dec14d2 100644 --- a/openslides/utils/cache.py +++ b/openslides/utils/cache.py @@ -254,25 +254,6 @@ class ElementCache: all_data[collection] = await restricter(user_id, all_data[collection]) return dict(all_data) - async def get_all_data_dict(self) -> Dict[str, Dict[int, Dict[str, Any]]]: - """ - Returns all data with a dict (id <-> element) per collection: - { - : { - : - } - } - """ - all_data: Dict[str, Dict[int, Dict[str, Any]]] = defaultdict(dict) - for element_id, data in (await self.cache_provider.get_all_data()).items(): - collection, id = split_element_id(element_id) - element = json.loads(data.decode()) - element.pop( - "_no_delete_on_restriction", False - ) # remove special field for get_data_since - all_data[collection][id] = element - return dict(all_data) - async def get_collection_data(self, collection: str) -> Dict[int, Dict[str, Any]]: """ Returns the data for one collection as dict: {id: } diff --git a/openslides/utils/cache_providers.py b/openslides/utils/cache_providers.py index e049a5211..d74c55378 100644 --- a/openslides/utils/cache_providers.py +++ b/openslides/utils/cache_providers.py @@ -460,7 +460,8 @@ class RedisCacheProvider: ) if reported_amount != read_only_redis_amount_replicas: logger.warn( - f"WAIT reported {reported_amount} replicas of {read_only_redis_amount_replicas} requested after {read_only_redis_wait_timeout} ms!" + f"WAIT reported {reported_amount} replicas of {read_only_redis_amount_replicas} " + + f"requested after {read_only_redis_wait_timeout} ms!" ) return result diff --git a/openslides/utils/projector.py b/openslides/utils/projector.py index 9e20fddcd..7b66e87c6 100644 --- a/openslides/utils/projector.py +++ b/openslides/utils/projector.py @@ -5,16 +5,14 @@ Functions that handel the registration of projector elements and the rendering of the data to present it on the projector. """ -from typing import Any, Awaitable, Callable, Dict, List +from collections import defaultdict +from typing import Any, Awaitable, Callable, Dict, List, Optional +from . import logging from .cache import element_cache -AllData = Dict[str, Dict[int, Dict[str, Any]]] -ProjectorSlide = Callable[[AllData, Dict[str, Any], int], Awaitable[Dict[str, Any]]] - - -projector_slides: Dict[str, ProjectorSlide] = {} +logger = logging.getLogger(__name__) class ProjectorElementException(Exception): @@ -23,6 +21,44 @@ class ProjectorElementException(Exception): """ +class ProjectorAllDataProvider: + NON_EXISTENT_MARKER = object() + + def __init__(self) -> None: + self.cache: Any = defaultdict(dict) # fuu you mypy + self.fetched_collection: Dict[str, bool] = {} + + async def get(self, collection: str, id: int) -> Optional[Dict[str, Any]]: + cache_data = self.cache[collection].get(id) + if cache_data is None: + data: Any = await element_cache.get_element_data(collection, id) + if data is None: + data = ProjectorAllDataProvider.NON_EXISTENT_MARKER + self.cache[collection][id] = data + elif cache_data == ProjectorAllDataProvider.NON_EXISTENT_MARKER: + return None + return self.cache[collection][id] + + async def get_collection(self, collection: str) -> Dict[int, Dict[str, Any]]: + if not self.fetched_collection.get(collection, False): + collection_data = await element_cache.get_collection_data(collection) + self.cache[collection] = collection_data + self.fetched_collection[collection] = True + return self.cache[collection] + + async def exists(self, collection: str, id: int) -> bool: + model = await self.get(collection, id) + return model is not None + + +ProjectorSlide = Callable[ + [ProjectorAllDataProvider, Dict[str, Any], int], Awaitable[Dict[str, Any]] +] + + +projector_slides: Dict[str, ProjectorSlide] = {} + + def register_projector_slide(name: str, slide: ProjectorSlide) -> None: """ Registers a projector slide. @@ -67,10 +103,11 @@ async def get_projector_data( if projector_ids is None: projector_ids = [] - all_data = await element_cache.get_all_data_dict() projector_data: Dict[int, List[Dict[str, Any]]] = {} + all_data_provider = ProjectorAllDataProvider() + projectors = await all_data_provider.get_collection("core/projector") - for projector_id, projector in all_data.get("core/projector", {}).items(): + for projector_id, projector in projectors.items(): if projector_ids and projector_id not in projector_ids: # only render the projector in question. continue @@ -83,7 +120,7 @@ async def get_projector_data( for element in projector["elements"]: projector_slide = projector_slides[element["name"]] try: - data = await projector_slide(all_data, element, projector_id) + data = await projector_slide(all_data_provider, element, projector_id) except ProjectorElementException as err: data = {"error": str(err)} projector_data[projector_id].append({"data": data, "element": element}) @@ -91,18 +128,23 @@ async def get_projector_data( return projector_data -async def get_config(all_data: AllData, key: str) -> Any: +async def get_config(all_data_provider: ProjectorAllDataProvider, key: str) -> Any: """ - Returns a config value from all_data. + Returns a config value from all_data_provider. + Triggers the cache early: It access `get_colelction` instead of `get`. It + allows for all successive queries for configs to be cached. """ from ..core.config import config config_id = (await config.async_get_key_to_id())[key] - return all_data[config.get_collection_string()][config_id]["value"] + configs = await all_data_provider.get_collection(config.get_collection_string()) + return configs[config_id]["value"] -def get_model(all_data: AllData, collection: str, id: Any) -> Dict[str, Any]: +async def get_model( + all_data_provider: ProjectorAllDataProvider, collection: str, id: Any +) -> Dict[str, Any]: """ Tries to get the model identified by the collection and id. If the id is invalid or the model not found, ProjectorElementExceptions will be raised. @@ -110,17 +152,19 @@ def get_model(all_data: AllData, collection: str, id: Any) -> Dict[str, Any]: if id is None: raise ProjectorElementException(f"id is required for {collection} slide") - try: - model = all_data[collection][id] - except KeyError: + model = await all_data_provider.get(collection, id) + if model is None: raise ProjectorElementException(f"{collection} with id {id} does not exist") return model -def get_models( - all_data: AllData, collection: str, ids: List[Any] +async def get_models( + all_data_provider: ProjectorAllDataProvider, collection: str, ids: List[Any] ) -> List[Dict[str, Any]]: """ Tries to fetch all given models. Models are required to be all of the collection `collection`. """ - return [get_model(all_data, collection, id) for id in ids] + logger.info( + f"Note: a call to `get_models` with {collection}/{ids}. This might be cache-intensive" + ) + return [await get_model(all_data_provider, collection, id) for id in ids]