ag-admin/OpenSlides
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rewrite projector code to be cache friendly

Browse Source 
This speeds up the requests/seconds by a factor of 100
This commit is contained in:
FinnStutzenstein 2020-05-15 11:47:43 +02:00
parent 23842fd496
commit bf88cea200
No known key found for this signature in database
GPG Key ID: 9042F605C6324654
11 changed files with 310 additions and 276 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
openslides/agenda/projector.py
View File

@ -3,9 +3,10 @@ from typing import Any, Dict, List, Union
from ..users.projector import get_user_name
from ..utils.projector import (
AllData,
ProjectorAllDataProvider,
ProjectorElementException,
get_config,
get_model,
register_projector_slide,
)
@ -15,20 +16,24 @@ from ..utils.projector import (
# side effects.
async def get_sorted_agenda_items(all_data: AllData) -> List[Dict[str, Any]]:
async def get_sorted_agenda_items(
agenda_items: Dict[int, Dict[str, Any]]
) -> List[Dict[str, Any]]:
"""
Returns all sorted agenda items by id first and then weight, resulting in
ordered items, if some have the same weight.
"""
return sorted(
sorted(all_data["agenda/item"].values(), key=lambda item: item["id"]),
sorted(agenda_items.values(), key=lambda item: item["id"]),
key=lambda item: item["weight"],
)
async def get_flat_tree(all_data: AllData, parent_id: int = 0) -> List[Dict[str, Any]]:
async def get_flat_tree(
agenda_items: Dict[int, Dict[str, Any]], parent_id: int = 0
) -> List[Dict[str, Any]]:
"""
Build the item tree from all_data.
Build the item tree from all_data_provider.
Only build the tree from elements unterneath parent_id.
@ -38,16 +43,16 @@ async def get_flat_tree(all_data: AllData, parent_id: int = 0) -> List[Dict[str,
# Build a dict from an item_id to all its children
children: Dict[int, List[int]] = defaultdict(list)
if "agenda/item" in all_data:
for item in await get_sorted_agenda_items(all_data):
for item in await get_sorted_agenda_items(agenda_items):
if item["type"] == 1: # only normal items
children[item["parent_id"] or 0].append(item["id"])
tree = []
async def get_children(item_ids: List[int], depth: int) -> None:
def build_tree(item_ids: List[int], depth: int) -> None:
for item_id in item_ids:
item = all_data["agenda/item"][item_id]
item = agenda_items[item_id]
title_information = item["title_information"]
title_information["_agenda_item_number"] = item["item_number"]
tree.append(
@ -57,25 +62,29 @@ async def get_flat_tree(all_data: AllData, parent_id: int = 0) -> List[Dict[str,
"depth": depth,
}
)
await get_children(children[item_id], depth + 1)
build_tree(children[item_id], depth + 1)
await get_children(children[parent_id], 0)
build_tree(children[parent_id], 0)
return tree
async def item_list_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
all_data_provider: ProjectorAllDataProvider,
element: Dict[str, Any],
projector_id: int,
) -> Dict[str, Any]:
"""
Item list slide.
Returns all root items or all children of an item.
"""
only_main_items = element.get("only_main_items", True)
# fetch all items, so they are cached:
all_agenda_items = await all_data_provider.get_collection("agenda/item")
only_main_items = element.get("only_main_items", True)
if only_main_items:
agenda_items = []
for item in await get_sorted_agenda_items(all_data):
for item in await get_sorted_agenda_items(all_agenda_items):
if item["parent_id"] is None and item["type"] == 1:
title_information = item["title_information"]
title_information["_agenda_item_number"] = item["item_number"]
@ -86,13 +95,15 @@ async def item_list_slide(
}
)
else:
agenda_items = await get_flat_tree(all_data)
agenda_items = await get_flat_tree(all_agenda_items)
return {"items": agenda_items}
async def list_of_speakers_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
all_data_provider: ProjectorAllDataProvider,
element: Dict[str, Any],
projector_id: int,
) -> Dict[str, Any]:
"""
List of speakers slide.
@ -104,35 +115,35 @@ async def list_of_speakers_slide(
if list_of_speakers_id is None:
raise ProjectorElementException("id is required for list of speakers slide")
return await get_list_of_speakers_slide_data(all_data, list_of_speakers_id)
return await get_list_of_speakers_slide_data(all_data_provider, list_of_speakers_id)
async def get_list_of_speakers_slide_data(
all_data: AllData, list_of_speakers_id: int
all_data_provider: ProjectorAllDataProvider, list_of_speakers_id: int
) -> Dict[str, Any]:
try:
list_of_speakers = all_data["agenda/list-of-speakers"][list_of_speakers_id]
except KeyError:
raise ProjectorElementException(
f"List of speakers {list_of_speakers_id} does not exist"
list_of_speakers = await get_model(
all_data_provider, "agenda/list-of-speakers", list_of_speakers_id
)
title_information = list_of_speakers["title_information"]
# try to get the agenda item for the content object (which must not exist)
agenda_item_id = all_data[list_of_speakers["content_object"]["collection"]][
list_of_speakers["content_object"]["id"]
].get("agenda_item_id")
if agenda_item_id:
title_information["_agenda_item_number"] = all_data["agenda/item"][
agenda_item_id
]["item_number"]
content_object = await get_model(
all_data_provider,
list_of_speakers["content_object"]["collection"],
list_of_speakers["content_object"]["id"],
)
agenda_item_id = content_object.get("agenda_item_id")
if agenda_item_id is not None:
agenda_item = await all_data_provider.get("agenda/item", agenda_item_id)
if agenda_item is not None:
title_information["_agenda_item_number"] = agenda_item["item_number"]
# Partition speaker objects to waiting, current and finished
speakers_waiting = []
speakers_finished = []
current_speaker = None
for speaker in list_of_speakers["speakers"]:
user = await get_user_name(all_data, speaker["user_id"])
user = await get_user_name(all_data_provider, speaker["user_id"])
formatted_speaker = {
"user": user,
"marked": speaker["marked"],
@ -151,8 +162,12 @@ async def get_list_of_speakers_slide_data(
speakers_waiting = sorted(speakers_waiting, key=lambda s: s["weight"])
speakers_finished = sorted(speakers_finished, key=lambda s: s["end_time"])
number_of_last_speakers = await get_config(all_data, "agenda_show_last_speakers")
number_of_next_speakers = await get_config(all_data, "agenda_show_next_speakers")
number_of_last_speakers = await get_config(
all_data_provider, "agenda_show_last_speakers"
)
number_of_next_speakers = await get_config(
all_data_provider, "agenda_show_next_speakers"
)
if number_of_last_speakers == 0:
speakers_finished = []
@ -174,7 +189,7 @@ async def get_list_of_speakers_slide_data(
async def get_current_list_of_speakers_id_for_projector(
all_data: AllData, projector: Dict[str, Any]
all_data_provider: ProjectorAllDataProvider, projector: Dict[str, Any]
) -> Union[int, None]:
"""
Search for elements, that do have a list of speakers:
@ -189,94 +204,88 @@ async def get_current_list_of_speakers_id_for_projector(
continue
collection = element["name"]
id = element["id"]
if collection not in all_data or id not in all_data[collection]:
model = await all_data_provider.get(collection, id)
if model is None:
continue
model = all_data[collection][id]
if "list_of_speakers_id" not in model:
continue
if not model["list_of_speakers_id"] in all_data["agenda/list-of-speakers"]:
list_of_speakers_id = model["list_of_speakers_id"]
los_exists = await all_data_provider.exists(
"agenda/list-of-speakers", list_of_speakers_id
)
if not los_exists:
continue
list_of_speakers_id = model["list_of_speakers_id"]
break
return list_of_speakers_id
async def get_reference_projector(
all_data: AllData, projector_id: int
all_data_provider: ProjectorAllDataProvider, projector_id: int
) -> Dict[str, Any]:
"""
Returns the reference projector to the given projector (by id)
"""
try:
this_projector = all_data["core/projector"][projector_id]
except KeyError:
raise ProjectorElementException(f"Projector {projector_id} does not exist")
this_projector = await get_model(all_data_provider, "core/projector", projector_id)
reference_projector_id = this_projector["reference_projector_id"] or projector_id
try:
reference_projector = all_data["core/projector"][reference_projector_id]
except KeyError:
raise ProjectorElementException(
f"Projector {reference_projector_id} does not exist"
)
return reference_projector
return await get_model(all_data_provider, "core/projector", reference_projector_id)
async def current_list_of_speakers_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
all_data_provider: ProjectorAllDataProvider,
element: Dict[str, Any],
projector_id: int,
) -> Dict[str, Any]:
"""
The current list of speakers slide. Creates the data for the given projector.
"""
reference_projector = await get_reference_projector(all_data, projector_id)
reference_projector = await get_reference_projector(all_data_provider, projector_id)
list_of_speakers_id = await get_current_list_of_speakers_id_for_projector(
all_data, reference_projector
all_data_provider, reference_projector
)
if list_of_speakers_id is None: # no element found
return {}
return await get_list_of_speakers_slide_data(all_data, list_of_speakers_id)
return await get_list_of_speakers_slide_data(all_data_provider, list_of_speakers_id)
async def current_speaker_chyron_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
all_data_provider: ProjectorAllDataProvider,
element: Dict[str, Any],
projector_id: int,
) -> Dict[str, Any]:
"""
Returns the username for the current speaker.
"""
# get projector for color information
projector = all_data["core/projector"][projector_id]
projector = await get_model(all_data_provider, "core/projector", projector_id)
slide_data = {
"background_color": projector["chyron_background_color"],
"font_color": projector["chyron_font_color"],
}
reference_projector = await get_reference_projector(all_data, projector_id)
reference_projector = await get_reference_projector(all_data_provider, projector_id)
list_of_speakers_id = await get_current_list_of_speakers_id_for_projector(
all_data, reference_projector
all_data_provider, reference_projector
)
if list_of_speakers_id is None: # no element found
return slide_data
# get list of speakers to search current speaker
try:
list_of_speakers = all_data["agenda/list-of-speakers"][list_of_speakers_id]
except KeyError:
raise ProjectorElementException(
f"List of speakers {list_of_speakers_id} does not exist"
list_of_speakers = await get_model(
all_data_provider, "agenda/list-of-speakers", list_of_speakers_id
)
# find current speaker
current_speaker = None
for speaker in list_of_speakers["speakers"]:
if speaker["begin_time"] is not None and speaker["end_time"] is None:
current_speaker = await get_user_name(all_data, speaker["user_id"])
current_speaker = await get_user_name(all_data_provider, speaker["user_id"])
break
if current_speaker is not None:

42
openslides/assignments/projector.py
View File

@ -1,25 +1,29 @@
from typing import Any, Dict, List
from ..users.projector import get_user_name
from ..utils.projector import AllData, get_model, get_models, register_projector_slide
from ..utils.projector import (
ProjectorAllDataProvider,
get_model,
get_models,
register_projector_slide,
)
from .models import AssignmentPoll
# Important: All functions have to be prune. This means, that thay can only
# access the data, that they get as argument and do not have any
# side effects.
async def assignment_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
all_data_provider: ProjectorAllDataProvider,
element: Dict[str, Any],
projector_id: int,
) -> Dict[str, Any]:
"""
Assignment slide.
"""
assignment = get_model(all_data, "assignments/assignment", element.get("id"))
assignment = await get_model(
all_data_provider, "assignments/assignment", element.get("id")
)
assignment_related_users: List[Dict[str, Any]] = [
{"user": await get_user_name(all_data, aru["user_id"])}
{"user": await get_user_name(all_data_provider, aru["user_id"])}
for aru in sorted(
assignment["assignment_related_users"], key=lambda aru: aru["weight"]
)
@ -36,13 +40,19 @@ async def assignment_slide(
async def assignment_poll_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
all_data_provider: ProjectorAllDataProvider,
element: Dict[str, Any],
projector_id: int,
) -> Dict[str, Any]:
"""
Poll slide.
"""
poll = get_model(all_data, "assignments/assignment-poll", element.get("id"))
assignment = get_model(all_data, "assignments/assignment", poll["assignment_id"])
poll = await get_model(
all_data_provider, "assignments/assignment-poll", element.get("id")
)
assignment = await get_model(
all_data_provider, "assignments/assignment", poll["assignment_id"]
)
poll_data = {
key: poll[key]
@ -60,10 +70,14 @@ async def assignment_poll_slide(
# Add options:
poll_data["options"] = []
options = get_models(all_data, "assignments/assignment-option", poll["options_id"])
options = await get_models(
all_data_provider, "assignments/assignment-option", poll["options_id"]
)
for option in sorted(options, key=lambda option: option["weight"]):
option_data: Dict[str, Any] = {
"user": {"short_name": await get_user_name(all_data, option["user_id"])}
"user": {
"short_name": await get_user_name(all_data_provider, option["user_id"])
}
}
if poll["state"] == AssignmentPoll.STATE_PUBLISHED:
option_data["yes"] = float(option["yes"])

42
openslides/core/projector.py
View File

@ -1,20 +1,17 @@
from typing import Any, Dict
from ..utils.projector import (
AllData,
ProjectorElementException,
ProjectorAllDataProvider,
get_config,
get_model,
register_projector_slide,
)
# Important: All functions have to be prune. This means, that thay can only
# access the data, that they get as argument and do not have any
# side effects.
async def countdown_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
all_data_provider: ProjectorAllDataProvider,
element: Dict[str, Any],
projector_id: int,
) -> Dict[str, Any]:
"""
Countdown slide.
@ -26,23 +23,21 @@ async def countdown_slide(
id: 5, # Countdown ID
}
"""
countdown_id = element.get("id") or 1
try:
countdown = all_data["core/countdown"][countdown_id]
except KeyError:
raise ProjectorElementException(f"Countdown {countdown_id} does not exist")
countdown = await get_model(all_data_provider, "core/countdown", element.get("id"))
return {
"description": countdown["description"],
"running": countdown["running"],
"countdown_time": countdown["countdown_time"],
"warning_time": await get_config(all_data, "agenda_countdown_warning_time"),
"warning_time": await get_config(
all_data_provider, "agenda_countdown_warning_time"
),
}
async def message_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
all_data_provider: ProjectorAllDataProvider,
element: Dict[str, Any],
projector_id: int,
) -> Dict[str, Any]:
"""
Message slide.
@ -54,16 +49,15 @@ async def message_slide(
id: 5, # ProjectorMessage ID
}
"""
message_id = element.get("id") or 1
try:
return all_data["core/projector-message"][message_id]
except KeyError:
raise ProjectorElementException(f"Message {message_id} does not exist")
return await get_model(
all_data_provider, "core/projector-message", element.get("id")
)
async def clock_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
all_data_provider: ProjectorAllDataProvider,
element: Dict[str, Any],
projector_id: int,
) -> Dict[str, Any]:
return {}

26
openslides/mediafiles/projector.py
View File

@ -1,35 +1,23 @@
from typing import Any, Dict
from ..utils.projector import (
AllData,
ProjectorElementException,
ProjectorAllDataProvider,
get_model,
register_projector_slide,
)
# Important: All functions have to be prune. This means, that thay can only
# access the data, that they get as argument and do not have any
# side effects.
async def mediafile_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
all_data_provider: ProjectorAllDataProvider,
element: Dict[str, Any],
projector_id: int,
) -> Dict[str, Any]:
"""
Slide for Mediafile.
"""
mediafile_id = element.get("id")
if mediafile_id is None:
raise ProjectorElementException("id is required for mediafile slide")
try:
mediafile = all_data["mediafiles/mediafile"][mediafile_id]
except KeyError:
raise ProjectorElementException(
f"mediafile with id {mediafile_id} does not exist"
mediafile = await get_model(
all_data_provider, "mediafiles/mediafile", element.get("id")
)
return {
"path": mediafile["path"],
"mimetype": mediafile["mimetype"],

151
openslides/motions/projector.py
View File

@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional
from ..users.projector import get_user_name
from ..utils.projector import (
AllData,
ProjectorAllDataProvider,
ProjectorElementException,
get_config,
get_model,
@ -14,33 +14,31 @@ from .models import MotionPoll
motion_placeholder_regex = re.compile(r"\[motion:(\d+)\]")
# Important: All functions have to be prune. This means, that thay can only
# access the data, that they get as argument and do not have any
# side effects.
async def get_state(
all_data: AllData, motion: Dict[str, Any], state_id_key: str
all_data_provider: ProjectorAllDataProvider,
motion: Dict[str, Any],
state_id_key: str,
) -> Dict[str, Any]:
"""
Returns a state element from one motion. Raises an error if the state does not exist.
"""
state = all_data["motions/state"].get(motion[state_id_key])
if not state:
state = await all_data_provider.get("motions/state", motion[state_id_key])
if state is None:
raise ProjectorElementException(
f"motion {motion['id']} can not be on the state with id {motion[state_id_key]}"
)
return state
async def get_amendment_merge_into_motion_diff(all_data, amendment):
async def get_amendment_merge_into_motion_diff(all_data_provider, amendment):
"""
HINT: This implementation should be consistent to showInDiffView() in ViewMotionAmendedParagraph.ts
"""
if amendment["state_id"] is None:
return 0
state = await get_state(all_data, amendment, "state_id")
state = await get_state(all_data_provider, amendment, "state_id")
if state["merge_amendment_into_final"] == -1:
return 0
if state["merge_amendment_into_final"] == 1:
@ -48,36 +46,37 @@ async def get_amendment_merge_into_motion_diff(all_data, amendment):
if amendment["recommendation_id"] is None:
return 0
recommendation = await get_state(all_data, amendment, "recommendation_id")
recommendation = await get_state(all_data_provider, amendment, "recommendation_id")
if recommendation["merge_amendment_into_final"] == 1:
return 1
return 0
async def get_amendment_merge_into_motion_final(all_data, amendment):
async def get_amendment_merge_into_motion_final(all_data_provider, amendment):
"""
HINT: This implementation should be consistent to showInFinalView() in ViewMotionAmendedParagraph.ts
"""
if amendment["state_id"] is None:
return 0
state = await get_state(all_data, amendment, "state_id")
state = await get_state(all_data_provider, amendment, "state_id")
if state["merge_amendment_into_final"] == 1:
return 1
return 0
async def get_amendments_for_motion(motion, all_data):
async def get_amendments_for_motion(motion, all_data_provider):
amendment_data = []
for amendment_id, amendment in all_data["motions/motion"].items():
all_motions = await all_data_provider.get_collection("motions/motion")
for amendment_id, amendment in all_motions.items():
if amendment["parent_id"] == motion["id"]:
merge_amendment_into_final = await get_amendment_merge_into_motion_final(
all_data, amendment
all_data_provider, amendment
)
merge_amendment_into_diff = await get_amendment_merge_into_motion_diff(
all_data, amendment
all_data_provider, amendment
)
amendment_data.append(
{
@ -92,8 +91,10 @@ async def get_amendments_for_motion(motion, all_data):
return amendment_data
async def get_amendment_base_motion(amendment, all_data):
motion = get_model(all_data, "motions/motion", amendment.get("parent_id"))
async def get_amendment_base_motion(amendment, all_data_provider):
motion = await get_model(
all_data_provider, "motions/motion", amendment.get("parent_id")
)
return {
"identifier": motion["identifier"],
@ -102,15 +103,17 @@ async def get_amendment_base_motion(amendment, all_data):
}
async def get_amendment_base_statute(amendment, all_data):
statute = get_model(
all_data, "motions/statute-paragraph", amendment.get("statute_paragraph_id")
async def get_amendment_base_statute(amendment, all_data_provider):
statute = await get_model(
all_data_provider,
"motions/statute-paragraph",
amendment.get("statute_paragraph_id"),
)
return {"title": statute["title"], "text": statute["text"]}
async def extend_reference_motion_dict(
all_data: AllData,
all_data_provider: ProjectorAllDataProvider,
recommendation: Optional[str],
referenced_motions: Dict[int, Dict[str, str]],
) -> None:
@ -127,15 +130,18 @@ async def extend_reference_motion_dict(
]
for id in referenced_ids:
# Put every referenced motion into the referenced_motions dict
if id not in referenced_motions and id in all_data["motions/motion"]:
referenced_motion = await all_data_provider.get("motions/motion", id)
if id not in referenced_motions and referenced_motion is not None:
referenced_motions[id] = {
"title": all_data["motions/motion"][id]["title"],
"identifier": all_data["motions/motion"][id]["identifier"],
"title": referenced_motion["title"],
"identifier": referenced_motion["identifier"],
}
async def motion_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
all_data_provider: ProjectorAllDataProvider,
element: Dict[str, Any],
projector_id: int,
) -> Dict[str, Any]:
"""
Motion slide.
@ -158,13 +164,16 @@ async def motion_slide(
"""
# Get motion
mode = element.get(
"mode", await get_config(all_data, "motions_recommendation_text_mode")
"mode", await get_config(all_data_provider, "motions_recommendation_text_mode")
)
motion = get_model(all_data, "motions/motion", element.get("id"))
# populate cache:
motion = await get_model(all_data_provider, "motions/motion", element.get("id"))
# Add submitters
submitters = [
await get_user_name(all_data, submitter["user_id"])
await get_user_name(all_data_provider, submitter["user_id"])
for submitter in sorted(
motion["submitters"], key=lambda submitter: submitter["weight"]
)
@ -172,14 +181,16 @@ async def motion_slide(
# Get some needed config values
show_meta_box = not await get_config(
all_data, "motions_disable_sidebox_on_projector"
all_data_provider, "motions_disable_sidebox_on_projector"
)
show_referring_motions = not await get_config(
all_data, "motions_hide_referring_motions"
all_data_provider, "motions_hide_referring_motions"
)
line_length = await get_config(all_data, "motions_line_length")
line_numbering_mode = await get_config(all_data, "motions_default_line_numbering")
motions_preamble = await get_config(all_data, "motions_preamble")
line_length = await get_config(all_data_provider, "motions_line_length")
line_numbering_mode = await get_config(
all_data_provider, "motions_default_line_numbering"
)
motions_preamble = await get_config(all_data_provider, "motions_preamble")
# Query all change-recommendation and amendment related things.
change_recommendations = [] # type: ignore
@ -187,17 +198,19 @@ async def motion_slide(
base_motion = None
base_statute = None
if motion["statute_paragraph_id"]:
base_statute = await get_amendment_base_statute(motion, all_data)
base_statute = await get_amendment_base_statute(motion, all_data_provider)
elif motion["parent_id"] is not None and motion["amendment_paragraphs"]:
base_motion = await get_amendment_base_motion(motion, all_data)
base_motion = await get_amendment_base_motion(motion, all_data_provider)
else:
for change_recommendation_id in motion["change_recommendations_id"]:
cr = all_data["motions/motion-change-recommendation"].get(
change_recommendation_id
cr = await get_model(
all_data_provider,
"motions/motion-change-recommendation",
change_recommendation_id,
)
if cr is not None and not cr["internal"]:
change_recommendations.append(cr)
amendments = await get_amendments_for_motion(motion, all_data)
amendments = await get_amendments_for_motion(motion, all_data_provider)
# The base return value. More fields will get added below.
return_value = {
@ -217,10 +230,10 @@ async def motion_slide(
"line_numbering_mode": line_numbering_mode,
}
if not await get_config(all_data, "motions_disable_text_on_projector"):
if not await get_config(all_data_provider, "motions_disable_text_on_projector"):
return_value["text"] = motion["text"]
if not await get_config(all_data, "motions_disable_reason_on_projector"):
if not await get_config(all_data_provider, "motions_disable_reason_on_projector"):
return_value["reason"] = motion["reason"]
if mode == "final":
@ -228,40 +241,46 @@ async def motion_slide(
# Add recommendation, if enabled in config (and the motion has one)
if (
not await get_config(all_data, "motions_disable_recommendation_on_projector")
not await get_config(
all_data_provider, "motions_disable_recommendation_on_projector"
)
and motion["recommendation_id"]
):
recommendation_state = await get_state(all_data, motion, "recommendation_id")
recommendation_state = await get_state(
all_data_provider, motion, "recommendation_id"
)
return_value["recommendation"] = recommendation_state["recommendation_label"]
if recommendation_state["show_recommendation_extension_field"]:
recommendation_extension = motion["recommendation_extension"]
# All title information for referenced motions in the recommendation
referenced_motions: Dict[int, Dict[str, str]] = {}
await extend_reference_motion_dict(
all_data, recommendation_extension, referenced_motions
all_data_provider, recommendation_extension, referenced_motions
)
return_value["recommendation_extension"] = recommendation_extension
return_value["referenced_motions"] = referenced_motions
if motion["statute_paragraph_id"]:
return_value["recommender"] = await get_config(
all_data, "motions_statute_recommendations_by"
all_data_provider, "motions_statute_recommendations_by"
)
else:
return_value["recommender"] = await get_config(
all_data, "motions_recommendations_by"
all_data_provider, "motions_recommendations_by"
)
if show_referring_motions:
# Add recommendation-referencing motions
return_value[
"recommendation_referencing_motions"
] = await get_recommendation_referencing_motions(all_data, motion["id"])
] = await get_recommendation_referencing_motions(
all_data_provider, motion["id"]
)
return return_value
async def get_recommendation_referencing_motions(
all_data: AllData, motion_id: int
all_data_provider: ProjectorAllDataProvider, motion_id: int
) -> Optional[List[Dict[str, Any]]]:
"""
Returns all title information for motions, that are referencing
@ -269,14 +288,15 @@ async def get_recommendation_referencing_motions(
motions, None is returned (instead of []).
"""
recommendation_referencing_motions = []
for motion in all_data["motions/motion"].values():
all_motions = await all_data_provider.get_collection("motions/motion")
for motion in all_motions.values():
# Motion must have a recommendation and a recommendaiton extension
if not motion["recommendation_id"] or not motion["recommendation_extension"]:
continue
# The recommendation must allow the extension field (there might be left-overs
# in a motions recommendation extension..)
recommendation = await get_state(all_data, motion, "recommendation_id")
recommendation = await get_state(all_data_provider, motion, "recommendation_id")
if not recommendation["show_recommendation_extension_field"]:
continue
@ -297,12 +317,16 @@ async def get_recommendation_referencing_motions(
async def motion_block_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
all_data_provider: ProjectorAllDataProvider,
element: Dict[str, Any],
projector_id: int,
) -> Dict[str, Any]:
"""
Motion block slide.
"""
motion_block = get_model(all_data, "motions/motion-block", element.get("id"))
motion_block = await get_model(
all_data_provider, "motions/motion-block", element.get("id")
)
# All motions in this motion block
motions = []
@ -311,7 +335,8 @@ async def motion_block_slide(
referenced_motions: Dict[int, Dict[str, str]] = {}
# Search motions.
for motion in all_data["motions/motion"].values():
all_motions = await all_data_provider.get_collection("motions/motion")
for motion in all_motions.values():
if motion["motion_block_id"] == motion_block["id"]:
motion_object = {
"title": motion["title"],
@ -320,7 +345,9 @@ async def motion_block_slide(
recommendation_id = motion["recommendation_id"]
if recommendation_id is not None:
recommendation = await get_state(all_data, motion, "recommendation_id")
recommendation = await get_state(
all_data_provider, motion, "recommendation_id"
)
motion_object["recommendation"] = {
"name": recommendation["recommendation_label"],
"css_class": recommendation["css_class"],
@ -328,7 +355,7 @@ async def motion_block_slide(
if recommendation["show_recommendation_extension_field"]:
recommendation_extension = motion["recommendation_extension"]
await extend_reference_motion_dict(
all_data, recommendation_extension, referenced_motions
all_data_provider, recommendation_extension, referenced_motions
)
motion_object["recommendation_extension"] = recommendation_extension
@ -342,13 +369,15 @@ async def motion_block_slide(
async def motion_poll_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
all_data_provider: ProjectorAllDataProvider,
element: Dict[str, Any],
projector_id: int,
) -> Dict[str, Any]:
"""
Poll slide.
"""
poll = get_model(all_data, "motions/motion-poll", element.get("id"))
motion = get_model(all_data, "motions/motion", poll["motion_id"])
poll = await get_model(all_data_provider, "motions/motion-poll", element.get("id"))
motion = await get_model(all_data_provider, "motions/motion", poll["motion_id"])
poll_data = {
key: poll[key]
@ -363,8 +392,8 @@ async def motion_poll_slide(
}
if poll["state"] == MotionPoll.STATE_PUBLISHED:
option = get_model(
all_data, "motions/motion-option", poll["options_id"][0]
option = await get_model(
all_data_provider, "motions/motion-option", poll["options_id"][0]
) # there can only be exactly one option
poll_data["options"] = [
{

31
openslides/topics/projector.py
View File

@ -1,19 +1,16 @@
from typing import Any, Dict
from ..utils.projector import (
AllData,
ProjectorElementException,
ProjectorAllDataProvider,
get_model,
register_projector_slide,
)
# Important: All functions have to be prune. This means, that thay can only
# access the data, that they get as argument and do not have any
# side effects.
async def topic_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
all_data_provider: ProjectorAllDataProvider,
element: Dict[str, Any],
projector_id: int,
) -> Dict[str, Any]:
"""
Topic slide.
@ -22,22 +19,8 @@ async def topic_slide(
* title
* text
"""
topic_id = element.get("id")
if topic_id is None:
raise ProjectorElementException("id is required for topic slide")
try:
topic = all_data["topics/topic"][topic_id]
except KeyError:
raise ProjectorElementException(f"topic with id {topic_id} does not exist")
item_id = topic["agenda_item_id"]
try:
item = all_data["agenda/item"][item_id]
except KeyError:
raise ProjectorElementException(f"item with id {item_id} does not exist")
topic = await get_model(all_data_provider, "topics/topic", element.get("id"))
item = await get_model(all_data_provider, "agenda/item", topic["agenda_item_id"])
return {
"title": topic["title"],
"text": topic["text"],

31
openslides/users/projector.py
View File

@ -1,19 +1,16 @@
from typing import Any, Dict, List
from typing import Any, Dict, List, Optional
from ..utils.projector import (
AllData,
ProjectorElementException,
ProjectorAllDataProvider,
get_model,
register_projector_slide,
)
# Important: All functions have to be prune. This means, that thay can only
# access the data, that they get as argument and do not have any
# side effects.
async def user_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
all_data_provider: ProjectorAllDataProvider,
element: Dict[str, Any],
projector_id: int,
) -> Dict[str, Any]:
"""
User slide.
@ -21,22 +18,16 @@ async def user_slide(
The returned dict can contain the following fields:
* user
"""
user_id = element.get("id")
if user_id is None:
raise ProjectorElementException("id is required for user slide")
return {"user": await get_user_name(all_data, user_id)}
return {"user": await get_user_name(all_data_provider, element.get("id"))}
async def get_user_name(all_data: AllData, user_id: int) -> str:
async def get_user_name(
all_data_provider: ProjectorAllDataProvider, user_id: Optional[int]
) -> str:
"""
Returns the short name for an user_id.
"""
try:
user = all_data["users/user"][user_id]
except KeyError:
raise ProjectorElementException(f"user with id {user_id} does not exist")
user = await get_model(all_data_provider, "users/user", user_id)
name_parts: List[str] = []
for name_part in ("title", "first_name", "last_name"):

10
openslides/utils/autoupdate.py
View File

@ -92,13 +92,13 @@ class AutoupdateBundle:
elements[full_data["id"]]["full_data"] = full_data
# Save histroy here using sync code.
save_history(self.elements)
save_history(self.element_iterator)
# Update cache and send autoupdate using async code.
async_to_sync(self.async_handle_collection_elements)()
@property
def elements(self) -> Iterable[AutoupdateElement]:
def element_iterator(self) -> Iterable[AutoupdateElement]:
""" Iterator for all elements in this bundle """
for elements in self.autoupdate_elements.values():
yield from elements.values()
@ -110,7 +110,7 @@ class AutoupdateBundle:
Returns the change_id
"""
cache_elements: Dict[str, Optional[Dict[str, Any]]] = {}
for element in self.elements:
for element in self.element_iterator:
element_id = get_element_id(element["collection_string"], element["id"])
full_data = element.get("full_data")
if full_data:
@ -253,7 +253,7 @@ class AutoupdateBundleMiddleware:
return response
def save_history(elements: Iterable[AutoupdateElement]) -> Iterable:
def save_history(element_iterator: Iterable[AutoupdateElement]) -> Iterable:
"""
Thin wrapper around the call of history saving manager method.
@ -261,4 +261,4 @@ def save_history(elements: Iterable[AutoupdateElement]) -> Iterable:
"""
from ..core.models import History
return History.objects.add_elements(elements)
return History.objects.add_elements(element_iterator)

19
openslides/utils/cache.py
View File

@ -254,25 +254,6 @@ class ElementCache:
all_data[collection] = await restricter(user_id, all_data[collection])
return dict(all_data)
async def get_all_data_dict(self) -> Dict[str, Dict[int, Dict[str, Any]]]:
"""
Returns all data with a dict (id <-> element) per collection:
{
<collection>: {
<id>: <element>
}
}
"""
all_data: Dict[str, Dict[int, Dict[str, Any]]] = defaultdict(dict)
for element_id, data in (await self.cache_provider.get_all_data()).items():
collection, id = split_element_id(element_id)
element = json.loads(data.decode())
element.pop(
"_no_delete_on_restriction", False
) # remove special field for get_data_since
all_data[collection][id] = element
return dict(all_data)
async def get_collection_data(self, collection: str) -> Dict[int, Dict[str, Any]]:
"""
Returns the data for one collection as dict: {id: <element>}

3
openslides/utils/cache_providers.py
View File

@ -460,7 +460,8 @@ class RedisCacheProvider:
)
if reported_amount != read_only_redis_amount_replicas:
logger.warn(
f"WAIT reported {reported_amount} replicas of {read_only_redis_amount_replicas} requested after {read_only_redis_wait_timeout} ms!"
f"WAIT reported {reported_amount} replicas of {read_only_redis_amount_replicas} "
+ f"requested after {read_only_redis_wait_timeout} ms!"
)
return result

82
openslides/utils/projector.py
View File

@ -5,16 +5,14 @@ Functions that handel the registration of projector elements and the rendering
of the data to present it on the projector.
"""
from typing import Any, Awaitable, Callable, Dict, List
from collections import defaultdict
from typing import Any, Awaitable, Callable, Dict, List, Optional
from . import logging
from .cache import element_cache
AllData = Dict[str, Dict[int, Dict[str, Any]]]
ProjectorSlide = Callable[[AllData, Dict[str, Any], int], Awaitable[Dict[str, Any]]]
projector_slides: Dict[str, ProjectorSlide] = {}
logger = logging.getLogger(__name__)
class ProjectorElementException(Exception):
@ -23,6 +21,44 @@ class ProjectorElementException(Exception):
"""
class ProjectorAllDataProvider:
NON_EXISTENT_MARKER = object()
def __init__(self) -> None:
self.cache: Any = defaultdict(dict) # fuu you mypy
self.fetched_collection: Dict[str, bool] = {}
async def get(self, collection: str, id: int) -> Optional[Dict[str, Any]]:
cache_data = self.cache[collection].get(id)
if cache_data is None:
data: Any = await element_cache.get_element_data(collection, id)
if data is None:
data = ProjectorAllDataProvider.NON_EXISTENT_MARKER
self.cache[collection][id] = data
elif cache_data == ProjectorAllDataProvider.NON_EXISTENT_MARKER:
return None
return self.cache[collection][id]
async def get_collection(self, collection: str) -> Dict[int, Dict[str, Any]]:
if not self.fetched_collection.get(collection, False):
collection_data = await element_cache.get_collection_data(collection)
self.cache[collection] = collection_data
self.fetched_collection[collection] = True
return self.cache[collection]
async def exists(self, collection: str, id: int) -> bool:
model = await self.get(collection, id)
return model is not None
ProjectorSlide = Callable[
[ProjectorAllDataProvider, Dict[str, Any], int], Awaitable[Dict[str, Any]]
]
projector_slides: Dict[str, ProjectorSlide] = {}
def register_projector_slide(name: str, slide: ProjectorSlide) -> None:
"""
Registers a projector slide.
@ -67,10 +103,11 @@ async def get_projector_data(
if projector_ids is None:
projector_ids = []
all_data = await element_cache.get_all_data_dict()
projector_data: Dict[int, List[Dict[str, Any]]] = {}
all_data_provider = ProjectorAllDataProvider()
projectors = await all_data_provider.get_collection("core/projector")
for projector_id, projector in all_data.get("core/projector", {}).items():
for projector_id, projector in projectors.items():
if projector_ids and projector_id not in projector_ids:
# only render the projector in question.
continue
@ -83,7 +120,7 @@ async def get_projector_data(
for element in projector["elements"]:
projector_slide = projector_slides[element["name"]]
try:
data = await projector_slide(all_data, element, projector_id)
data = await projector_slide(all_data_provider, element, projector_id)
except ProjectorElementException as err:
data = {"error": str(err)}
projector_data[projector_id].append({"data": data, "element": element})
@ -91,18 +128,23 @@ async def get_projector_data(
return projector_data
async def get_config(all_data: AllData, key: str) -> Any:
async def get_config(all_data_provider: ProjectorAllDataProvider, key: str) -> Any:
"""
Returns a config value from all_data.
Returns a config value from all_data_provider.
Triggers the cache early: It access `get_colelction` instead of `get`. It
allows for all successive queries for configs to be cached.
"""
from ..core.config import config
config_id = (await config.async_get_key_to_id())[key]
return all_data[config.get_collection_string()][config_id]["value"]
configs = await all_data_provider.get_collection(config.get_collection_string())
return configs[config_id]["value"]
def get_model(all_data: AllData, collection: str, id: Any) -> Dict[str, Any]:
async def get_model(
all_data_provider: ProjectorAllDataProvider, collection: str, id: Any
) -> Dict[str, Any]:
"""
Tries to get the model identified by the collection and id.
If the id is invalid or the model not found, ProjectorElementExceptions will be raised.
@ -110,17 +152,19 @@ def get_model(all_data: AllData, collection: str, id: Any) -> Dict[str, Any]:
if id is None:
raise ProjectorElementException(f"id is required for {collection} slide")
try:
model = all_data[collection][id]
except KeyError:
model = await all_data_provider.get(collection, id)
if model is None:
raise ProjectorElementException(f"{collection} with id {id} does not exist")
return model
def get_models(
all_data: AllData, collection: str, ids: List[Any]
async def get_models(
all_data_provider: ProjectorAllDataProvider, collection: str, ids: List[Any]
) -> List[Dict[str, Any]]:
"""
Tries to fetch all given models. Models are required to be all of the collection `collection`.
"""
return [get_model(all_data, collection, id) for id in ids]
logger.info(
f"Note: a call to `get_models` with {collection}/{ids}. This might be cache-intensive"
)
return [await get_model(all_data_provider, collection, id) for id in ids]