ag-admin/OpenSlides
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make the projector-system explicit async.

Browse Source 
Also lock config.build_key_to_id
This commit is contained in:
Oskar Hahn 2019-03-23 12:06:57 +01:00
parent 028c358a7f
commit 41aed15426
13 changed files with 126 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
openslides/agenda/projector.py
View File

@ -12,11 +12,10 @@ from ..utils.projector import (
# Important: All functions have to be prune. This means, that thay can only
# access the data, that they get as argument and do not have any
# side effects. They are called from an async context. So they have
# to be fast!
# side effects.
def get_sorted_agenda_items(all_data: AllData) -> List[Dict[str, Any]]:
async def get_sorted_agenda_items(all_data: AllData) -> List[Dict[str, Any]]:
"""
Returns all sorted agenda items by id first and then weight, resulting in
ordered items, if some have the same weight.
@ -27,7 +26,7 @@ def get_sorted_agenda_items(all_data: AllData) -> List[Dict[str, Any]]:
)
def get_flat_tree(all_data: AllData, parent_id: int = 0) -> List[Dict[str, Any]]:
async def get_flat_tree(all_data: AllData, parent_id: int = 0) -> List[Dict[str, Any]]:
"""
Build the item tree from all_data.
@ -40,13 +39,13 @@ def get_flat_tree(all_data: AllData, parent_id: int = 0) -> List[Dict[str, Any]]
# Build a dict from an item_id to all its children
children: Dict[int, List[int]] = defaultdict(list)
if "agenda/item" in all_data:
for item in get_sorted_agenda_items(all_data):
for item in await get_sorted_agenda_items(all_data):
if item["type"] == 1: # only normal items
children[item["parent_id"] or 0].append(item["id"])
tree = []
def get_children(item_ids: List[int], depth: int) -> None:
async def get_children(item_ids: List[int], depth: int) -> None:
for item_id in item_ids:
tree.append(
{
@ -60,13 +59,13 @@ def get_flat_tree(all_data: AllData, parent_id: int = 0) -> List[Dict[str, Any]]
"depth": depth,
}
)
get_children(children[item_id], depth + 1)
await get_children(children[item_id], depth + 1)
get_children(children[parent_id], 0)
await get_children(children[parent_id], 0)
return tree
def item_list_slide(
async def item_list_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
) -> Dict[str, Any]:
"""
@ -78,7 +77,7 @@ def item_list_slide(
if only_main_items:
agenda_items = []
for item in get_sorted_agenda_items(all_data):
for item in await get_sorted_agenda_items(all_data):
if item["parent_id"] is None and item["type"] == 1:
agenda_items.append(
{
@ -88,12 +87,12 @@ def item_list_slide(
}
)
else:
agenda_items = get_flat_tree(all_data)
agenda_items = await get_flat_tree(all_data)
return {"items": agenda_items}
def list_of_speakers_slide(
async def list_of_speakers_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
) -> Dict[str, Any]:
"""
@ -106,10 +105,12 @@ def list_of_speakers_slide(
if item_id is None:
raise ProjectorElementException("id is required for list of speakers slide")
return get_list_of_speakers_slide_data(all_data, item_id)
return await get_list_of_speakers_slide_data(all_data, item_id)
def get_list_of_speakers_slide_data(all_data: AllData, item_id: int) -> Dict[str, Any]:
async def get_list_of_speakers_slide_data(
all_data: AllData, item_id: int
) -> Dict[str, Any]:
try:
item = all_data["agenda/item"][item_id]
except KeyError:
@ -120,7 +121,7 @@ def get_list_of_speakers_slide_data(all_data: AllData, item_id: int) -> Dict[str
speakers_finished = []
current_speaker = None
for speaker in item["speakers"]:
user = get_user_name(all_data, speaker["user_id"])
user = await get_user_name(all_data, speaker["user_id"])
formatted_speaker = {
"user": user,
"marked": speaker["marked"],
@ -139,7 +140,7 @@ def get_list_of_speakers_slide_data(all_data: AllData, item_id: int) -> Dict[str
speakers_waiting = sorted(speakers_waiting, key=lambda s: s["weight"])
speakers_finished = sorted(speakers_finished, key=lambda s: s["end_time"])
number_of_last_speakers = get_config(all_data, "agenda_show_last_speakers")
number_of_last_speakers = await get_config(all_data, "agenda_show_last_speakers")
if number_of_last_speakers == 0:
speakers_finished = []
else:
@ -157,7 +158,7 @@ def get_list_of_speakers_slide_data(all_data: AllData, item_id: int) -> Dict[str
}
def get_current_item_id_for_projector(
async def get_current_item_id_for_projector(
all_data: AllData, projector: Dict[str, Any]
) -> Union[int, None]:
"""
@ -189,7 +190,9 @@ def get_current_item_id_for_projector(
return item_id
def get_reference_projector(all_data: AllData, projector_id: int) -> Dict[str, Any]:
async def get_reference_projector(
all_data: AllData, projector_id: int
) -> Dict[str, Any]:
"""
Returns the reference projector to the given projector (by id)
"""
@ -209,28 +212,28 @@ def get_reference_projector(all_data: AllData, projector_id: int) -> Dict[str, A
return reference_projector
def current_list_of_speakers_slide(
async def current_list_of_speakers_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
) -> Dict[str, Any]:
"""
The current list of speakers slide. Creates the data for the given projector.
"""
reference_projector = get_reference_projector(all_data, projector_id)
item_id = get_current_item_id_for_projector(all_data, reference_projector)
reference_projector = await get_reference_projector(all_data, projector_id)
item_id = await get_current_item_id_for_projector(all_data, reference_projector)
if item_id is None: # no element found
return {}
return get_list_of_speakers_slide_data(all_data, item_id)
return await get_list_of_speakers_slide_data(all_data, item_id)
def current_speaker_chyron_slide(
async def current_speaker_chyron_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
) -> Dict[str, Any]:
"""
Returns the username for the current speaker.
"""
reference_projector = get_reference_projector(all_data, projector_id)
item_id = get_current_item_id_for_projector(all_data, reference_projector)
reference_projector = await get_reference_projector(all_data, projector_id)
item_id = await get_current_item_id_for_projector(all_data, reference_projector)
if item_id is None: # no element found
return {}
@ -244,7 +247,7 @@ def current_speaker_chyron_slide(
current_speaker = None
for speaker in item["speakers"]:
if speaker["begin_time"] is not None and speaker["end_time"] is None:
current_speaker = get_user_name(all_data, speaker["user_id"])
current_speaker = await get_user_name(all_data, speaker["user_id"])
return {"current_speaker": current_speaker}

5
openslides/assignments/projector.py
View File

@ -5,11 +5,10 @@ from ..utils.projector import AllData, register_projector_slide
# Important: All functions have to be prune. This means, that thay can only
# access the data, that they get as argument and do not have any
# side effects. They are called from an async context. So they have
# to be fast!
# side effects.
def assignment_slide(
async def assignment_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
) -> Dict[str, Any]:
"""

17
openslides/core/config.py
View File

@ -1,3 +1,4 @@
import asyncio
from typing import Any, Callable, Dict, Iterable, Optional, TypeVar, Union, cast
from asgiref.sync import async_to_sync
@ -23,6 +24,8 @@ INPUT_TYPE_MAPPING = {
"translations": list,
}
build_key_to_id_lock = asyncio.Lock()
class ConfigHandler:
"""
@ -59,6 +62,15 @@ class ConfigHandler:
self.key_to_id = cast(Dict[str, int], self.key_to_id)
return self.key_to_id
async def async_get_key_to_id(self) -> Dict[str, int]:
"""
Like get_key_to_id but in an async context.
"""
if self.key_to_id is None:
await self.build_key_to_id()
self.key_to_id = cast(Dict[str, int], self.key_to_id)
return self.key_to_id
async def build_key_to_id(self) -> None:
"""
Build the key_to_id dict.
@ -68,6 +80,11 @@ class ConfigHandler:
This uses the element_cache. It expects, that the config values are in the database
before this is called.
"""
async with build_key_to_id_lock:
# Another cliend could have build the key_to_id_dict, check and return early
if self.key_to_id is not None:
return
config_full_data = await element_cache.get_collection_full_data(
self.get_collection_string()
)

9
openslides/core/projector.py
View File

@ -10,11 +10,10 @@ from ..utils.projector import (
# Important: All functions have to be prune. This means, that thay can only
# access the data, that they get as argument and do not have any
# side effects. They are called from an async context. So they have
# to be fast!
# side effects.
def countdown_slide(
async def countdown_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
) -> Dict[str, Any]:
"""
@ -42,7 +41,7 @@ def countdown_slide(
}
def message_slide(
async def message_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
) -> Dict[str, Any]:
"""
@ -63,7 +62,7 @@ def message_slide(
raise ProjectorElementException(f"Message {message_id} does not exist")
def clock_slide(
async def clock_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
) -> Dict[str, Any]:
return {}

5
openslides/mediafiles/projector.py
View File

@ -9,11 +9,10 @@ from ..utils.projector import (
# Important: All functions have to be prune. This means, that thay can only
# access the data, that they get as argument and do not have any
# side effects. They are called from an async context. So they have
# to be fast!
# side effects.
def mediafile_slide(
async def mediafile_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
) -> Dict[str, Any]:
"""

69
openslides/motions/projector.py
View File

@ -14,11 +14,10 @@ motion_placeholder_regex = re.compile(r"\[motion:(\d+)\]")
# Important: All functions have to be prune. This means, that thay can only
# access the data, that they get as argument and do not have any
# side effects. They are called from an async context. So they have
# to be fast!
# side effects.
def get_state(
async def get_state(
all_data: AllData, motion: Dict[str, Any], state_id: int
) -> Dict[str, Any]:
"""
@ -35,14 +34,14 @@ def get_state(
)
def get_amendment_merge_into_motion_diff(all_data, motion, amendment):
async def get_amendment_merge_into_motion_diff(all_data, motion, amendment):
"""
HINT: This implementation should be consistent to showInDiffView() in ViewMotionAmendedParagraph.ts
"""
if amendment["state_id"] is None:
return 0
state = get_state(all_data, motion, amendment["state_id"])
state = await get_state(all_data, motion, amendment["state_id"])
if state["merge_amendment_into_final"] == -1:
return 0
if state["merge_amendment_into_final"] == 1:
@ -50,35 +49,35 @@ def get_amendment_merge_into_motion_diff(all_data, motion, amendment):
if amendment["recommendation_id"] is None:
return 0
recommendation = get_state(all_data, motion, amendment["recommendation_id"])
recommendation = await get_state(all_data, motion, amendment["recommendation_id"])
if recommendation["merge_amendment_into_final"] == 1:
return 1
return 0
def get_amendment_merge_into_motion_final(all_data, motion, amendment):
async def get_amendment_merge_into_motion_final(all_data, motion, amendment):
"""
HINT: This implementation should be consistent to showInFinalView() in ViewMotionAmendedParagraph.ts
"""
if amendment["state_id"] is None:
return 0
state = get_state(all_data, motion, amendment["state_id"])
state = await get_state(all_data, motion, amendment["state_id"])
if state["merge_amendment_into_final"] == 1:
return 1
return 0
def get_amendments_for_motion(motion, all_data):
async def get_amendments_for_motion(motion, all_data):
amendment_data = []
for amendment_id, amendment in all_data["motions/motion"].items():
if amendment["parent_id"] == motion["id"]:
merge_amendment_into_final = get_amendment_merge_into_motion_final(
merge_amendment_into_final = await get_amendment_merge_into_motion_final(
all_data, motion, amendment
)
merge_amendment_into_diff = get_amendment_merge_into_motion_diff(
merge_amendment_into_diff = await get_amendment_merge_into_motion_diff(
all_data, motion, amendment
)
amendment_data.append(
@ -94,7 +93,7 @@ def get_amendments_for_motion(motion, all_data):
return amendment_data
def get_amendment_base_motion(amendment, all_data):
async def get_amendment_base_motion(amendment, all_data):
try:
motion = all_data["motions/motion"][amendment["parent_id"]]
except KeyError:
@ -108,7 +107,7 @@ def get_amendment_base_motion(amendment, all_data):
}
def get_amendment_base_statute(amendment, all_data):
async def get_amendment_base_statute(amendment, all_data):
try:
statute = all_data["motions/statute-paragraph"][
amendment["statute_paragraph_id"]
@ -120,7 +119,7 @@ def get_amendment_base_statute(amendment, all_data):
return {"title": statute["title"], "text": statute["text"]}
def extend_reference_motion_dict(
async def extend_reference_motion_dict(
all_data: AllData,
recommendation: str,
referenced_motions: Dict[int, Dict[str, str]],
@ -142,7 +141,7 @@ def extend_reference_motion_dict(
}
def motion_slide(
async def motion_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
) -> Dict[str, Any]:
"""
@ -163,7 +162,9 @@ def motion_slide(
* change_recommendations
* submitter
"""
mode = element.get("mode", get_config(all_data, "motions_recommendation_text_mode"))
mode = element.get(
"mode", await get_config(all_data, "motions_recommendation_text_mode")
)
motion_id = element.get("id")
if motion_id is None:
@ -174,20 +175,22 @@ def motion_slide(
except KeyError:
raise ProjectorElementException(f"motion with id {motion_id} does not exist")
show_meta_box = not get_config(all_data, "motions_disable_sidebox_on_projector")
line_length = get_config(all_data, "motions_line_length")
line_numbering_mode = get_config(all_data, "motions_default_line_numbering")
motions_preamble = get_config(all_data, "motions_preamble")
show_meta_box = not await get_config(
all_data, "motions_disable_sidebox_on_projector"
)
line_length = await get_config(all_data, "motions_line_length")
line_numbering_mode = await get_config(all_data, "motions_default_line_numbering")
motions_preamble = await get_config(all_data, "motions_preamble")
if motion["statute_paragraph_id"]:
change_recommendations = [] # type: ignore
amendments = [] # type: ignore
base_motion = None
base_statute = get_amendment_base_statute(motion, all_data)
base_statute = await get_amendment_base_statute(motion, all_data)
elif bool(motion["parent_id"]) and motion["amendment_paragraphs"]:
change_recommendations = []
amendments = []
base_motion = get_amendment_base_motion(motion, all_data)
base_motion = await get_amendment_base_motion(motion, all_data)
base_statute = None
else:
change_recommendations = list(
@ -195,7 +198,7 @@ def motion_slide(
lambda reco: reco["internal"] is False, motion["change_recommendations"]
)
)
amendments = get_amendments_for_motion(motion, all_data)
amendments = await get_amendments_for_motion(motion, all_data)
base_motion = None
base_statute = None
@ -215,7 +218,7 @@ def motion_slide(
"line_numbering_mode": line_numbering_mode,
}
if not get_config(all_data, "motions_disable_reason_on_projector"):
if not await get_config(all_data, "motions_disable_reason_on_projector"):
return_value["reason"] = motion["reason"]
if mode == "final":
@ -223,10 +226,12 @@ def motion_slide(
if show_meta_box:
if (
not get_config(all_data, "motions_disable_recommendation_on_projector")
not await get_config(
all_data, "motions_disable_recommendation_on_projector"
)
and motion["recommendation_id"]
):
recommendation_state = get_state(
recommendation_state = await get_state(
all_data, motion, motion["recommendation_id"]
)
return_value["recommendation"] = recommendation_state[
@ -236,18 +241,18 @@ def motion_slide(
recommendation_extension = motion["recommendation_extension"]
# All title information for referenced motions in the recommendation
referenced_motions: Dict[int, Dict[str, str]] = {}
extend_reference_motion_dict(
await extend_reference_motion_dict(
all_data, recommendation_extension, referenced_motions
)
return_value["recommendation_extension"] = recommendation_extension
return_value["referenced_motions"] = referenced_motions
return_value["recommender"] = get_config(
return_value["recommender"] = await get_config(
all_data, "motions_recommendations_by"
)
return_value["submitter"] = [
get_user_name(all_data, submitter["user_id"])
await get_user_name(all_data, submitter["user_id"])
for submitter in sorted(
motion["submitters"], key=lambda submitter: submitter["weight"]
)
@ -256,7 +261,7 @@ def motion_slide(
return return_value
def motion_block_slide(
async def motion_block_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
) -> Dict[str, Any]:
"""
@ -290,7 +295,7 @@ def motion_block_slide(
recommendation_id = motion["recommendation_id"]
if recommendation_id is not None:
recommendation = get_state(
recommendation = await get_state(
all_data, motion, motion["recommendation_id"]
)
motion_object["recommendation"] = {
@ -299,7 +304,7 @@ def motion_block_slide(
}
if recommendation["show_recommendation_extension_field"]:
recommendation_extension = motion["recommendation_extension"]
extend_reference_motion_dict(
await extend_reference_motion_dict(
all_data, recommendation_extension, referenced_motions
)
motion_object["recommendation_extension"] = recommendation_extension

5
openslides/topics/projector.py
View File

@ -9,11 +9,10 @@ from ..utils.projector import (
# Important: All functions have to be prune. This means, that thay can only
# access the data, that they get as argument and do not have any
# side effects. They are called from an async context. So they have
# to be fast!
# side effects.
def topic_slide(
async def topic_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
) -> Dict[str, Any]:
"""

7
openslides/users/projector.py
View File

@ -9,11 +9,10 @@ from ..utils.projector import (
# Important: All functions have to be prune. This means, that thay can only
# access the data, that they get as argument and do not have any
# side effects. They are called from an async context. So they have
# to be fast!
# side effects.
def user_slide(
async def user_slide(
all_data: AllData, element: Dict[str, Any], projector_id: int
) -> Dict[str, Any]:
"""
@ -30,7 +29,7 @@ def user_slide(
return {"user": get_user_name(all_data, user_id)}
def get_user_name(all_data: AllData, user_id: int) -> str:
async def get_user_name(all_data: AllData, user_id: int) -> str:
"""
Returns the short name for an user_id.
"""

7
openslides/utils/auth.py
View File

@ -1,4 +1,4 @@
from typing import Dict, List, Union, cast
from typing import List, Union
from asgiref.sync import async_to_sync
from django.apps import apps
@ -157,12 +157,9 @@ async def async_anonymous_is_enabled() -> bool:
"""
from ..core.config import config
if config.key_to_id is None:
await config.build_key_to_id()
config.key_to_id = cast(Dict[str, int], config.key_to_id)
element = await element_cache.get_element_full_data(
config.get_collection_string(),
config.key_to_id["general_system_enable_anonymous"],
(await config.async_get_key_to_id())["general_system_enable_anonymous"],
)
return False if element is None else element["value"]

14
openslides/utils/projector.py
View File

@ -5,13 +5,13 @@ Functions that handel the registration of projector elements and the rendering
of the data to present it on the projector.
"""
from typing import Any, Callable, Dict, List
from typing import Any, Awaitable, Callable, Dict, List
from .cache import element_cache
AllData = Dict[str, Dict[int, Dict[str, Any]]]
ProjectorSlide = Callable[[AllData, Dict[str, Any], int], Dict[str, Any]]
ProjectorSlide = Callable[[AllData, Dict[str, Any], int], Awaitable[Dict[str, Any]]]
projector_slides: Dict[str, ProjectorSlide] = {}
@ -83,7 +83,7 @@ async def get_projector_data(
for element in projector["elements"]:
projector_slide = projector_slides[element["name"]]
try:
data = projector_slide(all_data, element, projector_id)
data = await projector_slide(all_data, element, projector_id)
except ProjectorElementException as err:
data = {"error": str(err)}
projector_data[projector_id].append({"data": data, "element": element})
@ -91,12 +91,12 @@ async def get_projector_data(
return projector_data
def get_config(all_data: AllData, key: str) -> Any:
async def get_config(all_data: AllData, key: str) -> Any:
"""
Returns a config value from all_data.
"""
from ..core.config import config
return all_data[config.get_collection_string()][config.get_key_to_id()[key]][
"value"
]
config_id = (await config.async_get_key_to_id())[key]
return all_data[config.get_collection_string()][config_id]["value"]

9
tests/integration/helpers.py
View File

@ -90,16 +90,19 @@ class TProjector:
return elements
def slide1(
async def slide1(
all_data: AllData, element: Dict[str, Any], projector_id: int
) -> Dict[str, Any]:
"""
Slide that shows the general_event_name.
"""
return {"name": "slide1", "event_name": get_config(all_data, "general_event_name")}
return {
"name": "slide1",
"event_name": await get_config(all_data, "general_event_name"),
}
def slide2(
async def slide2(
all_data: AllData, element: Dict[str, Any], projector_id: int
) -> Dict[str, Any]:
return {"name": "slide2"}

10
tests/unit/agenda/test_projector.py
View File

@ -85,10 +85,11 @@ def all_data():
return all_data
def test_main_items(all_data):
@pytest.mark.asyncio
async def test_main_items(all_data):
element: Dict[str, Any] = {}
data = projector.item_list_slide(all_data, element, 1)
data = await projector.item_list_slide(all_data, element, 1)
assert data == {
"items": [
@ -106,10 +107,11 @@ def test_main_items(all_data):
}
def test_all_items(all_data):
@pytest.mark.asyncio
async def test_all_items(all_data):
element: Dict[str, Any] = {"only_main_items": False}
data = projector.item_list_slide(all_data, element, 1)
data = await projector.item_list_slide(all_data, element, 1)
assert data == {
"items": [

15
tests/unit/motions/test_projector.py
View File

@ -253,10 +253,11 @@ def all_data():
return return_value
def test_motion_slide(all_data):
@pytest.mark.asyncio
async def test_motion_slide(all_data):
element: Dict[str, Any] = {"id": 1}
data = projector.motion_slide(all_data, element, 1)
data = await projector.motion_slide(all_data, element, 1)
assert data == {
"identifier": "4",
@ -299,10 +300,11 @@ def test_motion_slide(all_data):
}
def test_amendment_slide(all_data):
@pytest.mark.asyncio
async def test_amendment_slide(all_data):
element: Dict[str, Any] = {"id": 2}
data = projector.motion_slide(all_data, element, 1)
data = await projector.motion_slide(all_data, element, 1)
assert data == {
"identifier": "Ä1",
@ -323,10 +325,11 @@ def test_amendment_slide(all_data):
}
def test_statute_amendment_slide(all_data):
@pytest.mark.asyncio
async def test_statute_amendment_slide(all_data):
element: Dict[str, Any] = {"id": 3}
data = projector.motion_slide(all_data, element, 1)
data = await projector.motion_slide(all_data, element, 1)
assert data == {
"identifier": None,