Merge pull request #5835 from reiterl/dev_check_json
Work on check_json.py
This commit is contained in:
Finn Stutzenstein
GitHub
commit
4794f063f8
24
.github/workflows/models.yml
vendored
24
.github/workflows/models.yml
vendored
@ -1,7 +1,9 @@
|
||||
name: Validate models.yml
|
||||
on: [push, pull_request]
|
||||
env:
|
||||
PYTHON_VERSION: 3.8.5
|
||||
jobs:
|
||||
validate:
|
||||
validate-models:
|
||||
name: Validate models.yml
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@ -21,3 +23,23 @@ jobs:
|
||||
|
||||
- name: Validate models.yml
|
||||
run: docs/modelsvalidator/modelsvalidator docs/models.yml
|
||||
|
||||
validate-data:
|
||||
name: Validate example-data.json and initial-data.json
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Install requirements
|
||||
working-directory: docs/datavalidator
|
||||
run: pip install -U -r requirements.txt
|
||||
|
||||
- name: Validate
|
||||
working-directory: docs/datavalidator
|
||||
run: python check_json.py
|
||||
|
||||
@ -248,7 +248,7 @@
|
||||
"temporary_user_ids": [],
|
||||
"guest_ids": [],
|
||||
"user_ids": [1],
|
||||
"reference_projector_id": 2,
|
||||
"reference_projector_id": 1,
|
||||
|
||||
"default_group_id": 1,
|
||||
"admin_group_id": 2
|
||||
@ -546,7 +546,7 @@
|
||||
"current_element_ids": [],
|
||||
"preview_projection_ids": [],
|
||||
"history_projection_ids": [],
|
||||
"used_as_reference_projector_meeting_id": null,
|
||||
"used_as_reference_projector_meeting_id": 1,
|
||||
"projectiondefault_ids": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13],
|
||||
"meeting_id": 1
|
||||
}],
|
||||
|
||||
384
docs/datavalidator/check_json.py
Normal file
384
docs/datavalidator/check_json.py
Normal file
@ -0,0 +1,384 @@
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
import fastjsonschema
|
||||
import yaml
|
||||
|
||||
MODELS_YML_PATH = "../../docs/models.yml"
|
||||
|
||||
CHECKED_FILES = [
|
||||
"../../docker/initial-data.json",
|
||||
"../../docs/example-data.json",
|
||||
]
|
||||
|
||||
SCHEMA = fastjsonschema.compile(
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Schema for initial and example data.",
|
||||
"type": "object",
|
||||
"patternProperties": {
|
||||
"^[a-z_]+$": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {"id": {"type": "number"}},
|
||||
"required": ["id"],
|
||||
},
|
||||
}
|
||||
},
|
||||
"additionalProperties": False,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class CheckException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def check_string(value: Any) -> bool:
|
||||
return value is None or isinstance(value, str)
|
||||
|
||||
|
||||
def check_number(value: Any) -> bool:
|
||||
return value is None or isinstance(value, int)
|
||||
|
||||
|
||||
def check_boolean(value: Any) -> bool:
|
||||
return value is None or value is False or value is True
|
||||
|
||||
|
||||
def check_string_list(value: Any) -> bool:
|
||||
return check_x_list(value, check_string)
|
||||
|
||||
|
||||
def check_number_list(value: Any) -> bool:
|
||||
return check_x_list(value, check_number)
|
||||
|
||||
|
||||
def check_x_list(value: Any, fn: Callable) -> bool:
|
||||
if value is None:
|
||||
return True
|
||||
if not isinstance(value, list):
|
||||
return False
|
||||
return all([fn(sv) for sv in value])
|
||||
|
||||
|
||||
def check_decimal(value: Any) -> bool:
|
||||
if value is None:
|
||||
return True
|
||||
if isinstance(value, str):
|
||||
pattern = r"^-?(\d|[1-9]\d+)\.\d{6}$"
|
||||
if re.match(pattern, value):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class Checker:
|
||||
def __init__(self, data: Dict[str, List[Any]]) -> None:
|
||||
self.data = data
|
||||
|
||||
with open(MODELS_YML_PATH, "rb") as x:
|
||||
models_yml = x.read()
|
||||
models_yml = models_yml.replace(" yes:".encode(), ' "yes":'.encode())
|
||||
models_yml = models_yml.replace(" no:".encode(), ' "no":'.encode())
|
||||
self.models = yaml.safe_load(models_yml)
|
||||
|
||||
self.errors: List[str] = []
|
||||
|
||||
def run_check(self) -> None:
|
||||
self.check_json()
|
||||
self.check_collections()
|
||||
for collection, models in self.data.items():
|
||||
for model in models:
|
||||
self.check_model(collection, model)
|
||||
if self.errors:
|
||||
errors = [f"\t{error}" for error in self.errors]
|
||||
raise CheckException("\n".join(errors))
|
||||
|
||||
def check_json(self) -> None:
|
||||
try:
|
||||
SCHEMA(self.data)
|
||||
except fastjsonschema.exceptions.JsonSchemaException as e:
|
||||
raise CheckException(f"JSON does not match schema: {str(e)}")
|
||||
|
||||
def check_collections(self) -> None:
|
||||
c1 = set(self.data.keys())
|
||||
c2 = set(self.models.keys())
|
||||
if c1 != c2:
|
||||
err = "Collections in JSON file do not match with models.yml."
|
||||
if c2 - c1:
|
||||
err += f" Missing collections: {', '.join(c2-c1)}."
|
||||
if c1 - c2:
|
||||
err += f" Invalid collections: {', '.join(c1-c2)}."
|
||||
raise CheckException(err)
|
||||
|
||||
def check_model(self, collection: str, model: Dict[str, Any]) -> None:
|
||||
model_fields = set(x for x in model.keys() if "$" not in x)
|
||||
collection_fields = set(
|
||||
x for x in self.models[collection].keys() if "$" not in x
|
||||
)
|
||||
|
||||
errors = False
|
||||
if collection_fields - model_fields:
|
||||
error = f"{collection}/{model['id']}: Missing fields {', '.join(collection_fields - model_fields)}"
|
||||
self.errors.append(error)
|
||||
errors = True
|
||||
if model_fields - collection_fields:
|
||||
error = f"{collection}/{model['id']}: Invalid fields {', '.join(model_fields - collection_fields)}"
|
||||
self.errors.append(error)
|
||||
errors = True
|
||||
|
||||
if not errors:
|
||||
self.check_types(model, collection)
|
||||
self.check_relations(model, collection)
|
||||
|
||||
def check_types(self, model: Dict[str, Any], collection: str) -> None:
|
||||
for field in model.keys():
|
||||
if "$" in field:
|
||||
continue
|
||||
|
||||
checker = None
|
||||
field_type = self.get_type_from_collection(field, collection)
|
||||
if field_type in (
|
||||
"string",
|
||||
"HTMLStrict",
|
||||
"HTMLPermissive",
|
||||
"generic-relation",
|
||||
):
|
||||
checker = check_string
|
||||
elif field_type in ("number", "timestamp", "relation"):
|
||||
checker = check_number
|
||||
elif field_type == "boolean":
|
||||
checker = check_boolean
|
||||
elif field_type in ("string[]", "generic-relation-list"):
|
||||
checker = check_string_list
|
||||
elif field_type in ("number[]", "relation-list"):
|
||||
checker = check_number_list
|
||||
elif field_type == "decimal(6)":
|
||||
checker = check_decimal
|
||||
elif field_type in (
|
||||
"JSON",
|
||||
"template",
|
||||
):
|
||||
pass
|
||||
else:
|
||||
raise NotImplementedError(f"TODO field type {field_type}")
|
||||
if checker is not None and not checker(model[field]):
|
||||
error = f"{collection}/{model['id']}/{field}: Type error: Type is not {field_type}"
|
||||
self.errors.append(error)
|
||||
|
||||
def get_type_from_collection(self, field: str, collection: str) -> str:
|
||||
field_value = self.models[collection][field]
|
||||
if isinstance(field_value, dict):
|
||||
return field_value["type"]
|
||||
return field_value
|
||||
|
||||
def check_relations(self, model: Dict[str, Any], collection: str) -> None:
|
||||
for field in model.keys():
|
||||
try:
|
||||
self.check_relation(model, collection, field)
|
||||
except CheckException as e:
|
||||
self.errors.append(
|
||||
f"{collection}/{model['id']}/{field} error: " + str(e)
|
||||
)
|
||||
|
||||
def check_relation(
|
||||
self, model: Dict[str, Any], collection: str, field: str
|
||||
) -> None:
|
||||
if "$" in field:
|
||||
return
|
||||
|
||||
field_type = self.get_type_from_collection(field, collection)
|
||||
basemsg = self.get_basemsg(collection, model["id"], field)
|
||||
|
||||
if field_type == "relation":
|
||||
foreign_id = model[field]
|
||||
if foreign_id is None:
|
||||
return
|
||||
|
||||
foreign_collection, foreign_field = self.get_to(field, collection)
|
||||
if "$" in foreign_field:
|
||||
return
|
||||
|
||||
self.check_reverse_relation(
|
||||
collection,
|
||||
model["id"],
|
||||
foreign_collection,
|
||||
foreign_id,
|
||||
foreign_field,
|
||||
basemsg,
|
||||
)
|
||||
|
||||
elif field_type == "relation-list":
|
||||
foreign_ids = model[field]
|
||||
if foreign_ids is None:
|
||||
return
|
||||
|
||||
foreign_collection, foreign_field = self.get_to(field, collection)
|
||||
if "$" in foreign_field:
|
||||
return
|
||||
|
||||
for foreign_id in foreign_ids:
|
||||
self.check_reverse_relation(
|
||||
collection,
|
||||
model["id"],
|
||||
foreign_collection,
|
||||
foreign_id,
|
||||
foreign_field,
|
||||
basemsg,
|
||||
)
|
||||
|
||||
elif field_type == "generic-relation" and model[field] is not None:
|
||||
foreign_collection, foreign_id = self.split_fqid(model[field])
|
||||
foreign_field = self.get_to_generic_case(
|
||||
collection, field, foreign_collection
|
||||
)
|
||||
|
||||
if "$" in foreign_field:
|
||||
return
|
||||
|
||||
self.check_reverse_relation(
|
||||
collection,
|
||||
model["id"],
|
||||
foreign_collection,
|
||||
foreign_id,
|
||||
foreign_field,
|
||||
basemsg,
|
||||
)
|
||||
|
||||
elif field_type == "generic-relation-list" and model[field] is not None:
|
||||
for fqid in model[field]:
|
||||
foreign_collection, foreign_id = self.split_fqid(fqid)
|
||||
foreign_field = self.get_to_generic_case(
|
||||
collection, field, foreign_collection
|
||||
)
|
||||
|
||||
if "$" in foreign_field:
|
||||
continue
|
||||
|
||||
self.check_reverse_relation(
|
||||
collection,
|
||||
model["id"],
|
||||
foreign_collection,
|
||||
foreign_id,
|
||||
foreign_field,
|
||||
basemsg,
|
||||
)
|
||||
|
||||
def get_to(self, field: str, collection: str) -> Tuple[str, str]:
|
||||
to = self.models[collection][field]["to"]
|
||||
return to.split("/")
|
||||
|
||||
def get_value(self, collection: str, id: int, field: str) -> Any:
|
||||
model = self.find_model(collection, id)
|
||||
if model is None:
|
||||
return None
|
||||
return model.get(field)
|
||||
|
||||
def find_model(self, collection: str, id: int) -> Optional[Dict[str, Any]]:
|
||||
c = self.data.get(collection, [])
|
||||
for model in c:
|
||||
if model["id"] == id:
|
||||
return model
|
||||
return None
|
||||
|
||||
def get_basemsg(self, collection: str, id: int, field: str) -> str:
|
||||
return f"{collection}/{id}/{field}: RelationError: "
|
||||
|
||||
def check_reverse_relation(
|
||||
self,
|
||||
collection: str,
|
||||
id: int,
|
||||
foreign_collection: str,
|
||||
foreign_id: int,
|
||||
foreign_field: str,
|
||||
basemsg: str,
|
||||
) -> None:
|
||||
foreign_value = self.get_value(foreign_collection, foreign_id, foreign_field)
|
||||
foreign_field_type = self.get_type_from_collection(
|
||||
foreign_field, foreign_collection
|
||||
)
|
||||
fqid = f"{collection}/{id}"
|
||||
error = False
|
||||
if foreign_field_type == "relation":
|
||||
error = foreign_value != id
|
||||
elif foreign_field_type == "relation-list":
|
||||
error = not foreign_value or id not in foreign_value
|
||||
elif foreign_field_type == "generic-relation":
|
||||
error = foreign_value != fqid
|
||||
elif foreign_field_type == "generic-relation-list":
|
||||
error = not foreign_value or fqid not in foreign_value
|
||||
else:
|
||||
raise NotImplementedError()
|
||||
|
||||
if error:
|
||||
self.errors.append(
|
||||
f"{basemsg} points to {foreign_collection}/{foreign_id}/{foreign_field},"
|
||||
" but the reverse relation for is corrupt"
|
||||
)
|
||||
|
||||
def split_fqid(self, fqid: str) -> Tuple[str, int]:
|
||||
try:
|
||||
collection, _id = fqid.split("/")
|
||||
id = int(_id)
|
||||
if collection not in self.models.keys():
|
||||
raise CheckException(f"Fqid {fqid} has an invalid collection")
|
||||
return collection, id
|
||||
except ValueError:
|
||||
raise CheckException(f"Fqid {fqid} is malformed")
|
||||
|
||||
def split_collectionfield(self, collectionfield: str) -> Tuple[str, str]:
|
||||
collection, field = collectionfield.split("/")
|
||||
if collection not in self.models.keys():
|
||||
raise CheckException(
|
||||
f"Collectionfield {collectionfield} has an invalid collection"
|
||||
)
|
||||
if (
|
||||
field not in self.models[collection]
|
||||
): # Note: this has to be adopted when supporting template fields
|
||||
raise CheckException(
|
||||
f"Collectionfield {collectionfield} has an invalid field"
|
||||
)
|
||||
return collection, field
|
||||
|
||||
def get_to_generic_case(
|
||||
self, collection: str, field: str, foreign_collection: str
|
||||
) -> str:
|
||||
""" Returns all reverse relations as collectionfields """
|
||||
to = self.models[collection][field]["to"]
|
||||
if isinstance(to, dict):
|
||||
if foreign_collection not in to["collections"]:
|
||||
raise CheckException(
|
||||
f"The collection {foreign_collection} is not supported "
|
||||
"as a reverse relation in {collection}/{field}"
|
||||
)
|
||||
return to["field"]
|
||||
|
||||
for cf in to:
|
||||
c, f = self.split_collectionfield(cf)
|
||||
if c == foreign_collection:
|
||||
return f
|
||||
|
||||
raise CheckException(
|
||||
f"The collection {foreign_collection} is not supported as a reverse relation in {collection}/{field}"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
failed = False
|
||||
for f in CHECKED_FILES:
|
||||
with open(f) as data:
|
||||
try:
|
||||
Checker(json.load(data)).run_check()
|
||||
except CheckException as e:
|
||||
print(f"Check for {f} failed:\n", e)
|
||||
failed = True
|
||||
else:
|
||||
print(f"Check for {f} successful.")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
2
docs/datavalidator/requirements.txt
Normal file
2
docs/datavalidator/requirements.txt
Normal file
@ -0,0 +1,2 @@
|
||||
fastjsonschema
|
||||
pyyaml
|
||||
6
docs/datavalidator/setup.cfg
Normal file
6
docs/datavalidator/setup.cfg
Normal file
@ -0,0 +1,6 @@
|
||||
[flake8]
|
||||
max-line-length = 120
|
||||
|
||||
[mypy]
|
||||
disallow_untyped_defs = true
|
||||
ignore_missing_imports = true
|
||||
@ -7,6 +7,10 @@
|
||||
"login_text": "Guten Morgen!",
|
||||
"theme": "openslides-theme",
|
||||
"custom_translations": [],
|
||||
"enable_electronic_voting": true,
|
||||
"reset_password_verbose_errors": true,
|
||||
"name": "Test Organization",
|
||||
"description": "",
|
||||
|
||||
"committee_ids": [1],
|
||||
"resource_ids": [1]
|
||||
@ -28,6 +32,7 @@
|
||||
"default_structure_level": "",
|
||||
"default_vote_weight": "1.000000",
|
||||
"last_email_send": null,
|
||||
"is_demo_user": false,
|
||||
|
||||
"organisation_management_level": "superadmin",
|
||||
|
||||
@ -88,6 +93,7 @@
|
||||
"default_structure_level": "",
|
||||
"default_vote_weight": "1.000000",
|
||||
"last_email_send": null,
|
||||
"is_demo_user": false,
|
||||
|
||||
"organisation_management_level": "",
|
||||
|
||||
@ -143,6 +149,7 @@
|
||||
"default_structure_level": "",
|
||||
"default_vote_weight": "1.000000",
|
||||
"last_email_send": null,
|
||||
"is_demo_user": false,
|
||||
|
||||
"organisation_management_level": "",
|
||||
|
||||
@ -230,13 +237,17 @@
|
||||
"conference_open_video": true,
|
||||
"conference_auto_connect_next_speakers": true,
|
||||
|
||||
"jitsi_room_name": "",
|
||||
"jitsi_domain": "",
|
||||
"jitsi_room_password": "",
|
||||
|
||||
"projector_default_countdown_time": 60,
|
||||
"projector_countdown_warning_time": 0,
|
||||
|
||||
"export_csv_encoding": "utf-8",
|
||||
"export_csv_separator": ",",
|
||||
"export_pdf_pagenumber_alignment": "center",
|
||||
"export_pdf_fontsize": "10",
|
||||
"export_pdf_fontsize": 10,
|
||||
"export_pdf_pagesize": "A4",
|
||||
|
||||
"agenda_show_subtitles": false,
|
||||
@ -333,7 +344,7 @@
|
||||
"poll_default_group_ids": [3],
|
||||
|
||||
"projector_ids": [1, 2],
|
||||
"projection_ids": [1, 2, 4, 6],
|
||||
"projection_ids": [1, 2, 4],
|
||||
"projectiondefault_ids": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
|
||||
"projector_message_ids": [1],
|
||||
"projector_countdown_ids": [1],
|
||||
@ -342,15 +353,18 @@
|
||||
"list_of_speakers_ids": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18],
|
||||
"speaker_ids": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13],
|
||||
"topic_ids": [1, 2, 3, 4, 5, 6, 7, 8],
|
||||
"group_ids": [1, 2, 3, 5, 6],
|
||||
"group_ids": [1, 2, 3, 4, 5],
|
||||
"mediafile_ids": [1, 2, 3],
|
||||
"motion_ids": [1, 2, 3, 4],
|
||||
"motion_submitter_ids": [1, 2, 3, 4],
|
||||
"motion_comment_section_ids": [1],
|
||||
"motion_comment_ids": [1],
|
||||
"motion_state_ids": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14],
|
||||
"motion_category_ids": [1, 2],
|
||||
"motion_block_ids": [1],
|
||||
"motion_workflow_ids": [1, 2],
|
||||
"motion_statute_paragraph_ids": [],
|
||||
"motion_change_recommendation_ids": [4, 5],
|
||||
"poll_ids": [1, 2, 3, 4, 5],
|
||||
"option_ids": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13],
|
||||
"vote_ids": [1, 2, 3, 4, 5, 6, 7, 8, 9],
|
||||
@ -392,6 +406,7 @@
|
||||
|
||||
"user_ids": [],
|
||||
"mediafile_access_group_ids": [],
|
||||
"mediafile_inherited_access_group_ids": [],
|
||||
"read_comment_section_ids": [],
|
||||
"write_comment_section_ids": [],
|
||||
"poll_ids": [],
|
||||
@ -458,7 +473,7 @@
|
||||
"meeting_id": 1
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"id": 4,
|
||||
"name": "Committees",
|
||||
"admin_group_for_meeting_id": null,
|
||||
"default_group_for_meeting_id": null,
|
||||
@ -477,16 +492,17 @@
|
||||
|
||||
"user_ids": [],
|
||||
"mediafile_access_group_ids": [],
|
||||
"mediafile_inherited_access_group_ids": [],
|
||||
"read_comment_section_ids": [],
|
||||
"write_comment_section_ids": [],
|
||||
"poll_ids": [],
|
||||
"used_as_motion_poll_default_id": null,
|
||||
"used_as_assignment_poll_default_id": 1,
|
||||
"used_as_assignment_poll_default_id": null,
|
||||
"used_as_poll_default_id": null,
|
||||
"meeting_id": 1
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"id": 5,
|
||||
"name": "Delegates",
|
||||
"admin_group_for_meeting_id": null,
|
||||
"default_group_for_meeting_id": null,
|
||||
@ -509,11 +525,12 @@
|
||||
|
||||
"user_ids": [2],
|
||||
"mediafile_access_group_ids": [],
|
||||
"mediafile_inherited_access_group_ids": [],
|
||||
"read_comment_section_ids": [1],
|
||||
"write_comment_section_ids": [1],
|
||||
"poll_ids": [],
|
||||
"used_as_motion_poll_default_id": null,
|
||||
"used_as_assignment_poll_default_id": null,
|
||||
"used_as_assignment_poll_default_id": 1,
|
||||
"used_as_poll_default_id": null,
|
||||
"meeting_id": 1
|
||||
}],
|
||||
@ -1198,6 +1215,9 @@
|
||||
"agenda_item_id": 3,
|
||||
"list_of_speakers_id": 3,
|
||||
"tag_ids": [],
|
||||
"option_ids": [],
|
||||
"projection_ids": [],
|
||||
"current_projector_ids": [],
|
||||
"meeting_id": 1
|
||||
},
|
||||
{
|
||||
@ -1209,6 +1229,9 @@
|
||||
"agenda_item_id": 4,
|
||||
"list_of_speakers_id": 4,
|
||||
"tag_ids": [],
|
||||
"option_ids": [],
|
||||
"projection_ids": [],
|
||||
"current_projector_ids": [],
|
||||
"meeting_id": 1
|
||||
},
|
||||
{
|
||||
@ -1220,6 +1243,9 @@
|
||||
"agenda_item_id": 5,
|
||||
"list_of_speakers_id": 5,
|
||||
"tag_ids": [],
|
||||
"option_ids": [],
|
||||
"projection_ids": [],
|
||||
"current_projector_ids": [],
|
||||
"meeting_id": 1
|
||||
},
|
||||
{
|
||||
@ -1231,6 +1257,9 @@
|
||||
"agenda_item_id": 6,
|
||||
"list_of_speakers_id": 6,
|
||||
"tag_ids": [],
|
||||
"option_ids": [],
|
||||
"projection_ids": [],
|
||||
"current_projector_ids": [],
|
||||
"meeting_id": 1
|
||||
},
|
||||
{
|
||||
@ -1242,6 +1271,9 @@
|
||||
"agenda_item_id": 7,
|
||||
"list_of_speakers_id": 7,
|
||||
"tag_ids": [],
|
||||
"option_ids": [],
|
||||
"projection_ids": [],
|
||||
"current_projector_ids": [],
|
||||
"meeting_id": 1
|
||||
},
|
||||
{
|
||||
@ -1253,6 +1285,9 @@
|
||||
"agenda_item_id": 8,
|
||||
"list_of_speakers_id": 8,
|
||||
"tag_ids": [],
|
||||
"option_ids": [],
|
||||
"projection_ids": [],
|
||||
"current_projector_ids": [],
|
||||
"meeting_id": 1
|
||||
},
|
||||
{
|
||||
@ -1264,6 +1299,9 @@
|
||||
"agenda_item_id": 9,
|
||||
"list_of_speakers_id": 9,
|
||||
"tag_ids": [],
|
||||
"option_ids": [],
|
||||
"projection_ids": [],
|
||||
"current_projector_ids": [],
|
||||
"meeting_id": 1
|
||||
},
|
||||
{
|
||||
@ -1275,12 +1313,16 @@
|
||||
"agenda_item_id": 10,
|
||||
"list_of_speakers_id": 10,
|
||||
"tag_ids": [],
|
||||
"option_ids": [],
|
||||
"projection_ids": [],
|
||||
"current_projector_ids": [],
|
||||
"meeting_id": 1
|
||||
}],
|
||||
"motion": [
|
||||
{
|
||||
"id": 1,
|
||||
"number": "A1",
|
||||
"number_value": 1,
|
||||
"sequential_number": 1,
|
||||
"title": "test",
|
||||
"text": "<ul>\n<li>Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque</li>\n<li>penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat ma</li>\n<li>ssa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, vene<br>\n<br>\nnatis vitae, justo. Null<br>\nam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi.Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim. Aliquam lorem ante, dapibu</li>\n<li>s in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius laoreet. Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sit amet adipiscing sem neque sed ipsum. Nam quam nunc, blandit vel, luctus pulvinar, hendrerit id, lorem.Maecenas nec odio et ante tincidunt tempus. Donec vitae sapien ut libero venenatis faucibus. Nullam quis ante. Etiam sit amet orci eget eros faucibus tincidunt. Duis leo. Sed fringilla mauris sit amet nibh. Donec sodales sagittis magna. Sed consequat, leo eget bibendum sodales, augue velit cursus nunc, quis gravida magna mi a libero. Fusce vulputate eleifend sapien. Vestibulum purus quam, scelerisque ut, mollis sed, nonummy id, metus. Nullam accumsan lorem in dui. Cras ultricies mi eu turpis hendrerit fringilla. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; In ac dui quis mi consectetuer lacinia. Nam pretium turpis et arcu. Duis arcu tortor, suscipit eget, imperdiet nec, imperdiet iaculis, ipsum. Sed aliquam ultrices mauris. Integer ante arcu, accumsan a, consectetuer eget, posuere ut, mauris. Praesent adipiscing. Phasellus ullamcorper ipsum rutrum nunc. Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut</li>\n</ul>",
|
||||
@ -1326,6 +1368,7 @@
|
||||
{
|
||||
"id": 2,
|
||||
"number": "1 - 1",
|
||||
"number_value": 1,
|
||||
"sequential_number": 2,
|
||||
"title": "Änderungsantrag zu 1",
|
||||
"text": "<p>lömk</p>",
|
||||
@ -1365,12 +1408,13 @@
|
||||
"attachment_ids": [],
|
||||
"projection_ids": [],
|
||||
"current_projector_ids": [],
|
||||
"personal_note_ids": [],
|
||||
"personal_note_ids": [1],
|
||||
"meeting_id": 1
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"number": "2",
|
||||
"number_value": 2,
|
||||
"sequential_number": 3,
|
||||
"title": "ohne",
|
||||
"text": "<p>sf</p>",
|
||||
@ -1416,6 +1460,7 @@
|
||||
{
|
||||
"id": 4,
|
||||
"number": "3",
|
||||
"number_value": 3,
|
||||
"sequential_number": 4,
|
||||
"title": "komplex",
|
||||
"text": "<p>sdf sdfpdfkw wef</p>\n\n<p>wepkf </p>\n\n<p>weüpfk </p>\n\n<p>weüpfdfg</p>",
|
||||
@ -1448,7 +1493,7 @@
|
||||
"option_ids": [],
|
||||
"change_recommendation_ids": [4],
|
||||
"statute_paragraph_id": null,
|
||||
"comments": [],
|
||||
"comment_ids": [],
|
||||
"agenda_item_id": 13,
|
||||
"list_of_speakers_id": 13,
|
||||
"tag_ids": [],
|
||||
@ -1507,8 +1552,8 @@
|
||||
"weight": 10000,
|
||||
|
||||
"comment_ids": [1],
|
||||
"read_group_ids": [3, 6],
|
||||
"write_group_ids": [3, 6],
|
||||
"read_group_ids": [3, 5],
|
||||
"write_group_ids": [3, 5],
|
||||
"meeting_id": 1
|
||||
}],
|
||||
"motion_category": [
|
||||
@ -1921,7 +1966,6 @@
|
||||
"state": "finished",
|
||||
"min_votes_amount": 1,
|
||||
"max_votes_amount": 1,
|
||||
"allow_multiple_votes_per_candidate": false,
|
||||
"global_yes": false,
|
||||
"global_no": false,
|
||||
"global_abstain": false,
|
||||
@ -1930,7 +1974,6 @@
|
||||
"votesvalid": "2.000000",
|
||||
"votesinvalid": "9.000000",
|
||||
"votescast": "2.000000",
|
||||
"user_has_voted": false,
|
||||
|
||||
"content_object_id": "motion/1",
|
||||
"option_ids": [1],
|
||||
@ -1950,7 +1993,6 @@
|
||||
"state": "created",
|
||||
"min_votes_amount": 1,
|
||||
"max_votes_amount": 1,
|
||||
"allow_multiple_votes_per_candidate": false,
|
||||
"global_yes": false,
|
||||
"global_no": false,
|
||||
"global_abstain": false,
|
||||
@ -1959,7 +2001,6 @@
|
||||
"votesvalid": null,
|
||||
"votesinvalid": null,
|
||||
"votescast": null,
|
||||
"user_has_voted": false,
|
||||
|
||||
"content_object_id": "motion/1",
|
||||
"option_ids": [3],
|
||||
@ -1979,7 +2020,6 @@
|
||||
"state": "created",
|
||||
"min_votes_amount": 1,
|
||||
"max_votes_amount": 1,
|
||||
"allow_multiple_votes_per_candidate": false,
|
||||
"global_yes": false,
|
||||
"global_no": true,
|
||||
"global_abstain": true,
|
||||
@ -1988,7 +2028,6 @@
|
||||
"votesvalid": null,
|
||||
"votesinvalid": null,
|
||||
"votescast": null,
|
||||
"user_has_voted": false,
|
||||
|
||||
"content_object_id": "assignment/1",
|
||||
"voted_ids": [],
|
||||
@ -2008,7 +2047,6 @@
|
||||
"state": "finished",
|
||||
"min_votes_amount": 1,
|
||||
"max_votes_amount": 1,
|
||||
"allow_multiple_votes_per_candidate": false,
|
||||
"global_yes": false,
|
||||
"global_no": true,
|
||||
"global_abstain": true,
|
||||
@ -2017,7 +2055,6 @@
|
||||
"votesvalid": "9.000000",
|
||||
"votesinvalid": "2.000000",
|
||||
"votescast": "16.000000",
|
||||
"user_has_voted": false,
|
||||
|
||||
"content_object_id": "assignment/1",
|
||||
"voted_ids": [],
|
||||
@ -2037,7 +2074,6 @@
|
||||
"state": "finished",
|
||||
"min_votes_amount": 1,
|
||||
"max_votes_amount": 1,
|
||||
"allow_multiple_votes_per_candidate": false,
|
||||
"global_yes": false,
|
||||
"global_no": true,
|
||||
"global_abstain": false,
|
||||
@ -2046,7 +2082,6 @@
|
||||
"votesvalid": "1.000000",
|
||||
"votesinvalid": "0.000000",
|
||||
"votescast": "1.000000",
|
||||
"user_has_voted": true,
|
||||
|
||||
"content_object_id": "assignment/2",
|
||||
"voted_ids": [1],
|
||||
@ -2064,6 +2099,7 @@
|
||||
"no": "4.000000",
|
||||
"abstain": "1.000000",
|
||||
"weight": 1,
|
||||
"text": null,
|
||||
|
||||
"poll_id": 1,
|
||||
"used_as_global_option_in_poll_id": null,
|
||||
@ -2077,10 +2113,12 @@
|
||||
"no": "0.000000",
|
||||
"abstain": "0.000000",
|
||||
"weight": 1,
|
||||
"text": null,
|
||||
|
||||
"poll_id": null,
|
||||
"used_as_global_option_in_poll_id": 1,
|
||||
"vote_ids": [0],
|
||||
"content_object_id": null,
|
||||
"vote_ids": [],
|
||||
"meeting_id": 1
|
||||
},
|
||||
{
|
||||
@ -2089,6 +2127,7 @@
|
||||
"no": "0.000000",
|
||||
"abstain": "0.000000",
|
||||
"weight": 1,
|
||||
"text": null,
|
||||
|
||||
"poll_id": 2,
|
||||
"used_as_global_option_in_poll_id": null,
|
||||
@ -2102,9 +2141,11 @@
|
||||
"no": "0.000000",
|
||||
"abstain": "0.000000",
|
||||
"weight": 1,
|
||||
"text": null,
|
||||
|
||||
"poll_id": null,
|
||||
"used_as_global_option_in_poll_id": 2,
|
||||
"content_object_id": null,
|
||||
"vote_ids": [],
|
||||
"meeting_id": 1
|
||||
},
|
||||
@ -2114,6 +2155,7 @@
|
||||
"no": "0.000000",
|
||||
"abstain": "0.000000",
|
||||
"weight": 1,
|
||||
"text": null,
|
||||
|
||||
"poll_id": 3,
|
||||
"used_as_global_option_in_poll_id": null,
|
||||
@ -2127,9 +2169,11 @@
|
||||
"no": "0.000000",
|
||||
"abstain": "0.000000",
|
||||
"weight": 1,
|
||||
"text": null,
|
||||
|
||||
"poll_id": null,
|
||||
"used_as_global_option_in_poll_id": 3,
|
||||
"content_object_id": null,
|
||||
"vote_ids": [],
|
||||
"meeting_id": 1
|
||||
},
|
||||
@ -2139,6 +2183,7 @@
|
||||
"no": "0.000000",
|
||||
"abstain": "0.000000",
|
||||
"weight": 1,
|
||||
"text": null,
|
||||
|
||||
"poll_id": 4,
|
||||
"used_as_global_option_in_poll_id": null,
|
||||
@ -2152,6 +2197,7 @@
|
||||
"no": "0.000000",
|
||||
"abstain": "0.000000",
|
||||
"weight": 2,
|
||||
"text": null,
|
||||
|
||||
"poll_id": 4,
|
||||
"used_as_global_option_in_poll_id": null,
|
||||
@ -2165,6 +2211,7 @@
|
||||
"no": "0.000000",
|
||||
"abstain": "0.000000",
|
||||
"weight": 3,
|
||||
"text": null,
|
||||
|
||||
"poll_id": 4,
|
||||
"used_as_global_option_in_poll_id": null,
|
||||
@ -2178,9 +2225,11 @@
|
||||
"no": "2.000000",
|
||||
"abstain": "1.000000",
|
||||
"weight": 1,
|
||||
"text": null,
|
||||
|
||||
"poll_id": null,
|
||||
"used_as_global_option_in_poll_id": 4,
|
||||
"content_object_id": null,
|
||||
"vote_ids": [7, 8],
|
||||
"meeting_id": 1
|
||||
},
|
||||
@ -2190,6 +2239,7 @@
|
||||
"no": "0.000000",
|
||||
"abstain": "0.000000",
|
||||
"weight": 1,
|
||||
"text": null,
|
||||
|
||||
"poll_id": 5,
|
||||
"used_as_global_option_in_poll_id": null,
|
||||
@ -2203,6 +2253,7 @@
|
||||
"no": "0.000000",
|
||||
"abstain": "0.000000",
|
||||
"weight": 2,
|
||||
"text": null,
|
||||
|
||||
"poll_id": 5,
|
||||
"used_as_global_option_in_poll_id": null,
|
||||
@ -2216,9 +2267,11 @@
|
||||
"no": "0.000000",
|
||||
"abstain": "0.000000",
|
||||
"weight": 1,
|
||||
"text": null,
|
||||
|
||||
"poll_id": null,
|
||||
"used_as_global_option_in_poll_id": 5,
|
||||
"content_object_id": null,
|
||||
"vote_ids": [],
|
||||
"meeting_id": 1
|
||||
}],
|
||||
@ -2405,7 +2458,7 @@
|
||||
"mimetype": null,
|
||||
"pdf_information": {},
|
||||
"create_timestamp": 1584513763,
|
||||
"has_inherited_access_groups": true,
|
||||
"is_public": false,
|
||||
|
||||
"access_group_ids": [2, 3],
|
||||
"inherited_access_group_ids": [2, 3],
|
||||
@ -2428,7 +2481,7 @@
|
||||
"mimetype": "text/plain",
|
||||
"pdf_information": {},
|
||||
"create_timestamp": 1584513771,
|
||||
"has_inherited_access_groups": false,
|
||||
"is_public": true,
|
||||
|
||||
"access_group_ids": [],
|
||||
"inherited_access_group_ids": [],
|
||||
@ -2451,7 +2504,7 @@
|
||||
"mimetype": "image/png",
|
||||
"pdf_information": {},
|
||||
"create_timestamp": 1584513791,
|
||||
"has_inherited_access_groups": true,
|
||||
"is_public": false,
|
||||
|
||||
"access_group_ids": [],
|
||||
"inherited_access_group_ids": [2, 3],
|
||||
@ -2513,8 +2566,8 @@
|
||||
"show_title": true,
|
||||
"show_logo": true,
|
||||
|
||||
"current_projection_ids": [6],
|
||||
"current_element_ids": ["clock/1"],
|
||||
"current_projection_ids": [],
|
||||
"current_element_ids": [],
|
||||
"preview_projection_ids": [],
|
||||
"history_projection_ids": [],
|
||||
"used_as_reference_projector_meeting_id": 1,
|
||||
@ -2535,7 +2588,7 @@
|
||||
"id": 2,
|
||||
"current_projector_id": null,
|
||||
"preview_projector_id": 1,
|
||||
"history_projector_history_id": null,
|
||||
"history_projector_id": null,
|
||||
"element_id": "motion/4",
|
||||
"options": {
|
||||
"mode": "diff"
|
||||
@ -2550,17 +2603,6 @@
|
||||
"element_id": "assignment/1",
|
||||
"options": {},
|
||||
"meeting_id": 1
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"current_projector_id": 1,
|
||||
"preview_projector_id": null,
|
||||
"history_projector_id": null,
|
||||
"element_id": "clock/1",
|
||||
"options": {
|
||||
"stable": true
|
||||
},
|
||||
"meeting_id": 1
|
||||
}],
|
||||
"projectiondefault": [
|
||||
{
|
||||
@ -2698,7 +2740,7 @@
|
||||
"title": "Countdown 1",
|
||||
"description": "",
|
||||
"default_time": 60,
|
||||
"countdown_time": 60.0,
|
||||
"countdown_time": 60,
|
||||
"running": false,
|
||||
|
||||
"projection_ids": [],
|
||||
|
||||
@ -1,100 +0,0 @@
|
||||
# This script requires fastjsonschema and pyyaml to be installed e. g. via pip.
|
||||
|
||||
import json
|
||||
import sys
|
||||
from typing import Any, Dict, Iterable
|
||||
|
||||
import fastjsonschema # type:ignore
|
||||
import yaml
|
||||
|
||||
MODELS_YML_PATH = "../../docs/models.yml"
|
||||
|
||||
CHECKED_FILES = [
|
||||
"../../docker/initial-data.json",
|
||||
"../../docs/example-data.json",
|
||||
]
|
||||
|
||||
SCHEMA = fastjsonschema.compile(
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Schema for initial and example data.",
|
||||
"type": "object",
|
||||
"patternProperties": {
|
||||
"^[a-z_]+$": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {"id": {"type": "number"}},
|
||||
"required": ["id"],
|
||||
},
|
||||
}
|
||||
},
|
||||
"additionalProperties": False,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class CheckException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def run_check(data: Dict) -> None:
|
||||
try:
|
||||
SCHEMA(data)
|
||||
except fastjsonschema.exceptions.JsonSchemaException as e:
|
||||
raise CheckException(f"JSON does not match schema: {str(e)}")
|
||||
check_collections(data.keys())
|
||||
for collection, elements in data.items():
|
||||
for element in elements:
|
||||
check_instance(collection, element)
|
||||
|
||||
|
||||
def get_models() -> Dict[str, Any]:
|
||||
with open(MODELS_YML_PATH, "rb") as x:
|
||||
models_yml = x.read()
|
||||
models_yml = models_yml.replace(" yes:".encode(), ' "yes":'.encode())
|
||||
models_yml = models_yml.replace(" no:".encode(), ' "no":'.encode())
|
||||
return yaml.safe_load(models_yml)
|
||||
|
||||
|
||||
def check_collections(collections: Iterable[str]) -> None:
|
||||
c1 = set(collections)
|
||||
c2 = set(get_models().keys())
|
||||
if c1 != c2:
|
||||
err = "Collections in JSON file do not match with models.yml."
|
||||
if c2 - c1:
|
||||
err += f" Missing collections: {', '.join(c2-c1)}."
|
||||
if c1 - c2:
|
||||
err += f" Invalid collections: {', '.join(c1-c2)}."
|
||||
raise CheckException(err)
|
||||
|
||||
|
||||
def check_instance(name: str, instance: Dict[str, Any]) -> None:
|
||||
collection = get_models()[name]
|
||||
for field_name in instance.keys():
|
||||
if "$" in field_name and not ("$_" in field_name or field_name[-1] == "$"):
|
||||
# Structured field.
|
||||
# TODO: Check this.
|
||||
continue
|
||||
if field_name not in collection.keys():
|
||||
raise CheckException(f"Bad field in {name}: {field_name}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
failed = False
|
||||
for f in CHECKED_FILES:
|
||||
with open(f) as data:
|
||||
try:
|
||||
run_check(json.load(data))
|
||||
except CheckException as e:
|
||||
print(f"Check for {f} failed:", e)
|
||||
failed = True
|
||||
else:
|
||||
print(f"Check for {f} successful.")
|
||||
if failed:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Reference in New Issue
Block a user