From 27d31189c19f31136a19f0207888b0c6b9cae339 Mon Sep 17 00:00:00 2001 From: Finn Stutzenstein Date: Wed, 4 Aug 2021 12:44:39 +0200 Subject: [PATCH] Enhance CI - redone the modelsvalidator - use the datavalidator from the backend, remove old code --- .github/workflows/models.yml | 57 +- docs/datavalidator/check_json.py | 668 ------------------ docs/datavalidator/export.json | 1 - docs/datavalidator/requirements.txt | 2 - docs/models.yml | 21 +- docs/modelsvalidator/README.md | 21 - docs/modelsvalidator/check/check.go | 159 ----- docs/modelsvalidator/check/check_test.go | 121 ---- docs/modelsvalidator/check/error.go | 38 - docs/modelsvalidator/go.mod | 5 - docs/modelsvalidator/go.sum | 6 - docs/modelsvalidator/main.go | 56 -- docs/modelsvalidator/requirements.txt | 1 + .../setup.cfg | 0 docs/modelsvalidator/validate.py | 375 ++++++++++ openslides-autoupdate-service | 2 +- openslides-backend | 2 +- openslides-client | 2 +- 18 files changed, 421 insertions(+), 1116 deletions(-) delete mode 100644 docs/datavalidator/check_json.py delete mode 100644 docs/datavalidator/export.json delete mode 100644 docs/datavalidator/requirements.txt delete mode 100644 docs/modelsvalidator/README.md delete mode 100644 docs/modelsvalidator/check/check.go delete mode 100644 docs/modelsvalidator/check/check_test.go delete mode 100644 docs/modelsvalidator/check/error.go delete mode 100644 docs/modelsvalidator/go.mod delete mode 100644 docs/modelsvalidator/go.sum delete mode 100644 docs/modelsvalidator/main.go create mode 100644 docs/modelsvalidator/requirements.txt rename docs/{datavalidator => modelsvalidator}/setup.cfg (100%) create mode 100644 docs/modelsvalidator/validate.py diff --git a/.github/workflows/models.yml b/.github/workflows/models.yml index 41c3e2c25..f04151749 100644 --- a/.github/workflows/models.yml +++ b/.github/workflows/models.yml @@ -2,34 +2,12 @@ name: Validate models.yml and initial and example data on: [push, pull_request] env: - PYTHON_VERSION: 3.8.5 - GO_VERSION: 1.16 + PYTHON_VERSION: 3.9.6 jobs: validate-models: name: Validate models.yml runs-on: ubuntu-latest steps: - - name: Set up Go - uses: actions/setup-go@v1 - with: - go-version: ${{ env.GO_VERSION }} - - - name: Check out code - uses: actions/checkout@v2 - - - name: Build validator - run: go build - working-directory: docs/modelsvalidator - env: - GO111MODULE: on - - - name: Validate models.yml - run: docs/modelsvalidator/modelsvalidator docs/models.yml - - validate-data: - name: Validate example-data.json and initial-data.json - runs-on: ubuntu-latest - steps: - name: Check out code uses: actions/checkout@v2 @@ -39,9 +17,32 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Install requirements - working-directory: docs/datavalidator - run: pip install -U -r requirements.txt + run: pip install -U -r docs/modelsvalidator/requirements.txt - - name: Validate - working-directory: docs/datavalidator - run: python check_json.py + - name: Validate models.yml + working-directory: docs/modelsvalidator + run: python validate.py + + validate-data: + name: Validate example-data.json and initial-data.json + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Checkout backend submodule + run: git submodule update --init openslides-backend/ + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install requirements + run: pip install -U -r openslides-backend/cli/requirements.txt + + - name: set pythonpath + run: echo "PYTHONPATH=openslides-backend" >> $GITHUB_ENV + + - name: Validate example-data.json + run: python openslides-backend/cli/check_json.py docs/example-data.json docker/initial-data.json diff --git a/docs/datavalidator/check_json.py b/docs/datavalidator/check_json.py deleted file mode 100644 index b02d80f55..000000000 --- a/docs/datavalidator/check_json.py +++ /dev/null @@ -1,668 +0,0 @@ -import json -import re -import sys -from collections import defaultdict -from typing import Any, Callable, Dict, List, Optional, Set, Tuple - -import fastjsonschema -import yaml - -MODELS_YML_PATH = "../../docs/models.yml" - -DEFAULT_FILES = [ - "../../docker/initial-data.json", - "../../docs/example-data.json", -] - -SCHEMA = fastjsonschema.compile( - { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Schema for initial and example data.", - "type": "object", - "patternProperties": { - "^[a-z_]+$": { - "type": "array", - "items": { - "type": "object", - "properties": {"id": {"type": "number"}}, - "required": ["id"], - }, - } - }, - "additionalProperties": False, - } -) - - -class CheckException(Exception): - pass - - -def check_string(value: Any) -> bool: - return value is None or isinstance(value, str) - - -color_regex = re.compile("^#[0-9a-f]{6}$") - - -def check_color(value: Any) -> bool: - return value is None or bool(isinstance(value, str) and color_regex.match(value)) - - -def check_number(value: Any) -> bool: - return value is None or type(value) == int - - -def check_float(value: Any) -> bool: - return value is None or type(value) in (int, float) - - -def check_boolean(value: Any) -> bool: - return value is None or value is False or value is True - - -def check_string_list(value: Any) -> bool: - return check_x_list(value, check_string) - - -def check_number_list(value: Any) -> bool: - return check_x_list(value, check_number) - - -def check_x_list(value: Any, fn: Callable) -> bool: - if value is None: - return True - if not isinstance(value, list): - return False - return all([fn(sv) for sv in value]) - - -def check_decimal(value: Any) -> bool: - if value is None: - return True - if isinstance(value, str): - pattern = r"^-?(\d|[1-9]\d+)\.\d{6}$" - if re.match(pattern, value): - return True - return False - - -def check_json(value: Any, root: bool = True) -> bool: - if value is None: - return True - if not root and (isinstance(value, int) or isinstance(value, str)): - return True - if isinstance(value, list): - return all(check_json(x, root=False) for x in value) - if isinstance(value, dict): - return all(check_json(x, root=False) for x in value.values()) - return False - - -checker_mapping = { - "string": check_string, - "HTMLStrict": check_string, - "HTMLPermissive": check_string, - "generic-relation": check_string, - "number": check_number, - "timestamp": check_number, - "relation": check_number, - "float": check_float, - "boolean": check_boolean, - "string[]": check_string_list, - "generic-relation-list": check_string_list, - "number[]": check_number_list, - "relation-list": check_number_list, - "decimal(6)": check_decimal, - "color": check_color, - "JSON": check_json, -} - - -class Checker: - def __init__(self, data: Dict[str, List[Any]], is_import: bool = False) -> None: - self.data = data - - with open(MODELS_YML_PATH, "rb") as x: - models_yml = x.read() - models_yml = models_yml.replace(" yes:".encode(), ' "yes":'.encode()) - models_yml = models_yml.replace(" no:".encode(), ' "no":'.encode()) - self.models = yaml.safe_load(models_yml) - if is_import: - self.modify_models_for_import() - - self.errors: List[str] = [] - - self.template_prefixes: Dict[ - str, Dict[str, Tuple[str, int, int]] - ] = defaultdict(dict) - self.generate_template_prefixes() - - def modify_models_for_import(self) -> None: - collection_allowlist = ( - "user", - "meeting", - "group", - "personal_note", - "tag", - "agenda_item", - "list_of_speakers", - "speaker", - "topic", - "motion", - "motion_submitter", - "motion_comment", - "motion_comment_section", - "motion_category", - "motion_block", - "motion_change_recommendation", - "motion_state", - "motion_workflow", - "motion_statute_paragraph", - "poll", - "option", - "vote", - "assignment", - "assignment_candidate", - "mediafile", - "projector", - "projection", - "projector_message", - "projector_countdown", - "chat_group", - ) - for collection in list(self.models.keys()): - if collection not in collection_allowlist: - del self.models[collection] - self.models["mediafile"]["blob"] = "string" - - def generate_template_prefixes(self) -> None: - for collection in self.models.keys(): - for field in self.models[collection]: - if not self.is_template_field(field): - continue - parts = field.split("$") - prefix = parts[0] - suffix = parts[1] - if prefix in self.template_prefixes[collection]: - raise ValueError( - f"the template prefix {prefix} is not unique within {collection}" - ) - self.template_prefixes[collection][prefix] = ( - field, - len(prefix), - len(suffix), - ) - - def is_template_field(self, field: str) -> bool: - return "$_" in field or field.endswith("$") - - def is_structured_field(self, field: str) -> bool: - return "$" in field and not self.is_template_field(field) - - def is_normal_field(self, field: str) -> bool: - return "$" not in field - - def is_calculated_field(self, field: str) -> bool: - return field == "content" - - def make_structured(self, field: str, replacement: Any) -> str: - if type(replacement) not in (str, int): - raise CheckException( - f"Invalid type {type(replacement)} for the replacement of field {field}" - ) - parts = field.split("$") - return parts[0] + "$" + str(replacement) + parts[1] - - def to_template_field( - self, collection: str, structured_field: str - ) -> Tuple[str, str]: - """Returns template_field, replacement""" - parts = structured_field.split("$") - descriptor = self.template_prefixes[collection].get(parts[0]) - if not descriptor: - raise CheckException( - f"Unknown template field for prefix {parts[0]} in collection {collection}" - ) - return ( - descriptor[0], - structured_field[descriptor[1] + 1 : len(structured_field) - descriptor[2]], - ) - - def run_check(self) -> None: - self.check_json() - self.check_collections() - for collection, models in self.data.items(): - for model in models: - self.check_model(collection, model) - if self.errors: - errors = [f"\t{error}" for error in self.errors] - raise CheckException("\n".join(errors)) - - def check_json(self) -> None: - try: - SCHEMA(self.data) - except fastjsonschema.exceptions.JsonSchemaException as e: - raise CheckException(f"JSON does not match schema: {str(e)}") - - def check_collections(self) -> None: - c1 = set(self.data.keys()) - c2 = set(self.models.keys()) - if c1 != c2: - err = "Collections in JSON file do not match with models.yml." - if c2 - c1: - err += f" Missing collections: {', '.join(c2-c1)}." - if c1 - c2: - err += f" Invalid collections: {', '.join(c1-c2)}." - raise CheckException(err) - - def check_model(self, collection: str, model: Dict[str, Any]) -> None: - errors = self.check_normal_fields(model, collection) - - if not errors: - errors = self.check_template_fields(model, collection) - - if not errors: - self.check_types(model, collection) - self.check_relations(model, collection) - - def check_normal_fields(self, model: Dict[str, Any], collection: str) -> bool: - model_fields = set( - x - for x in model.keys() - if self.is_normal_field(x) or self.is_template_field(x) - ) - collection_fields = set( - x - for x in self.models[collection].keys() - if self.is_normal_field(x) or self.is_template_field(x) - ) - calculated_fields = set( - x - for x in self.models[collection].keys() - if self.is_calculated_field(x) - ) - - errors = False - if collection_fields - model_fields - calculated_fields: - error = f"{collection}/{model['id']}: Missing fields {', '.join(collection_fields - model_fields)}" - self.errors.append(error) - errors = True - if model_fields - collection_fields: - error = f"{collection}/{model['id']}: Invalid fields {', '.join(model_fields - collection_fields)}" - self.errors.append(error) - errors = True - return errors - - def check_template_fields(self, model: Dict[str, Any], collection: str) -> bool: - """ - Only checks that for each replacement a structured field exists and - not too many structured fields. Does not check the content. - Returns True on errors. - """ - errors = False - for template_field in self.models[collection].keys(): - if not self.is_template_field(template_field): - continue - field_error = False - replacements = model[template_field] - if not isinstance(replacements, list): - self.errors.append( - f"{collection}/{model['id']}/{template_field}: Replacements for the template field must be a list" - ) - field_error = True - continue - for replacement in replacements: - if not isinstance(replacement, str): - self.errors.append( - f"{collection}/{model['id']}/{template_field}: Each replacement for the template field must be a string" - ) - field_error = True - if field_error: - error = True - continue - - replacement_collection = None - field_description = self.models[collection][template_field] - if isinstance(field_description, dict): - replacement_collection = field_description.get("replacement_collection") - - for replacement in replacements: - structured_field = self.make_structured(template_field, replacement) - if structured_field not in model: - self.errors.append( - f"{collection}/{model['id']}/{template_field}: Missing {structured_field} since it is given as a replacement" - ) - errors = True - - if replacement_collection: - try: - as_id = int(replacement) - except (TypeError, ValueError): - self.errors.append( - f"{collection}/{model['id']}/{template_field}: Replacement {replacement} is not an integer" - ) - if not self.find_model(replacement_collection, as_id): - self.errors.append( - f"{collection}/{model['id']}/{template_field}: Replacement {replacement} does not exist as a model of collection {replacement_collection}" - ) - - for field in model.keys(): - if self.is_structured_field(field): - try: - _template_field, _replacement = self.to_template_field( - collection, field - ) - if ( - template_field == _template_field - and _replacement not in model[template_field] - ): - self.errors.append( - f"{collection}/{model['id']}/{field}: Invalid structured field. Missing replacement {_replacement} in {template_field}" - ) - errors = True - except CheckException as e: - self.errors.append( - f"{collection}/{model['id']}/{field} error: " + str(e) - ) - errors = True - - return errors - - def check_types(self, model: Dict[str, Any], collection: str) -> None: - for field in model.keys(): - if self.is_template_field(field): - continue - - field_type = self.get_type_from_collection(field, collection) - enum = self.get_enum_from_collection_field(field, collection) - checker = checker_mapping.get(field_type) - if checker is None: - raise NotImplementedError( - f"TODO implement check for field type {field_type}" - ) - - if not checker(model[field]): - error = f"{collection}/{model['id']}/{field}: Type error: Type is not {field_type}" - self.errors.append(error) - - if enum and model[field] not in enum: - error = f"{collection}/{model['id']}/{field}: Value error: Value {model[field]} is not a valid enum value" - self.errors.append(error) - - def get_type_from_collection(self, field: str, collection: str) -> str: - if self.is_structured_field(field): - field, _ = self.to_template_field(collection, field) - - field_description = self.models[collection][field] - if isinstance(field_description, dict): - if field_description["type"] == "template": - if isinstance(field_description["fields"], dict): - return field_description["fields"]["type"] - return field_description["fields"] - return field_description["type"] - return field_description - - field_description = self.get_field_description(field, collection) - if field_description: - return field_description["type"] - return self.models[collection][field] - - def get_enum_from_collection_field( - self, field: str, collection: str - ) -> Optional[Set[str]]: - if self.is_structured_field(field): - field, _ = self.to_template_field(collection, field) - - field_description = self.models[collection][field] - if isinstance(field_description, dict): - if field_description["type"] == "template": - if isinstance(field_description["fields"], dict): - field_description = field_description["fields"] - else: - return None - if "enum" in field_description: - enum = set(field_description["enum"]) - if not field_description.get("required", False): - enum.add(None) - return enum - return None - - def check_relations(self, model: Dict[str, Any], collection: str) -> None: - for field in model.keys(): - try: - self.check_relation(model, collection, field) - except CheckException as e: - self.errors.append( - f"{collection}/{model['id']}/{field} error: " + str(e) - ) - - def check_relation( - self, model: Dict[str, Any], collection: str, field: str - ) -> None: - if self.is_template_field(field): - return - - field_type = self.get_type_from_collection(field, collection) - basemsg = f"{collection}/{model['id']}/{field}: Relation Error: " - - replacement = None - if self.is_structured_field(field): - _, replacement = self.to_template_field(collection, field) - - if field_type == "relation": - foreign_id = model[field] - if foreign_id is None: - return - - foreign_collection, foreign_field = self.get_to(field, collection) - - self.check_reverse_relation( - collection, - model["id"], - model, - foreign_collection, - foreign_id, - foreign_field, - basemsg, - replacement, - ) - - elif field_type == "relation-list": - foreign_ids = model[field] - if foreign_ids is None: - return - - foreign_collection, foreign_field = self.get_to(field, collection) - - for foreign_id in foreign_ids: - self.check_reverse_relation( - collection, - model["id"], - model, - foreign_collection, - foreign_id, - foreign_field, - basemsg, - replacement, - ) - - elif field_type == "generic-relation" and model[field] is not None: - foreign_collection, foreign_id = self.split_fqid(model[field]) - foreign_field = self.get_to_generic_case( - collection, field, foreign_collection - ) - - self.check_reverse_relation( - collection, - model["id"], - model, - foreign_collection, - foreign_id, - foreign_field, - basemsg, - replacement, - ) - - elif field_type == "generic-relation-list" and model[field] is not None: - for fqid in model[field]: - foreign_collection, foreign_id = self.split_fqid(fqid) - foreign_field = self.get_to_generic_case( - collection, field, foreign_collection - ) - - self.check_reverse_relation( - collection, - model["id"], - model, - foreign_collection, - foreign_id, - foreign_field, - basemsg, - replacement, - ) - - def get_to(self, field: str, collection: str) -> Tuple[str, str]: - if self.is_structured_field(field): - field, _ = self.to_template_field(collection, field) - - field_description = self.models[collection][field] - if field_description["type"] == "template": - to = field_description["fields"]["to"] - else: - to = field_description["to"] - return to.split("/") - - def find_model(self, collection: str, id: int) -> Optional[Dict[str, Any]]: - c = self.data.get(collection, []) - for model in c: - if model["id"] == id: - return model - return None - - def check_reverse_relation( - self, - collection: str, - id: int, - model: Dict[str, Any], - foreign_collection: str, - foreign_id: int, - foreign_field: str, - basemsg: str, - replacement: Optional[str], - ) -> None: - actual_foreign_field = foreign_field - if self.is_template_field(foreign_field): - if replacement: - actual_foreign_field = self.make_structured(foreign_field, replacement) - else: - replacement_collection = self.models[foreign_collection][foreign_field][ - "replacement_collection" - ] - replacement = model.get(f"{replacement_collection}_id") - if not replacement: - self.errors.append( - f"{basemsg} points to {foreign_collection}/{foreign_id}/{foreign_field}," - f" but there is no replacement for {replacement_collection}" - ) - actual_foreign_field = self.make_structured(foreign_field, replacement) - - foreign_model = self.find_model(foreign_collection, foreign_id) - foreign_value = ( - foreign_model.get(actual_foreign_field) - if foreign_model is not None - else None - ) - foreign_field_type = self.get_type_from_collection( - foreign_field, foreign_collection - ) - fqid = f"{collection}/{id}" - error = False - if foreign_field_type == "relation": - error = foreign_value != id - elif foreign_field_type == "relation-list": - error = not foreign_value or id not in foreign_value - elif foreign_field_type == "generic-relation": - error = foreign_value != fqid - elif foreign_field_type == "generic-relation-list": - error = not foreign_value or fqid not in foreign_value - else: - raise NotImplementedError() - - if error: - self.errors.append( - f"{basemsg} points to {foreign_collection}/{foreign_id}/{actual_foreign_field}," - " but the reverse relation for is corrupt" - ) - - def split_fqid(self, fqid: str) -> Tuple[str, int]: - try: - collection, _id = fqid.split("/") - id = int(_id) - if collection not in self.models.keys(): - raise CheckException(f"Fqid {fqid} has an invalid collection") - return collection, id - except (ValueError, AttributeError): - raise CheckException(f"Fqid {fqid} is malformed") - - def split_collectionfield(self, collectionfield: str) -> Tuple[str, str]: - collection, field = collectionfield.split("/") - if collection not in self.models.keys(): - raise CheckException( - f"Collectionfield {collectionfield} has an invalid collection" - ) - if ( - field not in self.models[collection] - ): # Note: this has to be adopted when supporting template fields - raise CheckException( - f"Collectionfield {collectionfield} has an invalid field" - ) - return collection, field - - def get_to_generic_case( - self, collection: str, field: str, foreign_collection: str - ) -> str: - """Returns all reverse relations as collectionfields""" - to = self.models[collection][field]["to"] - if isinstance(to, dict): - if foreign_collection not in to["collections"]: - raise CheckException( - f"The collection {foreign_collection} is not supported " - "as a reverse relation in {collection}/{field}" - ) - return to["field"] - - for cf in to: - c, f = self.split_collectionfield(cf) - if c == foreign_collection: - return f - - raise CheckException( - f"The collection {foreign_collection} is not supported as a reverse relation in {collection}/{field}" - ) - - -def main() -> int: - files = sys.argv[1:] - if not files: - files = DEFAULT_FILES - - is_import = "--import" in files - if is_import: - files = [x for x in files if x != "--import"] - - failed = False - for f in files: - with open(f) as data: - try: - Checker(json.load(data), is_import=is_import).run_check() - except CheckException as e: - print(f"Check for {f} failed:\n", e) - failed = True - else: - print(f"Check for {f} successful.") - return 1 if failed else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/docs/datavalidator/export.json b/docs/datavalidator/export.json deleted file mode 100644 index b23c2db6f..000000000 --- a/docs/datavalidator/export.json +++ /dev/null @@ -1 +0,0 @@ -{"meeting":[{"id":1,"projection_ids":[],"list_of_speakers_countdown_id":1,"poll_countdown_id":2,"default_group_id":1,"admin_group_id":2,"reference_projector_id":1,"welcome_title":"Welcome to OpenSlides","welcome_text":"[Space for your welcome text.]","name":"OpenSlides","description":"Presentation and assembly system","location":"","start_time":0,"end_time":0,"jitsi_domain":null,"jitsi_room_name":null,"jitsi_room_password":null,"enable_chat":true,"url_name":null,"template_for_committee_id":null,"enable_anonymous":false,"custom_translations":[],"conference_show":false,"conference_auto_connect":false,"conference_los_restriction":false,"conference_stream_url":"","conference_stream_poster_url":"","conference_open_microphone":false,"conference_open_video":false,"conference_auto_connect_next_speakers":0,"projector_countdown_default_time":60,"projector_countdown_warning_time":0,"export_csv_encoding":"utf-8","export_csv_separator":",","export_pdf_pagenumber_alignment":"center","export_pdf_fontsize":10,"export_pdf_pagesize":"A4","agenda_show_subtitles":true,"agenda_enable_numbering":true,"agenda_number_prefix":"","agenda_numeral_system":"arabic","agenda_item_creation":"default_yes","agenda_new_items_default_visibility":"internal","agenda_show_internal_items_on_projector":false,"list_of_speakers_amount_last_on_projector":0,"list_of_speakers_amount_next_on_projector":-1,"list_of_speakers_couple_countdown":true,"list_of_speakers_show_amount_of_speakers_on_slide":true,"list_of_speakers_present_users_only":false,"list_of_speakers_show_first_contribution":false,"list_of_speakers_enable_point_of_order_speakers":false,"list_of_speakers_enable_pro_contra_speech":false,"list_of_speakers_can_set_contribution_self":false,"list_of_speakers_speaker_note_for_everyone":false,"list_of_speakers_initially_closed":false,"motions_default_workflow_id":1,"motions_default_amendment_workflow_id":1,"motions_default_statute_amendment_workflow_id":1,"motions_preamble":"The assembly may decide:","motions_default_line_numbering":"outside","motions_line_length":85,"motions_reason_required":false,"motions_enable_text_on_projector":true,"motions_enable_reason_on_projector":true,"motions_enable_sidebox_on_projector":false,"motions_enable_recommendation_on_projector":true,"motions_show_referring_motions":true,"motions_show_sequential_number":true,"motions_recommendations_by":"","motions_statute_recommendations_by":"","motions_recommendation_text_mode":"diff","motions_default_sorting":"identifier","motions_number_type":"per_category","motions_number_min_digits":1,"motions_number_with_blank":false,"motions_statutes_enabled":false,"motions_amendments_enabled":false,"motions_amendments_in_main_list":true,"motions_amendments_of_amendments":false,"motions_amendments_prefix":"-","motions_amendments_text_mode":"paragraph","motions_amendments_multiple_paragraphs":true,"motions_supporters_min_amount":0,"motions_export_title":"Motions","motions_export_preamble":"","motions_export_submitter_recommendation":false,"motions_export_follow_recommendation":false,"motion_poll_ballot_paper_selection":"CUSTOM_NUMBER","motion_poll_ballot_paper_number":8,"motion_poll_default_type":"analog","motion_poll_default_100_percent_base":"YNA","motion_poll_default_majority_method":"simple","motion_poll_default_group_ids":[],"users_sort_by":"first_name","users_enable_presence_view":false,"users_enable_vote_weight":false,"users_allow_self_set_present":false,"users_pdf_welcometitle":"Welcome to OpenSlides","users_pdf_welcometext":"[Place for your welcome and help text.]","users_pdf_url":"https://127.0.0.1","users_pdf_wlan_ssid":"","users_pdf_wlan_password":"","users_pdf_wlan_encryption":"","users_email_sender":"OpenSlides","users_email_replyto":"","users_email_subject":"OpenSlides access data","users_email_body":"Dear {name},\n\nthis is your personal OpenSlides login:\n\n {url}\n username: {username}\n password: {password}\n\nThis email was generated automatically.","assignments_export_title":"Elections","assignments_export_preamble":"","assignment_poll_ballot_paper_selection":"CUSTOM_NUMBER","assignment_poll_ballot_paper_number":8,"assignment_poll_add_candidates_to_list_of_speakers":true,"assignment_poll_sort_poll_result_by_votes":true,"assignment_poll_default_type":"analog","assignment_poll_default_method":"Y","assignment_poll_default_100_percent_base":"valid","assignment_poll_default_majority_method":"simple","assignment_poll_default_group_ids":[],"poll_ballot_paper_selection":"CUSTOM_NUMBER","poll_ballot_paper_number":8,"poll_sort_poll_result_by_votes":true,"poll_default_type":"analog","poll_default_method":"Y","poll_default_100_percent_base":"YNA","poll_default_majority_method":"simple","poll_default_group_ids":[],"poll_couple_countdown":true,"projector_ids":[1],"projector_message_ids":[],"projector_countdown_ids":[1,2],"tag_ids":[],"agenda_item_ids":[],"list_of_speakers_ids":[1],"speaker_ids":[],"topic_ids":[],"group_ids":[3,5,4,1,2],"mediafile_ids":[1],"motion_ids":[],"motion_comment_section_ids":[],"motion_category_ids":[],"motion_block_ids":[],"motion_workflow_ids":[2,1],"motion_statute_paragraph_ids":[],"motion_comment_ids":[],"motion_submitter_ids":[],"motion_change_recommendation_ids":[],"motion_state_ids":[4,6,3,7,2,13,14,12,11,8,1,15,5,9,10],"poll_ids":[],"option_ids":[],"vote_ids":[],"assignment_ids":[],"assignment_candidate_ids":[],"personal_note_ids":[],"chat_group_ids":[],"all_projection_ids":[],"logo_$_id":[],"font_$_id":[],"committee_id":null,"default_meeting_for_committee_id":null,"organization_tag_ids":[],"present_user_ids":[],"user_ids":[1],"default_projector_$_id":["amendment","agenda_all_items","assignment_poll","motion_block","current_list_of_speakers","projector_countdowns","mediafile","topics","list_of_speakers","projector_message","motion_poll","motion","user","assignment","poll"],"default_projector_$amendment_id":1,"default_projector_$agenda_all_items_id":1,"default_projector_$assignment_poll_id":1,"default_projector_$motion_block_id":1,"default_projector_$current_list_of_speakers_id":1,"default_projector_$projector_countdowns_id":1,"default_projector_$mediafile_id":1,"default_projector_$topics_id":1,"default_projector_$list_of_speakers_id":1,"default_projector_$projector_message_id":1,"default_projector_$motion_poll_id":1,"default_projector_$motion_id":1,"default_projector_$user_id":1,"default_projector_$assignment_id":1,"default_projector_$poll_id":1}],"list_of_speakers":[{"id":1,"closed":false,"content_object_id":"mediafile/1","speaker_ids":[],"projection_ids":[],"meeting_id":1}],"vote":[],"poll":[],"option":[],"mediafile":[{"id":1,"title":"main.cpp","is_directory":false,"mimetype":"text/x-c++src","pdf_information":{},"parent_id":null,"list_of_speakers_id":1,"filename":"main.cpp","filesize":1125,"blob":null,"create_timestamp":1624521902,"access_group_ids":[],"is_public":true,"inherited_access_group_ids":[],"child_ids":[],"attachment_ids":[],"projection_ids":[],"used_as_logo_$_in_meeting_id":[],"used_as_font_$_in_meeting_id":[],"meeting_id":1}],"motion":[],"motion_state":[{"id":4,"name":"not decided","recommendation_label":"No decision","allow_support":false,"allow_create_poll":false,"allow_submitter_edit":false,"show_state_extension_field":false,"show_recommendation_extension_field":false,"workflow_id":1,"css_class":"grey","restrictions":[],"set_number":true,"merge_amendment_into_final":"do_not_merge","next_state_ids":[],"previous_state_ids":[1],"motion_ids":[],"motion_recommendation_ids":[],"first_state_of_workflow_id":null,"meeting_id":1},{"id":6,"name":"submitted","recommendation_label":null,"allow_support":true,"allow_create_poll":false,"allow_submitter_edit":false,"show_state_extension_field":false,"show_recommendation_extension_field":false,"workflow_id":2,"css_class":"lightblue","restrictions":[],"set_number":false,"merge_amendment_into_final":"undefined","next_state_ids":[7,10,15],"previous_state_ids":[5],"motion_ids":[],"motion_recommendation_ids":[],"first_state_of_workflow_id":null,"meeting_id":1},{"id":3,"name":"rejected","recommendation_label":"Rejection","allow_support":false,"allow_create_poll":false,"allow_submitter_edit":false,"show_state_extension_field":false,"show_recommendation_extension_field":false,"workflow_id":1,"css_class":"red","restrictions":[],"set_number":true,"merge_amendment_into_final":"do_not_merge","next_state_ids":[],"previous_state_ids":[1],"motion_ids":[],"motion_recommendation_ids":[],"first_state_of_workflow_id":null,"meeting_id":1},{"id":7,"name":"permitted","recommendation_label":"Permission","allow_support":false,"allow_create_poll":true,"allow_submitter_edit":false,"show_state_extension_field":false,"show_recommendation_extension_field":false,"workflow_id":2,"css_class":"lightblue","restrictions":[],"set_number":true,"merge_amendment_into_final":"undefined","next_state_ids":[8,9,10,11,12,13,14],"previous_state_ids":[6],"motion_ids":[],"motion_recommendation_ids":[],"first_state_of_workflow_id":null,"meeting_id":1},{"id":2,"name":"accepted","recommendation_label":"Acceptance","allow_support":false,"allow_create_poll":false,"allow_submitter_edit":false,"show_state_extension_field":false,"show_recommendation_extension_field":false,"workflow_id":1,"css_class":"green","restrictions":[],"set_number":true,"merge_amendment_into_final":"do_merge","next_state_ids":[],"previous_state_ids":[1],"motion_ids":[],"motion_recommendation_ids":[],"first_state_of_workflow_id":null,"meeting_id":1},{"id":13,"name":"refered to committee","recommendation_label":"Referral to committee","allow_support":false,"allow_create_poll":false,"allow_submitter_edit":false,"show_state_extension_field":false,"show_recommendation_extension_field":false,"workflow_id":2,"css_class":"grey","restrictions":[],"set_number":true,"merge_amendment_into_final":"do_not_merge","next_state_ids":[],"previous_state_ids":[7],"motion_ids":[],"motion_recommendation_ids":[],"first_state_of_workflow_id":null,"meeting_id":1},{"id":14,"name":"needs review","recommendation_label":null,"allow_support":false,"allow_create_poll":false,"allow_submitter_edit":false,"show_state_extension_field":false,"show_recommendation_extension_field":false,"workflow_id":2,"css_class":"grey","restrictions":[],"set_number":true,"merge_amendment_into_final":"do_not_merge","next_state_ids":[],"previous_state_ids":[7],"motion_ids":[],"motion_recommendation_ids":[],"first_state_of_workflow_id":null,"meeting_id":1},{"id":12,"name":"not concerned","recommendation_label":"No concernment","allow_support":false,"allow_create_poll":false,"allow_submitter_edit":false,"show_state_extension_field":false,"show_recommendation_extension_field":false,"workflow_id":2,"css_class":"grey","restrictions":[],"set_number":true,"merge_amendment_into_final":"do_not_merge","next_state_ids":[],"previous_state_ids":[7],"motion_ids":[],"motion_recommendation_ids":[],"first_state_of_workflow_id":null,"meeting_id":1},{"id":11,"name":"adjourned","recommendation_label":"Adjournment","allow_support":false,"allow_create_poll":false,"allow_submitter_edit":false,"show_state_extension_field":false,"show_recommendation_extension_field":false,"workflow_id":2,"css_class":"grey","restrictions":[],"set_number":true,"merge_amendment_into_final":"do_not_merge","next_state_ids":[],"previous_state_ids":[7],"motion_ids":[],"motion_recommendation_ids":[],"first_state_of_workflow_id":null,"meeting_id":1},{"id":8,"name":"accepted","recommendation_label":"Acceptance","allow_support":false,"allow_create_poll":false,"allow_submitter_edit":false,"show_state_extension_field":false,"show_recommendation_extension_field":false,"workflow_id":2,"css_class":"green","restrictions":[],"set_number":true,"merge_amendment_into_final":"do_merge","next_state_ids":[],"previous_state_ids":[7],"motion_ids":[],"motion_recommendation_ids":[],"first_state_of_workflow_id":null,"meeting_id":1},{"id":1,"name":"submitted","recommendation_label":null,"allow_support":true,"allow_create_poll":true,"allow_submitter_edit":false,"show_state_extension_field":false,"show_recommendation_extension_field":false,"workflow_id":1,"css_class":"lightblue","restrictions":[],"set_number":true,"merge_amendment_into_final":"undefined","next_state_ids":[2,3,4],"previous_state_ids":[],"motion_ids":[],"motion_recommendation_ids":[],"first_state_of_workflow_id":1,"meeting_id":1},{"id":15,"name":"rejected (not authorized)","recommendation_label":"Rejection (not authorized)","allow_support":false,"allow_create_poll":false,"allow_submitter_edit":false,"show_state_extension_field":false,"show_recommendation_extension_field":false,"workflow_id":2,"css_class":"grey","restrictions":[],"set_number":true,"merge_amendment_into_final":"do_not_merge","next_state_ids":[],"previous_state_ids":[6],"motion_ids":[],"motion_recommendation_ids":[],"first_state_of_workflow_id":null,"meeting_id":1},{"id":5,"name":"in progress","recommendation_label":null,"allow_support":false,"allow_create_poll":false,"allow_submitter_edit":true,"show_state_extension_field":false,"show_recommendation_extension_field":false,"workflow_id":2,"css_class":"lightblue","restrictions":[],"set_number":false,"merge_amendment_into_final":"undefined","next_state_ids":[6,10],"previous_state_ids":[],"motion_ids":[],"motion_recommendation_ids":[],"first_state_of_workflow_id":2,"meeting_id":1},{"id":9,"name":"rejected","recommendation_label":"Rejection","allow_support":false,"allow_create_poll":false,"allow_submitter_edit":false,"show_state_extension_field":false,"show_recommendation_extension_field":false,"workflow_id":2,"css_class":"red","restrictions":[],"set_number":true,"merge_amendment_into_final":"do_not_merge","next_state_ids":[],"previous_state_ids":[7],"motion_ids":[],"motion_recommendation_ids":[],"first_state_of_workflow_id":null,"meeting_id":1},{"id":10,"name":"withdrawed","recommendation_label":null,"allow_support":false,"allow_create_poll":false,"allow_submitter_edit":false,"show_state_extension_field":false,"show_recommendation_extension_field":false,"workflow_id":2,"css_class":"grey","restrictions":[],"set_number":true,"merge_amendment_into_final":"do_not_merge","next_state_ids":[],"previous_state_ids":[6,7,5],"motion_ids":[],"motion_recommendation_ids":[],"first_state_of_workflow_id":null,"meeting_id":1}],"motion_workflow":[{"id":2,"name":"Complex Workflow","first_state_id":5,"state_ids":[5,6,7,8,9,10,11,12,13,14,15],"default_workflow_meeting_id":null,"default_amendment_workflow_meeting_id":null,"default_statute_amendment_workflow_meeting_id":null,"meeting_id":1},{"id":1,"name":"Simple Workflow","first_state_id":1,"state_ids":[1,2,3,4],"default_workflow_meeting_id":1,"default_amendment_workflow_meeting_id":1,"default_statute_amendment_workflow_meeting_id":1,"meeting_id":1}],"projector_countdown":[{"id":1,"title":"list of speakers countdown","description":"created at the migration from OS3 to OS4","default_time":60,"countdown_time":60,"running":false,"used_as_list_of_speaker_countdown_meeting_id":1,"used_as_poll_countdown_meeting_id":null,"projection_ids":[],"meeting_id":1},{"id":2,"title":"poll countdown","description":"created at the migration from OS3 to OS4","default_time":60,"countdown_time":60,"running":false,"used_as_list_of_speaker_countdown_meeting_id":null,"used_as_poll_countdown_meeting_id":1,"projection_ids":[],"meeting_id":1}],"speaker":[],"personal_note":[],"motion_submitter":[],"assignment_candidate":[],"user":[{"id":1,"username":"admin","title":"","first_name":"","last_name":"Administrator","is_active":true,"default_password":"admin","gender":"","email":"","is_physical_person":true,"password":"","default_number":"","default_structure_level":"","default_vote_weight":"1.000000","last_email_send":null,"is_demo_user":false,"organization_management_level":null,"is_present_in_meeting_ids":[],"committee_ids":[],"committee_$_management_level":[],"comment_$":[],"number_$":[],"structure_level_$":[],"about_me_$":[],"vote_weight_$":[],"group_$_ids":["1"],"group_$1_ids":[2],"can_change_own_password":true,"speaker_$_ids":[],"personal_note_$_ids":[],"supported_motion_$_ids":[],"submitted_motion_$_ids":[],"poll_voted_$_ids":[],"option_$_ids":[],"vote_$_ids":[],"vote_delegated_vote_$_ids":[],"assignment_candidate_$_ids":[],"projection_$_ids":[],"vote_delegated_$_to_id":[],"vote_delegations_$_from_ids":[],"meeting_ids":[1]}],"motion_comment_section":[],"chat_group":[],"group":[{"id":3,"name":"Delegates","permissions":["assignment.can_nominate_other","mediafile.can_see","motion.can_support","motion.can_create_amendments","list_of_speakers.can_be_speaker","meeting.can_see_frontpage","assignment.can_nominate_self","motion.can_create","meeting.can_see_autopilot","projector.can_see","agenda_item.can_see_internal","user.can_see"],"user_ids":[],"default_group_for_meeting_id":null,"admin_group_for_meeting_id":null,"mediafile_access_group_ids":[],"mediafile_inherited_access_group_ids":[],"read_comment_section_ids":[],"write_comment_section_ids":[],"read_chat_group_ids":[],"write_chat_group_ids":[],"poll_ids":[],"used_as_motion_poll_default_id":null,"used_as_assignment_poll_default_id":null,"used_as_poll_default_id":null,"meeting_id":1},{"id":5,"name":"Committees","permissions":["list_of_speakers.can_see","mediafile.can_see","motion.can_support","motion.can_create_amendments","meeting.can_see_frontpage","motion.can_create","assignment.can_see","projector.can_see","agenda_item.can_see_internal","user.can_see"],"user_ids":[],"default_group_for_meeting_id":null,"admin_group_for_meeting_id":null,"mediafile_access_group_ids":[],"mediafile_inherited_access_group_ids":[],"read_comment_section_ids":[],"write_comment_section_ids":[],"read_chat_group_ids":[],"write_chat_group_ids":[],"poll_ids":[],"used_as_motion_poll_default_id":null,"used_as_assignment_poll_default_id":null,"used_as_poll_default_id":null,"meeting_id":1},{"id":4,"name":"Staff","permissions":["list_of_speakers.can_manage","meeting.can_see_frontpage","assignment.can_nominate_self","meeting.can_see_history","agenda_item.can_manage","assignment.can_manage","user.can_manage","mediafile.can_manage","list_of_speakers.can_be_speaker","projector.can_manage","motion.can_manage"],"user_ids":[],"default_group_for_meeting_id":null,"admin_group_for_meeting_id":null,"mediafile_access_group_ids":[],"mediafile_inherited_access_group_ids":[],"read_comment_section_ids":[],"write_comment_section_ids":[],"read_chat_group_ids":[],"write_chat_group_ids":[],"poll_ids":[],"used_as_motion_poll_default_id":null,"used_as_assignment_poll_default_id":null,"used_as_poll_default_id":null,"meeting_id":1},{"id":1,"name":"Default","permissions":["list_of_speakers.can_see","mediafile.can_see","meeting.can_see_frontpage","assignment.can_see","projector.can_see","agenda_item.can_see_internal","user.can_see","motion.can_see"],"user_ids":[],"default_group_for_meeting_id":1,"admin_group_for_meeting_id":null,"mediafile_access_group_ids":[],"mediafile_inherited_access_group_ids":[],"read_comment_section_ids":[],"write_comment_section_ids":[],"read_chat_group_ids":[],"write_chat_group_ids":[],"poll_ids":[],"used_as_motion_poll_default_id":null,"used_as_assignment_poll_default_id":null,"used_as_poll_default_id":null,"meeting_id":1},{"id":2,"name":"Admin","permissions":[],"user_ids":[1],"default_group_for_meeting_id":null,"admin_group_for_meeting_id":1,"mediafile_access_group_ids":[],"mediafile_inherited_access_group_ids":[],"read_comment_section_ids":[],"write_comment_section_ids":[],"read_chat_group_ids":[],"write_chat_group_ids":[],"poll_ids":[],"used_as_motion_poll_default_id":null,"used_as_assignment_poll_default_id":null,"used_as_poll_default_id":null,"meeting_id":1}],"projector":[{"id":1,"name":"Default projector","scale":0,"scroll":0,"width":1200,"aspect_ratio_numerator":16,"aspect_ratio_denominator":9,"color":"#000000","background_color":"#ffffff","header_background_color":"#317796","header_font_color":"#f5f5f5","header_h1_color":"#317796","chyron_background_color":"#317796","chyron_font_color":"#ffffff","show_header_footer":true,"show_title":true,"show_logo":true,"show_clock":false,"current_projection_ids":[],"preview_projection_ids":[],"history_projection_ids":[],"used_as_reference_projector_meeting_id":1,"used_as_default_$_in_meeting_id":["amendment","agenda_all_items","assignment_poll","motion_block","current_list_of_speakers","projector_countdowns","mediafile","topics","list_of_speakers","projector_message","motion_poll","motion","user","assignment","poll"],"meeting_id":1,"used_as_default_$amendment_in_meeting_id":1,"used_as_default_$agenda_all_items_in_meeting_id":1,"used_as_default_$assignment_poll_in_meeting_id":1,"used_as_default_$motion_block_in_meeting_id":1,"used_as_default_$current_list_of_speakers_in_meeting_id":1,"used_as_default_$projector_countdowns_in_meeting_id":1,"used_as_default_$mediafile_in_meeting_id":1,"used_as_default_$topics_in_meeting_id":1,"used_as_default_$list_of_speakers_in_meeting_id":1,"used_as_default_$projector_message_in_meeting_id":1,"used_as_default_$motion_poll_in_meeting_id":1,"used_as_default_$motion_in_meeting_id":1,"used_as_default_$user_in_meeting_id":1,"used_as_default_$assignment_in_meeting_id":1,"used_as_default_$poll_in_meeting_id":1}],"projector_message":[],"tag":[],"agenda_item":[],"topic":[],"motion_category":[],"motion_block":[],"motion_statute_paragraph":[],"motion_comment":[],"motion_change_recommendation":[],"assignment":[],"projection":[]} diff --git a/docs/datavalidator/requirements.txt b/docs/datavalidator/requirements.txt deleted file mode 100644 index da27c7d17..000000000 --- a/docs/datavalidator/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -fastjsonschema -pyyaml diff --git a/docs/models.yml b/docs/models.yml index 1efe12142..fbb2a02eb 100644 --- a/docs/models.yml +++ b/docs/models.yml @@ -52,7 +52,7 @@ # for all the fields that come from the template field. # JSON Schema Properties: # - You can add JSON Schema properties to the fields like `enum`, `description`, -# `maxLength` and `minimum` +# `items`, `maxLength` and `minimum` # Additional properties: # - The property `read_only` describes a field that can not be changed by an action. # - The property `default` describes the default value that is used for new objects. @@ -60,6 +60,9 @@ # string. If this field is given it must have some content. # - The property `equal_fields` describes fields that must have the same value in # the instance and the related instance. +# Restriction Mode: +# The field `restriction_mode` is required for every field. It puts the field into a +# restriction group. See https://github.com/OpenSlides/OpenSlides/wiki/Restrictions-Overview organization: id: @@ -463,6 +466,7 @@ meeting: restriction_mode: B imported_at: type: timestamp + restriction_mode: B # Configuration (only for the server owner) jitsi_domain: @@ -2393,10 +2397,10 @@ option: text: type: HTMLStrict restriction_mode: A - yes: + "yes": type: decimal(6) restriction_mode: B - no: + "no": type: decimal(6) restriction_mode: B abstain: @@ -2589,7 +2593,7 @@ mediafile: restriction_mode: B filename: type: string - descriptin: The uploaded filename. Will be used for downloading. Only writeable on create. + description: The uploaded filename. Will be used for downloading. Only writeable on create. restriction_mode: B mimetype: type: string @@ -2797,6 +2801,11 @@ projection: type: string restriction_mode: A + content: + type: JSON + calculated: true + restriction_mode: A + current_projector_id: type: relation to: projector/current_projection_ids @@ -2834,10 +2843,6 @@ projection: to: meeting/all_projection_ids required: true restriction_mode: A - content: - type: JSON - calculated: true - restriction_mode: A projector_message: id: diff --git a/docs/modelsvalidator/README.md b/docs/modelsvalidator/README.md deleted file mode 100644 index b71db6f45..000000000 --- a/docs/modelsvalidator/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Modelsvalidator - -Modelsvalidator is a tool to validate the models.yml file - - -## Run - -Build first: `go build ./...`. - -The tool requires the content of the models.yml. It can be provided via stdin, a -file system path or an url starting with http:// or https://. - - -``` -cat models.yml | modelsvalidator -modelsvalidator openslides/docs/models.yml -modelsvalidator https://raw.githubusercontent.com/OpenSlides/OpenSlides/openslides4-dev/docs/models.yml -``` - -The tool returns with status code 0 and no content, if the given content is -valid. It returns with a positive status code and some error messages if not. diff --git a/docs/modelsvalidator/check/check.go b/docs/modelsvalidator/check/check.go deleted file mode 100644 index fa8ce3cb5..000000000 --- a/docs/modelsvalidator/check/check.go +++ /dev/null @@ -1,159 +0,0 @@ -package check - -import ( - "fmt" - "strings" - - models "github.com/OpenSlides/openslides-models-to-go" -) - -// Check runs some checks on the given models. -func Check(data map[string]models.Model) error { - validators := []func(map[string]models.Model) error{ - validateTypes, - validateRelations, - validateTemplatePrefixes, - } - - errors := new(ErrorList) - for _, v := range validators { - if err := v(data); err != nil { - errors.append(err) - } - } - - if !errors.empty() { - return errors - } - return nil -} - -func validateTypes(data map[string]models.Model) error { - scalar := scalarTypes() - relation := relationTypes() - errs := &ErrorList{ - Name: "type validator", - intent: 1, - } - for modelName, model := range data { - for fieldName, field := range model.Fields { - if scalar[strings.TrimSuffix(field.Type, "[]")] { - continue - } - - if relation[field.Type] { - continue - } - - errs.append(fmt.Errorf("Unknown type `%s` in %s/%s", field.Type, modelName, fieldName)) - } - } - if errs.empty() { - return nil - } - return errs -} - -func validateRelations(data map[string]models.Model) error { - errs := &ErrorList{ - Name: "relation validator", - intent: 1, - } - relation := relationTypes() - for modelName, model := range data { - Next: - for fieldName, field := range model.Fields { - r := field.Relation() - if r == nil { - continue - } - - for _, c := range r.ToCollections() { - toModel, ok := data[c.Collection] - if !ok { - errs.append(fmt.Errorf("%s/%s directs to nonexisting model `%s`", modelName, fieldName, c.Collection)) - continue Next - } - - toField, ok := toModel.Fields[c.ToField.Name] - if !ok { - errs.append(fmt.Errorf("%s/%s directs to nonexisting collectionfield `%s/%s`", modelName, fieldName, c.Collection, c.ToField.Name)) - continue Next - } - - if !relation[toField.Type] { - errs.append(fmt.Errorf("%s/%s directs to `%s/%s`, but it is not a relation, but %s", modelName, fieldName, c.Collection, c.ToField.Name, toField.Type)) - continue Next - } - - } - } - } - if errs.empty() { - return nil - } - return errs -} - -func validateTemplatePrefixes(models map[string]models.Model) error { - errs := &ErrorList{ - Name: "template prefixes validator", - intent: 1, - } - for modelName, model := range models { - prefixes := map[string]bool{} - for fieldName := range model.Fields { - i := strings.Index(fieldName, "$") - if i < 0 { - continue - } - prefix := fieldName[0:i] - if prefixes[prefix] { - errs.append(fmt.Errorf("Duplicate template prefix %s in %s", prefix, modelName)) - } - prefixes[prefix] = true - } - } - if errs.empty() { - return nil - } - return errs -} - -// scalarTypes are the main types. All scalarTypes can be used as a list. -// JSON[], timestamp[] etc. -func scalarTypes() map[string]bool { - s := []string{ - "string", - "number", - "boolean", - "JSON", - "HTMLPermissive", - "HTMLStrict", - "float", - "decimal(6)", - "timestamp", - "color", - } - out := make(map[string]bool) - for _, t := range s { - out[t] = true - } - return out -} - -// relationTypes are realtion types in realtion to other fields. -func relationTypes() map[string]bool { - s := []string{ - "relation", - "relation-list", - "generic-relation", - "generic-relation-list", - "template", - } - out := make(map[string]bool) - for _, t := range s { - out[t] = true - } - return out -} diff --git a/docs/modelsvalidator/check/check_test.go b/docs/modelsvalidator/check/check_test.go deleted file mode 100644 index 5871da637..000000000 --- a/docs/modelsvalidator/check/check_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package check_test - -import ( - "errors" - "strings" - "testing" - - "github.com/OpenSlides/Openslides/modelsvalidator/check" - models "github.com/OpenSlides/openslides-models-to-go" -) - -const yamlUnknownFieldType = `--- -some_model: - field: unknown -` - -const yamlNonExistingModel = `--- -some_model: - no_other_model: - type: relation - to: not_existing/field -` - -const yamlNonExistingField = `--- -some_model: - no_other_field: - type: relation - to: other_model/bar -other_model: - foo: string -` - -const yamlDuplicateTemplatePrefix = `--- -some_model: - field_$_1: number - field_$_2: number -` - -const yamlWrongReverseRelaitonType = `--- -some_model: - other_model: - type: relation - to: other_model/field -other_model: - field: HTMLStrict -` - -func TestCheck(t *testing.T) { - for _, tt := range []struct { - name string - yaml string - err string - }{ - { - "unknown type", - yamlUnknownFieldType, - "Unknown type `unknown` in some_model/field", - }, - { - "non-existing model", - yamlNonExistingModel, - "some_model/no_other_model directs to nonexisting model `not_existing`", - }, - { - "non-existing Field", - yamlNonExistingField, - "some_model/no_other_field directs to nonexisting collectionfield `other_model/bar`", - }, - { - "duplicate template prefix", - yamlDuplicateTemplatePrefix, - "Duplicate template prefix field_ in some_model", - }, - { - "wrong reverse relation type", - yamlWrongReverseRelaitonType, - "some_model/other_model directs to `other_model/field`, but it is not a relation, but HTMLStrict", - }, - } { - t.Run(tt.name, func(t *testing.T) { - data, err := models.Unmarshal(strings.NewReader(tt.yaml)) - if err != nil { - t.Fatalf("Can not unmarshal yaml: %v", err) - } - gotErr := check.Check(data) - if tt.err == "" { - if gotErr != nil { - t.Errorf("Models.Check() returned an unexepcted error: %v", err) - } - return - } - - if gotErr == nil { - t.Fatalf("Models.Check() did not return an error, expected: %v", tt.err) - } - - var errList *check.ErrorList - if !errors.As(gotErr, &errList) { - t.Fatalf("Models.Check() did not return a ListError, got: %v", gotErr) - } - - var found bool - for _, err := range errList.Errs { - var errList *check.ErrorList - if !errors.As(err, &errList) { - continue - } - - for _, err := range errList.Errs { - if err.Error() == tt.err { - found = true - } - } - } - - if !found { - t.Errorf("Models.Check() returned:\n%v\n\nExpected something that contains:\n%v", gotErr, tt.err) - } - }) - } -} diff --git a/docs/modelsvalidator/check/error.go b/docs/modelsvalidator/check/error.go deleted file mode 100644 index 4349acfe7..000000000 --- a/docs/modelsvalidator/check/error.go +++ /dev/null @@ -1,38 +0,0 @@ -package check - -import ( - "fmt" - "strings" -) - -// ErrorList is an error that contains a list of other errors. -type ErrorList struct { - Name string - intent int - Errs []error -} - -func (e *ErrorList) append(err error) { - if err == nil { - return - } - - e.Errs = append(e.Errs, err) -} - -func (e ErrorList) Error() string { - intent := strings.Repeat(" ", e.intent) - var msgs []string - for _, err := range e.Errs { - msgs = append(msgs, fmt.Sprintf("%s* %v", intent, err)) - } - msg := strings.Join(msgs, "\n") - if e.Name != "" { - return fmt.Sprintf("%s:\n%s", e.Name, msg) - } - return msg -} - -func (e *ErrorList) empty() bool { - return len(e.Errs) == 0 -} diff --git a/docs/modelsvalidator/go.mod b/docs/modelsvalidator/go.mod deleted file mode 100644 index dd145f79a..000000000 --- a/docs/modelsvalidator/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module github.com/OpenSlides/Openslides/modelsvalidator - -go 1.15 - -require github.com/OpenSlides/openslides-models-to-go v0.2.0 diff --git a/docs/modelsvalidator/go.sum b/docs/modelsvalidator/go.sum deleted file mode 100644 index 80592ada3..000000000 --- a/docs/modelsvalidator/go.sum +++ /dev/null @@ -1,6 +0,0 @@ -github.com/OpenSlides/openslides-models-to-go v0.2.0 h1:1D+rD94GcUZAvmHLRyzJCIj7wklU+JBkoYifwtHH08s= -github.com/OpenSlides/openslides-models-to-go v0.2.0/go.mod h1:CriCefW5smTixhFfVLiuA8NgyMX4PAU5e3YpJHnaZx8= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/docs/modelsvalidator/main.go b/docs/modelsvalidator/main.go deleted file mode 100644 index 21980085f..000000000 --- a/docs/modelsvalidator/main.go +++ /dev/null @@ -1,56 +0,0 @@ -package main - -import ( - "fmt" - "io" - "log" - "net/http" - "os" - "strings" - - "github.com/OpenSlides/Openslides/modelsvalidator/check" - models "github.com/OpenSlides/openslides-models-to-go" -) - -func main() { - var content io.Reader = os.Stdin - if len(os.Args) > 1 { - c, err := openModels(os.Args[1]) - if err != nil { - log.Fatalf("Can not load content: %v", err) - } - defer c.Close() - content = c - } - - data, err := models.Unmarshal(content) - if err != nil { - log.Fatalf("Invalid model format: %v", err) - } - - if err := check.Check(data); err != nil { - log.Fatalf("Invalid model structure:\n\n%v", err) - } -} - -// openModels reads the model either from file or from an url. -func openModels(path string) (io.ReadCloser, error) { - if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { - return openModelsFromURL(path) - } - - return os.Open(path) -} - -func openModelsFromURL(url string) (io.ReadCloser, error) { - resp, err := http.Get(url) - if err != nil { - return nil, fmt.Errorf("requesting models from url: %w", err) - } - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("can not get models from url. Got status %s", resp.Status) - } - - return resp.Body, nil -} diff --git a/docs/modelsvalidator/requirements.txt b/docs/modelsvalidator/requirements.txt new file mode 100644 index 000000000..c3726e8bf --- /dev/null +++ b/docs/modelsvalidator/requirements.txt @@ -0,0 +1 @@ +pyyaml diff --git a/docs/datavalidator/setup.cfg b/docs/modelsvalidator/setup.cfg similarity index 100% rename from docs/datavalidator/setup.cfg rename to docs/modelsvalidator/setup.cfg diff --git a/docs/modelsvalidator/validate.py b/docs/modelsvalidator/validate.py new file mode 100644 index 000000000..6d9d08cac --- /dev/null +++ b/docs/modelsvalidator/validate.py @@ -0,0 +1,375 @@ +import json +import re +import sys +from collections import defaultdict +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union + +import yaml + +DEFAULT_FILES = [ + "../../docs/models.yml", +] + + +KEYSEPARATOR = "/" +_collection_regex = r"[a-z]([a-z_]+[a-z]+)?" +_field_regex = r"[a-z][a-z0-9_]*\$?[a-z0-9_]*" + +collectionfield_regex = re.compile( + f"^({_collection_regex}){KEYSEPARATOR}({_field_regex})$" +) +collection_regex = re.compile(f"^{_collection_regex}$") +field_regex = re.compile(f"^{_field_regex}$") + +decimal_regex = re.compile("^\d+\.\d{6}$") +color_regex = re.compile("^#[0-9a-f]{6}$") + +RELATION_TYPES = ( + "relation", + "relation-list", + "generic-relation", + "generic-relation-list", +) + +DATA_TYPES = ( + "string", + "number", + "string[]", + "number[]", + "boolean", + "JSON", + "HTMLStrict", + "HTMLPermissive", + "float", + "decimal(6)", + "timestamp", + "color", +) + + +VALID_TYPES = DATA_TYPES + RELATION_TYPES + ("template",) + +OPTIONAL_ATTRIBUTES = ( + "description", + "calculated", + "required", + "read_only", +) + + +class CheckException(Exception): + pass + + +class Checker: + def __init__(self, filepath: str) -> None: + with open(filepath, "rb") as x: + self.models = yaml.safe_load(x.read()) + self.errors: List[str] = [] + + def run_check(self) -> None: + self._run_checks() + if self.errors: + errors = [f"\t{error}" for error in self.errors] + raise CheckException("\n".join(errors)) + + def _run_checks(self) -> None: + for collection in self.models.keys(): + if not collection_regex.match(collection): + self.errors.append(f"Collection '{collection}' is not valid.") + if self.errors: + return + + for collection, fields in self.models.items(): + if not isinstance(fields, dict): + self.errors.append( + f"The fields of collection {collection} must be a dict." + ) + continue + for field_name, field in fields.items(): + if not field_regex.match(field_name): + self.errors.append( + f"Field name '{field_name}' of collection {collection} is not a valid field name." + ) + continue + if not isinstance(field, dict): + self.errors.append( + f"Field '{field_name}' of collection {collection} must be a dict." + ) + self.check_field(collection, field_name, field) + + if self.errors: + return + + for collection, fields in self.models.items(): + for field_name, field in fields.items(): + is_relation_field = field["type"] in RELATION_TYPES + is_template_relation_field = ( + field["type"] == "template" + and isinstance(field["fields"], dict) + and field["fields"]["type"] in RELATION_TYPES + ) + if not is_relation_field and not is_template_relation_field: + continue + error = self.check_relation(collection, field_name, field) + if error: + self.errors.append(error) + + def check_field( + self, + collection: str, + field_name: str, + field: Union[str, Dict[str, Any]], + nested: bool = False, + ) -> None: + collectionfield = f"{collection}{KEYSEPARATOR}{field_name}" + + if nested: + if isinstance(field, str): + field = {"type": field} + field[ + "restriction_mode" + ] = "A" # add restriction_mode to satisfy the checker below. + if field["type"] == "template": # no nested templates + self.errors.append(f"Nested template field in {collectionfield}") + return + + type = field.get("type") + if type not in VALID_TYPES: + self.errors.append( + f"Type '{type}' for collectionfield {collectionfield} is invalid." + ) + return + + required_attributes = [ + "type", + "restriction_mode", + ] + if type in RELATION_TYPES: + required_attributes.append("to") + if type == "template": + required_attributes.append("fields") + for attr in required_attributes: + if attr not in field: + self.errors.append( + f"Required attribute '{attr}' for collectionfield {collectionfield} is missing." + ) + return + + if field.get("calculated"): + return + + valid_attributes = list(OPTIONAL_ATTRIBUTES) + required_attributes + if type == "string[]": + valid_attributes.append("items") + if "items" in field and "enum" not in field["items"]: + self.errors.append( + f"'items' is missing an inner 'enum' for {collectionfield}" + ) + return + for value in field.get("items", {"enum": []})["enum"]: + self.validate_value_for_type("string", value, collectionfield) + if type == "JSON" and "default" in field: + try: + json.loads(json.dumps(field["default"])) + except: + self.errors.append( + f"Default value for {collectionfield}' is not valid json." + ) + if type == "number": + valid_attributes.append("minimum") + if not isinstance(field.get("minimum", 0), int): + self.errors.append(f"'minimum' for {collectionfield} is not a number.") + if type == "string": + valid_attributes.append("maxLength") + if not isinstance(field.get("maxLength", 0), int): + self.errors.append( + f"'maxLength' for {collectionfield} is not a number." + ) + if type in DATA_TYPES: + valid_attributes.append("default") + if "default" in field: + self.validate_value_for_type(type, field["default"], collectionfield) + valid_attributes.append("enum") + if "enum" in field: + if not isinstance(field["enum"], list): + self.errors.append(f"'enum' for {collectionfield}' is not a list.") + for value in field["enum"]: + self.validate_value_for_type(type, value, collectionfield) + + if type in RELATION_TYPES: + valid_attributes.append("on_delete") + if "on_delete" in field and field["on_delete"] not in ( + "CASCADE", + "PROTECT", + ): + self.errors.append( + f"invalid value for 'on_delete' for {collectionfield}" + ) + valid_attributes.append("equal_fields") + + if type == "template": + if "$" not in field_name: + self.errors.append( + f"The template field {collectionfield} is missing a $" + ) + valid_attributes.append("replacement_collection") + elif "$" in field_name and not nested: + print(field_name, field) + self.errors.append(f"The non-template field {collectionfield} contains a $") + + for attr in field.keys(): + if attr not in valid_attributes: + self.errors.append( + f"Attribute '{attr}' for collectionfield {collectionfield} is invalid." + ) + + if not isinstance(field.get("description", ""), str): + self.errors.append(f"Description of {collectionfield} must be a string.") + + if type == "template": + self.check_field(collection, field_name, field["fields"], nested=True) + + def validate_value_for_type( + self, type_str: str, value: Any, collectionfield: str + ) -> None: + basic_types = { + "string": str, + "number": int, + "boolean": bool, + "HTMLStrict": str, + "HTMLPermissive": str, + "timestamp": int, + } + if type_str in basic_types: + if type(value) != basic_types[type_str]: + self.errors.append( + f"Value '{value}' for {collectionfield}' is not a {type_str}." + ) + elif type_str in ("string[]", "number[]"): + if not isinstance(value, list): + self.errors.append( + f"Value '{value}' for {collectionfield}' is not a {type_str}." + ) + for x in value: + if type(x) != basic_types[type_str[:-2]]: + self.errors.append( + f"Listentry '{x}' for {collectionfield}' is not a {type_str[:-2]}." + ) + elif type_str == "JSON": + pass + elif type_str == "float": + if type(value) not in (int, float): + self.errors.append( + f"Value '{value}' for {collectionfield}' is not a float." + ) + elif type_str == "decimal(6)": + if not decimal_regex.match(value): + self.errors.append( + f"Value '{value}' for {collectionfield}' is not a decimal(6)." + ) + elif type_str == "color": + if not color_regex.match(value): + self.errors.append( + f"Value '{value}' for {collectionfield}' is not a color." + ) + else: + raise NotImplementedError(type_str) + + def check_relation( + self, collection: str, field_name: str, field: Dict[str, Any] + ) -> Optional[str]: + collectionfield = f"{collection}{KEYSEPARATOR}{field_name}" + if field["type"] == "template": + field = field["fields"] + to = field["to"] + + if isinstance(to, str): + if not collectionfield_regex.match(to): + return f"'to' of {collectionfield} is not a collectionfield." + return self.check_reverse(collectionfield, to) + elif isinstance(to, list): + for cf in to: + if not collectionfield_regex.match(cf): + return f"The collectionfield in 'to' of {collectionfield} is not valid." + error = self.check_reverse(collectionfield, cf) + if error: + return error + else: + to_field = to["field"] + if not field_regex.match(to_field): + return ( + f"The field '{to_field}' in 'to' of {collectionfield} is not valid." + ) + for c in to["collections"]: + if not collection_regex.match(c): + self.errors.append( + f"The collection '{c}' in 'to' of {collectionfield} is not a valid collection." + ) + error = self.check_reverse( + collectionfield, f"{c}{KEYSEPARATOR}{to['field']}" + ) + if error: + return error + return None + + def check_reverse( + self, from_collectionfield: str, to_collectionfield: str + ) -> Optional[str]: + to_unified = [] # a list of target collectionfields (unififed with all + # the different possibilities for the 'to' field) from the (expected) + # relation in to_collectionfield. The from_collectionfield must be in this + # list. + + to_collection, to_field_name = to_collectionfield.split(KEYSEPARATOR) + if to_collection not in self.models: + return f"The collection '{to_collection}' in 'to' of {from_collectionfield} is not a valid collection." + if to_field_name not in self.models[to_collection]: + return f"The collectionfield '{to_collectionfield}' in 'to' of {from_collectionfield} does not exist." + + to_field = self.models[to_collection][to_field_name] + if to_field["type"] == "template": + to_field = to_field["fields"] + if not isinstance(to_field, dict): + return f"The 'fields' of the template field '{to_collectionfield}' must be a dict to hold a relation." + if to_field["type"] not in RELATION_TYPES: + return f"{from_collectionfield} points to {to_collectionfield}, but {to_collectionfield} to is not a relation." + + to = to_field["to"] + if isinstance(to, str): + to_unified.append(to) + elif isinstance(to, list): + to_unified = to + else: + for c in to["collections"]: + to_unified.append(f"{c}{KEYSEPARATOR}{to['field']}") + + if from_collectionfield not in to_unified: + return f"{from_collectionfield} points to {to_collectionfield}, but {to_collectionfield} does not point back." + return None + + def split_collectionfield(self, collectionfield: str) -> Tuple[str, str]: + parts = collectionfield.split(KEYSEPARATOR) + return parts[0], parts[1] + + +def main() -> int: + files = sys.argv[1:] + if not files: + files = DEFAULT_FILES + + failed = False + for f in files: + with open(f) as data: + try: + Checker(f).run_check() + except CheckException as e: + print(f"Check for {f} failed:\n", e) + failed = True + else: + print(f"Check for {f} successful.") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/openslides-autoupdate-service b/openslides-autoupdate-service index 4bf228dd0..55fa585ff 160000 --- a/openslides-autoupdate-service +++ b/openslides-autoupdate-service @@ -1 +1 @@ -Subproject commit 4bf228dd0e246a466c403d31bd0158242c756f96 +Subproject commit 55fa585ff740c1bac9511228ec55a331997432ab diff --git a/openslides-backend b/openslides-backend index eb1e7fd9c..6f39f6074 160000 --- a/openslides-backend +++ b/openslides-backend @@ -1 +1 @@ -Subproject commit eb1e7fd9ce2463c1a40a33b1fc65fbcc503035d1 +Subproject commit 6f39f6074da8f4936f3127652097a4d1bafb4e32 diff --git a/openslides-client b/openslides-client index ad6032397..ce2b08ef5 160000 --- a/openslides-client +++ b/openslides-client @@ -1 +1 @@ -Subproject commit ad603239769e00345aa7855cb45ac71dfb08e396 +Subproject commit ce2b08ef5fc0d677a6a4f20c0ed9ec120181cadf