commit
b85a82bc61
57
.github/workflows/models.yml
vendored
57
.github/workflows/models.yml
vendored
@ -2,34 +2,12 @@
|
|||||||
name: Validate models.yml and initial and example data
|
name: Validate models.yml and initial and example data
|
||||||
on: [push, pull_request]
|
on: [push, pull_request]
|
||||||
env:
|
env:
|
||||||
PYTHON_VERSION: 3.8.5
|
PYTHON_VERSION: 3.9.6
|
||||||
GO_VERSION: 1.16
|
|
||||||
jobs:
|
jobs:
|
||||||
validate-models:
|
validate-models:
|
||||||
name: Validate models.yml
|
name: Validate models.yml
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Set up Go
|
|
||||||
uses: actions/setup-go@v1
|
|
||||||
with:
|
|
||||||
go-version: ${{ env.GO_VERSION }}
|
|
||||||
|
|
||||||
- name: Check out code
|
|
||||||
uses: actions/checkout@v2
|
|
||||||
|
|
||||||
- name: Build validator
|
|
||||||
run: go build
|
|
||||||
working-directory: docs/modelsvalidator
|
|
||||||
env:
|
|
||||||
GO111MODULE: on
|
|
||||||
|
|
||||||
- name: Validate models.yml
|
|
||||||
run: docs/modelsvalidator/modelsvalidator docs/models.yml
|
|
||||||
|
|
||||||
validate-data:
|
|
||||||
name: Validate example-data.json and initial-data.json
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Check out code
|
- name: Check out code
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
|
|
||||||
@ -39,9 +17,32 @@ jobs:
|
|||||||
python-version: ${{ env.PYTHON_VERSION }}
|
python-version: ${{ env.PYTHON_VERSION }}
|
||||||
|
|
||||||
- name: Install requirements
|
- name: Install requirements
|
||||||
working-directory: docs/datavalidator
|
run: pip install -U -r docs/modelsvalidator/requirements.txt
|
||||||
run: pip install -U -r requirements.txt
|
|
||||||
|
|
||||||
- name: Validate
|
- name: Validate models.yml
|
||||||
working-directory: docs/datavalidator
|
working-directory: docs/modelsvalidator
|
||||||
run: python check_json.py
|
run: python validate.py
|
||||||
|
|
||||||
|
validate-data:
|
||||||
|
name: Validate example-data.json and initial-data.json
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Check out code
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
|
||||||
|
- name: Checkout backend submodule
|
||||||
|
run: git submodule update --init openslides-backend/
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v2
|
||||||
|
with:
|
||||||
|
python-version: ${{ env.PYTHON_VERSION }}
|
||||||
|
|
||||||
|
- name: Install requirements
|
||||||
|
run: pip install -U -r openslides-backend/cli/requirements.txt
|
||||||
|
|
||||||
|
- name: set pythonpath
|
||||||
|
run: echo "PYTHONPATH=openslides-backend" >> $GITHUB_ENV
|
||||||
|
|
||||||
|
- name: Validate example-data.json
|
||||||
|
run: python openslides-backend/cli/check_json.py docs/example-data.json docker/initial-data.json
|
||||||
|
@ -1,668 +0,0 @@
|
|||||||
import json
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
from collections import defaultdict
|
|
||||||
from typing import Any, Callable, Dict, List, Optional, Set, Tuple
|
|
||||||
|
|
||||||
import fastjsonschema
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
MODELS_YML_PATH = "../../docs/models.yml"
|
|
||||||
|
|
||||||
DEFAULT_FILES = [
|
|
||||||
"../../docker/initial-data.json",
|
|
||||||
"../../docs/example-data.json",
|
|
||||||
]
|
|
||||||
|
|
||||||
SCHEMA = fastjsonschema.compile(
|
|
||||||
{
|
|
||||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
||||||
"title": "Schema for initial and example data.",
|
|
||||||
"type": "object",
|
|
||||||
"patternProperties": {
|
|
||||||
"^[a-z_]+$": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {"id": {"type": "number"}},
|
|
||||||
"required": ["id"],
|
|
||||||
},
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"additionalProperties": False,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class CheckException(Exception):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def check_string(value: Any) -> bool:
|
|
||||||
return value is None or isinstance(value, str)
|
|
||||||
|
|
||||||
|
|
||||||
color_regex = re.compile("^#[0-9a-f]{6}$")
|
|
||||||
|
|
||||||
|
|
||||||
def check_color(value: Any) -> bool:
|
|
||||||
return value is None or bool(isinstance(value, str) and color_regex.match(value))
|
|
||||||
|
|
||||||
|
|
||||||
def check_number(value: Any) -> bool:
|
|
||||||
return value is None or type(value) == int
|
|
||||||
|
|
||||||
|
|
||||||
def check_float(value: Any) -> bool:
|
|
||||||
return value is None or type(value) in (int, float)
|
|
||||||
|
|
||||||
|
|
||||||
def check_boolean(value: Any) -> bool:
|
|
||||||
return value is None or value is False or value is True
|
|
||||||
|
|
||||||
|
|
||||||
def check_string_list(value: Any) -> bool:
|
|
||||||
return check_x_list(value, check_string)
|
|
||||||
|
|
||||||
|
|
||||||
def check_number_list(value: Any) -> bool:
|
|
||||||
return check_x_list(value, check_number)
|
|
||||||
|
|
||||||
|
|
||||||
def check_x_list(value: Any, fn: Callable) -> bool:
|
|
||||||
if value is None:
|
|
||||||
return True
|
|
||||||
if not isinstance(value, list):
|
|
||||||
return False
|
|
||||||
return all([fn(sv) for sv in value])
|
|
||||||
|
|
||||||
|
|
||||||
def check_decimal(value: Any) -> bool:
|
|
||||||
if value is None:
|
|
||||||
return True
|
|
||||||
if isinstance(value, str):
|
|
||||||
pattern = r"^-?(\d|[1-9]\d+)\.\d{6}$"
|
|
||||||
if re.match(pattern, value):
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def check_json(value: Any, root: bool = True) -> bool:
|
|
||||||
if value is None:
|
|
||||||
return True
|
|
||||||
if not root and (isinstance(value, int) or isinstance(value, str)):
|
|
||||||
return True
|
|
||||||
if isinstance(value, list):
|
|
||||||
return all(check_json(x, root=False) for x in value)
|
|
||||||
if isinstance(value, dict):
|
|
||||||
return all(check_json(x, root=False) for x in value.values())
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
checker_mapping = {
|
|
||||||
"string": check_string,
|
|
||||||
"HTMLStrict": check_string,
|
|
||||||
"HTMLPermissive": check_string,
|
|
||||||
"generic-relation": check_string,
|
|
||||||
"number": check_number,
|
|
||||||
"timestamp": check_number,
|
|
||||||
"relation": check_number,
|
|
||||||
"float": check_float,
|
|
||||||
"boolean": check_boolean,
|
|
||||||
"string[]": check_string_list,
|
|
||||||
"generic-relation-list": check_string_list,
|
|
||||||
"number[]": check_number_list,
|
|
||||||
"relation-list": check_number_list,
|
|
||||||
"decimal(6)": check_decimal,
|
|
||||||
"color": check_color,
|
|
||||||
"JSON": check_json,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class Checker:
|
|
||||||
def __init__(self, data: Dict[str, List[Any]], is_import: bool = False) -> None:
|
|
||||||
self.data = data
|
|
||||||
|
|
||||||
with open(MODELS_YML_PATH, "rb") as x:
|
|
||||||
models_yml = x.read()
|
|
||||||
models_yml = models_yml.replace(" yes:".encode(), ' "yes":'.encode())
|
|
||||||
models_yml = models_yml.replace(" no:".encode(), ' "no":'.encode())
|
|
||||||
self.models = yaml.safe_load(models_yml)
|
|
||||||
if is_import:
|
|
||||||
self.modify_models_for_import()
|
|
||||||
|
|
||||||
self.errors: List[str] = []
|
|
||||||
|
|
||||||
self.template_prefixes: Dict[
|
|
||||||
str, Dict[str, Tuple[str, int, int]]
|
|
||||||
] = defaultdict(dict)
|
|
||||||
self.generate_template_prefixes()
|
|
||||||
|
|
||||||
def modify_models_for_import(self) -> None:
|
|
||||||
collection_allowlist = (
|
|
||||||
"user",
|
|
||||||
"meeting",
|
|
||||||
"group",
|
|
||||||
"personal_note",
|
|
||||||
"tag",
|
|
||||||
"agenda_item",
|
|
||||||
"list_of_speakers",
|
|
||||||
"speaker",
|
|
||||||
"topic",
|
|
||||||
"motion",
|
|
||||||
"motion_submitter",
|
|
||||||
"motion_comment",
|
|
||||||
"motion_comment_section",
|
|
||||||
"motion_category",
|
|
||||||
"motion_block",
|
|
||||||
"motion_change_recommendation",
|
|
||||||
"motion_state",
|
|
||||||
"motion_workflow",
|
|
||||||
"motion_statute_paragraph",
|
|
||||||
"poll",
|
|
||||||
"option",
|
|
||||||
"vote",
|
|
||||||
"assignment",
|
|
||||||
"assignment_candidate",
|
|
||||||
"mediafile",
|
|
||||||
"projector",
|
|
||||||
"projection",
|
|
||||||
"projector_message",
|
|
||||||
"projector_countdown",
|
|
||||||
"chat_group",
|
|
||||||
)
|
|
||||||
for collection in list(self.models.keys()):
|
|
||||||
if collection not in collection_allowlist:
|
|
||||||
del self.models[collection]
|
|
||||||
self.models["mediafile"]["blob"] = "string"
|
|
||||||
|
|
||||||
def generate_template_prefixes(self) -> None:
|
|
||||||
for collection in self.models.keys():
|
|
||||||
for field in self.models[collection]:
|
|
||||||
if not self.is_template_field(field):
|
|
||||||
continue
|
|
||||||
parts = field.split("$")
|
|
||||||
prefix = parts[0]
|
|
||||||
suffix = parts[1]
|
|
||||||
if prefix in self.template_prefixes[collection]:
|
|
||||||
raise ValueError(
|
|
||||||
f"the template prefix {prefix} is not unique within {collection}"
|
|
||||||
)
|
|
||||||
self.template_prefixes[collection][prefix] = (
|
|
||||||
field,
|
|
||||||
len(prefix),
|
|
||||||
len(suffix),
|
|
||||||
)
|
|
||||||
|
|
||||||
def is_template_field(self, field: str) -> bool:
|
|
||||||
return "$_" in field or field.endswith("$")
|
|
||||||
|
|
||||||
def is_structured_field(self, field: str) -> bool:
|
|
||||||
return "$" in field and not self.is_template_field(field)
|
|
||||||
|
|
||||||
def is_normal_field(self, field: str) -> bool:
|
|
||||||
return "$" not in field
|
|
||||||
|
|
||||||
def is_calculated_field(self, field: str) -> bool:
|
|
||||||
return field == "content"
|
|
||||||
|
|
||||||
def make_structured(self, field: str, replacement: Any) -> str:
|
|
||||||
if type(replacement) not in (str, int):
|
|
||||||
raise CheckException(
|
|
||||||
f"Invalid type {type(replacement)} for the replacement of field {field}"
|
|
||||||
)
|
|
||||||
parts = field.split("$")
|
|
||||||
return parts[0] + "$" + str(replacement) + parts[1]
|
|
||||||
|
|
||||||
def to_template_field(
|
|
||||||
self, collection: str, structured_field: str
|
|
||||||
) -> Tuple[str, str]:
|
|
||||||
"""Returns template_field, replacement"""
|
|
||||||
parts = structured_field.split("$")
|
|
||||||
descriptor = self.template_prefixes[collection].get(parts[0])
|
|
||||||
if not descriptor:
|
|
||||||
raise CheckException(
|
|
||||||
f"Unknown template field for prefix {parts[0]} in collection {collection}"
|
|
||||||
)
|
|
||||||
return (
|
|
||||||
descriptor[0],
|
|
||||||
structured_field[descriptor[1] + 1 : len(structured_field) - descriptor[2]],
|
|
||||||
)
|
|
||||||
|
|
||||||
def run_check(self) -> None:
|
|
||||||
self.check_json()
|
|
||||||
self.check_collections()
|
|
||||||
for collection, models in self.data.items():
|
|
||||||
for model in models:
|
|
||||||
self.check_model(collection, model)
|
|
||||||
if self.errors:
|
|
||||||
errors = [f"\t{error}" for error in self.errors]
|
|
||||||
raise CheckException("\n".join(errors))
|
|
||||||
|
|
||||||
def check_json(self) -> None:
|
|
||||||
try:
|
|
||||||
SCHEMA(self.data)
|
|
||||||
except fastjsonschema.exceptions.JsonSchemaException as e:
|
|
||||||
raise CheckException(f"JSON does not match schema: {str(e)}")
|
|
||||||
|
|
||||||
def check_collections(self) -> None:
|
|
||||||
c1 = set(self.data.keys())
|
|
||||||
c2 = set(self.models.keys())
|
|
||||||
if c1 != c2:
|
|
||||||
err = "Collections in JSON file do not match with models.yml."
|
|
||||||
if c2 - c1:
|
|
||||||
err += f" Missing collections: {', '.join(c2-c1)}."
|
|
||||||
if c1 - c2:
|
|
||||||
err += f" Invalid collections: {', '.join(c1-c2)}."
|
|
||||||
raise CheckException(err)
|
|
||||||
|
|
||||||
def check_model(self, collection: str, model: Dict[str, Any]) -> None:
|
|
||||||
errors = self.check_normal_fields(model, collection)
|
|
||||||
|
|
||||||
if not errors:
|
|
||||||
errors = self.check_template_fields(model, collection)
|
|
||||||
|
|
||||||
if not errors:
|
|
||||||
self.check_types(model, collection)
|
|
||||||
self.check_relations(model, collection)
|
|
||||||
|
|
||||||
def check_normal_fields(self, model: Dict[str, Any], collection: str) -> bool:
|
|
||||||
model_fields = set(
|
|
||||||
x
|
|
||||||
for x in model.keys()
|
|
||||||
if self.is_normal_field(x) or self.is_template_field(x)
|
|
||||||
)
|
|
||||||
collection_fields = set(
|
|
||||||
x
|
|
||||||
for x in self.models[collection].keys()
|
|
||||||
if self.is_normal_field(x) or self.is_template_field(x)
|
|
||||||
)
|
|
||||||
calculated_fields = set(
|
|
||||||
x
|
|
||||||
for x in self.models[collection].keys()
|
|
||||||
if self.is_calculated_field(x)
|
|
||||||
)
|
|
||||||
|
|
||||||
errors = False
|
|
||||||
if collection_fields - model_fields - calculated_fields:
|
|
||||||
error = f"{collection}/{model['id']}: Missing fields {', '.join(collection_fields - model_fields)}"
|
|
||||||
self.errors.append(error)
|
|
||||||
errors = True
|
|
||||||
if model_fields - collection_fields:
|
|
||||||
error = f"{collection}/{model['id']}: Invalid fields {', '.join(model_fields - collection_fields)}"
|
|
||||||
self.errors.append(error)
|
|
||||||
errors = True
|
|
||||||
return errors
|
|
||||||
|
|
||||||
def check_template_fields(self, model: Dict[str, Any], collection: str) -> bool:
|
|
||||||
"""
|
|
||||||
Only checks that for each replacement a structured field exists and
|
|
||||||
not too many structured fields. Does not check the content.
|
|
||||||
Returns True on errors.
|
|
||||||
"""
|
|
||||||
errors = False
|
|
||||||
for template_field in self.models[collection].keys():
|
|
||||||
if not self.is_template_field(template_field):
|
|
||||||
continue
|
|
||||||
field_error = False
|
|
||||||
replacements = model[template_field]
|
|
||||||
if not isinstance(replacements, list):
|
|
||||||
self.errors.append(
|
|
||||||
f"{collection}/{model['id']}/{template_field}: Replacements for the template field must be a list"
|
|
||||||
)
|
|
||||||
field_error = True
|
|
||||||
continue
|
|
||||||
for replacement in replacements:
|
|
||||||
if not isinstance(replacement, str):
|
|
||||||
self.errors.append(
|
|
||||||
f"{collection}/{model['id']}/{template_field}: Each replacement for the template field must be a string"
|
|
||||||
)
|
|
||||||
field_error = True
|
|
||||||
if field_error:
|
|
||||||
error = True
|
|
||||||
continue
|
|
||||||
|
|
||||||
replacement_collection = None
|
|
||||||
field_description = self.models[collection][template_field]
|
|
||||||
if isinstance(field_description, dict):
|
|
||||||
replacement_collection = field_description.get("replacement_collection")
|
|
||||||
|
|
||||||
for replacement in replacements:
|
|
||||||
structured_field = self.make_structured(template_field, replacement)
|
|
||||||
if structured_field not in model:
|
|
||||||
self.errors.append(
|
|
||||||
f"{collection}/{model['id']}/{template_field}: Missing {structured_field} since it is given as a replacement"
|
|
||||||
)
|
|
||||||
errors = True
|
|
||||||
|
|
||||||
if replacement_collection:
|
|
||||||
try:
|
|
||||||
as_id = int(replacement)
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
self.errors.append(
|
|
||||||
f"{collection}/{model['id']}/{template_field}: Replacement {replacement} is not an integer"
|
|
||||||
)
|
|
||||||
if not self.find_model(replacement_collection, as_id):
|
|
||||||
self.errors.append(
|
|
||||||
f"{collection}/{model['id']}/{template_field}: Replacement {replacement} does not exist as a model of collection {replacement_collection}"
|
|
||||||
)
|
|
||||||
|
|
||||||
for field in model.keys():
|
|
||||||
if self.is_structured_field(field):
|
|
||||||
try:
|
|
||||||
_template_field, _replacement = self.to_template_field(
|
|
||||||
collection, field
|
|
||||||
)
|
|
||||||
if (
|
|
||||||
template_field == _template_field
|
|
||||||
and _replacement not in model[template_field]
|
|
||||||
):
|
|
||||||
self.errors.append(
|
|
||||||
f"{collection}/{model['id']}/{field}: Invalid structured field. Missing replacement {_replacement} in {template_field}"
|
|
||||||
)
|
|
||||||
errors = True
|
|
||||||
except CheckException as e:
|
|
||||||
self.errors.append(
|
|
||||||
f"{collection}/{model['id']}/{field} error: " + str(e)
|
|
||||||
)
|
|
||||||
errors = True
|
|
||||||
|
|
||||||
return errors
|
|
||||||
|
|
||||||
def check_types(self, model: Dict[str, Any], collection: str) -> None:
|
|
||||||
for field in model.keys():
|
|
||||||
if self.is_template_field(field):
|
|
||||||
continue
|
|
||||||
|
|
||||||
field_type = self.get_type_from_collection(field, collection)
|
|
||||||
enum = self.get_enum_from_collection_field(field, collection)
|
|
||||||
checker = checker_mapping.get(field_type)
|
|
||||||
if checker is None:
|
|
||||||
raise NotImplementedError(
|
|
||||||
f"TODO implement check for field type {field_type}"
|
|
||||||
)
|
|
||||||
|
|
||||||
if not checker(model[field]):
|
|
||||||
error = f"{collection}/{model['id']}/{field}: Type error: Type is not {field_type}"
|
|
||||||
self.errors.append(error)
|
|
||||||
|
|
||||||
if enum and model[field] not in enum:
|
|
||||||
error = f"{collection}/{model['id']}/{field}: Value error: Value {model[field]} is not a valid enum value"
|
|
||||||
self.errors.append(error)
|
|
||||||
|
|
||||||
def get_type_from_collection(self, field: str, collection: str) -> str:
|
|
||||||
if self.is_structured_field(field):
|
|
||||||
field, _ = self.to_template_field(collection, field)
|
|
||||||
|
|
||||||
field_description = self.models[collection][field]
|
|
||||||
if isinstance(field_description, dict):
|
|
||||||
if field_description["type"] == "template":
|
|
||||||
if isinstance(field_description["fields"], dict):
|
|
||||||
return field_description["fields"]["type"]
|
|
||||||
return field_description["fields"]
|
|
||||||
return field_description["type"]
|
|
||||||
return field_description
|
|
||||||
|
|
||||||
field_description = self.get_field_description(field, collection)
|
|
||||||
if field_description:
|
|
||||||
return field_description["type"]
|
|
||||||
return self.models[collection][field]
|
|
||||||
|
|
||||||
def get_enum_from_collection_field(
|
|
||||||
self, field: str, collection: str
|
|
||||||
) -> Optional[Set[str]]:
|
|
||||||
if self.is_structured_field(field):
|
|
||||||
field, _ = self.to_template_field(collection, field)
|
|
||||||
|
|
||||||
field_description = self.models[collection][field]
|
|
||||||
if isinstance(field_description, dict):
|
|
||||||
if field_description["type"] == "template":
|
|
||||||
if isinstance(field_description["fields"], dict):
|
|
||||||
field_description = field_description["fields"]
|
|
||||||
else:
|
|
||||||
return None
|
|
||||||
if "enum" in field_description:
|
|
||||||
enum = set(field_description["enum"])
|
|
||||||
if not field_description.get("required", False):
|
|
||||||
enum.add(None)
|
|
||||||
return enum
|
|
||||||
return None
|
|
||||||
|
|
||||||
def check_relations(self, model: Dict[str, Any], collection: str) -> None:
|
|
||||||
for field in model.keys():
|
|
||||||
try:
|
|
||||||
self.check_relation(model, collection, field)
|
|
||||||
except CheckException as e:
|
|
||||||
self.errors.append(
|
|
||||||
f"{collection}/{model['id']}/{field} error: " + str(e)
|
|
||||||
)
|
|
||||||
|
|
||||||
def check_relation(
|
|
||||||
self, model: Dict[str, Any], collection: str, field: str
|
|
||||||
) -> None:
|
|
||||||
if self.is_template_field(field):
|
|
||||||
return
|
|
||||||
|
|
||||||
field_type = self.get_type_from_collection(field, collection)
|
|
||||||
basemsg = f"{collection}/{model['id']}/{field}: Relation Error: "
|
|
||||||
|
|
||||||
replacement = None
|
|
||||||
if self.is_structured_field(field):
|
|
||||||
_, replacement = self.to_template_field(collection, field)
|
|
||||||
|
|
||||||
if field_type == "relation":
|
|
||||||
foreign_id = model[field]
|
|
||||||
if foreign_id is None:
|
|
||||||
return
|
|
||||||
|
|
||||||
foreign_collection, foreign_field = self.get_to(field, collection)
|
|
||||||
|
|
||||||
self.check_reverse_relation(
|
|
||||||
collection,
|
|
||||||
model["id"],
|
|
||||||
model,
|
|
||||||
foreign_collection,
|
|
||||||
foreign_id,
|
|
||||||
foreign_field,
|
|
||||||
basemsg,
|
|
||||||
replacement,
|
|
||||||
)
|
|
||||||
|
|
||||||
elif field_type == "relation-list":
|
|
||||||
foreign_ids = model[field]
|
|
||||||
if foreign_ids is None:
|
|
||||||
return
|
|
||||||
|
|
||||||
foreign_collection, foreign_field = self.get_to(field, collection)
|
|
||||||
|
|
||||||
for foreign_id in foreign_ids:
|
|
||||||
self.check_reverse_relation(
|
|
||||||
collection,
|
|
||||||
model["id"],
|
|
||||||
model,
|
|
||||||
foreign_collection,
|
|
||||||
foreign_id,
|
|
||||||
foreign_field,
|
|
||||||
basemsg,
|
|
||||||
replacement,
|
|
||||||
)
|
|
||||||
|
|
||||||
elif field_type == "generic-relation" and model[field] is not None:
|
|
||||||
foreign_collection, foreign_id = self.split_fqid(model[field])
|
|
||||||
foreign_field = self.get_to_generic_case(
|
|
||||||
collection, field, foreign_collection
|
|
||||||
)
|
|
||||||
|
|
||||||
self.check_reverse_relation(
|
|
||||||
collection,
|
|
||||||
model["id"],
|
|
||||||
model,
|
|
||||||
foreign_collection,
|
|
||||||
foreign_id,
|
|
||||||
foreign_field,
|
|
||||||
basemsg,
|
|
||||||
replacement,
|
|
||||||
)
|
|
||||||
|
|
||||||
elif field_type == "generic-relation-list" and model[field] is not None:
|
|
||||||
for fqid in model[field]:
|
|
||||||
foreign_collection, foreign_id = self.split_fqid(fqid)
|
|
||||||
foreign_field = self.get_to_generic_case(
|
|
||||||
collection, field, foreign_collection
|
|
||||||
)
|
|
||||||
|
|
||||||
self.check_reverse_relation(
|
|
||||||
collection,
|
|
||||||
model["id"],
|
|
||||||
model,
|
|
||||||
foreign_collection,
|
|
||||||
foreign_id,
|
|
||||||
foreign_field,
|
|
||||||
basemsg,
|
|
||||||
replacement,
|
|
||||||
)
|
|
||||||
|
|
||||||
def get_to(self, field: str, collection: str) -> Tuple[str, str]:
|
|
||||||
if self.is_structured_field(field):
|
|
||||||
field, _ = self.to_template_field(collection, field)
|
|
||||||
|
|
||||||
field_description = self.models[collection][field]
|
|
||||||
if field_description["type"] == "template":
|
|
||||||
to = field_description["fields"]["to"]
|
|
||||||
else:
|
|
||||||
to = field_description["to"]
|
|
||||||
return to.split("/")
|
|
||||||
|
|
||||||
def find_model(self, collection: str, id: int) -> Optional[Dict[str, Any]]:
|
|
||||||
c = self.data.get(collection, [])
|
|
||||||
for model in c:
|
|
||||||
if model["id"] == id:
|
|
||||||
return model
|
|
||||||
return None
|
|
||||||
|
|
||||||
def check_reverse_relation(
|
|
||||||
self,
|
|
||||||
collection: str,
|
|
||||||
id: int,
|
|
||||||
model: Dict[str, Any],
|
|
||||||
foreign_collection: str,
|
|
||||||
foreign_id: int,
|
|
||||||
foreign_field: str,
|
|
||||||
basemsg: str,
|
|
||||||
replacement: Optional[str],
|
|
||||||
) -> None:
|
|
||||||
actual_foreign_field = foreign_field
|
|
||||||
if self.is_template_field(foreign_field):
|
|
||||||
if replacement:
|
|
||||||
actual_foreign_field = self.make_structured(foreign_field, replacement)
|
|
||||||
else:
|
|
||||||
replacement_collection = self.models[foreign_collection][foreign_field][
|
|
||||||
"replacement_collection"
|
|
||||||
]
|
|
||||||
replacement = model.get(f"{replacement_collection}_id")
|
|
||||||
if not replacement:
|
|
||||||
self.errors.append(
|
|
||||||
f"{basemsg} points to {foreign_collection}/{foreign_id}/{foreign_field},"
|
|
||||||
f" but there is no replacement for {replacement_collection}"
|
|
||||||
)
|
|
||||||
actual_foreign_field = self.make_structured(foreign_field, replacement)
|
|
||||||
|
|
||||||
foreign_model = self.find_model(foreign_collection, foreign_id)
|
|
||||||
foreign_value = (
|
|
||||||
foreign_model.get(actual_foreign_field)
|
|
||||||
if foreign_model is not None
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
foreign_field_type = self.get_type_from_collection(
|
|
||||||
foreign_field, foreign_collection
|
|
||||||
)
|
|
||||||
fqid = f"{collection}/{id}"
|
|
||||||
error = False
|
|
||||||
if foreign_field_type == "relation":
|
|
||||||
error = foreign_value != id
|
|
||||||
elif foreign_field_type == "relation-list":
|
|
||||||
error = not foreign_value or id not in foreign_value
|
|
||||||
elif foreign_field_type == "generic-relation":
|
|
||||||
error = foreign_value != fqid
|
|
||||||
elif foreign_field_type == "generic-relation-list":
|
|
||||||
error = not foreign_value or fqid not in foreign_value
|
|
||||||
else:
|
|
||||||
raise NotImplementedError()
|
|
||||||
|
|
||||||
if error:
|
|
||||||
self.errors.append(
|
|
||||||
f"{basemsg} points to {foreign_collection}/{foreign_id}/{actual_foreign_field},"
|
|
||||||
" but the reverse relation for is corrupt"
|
|
||||||
)
|
|
||||||
|
|
||||||
def split_fqid(self, fqid: str) -> Tuple[str, int]:
|
|
||||||
try:
|
|
||||||
collection, _id = fqid.split("/")
|
|
||||||
id = int(_id)
|
|
||||||
if collection not in self.models.keys():
|
|
||||||
raise CheckException(f"Fqid {fqid} has an invalid collection")
|
|
||||||
return collection, id
|
|
||||||
except (ValueError, AttributeError):
|
|
||||||
raise CheckException(f"Fqid {fqid} is malformed")
|
|
||||||
|
|
||||||
def split_collectionfield(self, collectionfield: str) -> Tuple[str, str]:
|
|
||||||
collection, field = collectionfield.split("/")
|
|
||||||
if collection not in self.models.keys():
|
|
||||||
raise CheckException(
|
|
||||||
f"Collectionfield {collectionfield} has an invalid collection"
|
|
||||||
)
|
|
||||||
if (
|
|
||||||
field not in self.models[collection]
|
|
||||||
): # Note: this has to be adopted when supporting template fields
|
|
||||||
raise CheckException(
|
|
||||||
f"Collectionfield {collectionfield} has an invalid field"
|
|
||||||
)
|
|
||||||
return collection, field
|
|
||||||
|
|
||||||
def get_to_generic_case(
|
|
||||||
self, collection: str, field: str, foreign_collection: str
|
|
||||||
) -> str:
|
|
||||||
"""Returns all reverse relations as collectionfields"""
|
|
||||||
to = self.models[collection][field]["to"]
|
|
||||||
if isinstance(to, dict):
|
|
||||||
if foreign_collection not in to["collections"]:
|
|
||||||
raise CheckException(
|
|
||||||
f"The collection {foreign_collection} is not supported "
|
|
||||||
"as a reverse relation in {collection}/{field}"
|
|
||||||
)
|
|
||||||
return to["field"]
|
|
||||||
|
|
||||||
for cf in to:
|
|
||||||
c, f = self.split_collectionfield(cf)
|
|
||||||
if c == foreign_collection:
|
|
||||||
return f
|
|
||||||
|
|
||||||
raise CheckException(
|
|
||||||
f"The collection {foreign_collection} is not supported as a reverse relation in {collection}/{field}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
files = sys.argv[1:]
|
|
||||||
if not files:
|
|
||||||
files = DEFAULT_FILES
|
|
||||||
|
|
||||||
is_import = "--import" in files
|
|
||||||
if is_import:
|
|
||||||
files = [x for x in files if x != "--import"]
|
|
||||||
|
|
||||||
failed = False
|
|
||||||
for f in files:
|
|
||||||
with open(f) as data:
|
|
||||||
try:
|
|
||||||
Checker(json.load(data), is_import=is_import).run_check()
|
|
||||||
except CheckException as e:
|
|
||||||
print(f"Check for {f} failed:\n", e)
|
|
||||||
failed = True
|
|
||||||
else:
|
|
||||||
print(f"Check for {f} successful.")
|
|
||||||
return 1 if failed else 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
File diff suppressed because one or more lines are too long
@ -1,2 +0,0 @@
|
|||||||
fastjsonschema
|
|
||||||
pyyaml
|
|
@ -52,7 +52,7 @@
|
|||||||
# for all the fields that come from the template field.
|
# for all the fields that come from the template field.
|
||||||
# JSON Schema Properties:
|
# JSON Schema Properties:
|
||||||
# - You can add JSON Schema properties to the fields like `enum`, `description`,
|
# - You can add JSON Schema properties to the fields like `enum`, `description`,
|
||||||
# `maxLength` and `minimum`
|
# `items`, `maxLength` and `minimum`
|
||||||
# Additional properties:
|
# Additional properties:
|
||||||
# - The property `read_only` describes a field that can not be changed by an action.
|
# - The property `read_only` describes a field that can not be changed by an action.
|
||||||
# - The property `default` describes the default value that is used for new objects.
|
# - The property `default` describes the default value that is used for new objects.
|
||||||
@ -60,6 +60,9 @@
|
|||||||
# string. If this field is given it must have some content.
|
# string. If this field is given it must have some content.
|
||||||
# - The property `equal_fields` describes fields that must have the same value in
|
# - The property `equal_fields` describes fields that must have the same value in
|
||||||
# the instance and the related instance.
|
# the instance and the related instance.
|
||||||
|
# Restriction Mode:
|
||||||
|
# The field `restriction_mode` is required for every field. It puts the field into a
|
||||||
|
# restriction group. See https://github.com/OpenSlides/OpenSlides/wiki/Restrictions-Overview
|
||||||
|
|
||||||
organization:
|
organization:
|
||||||
id:
|
id:
|
||||||
@ -463,6 +466,7 @@ meeting:
|
|||||||
restriction_mode: B
|
restriction_mode: B
|
||||||
imported_at:
|
imported_at:
|
||||||
type: timestamp
|
type: timestamp
|
||||||
|
restriction_mode: B
|
||||||
|
|
||||||
# Configuration (only for the server owner)
|
# Configuration (only for the server owner)
|
||||||
jitsi_domain:
|
jitsi_domain:
|
||||||
@ -2414,10 +2418,10 @@ option:
|
|||||||
text:
|
text:
|
||||||
type: HTMLStrict
|
type: HTMLStrict
|
||||||
restriction_mode: A
|
restriction_mode: A
|
||||||
yes:
|
"yes":
|
||||||
type: decimal(6)
|
type: decimal(6)
|
||||||
restriction_mode: B
|
restriction_mode: B
|
||||||
no:
|
"no":
|
||||||
type: decimal(6)
|
type: decimal(6)
|
||||||
restriction_mode: B
|
restriction_mode: B
|
||||||
abstain:
|
abstain:
|
||||||
@ -2610,7 +2614,7 @@ mediafile:
|
|||||||
restriction_mode: B
|
restriction_mode: B
|
||||||
filename:
|
filename:
|
||||||
type: string
|
type: string
|
||||||
descriptin: The uploaded filename. Will be used for downloading. Only writeable on create.
|
description: The uploaded filename. Will be used for downloading. Only writeable on create.
|
||||||
restriction_mode: B
|
restriction_mode: B
|
||||||
mimetype:
|
mimetype:
|
||||||
type: string
|
type: string
|
||||||
@ -2818,6 +2822,11 @@ projection:
|
|||||||
type: string
|
type: string
|
||||||
restriction_mode: A
|
restriction_mode: A
|
||||||
|
|
||||||
|
content:
|
||||||
|
type: JSON
|
||||||
|
calculated: true
|
||||||
|
restriction_mode: A
|
||||||
|
|
||||||
current_projector_id:
|
current_projector_id:
|
||||||
type: relation
|
type: relation
|
||||||
to: projector/current_projection_ids
|
to: projector/current_projection_ids
|
||||||
@ -2855,10 +2864,6 @@ projection:
|
|||||||
to: meeting/all_projection_ids
|
to: meeting/all_projection_ids
|
||||||
required: true
|
required: true
|
||||||
restriction_mode: A
|
restriction_mode: A
|
||||||
content:
|
|
||||||
type: JSON
|
|
||||||
calculated: true
|
|
||||||
restriction_mode: A
|
|
||||||
|
|
||||||
projector_message:
|
projector_message:
|
||||||
id:
|
id:
|
||||||
|
@ -1,21 +0,0 @@
|
|||||||
# Modelsvalidator
|
|
||||||
|
|
||||||
Modelsvalidator is a tool to validate the models.yml file
|
|
||||||
|
|
||||||
|
|
||||||
## Run
|
|
||||||
|
|
||||||
Build first: `go build ./...`.
|
|
||||||
|
|
||||||
The tool requires the content of the models.yml. It can be provided via stdin, a
|
|
||||||
file system path or an url starting with http:// or https://.
|
|
||||||
|
|
||||||
|
|
||||||
```
|
|
||||||
cat models.yml | modelsvalidator
|
|
||||||
modelsvalidator openslides/docs/models.yml
|
|
||||||
modelsvalidator https://raw.githubusercontent.com/OpenSlides/OpenSlides/openslides4-dev/docs/models.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
The tool returns with status code 0 and no content, if the given content is
|
|
||||||
valid. It returns with a positive status code and some error messages if not.
|
|
@ -1,159 +0,0 @@
|
|||||||
package check
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
models "github.com/OpenSlides/openslides-models-to-go"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Check runs some checks on the given models.
|
|
||||||
func Check(data map[string]models.Model) error {
|
|
||||||
validators := []func(map[string]models.Model) error{
|
|
||||||
validateTypes,
|
|
||||||
validateRelations,
|
|
||||||
validateTemplatePrefixes,
|
|
||||||
}
|
|
||||||
|
|
||||||
errors := new(ErrorList)
|
|
||||||
for _, v := range validators {
|
|
||||||
if err := v(data); err != nil {
|
|
||||||
errors.append(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !errors.empty() {
|
|
||||||
return errors
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateTypes(data map[string]models.Model) error {
|
|
||||||
scalar := scalarTypes()
|
|
||||||
relation := relationTypes()
|
|
||||||
errs := &ErrorList{
|
|
||||||
Name: "type validator",
|
|
||||||
intent: 1,
|
|
||||||
}
|
|
||||||
for modelName, model := range data {
|
|
||||||
for fieldName, field := range model.Fields {
|
|
||||||
if scalar[strings.TrimSuffix(field.Type, "[]")] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if relation[field.Type] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
errs.append(fmt.Errorf("Unknown type `%s` in %s/%s", field.Type, modelName, fieldName))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if errs.empty() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return errs
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateRelations(data map[string]models.Model) error {
|
|
||||||
errs := &ErrorList{
|
|
||||||
Name: "relation validator",
|
|
||||||
intent: 1,
|
|
||||||
}
|
|
||||||
relation := relationTypes()
|
|
||||||
for modelName, model := range data {
|
|
||||||
Next:
|
|
||||||
for fieldName, field := range model.Fields {
|
|
||||||
r := field.Relation()
|
|
||||||
if r == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, c := range r.ToCollections() {
|
|
||||||
toModel, ok := data[c.Collection]
|
|
||||||
if !ok {
|
|
||||||
errs.append(fmt.Errorf("%s/%s directs to nonexisting model `%s`", modelName, fieldName, c.Collection))
|
|
||||||
continue Next
|
|
||||||
}
|
|
||||||
|
|
||||||
toField, ok := toModel.Fields[c.ToField.Name]
|
|
||||||
if !ok {
|
|
||||||
errs.append(fmt.Errorf("%s/%s directs to nonexisting collectionfield `%s/%s`", modelName, fieldName, c.Collection, c.ToField.Name))
|
|
||||||
continue Next
|
|
||||||
}
|
|
||||||
|
|
||||||
if !relation[toField.Type] {
|
|
||||||
errs.append(fmt.Errorf("%s/%s directs to `%s/%s`, but it is not a relation, but %s", modelName, fieldName, c.Collection, c.ToField.Name, toField.Type))
|
|
||||||
continue Next
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if errs.empty() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return errs
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateTemplatePrefixes(models map[string]models.Model) error {
|
|
||||||
errs := &ErrorList{
|
|
||||||
Name: "template prefixes validator",
|
|
||||||
intent: 1,
|
|
||||||
}
|
|
||||||
for modelName, model := range models {
|
|
||||||
prefixes := map[string]bool{}
|
|
||||||
for fieldName := range model.Fields {
|
|
||||||
i := strings.Index(fieldName, "$")
|
|
||||||
if i < 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
prefix := fieldName[0:i]
|
|
||||||
if prefixes[prefix] {
|
|
||||||
errs.append(fmt.Errorf("Duplicate template prefix %s in %s", prefix, modelName))
|
|
||||||
}
|
|
||||||
prefixes[prefix] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if errs.empty() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return errs
|
|
||||||
}
|
|
||||||
|
|
||||||
// scalarTypes are the main types. All scalarTypes can be used as a list.
|
|
||||||
// JSON[], timestamp[] etc.
|
|
||||||
func scalarTypes() map[string]bool {
|
|
||||||
s := []string{
|
|
||||||
"string",
|
|
||||||
"number",
|
|
||||||
"boolean",
|
|
||||||
"JSON",
|
|
||||||
"HTMLPermissive",
|
|
||||||
"HTMLStrict",
|
|
||||||
"float",
|
|
||||||
"decimal(6)",
|
|
||||||
"timestamp",
|
|
||||||
"color",
|
|
||||||
}
|
|
||||||
out := make(map[string]bool)
|
|
||||||
for _, t := range s {
|
|
||||||
out[t] = true
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// relationTypes are realtion types in realtion to other fields.
|
|
||||||
func relationTypes() map[string]bool {
|
|
||||||
s := []string{
|
|
||||||
"relation",
|
|
||||||
"relation-list",
|
|
||||||
"generic-relation",
|
|
||||||
"generic-relation-list",
|
|
||||||
"template",
|
|
||||||
}
|
|
||||||
out := make(map[string]bool)
|
|
||||||
for _, t := range s {
|
|
||||||
out[t] = true
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
@ -1,121 +0,0 @@
|
|||||||
package check_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/OpenSlides/Openslides/modelsvalidator/check"
|
|
||||||
models "github.com/OpenSlides/openslides-models-to-go"
|
|
||||||
)
|
|
||||||
|
|
||||||
const yamlUnknownFieldType = `---
|
|
||||||
some_model:
|
|
||||||
field: unknown
|
|
||||||
`
|
|
||||||
|
|
||||||
const yamlNonExistingModel = `---
|
|
||||||
some_model:
|
|
||||||
no_other_model:
|
|
||||||
type: relation
|
|
||||||
to: not_existing/field
|
|
||||||
`
|
|
||||||
|
|
||||||
const yamlNonExistingField = `---
|
|
||||||
some_model:
|
|
||||||
no_other_field:
|
|
||||||
type: relation
|
|
||||||
to: other_model/bar
|
|
||||||
other_model:
|
|
||||||
foo: string
|
|
||||||
`
|
|
||||||
|
|
||||||
const yamlDuplicateTemplatePrefix = `---
|
|
||||||
some_model:
|
|
||||||
field_$_1: number
|
|
||||||
field_$_2: number
|
|
||||||
`
|
|
||||||
|
|
||||||
const yamlWrongReverseRelaitonType = `---
|
|
||||||
some_model:
|
|
||||||
other_model:
|
|
||||||
type: relation
|
|
||||||
to: other_model/field
|
|
||||||
other_model:
|
|
||||||
field: HTMLStrict
|
|
||||||
`
|
|
||||||
|
|
||||||
func TestCheck(t *testing.T) {
|
|
||||||
for _, tt := range []struct {
|
|
||||||
name string
|
|
||||||
yaml string
|
|
||||||
err string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
"unknown type",
|
|
||||||
yamlUnknownFieldType,
|
|
||||||
"Unknown type `unknown` in some_model/field",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"non-existing model",
|
|
||||||
yamlNonExistingModel,
|
|
||||||
"some_model/no_other_model directs to nonexisting model `not_existing`",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"non-existing Field",
|
|
||||||
yamlNonExistingField,
|
|
||||||
"some_model/no_other_field directs to nonexisting collectionfield `other_model/bar`",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"duplicate template prefix",
|
|
||||||
yamlDuplicateTemplatePrefix,
|
|
||||||
"Duplicate template prefix field_ in some_model",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"wrong reverse relation type",
|
|
||||||
yamlWrongReverseRelaitonType,
|
|
||||||
"some_model/other_model directs to `other_model/field`, but it is not a relation, but HTMLStrict",
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
data, err := models.Unmarshal(strings.NewReader(tt.yaml))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Can not unmarshal yaml: %v", err)
|
|
||||||
}
|
|
||||||
gotErr := check.Check(data)
|
|
||||||
if tt.err == "" {
|
|
||||||
if gotErr != nil {
|
|
||||||
t.Errorf("Models.Check() returned an unexepcted error: %v", err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if gotErr == nil {
|
|
||||||
t.Fatalf("Models.Check() did not return an error, expected: %v", tt.err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var errList *check.ErrorList
|
|
||||||
if !errors.As(gotErr, &errList) {
|
|
||||||
t.Fatalf("Models.Check() did not return a ListError, got: %v", gotErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
var found bool
|
|
||||||
for _, err := range errList.Errs {
|
|
||||||
var errList *check.ErrorList
|
|
||||||
if !errors.As(err, &errList) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, err := range errList.Errs {
|
|
||||||
if err.Error() == tt.err {
|
|
||||||
found = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !found {
|
|
||||||
t.Errorf("Models.Check() returned:\n%v\n\nExpected something that contains:\n%v", gotErr, tt.err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,38 +0,0 @@
|
|||||||
package check
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ErrorList is an error that contains a list of other errors.
|
|
||||||
type ErrorList struct {
|
|
||||||
Name string
|
|
||||||
intent int
|
|
||||||
Errs []error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *ErrorList) append(err error) {
|
|
||||||
if err == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
e.Errs = append(e.Errs, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e ErrorList) Error() string {
|
|
||||||
intent := strings.Repeat(" ", e.intent)
|
|
||||||
var msgs []string
|
|
||||||
for _, err := range e.Errs {
|
|
||||||
msgs = append(msgs, fmt.Sprintf("%s* %v", intent, err))
|
|
||||||
}
|
|
||||||
msg := strings.Join(msgs, "\n")
|
|
||||||
if e.Name != "" {
|
|
||||||
return fmt.Sprintf("%s:\n%s", e.Name, msg)
|
|
||||||
}
|
|
||||||
return msg
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *ErrorList) empty() bool {
|
|
||||||
return len(e.Errs) == 0
|
|
||||||
}
|
|
@ -1,5 +0,0 @@
|
|||||||
module github.com/OpenSlides/Openslides/modelsvalidator
|
|
||||||
|
|
||||||
go 1.15
|
|
||||||
|
|
||||||
require github.com/OpenSlides/openslides-models-to-go v0.2.0
|
|
@ -1,6 +0,0 @@
|
|||||||
github.com/OpenSlides/openslides-models-to-go v0.2.0 h1:1D+rD94GcUZAvmHLRyzJCIj7wklU+JBkoYifwtHH08s=
|
|
||||||
github.com/OpenSlides/openslides-models-to-go v0.2.0/go.mod h1:CriCefW5smTixhFfVLiuA8NgyMX4PAU5e3YpJHnaZx8=
|
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
|
||||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
|
|
||||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
|
@ -1,56 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"log"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/OpenSlides/Openslides/modelsvalidator/check"
|
|
||||||
models "github.com/OpenSlides/openslides-models-to-go"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
var content io.Reader = os.Stdin
|
|
||||||
if len(os.Args) > 1 {
|
|
||||||
c, err := openModels(os.Args[1])
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("Can not load content: %v", err)
|
|
||||||
}
|
|
||||||
defer c.Close()
|
|
||||||
content = c
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := models.Unmarshal(content)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("Invalid model format: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := check.Check(data); err != nil {
|
|
||||||
log.Fatalf("Invalid model structure:\n\n%v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// openModels reads the model either from file or from an url.
|
|
||||||
func openModels(path string) (io.ReadCloser, error) {
|
|
||||||
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
|
|
||||||
return openModelsFromURL(path)
|
|
||||||
}
|
|
||||||
|
|
||||||
return os.Open(path)
|
|
||||||
}
|
|
||||||
|
|
||||||
func openModelsFromURL(url string) (io.ReadCloser, error) {
|
|
||||||
resp, err := http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("requesting models from url: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if resp.StatusCode != 200 {
|
|
||||||
return nil, fmt.Errorf("can not get models from url. Got status %s", resp.Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
return resp.Body, nil
|
|
||||||
}
|
|
1
docs/modelsvalidator/requirements.txt
Normal file
1
docs/modelsvalidator/requirements.txt
Normal file
@ -0,0 +1 @@
|
|||||||
|
pyyaml
|
375
docs/modelsvalidator/validate.py
Normal file
375
docs/modelsvalidator/validate.py
Normal file
@ -0,0 +1,375 @@
|
|||||||
|
import json
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from collections import defaultdict
|
||||||
|
from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
DEFAULT_FILES = [
|
||||||
|
"../../docs/models.yml",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
KEYSEPARATOR = "/"
|
||||||
|
_collection_regex = r"[a-z]([a-z_]+[a-z]+)?"
|
||||||
|
_field_regex = r"[a-z][a-z0-9_]*\$?[a-z0-9_]*"
|
||||||
|
|
||||||
|
collectionfield_regex = re.compile(
|
||||||
|
f"^({_collection_regex}){KEYSEPARATOR}({_field_regex})$"
|
||||||
|
)
|
||||||
|
collection_regex = re.compile(f"^{_collection_regex}$")
|
||||||
|
field_regex = re.compile(f"^{_field_regex}$")
|
||||||
|
|
||||||
|
decimal_regex = re.compile("^\d+\.\d{6}$")
|
||||||
|
color_regex = re.compile("^#[0-9a-f]{6}$")
|
||||||
|
|
||||||
|
RELATION_TYPES = (
|
||||||
|
"relation",
|
||||||
|
"relation-list",
|
||||||
|
"generic-relation",
|
||||||
|
"generic-relation-list",
|
||||||
|
)
|
||||||
|
|
||||||
|
DATA_TYPES = (
|
||||||
|
"string",
|
||||||
|
"number",
|
||||||
|
"string[]",
|
||||||
|
"number[]",
|
||||||
|
"boolean",
|
||||||
|
"JSON",
|
||||||
|
"HTMLStrict",
|
||||||
|
"HTMLPermissive",
|
||||||
|
"float",
|
||||||
|
"decimal(6)",
|
||||||
|
"timestamp",
|
||||||
|
"color",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
VALID_TYPES = DATA_TYPES + RELATION_TYPES + ("template",)
|
||||||
|
|
||||||
|
OPTIONAL_ATTRIBUTES = (
|
||||||
|
"description",
|
||||||
|
"calculated",
|
||||||
|
"required",
|
||||||
|
"read_only",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class CheckException(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class Checker:
|
||||||
|
def __init__(self, filepath: str) -> None:
|
||||||
|
with open(filepath, "rb") as x:
|
||||||
|
self.models = yaml.safe_load(x.read())
|
||||||
|
self.errors: List[str] = []
|
||||||
|
|
||||||
|
def run_check(self) -> None:
|
||||||
|
self._run_checks()
|
||||||
|
if self.errors:
|
||||||
|
errors = [f"\t{error}" for error in self.errors]
|
||||||
|
raise CheckException("\n".join(errors))
|
||||||
|
|
||||||
|
def _run_checks(self) -> None:
|
||||||
|
for collection in self.models.keys():
|
||||||
|
if not collection_regex.match(collection):
|
||||||
|
self.errors.append(f"Collection '{collection}' is not valid.")
|
||||||
|
if self.errors:
|
||||||
|
return
|
||||||
|
|
||||||
|
for collection, fields in self.models.items():
|
||||||
|
if not isinstance(fields, dict):
|
||||||
|
self.errors.append(
|
||||||
|
f"The fields of collection {collection} must be a dict."
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
for field_name, field in fields.items():
|
||||||
|
if not field_regex.match(field_name):
|
||||||
|
self.errors.append(
|
||||||
|
f"Field name '{field_name}' of collection {collection} is not a valid field name."
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
if not isinstance(field, dict):
|
||||||
|
self.errors.append(
|
||||||
|
f"Field '{field_name}' of collection {collection} must be a dict."
|
||||||
|
)
|
||||||
|
self.check_field(collection, field_name, field)
|
||||||
|
|
||||||
|
if self.errors:
|
||||||
|
return
|
||||||
|
|
||||||
|
for collection, fields in self.models.items():
|
||||||
|
for field_name, field in fields.items():
|
||||||
|
is_relation_field = field["type"] in RELATION_TYPES
|
||||||
|
is_template_relation_field = (
|
||||||
|
field["type"] == "template"
|
||||||
|
and isinstance(field["fields"], dict)
|
||||||
|
and field["fields"]["type"] in RELATION_TYPES
|
||||||
|
)
|
||||||
|
if not is_relation_field and not is_template_relation_field:
|
||||||
|
continue
|
||||||
|
error = self.check_relation(collection, field_name, field)
|
||||||
|
if error:
|
||||||
|
self.errors.append(error)
|
||||||
|
|
||||||
|
def check_field(
|
||||||
|
self,
|
||||||
|
collection: str,
|
||||||
|
field_name: str,
|
||||||
|
field: Union[str, Dict[str, Any]],
|
||||||
|
nested: bool = False,
|
||||||
|
) -> None:
|
||||||
|
collectionfield = f"{collection}{KEYSEPARATOR}{field_name}"
|
||||||
|
|
||||||
|
if nested:
|
||||||
|
if isinstance(field, str):
|
||||||
|
field = {"type": field}
|
||||||
|
field[
|
||||||
|
"restriction_mode"
|
||||||
|
] = "A" # add restriction_mode to satisfy the checker below.
|
||||||
|
if field["type"] == "template": # no nested templates
|
||||||
|
self.errors.append(f"Nested template field in {collectionfield}")
|
||||||
|
return
|
||||||
|
|
||||||
|
type = field.get("type")
|
||||||
|
if type not in VALID_TYPES:
|
||||||
|
self.errors.append(
|
||||||
|
f"Type '{type}' for collectionfield {collectionfield} is invalid."
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
required_attributes = [
|
||||||
|
"type",
|
||||||
|
"restriction_mode",
|
||||||
|
]
|
||||||
|
if type in RELATION_TYPES:
|
||||||
|
required_attributes.append("to")
|
||||||
|
if type == "template":
|
||||||
|
required_attributes.append("fields")
|
||||||
|
for attr in required_attributes:
|
||||||
|
if attr not in field:
|
||||||
|
self.errors.append(
|
||||||
|
f"Required attribute '{attr}' for collectionfield {collectionfield} is missing."
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
if field.get("calculated"):
|
||||||
|
return
|
||||||
|
|
||||||
|
valid_attributes = list(OPTIONAL_ATTRIBUTES) + required_attributes
|
||||||
|
if type == "string[]":
|
||||||
|
valid_attributes.append("items")
|
||||||
|
if "items" in field and "enum" not in field["items"]:
|
||||||
|
self.errors.append(
|
||||||
|
f"'items' is missing an inner 'enum' for {collectionfield}"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
for value in field.get("items", {"enum": []})["enum"]:
|
||||||
|
self.validate_value_for_type("string", value, collectionfield)
|
||||||
|
if type == "JSON" and "default" in field:
|
||||||
|
try:
|
||||||
|
json.loads(json.dumps(field["default"]))
|
||||||
|
except:
|
||||||
|
self.errors.append(
|
||||||
|
f"Default value for {collectionfield}' is not valid json."
|
||||||
|
)
|
||||||
|
if type == "number":
|
||||||
|
valid_attributes.append("minimum")
|
||||||
|
if not isinstance(field.get("minimum", 0), int):
|
||||||
|
self.errors.append(f"'minimum' for {collectionfield} is not a number.")
|
||||||
|
if type == "string":
|
||||||
|
valid_attributes.append("maxLength")
|
||||||
|
if not isinstance(field.get("maxLength", 0), int):
|
||||||
|
self.errors.append(
|
||||||
|
f"'maxLength' for {collectionfield} is not a number."
|
||||||
|
)
|
||||||
|
if type in DATA_TYPES:
|
||||||
|
valid_attributes.append("default")
|
||||||
|
if "default" in field:
|
||||||
|
self.validate_value_for_type(type, field["default"], collectionfield)
|
||||||
|
valid_attributes.append("enum")
|
||||||
|
if "enum" in field:
|
||||||
|
if not isinstance(field["enum"], list):
|
||||||
|
self.errors.append(f"'enum' for {collectionfield}' is not a list.")
|
||||||
|
for value in field["enum"]:
|
||||||
|
self.validate_value_for_type(type, value, collectionfield)
|
||||||
|
|
||||||
|
if type in RELATION_TYPES:
|
||||||
|
valid_attributes.append("on_delete")
|
||||||
|
if "on_delete" in field and field["on_delete"] not in (
|
||||||
|
"CASCADE",
|
||||||
|
"PROTECT",
|
||||||
|
):
|
||||||
|
self.errors.append(
|
||||||
|
f"invalid value for 'on_delete' for {collectionfield}"
|
||||||
|
)
|
||||||
|
valid_attributes.append("equal_fields")
|
||||||
|
|
||||||
|
if type == "template":
|
||||||
|
if "$" not in field_name:
|
||||||
|
self.errors.append(
|
||||||
|
f"The template field {collectionfield} is missing a $"
|
||||||
|
)
|
||||||
|
valid_attributes.append("replacement_collection")
|
||||||
|
elif "$" in field_name and not nested:
|
||||||
|
print(field_name, field)
|
||||||
|
self.errors.append(f"The non-template field {collectionfield} contains a $")
|
||||||
|
|
||||||
|
for attr in field.keys():
|
||||||
|
if attr not in valid_attributes:
|
||||||
|
self.errors.append(
|
||||||
|
f"Attribute '{attr}' for collectionfield {collectionfield} is invalid."
|
||||||
|
)
|
||||||
|
|
||||||
|
if not isinstance(field.get("description", ""), str):
|
||||||
|
self.errors.append(f"Description of {collectionfield} must be a string.")
|
||||||
|
|
||||||
|
if type == "template":
|
||||||
|
self.check_field(collection, field_name, field["fields"], nested=True)
|
||||||
|
|
||||||
|
def validate_value_for_type(
|
||||||
|
self, type_str: str, value: Any, collectionfield: str
|
||||||
|
) -> None:
|
||||||
|
basic_types = {
|
||||||
|
"string": str,
|
||||||
|
"number": int,
|
||||||
|
"boolean": bool,
|
||||||
|
"HTMLStrict": str,
|
||||||
|
"HTMLPermissive": str,
|
||||||
|
"timestamp": int,
|
||||||
|
}
|
||||||
|
if type_str in basic_types:
|
||||||
|
if type(value) != basic_types[type_str]:
|
||||||
|
self.errors.append(
|
||||||
|
f"Value '{value}' for {collectionfield}' is not a {type_str}."
|
||||||
|
)
|
||||||
|
elif type_str in ("string[]", "number[]"):
|
||||||
|
if not isinstance(value, list):
|
||||||
|
self.errors.append(
|
||||||
|
f"Value '{value}' for {collectionfield}' is not a {type_str}."
|
||||||
|
)
|
||||||
|
for x in value:
|
||||||
|
if type(x) != basic_types[type_str[:-2]]:
|
||||||
|
self.errors.append(
|
||||||
|
f"Listentry '{x}' for {collectionfield}' is not a {type_str[:-2]}."
|
||||||
|
)
|
||||||
|
elif type_str == "JSON":
|
||||||
|
pass
|
||||||
|
elif type_str == "float":
|
||||||
|
if type(value) not in (int, float):
|
||||||
|
self.errors.append(
|
||||||
|
f"Value '{value}' for {collectionfield}' is not a float."
|
||||||
|
)
|
||||||
|
elif type_str == "decimal(6)":
|
||||||
|
if not decimal_regex.match(value):
|
||||||
|
self.errors.append(
|
||||||
|
f"Value '{value}' for {collectionfield}' is not a decimal(6)."
|
||||||
|
)
|
||||||
|
elif type_str == "color":
|
||||||
|
if not color_regex.match(value):
|
||||||
|
self.errors.append(
|
||||||
|
f"Value '{value}' for {collectionfield}' is not a color."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise NotImplementedError(type_str)
|
||||||
|
|
||||||
|
def check_relation(
|
||||||
|
self, collection: str, field_name: str, field: Dict[str, Any]
|
||||||
|
) -> Optional[str]:
|
||||||
|
collectionfield = f"{collection}{KEYSEPARATOR}{field_name}"
|
||||||
|
if field["type"] == "template":
|
||||||
|
field = field["fields"]
|
||||||
|
to = field["to"]
|
||||||
|
|
||||||
|
if isinstance(to, str):
|
||||||
|
if not collectionfield_regex.match(to):
|
||||||
|
return f"'to' of {collectionfield} is not a collectionfield."
|
||||||
|
return self.check_reverse(collectionfield, to)
|
||||||
|
elif isinstance(to, list):
|
||||||
|
for cf in to:
|
||||||
|
if not collectionfield_regex.match(cf):
|
||||||
|
return f"The collectionfield in 'to' of {collectionfield} is not valid."
|
||||||
|
error = self.check_reverse(collectionfield, cf)
|
||||||
|
if error:
|
||||||
|
return error
|
||||||
|
else:
|
||||||
|
to_field = to["field"]
|
||||||
|
if not field_regex.match(to_field):
|
||||||
|
return (
|
||||||
|
f"The field '{to_field}' in 'to' of {collectionfield} is not valid."
|
||||||
|
)
|
||||||
|
for c in to["collections"]:
|
||||||
|
if not collection_regex.match(c):
|
||||||
|
self.errors.append(
|
||||||
|
f"The collection '{c}' in 'to' of {collectionfield} is not a valid collection."
|
||||||
|
)
|
||||||
|
error = self.check_reverse(
|
||||||
|
collectionfield, f"{c}{KEYSEPARATOR}{to['field']}"
|
||||||
|
)
|
||||||
|
if error:
|
||||||
|
return error
|
||||||
|
return None
|
||||||
|
|
||||||
|
def check_reverse(
|
||||||
|
self, from_collectionfield: str, to_collectionfield: str
|
||||||
|
) -> Optional[str]:
|
||||||
|
to_unified = [] # a list of target collectionfields (unififed with all
|
||||||
|
# the different possibilities for the 'to' field) from the (expected)
|
||||||
|
# relation in to_collectionfield. The from_collectionfield must be in this
|
||||||
|
# list.
|
||||||
|
|
||||||
|
to_collection, to_field_name = to_collectionfield.split(KEYSEPARATOR)
|
||||||
|
if to_collection not in self.models:
|
||||||
|
return f"The collection '{to_collection}' in 'to' of {from_collectionfield} is not a valid collection."
|
||||||
|
if to_field_name not in self.models[to_collection]:
|
||||||
|
return f"The collectionfield '{to_collectionfield}' in 'to' of {from_collectionfield} does not exist."
|
||||||
|
|
||||||
|
to_field = self.models[to_collection][to_field_name]
|
||||||
|
if to_field["type"] == "template":
|
||||||
|
to_field = to_field["fields"]
|
||||||
|
if not isinstance(to_field, dict):
|
||||||
|
return f"The 'fields' of the template field '{to_collectionfield}' must be a dict to hold a relation."
|
||||||
|
if to_field["type"] not in RELATION_TYPES:
|
||||||
|
return f"{from_collectionfield} points to {to_collectionfield}, but {to_collectionfield} to is not a relation."
|
||||||
|
|
||||||
|
to = to_field["to"]
|
||||||
|
if isinstance(to, str):
|
||||||
|
to_unified.append(to)
|
||||||
|
elif isinstance(to, list):
|
||||||
|
to_unified = to
|
||||||
|
else:
|
||||||
|
for c in to["collections"]:
|
||||||
|
to_unified.append(f"{c}{KEYSEPARATOR}{to['field']}")
|
||||||
|
|
||||||
|
if from_collectionfield not in to_unified:
|
||||||
|
return f"{from_collectionfield} points to {to_collectionfield}, but {to_collectionfield} does not point back."
|
||||||
|
return None
|
||||||
|
|
||||||
|
def split_collectionfield(self, collectionfield: str) -> Tuple[str, str]:
|
||||||
|
parts = collectionfield.split(KEYSEPARATOR)
|
||||||
|
return parts[0], parts[1]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
files = sys.argv[1:]
|
||||||
|
if not files:
|
||||||
|
files = DEFAULT_FILES
|
||||||
|
|
||||||
|
failed = False
|
||||||
|
for f in files:
|
||||||
|
with open(f) as data:
|
||||||
|
try:
|
||||||
|
Checker(f).run_check()
|
||||||
|
except CheckException as e:
|
||||||
|
print(f"Check for {f} failed:\n", e)
|
||||||
|
failed = True
|
||||||
|
else:
|
||||||
|
print(f"Check for {f} successful.")
|
||||||
|
return 1 if failed else 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
@ -1 +1 @@
|
|||||||
Subproject commit 4bf228dd0e246a466c403d31bd0158242c756f96
|
Subproject commit 55fa585ff740c1bac9511228ec55a331997432ab
|
@ -1 +1 @@
|
|||||||
Subproject commit eb1e7fd9ce2463c1a40a33b1fc65fbcc503035d1
|
Subproject commit 6f39f6074da8f4936f3127652097a4d1bafb4e32
|
@ -1 +1 @@
|
|||||||
Subproject commit ad603239769e00345aa7855cb45ac71dfb08e396
|
Subproject commit ce2b08ef5fc0d677a6a4f20c0ed9ec120181cadf
|
Loading…
Reference in New Issue
Block a user