From e6111daec83c49fecaebb125374dfd6a3cfb8d41 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Jun 2021 10:16:51 +0200 Subject: [PATCH 01/18] created SchemasHub for handling schemas --- openpype/settings/entities/lib.py | 46 +++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index 05f4ea64f8c..9d13655a8fb 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -426,3 +426,49 @@ class OverrideState: DEFAULTS = OverrideStateItem(0, "Defaults") STUDIO = OverrideStateItem(1, "Studio overrides") PROJECT = OverrideStateItem(2, "Project Overrides") + + +class SchemasHub: + def __init__(self, schema_subfolder): + from openpype.settings import entities + + # Define known abstract classes + known_abstract_classes = ( + entities.BaseEntity, + entities.BaseItemEntity, + entities.ItemEntity, + entities.EndpointEntity, + entities.InputEntity, + entities.BaseEnumEntity + ) + + self._loaded_types = {} + _gui_types = [] + for attr in dir(entities): + item = getattr(entities, attr) + # Filter classes + if not inspect.isclass(item): + continue + + # Skip classes that do not inherit from BaseEntity + if not issubclass(item, entities.BaseEntity): + continue + + # Skip class that is abstract by design + if item in known_abstract_classes: + continue + + if inspect.isabstract(item): + # Create an object to get crash and get traceback + item() + + # Backwards compatibility + # Single entity may have multiple schema types + for schema_type in item.schema_types: + self._loaded_types[schema_type] = item + + if item.gui_type: + _gui_types.append(item) + self._gui_types = tuple(_gui_types) + + self._schema_subfolder = schema_subfolder From 070ba3070af4a61d240aeb4bef166c425c79b947 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Jun 2021 10:20:07 +0200 Subject: [PATCH 02/18] added api callbacks to SchemaHub --- openpype/settings/entities/lib.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index 9d13655a8fb..879e3d9cad2 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -472,3 +472,19 @@ def __init__(self, schema_subfolder): self._gui_types = tuple(_gui_types) self._schema_subfolder = schema_subfolder + + @property + def gui_types(self): + return self._gui_types + + def get_schema(self, schema_name): + pass + + def get_template(self, template_name): + pass + + def resolve_schema_data(self, schema_data): + pass + + def create_schema_object(self, schema_data, *args, **kwargs): + pass From e3c4c91f3e91ab9480408d3b1f5f5445297b6a57 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Jun 2021 10:21:41 +0200 Subject: [PATCH 03/18] use schema hub inside root entity --- openpype/settings/entities/root_entities.py | 87 ++++++--------------- 1 file changed, 22 insertions(+), 65 deletions(-) diff --git a/openpype/settings/entities/root_entities.py b/openpype/settings/entities/root_entities.py index 401d3980c9f..9bb32382fbf 100644 --- a/openpype/settings/entities/root_entities.py +++ b/openpype/settings/entities/root_entities.py @@ -1,7 +1,6 @@ import os import json import copy -import inspect from abc import abstractmethod @@ -10,8 +9,7 @@ NOT_SET, WRAPPER_TYPES, OverrideState, - get_studio_settings_schema, - get_project_settings_schema + SchemasHub ) from .exceptions import ( SchemaError, @@ -53,7 +51,12 @@ class RootEntity(BaseItemEntity): """ schema_types = ["root"] - def __init__(self, schema_data, reset): + def __init__(self, schema_hub, reset, main_schema_name=None): + self.schema_hub = schema_hub + if not main_schema_name: + main_schema_name = "schema_main" + schema_data = schema_hub.get_schema(main_schema_name) + super(RootEntity, self).__init__(schema_data) self._require_restart_callbacks = [] self._item_ids_require_restart = set() @@ -143,11 +146,13 @@ def _add_children(self, schema_data, first=True): child_obj = self.create_schema_object(children_schema, self) self.children.append(child_obj) added_children.append(child_obj) - if isinstance(child_obj, self._gui_types): + if isinstance(child_obj, self.schema_hub.gui_types): continue if child_obj.key in self.non_gui_children: - raise KeyError("Duplicated key \"{}\"".format(child_obj.key)) + raise KeyError( + "Duplicated key \"{}\"".format(child_obj.key) + ) self.non_gui_children[child_obj.key] = child_obj if not first: @@ -160,9 +165,6 @@ def _item_initalization(self): # Store `self` to `root_item` for children entities self.root_item = self - self._loaded_types = None - self._gui_types = None - # Children are stored by key as keys are immutable and are defined by # schema self.valid_value_types = (dict, ) @@ -201,54 +203,9 @@ def create_schema_object(self, schema_data, *args, **kwargs): Available entities are loaded on first run. Children entities can call this method. """ - if self._loaded_types is None: - # Load available entities - from openpype.settings import entities - - # Define known abstract classes - known_abstract_classes = ( - entities.BaseEntity, - entities.BaseItemEntity, - entities.ItemEntity, - entities.EndpointEntity, - entities.InputEntity, - entities.BaseEnumEntity - ) - - self._loaded_types = {} - _gui_types = [] - for attr in dir(entities): - item = getattr(entities, attr) - # Filter classes - if not inspect.isclass(item): - continue - - # Skip classes that do not inherit from BaseEntity - if not issubclass(item, entities.BaseEntity): - continue - - # Skip class that is abstract by design - if item in known_abstract_classes: - continue - - if inspect.isabstract(item): - # Create an object to get crash and get traceback - item() - - # Backwards compatibility - # Single entity may have multiple schema types - for schema_type in item.schema_types: - self._loaded_types[schema_type] = item - - if item.gui_type: - _gui_types.append(item) - self._gui_types = tuple(_gui_types) - - klass = self._loaded_types.get(schema_data["type"]) - if not klass: - raise KeyError("Unknown type \"{}\"".format(schema_data["type"])) - - return klass(schema_data, *args, **kwargs) + return self.schema_hub.create_schema_object( + schema_data, *args, **kwargs + ) def set_override_state(self, state): """Set override state and trigger it on children. @@ -492,13 +449,13 @@ class SystemSettings(RootEntity): and debugging purposes. """ def __init__( - self, set_studio_state=True, reset=True, schema_data=None + self, set_studio_state=True, reset=True, schema_hub=None ): - if schema_data is None: + if schema_hub is None: # Load system schemas - schema_data = get_studio_settings_schema() + schema_hub = SchemasHub("system_schema") - super(SystemSettings, self).__init__(schema_data, reset) + super(SystemSettings, self).__init__(schema_hub, reset) if set_studio_state: self.set_studio_state() @@ -605,17 +562,17 @@ def __init__( project_name=None, change_state=True, reset=True, - schema_data=None + schema_hub=None ): self._project_name = project_name self._system_settings_entity = None - if schema_data is None: + if schema_hub is None: # Load system schemas - schema_data = get_project_settings_schema() + schema_hub = SchemasHub("projects_schema") - super(ProjectSettings, self).__init__(schema_data, reset) + super(ProjectSettings, self).__init__(schema_hub, reset) if change_state: if self.project_name is None: From cb7e0c957ca94fa89162a242c7b64ab77df6e25b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Jun 2021 10:22:19 +0200 Subject: [PATCH 04/18] use resolving where templates can be used --- .../settings/entities/dict_immutable_keys_entity.py | 12 +++++++++++- openpype/settings/entities/root_entities.py | 13 ++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/openpype/settings/entities/dict_immutable_keys_entity.py b/openpype/settings/entities/dict_immutable_keys_entity.py index 052bbda4d08..c965dc3b5ab 100644 --- a/openpype/settings/entities/dict_immutable_keys_entity.py +++ b/openpype/settings/entities/dict_immutable_keys_entity.py @@ -1,4 +1,5 @@ import copy +import collections from .lib import ( WRAPPER_TYPES, @@ -138,7 +139,16 @@ def _add_children(self, schema_data, first=True): method when handling gui wrappers. """ added_children = [] - for children_schema in schema_data["children"]: + children_deque = collections.deque() + for _children_schema in schema_data["children"]: + children_schemas = self.schema_hub.resolve_schema_data( + _children_schema + ) + for children_schema in children_schemas: + children_deque.append(children_schema) + + while children_deque: + children_schema = children_deque.popleft() if children_schema["type"] in WRAPPER_TYPES: _children_schema = copy.deepcopy(children_schema) wrapper_children = self._add_children( diff --git a/openpype/settings/entities/root_entities.py b/openpype/settings/entities/root_entities.py index 9bb32382fbf..1833535a071 100644 --- a/openpype/settings/entities/root_entities.py +++ b/openpype/settings/entities/root_entities.py @@ -1,6 +1,7 @@ import os import json import copy +import collections from abc import abstractmethod @@ -133,7 +134,17 @@ def items(self): def _add_children(self, schema_data, first=True): added_children = [] - for children_schema in schema_data["children"]: + children_deque = collections.deque() + for _children_schema in schema_data["children"]: + children_schemas = self.schema_hub.resolve_schema_data( + _children_schema + ) + for children_schema in children_schemas: + children_deque.append(children_schema) + + while children_deque: + children_schema = children_deque.popleft() + if children_schema["type"] in WRAPPER_TYPES: _children_schema = copy.deepcopy(children_schema) wrapper_children = self._add_children( From d02e6eda745b35b6bf7c221b369d157a8b901bea Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Jun 2021 10:22:39 +0200 Subject: [PATCH 05/18] gave access to event hub for all entities --- openpype/settings/entities/base_entity.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openpype/settings/entities/base_entity.py b/openpype/settings/entities/base_entity.py index c6bff1ff47a..0e29a35e1f9 100644 --- a/openpype/settings/entities/base_entity.py +++ b/openpype/settings/entities/base_entity.py @@ -885,7 +885,11 @@ def schema_validations(self): def create_schema_object(self, *args, **kwargs): """Reference method for creation of entities defined in RootEntity.""" - return self.root_item.create_schema_object(*args, **kwargs) + return self.schema_hub.create_schema_object(*args, **kwargs) + + @property + def schema_hub(self): + return self.root_item.schema_hub def get_entity_from_path(self, path): return self.root_item.get_entity_from_path(path) From 0fc16b25767e8f7d614553def050b49552ba09a0 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Jun 2021 10:23:49 +0200 Subject: [PATCH 06/18] moved template filling functions under SchemaHub --- openpype/settings/entities/lib.py | 351 +++++++++++++++--------------- 1 file changed, 175 insertions(+), 176 deletions(-) diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index 879e3d9cad2..74de3e6ffa8 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -25,182 +25,6 @@ template_key_pattern = re.compile(r"(\{.*?[^{0]*\})") -def _pop_metadata_item(template): - found_idx = None - for idx, item in enumerate(template): - if not isinstance(item, dict): - continue - - for key in TEMPLATE_METADATA_KEYS: - if key in item: - found_idx = idx - break - - if found_idx is not None: - break - - metadata_item = {} - if found_idx is not None: - metadata_item = template.pop(found_idx) - return metadata_item - - -def _fill_schema_template_data( - template, template_data, skip_paths, required_keys=None, missing_keys=None -): - first = False - if required_keys is None: - first = True - - if "skip_paths" in template_data: - skip_paths = template_data["skip_paths"] - if not isinstance(skip_paths, list): - skip_paths = [skip_paths] - - # Cleanup skip paths (skip empty values) - skip_paths = [path for path in skip_paths if path] - - required_keys = set() - missing_keys = set() - - # Copy template data as content may change - template = copy.deepcopy(template) - - # Get metadata item from template - metadata_item = _pop_metadata_item(template) - - # Check for default values for template data - default_values = metadata_item.get(DEFAULT_VALUES_KEY) or {} - - for key, value in default_values.items(): - if key not in template_data: - template_data[key] = value - - if not template: - output = template - - elif isinstance(template, list): - # Store paths by first part if path - # - None value says that whole key should be skipped - skip_paths_by_first_key = {} - for path in skip_paths: - parts = path.split("/") - key = parts.pop(0) - if key not in skip_paths_by_first_key: - skip_paths_by_first_key[key] = [] - - value = "/".join(parts) - skip_paths_by_first_key[key].append(value or None) - - output = [] - for item in template: - # Get skip paths for children item - _skip_paths = [] - if not isinstance(item, dict): - pass - - elif item.get("type") in WRAPPER_TYPES: - _skip_paths = copy.deepcopy(skip_paths) - - elif skip_paths_by_first_key: - # Check if this item should be skipped - key = item.get("key") - if key and key in skip_paths_by_first_key: - _skip_paths = skip_paths_by_first_key[key] - # Skip whole item if None is in skip paths value - if None in _skip_paths: - continue - - output_item = _fill_schema_template_data( - item, template_data, _skip_paths, required_keys, missing_keys - ) - if output_item: - output.append(output_item) - - elif isinstance(template, dict): - output = {} - for key, value in template.items(): - output[key] = _fill_schema_template_data( - value, template_data, skip_paths, required_keys, missing_keys - ) - if output.get("type") in WRAPPER_TYPES and not output.get("children"): - return {} - - elif isinstance(template, STRING_TYPE): - # TODO find much better way how to handle filling template data - template = template.replace("{{", "__dbcb__").replace("}}", "__decb__") - for replacement_string in template_key_pattern.findall(template): - key = str(replacement_string[1:-1]) - required_keys.add(key) - if key not in template_data: - missing_keys.add(key) - continue - - value = template_data[key] - if replacement_string == template: - # Replace the value with value from templates data - # - with this is possible to set value with different type - template = value - else: - # Only replace the key in string - template = template.replace(replacement_string, value) - - output = template.replace("__dbcb__", "{").replace("__decb__", "}") - - else: - output = template - - if first and missing_keys: - raise SchemaTemplateMissingKeys(missing_keys, required_keys) - - return output - - -def _fill_schema_template(child_data, schema_collection, schema_templates): - template_name = child_data["name"] - template = schema_templates.get(template_name) - if template is None: - if template_name in schema_collection: - raise KeyError(( - "Schema \"{}\" is used as `schema_template`" - ).format(template_name)) - raise KeyError("Schema template \"{}\" was not found".format( - template_name - )) - - # Default value must be dictionary (NOT list) - # - empty list would not add any item if `template_data` are not filled - template_data = child_data.get("template_data") or {} - if isinstance(template_data, dict): - template_data = [template_data] - - skip_paths = child_data.get("skip_paths") or [] - if isinstance(skip_paths, STRING_TYPE): - skip_paths = [skip_paths] - - output = [] - for single_template_data in template_data: - try: - filled_child = _fill_schema_template_data( - template, single_template_data, skip_paths - ) - - except SchemaTemplateMissingKeys as exc: - raise SchemaTemplateMissingKeys( - exc.missing_keys, exc.required_keys, template_name - ) - - for item in filled_child: - filled_item = _fill_inner_schemas( - item, schema_collection, schema_templates - ) - if filled_item["type"] == "schema_template": - output.extend(_fill_schema_template( - filled_item, schema_collection, schema_templates - )) - else: - output.append(filled_item) - return output def _fill_inner_schemas(schema_data, schema_collection, schema_templates): @@ -488,3 +312,178 @@ def resolve_schema_data(self, schema_data): def create_schema_object(self, schema_data, *args, **kwargs): pass + + def _fill_schema_template(self, child_data, template_def): + template_name = child_data["name"] + + # Default value must be dictionary (NOT list) + # - empty list would not add any item if `template_data` are not filled + template_data = child_data.get("template_data") or {} + if isinstance(template_data, dict): + template_data = [template_data] + + skip_paths = child_data.get("skip_paths") or [] + if isinstance(skip_paths, STRING_TYPE): + skip_paths = [skip_paths] + + output = [] + for single_template_data in template_data: + try: + output.extend(self._fill_schema_template_data( + template_def, single_template_data, skip_paths + )) + + except SchemaTemplateMissingKeys as exc: + raise SchemaTemplateMissingKeys( + exc.missing_keys, exc.required_keys, template_name + ) + return output + + def _fill_schema_template_data( + self, + template, + template_data, + skip_paths, + required_keys=None, + missing_keys=None + ): + first = False + if required_keys is None: + first = True + + if "skip_paths" in template_data: + skip_paths = template_data["skip_paths"] + if not isinstance(skip_paths, list): + skip_paths = [skip_paths] + + # Cleanup skip paths (skip empty values) + skip_paths = [path for path in skip_paths if path] + + required_keys = set() + missing_keys = set() + + # Copy template data as content may change + template = copy.deepcopy(template) + + # Get metadata item from template + metadata_item = self._pop_metadata_item(template) + + # Check for default values for template data + default_values = metadata_item.get(DEFAULT_VALUES_KEY) or {} + + for key, value in default_values.items(): + if key not in template_data: + template_data[key] = value + + if not template: + output = template + + elif isinstance(template, list): + # Store paths by first part if path + # - None value says that whole key should be skipped + skip_paths_by_first_key = {} + for path in skip_paths: + parts = path.split("/") + key = parts.pop(0) + if key not in skip_paths_by_first_key: + skip_paths_by_first_key[key] = [] + + value = "/".join(parts) + skip_paths_by_first_key[key].append(value or None) + + output = [] + for item in template: + # Get skip paths for children item + _skip_paths = [] + if not isinstance(item, dict): + pass + + elif item.get("type") in WRAPPER_TYPES: + _skip_paths = copy.deepcopy(skip_paths) + + elif skip_paths_by_first_key: + # Check if this item should be skipped + key = item.get("key") + if key and key in skip_paths_by_first_key: + _skip_paths = skip_paths_by_first_key[key] + # Skip whole item if None is in skip paths value + if None in _skip_paths: + continue + + output_item = self._fill_schema_template_data( + item, + template_data, + _skip_paths, + required_keys, + missing_keys + ) + if output_item: + output.append(output_item) + + elif isinstance(template, dict): + output = {} + for key, value in template.items(): + output[key] = self._fill_schema_template_data( + value, + template_data, + skip_paths, + required_keys, + missing_keys + ) + if ( + output.get("type") in WRAPPER_TYPES + and not output.get("children") + ): + return {} + + elif isinstance(template, STRING_TYPE): + # TODO find much better way how to handle filling template data + template = ( + template + .replace("{{", "__dbcb__") + .replace("}}", "__decb__") + ) + for replacement_string in template_key_pattern.findall(template): + key = str(replacement_string[1:-1]) + required_keys.add(key) + if key not in template_data: + missing_keys.add(key) + continue + + value = template_data[key] + if replacement_string == template: + # Replace the value with value from templates data + # - with this is possible to set value with different type + template = value + else: + # Only replace the key in string + template = template.replace(replacement_string, value) + + output = template.replace("__dbcb__", "{").replace("__decb__", "}") + + else: + output = template + + if first and missing_keys: + raise SchemaTemplateMissingKeys(missing_keys, required_keys) + + return output + + def _pop_metadata_item(self, template_def): + found_idx = None + for idx, item in enumerate(template_def): + if not isinstance(item, dict): + continue + + for key in TEMPLATE_METADATA_KEYS: + if key in item: + found_idx = idx + break + + if found_idx is not None: + break + + metadata_item = {} + if found_idx is not None: + metadata_item = template_def.pop(found_idx) + return metadata_item From d0b32e129271806132d84cb37011dc69ba68ad76 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Jun 2021 10:24:29 +0200 Subject: [PATCH 07/18] implemented loading of schemas for schema hub --- openpype/settings/entities/lib.py | 64 +++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index 74de3e6ffa8..aae98067f78 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -296,6 +296,11 @@ def __init__(self, schema_subfolder): self._gui_types = tuple(_gui_types) self._schema_subfolder = schema_subfolder + self._crashed_on_load = {} + loaded_templates, loaded_schemas = self._load_schemas() + + self._loaded_templates = loaded_templates + self._loaded_schemas = loaded_schemas @property def gui_types(self): @@ -313,6 +318,65 @@ def resolve_schema_data(self, schema_data): def create_schema_object(self, schema_data, *args, **kwargs): pass + def _load_schemas(self): + dirpath = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "schemas", + self._schema_subfolder + ) + loaded_schemas = {} + loaded_templates = {} + for root, _, filenames in os.walk(dirpath): + for filename in filenames: + basename, ext = os.path.splitext(filename) + if ext != ".json": + continue + + filepath = os.path.join(root, filename) + with open(filepath, "r") as json_stream: + try: + schema_data = json.load(json_stream) + except Exception as exc: + msg = str(exc) + print("Unable to parse JSON file {}\n{}".format( + filepath, msg + )) + self._crashed_on_load[basename] = { + "filepath": filepath, + "message": msg + } + continue + + if basename in self._crashed_on_load: + crashed_item = self._crashed_on_load[basename] + raise KeyError(( + "Duplicated filename \"{}\"." + " One of them crashed on load \"{}\" {}" + ).format( + filename, + crashed_item["filpath"], + crashed_item["message"] + )) + + if isinstance(schema_data, list): + if basename in loaded_templates: + raise KeyError( + "Duplicated template filename \"{}\"".format( + filename + ) + ) + loaded_templates[basename] = schema_data + else: + if basename in loaded_schemas: + raise KeyError( + "Duplicated schema filename \"{}\"".format( + filename + ) + ) + loaded_schemas[basename] = schema_data + + return loaded_templates, loaded_schemas + def _fill_schema_template(self, child_data, template_def): template_name = child_data["name"] From 770e33d0f9a2e507f3499ead0b655a014ae4748b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Jun 2021 10:24:54 +0200 Subject: [PATCH 08/18] implemented get_template and get_schema --- openpype/settings/entities/lib.py | 38 +++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index aae98067f78..cdc154e4412 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -307,10 +307,44 @@ def gui_types(self): return self._gui_types def get_schema(self, schema_name): - pass + if schema_name not in self._loaded_schemas: + if schema_name in self._loaded_templates: + raise KeyError(( + "Template \"{}\" is used as `schema`" + ).format(schema_name)) + + elif schema_name in self._crashed_on_load: + crashed_item = self._crashed_on_load[schema_name] + raise KeyError( + "Unable to parse schema file \"{}\". {}".format( + crashed_item["filpath"], crashed_item["message"] + ) + ) + + raise KeyError( + "Schema \"{}\" was not found".format(schema_name) + ) + return copy.deepcopy(self._loaded_schemas[schema_name]) def get_template(self, template_name): - pass + if template_name not in self._loaded_templates: + if template_name in self._loaded_schemas: + raise KeyError(( + "Schema \"{}\" is used as `template`" + ).format(template_name)) + + elif template_name in self._crashed_on_load: + crashed_item = self._crashed_on_load[template_name] + raise KeyError( + "Unable to parse templace file \"{}\". {}".format( + crashed_item["filpath"], crashed_item["message"] + ) + ) + + raise KeyError( + "Template \"{}\" was not found".format(template_name) + ) + return copy.deepcopy(self._loaded_templates[template_name]) def resolve_schema_data(self, schema_data): pass From cae6f7e6209b879908ead4d31b75ab99e818b774 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Jun 2021 10:25:42 +0200 Subject: [PATCH 09/18] implemented create_schema_object which handle creation of entities by schema data --- openpype/settings/entities/lib.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index cdc154e4412..0e67e6500a3 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -350,7 +350,17 @@ def resolve_schema_data(self, schema_data): pass def create_schema_object(self, schema_data, *args, **kwargs): - pass + schema_type = schema_data["type"] + if schema_type in ("schema", "template", "schema_template"): + raise ValueError( + "Got unresolved schema data of type \"{}\"".format(schema_type) + ) + + klass = self._loaded_types.get(schema_type) + if not klass: + raise KeyError("Unknown type \"{}\"".format(schema_type)) + + return klass(schema_data, *args, **kwargs) def _load_schemas(self): dirpath = os.path.join( From 9ec64866a35473126a82e69bd7fde11758079e51 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Jun 2021 10:26:07 +0200 Subject: [PATCH 10/18] implemented resolving for schemas and template items --- openpype/settings/entities/lib.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index 0e67e6500a3..a57a391c3a7 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -347,7 +347,22 @@ def get_template(self, template_name): return copy.deepcopy(self._loaded_templates[template_name]) def resolve_schema_data(self, schema_data): - pass + schema_type = schema_data["type"] + if schema_type not in ("schema", "template", "schema_template"): + return [schema_data] + + if schema_type == "schema": + return self.resolve_schema_data( + self.get_schema(schema_data["name"]) + ) + + template_name = schema_data["name"] + template_def = self.get_template(template_name) + + filled_template = self._fill_schema_template( + schema_data, template_def + ) + return filled_template def create_schema_object(self, schema_data, *args, **kwargs): schema_type = schema_data["type"] From d4d1e177ae49e7bdbdc27821dd68d5619e64ee85 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Jun 2021 10:27:15 +0200 Subject: [PATCH 11/18] removed unused functions --- openpype/settings/entities/lib.py | 73 ------------------------------- 1 file changed, 73 deletions(-) diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index a57a391c3a7..933905a3b22 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -25,71 +25,6 @@ template_key_pattern = re.compile(r"(\{.*?[^{0]*\})") - - -def _fill_inner_schemas(schema_data, schema_collection, schema_templates): - if schema_data["type"] == "schema": - raise ValueError("First item in schema data can't be schema.") - - children_key = "children" - object_type_key = "object_type" - for item_key in (children_key, object_type_key): - children = schema_data.get(item_key) - if not children: - continue - - if object_type_key == item_key: - if not isinstance(children, dict): - continue - children = [children] - - new_children = [] - for child in children: - child_type = child["type"] - if child_type == "schema": - schema_name = child["name"] - if schema_name not in schema_collection: - if schema_name in schema_templates: - raise KeyError(( - "Schema template \"{}\" is used as `schema`" - ).format(schema_name)) - raise KeyError( - "Schema \"{}\" was not found".format(schema_name) - ) - - filled_child = _fill_inner_schemas( - schema_collection[schema_name], - schema_collection, - schema_templates - ) - - elif child_type in ("template", "schema_template"): - for filled_child in _fill_schema_template( - child, schema_collection, schema_templates - ): - new_children.append(filled_child) - continue - - else: - filled_child = _fill_inner_schemas( - child, schema_collection, schema_templates - ) - - new_children.append(filled_child) - - if item_key == object_type_key: - if len(new_children) != 1: - raise KeyError(( - "Failed to fill object type with type: {} | name {}" - ).format( - child_type, str(child.get("name")) - )) - new_children = new_children[0] - - schema_data[item_key] = new_children - return schema_data - - # TODO reimplement logic inside entities def validate_environment_groups_uniquenes( schema_data, env_groups=None, keys=None @@ -170,14 +105,6 @@ def get_gui_schema(subfolder, main_schema_name): return main_schema -def get_studio_settings_schema(): - return get_gui_schema("system_schema", "schema_main") - - -def get_project_settings_schema(): - return get_gui_schema("projects_schema", "schema_main") - - class OverrideStateItem: """Object used as item for `OverrideState` enum. From 4bc9aa821fb5e2b47a34513458c0ae957844305d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Jun 2021 10:41:53 +0200 Subject: [PATCH 12/18] add missing import --- openpype/settings/entities/lib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index 933905a3b22..437fa05acac 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -2,6 +2,7 @@ import re import json import copy +import inspect from .exceptions import ( SchemaTemplateMissingKeys, From 8f00b0eb2ff88f4826c37be6513af7dc2485139f Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Jun 2021 10:53:11 +0200 Subject: [PATCH 13/18] few smaller organization changes --- openpype/settings/entities/lib.py | 106 ++++++++++++++++++------------ 1 file changed, 63 insertions(+), 43 deletions(-) diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index 437fa05acac..6e1231e2f64 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -181,54 +181,26 @@ class OverrideState: class SchemasHub: - def __init__(self, schema_subfolder): - from openpype.settings import entities - - # Define known abstract classes - known_abstract_classes = ( - entities.BaseEntity, - entities.BaseItemEntity, - entities.ItemEntity, - entities.EndpointEntity, - entities.InputEntity, - entities.BaseEnumEntity - ) + def __init__(self, schema_subfolder, reset=True): + self._schema_subfolder = schema_subfolder self._loaded_types = {} - _gui_types = [] - for attr in dir(entities): - item = getattr(entities, attr) - # Filter classes - if not inspect.isclass(item): - continue - - # Skip classes that do not inherit from BaseEntity - if not issubclass(item, entities.BaseEntity): - continue + self._gui_types = tuple() - # Skip class that is abstract by design - if item in known_abstract_classes: - continue - - if inspect.isabstract(item): - # Create an object to get crash and get traceback - item() - - # Backwards compatibility - # Single entity may have multiple schema types - for schema_type in item.schema_types: - self._loaded_types[schema_type] = item + self._crashed_on_load = {} + self._loaded_templates = {} + self._loaded_schemas = {} - if item.gui_type: - _gui_types.append(item) - self._gui_types = tuple(_gui_types) + # It doesn't make sence to reload types on each reset as they can't be + # changed + self._load_types() - self._schema_subfolder = schema_subfolder - self._crashed_on_load = {} - loaded_templates, loaded_schemas = self._load_schemas() + # Trigger reset + if reset: + self.reset() - self._loaded_templates = loaded_templates - self._loaded_schemas = loaded_schemas + def reset(self): + self._load_schemas() @property def gui_types(self): @@ -305,7 +277,54 @@ def create_schema_object(self, schema_data, *args, **kwargs): return klass(schema_data, *args, **kwargs) + def _load_types(self): + from openpype.settings import entities + + # Define known abstract classes + known_abstract_classes = ( + entities.BaseEntity, + entities.BaseItemEntity, + entities.ItemEntity, + entities.EndpointEntity, + entities.InputEntity, + entities.BaseEnumEntity + ) + + self._loaded_types = {} + _gui_types = [] + for attr in dir(entities): + item = getattr(entities, attr) + # Filter classes + if not inspect.isclass(item): + continue + + # Skip classes that do not inherit from BaseEntity + if not issubclass(item, entities.BaseEntity): + continue + + # Skip class that is abstract by design + if item in known_abstract_classes: + continue + + if inspect.isabstract(item): + # Create an object to get crash and get traceback + item() + + # Backwards compatibility + # Single entity may have multiple schema types + for schema_type in item.schema_types: + self._loaded_types[schema_type] = item + + if item.gui_type: + _gui_types.append(item) + self._gui_types = tuple(_gui_types) + def _load_schemas(self): + # Refresh all affecting variables + self._crashed_on_load = {} + self._loaded_templates = {} + self._loaded_schemas = {} + dirpath = os.path.join( os.path.dirname(os.path.abspath(__file__)), "schemas", @@ -362,7 +381,8 @@ def _load_schemas(self): ) loaded_schemas[basename] = schema_data - return loaded_templates, loaded_schemas + self._loaded_templates = loaded_templates + self._loaded_schemas = loaded_schemas def _fill_schema_template(self, child_data, template_def): template_name = child_data["name"] From 568c6e5f61e9a912048f6e192a0f3b0d9b022d6e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Jun 2021 11:19:02 +0200 Subject: [PATCH 14/18] use shorter method names --- openpype/settings/entities/lib.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index 6e1231e2f64..2d384688772 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -259,7 +259,7 @@ def resolve_schema_data(self, schema_data): template_name = schema_data["name"] template_def = self.get_template(template_name) - filled_template = self._fill_schema_template( + filled_template = self._fill_template( schema_data, template_def ) return filled_template @@ -384,7 +384,7 @@ def _load_schemas(self): self._loaded_templates = loaded_templates self._loaded_schemas = loaded_schemas - def _fill_schema_template(self, child_data, template_def): + def _fill_template(self, child_data, template_def): template_name = child_data["name"] # Default value must be dictionary (NOT list) @@ -400,7 +400,7 @@ def _fill_schema_template(self, child_data, template_def): output = [] for single_template_data in template_data: try: - output.extend(self._fill_schema_template_data( + output.extend(self._fill_template_data( template_def, single_template_data, skip_paths )) @@ -410,7 +410,7 @@ def _fill_schema_template(self, child_data, template_def): ) return output - def _fill_schema_template_data( + def _fill_template_data( self, template, template_data, @@ -481,7 +481,7 @@ def _fill_schema_template_data( if None in _skip_paths: continue - output_item = self._fill_schema_template_data( + output_item = self._fill_template_data( item, template_data, _skip_paths, @@ -494,7 +494,7 @@ def _fill_schema_template_data( elif isinstance(template, dict): output = {} for key, value in template.items(): - output[key] = self._fill_schema_template_data( + output[key] = self._fill_template_data( value, template_data, skip_paths, From 083dd58b3937edfb6c6903aacbd7ed82ea298c90 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Jun 2021 11:20:07 +0200 Subject: [PATCH 15/18] handle wrapper types in create object --- openpype/settings/entities/lib.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index 2d384688772..e6b73b70661 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -271,6 +271,12 @@ def create_schema_object(self, schema_data, *args, **kwargs): "Got unresolved schema data of type \"{}\"".format(schema_type) ) + if schema_type in WRAPPER_TYPES: + raise ValueError(( + "Function `create_schema_object` can't create entities" + " of any wrapper type. Got type: \"{}\"" + ).format(schema_type)) + klass = self._loaded_types.get(schema_type) if not klass: raise KeyError("Unknown type \"{}\"".format(schema_type)) From 9cfd8af2bf341dfe8a603dac84c21690d793c2b5 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 24 Jun 2021 11:20:15 +0200 Subject: [PATCH 16/18] added brief docstrings --- openpype/settings/entities/lib.py | 143 +++++++++++++----------------- 1 file changed, 63 insertions(+), 80 deletions(-) diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index e6b73b70661..31071a2d309 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -26,86 +26,6 @@ template_key_pattern = re.compile(r"(\{.*?[^{0]*\})") -# TODO reimplement logic inside entities -def validate_environment_groups_uniquenes( - schema_data, env_groups=None, keys=None -): - is_first = False - if env_groups is None: - is_first = True - env_groups = {} - keys = [] - - my_keys = copy.deepcopy(keys) - key = schema_data.get("key") - if key: - my_keys.append(key) - - env_group_key = schema_data.get("env_group_key") - if env_group_key: - if env_group_key not in env_groups: - env_groups[env_group_key] = [] - env_groups[env_group_key].append("/".join(my_keys)) - - children = schema_data.get("children") - if not children: - return - - for child in children: - validate_environment_groups_uniquenes( - child, env_groups, copy.deepcopy(my_keys) - ) - - if is_first: - invalid = {} - for env_group_key, key_paths in env_groups.items(): - if len(key_paths) > 1: - invalid[env_group_key] = key_paths - - if invalid: - raise SchemaDuplicatedEnvGroupKeys(invalid) - - -def validate_schema(schema_data): - validate_environment_groups_uniquenes(schema_data) - - -def get_gui_schema(subfolder, main_schema_name): - dirpath = os.path.join( - os.path.dirname(__file__), - "schemas", - subfolder - ) - loaded_schemas = {} - loaded_schema_templates = {} - for root, _, filenames in os.walk(dirpath): - for filename in filenames: - basename, ext = os.path.splitext(filename) - if ext != ".json": - continue - - filepath = os.path.join(root, filename) - with open(filepath, "r") as json_stream: - try: - schema_data = json.load(json_stream) - except Exception as exc: - raise ValueError(( - "Unable to parse JSON file {}\n{}" - ).format(filepath, str(exc))) - if isinstance(schema_data, list): - loaded_schema_templates[basename] = schema_data - else: - loaded_schemas[basename] = schema_data - - main_schema = _fill_inner_schemas( - loaded_schemas[main_schema_name], - loaded_schemas, - loaded_schema_templates - ) - validate_schema(main_schema) - return main_schema - - class OverrideStateItem: """Object used as item for `OverrideState` enum. @@ -207,6 +127,7 @@ def gui_types(self): return self._gui_types def get_schema(self, schema_name): + """Get schema definition data by it's name.""" if schema_name not in self._loaded_schemas: if schema_name in self._loaded_templates: raise KeyError(( @@ -227,6 +148,7 @@ def get_schema(self, schema_name): return copy.deepcopy(self._loaded_schemas[schema_name]) def get_template(self, template_name): + """Get template definition data by it's name.""" if template_name not in self._loaded_templates: if template_name in self._loaded_schemas: raise KeyError(( @@ -247,6 +169,19 @@ def get_template(self, template_name): return copy.deepcopy(self._loaded_templates[template_name]) def resolve_schema_data(self, schema_data): + """Resolve single item schema data as few types can be expanded. + + This is mainly for 'schema' and 'template' types. Type 'schema' does + not have entity representation and 'template' may contain more than one + output schemas. + + In other cases is retuned passed schema item in list. + + Goal is to have schema and template resolving at one place. + + Returns: + list: Resolved schema data. + """ schema_type = schema_data["type"] if schema_type not in ("schema", "template", "schema_template"): return [schema_data] @@ -265,6 +200,19 @@ def resolve_schema_data(self, schema_data): return filled_template def create_schema_object(self, schema_data, *args, **kwargs): + """Create entity for passed schema data. + + Args: + schema_data(dict): Schema definition of settings entity. + + Returns: + ItemEntity: Created entity for passed schema data item. + + Raises: + ValueError: When 'schema', 'template' or any of wrapper types are + passed. + KeyError: When type of passed schema is not known. + """ schema_type = schema_data["type"] if schema_type in ("schema", "template", "schema_template"): raise ValueError( @@ -284,6 +232,8 @@ def create_schema_object(self, schema_data, *args, **kwargs): return klass(schema_data, *args, **kwargs) def _load_types(self): + """Prepare entity types for cretion of their objects.""" + from openpype.settings import entities # Define known abstract classes @@ -326,6 +276,8 @@ def _load_types(self): self._gui_types = tuple(_gui_types) def _load_schemas(self): + """Load schema definitions from json files.""" + # Refresh all affecting variables self._crashed_on_load = {} self._loaded_templates = {} @@ -391,6 +343,30 @@ def _load_schemas(self): self._loaded_schemas = loaded_schemas def _fill_template(self, child_data, template_def): + """Fill template based on schema definition and template definition. + + Based on `child_data` is `template_def` modified and result is + returned. + + Template definition may have defined data to fill which + should be filled with data from child data. + + Child data may contain more than one output definition of an template. + + Child data can define paths to skip. Path is full path of an item + which won't be returned. + + TODO: + Be able to handle wrapper items here. + + Args: + child_data(dict): Schema data of template item. + template_def(dict): Template definition that will be filled with + child_data. + + Returns: + list: Resolved template always returns list of schemas. + """ template_name = child_data["name"] # Default value must be dictionary (NOT list) @@ -424,6 +400,7 @@ def _fill_template_data( required_keys=None, missing_keys=None ): + """Fill template values with data from schema data.""" first = False if required_keys is None: first = True @@ -547,6 +524,12 @@ def _fill_template_data( return output def _pop_metadata_item(self, template_def): + """Pop template metadata from template definition. + + Template metadata may define default values if are not passed from + schema data. + """ + found_idx = None for idx, item in enumerate(template_def): if not isinstance(item, dict): From d91f47084860a8c40b89f37506689a457bc99e2a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Jun 2021 14:11:26 +0200 Subject: [PATCH 17/18] handle full value replacement in template --- openpype/settings/entities/lib.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index 31071a2d309..d747c3e85e4 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -497,6 +497,7 @@ def _fill_template_data( .replace("{{", "__dbcb__") .replace("}}", "__decb__") ) + full_replacement = False for replacement_string in template_key_pattern.findall(template): key = str(replacement_string[1:-1]) required_keys.add(key) @@ -509,11 +510,19 @@ def _fill_template_data( # Replace the value with value from templates data # - with this is possible to set value with different type template = value + full_replacement = True else: # Only replace the key in string template = template.replace(replacement_string, value) - output = template.replace("__dbcb__", "{").replace("__decb__", "}") + if not full_replacement: + output = ( + template + .replace("__dbcb__", "{") + .replace("__decb__", "}") + ) + else: + output = template else: output = template From b21f827790727433393397c2931d21efb099594a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 30 Jun 2021 11:22:43 +0200 Subject: [PATCH 18/18] added few docstrings --- openpype/settings/entities/lib.py | 53 ++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index d747c3e85e4..42a08232b97 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -127,7 +127,16 @@ def gui_types(self): return self._gui_types def get_schema(self, schema_name): - """Get schema definition data by it's name.""" + """Get schema definition data by it's name. + + Returns: + dict: Copy of schema loaded from json files. + + Raises: + KeyError: When schema name is stored in loaded templates or json + file was not possible to parse or when schema name was not + found. + """ if schema_name not in self._loaded_schemas: if schema_name in self._loaded_templates: raise KeyError(( @@ -148,7 +157,16 @@ def get_schema(self, schema_name): return copy.deepcopy(self._loaded_schemas[schema_name]) def get_template(self, template_name): - """Get template definition data by it's name.""" + """Get template definition data by it's name. + + Returns: + list: Copy of template items loaded from json files. + + Raises: + KeyError: When template name is stored in loaded schemas or json + file was not possible to parse or when template name was not + found. + """ if template_name not in self._loaded_templates: if template_name in self._loaded_schemas: raise KeyError(( @@ -232,7 +250,16 @@ def create_schema_object(self, schema_data, *args, **kwargs): return klass(schema_data, *args, **kwargs) def _load_types(self): - """Prepare entity types for cretion of their objects.""" + """Prepare entity types for cretion of their objects. + + Currently all classes in `openpype.settings.entities` that inherited + from `BaseEntity` are stored as loaded types. GUI types are stored to + separated attribute to not mess up api access of entities. + + TODOs: + Add more dynamic way how to add custom types from anywhere and + better handling of abstract classes. Skipping them is dangerous. + """ from openpype.settings import entities @@ -400,7 +427,25 @@ def _fill_template_data( required_keys=None, missing_keys=None ): - """Fill template values with data from schema data.""" + """Fill template values with data from schema data. + + Template has more abilities than schemas. It is expected that template + will be used at multiple places (but may not). Schema represents + exactly one entity and it's children but template may represent more + entities. + + Template can have "keys to fill" from their definition. Some key may be + required and some may be optional because template has their default + values defined. + + Template also have ability to "skip paths" which means to skip entities + from it's content. A template can be used across multiple places with + different requirements. + + Raises: + SchemaTemplateMissingKeys: When fill data do not contain all + required keys for template. + """ first = False if required_keys is None: first = True