From ac33b567b4e6813afa695ef784639fe2e069dfcd Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Sat, 24 Jun 2023 14:34:18 +0900 Subject: [PATCH 1/9] start on multi-dimensional typeDSL --- schema_salad/metaschema/typedsl_res.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/schema_salad/metaschema/typedsl_res.yml b/schema_salad/metaschema/typedsl_res.yml index de5d62bae..8d347b840 100644 --- a/schema_salad/metaschema/typedsl_res.yml +++ b/schema_salad/metaschema/typedsl_res.yml @@ -6,9 +6,14 @@ * If the type ends with a question mark `?`, the question mark is stripped off and the type is expanded to a union with `null` * If the type ends with square brackets `[]` it is expanded to an array with items of the preceding type symbol - * The type may end with both `[]?` to indicate it is an optional array. + * The type may end with both square brackets with one question mark (`[]?`) to indicate it is an optional array. * Identifier resolution is applied after type DSL expansion. + Starting with Schema Salad version 1.3, fields tagged with `typeDSL: true` in `jsonldPredicate` have the following additional behavior: + + * Square brackes `[]` can be repeated to indicate 2, 3, or more dimensional array types. + * These multi-dimensional arrays, like 1-dimensional arrays, can be combined with `?` (for example, `[][]?`) to indicate that it is an optional multi-dimensional array. + ### Type DSL example Given the following schema: From fff6008e5bc87716d3936f6045666111050eda40 Mon Sep 17 00:00:00 2001 From: Tomoya Tanjo Date: Mon, 26 Jun 2023 12:08:23 +0900 Subject: [PATCH 2/9] Fix cwl-v1.3#10 - suppert nested typeDSL --- schema_salad/metaschema.py | 44 +++++++++++--------------- schema_salad/python_codegen_support.py | 44 +++++++++++--------------- schema_salad/ref_resolver.py | 44 +++++++++++++++----------- schema_salad/tests/metaschema-pre.yml | 4 +-- schema_salad/tests/test_makedoc.py | 2 +- 5 files changed, 67 insertions(+), 71 deletions(-) diff --git a/schema_salad/metaschema.py b/schema_salad/metaschema.py index 1b5c17e17..77ec43989 100644 --- a/schema_salad/metaschema.py +++ b/schema_salad/metaschema.py @@ -6,7 +6,6 @@ import logging import os import pathlib -import re import tempfile import uuid as _uuid__ # pylint: disable=unused-import # noqa: F401 import xml.sax # nosec @@ -574,8 +573,6 @@ def load(self, doc, baseuri, loadingOptions, docRoot=None): class _TypeDSLLoader(_Loader): - typeDSLregex = re.compile(r"^([^[?]+)(\[\])?(\?)?$") - def __init__(self, inner, refScope): # type: (_Loader, Union[int, None]) -> None self.inner = inner @@ -587,28 +584,25 @@ def resolve( baseuri, # type: str loadingOptions, # type: LoadingOptions ): - # type: (...) -> Union[List[Union[Dict[str, str], str]], Dict[str, str], str] - m = self.typeDSLregex.match(doc) - if m: - group1 = m.group(1) - assert group1 is not None # nosec - first = expand_url(group1, baseuri, loadingOptions, False, True, self.refScope) - second = third = None - if bool(m.group(2)): - second = {"type": "array", "items": first} - # second = CommentedMap((("type", "array"), - # ("items", first))) - # second.lc.add_kv_line_col("type", lc) - # second.lc.add_kv_line_col("items", lc) - # second.lc.filename = filename - if bool(m.group(3)): - third = ["null", second or first] - # third = CommentedSeq(["null", second or first]) - # third.lc.add_kv_line_col(0, lc) - # third.lc.add_kv_line_col(1, lc) - # third.lc.filename = filename - return third or second or first - return doc + # type: (...) -> Union[List[Union[Dict[str, Any], str]], Dict[str, Any], str] + doc_ = doc + optional = False + if doc_.endswith("?"): + optional = True + doc_ = doc_[0:-1] + + if doc_.endswith("[]"): + items = self.resolve(doc_[0:-2], baseuri, loadingOptions) + if isinstance(items, str): + items = expand_url(items, baseuri, loadingOptions, False, True, self.refScope) + expanded = {"type": "array", "items": items} # type: Union[Dict[str, Any], str] + else: + expanded = expand_url(doc_, baseuri, loadingOptions, False, True, self.refScope) + + if optional: + return ["null", expanded] + else: + return expanded def load(self, doc, baseuri, loadingOptions, docRoot=None): # type: (Any, str, LoadingOptions, Optional[str]) -> Any diff --git a/schema_salad/python_codegen_support.py b/schema_salad/python_codegen_support.py index b81cb52b9..f4e225041 100644 --- a/schema_salad/python_codegen_support.py +++ b/schema_salad/python_codegen_support.py @@ -3,7 +3,6 @@ import logging import os import pathlib -import re import tempfile import uuid as _uuid__ # pylint: disable=unused-import # noqa: F401 import xml.sax # nosec @@ -571,8 +570,6 @@ def load(self, doc, baseuri, loadingOptions, docRoot=None): class _TypeDSLLoader(_Loader): - typeDSLregex = re.compile(r"^([^[?]+)(\[\])?(\?)?$") - def __init__(self, inner, refScope): # type: (_Loader, Union[int, None]) -> None self.inner = inner @@ -584,28 +581,25 @@ def resolve( baseuri, # type: str loadingOptions, # type: LoadingOptions ): - # type: (...) -> Union[List[Union[Dict[str, str], str]], Dict[str, str], str] - m = self.typeDSLregex.match(doc) - if m: - group1 = m.group(1) - assert group1 is not None # nosec - first = expand_url(group1, baseuri, loadingOptions, False, True, self.refScope) - second = third = None - if bool(m.group(2)): - second = {"type": "array", "items": first} - # second = CommentedMap((("type", "array"), - # ("items", first))) - # second.lc.add_kv_line_col("type", lc) - # second.lc.add_kv_line_col("items", lc) - # second.lc.filename = filename - if bool(m.group(3)): - third = ["null", second or first] - # third = CommentedSeq(["null", second or first]) - # third.lc.add_kv_line_col(0, lc) - # third.lc.add_kv_line_col(1, lc) - # third.lc.filename = filename - return third or second or first - return doc + # type: (...) -> Union[List[Union[Dict[str, Any], str]], Dict[str, Any], str] + doc_ = doc + optional = False + if doc_.endswith("?"): + optional = True + doc_ = doc_[0:-1] + + if doc_.endswith("[]"): + items = self.resolve(doc_[0:-2], baseuri, loadingOptions) + if isinstance(items, str): + items = expand_url(items, baseuri, loadingOptions, False, True, self.refScope) + expanded = {"type": "array", "items": items} # type: Union[Dict[str, Any], str] + else: + expanded = expand_url(doc_, baseuri, loadingOptions, False, True, self.refScope) + + if optional: + return ["null", expanded] + else: + return expanded def load(self, doc, baseuri, loadingOptions, docRoot=None): # type: (Any, str, LoadingOptions, Optional[str]) -> Any diff --git a/schema_salad/ref_resolver.py b/schema_salad/ref_resolver.py index b06e51b27..d71f7a475 100644 --- a/schema_salad/ref_resolver.py +++ b/schema_salad/ref_resolver.py @@ -51,7 +51,6 @@ ) _logger = logging.getLogger("salad") -typeDSLregex = re.compile(r"^([^[?]+)(\[\])?(\?)?$") def file_uri(path: str, split_frag: bool = False) -> str: @@ -631,23 +630,32 @@ def _type_dsl( if not isinstance(t, str): return t - m = typeDSLregex.match(t) - if not m: - return t - first = m.group(1) - assert first # nosec - second = third = None - if bool(m.group(2)): - second = CommentedMap((("type", "array"), ("items", first))) - second.lc.add_kv_line_col("type", lc) - second.lc.add_kv_line_col("items", lc) - second.lc.filename = filename - if bool(m.group(3)): - third = CommentedSeq(["null", second or first]) - third.lc.add_kv_line_col(0, lc) - third.lc.add_kv_line_col(1, lc) - third.lc.filename = filename - return third or second or first + t_ = t + optional = False + if t_.endswith("?"): + optional = True + t_ = t_[0:-1] + + if t_.endswith("[]"): + items = self._type_dsl(t_[0:-2], lc, filename) + cmap = CommentedMap((("type", "array"), ("items", items))) + cmap.lc.add_kv_line_col("type", lc) + cmap.lc.add_kv_line_col("items", lc) + cmap.lc.filename = filename + expanded: Union[str, CommentedMap, CommentedSeq] = cmap + else: + expanded = t_ + + if optional: + cs = CommentedSeq(["null", expanded]) + cs.lc.add_kv_line_col(0, lc) + cs.lc.add_kv_line_col(1, lc) + cs.lc.filename = filename + ret: Union[str, CommentedMap, CommentedSeq] = cs + else: + ret = expanded + + return ret def _secondaryFile_dsl( self, diff --git a/schema_salad/tests/metaschema-pre.yml b/schema_salad/tests/metaschema-pre.yml index 710ddca1b..f35d994b1 100644 --- a/schema_salad/tests/metaschema-pre.yml +++ b/schema_salad/tests/metaschema-pre.yml @@ -40,7 +40,7 @@ "```\n\nThis becomes:\n\n```\n", "{\n \"mapped\": [\n {\n \"value\": \"daphne\",\n \"key\": \"fred\"\n },\n {\n \"value\": \"scooby\",\n \"key\": \"shaggy\"\n }\n ]\n}\n", "```\n", - "## Domain Specific Language for types\n\nFields may be tagged `typeDSL: true` in `jsonldPredicate`. If so, the field is expanded using the\nfollowing micro-DSL for schema salad types:\n\n* If the type ends with a question mark `?`, the question mark is stripped off and the type is expanded to a union with `null`\n* If the type ends with square brackets `[]` it is expanded to an array with items of the preceding type symbol\n* The type may end with both `[]?` to indicate it is an optional array.\n* Identifier resolution is applied after type DSL expansion.\n\n### Type DSL example\n\nGiven the following schema:\n\n```\n", + "## Domain Specific Language for types\n\nFields may be tagged `typeDSL: true` in `jsonldPredicate`. If so, the field is expanded using the\nfollowing micro-DSL for schema salad types:\n\n* If the type ends with a question mark `?`, the question mark is stripped off and the type is expanded to a union with `null`\n* If the type ends with square brackets `[]` it is expanded to an array with items of the preceding type symbol\n* The type may end with both square brackets with one question mark (`[]?`) to indicate it is an optional array.\n* Identifier resolution is applied after type DSL expansion.\n\nStarting with Schema Salad version 1.3, fields tagged with `typeDSL: true` in `jsonldPredicate` have the following additional behavior:\n\n* Square brackes `[]` can be repeated to indicate 2, 3, or more dimensional array types.\n* These multi-dimensional arrays, like 1-dimensional arrays, can be combined with `?` (for example, `[][]?`) to indicate that it is an optional multi-dimensional array.\n\n### Type DSL example\n\nGiven the following schema:\n\n```\n", "{\n \"$graph\": [\n {\"$import\": \"metaschema_base.yml\"},\n {\n \"name\": \"TypeDSLExample\",\n \"type\": \"record\",\n \"documentRoot\": true,\n \"fields\": [{\n \"name\": \"extype\",\n \"type\": \"string\",\n \"jsonldPredicate\": {\n _type: \"@vocab\",\n \"typeDSL\": true\n }\n }]\n }]\n}\n", "```\n\nProcess the following example:\n\n```\n", "[{\n \"extype\": \"string\"\n}, {\n \"extype\": \"string?\"\n}, {\n \"extype\": \"string[]\"\n}, {\n \"extype\": \"string[]?\"\n}]\n", @@ -674,4 +674,4 @@ } ] } -] +] \ No newline at end of file diff --git a/schema_salad/tests/test_makedoc.py b/schema_salad/tests/test_makedoc.py index 423bb6a5d..947ce8963 100644 --- a/schema_salad/tests/test_makedoc.py +++ b/schema_salad/tests/test_makedoc.py @@ -239,5 +239,5 @@ def test_detect_changes_in_html(metaschema_doc: str, tmp_path: Path) -> None: with open(result, "w") as h: h.write(metaschema_doc) assert ( - hasher.hexdigest() == "4e67cb0a28829f9e14ecce93183fcf89fedb2446ae6cd717ccb2966609fe76a2" + hasher.hexdigest() == "360fcb7860e6ccc15db70a20f95247a0b5ee729dd3446747463e68054e186e59" ), result From 5e9dde3582d869ae42d8f85931876f93cb3011dc Mon Sep 17 00:00:00 2001 From: Tomoya Tanjo Date: Mon, 26 Jun 2023 12:13:11 +0900 Subject: [PATCH 3/9] Add newline to metaschema-pre.yml --- schema_salad/tests/metaschema-pre.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema_salad/tests/metaschema-pre.yml b/schema_salad/tests/metaschema-pre.yml index f35d994b1..6224075d5 100644 --- a/schema_salad/tests/metaschema-pre.yml +++ b/schema_salad/tests/metaschema-pre.yml @@ -674,4 +674,4 @@ } ] } -] \ No newline at end of file +] From 510ebce9dc08f65dd02b7fcd8a5db48d2eae1710 Mon Sep 17 00:00:00 2001 From: Tomoya Tanjo Date: Mon, 26 Jun 2023 16:01:28 +0900 Subject: [PATCH 4/9] Add a test for nested typeDSL --- schema_salad/tests/test_examples.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/schema_salad/tests/test_examples.py b/schema_salad/tests/test_examples.py index dcbf3eecc..c3e8dcfdf 100644 --- a/schema_salad/tests/test_examples.py +++ b/schema_salad/tests/test_examples.py @@ -355,6 +355,9 @@ def test_typedsl_ref() -> None: ra, _ = ldr.resolve_all(cmap({"type": "File[]?"}), "") assert {"type": ["null", {"items": "File", "type": "array"}]} == ra + ra, _ = ldr.resolve_all(cmap({"type": "File[][]"}), "") + assert {"type": {"items": {"items": "File", "type": "array"}, "type": "array"}} == ra + def test_secondaryFile_dsl_ref() -> None: ldr = Loader({}) From 949d59b944414d35dc5a9ca22274e46bea2ad9bf Mon Sep 17 00:00:00 2001 From: Tomoya Tanjo Date: Wed, 28 Jun 2023 11:26:15 +0900 Subject: [PATCH 5/9] =?UTF-8?q?Tentative=20support=20for=20switching=20nes?= =?UTF-8?q?ted=20TypeDSL=20with=E3=80=80`saladVersion`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- schema_salad/metaschema.py | 21 ++++++++++++++++++--- schema_salad/python_codegen_support.py | 21 ++++++++++++++++++--- schema_salad/ref_resolver.py | 9 +++++++-- schema_salad/tests/test_examples.py | 17 +++++++++++++++++ 4 files changed, 60 insertions(+), 8 deletions(-) diff --git a/schema_salad/metaschema.py b/schema_salad/metaschema.py index 77ec43989..7407a7153 100644 --- a/schema_salad/metaschema.py +++ b/schema_salad/metaschema.py @@ -59,6 +59,7 @@ class LoadingOptions: cache: CacheType imports: List[str] includes: List[str] + salad_version: str def __init__( self, @@ -73,6 +74,7 @@ def __init__( idx: Optional[IdxType] = None, imports: Optional[List[str]] = None, includes: Optional[List[str]] = None, + salad_version: Optional[str] = None, ) -> None: """Create a LoadingOptions object.""" self.original_doc = original_doc @@ -138,6 +140,11 @@ def __init__( self.vocab = _vocab self.rvocab = _rvocab + if salad_version: + self.salad_version = salad_version + else: + self.salad_version = "v1.0" + if namespaces is not None: self.vocab = self.vocab.copy() self.rvocab = self.rvocab.copy() @@ -592,9 +599,14 @@ def resolve( doc_ = doc_[0:-1] if doc_.endswith("[]"): - items = self.resolve(doc_[0:-2], baseuri, loadingOptions) - if isinstance(items, str): - items = expand_url(items, baseuri, loadingOptions, False, True, self.refScope) + salad_versions = [int(v) for v in loadingOptions.salad_version[1:].split(".")] + items = "" # type: Union[list[Union[dict[str, Any], str]], dict[str, Any], str] + if salad_versions < [1, 3]: + items = expand_url(doc_[0:-2], baseuri, loadingOptions, False, True, self.refScope) + else: + items = self.resolve(doc_[0:-2], baseuri, loadingOptions) + if isinstance(items, str): + items = expand_url(items, baseuri, loadingOptions, False, True, self.refScope) expanded = {"type": "array", "items": items} # type: Union[Dict[str, Any], str] else: expanded = expand_url(doc_, baseuri, loadingOptions, False, True, self.refScope) @@ -708,6 +720,9 @@ def _document_load( loadingOptions, ) + if "saladVersion" in doc: + loadingOptions.salad_version = doc["saladVersion"] + if docuri != baseuri: loadingOptions.idx[docuri] = loadingOptions.idx[baseuri] diff --git a/schema_salad/python_codegen_support.py b/schema_salad/python_codegen_support.py index f4e225041..a6c0b3642 100644 --- a/schema_salad/python_codegen_support.py +++ b/schema_salad/python_codegen_support.py @@ -56,6 +56,7 @@ class LoadingOptions: cache: CacheType imports: List[str] includes: List[str] + salad_version: str def __init__( self, @@ -70,6 +71,7 @@ def __init__( idx: Optional[IdxType] = None, imports: Optional[List[str]] = None, includes: Optional[List[str]] = None, + salad_version: Optional[str] = None, ) -> None: """Create a LoadingOptions object.""" self.original_doc = original_doc @@ -135,6 +137,11 @@ def __init__( self.vocab = _vocab self.rvocab = _rvocab + if salad_version: + self.salad_version = salad_version + else: + self.salad_version = "v1.0" + if namespaces is not None: self.vocab = self.vocab.copy() self.rvocab = self.rvocab.copy() @@ -589,9 +596,14 @@ def resolve( doc_ = doc_[0:-1] if doc_.endswith("[]"): - items = self.resolve(doc_[0:-2], baseuri, loadingOptions) - if isinstance(items, str): - items = expand_url(items, baseuri, loadingOptions, False, True, self.refScope) + salad_versions = [int(v) for v in loadingOptions.salad_version[1:].split(".")] + items = "" # type: Union[list[Union[dict[str, Any], str]], dict[str, Any], str] + if salad_versions < [1, 3]: + items = expand_url(doc_[0:-2], baseuri, loadingOptions, False, True, self.refScope) + else: + items = self.resolve(doc_[0:-2], baseuri, loadingOptions) + if isinstance(items, str): + items = expand_url(items, baseuri, loadingOptions, False, True, self.refScope) expanded = {"type": "array", "items": items} # type: Union[Dict[str, Any], str] else: expanded = expand_url(doc_, baseuri, loadingOptions, False, True, self.refScope) @@ -705,6 +717,9 @@ def _document_load( loadingOptions, ) + if "saladVersion" in doc: + loadingOptions.salad_version = doc["saladVersion"] + if docuri != baseuri: loadingOptions.idx[docuri] = loadingOptions.idx[baseuri] diff --git a/schema_salad/ref_resolver.py b/schema_salad/ref_resolver.py index d71f7a475..702a35217 100644 --- a/schema_salad/ref_resolver.py +++ b/schema_salad/ref_resolver.py @@ -205,6 +205,7 @@ def __init__( self.subscopes: Dict[str, str] = {} self.secondaryFile_dsl_fields: Set[str] = set() self.allow_attachments = allow_attachments + self.salad_version = "v1.0" self.add_context(ctx) @@ -637,8 +638,12 @@ def _type_dsl( t_ = t_[0:-1] if t_.endswith("[]"): - items = self._type_dsl(t_[0:-2], lc, filename) - cmap = CommentedMap((("type", "array"), ("items", items))) + salad_versions = [int(v) for v in self.salad_version[1:].split(".")] + if salad_versions < [1, 3]: + cmap = CommentedMap((("type", "array"), ("items", t_[0:-2]))) + else: + items = self._type_dsl(t_[0:-2], lc, filename) + cmap = CommentedMap((("type", "array"), ("items", items))) cmap.lc.add_kv_line_col("type", lc) cmap.lc.add_kv_line_col("items", lc) cmap.lc.filename = filename diff --git a/schema_salad/tests/test_examples.py b/schema_salad/tests/test_examples.py index c3e8dcfdf..fac94148f 100644 --- a/schema_salad/tests/test_examples.py +++ b/schema_salad/tests/test_examples.py @@ -355,6 +355,23 @@ def test_typedsl_ref() -> None: ra, _ = ldr.resolve_all(cmap({"type": "File[]?"}), "") assert {"type": ["null", {"items": "File", "type": "array"}]} == ra + assert ldr.salad_version == "v1.0" + ra, _ = ldr.resolve_all(cmap({"type": "File[][]"}), "") + assert {"type": {"items": "File[]", "type": "array"}} == ra + + +def test_nested_typedsl_ref() -> None: + ldr = Loader({}) + ldr.add_context( + { + "File": "http://example.com/File", + "null": "http://example.com/null", + "array": "http://example.com/array", + "type": {"@type": "@vocab", "typeDSL": True}, + } + ) + ldr.salad_version = "v1.3" + ra, _ = ldr.resolve_all(cmap({"type": "File[][]"}), "") assert {"type": {"items": {"items": "File", "type": "array"}, "type": "array"}} == ra From 2c1c88058b408fb1963d17e5eb45858772b66dc6 Mon Sep 17 00:00:00 2001 From: Tomoya Tanjo Date: Wed, 28 Jun 2023 11:36:10 +0900 Subject: [PATCH 6/9] Fix typo --- schema_salad/metaschema.py | 2 +- schema_salad/python_codegen_support.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/schema_salad/metaschema.py b/schema_salad/metaschema.py index 7407a7153..e1159bb33 100644 --- a/schema_salad/metaschema.py +++ b/schema_salad/metaschema.py @@ -600,7 +600,7 @@ def resolve( if doc_.endswith("[]"): salad_versions = [int(v) for v in loadingOptions.salad_version[1:].split(".")] - items = "" # type: Union[list[Union[dict[str, Any], str]], dict[str, Any], str] + items = "" # type: Union[List[Union[dict[str, Any], str]], dict[str, Any], str] if salad_versions < [1, 3]: items = expand_url(doc_[0:-2], baseuri, loadingOptions, False, True, self.refScope) else: diff --git a/schema_salad/python_codegen_support.py b/schema_salad/python_codegen_support.py index a6c0b3642..fa12723b0 100644 --- a/schema_salad/python_codegen_support.py +++ b/schema_salad/python_codegen_support.py @@ -597,7 +597,7 @@ def resolve( if doc_.endswith("[]"): salad_versions = [int(v) for v in loadingOptions.salad_version[1:].split(".")] - items = "" # type: Union[list[Union[dict[str, Any], str]], dict[str, Any], str] + items = "" # type: Union[List[Union[dict[str, Any], str]], dict[str, Any], str] if salad_versions < [1, 3]: items = expand_url(doc_[0:-2], baseuri, loadingOptions, False, True, self.refScope) else: From dad9beaa9804560a9e9d94407786e833d65f3cf3 Mon Sep 17 00:00:00 2001 From: Tomoya Tanjo Date: Wed, 28 Jun 2023 11:39:27 +0900 Subject: [PATCH 7/9] Fix typo --- schema_salad/metaschema.py | 2 +- schema_salad/python_codegen_support.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/schema_salad/metaschema.py b/schema_salad/metaschema.py index e1159bb33..bda2d0af6 100644 --- a/schema_salad/metaschema.py +++ b/schema_salad/metaschema.py @@ -600,7 +600,7 @@ def resolve( if doc_.endswith("[]"): salad_versions = [int(v) for v in loadingOptions.salad_version[1:].split(".")] - items = "" # type: Union[List[Union[dict[str, Any], str]], dict[str, Any], str] + items = "" # type: Union[List[Union[Dict[str, Any], str]], Dict[str, Any], str] if salad_versions < [1, 3]: items = expand_url(doc_[0:-2], baseuri, loadingOptions, False, True, self.refScope) else: diff --git a/schema_salad/python_codegen_support.py b/schema_salad/python_codegen_support.py index fa12723b0..017a12f3c 100644 --- a/schema_salad/python_codegen_support.py +++ b/schema_salad/python_codegen_support.py @@ -597,7 +597,7 @@ def resolve( if doc_.endswith("[]"): salad_versions = [int(v) for v in loadingOptions.salad_version[1:].split(".")] - items = "" # type: Union[List[Union[dict[str, Any], str]], dict[str, Any], str] + items = "" # type: Union[List[Union[Dict[str, Any], str]], Dict[str, Any], str] if salad_versions < [1, 3]: items = expand_url(doc_[0:-2], baseuri, loadingOptions, False, True, self.refScope) else: From 15833b9592b199b69d7bd565d51523a5fb95d5f2 Mon Sep 17 00:00:00 2001 From: Tomoya Tanjo Date: Thu, 29 Jun 2023 15:32:26 +0900 Subject: [PATCH 8/9] Fix runtime and code-generated handling of nested typeDSL --- schema_salad/codegen.py | 6 +++++- schema_salad/main.py | 5 ++++- schema_salad/metaschema.py | 26 +++++++++----------------- schema_salad/python_codegen.py | 4 +++- schema_salad/python_codegen_support.py | 17 ++++------------- schema_salad/ref_resolver.py | 8 +++++++- schema_salad/schema.py | 3 ++- schema_salad/tests/test_examples.py | 6 ++---- 8 files changed, 36 insertions(+), 39 deletions(-) diff --git a/schema_salad/codegen.py b/schema_salad/codegen.py index b74f00690..7f50558c1 100644 --- a/schema_salad/codegen.py +++ b/schema_salad/codegen.py @@ -57,6 +57,8 @@ def codegen( else ".".join(list(reversed(sp.netloc.split("."))) + sp.path.strip("/").split("/")) ) info = parser_info or pkg + salad_version = schema_metadata.get("saladVersion", "v1.1") + if lang in set(["python", "cpp", "dlang"]): if target: dest: Union[TextIOWrapper, TextIO] = open(target, mode="w", encoding="utf-8") @@ -83,7 +85,9 @@ def codegen( ) gen.parse(j) return - gen = PythonCodeGen(dest, copyright=copyright, parser_info=info) + gen = PythonCodeGen( + dest, copyright=copyright, parser_info=info, salad_version=salad_version + ) elif lang == "java": gen = JavaCodeGen( diff --git a/schema_salad/main.py b/schema_salad/main.py index eb440b677..704bdc264 100644 --- a/schema_salad/main.py +++ b/schema_salad/main.py @@ -299,7 +299,10 @@ def main(argsl: Optional[List[str]] = None) -> int: raise ValidationException(f"Expected a CommentedSeq, got {type(schema_doc)}: {schema_doc}.") # Create the loader that will be used to load the target document. - document_loader = Loader(schema_ctx, skip_schemas=args.skip_schemas) + schema_version = schema_metadata.get("saladVersion", None) + document_loader = Loader( + schema_ctx, skip_schemas=args.skip_schemas, salad_version=schema_version + ) if args.codegen: codegen.codegen( diff --git a/schema_salad/metaschema.py b/schema_salad/metaschema.py index bda2d0af6..411705af7 100644 --- a/schema_salad/metaschema.py +++ b/schema_salad/metaschema.py @@ -59,7 +59,6 @@ class LoadingOptions: cache: CacheType imports: List[str] includes: List[str] - salad_version: str def __init__( self, @@ -74,7 +73,6 @@ def __init__( idx: Optional[IdxType] = None, imports: Optional[List[str]] = None, includes: Optional[List[str]] = None, - salad_version: Optional[str] = None, ) -> None: """Create a LoadingOptions object.""" self.original_doc = original_doc @@ -140,11 +138,6 @@ def __init__( self.vocab = _vocab self.rvocab = _rvocab - if salad_version: - self.salad_version = salad_version - else: - self.salad_version = "v1.0" - if namespaces is not None: self.vocab = self.vocab.copy() self.rvocab = self.rvocab.copy() @@ -580,10 +573,11 @@ def load(self, doc, baseuri, loadingOptions, docRoot=None): class _TypeDSLLoader(_Loader): - def __init__(self, inner, refScope): - # type: (_Loader, Union[int, None]) -> None + def __init__(self, inner, refScope, salad_version): + # type: (_Loader, Union[int, None], str) -> None self.inner = inner self.refScope = refScope + self.salad_version = salad_version def resolve( self, @@ -599,7 +593,7 @@ def resolve( doc_ = doc_[0:-1] if doc_.endswith("[]"): - salad_versions = [int(v) for v in loadingOptions.salad_version[1:].split(".")] + salad_versions = [int(v) for v in self.salad_version[1:].split(".")] items = "" # type: Union[List[Union[Dict[str, Any], str]], Dict[str, Any], str] if salad_versions < [1, 3]: items = expand_url(doc_[0:-2], baseuri, loadingOptions, False, True, self.refScope) @@ -720,9 +714,6 @@ def _document_load( loadingOptions, ) - if "saladVersion" in doc: - loadingOptions.salad_version = doc["saladVersion"] - if docuri != baseuri: loadingOptions.idx[docuri] = loadingOptions.idx[baseuri] @@ -3585,6 +3576,7 @@ def save( typedsl_union_of_PrimitiveTypeLoader_or_RecordSchemaLoader_or_EnumSchemaLoader_or_ArraySchemaLoader_or_strtype_or_array_of_union_of_PrimitiveTypeLoader_or_RecordSchemaLoader_or_EnumSchemaLoader_or_ArraySchemaLoader_or_strtype_2 = _TypeDSLLoader( union_of_PrimitiveTypeLoader_or_RecordSchemaLoader_or_EnumSchemaLoader_or_ArraySchemaLoader_or_strtype_or_array_of_union_of_PrimitiveTypeLoader_or_RecordSchemaLoader_or_EnumSchemaLoader_or_ArraySchemaLoader_or_strtype, 2, + "v1.1", ) array_of_RecordFieldLoader = _ArrayLoader(RecordFieldLoader) union_of_None_type_or_array_of_RecordFieldLoader = _UnionLoader( @@ -3597,7 +3589,7 @@ def save( union_of_None_type_or_array_of_RecordFieldLoader, "name", "type" ) Record_nameLoader = _EnumLoader(("record",), "Record_name") -typedsl_Record_nameLoader_2 = _TypeDSLLoader(Record_nameLoader, 2) +typedsl_Record_nameLoader_2 = _TypeDSLLoader(Record_nameLoader, 2, "v1.1") union_of_None_type_or_strtype = _UnionLoader( ( None_type, @@ -3609,7 +3601,7 @@ def save( ) uri_array_of_strtype_True_False_None = _URILoader(array_of_strtype, True, False, None) Enum_nameLoader = _EnumLoader(("enum",), "Enum_name") -typedsl_Enum_nameLoader_2 = _TypeDSLLoader(Enum_nameLoader, 2) +typedsl_Enum_nameLoader_2 = _TypeDSLLoader(Enum_nameLoader, 2, "v1.1") uri_union_of_PrimitiveTypeLoader_or_RecordSchemaLoader_or_EnumSchemaLoader_or_ArraySchemaLoader_or_strtype_or_array_of_union_of_PrimitiveTypeLoader_or_RecordSchemaLoader_or_EnumSchemaLoader_or_ArraySchemaLoader_or_strtype_False_True_2 = _URILoader( union_of_PrimitiveTypeLoader_or_RecordSchemaLoader_or_EnumSchemaLoader_or_ArraySchemaLoader_or_strtype_or_array_of_union_of_PrimitiveTypeLoader_or_RecordSchemaLoader_or_EnumSchemaLoader_or_ArraySchemaLoader_or_strtype, False, @@ -3617,7 +3609,7 @@ def save( 2, ) Array_nameLoader = _EnumLoader(("array",), "Array_name") -typedsl_Array_nameLoader_2 = _TypeDSLLoader(Array_nameLoader, 2) +typedsl_Array_nameLoader_2 = _TypeDSLLoader(Array_nameLoader, 2, "v1.1") union_of_None_type_or_booltype = _UnionLoader( ( None_type, @@ -3674,7 +3666,7 @@ def save( union_of_None_type_or_array_of_SpecializeDefLoader, "specializeFrom", "specializeTo" ) Documentation_nameLoader = _EnumLoader(("documentation",), "Documentation_name") -typedsl_Documentation_nameLoader_2 = _TypeDSLLoader(Documentation_nameLoader, 2) +typedsl_Documentation_nameLoader_2 = _TypeDSLLoader(Documentation_nameLoader, 2, "v1.1") union_of_SaladRecordSchemaLoader_or_SaladEnumSchemaLoader_or_DocumentationLoader = ( _UnionLoader( ( diff --git a/schema_salad/python_codegen.py b/schema_salad/python_codegen.py index fb23a4798..e653227a2 100644 --- a/schema_salad/python_codegen.py +++ b/schema_salad/python_codegen.py @@ -82,6 +82,7 @@ def __init__( out: IO[str], copyright: Optional[str], parser_info: str, + salad_version: str, ) -> None: super().__init__() self.out = out @@ -90,6 +91,7 @@ def __init__( self.idfield = "" self.copyright = copyright self.parser_info = parser_info + self.salad_version = salad_version @staticmethod def safe_name(name: str) -> str: @@ -629,7 +631,7 @@ def typedsl_loader(self, inner: TypeDef, ref_scope: Optional[int]) -> TypeDef: return self.declare_type( TypeDef( f"typedsl_{self.safe_name(inner.name)}_{ref_scope}", - f"_TypeDSLLoader({self.safe_name(inner.name)}, {ref_scope})", + f"_TypeDSLLoader({self.safe_name(inner.name)}, {ref_scope}, '{self.salad_version}')", ) ) diff --git a/schema_salad/python_codegen_support.py b/schema_salad/python_codegen_support.py index 017a12f3c..09df821a4 100644 --- a/schema_salad/python_codegen_support.py +++ b/schema_salad/python_codegen_support.py @@ -56,7 +56,6 @@ class LoadingOptions: cache: CacheType imports: List[str] includes: List[str] - salad_version: str def __init__( self, @@ -71,7 +70,6 @@ def __init__( idx: Optional[IdxType] = None, imports: Optional[List[str]] = None, includes: Optional[List[str]] = None, - salad_version: Optional[str] = None, ) -> None: """Create a LoadingOptions object.""" self.original_doc = original_doc @@ -137,11 +135,6 @@ def __init__( self.vocab = _vocab self.rvocab = _rvocab - if salad_version: - self.salad_version = salad_version - else: - self.salad_version = "v1.0" - if namespaces is not None: self.vocab = self.vocab.copy() self.rvocab = self.rvocab.copy() @@ -577,10 +570,11 @@ def load(self, doc, baseuri, loadingOptions, docRoot=None): class _TypeDSLLoader(_Loader): - def __init__(self, inner, refScope): - # type: (_Loader, Union[int, None]) -> None + def __init__(self, inner, refScope, salad_version): + # type: (_Loader, Union[int, None], str) -> None self.inner = inner self.refScope = refScope + self.salad_version = salad_version def resolve( self, @@ -596,7 +590,7 @@ def resolve( doc_ = doc_[0:-1] if doc_.endswith("[]"): - salad_versions = [int(v) for v in loadingOptions.salad_version[1:].split(".")] + salad_versions = [int(v) for v in self.salad_version[1:].split(".")] items = "" # type: Union[List[Union[Dict[str, Any], str]], Dict[str, Any], str] if salad_versions < [1, 3]: items = expand_url(doc_[0:-2], baseuri, loadingOptions, False, True, self.refScope) @@ -717,9 +711,6 @@ def _document_load( loadingOptions, ) - if "saladVersion" in doc: - loadingOptions.salad_version = doc["saladVersion"] - if docuri != baseuri: loadingOptions.idx[docuri] = loadingOptions.idx[baseuri] diff --git a/schema_salad/ref_resolver.py b/schema_salad/ref_resolver.py index 702a35217..4de95fb8e 100644 --- a/schema_salad/ref_resolver.py +++ b/schema_salad/ref_resolver.py @@ -140,6 +140,7 @@ def SubLoader(loader: "Loader") -> "Loader": url_fields=loader.url_fields, allow_attachments=loader.allow_attachments, session=loader.session, + salad_version=loader.salad_version, ) @@ -157,6 +158,7 @@ def __init__( url_fields: Optional[Set[str]] = None, allow_attachments: Optional[AttachmentsType] = None, doc_cache: Union[str, bool] = True, + salad_version: Optional[str] = None, ) -> None: self.idx: IdxType = ( NormDict(lambda url: urllib.parse.urlsplit(url).geturl()) if idx is None else idx @@ -205,7 +207,11 @@ def __init__( self.subscopes: Dict[str, str] = {} self.secondaryFile_dsl_fields: Set[str] = set() self.allow_attachments = allow_attachments - self.salad_version = "v1.0" + + if salad_version: + self.salad_version = salad_version + else: + self.salad_version = "v1.1" self.add_context(ctx) diff --git a/schema_salad/schema.py b/schema_salad/schema.py index 19d9225bd..545f2989e 100644 --- a/schema_salad/schema.py +++ b/schema_salad/schema.py @@ -177,7 +177,8 @@ def get_metaschema() -> Tuple[Names, List[Dict[str, str]], Loader]: }, "typeDSL": saladp + "JsonldPredicate/typeDSL", "xsd": "http://www.w3.org/2001/XMLSchema#", - } + }, + salad_version="v1.3", ) for salad in SALAD_FILES: diff --git a/schema_salad/tests/test_examples.py b/schema_salad/tests/test_examples.py index fac94148f..bc472a330 100644 --- a/schema_salad/tests/test_examples.py +++ b/schema_salad/tests/test_examples.py @@ -333,7 +333,7 @@ def test_yaml_float_test() -> None: def test_typedsl_ref() -> None: - ldr = Loader({}) + ldr = Loader({}, salad_version="v1.1") ldr.add_context( { "File": "http://example.com/File", @@ -355,13 +355,12 @@ def test_typedsl_ref() -> None: ra, _ = ldr.resolve_all(cmap({"type": "File[]?"}), "") assert {"type": ["null", {"items": "File", "type": "array"}]} == ra - assert ldr.salad_version == "v1.0" ra, _ = ldr.resolve_all(cmap({"type": "File[][]"}), "") assert {"type": {"items": "File[]", "type": "array"}} == ra def test_nested_typedsl_ref() -> None: - ldr = Loader({}) + ldr = Loader({}, salad_version="v1.3") ldr.add_context( { "File": "http://example.com/File", @@ -370,7 +369,6 @@ def test_nested_typedsl_ref() -> None: "type": {"@type": "@vocab", "typeDSL": True}, } ) - ldr.salad_version = "v1.3" ra, _ = ldr.resolve_all(cmap({"type": "File[][]"}), "") assert {"type": {"items": {"items": "File", "type": "array"}, "type": "array"}} == ra From 77c2483b17190c964eb349485467ee35605d2126 Mon Sep 17 00:00:00 2001 From: Tomoya Tanjo Date: Fri, 30 Jun 2023 13:06:07 +0900 Subject: [PATCH 9/9] Fix the old typeDSL handling for better message --- schema_salad/metaschema.py | 9 +++++++-- schema_salad/python_codegen_support.py | 9 +++++++-- schema_salad/ref_resolver.py | 9 +++++++-- schema_salad/tests/test_examples.py | 5 +++-- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/schema_salad/metaschema.py b/schema_salad/metaschema.py index 411705af7..d86c23fb5 100644 --- a/schema_salad/metaschema.py +++ b/schema_salad/metaschema.py @@ -595,10 +595,15 @@ def resolve( if doc_.endswith("[]"): salad_versions = [int(v) for v in self.salad_version[1:].split(".")] items = "" # type: Union[List[Union[Dict[str, Any], str]], Dict[str, Any], str] + rest = doc_[0:-2] if salad_versions < [1, 3]: - items = expand_url(doc_[0:-2], baseuri, loadingOptions, False, True, self.refScope) + if rest.endswith("[]"): + # To show the error message with the original type + return doc + else: + items = expand_url(rest, baseuri, loadingOptions, False, True, self.refScope) else: - items = self.resolve(doc_[0:-2], baseuri, loadingOptions) + items = self.resolve(rest, baseuri, loadingOptions) if isinstance(items, str): items = expand_url(items, baseuri, loadingOptions, False, True, self.refScope) expanded = {"type": "array", "items": items} # type: Union[Dict[str, Any], str] diff --git a/schema_salad/python_codegen_support.py b/schema_salad/python_codegen_support.py index 09df821a4..afd0cbdc7 100644 --- a/schema_salad/python_codegen_support.py +++ b/schema_salad/python_codegen_support.py @@ -592,10 +592,15 @@ def resolve( if doc_.endswith("[]"): salad_versions = [int(v) for v in self.salad_version[1:].split(".")] items = "" # type: Union[List[Union[Dict[str, Any], str]], Dict[str, Any], str] + rest = doc_[0:-2] if salad_versions < [1, 3]: - items = expand_url(doc_[0:-2], baseuri, loadingOptions, False, True, self.refScope) + if rest.endswith("[]"): + # To show the error message with the original type + return doc + else: + items = expand_url(rest, baseuri, loadingOptions, False, True, self.refScope) else: - items = self.resolve(doc_[0:-2], baseuri, loadingOptions) + items = self.resolve(rest, baseuri, loadingOptions) if isinstance(items, str): items = expand_url(items, baseuri, loadingOptions, False, True, self.refScope) expanded = {"type": "array", "items": items} # type: Union[Dict[str, Any], str] diff --git a/schema_salad/ref_resolver.py b/schema_salad/ref_resolver.py index 4de95fb8e..763e22058 100644 --- a/schema_salad/ref_resolver.py +++ b/schema_salad/ref_resolver.py @@ -645,10 +645,15 @@ def _type_dsl( if t_.endswith("[]"): salad_versions = [int(v) for v in self.salad_version[1:].split(".")] + rest = t_[0:-2] if salad_versions < [1, 3]: - cmap = CommentedMap((("type", "array"), ("items", t_[0:-2]))) + if rest.endswith("[]"): + # To show the error message with the original type + return t + else: + cmap = CommentedMap((("type", "array"), ("items", rest))) else: - items = self._type_dsl(t_[0:-2], lc, filename) + items = self._type_dsl(rest, lc, filename) cmap = CommentedMap((("type", "array"), ("items", items))) cmap.lc.add_kv_line_col("type", lc) cmap.lc.add_kv_line_col("items", lc) diff --git a/schema_salad/tests/test_examples.py b/schema_salad/tests/test_examples.py index bc472a330..079706cb5 100644 --- a/schema_salad/tests/test_examples.py +++ b/schema_salad/tests/test_examples.py @@ -9,6 +9,7 @@ import schema_salad.main import schema_salad.schema +from schema_salad.exceptions import ValidationException from schema_salad.jsonld_context import makerdf from schema_salad.ref_resolver import Loader, file_uri, uri_file_path from schema_salad.sourceline import SourceLine, cmap @@ -355,8 +356,8 @@ def test_typedsl_ref() -> None: ra, _ = ldr.resolve_all(cmap({"type": "File[]?"}), "") assert {"type": ["null", {"items": "File", "type": "array"}]} == ra - ra, _ = ldr.resolve_all(cmap({"type": "File[][]"}), "") - assert {"type": {"items": "File[]", "type": "array"}} == ra + with pytest.raises(ValidationException): + ldr.resolve_all(cmap({"type": "File[][]"}), "") def test_nested_typedsl_ref() -> None: