From cf14f2efdd34232e9edbc94bad5665993dea74aa Mon Sep 17 00:00:00 2001 From: Michael Franklin Date: Wed, 20 May 2020 16:09:30 +1000 Subject: [PATCH 1/7] Add optional_fields set to codegen begin_class --- schema_salad/codegen_base.py | 1 + schema_salad/python_codegen.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/schema_salad/codegen_base.py b/schema_salad/codegen_base.py index 32f2f18d..dbc017ff 100644 --- a/schema_salad/codegen_base.py +++ b/schema_salad/codegen_base.py @@ -76,6 +76,7 @@ def begin_class( abstract: bool, field_names: MutableSequence[str], idfield: str, + optional_fields: MutableSequence[str], ) -> None: """Produce the header for the given class.""" raise NotImplementedError() diff --git a/schema_salad/python_codegen.py b/schema_salad/python_codegen.py index 25aacce1..e38f6863 100644 --- a/schema_salad/python_codegen.py +++ b/schema_salad/python_codegen.py @@ -80,6 +80,7 @@ def begin_class( abstract, # type: bool field_names, # type: MutableSequence[str] idfield, # type: str + optional_fields, # type: MutableSequence[str] ): # type: (...) -> None classname = self.safe_name(classname) @@ -102,11 +103,21 @@ def begin_class( self.out.write(" pass\n\n\n") return + required_field_names = [f for f in field_names if f not in optional_fields] + optional_field_names = [f for f in field_names if f in optional_fields] + safe_inits = [" self,"] # type: List[str] safe_inits.extend( [ " {}, # type: Any".format(self.safe_name(f)) - for f in field_names + for f in required_field_names + if f != "class" + ] + ) + safe_inits.extend( + [ + " {}=None, # type: Any".format(self.safe_name(f)) + for f in optional_field_names if f != "class" ] ) From 388f561352bedde86e2ea901e6af30785342c98a Mon Sep 17 00:00:00 2001 From: Michael Franklin Date: Wed, 20 May 2020 16:12:05 +1000 Subject: [PATCH 2/7] Determine optional_fields for begin_class codegen --- schema_salad/codegen.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/schema_salad/codegen.py b/schema_salad/codegen.py index 762178c4..e409610b 100644 --- a/schema_salad/codegen.py +++ b/schema_salad/codegen.py @@ -1,7 +1,7 @@ """Generate langauge specific loaders for a particular SALAD schema.""" import sys from io import open -from typing import Any, Dict, List, MutableMapping, Optional +from typing import Any, Dict, List, MutableMapping, MutableSequence, Optional from . import schema from .codegen_base import CodeGenBase @@ -63,9 +63,17 @@ def codegen( document_roots.append(rec["name"]) field_names = [] + optional_fields = set() for field in rec.get("fields", []): - field_names.append(shortname(field["name"])) - + field_name = shortname(field["name"]) + field_names.append(field_name) + tp = field["type"] + if ( + isinstance(tp, MutableSequence) + and tp[0] == "https://w3id.org/cwl/salad#null" + ): + optional_fields.add(field_name) + idfield = "" for field in rec.get("fields", []): if field.get("jsonldPredicate") == "@id": @@ -78,6 +86,7 @@ def codegen( rec.get("abstract", False), field_names, idfield, + optional_fields ) gen.add_vocab(shortname(rec["name"]), rec["name"]) From eb3d6bd4aba3d2c415d9d0093220182639bbbff5 Mon Sep 17 00:00:00 2001 From: Michael Franklin Date: Wed, 20 May 2020 16:14:53 +1000 Subject: [PATCH 3/7] Add unimplemented optional_fields to java_codegen --- schema_salad/java_codegen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/schema_salad/java_codegen.py b/schema_salad/java_codegen.py index bb463aad..16c51834 100644 --- a/schema_salad/java_codegen.py +++ b/schema_salad/java_codegen.py @@ -154,6 +154,7 @@ def begin_class( abstract: bool, field_names: MutableSequence[str], idfield: str, + optional_fields: MutableSequence[str], ) -> None: cls = self.interface_name(classname) self.current_class = cls From 8986a678a3f098df40e91e784f64f215a75469c1 Mon Sep 17 00:00:00 2001 From: Michael Franklin Date: Wed, 20 May 2020 16:15:27 +1000 Subject: [PATCH 4/7] Type annotations for begin_class on python codgen --- schema_salad/python_codegen.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/schema_salad/python_codegen.py b/schema_salad/python_codegen.py index e38f6863..735ba5fe 100644 --- a/schema_salad/python_codegen.py +++ b/schema_salad/python_codegen.py @@ -74,13 +74,13 @@ def prologue(self): def begin_class( self, # pylint: disable=too-many-arguments - classname, # type: str - extends, # type: MutableSequence[str] - doc, # type: str - abstract, # type: bool - field_names, # type: MutableSequence[str] - idfield, # type: str - optional_fields, # type: MutableSequence[str] + classname: str, + extends: MutableSequence[str], + doc: str, + abstract: bool, + field_names: MutableSequence[str], + idfield: str, + optional_fields: MutableSequence[str], ): # type: (...) -> None classname = self.safe_name(classname) From 069193a3c06382f3bd2860616cd72a8f5a2c9055 Mon Sep 17 00:00:00 2001 From: Michael Franklin Date: Wed, 20 May 2020 21:10:56 +1000 Subject: [PATCH 5/7] Fix mypy type error --- schema_salad/codegen_base.py | 4 ++-- schema_salad/java_codegen.py | 4 ++-- schema_salad/python_codegen.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/schema_salad/codegen_base.py b/schema_salad/codegen_base.py index dbc017ff..e0a96362 100644 --- a/schema_salad/codegen_base.py +++ b/schema_salad/codegen_base.py @@ -1,6 +1,6 @@ """Base class for the generation of loaders from schema-salad definitions.""" import collections -from typing import Any, Dict, List, MutableSequence, Optional, Union +from typing import Any, Dict, List, MutableSequence, Optional, Union, Set from . import schema @@ -76,7 +76,7 @@ def begin_class( abstract: bool, field_names: MutableSequence[str], idfield: str, - optional_fields: MutableSequence[str], + optional_fields: Set[str], ) -> None: """Produce the header for the given class.""" raise NotImplementedError() diff --git a/schema_salad/java_codegen.py b/schema_salad/java_codegen.py index 16c51834..6dedd3fc 100644 --- a/schema_salad/java_codegen.py +++ b/schema_salad/java_codegen.py @@ -4,7 +4,7 @@ import string from io import StringIO from io import open as io_open -from typing import Any, Dict, List, MutableMapping, MutableSequence, Optional, Union +from typing import Any, Dict, List, MutableMapping, MutableSequence, Optional, Union, Set from urllib.parse import urlsplit import pkg_resources @@ -154,7 +154,7 @@ def begin_class( abstract: bool, field_names: MutableSequence[str], idfield: str, - optional_fields: MutableSequence[str], + optional_fields: Set[str], ) -> None: cls = self.interface_name(classname) self.current_class = cls diff --git a/schema_salad/python_codegen.py b/schema_salad/python_codegen.py index 735ba5fe..0af552af 100644 --- a/schema_salad/python_codegen.py +++ b/schema_salad/python_codegen.py @@ -1,6 +1,6 @@ """Python code generator for a given schema salad definition.""" from io import StringIO -from typing import IO, Any, Dict, List, MutableMapping, MutableSequence, Optional, Union +from typing import IO, Any, Dict, List, MutableMapping, MutableSequence, Optional, Union, Set from pkg_resources import resource_stream @@ -80,7 +80,7 @@ def begin_class( abstract: bool, field_names: MutableSequence[str], idfield: str, - optional_fields: MutableSequence[str], + optional_fields: Set[str], ): # type: (...) -> None classname = self.safe_name(classname) From 4970eb9a00b1d0af59645760eb71025d8d4e38ce Mon Sep 17 00:00:00 2001 From: Michael Franklin Date: Wed, 20 May 2020 22:17:30 +1000 Subject: [PATCH 6/7] Make format --- schema_salad/codegen.py | 4 ++-- schema_salad/java_codegen.py | 11 ++++++++++- schema_salad/python_codegen.py | 12 +++++++++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/schema_salad/codegen.py b/schema_salad/codegen.py index e409610b..3952737e 100644 --- a/schema_salad/codegen.py +++ b/schema_salad/codegen.py @@ -73,7 +73,7 @@ def codegen( and tp[0] == "https://w3id.org/cwl/salad#null" ): optional_fields.add(field_name) - + idfield = "" for field in rec.get("fields", []): if field.get("jsonldPredicate") == "@id": @@ -86,7 +86,7 @@ def codegen( rec.get("abstract", False), field_names, idfield, - optional_fields + optional_fields, ) gen.add_vocab(shortname(rec["name"]), rec["name"]) diff --git a/schema_salad/java_codegen.py b/schema_salad/java_codegen.py index 6dedd3fc..6c73aa7c 100644 --- a/schema_salad/java_codegen.py +++ b/schema_salad/java_codegen.py @@ -4,7 +4,16 @@ import string from io import StringIO from io import open as io_open -from typing import Any, Dict, List, MutableMapping, MutableSequence, Optional, Union, Set +from typing import ( + Any, + Dict, + List, + MutableMapping, + MutableSequence, + Optional, + Union, + Set, +) from urllib.parse import urlsplit import pkg_resources diff --git a/schema_salad/python_codegen.py b/schema_salad/python_codegen.py index 0af552af..b05170ac 100644 --- a/schema_salad/python_codegen.py +++ b/schema_salad/python_codegen.py @@ -1,6 +1,16 @@ """Python code generator for a given schema salad definition.""" from io import StringIO -from typing import IO, Any, Dict, List, MutableMapping, MutableSequence, Optional, Union, Set +from typing import ( + IO, + Any, + Dict, + List, + MutableMapping, + MutableSequence, + Optional, + Union, + Set, +) from pkg_resources import resource_stream From b9a30fff334dc135845fcfb0b0e092d1a3acfa78 Mon Sep 17 00:00:00 2001 From: Michael Franklin Date: Thu, 21 May 2020 08:30:34 +1000 Subject: [PATCH 7/7] Make doc.load use kwargs instead of ordered args This is because in the python codegen change we modified the order of the params to allow the optional to be (=None) and only optionally specified. With this change, the cwl-util tests pass --- schema_salad/python_codegen.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/schema_salad/python_codegen.py b/schema_salad/python_codegen.py index b05170ac..2ad66509 100644 --- a/schema_salad/python_codegen.py +++ b/schema_salad/python_codegen.py @@ -258,10 +258,12 @@ def end_class(self, classname, field_names): " attrs = frozenset({attrs})\n".format(attrs=field_names) ) - safe_inits = [ + safe_init_fields = [ self.safe_name(f) for f in field_names if f != "class" ] # type: List[str] + safe_inits = [f + "=" + f for f in safe_init_fields] + safe_inits.extend( ["extension_fields=extension_fields", "loadingOptions=loadingOptions"] )