From e983f043518e9226a3a22c500d25fedbc593ad74 Mon Sep 17 00:00:00 2001 From: Nementon Date: Sun, 7 Feb 2021 20:23:10 +0100 Subject: [PATCH 01/39] parser / properties / resolve local $ref --- CHANGELOG.md | 1 + .../parser/properties/__init__.py | 87 +++++++- .../test_parser/test_properties/test_init.py | 204 +++++++++++++++++- 3 files changed, 286 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffb6c8f5d..8259dbf6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Additions +- Add support for properties local reference ($ref) resolution - New `--meta` command line option for specifying what type of metadata should be generated: - `poetry` is the default value, same behavior you're used to in previous versions - `setup` will generate a pyproject.toml with no Poetry information, and instead create a `setup.py` with the diff --git a/openapi_python_client/parser/properties/__init__.py b/openapi_python_client/parser/properties/__init__.py index 992d726df..f871c3fcb 100644 --- a/openapi_python_client/parser/properties/__init__.py +++ b/openapi_python_client/parser/properties/__init__.py @@ -525,12 +525,83 @@ def update_schemas_with_data(name: str, data: oai.Schema, schemas: Schemas) -> U return schemas +def resolve_reference_and_update_schemas( + name: str, data: oai.Reference, schemas: Schemas, references_by_name: Dict[str, oai.Reference] +) -> Union[Schemas, PropertyError]: + if _is_local_reference(data): + return _resolve_local_reference_schema(name, data, schemas, references_by_name) + else: + return _resolve_remote_reference_schema(name, data, schemas, references_by_name) + + +def _resolve_local_reference_schema( + name: str, data: oai.Reference, schemas: Schemas, references_by_name: Dict[str, oai.Reference] +) -> Union[Schemas, PropertyError]: + resolved_model_or_enum = _resolve_model_or_enum_reference(name, data, schemas, references_by_name) + + if resolved_model_or_enum: + model_name = utils.pascal_case(name) + + if isinstance(resolved_model_or_enum, EnumProperty): + schemas.enums[model_name] = resolved_model_or_enum + + elif isinstance(resolved_model_or_enum, ModelProperty): + schemas.models[model_name] = resolved_model_or_enum + + return schemas + else: + return PropertyError(data=data, detail="Failed to resolve local reference schemas.") + + +def _resolve_model_or_enum_reference( + name: str, data: oai.Reference, schemas: Schemas, references_by_name: Dict[str, oai.Reference] +) -> Union[EnumProperty, ModelProperty, None]: + target_model = _reference_model_name(data) + target_name = _reference_name(data) + + if target_model == name or target_name == name: + return None # Avoid infinite loop + + if target_name in references_by_name: + return _resolve_model_or_enum_reference( + target_name, references_by_name[target_name], schemas, references_by_name + ) + + if target_model in schemas.enums: + return schemas.enums[target_model] + elif target_model in schemas.models: + return schemas.models[target_model] + + return None + + +def _resolve_remote_reference_schema( + name: str, data: oai.Reference, schemas: Schemas, references_by_name: Dict[str, oai.Reference] +) -> Union[Schemas, PropertyError]: + return PropertyError(data=data, detail="Remote reference schemas are not supported.") + + +def _is_local_reference(reference: oai.Reference) -> bool: + return reference.ref.startswith("#", 0) + + +def _reference_model_name(reference: oai.Reference) -> str: + return utils.pascal_case(_reference_name(reference)) + + +def _reference_name(reference: oai.Reference) -> str: + parts = reference.ref.split("/") + return parts[-1] + + def build_schemas(*, components: Dict[str, Union[oai.Reference, oai.Schema]]) -> Schemas: """ Get a list of Schemas from an OpenAPI dict """ schemas = Schemas() to_process: Iterable[Tuple[str, Union[oai.Reference, oai.Schema]]] = components.items() processing = True errors: List[PropertyError] = [] + references_by_name: Dict[str, oai.Reference] = dict() + references_to_process: List[Tuple[str, oai.Reference]] = list() # References could have forward References so keep going as long as we are making progress while processing: @@ -540,16 +611,26 @@ def build_schemas(*, components: Dict[str, Union[oai.Reference, oai.Schema]]) -> # Only accumulate errors from the last round, since we might fix some along the way for name, data in to_process: if isinstance(data, oai.Reference): - schemas.errors.append(PropertyError(data=data, detail="Reference schemas are not supported.")) + references_by_name[name] = data + references_to_process.append((name, data)) continue + schemas_or_err = update_schemas_with_data(name, data, schemas) + if isinstance(schemas_or_err, PropertyError): next_round.append((name, data)) errors.append(schemas_or_err) else: schemas = schemas_or_err - processing = True # We made some progress this round, do another after it's done + processing = True # We made some progress this round, do another after it's donea + to_process = next_round - schemas.errors.extend(errors) + for name, reference in references_to_process: + schemas_or_err = resolve_reference_and_update_schemas(name, reference, schemas, references_by_name) + + if isinstance(schemas_or_err, PropertyError): + errors.append(schemas_or_err) + + schemas.errors.extend(errors) return schemas diff --git a/tests/test_parser/test_properties/test_init.py b/tests/test_parser/test_properties/test_init.py index ee07d3973..859dbfa65 100644 --- a/tests/test_parser/test_properties/test_init.py +++ b/tests/test_parser/test_properties/test_init.py @@ -1020,13 +1020,43 @@ def test_build_schemas(mocker): assert result.errors == [error] -def test_build_parse_error_on_reference(): +def test_build_parse_error_on_unknown_local_reference(): from openapi_python_client.parser.openapi import build_schemas - ref_schema = oai.Reference.construct() + ref_schema = oai.Reference.construct(ref="#/foobar") in_data = {"1": ref_schema} result = build_schemas(components=in_data) - assert result.errors[0] == PropertyError(data=ref_schema, detail="Reference schemas are not supported.") + assert result.errors[0] == PropertyError(data=ref_schema, detail="Failed to resolve local reference schemas.") + + +def test_build_parse_success_on_known_local_reference(mocker): + from openapi_python_client.parser.openapi import build_schemas + + build_model_property = mocker.patch(f"{MODULE_NAME}.build_model_property") + schemas = mocker.MagicMock() + build_enum_property = mocker.patch(f"{MODULE_NAME}.build_enum_property", return_value=(mocker.MagicMock(), schemas)) + in_data = {"1": oai.Reference.construct(ref="#/foobar"), "foobar": mocker.MagicMock(enum=["val1", "val2", "val3"])} + + result = build_schemas(components=in_data) + + assert len(result.errors) == 0 + assert result.enums["1"] == result.enums["foobar"] + + +def test_build_parse_error_on_remote_reference(): + from openapi_python_client.parser.openapi import build_schemas + + ref_schemas = [ + oai.Reference.construct(ref="http://foobar/../foobar.yaml#/foobar"), + oai.Reference.construct(ref="https://foobar/foobar.yaml#/foobar"), + oai.Reference.construct(ref="../foobar.yaml#/foobar"), + oai.Reference.construct(ref="foobar.yaml#/foobar"), + oai.Reference.construct(ref="//foobar#/foobar"), + ] + for ref_schema in ref_schemas: + in_data = {"1": ref_schema} + result = build_schemas(components=in_data) + assert result.errors[0] == PropertyError(data=ref_schema, detail="Remote reference schemas are not supported.") def test_build_enums(mocker): @@ -1216,3 +1246,171 @@ def test_build_enum_property_bad_default(): assert schemas == schemas assert err == PropertyError(detail="B is an invalid default for enum Existing", data=data) + + +def test__is_local_reference(): + from openapi_python_client.parser.properties import _is_local_reference + + data_set = [ + ("//foobar#foobar", False), + ("foobar#/foobar", False), + ("foobar.json", False), + ("foobar.yaml", False), + ("../foo/bar.json#/foobar", False), + ("#/foobar", True), + ("#/foo/bar", True), + ] + + for data, expected_result in data_set: + ref = oai.Reference.construct(ref=data) + assert _is_local_reference(ref) == expected_result + + +def test__reference_name(): + from openapi_python_client.parser.properties import _reference_name + + data_set = [ + ("#/foobar", "foobar"), + ("#/foo/bar", "bar"), + ] + + for data, expected_result in data_set: + ref = oai.Reference.construct(ref=data) + assert _reference_name(ref) == expected_result + + +def test__reference_model_name(): + from openapi_python_client.parser.properties import _reference_model_name + + data_set = [ + ("#/foobar", "Foobar"), + ("#/foo/bar", "Bar"), + ] + + for data, expected_result in data_set: + ref = oai.Reference.construct(ref=data) + assert _reference_model_name(ref) == expected_result + + +def test__resolve_model_or_enum_reference(mocker): + from openapi_python_client.parser.properties import _resolve_model_or_enum_reference + from openapi_python_client.parser.properties.schemas import Schemas + + references_by_name = { + "FooBarReferenceLoop": oai.Reference.construct(ref="#/foobar"), + "FooBarDeeperReferenceLoop": oai.Reference.construct(ref="#/FooBarReferenceLoop"), + "BarFooReferenceLoop": oai.Reference.construct(ref="#/barfoo"), + "BarFooDeeperReferenceLoop": oai.Reference.construct(ref="#/BarFooReferenceLoop"), + "InfiniteReferenceLoop": oai.Reference.construct(ref="#/InfiniteReferenceLoop"), + "UnknownReference": oai.Reference.construct(ref="#/unknown"), + } + schemas = Schemas(enums={"Foobar": 1}, models={"Barfoo": 2}) + + res_1 = _resolve_model_or_enum_reference( + "FooBarReferenceLoop", references_by_name["FooBarReferenceLoop"], schemas, references_by_name + ) + res_2 = _resolve_model_or_enum_reference( + "FooBarDeeperReferenceLoop", references_by_name["FooBarDeeperReferenceLoop"], schemas, references_by_name + ) + res_3 = _resolve_model_or_enum_reference( + "BarFooReferenceLoop", references_by_name["BarFooReferenceLoop"], schemas, references_by_name + ) + res_4 = _resolve_model_or_enum_reference( + "BarFooDeeperReferenceLoop", references_by_name["BarFooDeeperReferenceLoop"], schemas, references_by_name + ) + res_5 = _resolve_model_or_enum_reference( + "InfiniteReferenceLoop", references_by_name["InfiniteReferenceLoop"], schemas, references_by_name + ) + res_6 = _resolve_model_or_enum_reference( + "UnknownReference", references_by_name["UnknownReference"], schemas, references_by_name + ) + + assert res_1 == schemas.enums["Foobar"] + assert res_2 == schemas.enums["Foobar"] + assert res_3 == schemas.models["Barfoo"] + assert res_4 == schemas.models["Barfoo"] + assert res_5 == None + assert res_6 == None + + +def test__resolve_local_reference_schema(mocker): + from openapi_python_client.parser.properties import _resolve_local_reference_schema + from openapi_python_client.parser.properties.enum_property import EnumProperty + from openapi_python_client.parser.properties.model_property import ModelProperty + from openapi_python_client.parser.properties.schemas import Schemas + + references_by_name = { + "FooBarReferenceLoop": oai.Reference.construct(ref="#/foobar"), + "FooBarDeeperReferenceLoop": oai.Reference.construct(ref="#/FooBarReferenceLoop"), + "fooBarLowerCaseReferenceLoop": oai.Reference.construct(ref="#/foobar"), + "fooBarLowerCaseDeeperReferenceLoop": oai.Reference.construct(ref="#/fooBarLowerCaseReferenceLoop"), + "BarFooReferenceLoop": oai.Reference.construct(ref="#/barfoo"), + "BarFooDeeperReferenceLoop": oai.Reference.construct(ref="#/BarFooReferenceLoop"), + "InfiniteReferenceLoop": oai.Reference.construct(ref="#/InfiniteReferenceLoop"), + "UnknownReference": oai.Reference.construct(ref="#/unknown"), + } + schemas = Schemas( + enums={ + "Foobar": EnumProperty( + name="Foobar", + required=False, + nullable=True, + default="foobar", + values=["foobar"], + value_type="str", + reference="", + ) + }, + models={ + "Barfoo": ModelProperty( + name="Barfoo", + required=False, + nullable=True, + default="barfoo", + reference="", + required_properties=[], + optional_properties=[], + description="", + relative_imports=[], + additional_properties=[], + ) + }, + ) + + res_1 = _resolve_local_reference_schema( + "FooBarReferenceLoop", references_by_name["FooBarReferenceLoop"], schemas, references_by_name + ) + res_2 = _resolve_local_reference_schema( + "FooBarDeeperReferenceLoop", references_by_name["FooBarDeeperReferenceLoop"], schemas, references_by_name + ) + res_3 = _resolve_local_reference_schema( + "BarFooReferenceLoop", references_by_name["BarFooReferenceLoop"], schemas, references_by_name + ) + res_4 = _resolve_local_reference_schema( + "BarFooDeeperReferenceLoop", references_by_name["BarFooDeeperReferenceLoop"], schemas, references_by_name + ) + res_5 = _resolve_local_reference_schema( + "fooBarLowerCaseReferenceLoop", references_by_name["fooBarLowerCaseReferenceLoop"], schemas, references_by_name + ) + res_6 = _resolve_local_reference_schema( + "fooBarLowerCaseDeeperReferenceLoop", + references_by_name["fooBarLowerCaseDeeperReferenceLoop"], + schemas, + references_by_name, + ) + res_7 = _resolve_local_reference_schema( + "InfiniteReferenceLoop", references_by_name["InfiniteReferenceLoop"], schemas, references_by_name + ) + res_8 = _resolve_local_reference_schema( + "UnknownReference", references_by_name["UnknownReference"], schemas, references_by_name + ) + + assert res_1 == res_2 == res_3 == res_4 == res_5 == res_6 == schemas + assert schemas.enums["FooBarReferenceLoop"] == schemas.enums["Foobar"] + assert schemas.enums["FooBarDeeperReferenceLoop"] == schemas.enums["Foobar"] + assert schemas.models["BarFooReferenceLoop"] == schemas.models["Barfoo"] + assert schemas.models["BarFooDeeperReferenceLoop"] == schemas.models["Barfoo"] + assert schemas.enums["FooBarLowerCaseReferenceLoop"] == schemas.enums["Foobar"] + assert schemas.enums["FooBarLowerCaseDeeperReferenceLoop"] == schemas.enums["Foobar"] + assert isinstance(res_7, PropertyError) + assert isinstance(res_8, PropertyError) From 49c1ed1c6ba5073e7332e839753e51faae909b77 Mon Sep 17 00:00:00 2001 From: Nementon Date: Fri, 19 Feb 2021 04:09:47 +0100 Subject: [PATCH 02/39] parser/ propertoes / add `LazyReferencePropertyProxy`: lazy resolve reference properties --- .../parser/properties/__init__.py | 103 +++++++++++++++++- 1 file changed, 97 insertions(+), 6 deletions(-) diff --git a/openapi_python_client/parser/properties/__init__.py b/openapi_python_client/parser/properties/__init__.py index f871c3fcb..2f8aab315 100644 --- a/openapi_python_client/parser/properties/__init__.py +++ b/openapi_python_client/parser/properties/__init__.py @@ -1,5 +1,6 @@ +import copy from itertools import chain -from typing import Any, ClassVar, Dict, Generic, Iterable, Iterator, List, Optional, Set, Tuple, TypeVar, Union +from typing import Any, ClassVar, Dict, Generic, Iterable, Iterator, List, Optional, Set, Tuple, TypeVar, Union, cast import attr @@ -14,6 +15,83 @@ from .schemas import Schemas +class LazyReferencePropertyProxy: + + __GLOBAL_SCHEMAS_REF: Schemas = Schemas() + __PROXIES: List[Tuple["LazyReferencePropertyProxy", oai.Reference]] = [] + + @classmethod + def update_schemas(cls, schemas: Schemas) -> None: + cls.__GLOBAL_SCHEMAS_REF = schemas + + @classmethod + def create(cls, name: str, required: bool, data: oai.Reference, parent_name: str) -> "LazyReferencePropertyProxy": + proxy = LazyReferencePropertyProxy(name, required, data, parent_name) + cls.__PROXIES.append((proxy, data)) + return proxy + + @classmethod + def created_proxies(cls) -> List[Tuple["LazyReferencePropertyProxy", oai.Reference]]: + return cls.__PROXIES + + @classmethod + def flush_internal_references(cls) -> None: + cls.__PROXIES = [] + cls.__GLOBAL_SCHEMAS_REF = Schemas() + + def __init__(self, name: str, required: bool, data: oai.Reference, parent_name: str): + self._name = name + self._required = required + self._data = data + self._parent_name = parent_name + self._reference: Reference = Reference.from_ref(data.ref) + self._reference_to_itself: bool = self._reference.class_name == parent_name + self._resolved: Union[Property, None] = None + + def get_instance_type_string(self) -> str: + return self.get_type_string(no_optional=True) + + def get_type_string(self, no_optional: bool = False) -> str: + resolved = self.resolve() + if resolved: + return resolved.get_type_string(no_optional) + return "LazyReferencePropertyProxy" + + def get_imports(self, *, prefix: str) -> Set[str]: + resolved = self.resolve() + if resolved: + return resolved.get_imports(prefix=prefix) + return set() + + def __copy__(self) -> Property: + resolved = cast(Property, self.resolve(False)) + return copy.copy(resolved) + + def __deepcopy__(self, memo: Any) -> Property: + resolved = cast(Property, self.resolve(False)) + return copy.deepcopy(resolved, memo) + + def __getattr__(self, name: str) -> Any: + resolved = self.resolve(False) + return resolved.__getattribute__(name) + + def resolve(self, allow_lazyness: bool = True) -> Union[Property, None]: + if not self._resolved: + schemas = LazyReferencePropertyProxy.__GLOBAL_SCHEMAS_REF + class_name = self._reference.class_name + if schemas: + existing = schemas.enums.get(class_name) or schemas.models.get(class_name) + if existing: + self._resolved = attr.evolve(existing, required=self._required, name=self._name) + + if self._resolved: + return self._resolved + elif allow_lazyness: + return None + else: + raise RuntimeError(f"Reference {self._data} shall have been resolved.") + + @attr.s(auto_attribs=True, frozen=True) class NoneProperty(Property): """ A property that is always None (used for empty schemas) """ @@ -441,6 +519,9 @@ def _property_from_data( """ Generate a Property from the OpenAPI dictionary representation of it """ name = utils.remove_string_escapes(name) if isinstance(data, oai.Reference): + if not _is_local_reference(data): + return PropertyError(data=data, detail="Remote reference schemas are not supported."), schemas + reference = Reference.from_ref(data.ref) existing = schemas.enums.get(reference.class_name) or schemas.models.get(reference.class_name) if existing: @@ -448,7 +529,9 @@ def _property_from_data( attr.evolve(existing, required=required, name=name), schemas, ) - return PropertyError(data=data, detail="Could not find reference in parsed models or enums"), schemas + else: + return cast(Property, LazyReferencePropertyProxy.create(name, required, data, parent_name)), schemas + if data.enum: return build_enum_property( data=data, name=name, required=required, schemas=schemas, enum=data.enum, parent_name=parent_name @@ -519,6 +602,7 @@ def update_schemas_with_data(name: str, data: oai.Schema, schemas: Schemas) -> U ) else: prop, schemas = build_model_property(data=data, name=name, schemas=schemas, required=True, parent_name=None) + if isinstance(prop, PropertyError): return prop else: @@ -602,6 +686,7 @@ def build_schemas(*, components: Dict[str, Union[oai.Reference, oai.Schema]]) -> errors: List[PropertyError] = [] references_by_name: Dict[str, oai.Reference] = dict() references_to_process: List[Tuple[str, oai.Reference]] = list() + LazyReferencePropertyProxy.flush_internal_references() # Cleanup side effects # References could have forward References so keep going as long as we are making progress while processing: @@ -626,11 +711,17 @@ def build_schemas(*, components: Dict[str, Union[oai.Reference, oai.Schema]]) -> to_process = next_round - for name, reference in references_to_process: - schemas_or_err = resolve_reference_and_update_schemas(name, reference, schemas, references_by_name) + for name, reference in references_to_process: + schemas_or_err = resolve_reference_and_update_schemas(name, reference, schemas, references_by_name) - if isinstance(schemas_or_err, PropertyError): - errors.append(schemas_or_err) + if isinstance(schemas_or_err, PropertyError): + errors.append(schemas_or_err) schemas.errors.extend(errors) + LazyReferencePropertyProxy.update_schemas(schemas) + for reference_proxy, data in LazyReferencePropertyProxy.created_proxies(): + if not reference_proxy.resolve(): + schemas.errors.append( + PropertyError(data=data, detail="Could not find reference in parsed models or enums.") + ) return schemas From b27a003cd07aaa1416dad3ef0d36f03d7c91eb90 Mon Sep 17 00:00:00 2001 From: Nementon Date: Sat, 20 Feb 2021 19:20:03 +0100 Subject: [PATCH 03/39] test / parser / properties / correct references arranged data, use local ones --- tests/test_parser/test_properties/test_init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_parser/test_properties/test_init.py b/tests/test_parser/test_properties/test_init.py index 859dbfa65..0fe7ec30b 100644 --- a/tests/test_parser/test_properties/test_init.py +++ b/tests/test_parser/test_properties/test_init.py @@ -548,7 +548,7 @@ def test_property_from_data_ref_enum(self): from openapi_python_client.parser.properties import EnumProperty, Reference, Schemas, property_from_data name = "some_enum" - data = oai.Reference.construct(ref="MyEnum") + data = oai.Reference.construct(ref="#MyEnum") existing_enum = EnumProperty( name="an_enum", required=True, @@ -579,7 +579,7 @@ def test_property_from_data_ref_model(self): name = "new_name" required = False class_name = "MyModel" - data = oai.Reference.construct(ref=class_name) + data = oai.Reference.construct(ref=f"#{class_name}") existing_model = ModelProperty( name="old_name", required=True, From 32698b0d2cc69ddf9b2c191660a815686cfa8afb Mon Sep 17 00:00:00 2001 From: Nementon Date: Sat, 20 Feb 2021 23:39:58 +0100 Subject: [PATCH 04/39] test / parser / properties / add `LazyReferencePropertyProxy` tests --- .../test_parser/test_properties/test_init.py | 300 +++++++++++++++++- 1 file changed, 297 insertions(+), 3 deletions(-) diff --git a/tests/test_parser/test_properties/test_init.py b/tests/test_parser/test_properties/test_init.py index 0fe7ec30b..42ef29931 100644 --- a/tests/test_parser/test_properties/test_init.py +++ b/tests/test_parser/test_properties/test_init.py @@ -611,7 +611,12 @@ def test_property_from_data_ref_model(self): assert schemas == new_schemas def test_property_from_data_ref_not_found(self, mocker): - from openapi_python_client.parser.properties import PropertyError, Schemas, property_from_data + from openapi_python_client.parser.properties import ( + LazyReferencePropertyProxy, + PropertyError, + Schemas, + property_from_data, + ) name = mocker.MagicMock() required = mocker.MagicMock() @@ -624,8 +629,13 @@ def test_property_from_data_ref_not_found(self, mocker): name=name, required=required, data=data, schemas=schemas, parent_name="parent" ) - from_ref.assert_called_once_with(data.ref) - assert prop == PropertyError(data=data, detail="Could not find reference in parsed models or enums") + from_ref.assert_called_with(data.ref) + assert isinstance(prop, LazyReferencePropertyProxy) + assert prop.resolve() == None + with pytest.raises(RuntimeError): + prop.resolve(False) + with pytest.raises(RuntimeError): + prop.template assert schemas == new_schemas def test_property_from_data_string(self, mocker): @@ -1414,3 +1424,287 @@ def test__resolve_local_reference_schema(mocker): assert schemas.enums["FooBarLowerCaseDeeperReferenceLoop"] == schemas.enums["Foobar"] assert isinstance(res_7, PropertyError) assert isinstance(res_8, PropertyError) + + +def _base_api_data(): + return """ +--- +openapi: 3.0.2 +info: + title: test + description: test + version: 1.0.0 +paths: + /tests/: + get: + operationId: getTests + description: test + responses: + '200': + description: test + content: + application/json: + schema: + $ref: '#/components/schemas/fooBarModel' +""" + + +def test_lazy_proxy_reference_unresolved(): + import copy + + import openapi_python_client.schema as oai + from openapi_python_client.parser.properties import LazyReferencePropertyProxy, Schemas + + LazyReferencePropertyProxy.update_schemas(Schemas()) + lazy_reference_proxy = LazyReferencePropertyProxy.create( + "childProperty", False, oai.Reference(ref="#/foobar"), "AModel" + ) + + assert lazy_reference_proxy.get_instance_type_string() == "LazyReferencePropertyProxy" + assert lazy_reference_proxy.get_type_string(no_optional=False) == "LazyReferencePropertyProxy" + assert lazy_reference_proxy.get_type_string(no_optional=True) == "LazyReferencePropertyProxy" + assert lazy_reference_proxy.get_imports(prefix="..") == set() + assert lazy_reference_proxy.resolve() == None + with pytest.raises(RuntimeError): + lazy_reference_proxy.resolve(False) + with pytest.raises(RuntimeError): + copy.copy(lazy_reference_proxy) + with pytest.raises(RuntimeError): + copy.deepcopy(lazy_reference_proxy) + with pytest.raises(RuntimeError): + lazy_reference_proxy.name + with pytest.raises(RuntimeError): + lazy_reference_proxy.required + with pytest.raises(RuntimeError): + lazy_reference_proxy.nullable + with pytest.raises(RuntimeError): + lazy_reference_proxy.default + with pytest.raises(RuntimeError): + lazy_reference_proxy.python_name + with pytest.raises(RuntimeError): + lazy_reference_proxy.template + + +def test_lazy_proxy_reference_resolved(): + import copy + + import yaml + + import openapi_python_client.schema as oai + from openapi_python_client.parser.properties import LazyReferencePropertyProxy, Schemas, build_schemas + + data = yaml.safe_load( + f""" +{_base_api_data()} +components: + schemas: + fooBar: + type: object + properties: + childSettings: + type: number +""" + ) + openapi = oai.OpenAPI.parse_obj(data) + schemas = build_schemas(components=openapi.components.schemas) + foobar = schemas.models.get("FooBar") + + LazyReferencePropertyProxy.update_schemas(schemas) + lazy_reference_proxy = LazyReferencePropertyProxy.create( + "childProperty", True, oai.Reference(ref="#/components/schemas/fooBar"), "AModel" + ) + + assert foobar + assert lazy_reference_proxy.get_instance_type_string() == foobar.get_instance_type_string() + assert lazy_reference_proxy.get_type_string(no_optional=False) == foobar.get_type_string(no_optional=False) + assert lazy_reference_proxy.get_type_string(no_optional=True) == foobar.get_type_string(no_optional=True) + assert lazy_reference_proxy.get_imports(prefix="..") == foobar.get_imports(prefix="..") + assert lazy_reference_proxy.name == "childProperty" and foobar.name == "fooBar" + assert lazy_reference_proxy.nullable == foobar.nullable + assert lazy_reference_proxy.default == foobar.default + assert lazy_reference_proxy.python_name == "child_property" and foobar.python_name == "foo_bar" + assert lazy_reference_proxy.template == foobar.template + try: + copy.copy(lazy_reference_proxy) + copy.deepcopy(lazy_reference_proxy) + except Exception as e: + pytest.fail(e) + + +def test_build_schemas_resolve_inner_property_remote_reference(): + import yaml + + import openapi_python_client.schema as oai + from openapi_python_client.parser.properties import Schemas, build_schemas + + data = yaml.safe_load( + f""" +{_base_api_data()} +components: + schemas: + fooBar: + type: object + properties: + childSettings: + type: array + items: + $ref: 'AnOtherDocument#/components/schemas/bar' +""" + ) + openapi = oai.OpenAPI.parse_obj(data) + + schemas = build_schemas(components=openapi.components.schemas) + + assert len(schemas.errors) == 1 + assert schemas.errors[0] == PropertyError( + data=oai.Reference(ref="AnOtherDocument#/components/schemas/bar"), + detail="invalid data in items of array childSettings", + ) + + +def test_build_schemas_lazy_resolve_known_inner_property_local_reference(): + import yaml + + import openapi_python_client.schema as oai + from openapi_python_client.parser.properties import Schemas, build_schemas + + data = yaml.safe_load( + f""" +{_base_api_data()} +components: + schemas: + fooBar: + type: object + properties: + childSettings: + type: array + items: + $ref: '#/components/schemas/bar' + bar: + type: object + properties: + a_prop: + type: number +""" + ) + openapi = oai.OpenAPI.parse_obj(data) + + schemas = build_schemas(components=openapi.components.schemas) + + foo_bar = schemas.models.get("FooBar") + bar = schemas.models.get("Bar") + assert len(schemas.errors) == 0 + assert foo_bar and bar + child_settings = foo_bar.optional_properties[0] + assert child_settings.inner_property.reference == bar.reference + + +def test_build_schemas_lazy_resolve_known_inner_property_local_reference_with_loop(): + import yaml + + import openapi_python_client.schema as oai + from openapi_python_client.parser.properties import Schemas, build_schemas + + data = yaml.safe_load( + f""" +{_base_api_data()} +components: + schemas: + fooBar: + type: object + properties: + childSettings: + type: array + items: + $ref: '#/components/schemas/barDeeperLoop' + + barDeeperLoop: + $ref: '#/components/schemas/barLoop' + barLoop: + $ref: '#/components/schemas/bar' + bar: + type: object + properties: + a_prop: + type: number + +""" + ) + openapi = oai.OpenAPI.parse_obj(data) + + schemas = build_schemas(components=openapi.components.schemas) + + foo_bar = schemas.models.get("FooBar") + bar_deeper_loop = schemas.models.get("BarDeeperLoop") + bar_loop = schemas.models.get("BarLoop") + bar = schemas.models.get("Bar") + assert len(schemas.errors) == 0 + assert foo_bar and bar_deeper_loop and bar_loop and bar + assert bar == bar_deeper_loop == bar_loop + + child_settings = foo_bar.optional_properties[0] + assert child_settings.inner_property.reference == bar.reference + assert child_settings.inner_property.reference == bar_loop.reference + assert child_settings.inner_property.reference == bar_deeper_loop.reference + + +def test_build_schemas_lazy_resolve_inner_property_self_local_reference(): + import yaml + + import openapi_python_client.schema as oai + from openapi_python_client.parser.properties import Schemas, build_schemas + + data = yaml.safe_load( + f""" +{_base_api_data()} +components: + schemas: + fooBar: + type: object + properties: + childSettings: + type: array + items: + $ref: '#/components/schemas/fooBar' +""" + ) + openapi = oai.OpenAPI.parse_obj(data) + + schemas = build_schemas(components=openapi.components.schemas) + + foo_bar = schemas.models.get("FooBar") + assert len(schemas.errors) == 0 + assert foo_bar + child_settings = foo_bar.optional_properties[0] + assert child_settings.inner_property.reference == foo_bar.reference + + +def test_build_schemas_lazy_resolve_unknown_inner_property_local_reference(): + import yaml + + import openapi_python_client.schema as oai + from openapi_python_client.parser.properties import Schemas, build_schemas + + data = yaml.safe_load( + f""" +{_base_api_data()} +components: + schemas: + fooBar: + type: object + properties: + childSettings: + type: array + items: + $ref: '#/components/schemas/noexist' +""" + ) + openapi = oai.OpenAPI.parse_obj(data) + + schemas = build_schemas(components=openapi.components.schemas) + + assert len(schemas.errors) == 1 + assert schemas.errors[0] == PropertyError( + detail="Could not find reference in parsed models or enums.", + data=oai.Reference(ref="#/components/schemas/noexist"), + ) From f7940b62e88457a9076ed6d5f5c86d402925244f Mon Sep 17 00:00:00 2001 From: Nementon Date: Sat, 20 Feb 2021 23:53:12 +0100 Subject: [PATCH 05/39] parser / properties / tiny code clean up --- openapi_python_client/parser/properties/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openapi_python_client/parser/properties/__init__.py b/openapi_python_client/parser/properties/__init__.py index 2f8aab315..9e29eb515 100644 --- a/openapi_python_client/parser/properties/__init__.py +++ b/openapi_python_client/parser/properties/__init__.py @@ -670,7 +670,7 @@ def _is_local_reference(reference: oai.Reference) -> bool: def _reference_model_name(reference: oai.Reference) -> str: - return utils.pascal_case(_reference_name(reference)) + return Reference.from_ref(reference.ref).class_name def _reference_name(reference: oai.Reference) -> str: From 7b373d2f0a6f1a508b14be8ccbbce25a66325c67 Mon Sep 17 00:00:00 2001 From: Nementon Date: Sun, 21 Feb 2021 01:33:19 +0100 Subject: [PATCH 06/39] parser / properties / use lazy reference only for reference to themself --- .../parser/properties/__init__.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/openapi_python_client/parser/properties/__init__.py b/openapi_python_client/parser/properties/__init__.py index 9e29eb515..b2f474dc6 100644 --- a/openapi_python_client/parser/properties/__init__.py +++ b/openapi_python_client/parser/properties/__init__.py @@ -79,10 +79,9 @@ def resolve(self, allow_lazyness: bool = True) -> Union[Property, None]: if not self._resolved: schemas = LazyReferencePropertyProxy.__GLOBAL_SCHEMAS_REF class_name = self._reference.class_name - if schemas: - existing = schemas.enums.get(class_name) or schemas.models.get(class_name) - if existing: - self._resolved = attr.evolve(existing, required=self._required, name=self._name) + existing = schemas.enums.get(class_name) or schemas.models.get(class_name) + if existing: + self._resolved = attr.evolve(existing, required=self._required, name=self._name) if self._resolved: return self._resolved @@ -530,7 +529,10 @@ def _property_from_data( schemas, ) else: - return cast(Property, LazyReferencePropertyProxy.create(name, required, data, parent_name)), schemas + if Reference.from_ref(f"#{parent_name}").class_name == reference.class_name: + return cast(Property, LazyReferencePropertyProxy.create(name, required, data, parent_name)), schemas + else: + return PropertyError(data=data, detail="Could not find reference in parsed models or enums."), schemas if data.enum: return build_enum_property( @@ -707,15 +709,15 @@ def build_schemas(*, components: Dict[str, Union[oai.Reference, oai.Schema]]) -> errors.append(schemas_or_err) else: schemas = schemas_or_err - processing = True # We made some progress this round, do another after it's donea + processing = True # We made some progress this round, do another after it's done to_process = next_round - for name, reference in references_to_process: - schemas_or_err = resolve_reference_and_update_schemas(name, reference, schemas, references_by_name) + for name, reference in references_to_process: + schemas_or_err = resolve_reference_and_update_schemas(name, reference, schemas, references_by_name) - if isinstance(schemas_or_err, PropertyError): - errors.append(schemas_or_err) + if isinstance(schemas_or_err, PropertyError): + errors.append(schemas_or_err) schemas.errors.extend(errors) LazyReferencePropertyProxy.update_schemas(schemas) From ebb8e62cbfd565c55b8802f434f62f3d13d3e8f7 Mon Sep 17 00:00:00 2001 From: Nementon Date: Sun, 21 Feb 2021 01:34:05 +0100 Subject: [PATCH 07/39] test / parser / properties / correct `LazyReferencePropertyProxy` tests --- tests/test_parser/test_properties/test_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_parser/test_properties/test_init.py b/tests/test_parser/test_properties/test_init.py index 42ef29931..27bbf71b9 100644 --- a/tests/test_parser/test_properties/test_init.py +++ b/tests/test_parser/test_properties/test_init.py @@ -1705,6 +1705,6 @@ def test_build_schemas_lazy_resolve_unknown_inner_property_local_reference(): assert len(schemas.errors) == 1 assert schemas.errors[0] == PropertyError( - detail="Could not find reference in parsed models or enums.", + detail="invalid data in items of array childSettings", data=oai.Reference(ref="#/components/schemas/noexist"), ) From 0f445ac238718b7cabe460d0a7dce446a84cf916 Mon Sep 17 00:00:00 2001 From: Nementon Date: Sun, 21 Feb 2021 15:02:00 +0100 Subject: [PATCH 08/39] parser / propertie / add support for indirect reference to itself resolution --- .../parser/properties/__init__.py | 162 +++++++++++++++--- 1 file changed, 139 insertions(+), 23 deletions(-) diff --git a/openapi_python_client/parser/properties/__init__.py b/openapi_python_client/parser/properties/__init__.py index b2f474dc6..5bc5b156a 100644 --- a/openapi_python_client/parser/properties/__init__.py +++ b/openapi_python_client/parser/properties/__init__.py @@ -72,8 +72,13 @@ def __deepcopy__(self, memo: Any) -> Property: return copy.deepcopy(resolved, memo) def __getattr__(self, name: str) -> Any: - resolved = self.resolve(False) - return resolved.__getattribute__(name) + if name == "nullable": + return not self._required + elif name == "required": + return self._required + else: + resolved = self.resolve(False) + return resolved.__getattribute__(name) def resolve(self, allow_lazyness: bool = True) -> Union[Property, None]: if not self._resolved: @@ -312,7 +317,13 @@ def _string_based_property( def build_model_property( - *, data: oai.Schema, name: str, schemas: Schemas, required: bool, parent_name: Optional[str] + *, + data: oai.Schema, + name: str, + schemas: Schemas, + required: bool, + parent_name: Optional[str], + lazy_references: Dict[str, oai.Reference], ) -> Tuple[Union[ModelProperty, PropertyError], Schemas]: """ A single ModelProperty from its OAI data @@ -336,7 +347,12 @@ def build_model_property( for key, value in (data.properties or {}).items(): prop_required = key in required_set prop, schemas = property_from_data( - name=key, required=prop_required, data=value, schemas=schemas, parent_name=class_name + name=key, + required=prop_required, + data=value, + schemas=schemas, + parent_name=class_name, + lazy_references=lazy_references, ) if isinstance(prop, PropertyError): return prop, schemas @@ -362,6 +378,7 @@ def build_model_property( data=data.additionalProperties, schemas=schemas, parent_name=class_name, + lazy_references=lazy_references, ) if isinstance(additional_properties, PropertyError): return additional_properties, schemas @@ -462,12 +479,23 @@ def build_enum_property( def build_union_property( - *, data: oai.Schema, name: str, required: bool, schemas: Schemas, parent_name: str + *, + data: oai.Schema, + name: str, + required: bool, + schemas: Schemas, + parent_name: str, + lazy_references: Dict[str, oai.Reference], ) -> Tuple[Union[UnionProperty, PropertyError], Schemas]: sub_properties: List[Property] = [] for sub_prop_data in chain(data.anyOf, data.oneOf): sub_prop, schemas = property_from_data( - name=name, required=required, data=sub_prop_data, schemas=schemas, parent_name=parent_name + name=name, + required=required, + data=sub_prop_data, + schemas=schemas, + parent_name=parent_name, + lazy_references=lazy_references, ) if isinstance(sub_prop, PropertyError): return PropertyError(detail=f"Invalid property in union {name}", data=sub_prop_data), schemas @@ -487,12 +515,23 @@ def build_union_property( def build_list_property( - *, data: oai.Schema, name: str, required: bool, schemas: Schemas, parent_name: str + *, + data: oai.Schema, + name: str, + required: bool, + schemas: Schemas, + parent_name: str, + lazy_references: Dict[str, oai.Reference], ) -> Tuple[Union[ListProperty[Any], PropertyError], Schemas]: if data.items is None: return PropertyError(data=data, detail="type array must have items defined"), schemas inner_prop, schemas = property_from_data( - name=f"{name}_item", required=True, data=data.items, schemas=schemas, parent_name=parent_name + name=f"{name}_item", + required=True, + data=data.items, + schemas=schemas, + parent_name=parent_name, + lazy_references=lazy_references, ) if isinstance(inner_prop, PropertyError): return PropertyError(data=inner_prop.data, detail=f"invalid data in items of array {name}"), schemas @@ -514,6 +553,7 @@ def _property_from_data( data: Union[oai.Reference, oai.Schema], schemas: Schemas, parent_name: str, + lazy_references: Dict[str, oai.Reference], ) -> Tuple[Union[Property, PropertyError], Schemas]: """ Generate a Property from the OpenAPI dictionary representation of it """ name = utils.remove_string_escapes(name) @@ -529,17 +569,40 @@ def _property_from_data( schemas, ) else: - if Reference.from_ref(f"#{parent_name}").class_name == reference.class_name: + + def lookup_is_reference_to_itself( + ref_name: str, owner_class_name: str, lazy_references: Dict[str, oai.Reference] + ) -> bool: + if ref_name in lazy_references: + next_ref_name = _reference_name(lazy_references[ref_name]) + return lookup_is_reference_to_itself(next_ref_name, owner_class_name, lazy_references) + + return ref_name.casefold() == owner_class_name.casefold() + + reference_name = _reference_name(data) + if lookup_is_reference_to_itself(reference_name, parent_name, lazy_references): return cast(Property, LazyReferencePropertyProxy.create(name, required, data, parent_name)), schemas else: return PropertyError(data=data, detail="Could not find reference in parsed models or enums."), schemas if data.enum: return build_enum_property( - data=data, name=name, required=required, schemas=schemas, enum=data.enum, parent_name=parent_name + data=data, + name=name, + required=required, + schemas=schemas, + enum=data.enum, + parent_name=parent_name, ) if data.anyOf or data.oneOf: - return build_union_property(data=data, name=name, required=required, schemas=schemas, parent_name=parent_name) + return build_union_property( + data=data, + name=name, + required=required, + schemas=schemas, + parent_name=parent_name, + lazy_references=lazy_references, + ) if not data.type: return NoneProperty(name=name, required=required, nullable=False, default=None), schemas @@ -576,9 +639,23 @@ def _property_from_data( schemas, ) elif data.type == "array": - return build_list_property(data=data, name=name, required=required, schemas=schemas, parent_name=parent_name) + return build_list_property( + data=data, + name=name, + required=required, + schemas=schemas, + parent_name=parent_name, + lazy_references=lazy_references, + ) elif data.type == "object": - return build_model_property(data=data, name=name, schemas=schemas, required=required, parent_name=parent_name) + return build_model_property( + data=data, + name=name, + schemas=schemas, + required=required, + parent_name=parent_name, + lazy_references=lazy_references, + ) return PropertyError(data=data, detail=f"unknown type {data.type}"), schemas @@ -589,21 +666,41 @@ def property_from_data( data: Union[oai.Reference, oai.Schema], schemas: Schemas, parent_name: str, + lazy_references: Optional[Dict[str, oai.Reference]] = None, ) -> Tuple[Union[Property, PropertyError], Schemas]: + if lazy_references is None: + lazy_references = dict() + try: - return _property_from_data(name=name, required=required, data=data, schemas=schemas, parent_name=parent_name) + return _property_from_data( + name=name, + required=required, + data=data, + schemas=schemas, + parent_name=parent_name, + lazy_references=lazy_references, + ) except ValidationError: return PropertyError(detail="Failed to validate default value", data=data), schemas -def update_schemas_with_data(name: str, data: oai.Schema, schemas: Schemas) -> Union[Schemas, PropertyError]: +def update_schemas_with_data( + name: str, data: oai.Schema, schemas: Schemas, lazy_references: Dict[str, oai.Reference] +) -> Union[Schemas, PropertyError]: prop: Union[PropertyError, ModelProperty, EnumProperty] if data.enum is not None: prop, schemas = build_enum_property( - data=data, name=name, required=True, schemas=schemas, enum=data.enum, parent_name=None + data=data, + name=name, + required=True, + schemas=schemas, + enum=data.enum, + parent_name=None, ) else: - prop, schemas = build_model_property(data=data, name=name, schemas=schemas, required=True, parent_name=None) + prop, schemas = build_model_property( + data=data, name=name, schemas=schemas, required=True, parent_name=None, lazy_references=lazy_references + ) if isinstance(prop, PropertyError): return prop @@ -686,23 +783,31 @@ def build_schemas(*, components: Dict[str, Union[oai.Reference, oai.Schema]]) -> to_process: Iterable[Tuple[str, Union[oai.Reference, oai.Schema]]] = components.items() processing = True errors: List[PropertyError] = [] - references_by_name: Dict[str, oai.Reference] = dict() - references_to_process: List[Tuple[str, oai.Reference]] = list() LazyReferencePropertyProxy.flush_internal_references() # Cleanup side effects + lazy_self_references: Dict[str, oai.Reference] = dict() + visited: List[str] = [] # References could have forward References so keep going as long as we are making progress while processing: + references_by_name: Dict[str, oai.Reference] = dict() + references_to_process: List[Tuple[str, oai.Reference]] = list() processing = False errors = [] next_round = [] + # Only accumulate errors from the last round, since we might fix some along the way for name, data in to_process: + visited.append(name) + if isinstance(data, oai.Reference): - references_by_name[name] = data - references_to_process.append((name, data)) + class_name = _reference_model_name(data) + + if not schemas.models.get(class_name) and not schemas.enums.get(class_name): + references_by_name[name] = data + references_to_process.append((name, data)) continue - schemas_or_err = update_schemas_with_data(name, data, schemas) + schemas_or_err = update_schemas_with_data(name, data, schemas, lazy_self_references) if isinstance(schemas_or_err, PropertyError): next_round.append((name, data)) @@ -717,7 +822,18 @@ def build_schemas(*, components: Dict[str, Union[oai.Reference, oai.Schema]]) -> schemas_or_err = resolve_reference_and_update_schemas(name, reference, schemas, references_by_name) if isinstance(schemas_or_err, PropertyError): - errors.append(schemas_or_err) + if _reference_name(reference) in visited: + # It's a reference to an already visited Enum|Model; not yet resolved + # It's an indirect reference toward this Enum|Model; + # It will be lazy proxified and resolved later on + lazy_self_references[name] = reference + else: + errors.append(schemas_or_err) + + for name in lazy_self_references.keys(): + schemas_or_err = resolve_reference_and_update_schemas( + name, lazy_self_references[name], schemas, references_by_name + ) schemas.errors.extend(errors) LazyReferencePropertyProxy.update_schemas(schemas) From 1970fd65cfe3949462a1c7e2ebcfc54534d9b844 Mon Sep 17 00:00:00 2001 From: Nementon Date: Sun, 21 Feb 2021 15:08:03 +0100 Subject: [PATCH 09/39] parser / properties / rename `_reference_name` -> `_reference_pointer_name` --- openapi_python_client/parser/properties/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openapi_python_client/parser/properties/__init__.py b/openapi_python_client/parser/properties/__init__.py index 5bc5b156a..f722cc269 100644 --- a/openapi_python_client/parser/properties/__init__.py +++ b/openapi_python_client/parser/properties/__init__.py @@ -574,12 +574,12 @@ def lookup_is_reference_to_itself( ref_name: str, owner_class_name: str, lazy_references: Dict[str, oai.Reference] ) -> bool: if ref_name in lazy_references: - next_ref_name = _reference_name(lazy_references[ref_name]) + next_ref_name = _reference_pointer_name(lazy_references[ref_name]) return lookup_is_reference_to_itself(next_ref_name, owner_class_name, lazy_references) return ref_name.casefold() == owner_class_name.casefold() - reference_name = _reference_name(data) + reference_name = _reference_pointer_name(data) if lookup_is_reference_to_itself(reference_name, parent_name, lazy_references): return cast(Property, LazyReferencePropertyProxy.create(name, required, data, parent_name)), schemas else: @@ -740,7 +740,7 @@ def _resolve_model_or_enum_reference( name: str, data: oai.Reference, schemas: Schemas, references_by_name: Dict[str, oai.Reference] ) -> Union[EnumProperty, ModelProperty, None]: target_model = _reference_model_name(data) - target_name = _reference_name(data) + target_name = _reference_pointer_name(data) if target_model == name or target_name == name: return None # Avoid infinite loop @@ -772,7 +772,7 @@ def _reference_model_name(reference: oai.Reference) -> str: return Reference.from_ref(reference.ref).class_name -def _reference_name(reference: oai.Reference) -> str: +def _reference_pointer_name(reference: oai.Reference) -> str: parts = reference.ref.split("/") return parts[-1] @@ -822,7 +822,7 @@ def build_schemas(*, components: Dict[str, Union[oai.Reference, oai.Schema]]) -> schemas_or_err = resolve_reference_and_update_schemas(name, reference, schemas, references_by_name) if isinstance(schemas_or_err, PropertyError): - if _reference_name(reference) in visited: + if _reference_pointer_name(reference) in visited: # It's a reference to an already visited Enum|Model; not yet resolved # It's an indirect reference toward this Enum|Model; # It will be lazy proxified and resolved later on From ce6bb1ee9b95c0b1d1529cd843988728270e10be Mon Sep 17 00:00:00 2001 From: Nementon Date: Sun, 21 Feb 2021 15:47:06 +0100 Subject: [PATCH 10/39] test / parser / properties / correct API breaking changes --- .../test_parser/test_properties/test_init.py | 109 +++++++++++++----- 1 file changed, 77 insertions(+), 32 deletions(-) diff --git a/tests/test_parser/test_properties/test_init.py b/tests/test_parser/test_properties/test_init.py index 27bbf71b9..144379d0e 100644 --- a/tests/test_parser/test_properties/test_init.py +++ b/tests/test_parser/test_properties/test_init.py @@ -626,16 +626,16 @@ def test_property_from_data_ref_not_found(self, mocker): schemas = Schemas() prop, new_schemas = property_from_data( - name=name, required=required, data=data, schemas=schemas, parent_name="parent" + name=name, + required=required, + data=data, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) - from_ref.assert_called_with(data.ref) - assert isinstance(prop, LazyReferencePropertyProxy) - assert prop.resolve() == None - with pytest.raises(RuntimeError): - prop.resolve(False) - with pytest.raises(RuntimeError): - prop.template + from_ref.assert_called_once_with(data.ref) + assert prop == PropertyError(data=data, detail="Could not find reference in parsed models or enums.") assert schemas == new_schemas def test_property_from_data_string(self, mocker): @@ -708,7 +708,12 @@ def test_property_from_data_array(self, mocker): assert response == build_list_property.return_value build_list_property.assert_called_once_with( - data=data, name=name, required=required, schemas=schemas, parent_name="parent" + data=data, + name=name, + required=required, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) def test_property_from_data_object(self, mocker): @@ -727,7 +732,12 @@ def test_property_from_data_object(self, mocker): assert response == build_model_property.return_value build_model_property.assert_called_once_with( - data=data, name=name, required=required, schemas=schemas, parent_name="parent" + data=data, + name=name, + required=required, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) def test_property_from_data_union(self, mocker): @@ -749,7 +759,12 @@ def test_property_from_data_union(self, mocker): assert response == build_union_property.return_value build_union_property.assert_called_once_with( - data=data, name=name, required=required, schemas=schemas, parent_name="parent" + data=data, + name=name, + required=required, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) def test_property_from_data_unsupported_type(self, mocker): @@ -804,7 +819,12 @@ def test_build_list_property_no_items(self, mocker): schemas = properties.Schemas() p, new_schemas = properties.build_list_property( - name=name, required=required, data=data, schemas=schemas, parent_name="parent" + name=name, + required=required, + data=data, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) assert p == PropertyError(data=data, detail="type array must have items defined") @@ -827,14 +847,24 @@ def test_build_list_property_invalid_items(self, mocker): ) p, new_schemas = properties.build_list_property( - name=name, required=required, data=data, schemas=schemas, parent_name="parent" + name=name, + required=required, + data=data, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) assert p == PropertyError(data="blah", detail=f"invalid data in items of array {name}") assert new_schemas == second_schemas assert schemas != new_schemas, "Schema was mutated" property_from_data.assert_called_once_with( - name=f"{name}_item", required=True, data=data.items, schemas=schemas, parent_name="parent" + name=f"{name}_item", + required=True, + data=data.items, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) def test_build_list_property(self, mocker): @@ -855,7 +885,12 @@ def test_build_list_property(self, mocker): mocker.patch("openapi_python_client.utils.to_valid_python_identifier", return_value=name) p, new_schemas = properties.build_list_property( - name=name, required=required, data=data, schemas=schemas, parent_name="parent" + name=name, + required=required, + data=data, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) assert isinstance(p, properties.ListProperty) @@ -863,7 +898,12 @@ def test_build_list_property(self, mocker): assert new_schemas == second_schemas assert schemas != new_schemas, "Schema was mutated" property_from_data.assert_called_once_with( - name=f"{name}_item", required=True, data=data.items, schemas=schemas, parent_name="parent" + name=f"{name}_item", + required=True, + data=data.items, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) @@ -1019,10 +1059,18 @@ def test_build_schemas(mocker): build_model_property.assert_has_calls( [ - mocker.call(data=in_data["1"], name="1", schemas=Schemas(), required=True, parent_name=None), - mocker.call(data=in_data["2"], name="2", schemas=schemas_1, required=True, parent_name=None), - mocker.call(data=in_data["3"], name="3", schemas=schemas_2, required=True, parent_name=None), - mocker.call(data=in_data["3"], name="3", schemas=schemas_2, required=True, parent_name=None), + mocker.call( + data=in_data["1"], name="1", schemas=Schemas(), required=True, parent_name=None, lazy_references=dict() + ), + mocker.call( + data=in_data["2"], name="2", schemas=schemas_1, required=True, parent_name=None, lazy_references=dict() + ), + mocker.call( + data=in_data["3"], name="3", schemas=schemas_2, required=True, parent_name=None, lazy_references=dict() + ), + mocker.call( + data=in_data["3"], name="3", schemas=schemas_2, required=True, parent_name=None, lazy_references=dict() + ), ] ) # schemas_3 was the last to come back from build_model_property, but it should be ignored because it's an error @@ -1118,6 +1166,7 @@ def test_build_model_property(additional_properties_schema, expected_additional_ schemas=schemas, required=True, parent_name="parent", + lazy_references=dict(), ) assert new_schemas != schemas @@ -1164,6 +1213,7 @@ def test_build_model_property_conflict(): schemas=schemas, required=True, parent_name=None, + lazy_references=dict(), ) assert new_schemas == schemas @@ -1181,11 +1231,7 @@ def test_build_model_property_bad_prop(): schemas = Schemas(models={"OtherModel": None}) err, new_schemas = build_model_property( - data=data, - name="prop", - schemas=schemas, - required=True, - parent_name=None, + data=data, name="prop", schemas=schemas, required=True, parent_name=None, lazy_references=dict() ) assert new_schemas == schemas @@ -1210,6 +1256,7 @@ def test_build_model_property_bad_additional_props(): schemas=schemas, required=True, parent_name=None, + lazy_references=dict(), ) assert new_schemas == schemas @@ -1276,8 +1323,8 @@ def test__is_local_reference(): assert _is_local_reference(ref) == expected_result -def test__reference_name(): - from openapi_python_client.parser.properties import _reference_name +def test__reference_pointer_name(): + from openapi_python_client.parser.properties import _reference_pointer_name data_set = [ ("#/foobar", "foobar"), @@ -1286,7 +1333,7 @@ def test__reference_name(): for data, expected_result in data_set: ref = oai.Reference.construct(ref=data) - assert _reference_name(ref) == expected_result + assert _reference_pointer_name(ref) == expected_result def test__reference_model_name(): @@ -1465,6 +1512,8 @@ def test_lazy_proxy_reference_unresolved(): assert lazy_reference_proxy.get_type_string(no_optional=True) == "LazyReferencePropertyProxy" assert lazy_reference_proxy.get_imports(prefix="..") == set() assert lazy_reference_proxy.resolve() == None + assert lazy_reference_proxy.required == False + assert lazy_reference_proxy.nullable == True with pytest.raises(RuntimeError): lazy_reference_proxy.resolve(False) with pytest.raises(RuntimeError): @@ -1473,10 +1522,6 @@ def test_lazy_proxy_reference_unresolved(): copy.deepcopy(lazy_reference_proxy) with pytest.raises(RuntimeError): lazy_reference_proxy.name - with pytest.raises(RuntimeError): - lazy_reference_proxy.required - with pytest.raises(RuntimeError): - lazy_reference_proxy.nullable with pytest.raises(RuntimeError): lazy_reference_proxy.default with pytest.raises(RuntimeError): From d3d7bbec05861009172fbd312a7f3ee9ef18b20f Mon Sep 17 00:00:00 2001 From: Nementon Date: Sun, 21 Feb 2021 17:40:05 +0100 Subject: [PATCH 11/39] parser / properties / tests cleanup + behaviour fixes --- .../parser/properties/__init__.py | 25 ++-- .../test_parser/test_properties/test_init.py | 113 +++--------------- 2 files changed, 36 insertions(+), 102 deletions(-) diff --git a/openapi_python_client/parser/properties/__init__.py b/openapi_python_client/parser/properties/__init__.py index f722cc269..2674531ee 100644 --- a/openapi_python_client/parser/properties/__init__.py +++ b/openapi_python_client/parser/properties/__init__.py @@ -571,11 +571,17 @@ def _property_from_data( else: def lookup_is_reference_to_itself( - ref_name: str, owner_class_name: str, lazy_references: Dict[str, oai.Reference] + ref_name: str, + owner_class_name: str, + lazy_references: Dict[str, oai.Reference], ) -> bool: if ref_name in lazy_references: next_ref_name = _reference_pointer_name(lazy_references[ref_name]) - return lookup_is_reference_to_itself(next_ref_name, owner_class_name, lazy_references) + return lookup_is_reference_to_itself( + next_ref_name, + owner_class_name, + lazy_references, + ) return ref_name.casefold() == owner_class_name.casefold() @@ -783,14 +789,14 @@ def build_schemas(*, components: Dict[str, Union[oai.Reference, oai.Schema]]) -> to_process: Iterable[Tuple[str, Union[oai.Reference, oai.Schema]]] = components.items() processing = True errors: List[PropertyError] = [] - LazyReferencePropertyProxy.flush_internal_references() # Cleanup side effects lazy_self_references: Dict[str, oai.Reference] = dict() visited: List[str] = [] + references_by_name: Dict[str, oai.Reference] = dict() + references_to_process: List[Tuple[str, oai.Reference]] = list() + LazyReferencePropertyProxy.flush_internal_references() # Cleanup side effects # References could have forward References so keep going as long as we are making progress while processing: - references_by_name: Dict[str, oai.Reference] = dict() - references_to_process: List[Tuple[str, oai.Reference]] = list() processing = False errors = [] next_round = [] @@ -822,24 +828,29 @@ def build_schemas(*, components: Dict[str, Union[oai.Reference, oai.Schema]]) -> schemas_or_err = resolve_reference_and_update_schemas(name, reference, schemas, references_by_name) if isinstance(schemas_or_err, PropertyError): - if _reference_pointer_name(reference) in visited: + if _reference_pointer_name(reference) in visited and name not in lazy_self_references: # It's a reference to an already visited Enum|Model; not yet resolved # It's an indirect reference toward this Enum|Model; # It will be lazy proxified and resolved later on lazy_self_references[name] = reference + processing = True else: errors.append(schemas_or_err) + schemas.errors.extend(errors) + for name in lazy_self_references.keys(): schemas_or_err = resolve_reference_and_update_schemas( name, lazy_self_references[name], schemas, references_by_name ) + if isinstance(schemas_or_err, PropertyError): + schemas.errors.extend(errors) - schemas.errors.extend(errors) LazyReferencePropertyProxy.update_schemas(schemas) for reference_proxy, data in LazyReferencePropertyProxy.created_proxies(): if not reference_proxy.resolve(): schemas.errors.append( PropertyError(data=data, detail="Could not find reference in parsed models or enums.") ) + return schemas diff --git a/tests/test_parser/test_properties/test_init.py b/tests/test_parser/test_properties/test_init.py index 144379d0e..baef3b6a8 100644 --- a/tests/test_parser/test_properties/test_init.py +++ b/tests/test_parser/test_properties/test_init.py @@ -1492,7 +1492,7 @@ def _base_api_data(): content: application/json: schema: - $ref: '#/components/schemas/fooBarModel' + $ref: '#/components/schemas/fooBar' """ @@ -1607,93 +1607,7 @@ def test_build_schemas_resolve_inner_property_remote_reference(): ) -def test_build_schemas_lazy_resolve_known_inner_property_local_reference(): - import yaml - - import openapi_python_client.schema as oai - from openapi_python_client.parser.properties import Schemas, build_schemas - - data = yaml.safe_load( - f""" -{_base_api_data()} -components: - schemas: - fooBar: - type: object - properties: - childSettings: - type: array - items: - $ref: '#/components/schemas/bar' - bar: - type: object - properties: - a_prop: - type: number -""" - ) - openapi = oai.OpenAPI.parse_obj(data) - - schemas = build_schemas(components=openapi.components.schemas) - - foo_bar = schemas.models.get("FooBar") - bar = schemas.models.get("Bar") - assert len(schemas.errors) == 0 - assert foo_bar and bar - child_settings = foo_bar.optional_properties[0] - assert child_settings.inner_property.reference == bar.reference - - -def test_build_schemas_lazy_resolve_known_inner_property_local_reference_with_loop(): - import yaml - - import openapi_python_client.schema as oai - from openapi_python_client.parser.properties import Schemas, build_schemas - - data = yaml.safe_load( - f""" -{_base_api_data()} -components: - schemas: - fooBar: - type: object - properties: - childSettings: - type: array - items: - $ref: '#/components/schemas/barDeeperLoop' - - barDeeperLoop: - $ref: '#/components/schemas/barLoop' - barLoop: - $ref: '#/components/schemas/bar' - bar: - type: object - properties: - a_prop: - type: number - -""" - ) - openapi = oai.OpenAPI.parse_obj(data) - - schemas = build_schemas(components=openapi.components.schemas) - - foo_bar = schemas.models.get("FooBar") - bar_deeper_loop = schemas.models.get("BarDeeperLoop") - bar_loop = schemas.models.get("BarLoop") - bar = schemas.models.get("Bar") - assert len(schemas.errors) == 0 - assert foo_bar and bar_deeper_loop and bar_loop and bar - assert bar == bar_deeper_loop == bar_loop - - child_settings = foo_bar.optional_properties[0] - assert child_settings.inner_property.reference == bar.reference - assert child_settings.inner_property.reference == bar_loop.reference - assert child_settings.inner_property.reference == bar_deeper_loop.reference - - -def test_build_schemas_lazy_resolve_inner_property_self_local_reference(): +def test_build_schemas_lazy_resolve_inner_property_self_direct_reference(): import yaml import openapi_python_client.schema as oai @@ -1724,7 +1638,7 @@ def test_build_schemas_lazy_resolve_inner_property_self_local_reference(): assert child_settings.inner_property.reference == foo_bar.reference -def test_build_schemas_lazy_resolve_unknown_inner_property_local_reference(): +def test_build_schemas_lazy_resolve_known_inner_property_self_indirect_reference(): import yaml import openapi_python_client.schema as oai @@ -1740,16 +1654,25 @@ def test_build_schemas_lazy_resolve_unknown_inner_property_local_reference(): properties: childSettings: type: array + description: test items: - $ref: '#/components/schemas/noexist' + $ref: '#/components/schemas/FoobarSelfIndirectReference' + FoobarSelfIndirectReference: + $ref: '#/components/schemas/foobarSelfDeeperIndirectReference' + foobarSelfDeeperIndirectReference: + $ref: '#/components/schemas/fooBar' """ ) openapi = oai.OpenAPI.parse_obj(data) schemas = build_schemas(components=openapi.components.schemas) - assert len(schemas.errors) == 1 - assert schemas.errors[0] == PropertyError( - detail="invalid data in items of array childSettings", - data=oai.Reference(ref="#/components/schemas/noexist"), - ) + assert len(schemas.errors) == 0 + foobar = schemas.models.get("FooBar") + foobar_indirect_ref = schemas.models.get("FoobarSelfIndirectReference") + foobar_deep_indirect_ref = schemas.models.get("FoobarSelfDeeperIndirectReference") + assert foobar is not None and foobar_indirect_ref is not None and foobar_deep_indirect_ref is not None + assert foobar == foobar_indirect_ref == foobar_deep_indirect_ref + + child_settings = foobar.optional_properties[0] + assert child_settings.inner_property.reference == foobar.reference From 1de31903fa679ba24604835b1f7348cc5002c67b Mon Sep 17 00:00:00 2001 From: Nementon Date: Sun, 21 Feb 2021 20:00:43 +0100 Subject: [PATCH 12/39] parser / properties / LazyReferencePropertyProxy: quote resolved type string for typing --- openapi_python_client/parser/properties/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openapi_python_client/parser/properties/__init__.py b/openapi_python_client/parser/properties/__init__.py index 2674531ee..cc01b80d3 100644 --- a/openapi_python_client/parser/properties/__init__.py +++ b/openapi_python_client/parser/properties/__init__.py @@ -63,6 +63,11 @@ def get_imports(self, *, prefix: str) -> Set[str]: return resolved.get_imports(prefix=prefix) return set() + def to_string(self) -> str: + resolved = cast(Property, self.resolve(False)) + p_repr = resolved.to_string() + return p_repr.replace(f"{self._parent_name}", f"'{self._parent_name}'") + def __copy__(self) -> Property: resolved = cast(Property, self.resolve(False)) return copy.copy(resolved) From c7a1371c10fc3929049b4902a7c01ed001fa31ff Mon Sep 17 00:00:00 2001 From: Nementon Date: Sun, 21 Feb 2021 23:30:43 +0100 Subject: [PATCH 13/39] parser / properties / LazyReferencePropertyProxy: enforce nullable to `false` - Having nullable to `true` lead to invalid source generation Since `LazyReferencePropertyProxy` is only made to be used for inner property reference, it seems unamrfull It's ~nullability shall be define on its parent --- openapi_python_client/parser/properties/__init__.py | 2 +- tests/test_parser/test_properties/test_init.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openapi_python_client/parser/properties/__init__.py b/openapi_python_client/parser/properties/__init__.py index cc01b80d3..8ecad0e60 100644 --- a/openapi_python_client/parser/properties/__init__.py +++ b/openapi_python_client/parser/properties/__init__.py @@ -78,7 +78,7 @@ def __deepcopy__(self, memo: Any) -> Property: def __getattr__(self, name: str) -> Any: if name == "nullable": - return not self._required + return False elif name == "required": return self._required else: diff --git a/tests/test_parser/test_properties/test_init.py b/tests/test_parser/test_properties/test_init.py index baef3b6a8..2edc7fa17 100644 --- a/tests/test_parser/test_properties/test_init.py +++ b/tests/test_parser/test_properties/test_init.py @@ -1513,7 +1513,7 @@ def test_lazy_proxy_reference_unresolved(): assert lazy_reference_proxy.get_imports(prefix="..") == set() assert lazy_reference_proxy.resolve() == None assert lazy_reference_proxy.required == False - assert lazy_reference_proxy.nullable == True + assert lazy_reference_proxy.nullable == False with pytest.raises(RuntimeError): lazy_reference_proxy.resolve(False) with pytest.raises(RuntimeError): From 8334c03889e85f5410e1b50d5cddb3ffc52c4e7f Mon Sep 17 00:00:00 2001 From: Nementon Date: Sun, 21 Feb 2021 19:10:27 +0100 Subject: [PATCH 14/39] golden records / regen / add indirect, self, indirect-self model reference --- .../custom_e2e/models/a_model.py | 39 +++++++++++++++++++ .../my_test_api_client/models/a_model.py | 39 +++++++++++++++++++ end_to_end_tests/openapi.json | 23 ++++++++++- 3 files changed, 100 insertions(+), 1 deletion(-) diff --git a/end_to_end_tests/golden-record-custom/custom_e2e/models/a_model.py b/end_to_end_tests/golden-record-custom/custom_e2e/models/a_model.py index 9237d2428..97ae9d56d 100644 --- a/end_to_end_tests/golden-record-custom/custom_e2e/models/a_model.py +++ b/end_to_end_tests/golden-record-custom/custom_e2e/models/a_model.py @@ -27,6 +27,9 @@ class AModel: a_nullable_date: Optional[datetime.date] required_nullable: Optional[str] nullable_model: Optional[AModelNullableModel] + an_enum_indirect_ref: Union[Unset, AnEnum] = UNSET + direct_ref_to_itself: Union["AModel", Unset] = UNSET + indirect_ref_to_itself: Union["AModel", Unset] = UNSET nested_list_of_enums: Union[Unset, List[List[DifferentEnum]]] = UNSET a_not_required_date: Union[Unset, datetime.date] = UNSET attr_1_leading_digit: Union[Unset, str] = UNSET @@ -48,6 +51,18 @@ def to_dict(self) -> Dict[str, Any]: required_not_nullable = self.required_not_nullable model = self.model.to_dict() + an_enum_indirect_ref: Union[Unset, AnEnum] = UNSET + if not isinstance(self.an_enum_indirect_ref, Unset): + an_enum_indirect_ref = self.an_enum_indirect_ref + + direct_ref_to_itself: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.direct_ref_to_itself, Unset): + direct_ref_to_itself = self.direct_ref_to_itself.to_dict() + + indirect_ref_to_itself: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.indirect_ref_to_itself, Unset): + indirect_ref_to_itself = self.indirect_ref_to_itself.to_dict() + nested_list_of_enums: Union[Unset, List[Any]] = UNSET if not isinstance(self.nested_list_of_enums, Unset): nested_list_of_enums = [] @@ -94,6 +109,12 @@ def to_dict(self) -> Dict[str, Any]: "nullable_model": nullable_model, } ) + if an_enum_indirect_ref is not UNSET: + field_dict["an_enum_indirect_ref"] = an_enum_indirect_ref + if direct_ref_to_itself is not UNSET: + field_dict["direct_ref_to_itself"] = direct_ref_to_itself + if indirect_ref_to_itself is not UNSET: + field_dict["indirect_ref_to_itself"] = indirect_ref_to_itself if nested_list_of_enums is not UNSET: field_dict["nested_list_of_enums"] = nested_list_of_enums if a_not_required_date is not UNSET: @@ -137,6 +158,21 @@ def _parse_a_camel_date_time(data: Any) -> Union[datetime.datetime, datetime.dat model = AModelModel.from_dict(d.pop("model")) + an_enum_indirect_ref: Union[Unset, AnEnum] = UNSET + _an_enum_indirect_ref = d.pop("an_enum_indirect_ref", UNSET) + if not isinstance(_an_enum_indirect_ref, Unset): + an_enum_indirect_ref = AnEnum(_an_enum_indirect_ref) + + direct_ref_to_itself: Union[AModel, Unset] = UNSET + _direct_ref_to_itself = d.pop("direct_ref_to_itself", UNSET) + if not isinstance(_direct_ref_to_itself, Unset): + direct_ref_to_itself = AModel.from_dict(_direct_ref_to_itself) + + indirect_ref_to_itself: Union[AModel, Unset] = UNSET + _indirect_ref_to_itself = d.pop("indirect_ref_to_itself", UNSET) + if not isinstance(_indirect_ref_to_itself, Unset): + indirect_ref_to_itself = AModel.from_dict(_indirect_ref_to_itself) + nested_list_of_enums = [] _nested_list_of_enums = d.pop("nested_list_of_enums", UNSET) for nested_list_of_enums_item_data in _nested_list_of_enums or []: @@ -188,6 +224,9 @@ def _parse_a_camel_date_time(data: Any) -> Union[datetime.datetime, datetime.dat a_date=a_date, required_not_nullable=required_not_nullable, model=model, + an_enum_indirect_ref=an_enum_indirect_ref, + direct_ref_to_itself=direct_ref_to_itself, + indirect_ref_to_itself=indirect_ref_to_itself, nested_list_of_enums=nested_list_of_enums, a_nullable_date=a_nullable_date, a_not_required_date=a_not_required_date, diff --git a/end_to_end_tests/golden-record/my_test_api_client/models/a_model.py b/end_to_end_tests/golden-record/my_test_api_client/models/a_model.py index 9237d2428..97ae9d56d 100644 --- a/end_to_end_tests/golden-record/my_test_api_client/models/a_model.py +++ b/end_to_end_tests/golden-record/my_test_api_client/models/a_model.py @@ -27,6 +27,9 @@ class AModel: a_nullable_date: Optional[datetime.date] required_nullable: Optional[str] nullable_model: Optional[AModelNullableModel] + an_enum_indirect_ref: Union[Unset, AnEnum] = UNSET + direct_ref_to_itself: Union["AModel", Unset] = UNSET + indirect_ref_to_itself: Union["AModel", Unset] = UNSET nested_list_of_enums: Union[Unset, List[List[DifferentEnum]]] = UNSET a_not_required_date: Union[Unset, datetime.date] = UNSET attr_1_leading_digit: Union[Unset, str] = UNSET @@ -48,6 +51,18 @@ def to_dict(self) -> Dict[str, Any]: required_not_nullable = self.required_not_nullable model = self.model.to_dict() + an_enum_indirect_ref: Union[Unset, AnEnum] = UNSET + if not isinstance(self.an_enum_indirect_ref, Unset): + an_enum_indirect_ref = self.an_enum_indirect_ref + + direct_ref_to_itself: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.direct_ref_to_itself, Unset): + direct_ref_to_itself = self.direct_ref_to_itself.to_dict() + + indirect_ref_to_itself: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.indirect_ref_to_itself, Unset): + indirect_ref_to_itself = self.indirect_ref_to_itself.to_dict() + nested_list_of_enums: Union[Unset, List[Any]] = UNSET if not isinstance(self.nested_list_of_enums, Unset): nested_list_of_enums = [] @@ -94,6 +109,12 @@ def to_dict(self) -> Dict[str, Any]: "nullable_model": nullable_model, } ) + if an_enum_indirect_ref is not UNSET: + field_dict["an_enum_indirect_ref"] = an_enum_indirect_ref + if direct_ref_to_itself is not UNSET: + field_dict["direct_ref_to_itself"] = direct_ref_to_itself + if indirect_ref_to_itself is not UNSET: + field_dict["indirect_ref_to_itself"] = indirect_ref_to_itself if nested_list_of_enums is not UNSET: field_dict["nested_list_of_enums"] = nested_list_of_enums if a_not_required_date is not UNSET: @@ -137,6 +158,21 @@ def _parse_a_camel_date_time(data: Any) -> Union[datetime.datetime, datetime.dat model = AModelModel.from_dict(d.pop("model")) + an_enum_indirect_ref: Union[Unset, AnEnum] = UNSET + _an_enum_indirect_ref = d.pop("an_enum_indirect_ref", UNSET) + if not isinstance(_an_enum_indirect_ref, Unset): + an_enum_indirect_ref = AnEnum(_an_enum_indirect_ref) + + direct_ref_to_itself: Union[AModel, Unset] = UNSET + _direct_ref_to_itself = d.pop("direct_ref_to_itself", UNSET) + if not isinstance(_direct_ref_to_itself, Unset): + direct_ref_to_itself = AModel.from_dict(_direct_ref_to_itself) + + indirect_ref_to_itself: Union[AModel, Unset] = UNSET + _indirect_ref_to_itself = d.pop("indirect_ref_to_itself", UNSET) + if not isinstance(_indirect_ref_to_itself, Unset): + indirect_ref_to_itself = AModel.from_dict(_indirect_ref_to_itself) + nested_list_of_enums = [] _nested_list_of_enums = d.pop("nested_list_of_enums", UNSET) for nested_list_of_enums_item_data in _nested_list_of_enums or []: @@ -188,6 +224,9 @@ def _parse_a_camel_date_time(data: Any) -> Union[datetime.datetime, datetime.dat a_date=a_date, required_not_nullable=required_not_nullable, model=model, + an_enum_indirect_ref=an_enum_indirect_ref, + direct_ref_to_itself=direct_ref_to_itself, + indirect_ref_to_itself=indirect_ref_to_itself, nested_list_of_enums=nested_list_of_enums, a_nullable_date=a_nullable_date, a_not_required_date=a_not_required_date, diff --git a/end_to_end_tests/openapi.json b/end_to_end_tests/openapi.json index fcd83e460..db4ae351a 100644 --- a/end_to_end_tests/openapi.json +++ b/end_to_end_tests/openapi.json @@ -675,6 +675,15 @@ "required": ["an_enum_value", "aCamelDateTime", "a_date", "a_nullable_date", "required_nullable", "required_not_nullable", "model", "nullable_model"], "type": "object", "properties": { + "an_enum_indirect_ref": { + "$ref": "#/components/schemas/AnEnumDeeperIndirectReference" + }, + "direct_ref_to_itself": { + "$ref": "#/components/schemas/AModel" + }, + "indirect_ref_to_itself": { + "$ref": "#/components/schemas/AModelDeeperIndirectReference" + }, "an_enum_value": { "$ref": "#/components/schemas/AnEnum" }, @@ -716,7 +725,7 @@ "a_not_required_date": { "title": "A Nullable Date", "type": "string", - "format": "date", + "format": "date" }, "1_leading_digit": { "title": "Leading Digit", @@ -782,11 +791,23 @@ "description": "A Model for testing all the ways custom objects can be used ", "additionalProperties": false }, + "AModelIndirectReference": { + "$ref": "#/components/schemas/AModel" + }, + "AModelDeeperIndirectReference": { + "$ref": "#/components/schemas/AModelIndirectReference" + }, + "AnEnumIndirectReference": { + "$ref": "#/components/schemas/AnEnum" + }, "AnEnum": { "title": "AnEnum", "enum": ["FIRST_VALUE", "SECOND_VALUE"], "description": "For testing Enums in all the ways they can be used " }, + "AnEnumDeeperIndirectReference": { + "$ref": "#/components/schemas/AnEnumIndirectReference" + }, "AnIntEnum": { "title": "AnIntEnum", "enum": [-1, 1, 2], From d0a2e6e1bf2e10c4d850ea14eee68e7d26ba1ac4 Mon Sep 17 00:00:00 2001 From: Nementon Date: Sun, 21 Feb 2021 19:35:23 +0100 Subject: [PATCH 15/39] update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8259dbf6f..da2009ef8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Additions -- Add support for properties local reference ($ref) resolution +- Add support of properties indirect reference ($ref) resolution; Add support of inner properties direct and indirect reference resolution to its owner model/enum (#329). Thanks @p1-ra! - New `--meta` command line option for specifying what type of metadata should be generated: - `poetry` is the default value, same behavior you're used to in previous versions - `setup` will generate a pyproject.toml with no Poetry information, and instead create a `setup.py` with the From 1e996d2849053203362e68ce8d9c76d6733eef63 Mon Sep 17 00:00:00 2001 From: Nementon Date: Fri, 5 Feb 2021 22:58:03 +0100 Subject: [PATCH 16/39] bootstrap schemas resolver --- openapi_python_client/resolver/__init__.py | 0 openapi_python_client/resolver/data_loader.py | 22 +++++ openapi_python_client/resolver/reference.py | 22 +++++ .../resolver/resolved_schema.py | 25 +++++ .../resolver/resolver_types.py | 3 + .../resolver/schema_resolver.py | 98 +++++++++++++++++++ 6 files changed, 170 insertions(+) create mode 100644 openapi_python_client/resolver/__init__.py create mode 100644 openapi_python_client/resolver/data_loader.py create mode 100644 openapi_python_client/resolver/reference.py create mode 100644 openapi_python_client/resolver/resolved_schema.py create mode 100644 openapi_python_client/resolver/resolver_types.py create mode 100644 openapi_python_client/resolver/schema_resolver.py diff --git a/openapi_python_client/resolver/__init__.py b/openapi_python_client/resolver/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openapi_python_client/resolver/data_loader.py b/openapi_python_client/resolver/data_loader.py new file mode 100644 index 000000000..ab899d4b3 --- /dev/null +++ b/openapi_python_client/resolver/data_loader.py @@ -0,0 +1,22 @@ +import yaml +from .resolver_types import SchemaData + +class DataLoader: + + @classmethod + def load(cls, path: str, data: bytes) -> SchemaData: + data_type = path.split('.')[-1] + + if data_type == 'json': + return cls.load_json(data) + else: + return cls.load_yaml(data) + + @classmethod + def load_json(cls, data: bytes) -> SchemaData: + raise NotImplementedError() + + @classmethod + def load_yaml(cls, data: bytes) -> SchemaData: + return yaml.safe_load(data) + diff --git a/openapi_python_client/resolver/reference.py b/openapi_python_client/resolver/reference.py new file mode 100644 index 000000000..3d387ec98 --- /dev/null +++ b/openapi_python_client/resolver/reference.py @@ -0,0 +1,22 @@ +class Reference: + + def __init__(self, reference): + self._ref = reference + + @property + def value(self) -> str: + return self._ref + + def is_relative_reference(self): + return self.is_remote_ref() and not self.is_url_reference() + + def is_url_reference(self): + return self.is_remote_ref() and (self._ref.startswith('//', 0) or self._ref.startswith('http', 0)) + + def is_remote_ref(self): + return not self.is_local_ref() + + def is_local_ref(self): + return self._ref.startswith('#', 0) + + diff --git a/openapi_python_client/resolver/resolved_schema.py b/openapi_python_client/resolver/resolved_schema.py new file mode 100644 index 000000000..19eb47d2b --- /dev/null +++ b/openapi_python_client/resolver/resolved_schema.py @@ -0,0 +1,25 @@ +from typing import Any, Dict, Optional, Sequence, Union +from .resolver_types import SchemaData + + +class ResolvedSchema: + + def __init__(self, root, refs, errors): + self._root: SchemaData = root + self._refs: Dict[str, SchemaData] = refs + self._errors: Sequense[str] = errors + + self._resolved_schema: SchemaData = self._root + self._process() + + @property + def schema(self) -> SchemaData: + return self._resolved_schema + + @property + def errors(self) -> Sequence[str]: + return self._errors.copy() + + def _process(self): + pass + diff --git a/openapi_python_client/resolver/resolver_types.py b/openapi_python_client/resolver/resolver_types.py new file mode 100644 index 000000000..baf35099b --- /dev/null +++ b/openapi_python_client/resolver/resolver_types.py @@ -0,0 +1,3 @@ +from typing import Any, Dict, NewType + +SchemaData = NewType('SchemaData', Dict[str, Any]) diff --git a/openapi_python_client/resolver/schema_resolver.py b/openapi_python_client/resolver/schema_resolver.py new file mode 100644 index 000000000..2ac040c52 --- /dev/null +++ b/openapi_python_client/resolver/schema_resolver.py @@ -0,0 +1,98 @@ +import httpcore +import httpx +import urllib +import logging + +from typing import Any, Dict, Optional, Sequence, Union, Generator, NewType +from pathlib import Path + +from .resolver_types import SchemaData +from .reference import Reference +from .resolved_schema import ResolvedSchema +from .data_loader import DataLoader + +class SchemaResolver: + + def __init__(self, url_or_path: Union[str, Path]): + if not url_or_path: + raise ValueError('Invalid document root reference, it shall be an remote url or local file path') + + self._root_path: Optional[Path] = None + self._root_path_dir: Optional[Path] = None + self._root_url: Optional[str] = None + self._root_url_scheme: Optional[str] = None + + if isinstance(url_or_path, Path): + self._root_path = url_or_path.absolute() + self._root_path_dir = self._root_path.parent + else: + self._root_url = url_or_path + self._root_url_scheme = urllib.parse.urlparse(url_or_path).scheme + + def resolve(self, recursive: bool = True) -> ResolvedSchema: + root_schema: SchemaData + external_schemas: Dict[str, SchemaData] = {} + errors: Sequence[str] = [] + + if self._root_path: + root_schema = self._fetch_remote_file_path(self._root_path) + else: + root_schema = self._fetch_url_reference(self._root_url) + + self._resolve_schema_references(root_schema, external_schemas, errors, recursive) + return ResolvedSchema(root_schema, external_schemas, errors) + + def _resolve_schema_references(self, root: SchemaData, external_schemas: Dict[str, SchemaData], errors: Sequence[str], recursive: bool) -> Sequence[SchemaData]: + + for ref in self._lookup_schema_references(root): + if ref.is_local_ref(): + continue + + try: + path = ref.value.split('#')[0] + if path in external_schemas: + continue + + if ref.is_url_reference(): + external_schemas[path] = self._fetch_url_reference(path) + else: + external_schemas[path] = self._fetch_remote_reference(path) + + if recursive: + self._resolve_schema_references(external_schemas[path], external_schemas, errors, recursive) + + except Exception as e: + errors.append('Failed to gather external reference data of {0}'.format(ref.value)) + logging.exception('Failed to gather external reference data of {0}'.format(ref.value)) + + def _fetch_remote_reference(self, relative_path: str) -> SchemaData: + if self._root_path: + abs_path = self._root_path_dir.joinpath(relative_path) + return self._fetch_remote_file_path(abs_path) + else: + abs_url = urllib.parse.urljoin(self._root_url, relative_path) + return self._fetch_url_reference(abs_url) + + def _fetch_remote_file_path(self, path: Path) -> SchemaData: + logging.info('Fetching remote ref file path > {0}'.format(path)) + return DataLoader.load(str(path), path.read_bytes()) + + def _fetch_url_reference(self, url: str) -> SchemaData: + if url.startswith('//', 0): + url = "{0}{1}".format(self._root_url_scheme, url) + + logging.info('Fetching remote ref url > {0}'.format(url)) + return DataLoader.load(url, httpx.get(url).content) + + def _lookup_schema_references(self, attr: Any) -> Generator[Reference, None, None]: + if isinstance(attr, dict): + for key, val in attr.items(): + if key == '$ref': + yield Reference(val) + else: + yield from self._lookup_schema_references(val) + + elif isinstance(attr, list): + for val in attr: + yield from self._lookup_schema_references(val) + From b18346085df891c7e605705f81440249a556f415 Mon Sep 17 00:00:00 2001 From: Nementon Date: Sat, 6 Feb 2021 00:54:30 +0100 Subject: [PATCH 17/39] __init__ / _get_document: use SchemaResolver --- openapi_python_client/__init__.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/openapi_python_client/__init__.py b/openapi_python_client/__init__.py index b5ad8afeb..ec0d61ae5 100644 --- a/openapi_python_client/__init__.py +++ b/openapi_python_client/__init__.py @@ -17,6 +17,7 @@ from .parser import GeneratorData, import_string_from_reference from .parser.errors import GeneratorError from .utils import snake_case +from .resolver.schema_resolver import SchemaResolver if sys.version_info.minor < 8: # version did not exist before 3.8, need to use a backport from importlib_metadata import version @@ -318,20 +319,19 @@ def update_existing_client( def _get_document(*, url: Optional[str], path: Optional[Path]) -> Union[Dict[str, Any], GeneratorError]: - yaml_bytes: bytes if url is not None and path is not None: return GeneratorError(header="Provide URL or Path, not both.") - if url is not None: - try: - response = httpx.get(url) - yaml_bytes = response.content - except (httpx.HTTPError, httpcore.NetworkError): - return GeneratorError(header="Could not get OpenAPI document from provided URL") - elif path is not None: - yaml_bytes = path.read_bytes() - else: + + if url is None and path is None: return GeneratorError(header="No URL or Path provided") + + source: Union[str, Path] = url if url is not None else path try: - return yaml.safe_load(yaml_bytes) - except yaml.YAMLError: + resolver = SchemaResolver(source) + result = resolver.resolve() + if len(result.errors) > 0: + return GeneratorError(header=errors.join('; ')) + except Exception as e: return GeneratorError(header="Invalid YAML from provided source") + + return result.schema From cedb946afc72f83c557c1cd900987a2533f10966 Mon Sep 17 00:00:00 2001 From: Nementon Date: Sat, 6 Feb 2021 23:16:48 +0100 Subject: [PATCH 18/39] resolver / wip resolve remote ref to local ones --- openapi_python_client/__init__.py | 13 +- .../resolver/.resolved_schema.py.swp | Bin 0 -> 12288 bytes .../resolver/.schema_resolver.py.swp | Bin 0 -> 16384 bytes openapi_python_client/resolver/data_loader.py | 8 +- openapi_python_client/resolver/reference.py | 41 +++- .../resolver/resolved_schema.py | 191 +++++++++++++++++- .../resolver/resolver_types.py | 2 +- .../resolver/schema_resolver.py | 68 ++++--- 8 files changed, 259 insertions(+), 64 deletions(-) create mode 100644 openapi_python_client/resolver/.resolved_schema.py.swp create mode 100644 openapi_python_client/resolver/.schema_resolver.py.swp diff --git a/openapi_python_client/__init__.py b/openapi_python_client/__init__.py index ec0d61ae5..9fa6ec9ed 100644 --- a/openapi_python_client/__init__.py +++ b/openapi_python_client/__init__.py @@ -5,19 +5,16 @@ import sys from enum import Enum from pathlib import Path -from typing import Any, Dict, Optional, Sequence, Union +from typing import Any, Dict, Optional, Sequence, Union, cast -import httpcore -import httpx -import yaml from jinja2 import BaseLoader, ChoiceLoader, Environment, FileSystemLoader, PackageLoader from openapi_python_client import utils from .parser import GeneratorData, import_string_from_reference from .parser.errors import GeneratorError -from .utils import snake_case from .resolver.schema_resolver import SchemaResolver +from .utils import snake_case if sys.version_info.minor < 8: # version did not exist before 3.8, need to use a backport from importlib_metadata import version @@ -325,13 +322,13 @@ def _get_document(*, url: Optional[str], path: Optional[Path]) -> Union[Dict[str if url is None and path is None: return GeneratorError(header="No URL or Path provided") - source: Union[str, Path] = url if url is not None else path + source = cast(Union[str, Path], (url if url is not None else path)) try: resolver = SchemaResolver(source) result = resolver.resolve() if len(result.errors) > 0: - return GeneratorError(header=errors.join('; ')) - except Exception as e: + return GeneratorError(header="; ".join(result.errors)) + except Exception: return GeneratorError(header="Invalid YAML from provided source") return result.schema diff --git a/openapi_python_client/resolver/.resolved_schema.py.swp b/openapi_python_client/resolver/.resolved_schema.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..00668b71c19c357928e09fa02edd8dc8599e5a9c GIT binary patch literal 12288 zcmeI2O=}b}7{^n=n{7p}3aK92h1r)XN?lMa^rBX}bX^g&#OZ9NJ2;!9NwzQ-Jz6jN z1qACC@gjl;&sO{>o_vzbbl1XqFZ%@k%QvX)Rl;VUA2g-nB32o@wU;m}Wo{}< zT@`j9MLo$gC|_Awu(Hysxsmw+*Uj>!&rCAl!?>Y}sr7LK3Ct{kkxlDs^SzrlR{fv) z^=nJ?@}=I);=sX?01`j~NB{{S0VIF~kiaYwuxgIHfx*v}!(S<%bElr=M|>awB!C2v z01`j~NB{{S0VIF~kN^@u0%wpw+$ZG61wx+9L-X+afA#GPDbT*_A?#WYX3<|5%{#qd0hXl$)+({)kQ+T$=6?L*y5 zmeOSv+ksi8nV>GJ!L*dN?LB;Qr6ptxDJS!2Zb!O`;aW>=8i`cJO6?>zSbZ}!TE9oT ze1GKndv2`W2_^1Bdd0PctJTnsU;iXNbqAX`57$<>l0A{yU(w!(Z9Fw^n64Tpbm`KS zQ-@pM)n)HC?Ve<$cPk$tGww6Qg&D!!&t}4G5E1Oe6>zI6lA1Q%$P()iL=!c3AOz0O z5br$>aAz^$=Db`J&dZ+;%S2`wD-Yc5MLthdHsDSy$zx;ab~#{}3(O?da!4E1b$4b9 p#SP7dN@_b%GPuEJCibdb_)0~7jEHV2n@JJpFid&65$y>uk-rS)BC`Mh literal 0 HcmV?d00001 diff --git a/openapi_python_client/resolver/.schema_resolver.py.swp b/openapi_python_client/resolver/.schema_resolver.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..b10291ff1c3357ff534fca995394c8dccffac1e0 GIT binary patch literal 16384 zcmeHNO^g&p6fP0}K#<5mFDAtTnb~BvcafOjBuj!W7!1N@`SGxjrgyq#x230hsIFOd zvRRGsq!{nw-M@=^(1XT{5i~K;i(E{+XcPm+7%v{Y_`R;`pZQ}+^kAeq`L?&JUcGwt z)vJ0{HN7x(>hLpspEJeqY-8;5caNO?UFsSOzQumI2Fv zWxz6E8L$jk2L6K#c=(-vJM4Oc4*dB2KRf^b{s3b?0#|`6z-6EdoCRE9AMg-xV=H4{ z09Szbf%kwF;56{t7RG)8J_23>P61=U4&cxG8T$-)8CV78fQ`Ukn;H8UxC98`3E%** z8~A<`W8VN5fmeVj;KqH7T?Y;VGr(rxmwO=xd<}dGyasr{4DclIFz^*FGCl+Z-~)$% zt9LW@E${(w6u5R5V?O||18YDXV89<68G9R$zZ-Q%X9460M{8V)G=hoNw(Kr(N@0mKwt&~D_WW;AYhb-2 zb!x5RfQpulLO^#M_LC*K@gNRq2gFI0TnN-s zwK6$bsqv{Yb4{`<)d7plP-@apx6TtKQ_ha3-?k7B>$8J8)wZXW#%r8jjuf8nE_9W^ zuol*>Pfs%lMXI5Xq*S+7ln%b`g67qwN-jy5p{IZAe3s0iu^-wGqS=hJ=5-h;-a>(RD(g1}G%p=t z&+ztg2kMz#`2gq{KG45GSI!K%rB*O;@cmr24h?!YDNB3Qbb@p`mZY(&4VXjnv@i!z zM+_7~5w$>(r{B2V*+^Pa$!T&x?P5;S4g*!K?5vE>Po3$}tz>tIDMQUD`Vz(sFYS9d z^ihCBi)5j8nvrpiB0)w#I#dAXAouQv6|OPa^#1p*P*gF z-=WY?BFHY>j4$9LO1<1>BE`BpB)rb)so-m&Y8us&G~f(cl{H^@`g+spmeiHZ?~A{s z>U~q9*IA7jCL$N+4bF(s8VA<+lG5JE0LSSoE4kdz0fO=Kg`W;M{P4j>rPa0Exl4wZ zT_-Ikr!zihB5$8RK-CF}rJ15|?|>JREpLhADfN$-P8~u(3lcq%HVwJ72XkYo1EQm4 zZ=y}B$tvW6gJV@Pf(C$MDC+j*Qq;%RFINNO`$c*ibP%QEu%kq^@Dq$^X`5T>PEGG# zO52gI&q28g^c%XQ)b6CaN$C}pw(Z6+8XL@>teiTd z6`&fOPK>QMD%H{zp_Y3Uf`D1fYnqN)agf5nNnALAh8Z&o%&sm=f=slUrP7mAInjuv zV08Zf6=(HbILp)dAIISIxsLQ(z?;B}fDh~jNNy)!AIpGcz%pPNunbrRECZGS%YbFT zGGH074E*mHpzD(Iqkn}>?;P|E&&V5MT&NWHAf$pWmywzM@LJr$iLm!BO8?{<21$Oy QT(*Gz{|8o3b$#vjH;XGpFaQ7m literal 0 HcmV?d00001 diff --git a/openapi_python_client/resolver/data_loader.py b/openapi_python_client/resolver/data_loader.py index ab899d4b3..aaa0fd26b 100644 --- a/openapi_python_client/resolver/data_loader.py +++ b/openapi_python_client/resolver/data_loader.py @@ -1,13 +1,14 @@ import yaml + from .resolver_types import SchemaData -class DataLoader: +class DataLoader: @classmethod def load(cls, path: str, data: bytes) -> SchemaData: - data_type = path.split('.')[-1] + data_type = path.split(".")[-1].casefold() - if data_type == 'json': + if data_type == "json": return cls.load_json(data) else: return cls.load_yaml(data) @@ -19,4 +20,3 @@ def load_json(cls, data: bytes) -> SchemaData: @classmethod def load_yaml(cls, data: bytes) -> SchemaData: return yaml.safe_load(data) - diff --git a/openapi_python_client/resolver/reference.py b/openapi_python_client/resolver/reference.py index 3d387ec98..b38b82160 100644 --- a/openapi_python_client/resolver/reference.py +++ b/openapi_python_client/resolver/reference.py @@ -1,22 +1,43 @@ -class Reference: +import urllib +from typing import Union + - def __init__(self, reference): +class Reference: + def __init__(self, reference: str): self._ref = reference + @property + def remote_relative_path(self) -> Union[str, None]: + if self.is_remote_ref(): + return self._ref.split("#")[0] + return None + + @property + def path_parent(self) -> str: + path = self.path + parts = path.split("/") + parts.pop() + return "/".join(parts) + + @property + def path(self) -> str: + d = self._ref.split("#")[-1] + d = urllib.parse.unquote(d) + d = d.replace("~1", "/") + return d + @property def value(self) -> str: return self._ref - def is_relative_reference(self): + def is_relative_reference(self) -> bool: return self.is_remote_ref() and not self.is_url_reference() - def is_url_reference(self): - return self.is_remote_ref() and (self._ref.startswith('//', 0) or self._ref.startswith('http', 0)) + def is_url_reference(self) -> bool: + return self.is_remote_ref() and (self._ref.startswith("//", 0) or self._ref.startswith("http", 0)) - def is_remote_ref(self): + def is_remote_ref(self) -> bool: return not self.is_local_ref() - - def is_local_ref(self): - return self._ref.startswith('#', 0) - + def is_local_ref(self) -> bool: + return self._ref.startswith("#", 0) diff --git a/openapi_python_client/resolver/resolved_schema.py b/openapi_python_client/resolver/resolved_schema.py index 19eb47d2b..35c41facb 100644 --- a/openapi_python_client/resolver/resolved_schema.py +++ b/openapi_python_client/resolver/resolved_schema.py @@ -1,25 +1,196 @@ -from typing import Any, Dict, Optional, Sequence, Union +import hashlib +from typing import Any, Dict, Generator, List, Tuple, Union, cast + +from .reference import Reference from .resolver_types import SchemaData class ResolvedSchema: - - def __init__(self, root, refs, errors): + def __init__(self, root: SchemaData, refs: Dict[str, SchemaData], errors: List[str]): self._root: SchemaData = root self._refs: Dict[str, SchemaData] = refs - self._errors: Sequense[str] = errors + self._errors: List[str] = errors + self._resolved_remotes_components: SchemaData = cast(SchemaData, {}) - self._resolved_schema: SchemaData = self._root - self._process() + self._resolved_schema: SchemaData = cast(SchemaData, {}) + if len(self._errors) == 0: + self._process() @property def schema(self) -> SchemaData: - return self._resolved_schema + return self._root @property - def errors(self) -> Sequence[str]: + def errors(self) -> List[str]: return self._errors.copy() - def _process(self): - pass + def _process(self) -> None: + self._process_remote_paths() + self._process_remote_components(self._root) + self._root.update(self._resolved_remotes_components) + + def _process_remote_paths(self) -> None: + refs_to_replace = [] + for owner, ref_key, ref_val in self._lookup_schema_references_in(self._root, "paths"): + ref = Reference(ref_val) + + if ref.is_local_ref(): + continue + + remote_path = ref.remote_relative_path + path = ref.path + + if remote_path not in self._refs: + self._errors.append("Failed to resolve remote reference > {0}".format(remote_path)) + else: + remote_schema = self._refs[remote_path] + remote_value = self._lookup_dict(remote_schema, path) + if not remote_value: + self._errors.append("Failed to read remote value {}, in remote ref {}".format(path, remote_path)) + else: + refs_to_replace.append((owner, remote_schema, remote_value)) + + for owner, remote_schema, remote_value in refs_to_replace: + self._process_remote_components(remote_schema, remote_value, 1) + self._replace_reference_with(owner, remote_value) + + def _process_remote_components( + self, owner: SchemaData, subpart: Union[SchemaData, None] = None, depth: int = 0 + ) -> None: + target = subpart if subpart else owner + + for parent, ref_key, ref_val in self._lookup_schema_references(target): + ref = Reference(ref_val) + + if ref.is_local_ref(): + # print('Found local reference >> {0}'.format(ref.value)) + if depth > 0: + self._transform_to_local_components(owner, ref) + else: + remote_path = ref.remote_relative_path + if remote_path not in self._refs: + self._errors.append("Failed to resolve remote reference > {0}".format(remote_path)) + else: + remote_owner = self._refs[remote_path] + self._transform_to_local_components(remote_owner, ref) + self._transform_to_local_ref(parent, ref) + + def _transform_to_local_components(self, owner: SchemaData, ref: Reference) -> None: + self._ensure_components_dir_exists(ref) + + # print('Processing remote component > {0}'.format(ref.value)) + remote_component = self._lookup_dict(owner, ref.path) + root_components_dir = self._lookup_dict(self._resolved_remotes_components, ref.path_parent) + component_name = ref.path.split("/")[-1] + + if component_name == "SorTransparentContainer" or component_name == "sorTransparentContainer": + print(ref.value) + + if remote_component is None: + print("Weirdy relookup of >> {0}".format(ref.value)) + assert ref.is_local_ref() and self._lookup_dict(self._resolved_remotes_components, ref.path) + return + + if "$ref" in remote_component: + subref = Reference(remote_component["$ref"]) + if not subref.is_local_ref(): + print("Lookup remote ref >>> {0}".format(subref.value)) + return self._process_remote_components(remote_component) + + if root_components_dir: + if component_name in root_components_dir: + local_component_hash = self._reference_schema_hash(root_components_dir[component_name]) + remote_component_hash = self._reference_schema_hash(remote_component) + + if local_component_hash == remote_component_hash: + return + else: + pass + # print('=' * 120) + # print('TODO: Find compoment collision to handle on >>> {0}'.format(ref.path)) + # print('Local componente {0} >> {1}'.format(local_component_hash, root_components_dir[component_name])) + # print('') + # print('Remote componente {0} >> {1}'.format(remote_component_hash, remote_component)) + # print('=' * 120) + else: + root_components_dir[component_name] = remote_component + self._process_remote_components(owner, remote_component, 2) + + def _ensure_components_dir_exists(self, ref: Reference) -> None: + cursor = self._resolved_remotes_components + for key in ref.path_parent.split("/"): + if key == "": + continue + + if key not in cursor: + cursor[key] = {} + + cursor = cursor[key] + + def _transform_to_local_ref(self, owner: Dict[str, Any], ref: Reference) -> None: + owner["$ref"] = "#{0}".format(ref.path) + + def _lookup_dict(self, attr: SchemaData, query: str) -> Union[SchemaData, None]: + cursor = attr + query_parts = [] + + if query.startswith("/paths"): + query_parts = ["paths", query.replace("/paths//", "/").replace("/paths", "")] + else: + query_parts = query.split("/") + + for key in query_parts: + if key == "": + continue + + if isinstance(cursor, dict) and key in cursor: + cursor = cursor[key] + else: + return None + return cursor + + def _replace_reference_with(self, root: Dict[str, Any], new_value: Dict[str, Any]) -> None: + for key in new_value: + root[key] = new_value[key] + + root.pop("$ref") + + def _lookup_schema_references_in( + self, attr: SchemaData, path: str + ) -> Generator[Tuple[SchemaData, str, Any], None, None]: + if not isinstance(attr, dict) or path not in attr: + return + + yield from self._lookup_schema_references(attr[path]) + + def _lookup_schema_references(self, attr: Any) -> Generator[Tuple[SchemaData, str, str], None, None]: + if isinstance(attr, dict): + for key, val in attr.items(): + if key == "$ref": + yield cast(SchemaData, attr), cast(str, key), cast(str, val) + else: + yield from self._lookup_schema_references(val) + + elif isinstance(attr, list): + for val in attr: + yield from self._lookup_schema_references(val) + + def _reference_schema_hash(self, schema: Dict[str, Any]) -> str: + md5 = hashlib.md5() + hash_elms = [] + for key in schema.keys(): + if key == "description": + continue + + if key == "type": + hash_elms.append(schema[key]) + + if key == "allOf": + for item in schema[key]: + hash_elms.append(str(item)) + + hash_elms.append(key) + hash_elms.sort() + md5.update(";".join(hash_elms).encode("utf-8")) + return md5.hexdigest() diff --git a/openapi_python_client/resolver/resolver_types.py b/openapi_python_client/resolver/resolver_types.py index baf35099b..84f6cea5b 100644 --- a/openapi_python_client/resolver/resolver_types.py +++ b/openapi_python_client/resolver/resolver_types.py @@ -1,3 +1,3 @@ from typing import Any, Dict, NewType -SchemaData = NewType('SchemaData', Dict[str, Any]) +SchemaData = NewType("SchemaData", Dict[str, Any]) diff --git a/openapi_python_client/resolver/schema_resolver.py b/openapi_python_client/resolver/schema_resolver.py index 2ac040c52..653bb2dfd 100644 --- a/openapi_python_client/resolver/schema_resolver.py +++ b/openapi_python_client/resolver/schema_resolver.py @@ -1,26 +1,25 @@ -import httpcore -import httpx -import urllib import logging - -from typing import Any, Dict, Optional, Sequence, Union, Generator, NewType +import urllib from pathlib import Path +from typing import Any, Dict, Generator, List, Union -from .resolver_types import SchemaData +import httpx + +from .data_loader import DataLoader from .reference import Reference from .resolved_schema import ResolvedSchema -from .data_loader import DataLoader +from .resolver_types import SchemaData -class SchemaResolver: +class SchemaResolver: def __init__(self, url_or_path: Union[str, Path]): if not url_or_path: - raise ValueError('Invalid document root reference, it shall be an remote url or local file path') - - self._root_path: Optional[Path] = None - self._root_path_dir: Optional[Path] = None - self._root_url: Optional[str] = None - self._root_url_scheme: Optional[str] = None + raise ValueError("Invalid document root reference, it shall be an remote url or local file path") + + self._root_path: Union[Path, None] = None + self._root_path_dir: Union[Path, None] = None + self._root_url: Union[str, None] = None + self._root_url_scheme: Union[str, None] = None if isinstance(url_or_path, Path): self._root_path = url_or_path.absolute() @@ -28,28 +27,32 @@ def __init__(self, url_or_path: Union[str, Path]): else: self._root_url = url_or_path self._root_url_scheme = urllib.parse.urlparse(url_or_path).scheme - + def resolve(self, recursive: bool = True) -> ResolvedSchema: + assert self._root_path or self._root_url + root_schema: SchemaData external_schemas: Dict[str, SchemaData] = {} - errors: Sequence[str] = [] + errors: List[str] = [] if self._root_path: root_schema = self._fetch_remote_file_path(self._root_path) - else: + elif self._root_url: root_schema = self._fetch_url_reference(self._root_url) self._resolve_schema_references(root_schema, external_schemas, errors, recursive) return ResolvedSchema(root_schema, external_schemas, errors) - def _resolve_schema_references(self, root: SchemaData, external_schemas: Dict[str, SchemaData], errors: Sequence[str], recursive: bool) -> Sequence[SchemaData]: + def _resolve_schema_references( + self, root: SchemaData, external_schemas: Dict[str, SchemaData], errors: List[str], recursive: bool + ) -> None: for ref in self._lookup_schema_references(root): if ref.is_local_ref(): continue try: - path = ref.value.split('#')[0] + path = ref.value.split("#")[0] if path in external_schemas: continue @@ -61,33 +64,37 @@ def _resolve_schema_references(self, root: SchemaData, external_schemas: Dict[st if recursive: self._resolve_schema_references(external_schemas[path], external_schemas, errors, recursive) - except Exception as e: - errors.append('Failed to gather external reference data of {0}'.format(ref.value)) - logging.exception('Failed to gather external reference data of {0}'.format(ref.value)) - + except Exception: + errors.append("Failed to gather external reference data of {0}".format(ref.value)) + logging.exception("Failed to gather external reference data of {0}".format(ref.value)) + def _fetch_remote_reference(self, relative_path: str) -> SchemaData: - if self._root_path: + assert self._root_path_dir or self._root_url + + if self._root_path_dir: abs_path = self._root_path_dir.joinpath(relative_path) return self._fetch_remote_file_path(abs_path) - else: + elif self._root_url: abs_url = urllib.parse.urljoin(self._root_url, relative_path) return self._fetch_url_reference(abs_url) + else: + raise RuntimeError("Bad object initilalization") def _fetch_remote_file_path(self, path: Path) -> SchemaData: - logging.info('Fetching remote ref file path > {0}'.format(path)) + logging.info("Fetching remote ref file path > {0}".format(path)) return DataLoader.load(str(path), path.read_bytes()) def _fetch_url_reference(self, url: str) -> SchemaData: - if url.startswith('//', 0): - url = "{0}{1}".format(self._root_url_scheme, url) + if url.startswith("//", 0): + url = "{0}:{1}".format(self._root_url_scheme, url) - logging.info('Fetching remote ref url > {0}'.format(url)) + logging.info("Fetching remote ref url > {0}".format(url)) return DataLoader.load(url, httpx.get(url).content) def _lookup_schema_references(self, attr: Any) -> Generator[Reference, None, None]: if isinstance(attr, dict): for key, val in attr.items(): - if key == '$ref': + if key == "$ref": yield Reference(val) else: yield from self._lookup_schema_references(val) @@ -95,4 +102,3 @@ def _lookup_schema_references(self, attr: Any) -> Generator[Reference, None, Non elif isinstance(attr, list): for val in attr: yield from self._lookup_schema_references(val) - From e38a6ad4608e0ee6648d1fefd217ad2b0939137d Mon Sep 17 00:00:00 2001 From: Nementon Date: Wed, 10 Feb 2021 20:02:42 +0100 Subject: [PATCH 19/39] correct tests breaking changes --- openapi_python_client/__init__.py | 5 +++++ .../resolver/schema_resolver.py | 18 +++++++++++++----- tests/test___init__.py | 17 ++++++++++------- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/openapi_python_client/__init__.py b/openapi_python_client/__init__.py index 9fa6ec9ed..61ae6914b 100644 --- a/openapi_python_client/__init__.py +++ b/openapi_python_client/__init__.py @@ -3,10 +3,13 @@ import shutil import subprocess import sys +import urllib from enum import Enum from pathlib import Path from typing import Any, Dict, Optional, Sequence, Union, cast +import httpcore +import httpx from jinja2 import BaseLoader, ChoiceLoader, Environment, FileSystemLoader, PackageLoader from openapi_python_client import utils @@ -328,6 +331,8 @@ def _get_document(*, url: Optional[str], path: Optional[Path]) -> Union[Dict[str result = resolver.resolve() if len(result.errors) > 0: return GeneratorError(header="; ".join(result.errors)) + except (httpx.HTTPError, httpcore.NetworkError, urllib.error.URLError): + return GeneratorError(header="Could not get OpenAPI document from provided URL") except Exception: return GeneratorError(header="Invalid YAML from provided source") diff --git a/openapi_python_client/resolver/schema_resolver.py b/openapi_python_client/resolver/schema_resolver.py index 653bb2dfd..b720052f8 100644 --- a/openapi_python_client/resolver/schema_resolver.py +++ b/openapi_python_client/resolver/schema_resolver.py @@ -1,7 +1,7 @@ import logging import urllib from pathlib import Path -from typing import Any, Dict, Generator, List, Union +from typing import Any, Dict, Generator, List, Union, cast import httpx @@ -21,12 +21,20 @@ def __init__(self, url_or_path: Union[str, Path]): self._root_url: Union[str, None] = None self._root_url_scheme: Union[str, None] = None - if isinstance(url_or_path, Path): + if self._isapath(url_or_path): + url_or_path = cast(Path, url_or_path) self._root_path = url_or_path.absolute() self._root_path_dir = self._root_path.parent else: + url_or_path = cast(str, url_or_path) self._root_url = url_or_path - self._root_url_scheme = urllib.parse.urlparse(url_or_path).scheme + try: + self._root_url_scheme = urllib.parse.urlparse(url_or_path).scheme + except Exception: + raise urllib.error.URLError(f"Coult not parse URL > {url_or_path}") + + def _isapath(self, url_or_path: Union[str, Path]) -> bool: + return isinstance(url_or_path, Path) def resolve(self, recursive: bool = True) -> ResolvedSchema: assert self._root_path or self._root_url @@ -81,14 +89,14 @@ def _fetch_remote_reference(self, relative_path: str) -> SchemaData: raise RuntimeError("Bad object initilalization") def _fetch_remote_file_path(self, path: Path) -> SchemaData: - logging.info("Fetching remote ref file path > {0}".format(path)) + logging.info(f"Fetching remote ref file path > {path}") return DataLoader.load(str(path), path.read_bytes()) def _fetch_url_reference(self, url: str) -> SchemaData: if url.startswith("//", 0): url = "{0}:{1}".format(self._root_url_scheme, url) - logging.info("Fetching remote ref url > {0}".format(url)) + logging.info(f"Fetching remote ref url > {url}") return DataLoader.load(url, httpx.get(url).content) def _lookup_schema_references(self, attr: Any) -> Generator[Reference, None, None]: diff --git a/tests/test___init__.py b/tests/test___init__.py index 3d2547d89..44988ec2e 100644 --- a/tests/test___init__.py +++ b/tests/test___init__.py @@ -1,4 +1,5 @@ import pathlib +from urllib.parse import ParseResult import httpcore import jinja2 @@ -169,7 +170,7 @@ def test__get_document_url_and_path(self, mocker): loads.assert_not_called() def test__get_document_bad_url(self, mocker): - get = mocker.patch("httpx.get", side_effect=httpcore.NetworkError) + get = mocker.patch("httpx.get") Path = mocker.patch("openapi_python_client.Path") loads = mocker.patch("yaml.safe_load") @@ -179,7 +180,7 @@ def test__get_document_bad_url(self, mocker): result = _get_document(url=url, path=None) assert result == GeneratorError(header="Could not get OpenAPI document from provided URL") - get.assert_called_once_with(url) + get.assert_not_called() Path.assert_not_called() loads.assert_not_called() @@ -190,7 +191,7 @@ def test__get_document_url_no_path(self, mocker): from openapi_python_client import _get_document - url = mocker.MagicMock() + url = "http://localhost/" _get_document(url=url, path=None) get.assert_called_once_with(url) @@ -200,6 +201,7 @@ def test__get_document_url_no_path(self, mocker): def test__get_document_path_no_url(self, mocker): get = mocker.patch("httpx.get") loads = mocker.patch("yaml.safe_load") + mocker.patch("openapi_python_client.resolver.schema_resolver.SchemaResolver._isapath", return_value=True) from openapi_python_client import _get_document @@ -207,12 +209,13 @@ def test__get_document_path_no_url(self, mocker): _get_document(url=None, path=path) get.assert_not_called() - path.read_bytes.assert_called_once() - loads.assert_called_once_with(path.read_bytes()) + path.absolute().read_bytes.assert_called_once() + loads.assert_called_once_with(path.absolute().read_bytes()) def test__get_document_bad_yaml(self, mocker): get = mocker.patch("httpx.get") loads = mocker.patch("yaml.safe_load", side_effect=yaml.YAMLError) + mocker.patch("openapi_python_client.resolver.schema_resolver.SchemaResolver._isapath", return_value=True) from openapi_python_client import _get_document @@ -220,8 +223,8 @@ def test__get_document_bad_yaml(self, mocker): result = _get_document(url=None, path=path) get.assert_not_called() - path.read_bytes.assert_called_once() - loads.assert_called_once_with(path.read_bytes()) + path.absolute().read_bytes.assert_called_once() + loads.assert_called_once_with(path.absolute().read_bytes()) assert result == GeneratorError(header="Invalid YAML from provided source") From b60c3827fdd825926aa0c6d3e49cb88b87edfcef Mon Sep 17 00:00:00 2001 From: Nementon Date: Thu, 11 Feb 2021 18:30:30 +0100 Subject: [PATCH 20/39] resolver / refactor (squash me) --- .../resolver/.resolved_schema.py.swp | Bin 12288 -> 0 bytes .../resolver/.schema_resolver.py.swp | Bin 16384 -> 0 bytes openapi_python_client/resolver/pointer.py | 48 +++++++++++++++ openapi_python_client/resolver/reference.py | 58 ++++++++++-------- .../resolver/resolved_schema.py | 24 +++++--- .../resolver/schema_resolver.py | 8 ++- 6 files changed, 101 insertions(+), 37 deletions(-) delete mode 100644 openapi_python_client/resolver/.resolved_schema.py.swp delete mode 100644 openapi_python_client/resolver/.schema_resolver.py.swp create mode 100644 openapi_python_client/resolver/pointer.py diff --git a/openapi_python_client/resolver/.resolved_schema.py.swp b/openapi_python_client/resolver/.resolved_schema.py.swp deleted file mode 100644 index 00668b71c19c357928e09fa02edd8dc8599e5a9c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2O=}b}7{^n=n{7p}3aK92h1r)XN?lMa^rBX}bX^g&#OZ9NJ2;!9NwzQ-Jz6jN z1qACC@gjl;&sO{>o_vzbbl1XqFZ%@k%QvX)Rl;VUA2g-nB32o@wU;m}Wo{}< zT@`j9MLo$gC|_Awu(Hysxsmw+*Uj>!&rCAl!?>Y}sr7LK3Ct{kkxlDs^SzrlR{fv) z^=nJ?@}=I);=sX?01`j~NB{{S0VIF~kiaYwuxgIHfx*v}!(S<%bElr=M|>awB!C2v z01`j~NB{{S0VIF~kN^@u0%wpw+$ZG61wx+9L-X+afA#GPDbT*_A?#WYX3<|5%{#qd0hXl$)+({)kQ+T$=6?L*y5 zmeOSv+ksi8nV>GJ!L*dN?LB;Qr6ptxDJS!2Zb!O`;aW>=8i`cJO6?>zSbZ}!TE9oT ze1GKndv2`W2_^1Bdd0PctJTnsU;iXNbqAX`57$<>l0A{yU(w!(Z9Fw^n64Tpbm`KS zQ-@pM)n)HC?Ve<$cPk$tGww6Qg&D!!&t}4G5E1Oe6>zI6lA1Q%$P()iL=!c3AOz0O z5br$>aAz^$=Db`J&dZ+;%S2`wD-Yc5MLthdHsDSy$zx;ab~#{}3(O?da!4E1b$4b9 p#SP7dN@_b%GPuEJCibdb_)0~7jEHV2n@JJpFid&65$y>uk-rS)BC`Mh diff --git a/openapi_python_client/resolver/.schema_resolver.py.swp b/openapi_python_client/resolver/.schema_resolver.py.swp deleted file mode 100644 index b10291ff1c3357ff534fca995394c8dccffac1e0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHNO^g&p6fP0}K#<5mFDAtTnb~BvcafOjBuj!W7!1N@`SGxjrgyq#x230hsIFOd zvRRGsq!{nw-M@=^(1XT{5i~K;i(E{+XcPm+7%v{Y_`R;`pZQ}+^kAeq`L?&JUcGwt z)vJ0{HN7x(>hLpspEJeqY-8;5caNO?UFsSOzQumI2Fv zWxz6E8L$jk2L6K#c=(-vJM4Oc4*dB2KRf^b{s3b?0#|`6z-6EdoCRE9AMg-xV=H4{ z09Szbf%kwF;56{t7RG)8J_23>P61=U4&cxG8T$-)8CV78fQ`Ukn;H8UxC98`3E%** z8~A<`W8VN5fmeVj;KqH7T?Y;VGr(rxmwO=xd<}dGyasr{4DclIFz^*FGCl+Z-~)$% zt9LW@E${(w6u5R5V?O||18YDXV89<68G9R$zZ-Q%X9460M{8V)G=hoNw(Kr(N@0mKwt&~D_WW;AYhb-2 zb!x5RfQpulLO^#M_LC*K@gNRq2gFI0TnN-s zwK6$bsqv{Yb4{`<)d7plP-@apx6TtKQ_ha3-?k7B>$8J8)wZXW#%r8jjuf8nE_9W^ zuol*>Pfs%lMXI5Xq*S+7ln%b`g67qwN-jy5p{IZAe3s0iu^-wGqS=hJ=5-h;-a>(RD(g1}G%p=t z&+ztg2kMz#`2gq{KG45GSI!K%rB*O;@cmr24h?!YDNB3Qbb@p`mZY(&4VXjnv@i!z zM+_7~5w$>(r{B2V*+^Pa$!T&x?P5;S4g*!K?5vE>Po3$}tz>tIDMQUD`Vz(sFYS9d z^ihCBi)5j8nvrpiB0)w#I#dAXAouQv6|OPa^#1p*P*gF z-=WY?BFHY>j4$9LO1<1>BE`BpB)rb)so-m&Y8us&G~f(cl{H^@`g+spmeiHZ?~A{s z>U~q9*IA7jCL$N+4bF(s8VA<+lG5JE0LSSoE4kdz0fO=Kg`W;M{P4j>rPa0Exl4wZ zT_-Ikr!zihB5$8RK-CF}rJ15|?|>JREpLhADfN$-P8~u(3lcq%HVwJ72XkYo1EQm4 zZ=y}B$tvW6gJV@Pf(C$MDC+j*Qq;%RFINNO`$c*ibP%QEu%kq^@Dq$^X`5T>PEGG# zO52gI&q28g^c%XQ)b6CaN$C}pw(Z6+8XL@>teiTd z6`&fOPK>QMD%H{zp_Y3Uf`D1fYnqN)agf5nNnALAh8Z&o%&sm=f=slUrP7mAInjuv zV08Zf6=(HbILp)dAIISIxsLQ(z?;B}fDh~jNNy)!AIpGcz%pPNunbrRECZGS%YbFT zGGH074E*mHpzD(Iqkn}>?;P|E&&V5MT&NWHAf$pWmywzM@LJr$iLm!BO8?{<21$Oy QT(*Gz{|8o3b$#vjH;XGpFaQ7m diff --git a/openapi_python_client/resolver/pointer.py b/openapi_python_client/resolver/pointer.py new file mode 100644 index 000000000..36874e294 --- /dev/null +++ b/openapi_python_client/resolver/pointer.py @@ -0,0 +1,48 @@ +import urllib.parse +from typing import List, Union + + +class Pointer: + """ https://tools.ietf.org/html/rfc6901 """ + + def __init__(self, pointer: str) -> None: + if pointer is None or pointer != "" and not pointer.startswith("/"): + raise ValueError(f'Invalid pointer value {pointer}, it must match: *( "/" reference-token )') + + self._pointer = pointer + + @property + def value(self) -> str: + return self._pointer + + @property + def parent(self) -> Union["Pointer", None]: + tokens = self.tokens(False) + + if len(tokens) > 1: + tokens.pop() + return Pointer("/".join(tokens)) + else: + assert tokens[-1] == "" + return None + + def tokens(self, unescape: bool = True) -> List[str]: + tokens = [] + + if unescape: + for token in self._pointer.split("/"): + tokens.append(self._unescape(token)) + else: + tokens = self._pointer.split("/") + + return tokens + + @property + def unescapated_value(self) -> str: + return self._unescape(self._pointer) + + def _unescape(self, data: str) -> str: + data = urllib.parse.unquote(data) + data = data.replace("~1", "/") + data = data.replace("~0", "~") + return data diff --git a/openapi_python_client/resolver/reference.py b/openapi_python_client/resolver/reference.py index b38b82160..534232bcd 100644 --- a/openapi_python_client/resolver/reference.py +++ b/openapi_python_client/resolver/reference.py @@ -1,43 +1,51 @@ -import urllib -from typing import Union +import urllib.parse + +from .pointer import Pointer class Reference: + """ https://tools.ietf.org/html/draft-pbryan-zyp-json-ref-03 """ + def __init__(self, reference: str): self._ref = reference + self._parsed_ref = urllib.parse.urlparse(reference) @property - def remote_relative_path(self) -> Union[str, None]: - if self.is_remote_ref(): - return self._ref.split("#")[0] - return None + def path(self) -> str: + return urllib.parse.urldefrag(self._parsed_ref.geturl()).url @property - def path_parent(self) -> str: - path = self.path - parts = path.split("/") - parts.pop() - return "/".join(parts) + def pointer(self) -> Pointer: + frag = self._parsed_ref.fragment + if self.is_url() and frag != "" and not frag.startswith("/"): + frag = f"/{frag}" - @property - def path(self) -> str: - d = self._ref.split("#")[-1] - d = urllib.parse.unquote(d) - d = d.replace("~1", "/") - return d + return Pointer(frag) + + def is_relative(self) -> bool: + """ return True if reference path is a relative path """ + return not self.is_absolute() + + def is_absolute(self) -> bool: + """ return True is reference path is an absolute path """ + return self._parsed_ref.netloc != "" @property def value(self) -> str: return self._ref - def is_relative_reference(self) -> bool: - return self.is_remote_ref() and not self.is_url_reference() + def is_url(self) -> bool: + """ return True if the reference path is pointing to an external url location """ + return self.is_remote() and self._parsed_ref.netloc != "" - def is_url_reference(self) -> bool: - return self.is_remote_ref() and (self._ref.startswith("//", 0) or self._ref.startswith("http", 0)) + def is_remote(self) -> bool: + """ return True if the reference pointer is pointing to a remote document """ + return not self.is_local() - def is_remote_ref(self) -> bool: - return not self.is_local_ref() + def is_local(self) -> bool: + """ return True if the reference pointer is pointing to the current document """ + return self._parsed_ref.path == "" - def is_local_ref(self) -> bool: - return self._ref.startswith("#", 0) + def is_full_document(self) -> bool: + """ return True if the reference pointer is pointing to the whole document content """ + return self.pointer.parent is None diff --git a/openapi_python_client/resolver/resolved_schema.py b/openapi_python_client/resolver/resolved_schema.py index 35c41facb..7e528e694 100644 --- a/openapi_python_client/resolver/resolved_schema.py +++ b/openapi_python_client/resolver/resolved_schema.py @@ -34,10 +34,10 @@ def _process_remote_paths(self) -> None: for owner, ref_key, ref_val in self._lookup_schema_references_in(self._root, "paths"): ref = Reference(ref_val) - if ref.is_local_ref(): + if ref.is_local(): continue - remote_path = ref.remote_relative_path + remote_path = ref.pointer.value path = ref.path if remote_path not in self._refs: @@ -62,12 +62,12 @@ def _process_remote_components( for parent, ref_key, ref_val in self._lookup_schema_references(target): ref = Reference(ref_val) - if ref.is_local_ref(): + if ref.is_local(): # print('Found local reference >> {0}'.format(ref.value)) if depth > 0: self._transform_to_local_components(owner, ref) else: - remote_path = ref.remote_relative_path + remote_path = ref.pointer.value if remote_path not in self._refs: self._errors.append("Failed to resolve remote reference > {0}".format(remote_path)) else: @@ -80,20 +80,23 @@ def _transform_to_local_components(self, owner: SchemaData, ref: Reference) -> N # print('Processing remote component > {0}'.format(ref.value)) remote_component = self._lookup_dict(owner, ref.path) - root_components_dir = self._lookup_dict(self._resolved_remotes_components, ref.path_parent) - component_name = ref.path.split("/")[-1] + pointer_parent = ref.pointer.parent + + if pointer_parent is not None: + root_components_dir = self._lookup_dict(self._resolved_remotes_components, pointer_parent.value) + component_name = ref.path.split("/")[-1] if component_name == "SorTransparentContainer" or component_name == "sorTransparentContainer": print(ref.value) if remote_component is None: print("Weirdy relookup of >> {0}".format(ref.value)) - assert ref.is_local_ref() and self._lookup_dict(self._resolved_remotes_components, ref.path) + assert ref.is_local() and self._lookup_dict(self._resolved_remotes_components, ref.path) return if "$ref" in remote_component: subref = Reference(remote_component["$ref"]) - if not subref.is_local_ref(): + if not subref.is_local(): print("Lookup remote ref >>> {0}".format(subref.value)) return self._process_remote_components(remote_component) @@ -118,7 +121,10 @@ def _transform_to_local_components(self, owner: SchemaData, ref: Reference) -> N def _ensure_components_dir_exists(self, ref: Reference) -> None: cursor = self._resolved_remotes_components - for key in ref.path_parent.split("/"): + pointer_dir = ref.pointer.parent + assert pointer_dir is not None + + for key in pointer_dir.value.split("/"): # noqa if key == "": continue diff --git a/openapi_python_client/resolver/schema_resolver.py b/openapi_python_client/resolver/schema_resolver.py index b720052f8..8c9507463 100644 --- a/openapi_python_client/resolver/schema_resolver.py +++ b/openapi_python_client/resolver/schema_resolver.py @@ -30,7 +30,9 @@ def __init__(self, url_or_path: Union[str, Path]): self._root_url = url_or_path try: self._root_url_scheme = urllib.parse.urlparse(url_or_path).scheme - except Exception: + if self._root_url_scheme not in ["http", "https"]: + raise ValueError(f"Unsupported URL scheme '{self._root_url_scheme}', expecting http or https") + except (TypeError, AttributeError): raise urllib.error.URLError(f"Coult not parse URL > {url_or_path}") def _isapath(self, url_or_path: Union[str, Path]) -> bool: @@ -56,7 +58,7 @@ def _resolve_schema_references( ) -> None: for ref in self._lookup_schema_references(root): - if ref.is_local_ref(): + if ref.is_local(): continue try: @@ -64,7 +66,7 @@ def _resolve_schema_references( if path in external_schemas: continue - if ref.is_url_reference(): + if ref.is_url(): external_schemas[path] = self._fetch_url_reference(path) else: external_schemas[path] = self._fetch_remote_reference(path) From 229f75f072f013db9994d9c9c1d2e9669b32d2a1 Mon Sep 17 00:00:00 2001 From: Nementon Date: Thu, 11 Feb 2021 18:31:01 +0100 Subject: [PATCH 21/39] resolver / add reference tests --- .../test_resolver/test_resolver_reference.py | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 tests/test_resolver/test_resolver_reference.py diff --git a/tests/test_resolver/test_resolver_reference.py b/tests/test_resolver/test_resolver_reference.py new file mode 100644 index 000000000..6782426f3 --- /dev/null +++ b/tests/test_resolver/test_resolver_reference.py @@ -0,0 +1,212 @@ +import pytest + + +def get_data_set(): + # https://swagger.io/docs/specification/using-ref/ + return { + "local_references": ["#/definitions/myElement"], + "remote_references": [ + "document.json#/myElement", + "../document.json#/myElement", + "../another-folder/document.json#/myElement", + ], + "url_references": [ + "http://path/to/your/resource", + "http://path/to/your/resource.json#myElement", + "//anotherserver.com/files/example.json", + ], + "relative_references": [ + "#/definitions/myElement", + "document.json#/myElement", + "../document.json#/myElement", + "../another-folder/document.json#/myElement", + ], + "absolute_references": [ + "http://path/to/your/resource", + "http://path/to/your/resource.json#myElement", + "//anotherserver.com/files/example.json", + ], + "full_document_references": [ + "http://path/to/your/resource", + "//anotherserver.com/files/example.json", + ], + "not_full_document_references": [ + "#/definitions/myElement", + "document.json#/myElement", + "../document.json#/myElement", + "../another-folder/document.json#/myElement", + "http://path/to/your/resource.json#myElement", + ], + "path_by_reference": { + "#/definitions/myElement": "", + "document.json#/myElement": "document.json", + "../document.json#/myElement": "../document.json", + "../another-folder/document.json#/myElement": "../another-folder/document.json", + "http://path/to/your/resource": "http://path/to/your/resource", + "http://path/to/your/resource.json#myElement": "http://path/to/your/resource.json", + "//anotherserver.com/files/example.json": "//anotherserver.com/files/example.json", + }, + "pointer_by_reference": { + "#/definitions/myElement": "/definitions/myElement", + "document.json#/myElement": "/myElement", + "../document.json#/myElement": "/myElement", + "../another-folder/document.json#/myElement": "/myElement", + "http://path/to/your/resource": "", + "http://path/to/your/resource.json#myElement": "/myElement", + "//anotherserver.com/files/example.json": "", + }, + "pointerparent_by_reference": { + "#/definitions/myElement": "/definitions", + "document.json#/myElement": "", + "../document.json#/myElement": "", + "../another-folder/document.json#/myElement": "", + "http://path/to/your/resource": None, + "http://path/to/your/resource.json#myElement": "", + "//anotherserver.com/files/example.json": None, + }, + } + + +def test_is_local(): + from openapi_python_client.resolver.reference import Reference + + data_set = get_data_set() + + for ref_str in data_set["local_references"]: + ref = Reference(ref_str) + assert ref.is_local() == True + + for ref_str in data_set["remote_references"]: + ref = Reference(ref_str) + assert ref.is_local() == False + + for ref_str in data_set["url_references"]: + ref = Reference(ref_str) + assert ref.is_local() == False + + +def test_is_remote(): + from openapi_python_client.resolver.reference import Reference + + data_set = get_data_set() + + for ref_str in data_set["local_references"]: + ref = Reference(ref_str) + assert ref.is_remote() == False + + for ref_str in data_set["remote_references"]: + ref = Reference(ref_str) + assert ref.is_remote() == True + + for ref_str in data_set["url_references"]: + ref = Reference(ref_str) + assert ref.is_remote() == True + + +def test_is_url(): + from openapi_python_client.resolver.reference import Reference + + data_set = get_data_set() + + for ref_str in data_set["local_references"]: + ref = Reference(ref_str) + assert ref.is_url() == False + + for ref_str in data_set["remote_references"]: + ref = Reference(ref_str) + assert ref.is_url() == False + + for ref_str in data_set["url_references"]: + ref = Reference(ref_str) + assert ref.is_url() == True + + +def test_is_absolute(): + from openapi_python_client.resolver.reference import Reference + + data_set = get_data_set() + + for ref_str in data_set["absolute_references"]: + ref = Reference(ref_str) + assert ref.is_absolute() == True + + for ref_str in data_set["relative_references"]: + ref = Reference(ref_str) + assert ref.is_absolute() == False + + +def test_is_relative(): + from openapi_python_client.resolver.reference import Reference + + data_set = get_data_set() + + for ref_str in data_set["absolute_references"]: + ref = Reference(ref_str) + assert ref.is_relative() == False + + for ref_str in data_set["relative_references"]: + ref = Reference(ref_str) + assert ref.is_relative() == True + + +def test_pointer(): + from openapi_python_client.resolver.reference import Reference + + data_set = get_data_set() + + for ref_str in data_set["pointer_by_reference"].keys(): + ref = Reference(ref_str) + pointer = data_set["pointer_by_reference"][ref_str] + assert ref.pointer.value == pointer + + +def test_pointer_parent(): + from openapi_python_client.resolver.reference import Reference + + data_set = get_data_set() + + for ref_str in data_set["pointerparent_by_reference"].keys(): + ref = Reference(ref_str) + pointer_parent = data_set["pointerparent_by_reference"][ref_str] + + if pointer_parent is not None: + assert ref.pointer.parent.value == pointer_parent + else: + assert ref.pointer.parent == None + + +def test_path(): + from openapi_python_client.resolver.reference import Reference + + data_set = get_data_set() + + for ref_str in data_set["path_by_reference"].keys(): + ref = Reference(ref_str) + path = data_set["path_by_reference"][ref_str] + assert ref.path == path + + +def test_is_full_document(): + from openapi_python_client.resolver.reference import Reference + + data_set = get_data_set() + + for ref_str in data_set["full_document_references"]: + ref = Reference(ref_str) + assert ref.is_full_document() == True + assert ref.pointer.parent == None + + for ref_str in data_set["not_full_document_references"]: + ref = Reference(ref_str) + assert ref.is_full_document() == False + assert ref.pointer.parent != None + + +def test_value(): + from openapi_python_client.resolver.reference import Reference + + ref = Reference("fooBaR") + assert ref.value == "fooBaR" + + ref = Reference("FooBAR") + assert ref.value == "FooBAR" From 325a8a709d1cc0869726b15cd1cc2244e4ac9be4 Mon Sep 17 00:00:00 2001 From: Nementon Date: Thu, 11 Feb 2021 18:31:21 +0100 Subject: [PATCH 22/39] resolver / add data_loader tests --- .../test_resolver_data_loader.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/test_resolver/test_resolver_data_loader.py diff --git a/tests/test_resolver/test_resolver_data_loader.py b/tests/test_resolver/test_resolver_data_loader.py new file mode 100644 index 000000000..ed20dd95f --- /dev/null +++ b/tests/test_resolver/test_resolver_data_loader.py @@ -0,0 +1,50 @@ +import pytest + + +def test_load(mocker): + from openapi_python_client.resolver.data_loader import DataLoader + + dl_load_json = mocker.patch("openapi_python_client.resolver.data_loader.DataLoader.load_json") + dl_load_yaml = mocker.patch("openapi_python_client.resolver.data_loader.DataLoader.load_yaml") + + content = mocker.MagicMock() + DataLoader.load("foobar.json", content) + dl_load_json.assert_called_once_with(content) + + content = mocker.MagicMock() + DataLoader.load("foobar.jSoN", content) + dl_load_json.assert_called_with(content) + + content = mocker.MagicMock() + DataLoader.load("foobar.yaml", content) + dl_load_yaml.assert_called_once_with(content) + + content = mocker.MagicMock() + DataLoader.load("foobar.yAmL", content) + dl_load_yaml.assert_called_with(content) + + content = mocker.MagicMock() + DataLoader.load("foobar.ymL", content) + dl_load_yaml.assert_called_with(content) + + content = mocker.MagicMock() + DataLoader.load("foobar", content) + dl_load_yaml.assert_called_with(content) + + +def test_load_yaml(mocker): + from openapi_python_client.resolver.data_loader import DataLoader + + yaml_safeload = mocker.patch("yaml.safe_load") + + content = mocker.MagicMock() + DataLoader.load_yaml(content) + yaml_safeload.assert_called_once_with(content) + + +def test_load_json(mocker): + from openapi_python_client.resolver.data_loader import DataLoader + + content = mocker.MagicMock() + with pytest.raises(NotImplementedError): + DataLoader.load_json(content) From 582115c06f5e25d7a84dbb84e7773e66eb9af96c Mon Sep 17 00:00:00 2001 From: Nementon Date: Thu, 11 Feb 2021 18:31:50 +0100 Subject: [PATCH 23/39] resolver / add schema_resolver tests (wip) --- .../test_resolver_schema_resolver.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 tests/test_resolver/test_resolver_schema_resolver.py diff --git a/tests/test_resolver/test_resolver_schema_resolver.py b/tests/test_resolver/test_resolver_schema_resolver.py new file mode 100644 index 000000000..3be3267dd --- /dev/null +++ b/tests/test_resolver/test_resolver_schema_resolver.py @@ -0,0 +1,22 @@ +import urllib + +import pytest + + +def test___init__invalid_data(mocker): + from openapi_python_client.resolver.schema_resolver import SchemaResolver + + with pytest.raises(ValueError): + SchemaResolver(None) + + invalid_url = "foobar" + with pytest.raises(ValueError): + SchemaResolver(invalid_url) + + invalid_url = 42 + with pytest.raises(urllib.error.URLError): + SchemaResolver(invalid_url) + + invalid_url = mocker.Mock() + with pytest.raises(urllib.error.URLError): + SchemaResolver(invalid_url) From 91eb471ce8f154852452db9468c77e673fd074c8 Mon Sep 17 00:00:00 2001 From: Nementon Date: Fri, 12 Feb 2021 18:38:51 +0100 Subject: [PATCH 24/39] resovler / add pointer tests --- tests/test_resolver/test_resolver_pointer.py | 97 ++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 tests/test_resolver/test_resolver_pointer.py diff --git a/tests/test_resolver/test_resolver_pointer.py b/tests/test_resolver/test_resolver_pointer.py new file mode 100644 index 000000000..92e1ded35 --- /dev/null +++ b/tests/test_resolver/test_resolver_pointer.py @@ -0,0 +1,97 @@ +import pytest + + +def get_data_set(): + # https://tools.ietf.org/html/rfc6901 + return { + "valid_pointers": [ + "/myElement", + "/definitions/myElement", + "", + "/foo", + "/foo/0", + "/", + "/a~1b", + "/c%d", + "/e^f", + "/g|h", + "/i\\j" '/k"l', + "/ ", + "/m~0n", + "/m~01", + ], + "invalid_pointers": ["../foo", "foobar", None], + "tokens_by_pointer": { + "/myElement": ["", "myElement"], + "/definitions/myElement": ["", "definitions", "myElement"], + "": [""], + "/foo": ["", "foo"], + "/foo/0": ["", "foo", "0"], + "/": ["", ""], + "/a~1b": ["", "a/b"], + "/c%d": ["", "c%d"], + "/e^f": ["", "e^f"], + "/g|h": ["", "g|h"], + "/i\\j": ["", "i\\j"], + '/k"l': ["", 'k"l'], + "/ ": ["", " "], + "/m~0n": ["", "m~n"], + "/m~01": ["", "m~1"], + }, + } + + +def test___init__(): + from openapi_python_client.resolver.pointer import Pointer + + data_set = get_data_set() + + for pointer_str in data_set["valid_pointers"]: + p = Pointer(pointer_str) + assert p.value != None + assert p.value == pointer_str + + for pointer_str in data_set["invalid_pointers"]: + with pytest.raises(ValueError): + p = Pointer(pointer_str) + + +def test_token(): + from openapi_python_client.resolver.pointer import Pointer + + data_set = get_data_set() + + for pointer_str in data_set["tokens_by_pointer"].keys(): + p = Pointer(pointer_str) + expected_tokens = data_set["tokens_by_pointer"][pointer_str] + + for idx, token in enumerate(p.tokens()): + assert expected_tokens[idx] == token + + +def test_parent(): + from openapi_python_client.resolver.pointer import Pointer + + data_set = get_data_set() + + for pointer_str in data_set["tokens_by_pointer"].keys(): + p = Pointer(pointer_str) + expected_tokens = data_set["tokens_by_pointer"][pointer_str] + + while p.parent is not None: + p = p.parent + expected_tokens.pop() + assert p.tokens()[-1] == expected_tokens[-1] + assert len(p.tokens()) == len(expected_tokens) + + assert len(expected_tokens) == 1 + assert expected_tokens[-1] == "" + + +def test__unescape_and__escape(): + from openapi_python_client.resolver.pointer import Pointer + + escaped_unescaped_values = [("/m~0n", "/m~n"), ("/m~01", "/m~1"), ("/a~1b", "/a/b"), ("/foobar", "/foobar")] + + for escaped, unescaped in escaped_unescaped_values: + assert Pointer(escaped).unescapated_value == unescaped From 08beec544fee4fb0905e6d21f1dc4137159587c6 Mon Sep 17 00:00:00 2001 From: Nementon Date: Sun, 14 Feb 2021 21:54:39 +0100 Subject: [PATCH 25/39] resolver / refactor (squash me) --- .../resolver/schema_resolver.py | 75 +++++++++++++------ 1 file changed, 53 insertions(+), 22 deletions(-) diff --git a/openapi_python_client/resolver/schema_resolver.py b/openapi_python_client/resolver/schema_resolver.py index 8c9507463..5a1c602e7 100644 --- a/openapi_python_client/resolver/schema_resolver.py +++ b/openapi_python_client/resolver/schema_resolver.py @@ -17,17 +17,18 @@ def __init__(self, url_or_path: Union[str, Path]): raise ValueError("Invalid document root reference, it shall be an remote url or local file path") self._root_path: Union[Path, None] = None - self._root_path_dir: Union[Path, None] = None self._root_url: Union[str, None] = None self._root_url_scheme: Union[str, None] = None + self._parent_path: str if self._isapath(url_or_path): url_or_path = cast(Path, url_or_path) self._root_path = url_or_path.absolute() - self._root_path_dir = self._root_path.parent + self._parent_path = str(self._root_path.parent) else: url_or_path = cast(str, url_or_path) self._root_url = url_or_path + self._parent_path = url_or_path try: self._root_url_scheme = urllib.parse.urlparse(url_or_path).scheme if self._root_url_scheme not in ["http", "https"]: @@ -44,17 +45,23 @@ def resolve(self, recursive: bool = True) -> ResolvedSchema: root_schema: SchemaData external_schemas: Dict[str, SchemaData] = {} errors: List[str] = [] + parent: str if self._root_path: root_schema = self._fetch_remote_file_path(self._root_path) elif self._root_url: root_schema = self._fetch_url_reference(self._root_url) - self._resolve_schema_references(root_schema, external_schemas, errors, recursive) + self._resolve_schema_references(self._parent_path, root_schema, external_schemas, errors, recursive) return ResolvedSchema(root_schema, external_schemas, errors) def _resolve_schema_references( - self, root: SchemaData, external_schemas: Dict[str, SchemaData], errors: List[str], recursive: bool + self, + parent: str, + root: SchemaData, + external_schemas: Dict[str, SchemaData], + errors: List[str], + recursive: bool, ) -> None: for ref in self._lookup_schema_references(root): @@ -62,33 +69,57 @@ def _resolve_schema_references( continue try: - path = ref.value.split("#")[0] + path = self._absolute_path(ref.path, parent) + parent = self._parent(path) + if path in external_schemas: continue - if ref.is_url(): - external_schemas[path] = self._fetch_url_reference(path) - else: - external_schemas[path] = self._fetch_remote_reference(path) + external_schemas[path] = self._fetch_remote_reference(path) if recursive: - self._resolve_schema_references(external_schemas[path], external_schemas, errors, recursive) + self._resolve_schema_references(parent, external_schemas[path], external_schemas, errors, recursive) except Exception: - errors.append("Failed to gather external reference data of {0}".format(ref.value)) - logging.exception("Failed to gather external reference data of {0}".format(ref.value)) + errors.append(f"Failed to gather external reference data of {ref.value} from {path}") + logging.exception(f"Failed to gather external reference data of {ref.value} from {path}") - def _fetch_remote_reference(self, relative_path: str) -> SchemaData: - assert self._root_path_dir or self._root_url + def _parent(self, abs_path: str) -> str: + if abs_path.startswith("http", 0): + return urllib.parse.urljoin(f"{abs_path}/", "..") + else: + path = Path(abs_path) + return str(path.parent) + + def _absolute_path(self, relative_path: str, parent: str) -> str: + if relative_path.startswith("http", 0): + return relative_path + + if relative_path.startswith("//"): + if parent.startswith("http"): + scheme = urllib.parse.urlparse(parent).scheme + return f"{scheme}:{relative_path}" + else: + scheme = self._root_url_scheme or "http" + return f"{scheme}:{relative_path}" + + if parent.startswith("http"): + return urllib.parse.urljoin(parent, relative_path) + else: + parent_dir = Path(parent) + abs_path = parent_dir.joinpath(relative_path) + abs_path = abs_path.resolve() + return str(abs_path) - if self._root_path_dir: - abs_path = self._root_path_dir.joinpath(relative_path) - return self._fetch_remote_file_path(abs_path) - elif self._root_url: - abs_url = urllib.parse.urljoin(self._root_url, relative_path) - return self._fetch_url_reference(abs_url) + def _fetch_remote_reference(self, abs_path: str) -> SchemaData: + res: SchemaData + + if abs_path.startswith("http"): + res = self._fetch_url_reference(abs_path) else: - raise RuntimeError("Bad object initilalization") + res = self._fetch_remote_file_path(Path(abs_path)) + + return res def _fetch_remote_file_path(self, path: Path) -> SchemaData: logging.info(f"Fetching remote ref file path > {path}") @@ -96,7 +127,7 @@ def _fetch_remote_file_path(self, path: Path) -> SchemaData: def _fetch_url_reference(self, url: str) -> SchemaData: if url.startswith("//", 0): - url = "{0}:{1}".format(self._root_url_scheme, url) + url = "{0}:{1}".format((self._root_url_scheme or "http"), url) logging.info(f"Fetching remote ref url > {url}") return DataLoader.load(url, httpx.get(url).content) From 97a2e56b94aa0cab9984f2bb89c9c58fe812055c Mon Sep 17 00:00:00 2001 From: Nementon Date: Sun, 14 Feb 2021 21:55:09 +0100 Subject: [PATCH 26/39] resolver / add schema_resolver tests (squash me) --- .../test_resolver_schema_resolver.py | 245 ++++++++++++++++++ 1 file changed, 245 insertions(+) diff --git a/tests/test_resolver/test_resolver_schema_resolver.py b/tests/test_resolver/test_resolver_schema_resolver.py index 3be3267dd..36caa3d7e 100644 --- a/tests/test_resolver/test_resolver_schema_resolver.py +++ b/tests/test_resolver/test_resolver_schema_resolver.py @@ -1,4 +1,6 @@ +import pathlib import urllib +import urllib.parse import pytest @@ -20,3 +22,246 @@ def test___init__invalid_data(mocker): invalid_url = mocker.Mock() with pytest.raises(urllib.error.URLError): SchemaResolver(invalid_url) + + +def test__init_with_filepath(mocker): + mocker.patch("openapi_python_client.resolver.schema_resolver.SchemaResolver._isapath", return_value=True) + mocker.patch("openapi_python_client.resolver.schema_resolver.DataLoader.load", return_value={}) + path = mocker.MagicMock() + + from openapi_python_client.resolver.schema_resolver import SchemaResolver + + resolver = SchemaResolver(path) + resolver.resolve() + + path.absolute().read_bytes.assert_called_once() + + +def test__init_with_url(mocker): + mocker.patch("openapi_python_client.resolver.schema_resolver.DataLoader.load", return_value={}) + url_parse = mocker.patch( + "urllib.parse.urlparse", + return_value=urllib.parse.ParseResult( + scheme="http", netloc="foobar.io", path="foo", params="", query="", fragment="/bar" + ), + ) + get = mocker.patch("httpx.get") + + url = mocker.MagicMock() + + from openapi_python_client.resolver.schema_resolver import SchemaResolver + + resolver = SchemaResolver(url) + resolver.resolve() + + url_parse.assert_called_once_with(url) + get.assert_called_once() + + +def test__resolve_schema_references_with_path(mocker): + read_bytes = mocker.patch("pathlib.Path.read_bytes") + + from openapi_python_client.resolver.schema_resolver import SchemaResolver + + path = pathlib.Path("/foo/bar/foobar") + path_parent = str(path.parent) + schema = {"foo": {"$ref": "foobar#/foobar"}} + external_schemas = {} + errors = [] + + def _datalaod_mocked_result(path, data): + if path == "/foo/bar/foobar": + return {"foobar": "bar", "bar": {"$ref": "bar#/foobar"}, "local": {"$ref": "#/toto"}} + + if path == "/foo/bar/bar": + return {"foobar": "bar", "bar": {"$ref": "../bar#/foobar"}} + + if path == "/foo/bar": + return {"foobar": "bar/bar", "bar": {"$ref": "/barfoo.io/foobar#foobar"}} + + if path == "/barfoo.io/foobar": + return {"foobar": "barfoo.io/foobar", "bar": {"$ref": "./bar#foobar"}} + + if path == "/barfoo.io/bar": + return {"foobar": "barfoo.io/bar", "bar": {"$ref": "/bar.foo/foobar"}} + + if path == "/bar.foo/foobar": + return {"foobar": "bar.foo/foobar", "bar": {"$ref": "/foo.bar/foobar"}} + + if path == "/foo.bar/foobar": + return {"foobar": "foo.bar/foobar", "bar": {"$ref": "/foo/bar/foobar"}} # Loop to first path + + raise ValueError(f"Unexpected path {path}") + + mocker.patch("openapi_python_client.resolver.schema_resolver.DataLoader.load", _datalaod_mocked_result) + resolver = SchemaResolver(path) + resolver._resolve_schema_references(path_parent, schema, external_schemas, errors, True) + + assert len(errors) == 0 + assert "/foo/bar/foobar" in external_schemas + assert "/foo/bar/bar" in external_schemas + assert "/foo/bar" in external_schemas + assert "/barfoo.io/foobar" in external_schemas + assert "/barfoo.io/bar" in external_schemas + assert "/bar.foo/foobar" in external_schemas + assert "/foo.bar/foobar" in external_schemas + + +def test__resolve_schema_references_with_url(mocker): + get = mocker.patch("httpx.get") + + from openapi_python_client.resolver.schema_resolver import SchemaResolver + + url = "http://foobar.io/foo/bar/foobar" + url_parent = "http://foobar.io/foo/bar/" + schema = {"foo": {"$ref": "foobar#/foobar"}} + external_schemas = {} + errors = [] + + def _datalaod_mocked_result(url, data): + if url == "http://foobar.io/foo/bar/foobar": + return {"foobar": "bar", "bar": {"$ref": "bar#/foobar"}, "local": {"$ref": "#/toto"}} + + if url == "http://foobar.io/foo/bar/bar": + return {"foobar": "bar", "bar": {"$ref": "../bar#/foobar"}} + + if url == "http://foobar.io/foo/bar": + return {"foobar": "bar/bar", "bar": {"$ref": "//barfoo.io/foobar#foobar"}} + + if url == "http://barfoo.io/foobar": + return {"foobar": "barfoo.io/foobar", "bar": {"$ref": "./bar#foobar"}} + + if url == "http://barfoo.io/bar": + return {"foobar": "barfoo.io/bar", "bar": {"$ref": "https://bar.foo/foobar"}} + + if url == "https://bar.foo/foobar": + return {"foobar": "bar.foo/foobar", "bar": {"$ref": "//foo.bar/foobar"}} + + if url == "https://foo.bar/foobar": + return {"foobar": "foo.bar/foobar", "bar": {"$ref": "http://foobar.io/foo/bar/foobar"}} # Loop to first uri + + raise ValueError(f"Unexpected url {url}") + + mocker.patch("openapi_python_client.resolver.schema_resolver.DataLoader.load", _datalaod_mocked_result) + + resolver = SchemaResolver(url) + resolver._resolve_schema_references(url_parent, schema, external_schemas, errors, True) + + assert len(errors) == 0 + assert "http://foobar.io/foo/bar/bar" in external_schemas + assert "http://foobar.io/foo/bar" in external_schemas + assert "http://barfoo.io/foobar" in external_schemas + assert "http://barfoo.io/foobar" in external_schemas + assert "http://barfoo.io/bar" in external_schemas + assert "https://bar.foo/foobar" in external_schemas + assert "https://foo.bar/foobar" in external_schemas + + +def test__resolve_schema_references_mix_path_and_url(mocker): + read_bytes = mocker.patch("pathlib.Path.read_bytes") + get = mocker.patch("httpx.get") + + from openapi_python_client.resolver.schema_resolver import SchemaResolver + + path = pathlib.Path("/foo/bar/foobar") + path_parent = str(path.parent) + schema = {"foo": {"$ref": "foobar#/foobar"}} + external_schemas = {} + errors = [] + + def _datalaod_mocked_result(path, data): + if path == "/foo/bar/foobar": + return {"foobar": "bar", "bar": {"$ref": "bar#/foobar"}, "local": {"$ref": "#/toto"}} + + if path == "/foo/bar/bar": + return {"foobar": "bar", "bar": {"$ref": "../bar#/foobar"}} + + if path == "/foo/bar": + return {"foobar": "bar/bar", "bar": {"$ref": "//barfoo.io/foobar#foobar"}} + + if path == "http://barfoo.io/foobar": + return {"foobar": "barfoo.io/foobar", "bar": {"$ref": "./bar#foobar"}} + + if path == "http://barfoo.io/bar": + return {"foobar": "barfoo.io/bar", "bar": {"$ref": "https://bar.foo/foobar"}} + + if path == "https://bar.foo/foobar": + return {"foobar": "bar.foo/foobar", "bar": {"$ref": "//foo.bar/foobar"}} + + if path == "https://foo.bar/foobar": + return {"foobar": "foo.bar/foobar"} + + raise ValueError(f"Unexpected path {path}") + + mocker.patch("openapi_python_client.resolver.schema_resolver.DataLoader.load", _datalaod_mocked_result) + resolver = SchemaResolver(path) + resolver._resolve_schema_references(path_parent, schema, external_schemas, errors, True) + + assert len(errors) == 0 + assert "/foo/bar/foobar" in external_schemas + assert "/foo/bar/bar" in external_schemas + assert "/foo/bar" in external_schemas + assert "http://barfoo.io/foobar" in external_schemas + assert "http://barfoo.io/bar" in external_schemas + assert "https://bar.foo/foobar" in external_schemas + assert "https://foo.bar/foobar" in external_schemas + + +def test__resolve_schema_references_with_error(mocker): + get = mocker.patch("httpx.get") + + import httpcore + + from openapi_python_client.resolver.schema_resolver import SchemaResolver + + url = "http://foobar.io/foo/bar/foobar" + url_parent = "http://foobar.io/foo/bar/" + schema = {"foo": {"$ref": "foobar#/foobar"}} + external_schemas = {} + errors = [] + + def _datalaod_mocked_result(url, data): + if url == "http://foobar.io/foo/bar/foobar": + return { + "foobar": "bar", + "bar": {"$ref": "bar#/foobar"}, + "barfoor": {"$ref": "barfoo#foobar"}, + "local": {"$ref": "#/toto"}, + } + + if url == "http://foobar.io/foo/bar/bar": + raise httpcore.NetworkError("mocked error") + + if url == "http://foobar.io/foo/bar/barfoo": + return {"foobar": "foo/bar/barfoo", "bar": {"$ref": "//barfoo.io/foobar#foobar"}} + + if url == "http://barfoo.io/foobar": + return {"foobar": "foobar"} + + mocker.patch("openapi_python_client.resolver.schema_resolver.DataLoader.load", _datalaod_mocked_result) + resolver = SchemaResolver(url) + resolver._resolve_schema_references(url_parent, schema, external_schemas, errors, True) + + assert len(errors) == 1 + assert errors[0] == "Failed to gather external reference data of bar#/foobar from http://foobar.io/foo/bar/bar" + assert "http://foobar.io/foo/bar/bar" not in external_schemas + assert "http://foobar.io/foo/bar/foobar" in external_schemas + assert "http://foobar.io/foo/bar/barfoo" in external_schemas + assert "http://barfoo.io/foobar" in external_schemas + + +def test___lookup_schema_references(): + from openapi_python_client.resolver.schema_resolver import SchemaResolver + + data_set = { + "foo": {"$ref": "#/ref_1"}, + "bar": {"foobar": {"$ref": "#/ref_2"}}, + "foobar": [{"foo": {"$ref": "#/ref_3"}}, {"bar": [{"foobar": {"$ref": "#/ref_4"}}]}], + } + + resolver = SchemaResolver("http://foobar.io") + expected_references = sorted([f"#/ref_{x}" for x in range(1, 5)]) + references = sorted([x.value for x in resolver._lookup_schema_references(data_set)]) + + for idx, ref in enumerate(references): + assert expected_references[idx] == ref From ca2dc416f397d1db478a484fd5235d1fb4232532 Mon Sep 17 00:00:00 2001 From: avy Date: Mon, 8 Mar 2021 16:32:27 +0100 Subject: [PATCH 27/39] fix absolute paths feature bugs --- openapi_python_client/resolver/reference.py | 19 +++- .../resolver/resolved_schema.py | 46 +++++---- .../resolver/schema_resolver.py | 2 +- .../test_resolver_resolved_schema.py | 96 +++++++++++++++++++ 4 files changed, 141 insertions(+), 22 deletions(-) create mode 100644 tests/test_resolver/test_resolver_resolved_schema.py diff --git a/openapi_python_client/resolver/reference.py b/openapi_python_client/resolver/reference.py index 534232bcd..dbd5bd007 100644 --- a/openapi_python_client/resolver/reference.py +++ b/openapi_python_client/resolver/reference.py @@ -1,4 +1,6 @@ import urllib.parse +from pathlib import Path +from typing import Union from .pointer import Pointer @@ -6,14 +8,29 @@ class Reference: """ https://tools.ietf.org/html/draft-pbryan-zyp-json-ref-03 """ - def __init__(self, reference: str): + def __init__(self, reference: str, parent: str = None): self._ref = reference self._parsed_ref = urllib.parse.urlparse(reference) + self._parent = parent @property def path(self) -> str: return urllib.parse.urldefrag(self._parsed_ref.geturl()).url + @property + def abs_path(self) -> str: + if self._parent: + parent_dir = Path(self._parent) + abs_path = parent_dir.joinpath(self.path) + abs_path = abs_path.resolve() + return str(abs_path) + else: + return self.path + + @property + def parent(self) -> Union[str, None]: + return self._parent + @property def pointer(self) -> Pointer: frag = self._parsed_ref.fragment diff --git a/openapi_python_client/resolver/resolved_schema.py b/openapi_python_client/resolver/resolved_schema.py index 7e528e694..6cd37415b 100644 --- a/openapi_python_client/resolver/resolved_schema.py +++ b/openapi_python_client/resolver/resolved_schema.py @@ -6,11 +6,12 @@ class ResolvedSchema: - def __init__(self, root: SchemaData, refs: Dict[str, SchemaData], errors: List[str]): + def __init__(self, root: SchemaData, refs: Dict[str, SchemaData], errors: List[str], parent: str): self._root: SchemaData = root self._refs: Dict[str, SchemaData] = refs self._errors: List[str] = errors self._resolved_remotes_components: SchemaData = cast(SchemaData, {}) + self._parent = parent self._resolved_schema: SchemaData = cast(SchemaData, {}) if len(self._errors) == 0: @@ -24,21 +25,29 @@ def schema(self) -> SchemaData: def errors(self) -> List[str]: return self._errors.copy() + def _dict_deep_update(self, d: Dict[str, Any], u: Dict[str, Any]) -> Dict[str, Any]: + for k, v in u.items(): + if isinstance(v, Dict): + d[k] = self._dict_deep_update(d.get(k, {}), v) + else: + d[k] = v + return d + def _process(self) -> None: self._process_remote_paths() - self._process_remote_components(self._root) - self._root.update(self._resolved_remotes_components) + self._process_remote_components(self._root, parent_path=self._parent) + self._dict_deep_update(self._root, self._resolved_remotes_components) def _process_remote_paths(self) -> None: refs_to_replace = [] for owner, ref_key, ref_val in self._lookup_schema_references_in(self._root, "paths"): - ref = Reference(ref_val) + ref = Reference(ref_val, self._parent) if ref.is_local(): continue - remote_path = ref.pointer.value - path = ref.path + remote_path = ref.abs_path + path = ref.pointer.unescapated_value if remote_path not in self._refs: self._errors.append("Failed to resolve remote reference > {0}".format(remote_path)) @@ -51,23 +60,23 @@ def _process_remote_paths(self) -> None: refs_to_replace.append((owner, remote_schema, remote_value)) for owner, remote_schema, remote_value in refs_to_replace: - self._process_remote_components(remote_schema, remote_value, 1) + self._process_remote_components(remote_schema, remote_value, 1, self._parent) self._replace_reference_with(owner, remote_value) def _process_remote_components( - self, owner: SchemaData, subpart: Union[SchemaData, None] = None, depth: int = 0 + self, owner: SchemaData, subpart: Union[SchemaData, None] = None, depth: int = 0, parent_path: str = None ) -> None: target = subpart if subpart else owner for parent, ref_key, ref_val in self._lookup_schema_references(target): - ref = Reference(ref_val) + ref = Reference(ref_val, parent_path) if ref.is_local(): # print('Found local reference >> {0}'.format(ref.value)) if depth > 0: self._transform_to_local_components(owner, ref) else: - remote_path = ref.pointer.value + remote_path = ref.abs_path if remote_path not in self._refs: self._errors.append("Failed to resolve remote reference > {0}".format(remote_path)) else: @@ -79,15 +88,12 @@ def _transform_to_local_components(self, owner: SchemaData, ref: Reference) -> N self._ensure_components_dir_exists(ref) # print('Processing remote component > {0}'.format(ref.value)) - remote_component = self._lookup_dict(owner, ref.path) + remote_component = self._lookup_dict(owner, ref.pointer.value) pointer_parent = ref.pointer.parent if pointer_parent is not None: root_components_dir = self._lookup_dict(self._resolved_remotes_components, pointer_parent.value) - component_name = ref.path.split("/")[-1] - - if component_name == "SorTransparentContainer" or component_name == "sorTransparentContainer": - print(ref.value) + component_name = ref.pointer.value.split("/")[-1] if remote_component is None: print("Weirdy relookup of >> {0}".format(ref.value)) @@ -95,12 +101,12 @@ def _transform_to_local_components(self, owner: SchemaData, ref: Reference) -> N return if "$ref" in remote_component: - subref = Reference(remote_component["$ref"]) + subref = Reference(remote_component["$ref"], ref.parent) if not subref.is_local(): print("Lookup remote ref >>> {0}".format(subref.value)) - return self._process_remote_components(remote_component) + self._process_remote_components(remote_component, parent_path=ref.parent) - if root_components_dir: + if root_components_dir is not None: if component_name in root_components_dir: local_component_hash = self._reference_schema_hash(root_components_dir[component_name]) remote_component_hash = self._reference_schema_hash(remote_component) @@ -117,7 +123,7 @@ def _transform_to_local_components(self, owner: SchemaData, ref: Reference) -> N # print('=' * 120) else: root_components_dir[component_name] = remote_component - self._process_remote_components(owner, remote_component, 2) + self._process_remote_components(owner, remote_component, 2, ref.parent) def _ensure_components_dir_exists(self, ref: Reference) -> None: cursor = self._resolved_remotes_components @@ -134,7 +140,7 @@ def _ensure_components_dir_exists(self, ref: Reference) -> None: cursor = cursor[key] def _transform_to_local_ref(self, owner: Dict[str, Any], ref: Reference) -> None: - owner["$ref"] = "#{0}".format(ref.path) + owner["$ref"] = "#{0}".format(ref.pointer.value) def _lookup_dict(self, attr: SchemaData, query: str) -> Union[SchemaData, None]: cursor = attr diff --git a/openapi_python_client/resolver/schema_resolver.py b/openapi_python_client/resolver/schema_resolver.py index 5a1c602e7..2d20952b1 100644 --- a/openapi_python_client/resolver/schema_resolver.py +++ b/openapi_python_client/resolver/schema_resolver.py @@ -53,7 +53,7 @@ def resolve(self, recursive: bool = True) -> ResolvedSchema: root_schema = self._fetch_url_reference(self._root_url) self._resolve_schema_references(self._parent_path, root_schema, external_schemas, errors, recursive) - return ResolvedSchema(root_schema, external_schemas, errors) + return ResolvedSchema(root_schema, external_schemas, errors, self._parent_path) def _resolve_schema_references( self, diff --git a/tests/test_resolver/test_resolver_resolved_schema.py b/tests/test_resolver/test_resolver_resolved_schema.py new file mode 100644 index 000000000..b31111d2d --- /dev/null +++ b/tests/test_resolver/test_resolver_resolved_schema.py @@ -0,0 +1,96 @@ +import pathlib +import urllib +import urllib.parse + +import pytest + + +def test__resolved_schema_with_resolved_external_references(): + + from openapi_python_client.resolver.resolved_schema import ResolvedSchema + + root_schema = {"foobar": {"$ref": "foobar.yaml#/foo"}} + external_schemas = {"/home/user/foobar.yaml": {"foo": {"description": "foobar_description"}}} + errors = [] + + resolved_schema = ResolvedSchema(root_schema, external_schemas, errors, "/home/user").schema + + assert len(errors) == 0 + assert "foo" in resolved_schema + assert "foobar" in resolved_schema + assert "$ref" in resolved_schema["foobar"] + assert "#/foo" in resolved_schema["foobar"]["$ref"] + assert "description" in resolved_schema["foo"] + assert "foobar_description" in resolved_schema["foo"]["description"] + + +def test__resolved_schema_with_absolute_paths(): + + from openapi_python_client.resolver.resolved_schema import ResolvedSchema + + root_schema = {"foobar": {"$ref": "foobar.yaml#/foo"}, "barfoo": {"$ref": "../barfoo.yaml#/bar"}} + + external_schemas = { + "/home/user/foobar.yaml": {"foo": {"description": "foobar_description"}}, + "/home/barfoo.yaml": {"bar": {"description": "barfoo_description"}}, + } + + errors = [] + + resolved_schema = ResolvedSchema(root_schema, external_schemas, errors, "/home/user").schema + + assert len(errors) == 0 + assert "foo" in resolved_schema + assert "bar" in resolved_schema + assert "foobar" in resolved_schema + assert "barfoo" in resolved_schema + assert "$ref" in resolved_schema["foobar"] + assert "#/foo" in resolved_schema["foobar"]["$ref"] + assert "$ref" in resolved_schema["barfoo"] + assert "#/bar" in resolved_schema["barfoo"]["$ref"] + assert "description" in resolved_schema["foo"] + assert "foobar_description" in resolved_schema["foo"]["description"] + assert "description" in resolved_schema["bar"] + assert "barfoo_description" in resolved_schema["bar"]["description"] + + +def test__resolved_schema_with_conflicts(): + + from openapi_python_client.resolver.resolved_schema import ResolvedSchema + + root_schema = { + "foobar": {"$ref": "first_instance.yaml#/foo"}, + "barfoo": {"$ref": "second_instance.yaml#/foo"}, + "foobarfoo": {"$ref": "second_instance.yaml#/foo"}, + "barfoobar": {"$ref": "first_instance.yaml#/foo"}, + } + + external_schemas = { + "/home/user/first_instance.yaml": {"foo": {"description": "foo_first_description"}}, + "/home/user/second_instance.yaml": {"foo": {"description": "foo_second_description"}}, + } + + current_result = { + "foobar": {"$ref": "#/foo"}, + "barfoo": {"$ref": "#/foo"}, + "foobarfoo": {"$ref": "#/foo"}, + "barfoobar": {"$ref": "#/foo"}, + "foo": {"description": "foo_first_description"}, + } + + desired_result = { + "foobar": {"$ref": "#/foo"}, + "barfoo": {"$ref": "#/foo2"}, + "foobarfoo": {"$ref": "#/foo2"}, + "barfoobar": {"$ref": "#/foo"}, + "foo": {"description": "foo_first_description"}, + "foo2": {"description": "foo_second_description"}, + } + + errors = [] + + resolved_schema = ResolvedSchema(root_schema, external_schemas, errors, "/home/user").schema + + print(resolved_schema) + assert len(errors) == 0 + assert resolved_schema == desired_result From 250fa6a762d039c9c3de9a42ab2632a5eb7881c0 Mon Sep 17 00:00:00 2001 From: avy Date: Fri, 12 Mar 2021 17:11:30 +0100 Subject: [PATCH 28/39] add collision_resolver --- .../resolver/collision_resolver.py | 156 ++++++++++++++++++ .../resolver/resolved_schema.py | 38 +---- .../resolver/schema_resolver.py | 2 + .../test_resolver_collision_resolver.py | 58 +++++++ .../test_resolver_resolved_schema.py | 42 ----- 5 files changed, 222 insertions(+), 74 deletions(-) create mode 100644 openapi_python_client/resolver/collision_resolver.py create mode 100644 tests/test_resolver/test_resolver_collision_resolver.py diff --git a/openapi_python_client/resolver/collision_resolver.py b/openapi_python_client/resolver/collision_resolver.py new file mode 100644 index 000000000..1fbf117c6 --- /dev/null +++ b/openapi_python_client/resolver/collision_resolver.py @@ -0,0 +1,156 @@ +import hashlib +from typing import Any, Dict, List, Tuple + +from .reference import Reference +from .resolver_types import SchemaData + + +class CollisionResolver: + def __init__(self, root: SchemaData, refs: Dict[str, SchemaData], errors: List[str], parent: str): + self._root: SchemaData = root + self._refs: Dict[str, SchemaData] = refs + self._errors: List[str] = errors + self._parent = parent + self._refs_index: Dict[str, str] = dict() + self._keys_to_replace: Dict[Reference, Tuple[int, SchemaData]] = dict() + self._debug = set() + + def _browse_schema(self, attr: Any, root_attr: Any) -> None: + if isinstance(attr, dict): + attr_copy = {**attr} # Create a shallow copy + for key, val in attr_copy.items(): + if key == "$ref": + ref = Reference(val, self._parent) + value = ref.pointer.value + + assert value + + schema = self._get_from_ref(ref, root_attr) + # if value == '/components/schemas/EeSubscription': + # print(schema) + # print() + hashed_schema = self._reference_schema_hash(schema) + + if value in self._refs_index.keys(): + if self._refs_index[value] != hashed_schema: + self._debug.add(ref.pointer.value) + if ref.is_local(): + self._increment_ref(ref, root_attr, hashed_schema, attr, key) + else: + assert ref.abs_path in self._refs.keys() + self._increment_ref(ref, self._refs[ref.abs_path], hashed_schema, attr, key) + else: + self._refs_index[value] = hashed_schema + else: + self._browse_schema(val, root_attr) + + elif isinstance(attr, list): + for val in attr: + self._browse_schema(val, root_attr) + + def _get_from_ref(self, ref: Reference, attr: SchemaData) -> SchemaData: + if ref.is_local(): + cursor = attr + else: + assert ref.abs_path in self._refs.keys() + cursor = self._refs[ref.abs_path] + query = ref.pointer.unescapated_value + query_parts = [] + + if query.startswith("/paths"): + query_parts = ["paths", query.replace("/paths//", "/").replace("/paths", "")] + else: + query_parts = query.split("/") + + for key in query_parts: + if key == "": + continue + + if isinstance(cursor, dict) and key in cursor: + cursor = cursor[key] + else: + print('ERROR') + + # if list(cursor) == ['$ref']: + # ref = Reference(cursor['$ref'],self._parent) + # if ref.is_remote(): + # print('remote_ref') + # print(ref.value) + # attr = self._refs[ref.abs_path] + # return self._get_from_ref(ref,attr) + + return cursor + + def _increment_ref( + self, ref: Reference, schema: SchemaData, hashed_schema: str, attr: Dict[str, Any], key: str + ) -> None: + i = 2 + value = ref.pointer.value + incremented_value = value + "_" + str(i) + + while incremented_value in self._refs_index.keys(): + if self._refs_index[incremented_value] == hashed_schema: + attr[key] = ref.value + "_" + str(i) + return + else: + i = i + 1 + incremented_value = value + "_" + str(i) + + attr[key] = ref.value + "_" + str(i) + self._refs_index[incremented_value] = hashed_schema + self._keys_to_replace[ref] = (i, schema) + + def _modify_root_ref_name(self, ref_pointer: str, i: int, attr: SchemaData) -> None: + cursor = attr + query = ref_pointer + query_parts = [] + + if query.startswith("/paths"): + query_parts = ["paths", query.replace("/paths//", "/").replace("/paths", "")] + else: + query_parts = query.split("/") + + last_key = query_parts[-1] + + for key in query_parts: + if key == "": + continue + + if key == last_key: + assert key in cursor + cursor[key + "_" + str(i)] = cursor.pop(key) + return + + if isinstance(cursor, dict) and key in cursor: + cursor = cursor[key] + else: + return + + def resolve(self) -> None: + debug = set() + self._browse_schema(self._root, self._root) + for file, schema in self._refs.items(): + self._browse_schema(schema, schema) + for a, b in self._keys_to_replace.items(): + debug.add(a.pointer.value) + self._modify_root_ref_name(a.pointer.value, b[0], b[1]) + + print(self._debug) + + def _reference_schema_hash(self, schema: Dict[str, Any]) -> str: + md5 = hashlib.md5() + hash_elms = [] + for key in schema.keys(): + if key == "description": + hash_elms.append(schema[key]) + if key == "type": + hash_elms.append(schema[key]) + if key == "allOf": + for item in schema[key]: + hash_elms.append(str(item)) + + hash_elms.append(key) + + hash_elms.sort() + md5.update(";".join(hash_elms).encode("utf-8")) + return md5.hexdigest() diff --git a/openapi_python_client/resolver/resolved_schema.py b/openapi_python_client/resolver/resolved_schema.py index 6cd37415b..9d0fa9066 100644 --- a/openapi_python_client/resolver/resolved_schema.py +++ b/openapi_python_client/resolver/resolved_schema.py @@ -1,4 +1,3 @@ -import hashlib from typing import Any, Dict, Generator, List, Tuple, Union, cast from .reference import Reference @@ -96,7 +95,7 @@ def _transform_to_local_components(self, owner: SchemaData, ref: Reference) -> N component_name = ref.pointer.value.split("/")[-1] if remote_component is None: - print("Weirdy relookup of >> {0}".format(ref.value)) + print("Weird relookup of >> {0}".format(ref.value)) assert ref.is_local() and self._lookup_dict(self._resolved_remotes_components, ref.path) return @@ -108,19 +107,14 @@ def _transform_to_local_components(self, owner: SchemaData, ref: Reference) -> N if root_components_dir is not None: if component_name in root_components_dir: - local_component_hash = self._reference_schema_hash(root_components_dir[component_name]) - remote_component_hash = self._reference_schema_hash(remote_component) - - if local_component_hash == remote_component_hash: + if remote_component == root_components_dir[component_name]: return else: + print("FOUND COLLISION IN RESOLVED SCHEMA, SHOULD NOT HAPPEN") + # print(component_name) + # print(remote_component) + # print(root_components_dir[component_name]) pass - # print('=' * 120) - # print('TODO: Find compoment collision to handle on >>> {0}'.format(ref.path)) - # print('Local componente {0} >> {1}'.format(local_component_hash, root_components_dir[component_name])) - # print('') - # print('Remote componente {0} >> {1}'.format(remote_component_hash, remote_component)) - # print('=' * 120) else: root_components_dir[component_name] = remote_component self._process_remote_components(owner, remote_component, 2, ref.parent) @@ -186,23 +180,3 @@ def _lookup_schema_references(self, attr: Any) -> Generator[Tuple[SchemaData, st elif isinstance(attr, list): for val in attr: yield from self._lookup_schema_references(val) - - def _reference_schema_hash(self, schema: Dict[str, Any]) -> str: - md5 = hashlib.md5() - hash_elms = [] - for key in schema.keys(): - if key == "description": - continue - - if key == "type": - hash_elms.append(schema[key]) - - if key == "allOf": - for item in schema[key]: - hash_elms.append(str(item)) - - hash_elms.append(key) - - hash_elms.sort() - md5.update(";".join(hash_elms).encode("utf-8")) - return md5.hexdigest() diff --git a/openapi_python_client/resolver/schema_resolver.py b/openapi_python_client/resolver/schema_resolver.py index 2d20952b1..3521842fb 100644 --- a/openapi_python_client/resolver/schema_resolver.py +++ b/openapi_python_client/resolver/schema_resolver.py @@ -5,6 +5,7 @@ import httpx +from .collision_resolver import CollisionResolver from .data_loader import DataLoader from .reference import Reference from .resolved_schema import ResolvedSchema @@ -53,6 +54,7 @@ def resolve(self, recursive: bool = True) -> ResolvedSchema: root_schema = self._fetch_url_reference(self._root_url) self._resolve_schema_references(self._parent_path, root_schema, external_schemas, errors, recursive) + CollisionResolver(root_schema, external_schemas, errors, self._parent_path).resolve() return ResolvedSchema(root_schema, external_schemas, errors, self._parent_path) def _resolve_schema_references( diff --git a/tests/test_resolver/test_resolver_collision_resolver.py b/tests/test_resolver/test_resolver_collision_resolver.py new file mode 100644 index 000000000..4f0ccf37d --- /dev/null +++ b/tests/test_resolver/test_resolver_collision_resolver.py @@ -0,0 +1,58 @@ +import pathlib +import urllib +import urllib.parse + +import pytest + + +def test__collision_resolver(): + + from openapi_python_client.resolver.collision_resolver import CollisionResolver + from openapi_python_client.resolver.resolved_schema import ResolvedSchema + + root_schema = { + "foobar": {"$ref": "first_instance.yaml#/foo"}, + "barfoo": {"$ref": "second_instance.yaml#/foo"}, + "barbarfoo": {"$ref": "third_instance.yaml#/foo"}, + "foobarfoo": {"$ref": "second_instance.yaml#/foo"}, + "barfoobar": {"$ref": "first_instance.yaml#/bar/foo"}, + "localref": {"$ref": "#/local_ref"}, + "local_ref": {"description": "a local ref"}, + "last": {"$ref": "first_instance.yaml#/fourth_instance"}, + } + + external_schemas = { + "/home/user/first_instance.yaml": { + "foo": {"description": "foo_first_description"}, + "bar": {"foo": {"description": "nested foo"}}, + "fourth_instance": {"$ref": "fourth_instance.yaml#/foo"}, + }, + "/home/user/second_instance.yaml": {"foo": {"description": "foo_second_description"}}, + "/home/user/third_instance.yaml": {"foo": {"description": "foo_third_description"}}, + "/home/user/fourth_instance.yaml": {"foo": {"description": "foo_fourth_description"}}, + } + + desired_result = { + "foobar": {"$ref": "#/foo"}, + "barfoo": {"$ref": "#/foo_2"}, + "barbarfoo": {"$ref": "#/foo_3"}, + "foobarfoo": {"$ref": "#/foo_2"}, + "barfoobar": {"$ref": "#/bar/foo"}, + "localref": {"$ref": "#/local_ref"}, + "local_ref": {"description": "a local ref"}, + "last": {"$ref": "#/fourth_instance"}, + "foo": {"description": "foo_first_description"}, + "foo_2": {"description": "foo_second_description"}, + "foo_3": {"description": "foo_third_description"}, + "bar": {"foo": {"description": "nested foo"}}, + "foo_4": {"description": "foo_fourth_description"}, + "fourth_instance": {"$ref": "#/foo_4"}, + } + errors = [] + + CollisionResolver(root_schema, external_schemas, errors, "/home/user").resolve() + resolved_schema = ResolvedSchema(root_schema, external_schemas, errors, "/home/user").schema + + print(resolved_schema) + assert len(errors) == 0 + assert resolved_schema == desired_result diff --git a/tests/test_resolver/test_resolver_resolved_schema.py b/tests/test_resolver/test_resolver_resolved_schema.py index b31111d2d..5728a24eb 100644 --- a/tests/test_resolver/test_resolver_resolved_schema.py +++ b/tests/test_resolver/test_resolver_resolved_schema.py @@ -52,45 +52,3 @@ def test__resolved_schema_with_absolute_paths(): assert "foobar_description" in resolved_schema["foo"]["description"] assert "description" in resolved_schema["bar"] assert "barfoo_description" in resolved_schema["bar"]["description"] - - -def test__resolved_schema_with_conflicts(): - - from openapi_python_client.resolver.resolved_schema import ResolvedSchema - - root_schema = { - "foobar": {"$ref": "first_instance.yaml#/foo"}, - "barfoo": {"$ref": "second_instance.yaml#/foo"}, - "foobarfoo": {"$ref": "second_instance.yaml#/foo"}, - "barfoobar": {"$ref": "first_instance.yaml#/foo"}, - } - - external_schemas = { - "/home/user/first_instance.yaml": {"foo": {"description": "foo_first_description"}}, - "/home/user/second_instance.yaml": {"foo": {"description": "foo_second_description"}}, - } - - current_result = { - "foobar": {"$ref": "#/foo"}, - "barfoo": {"$ref": "#/foo"}, - "foobarfoo": {"$ref": "#/foo"}, - "barfoobar": {"$ref": "#/foo"}, - "foo": {"description": "foo_first_description"}, - } - - desired_result = { - "foobar": {"$ref": "#/foo"}, - "barfoo": {"$ref": "#/foo2"}, - "foobarfoo": {"$ref": "#/foo2"}, - "barfoobar": {"$ref": "#/foo"}, - "foo": {"description": "foo_first_description"}, - "foo2": {"description": "foo_second_description"}, - } - - errors = [] - - resolved_schema = ResolvedSchema(root_schema, external_schemas, errors, "/home/user").schema - - print(resolved_schema) - assert len(errors) == 0 - assert resolved_schema == desired_result From bb61e8afc15416d8f5eecedfc5f061fad98bd358 Mon Sep 17 00:00:00 2001 From: avy Date: Mon, 15 Mar 2021 13:56:24 +0100 Subject: [PATCH 29/39] =?UTF-8?q?fix=20collision=20issues=20with=20same=20?= =?UTF-8?q?schema=20in=20two=20different=20files=C3=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resolver/collision_resolver.py | 43 ++++++++----------- .../test_resolver_collision_resolver.py | 3 ++ 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/openapi_python_client/resolver/collision_resolver.py b/openapi_python_client/resolver/collision_resolver.py index 1fbf117c6..ca4bfdba9 100644 --- a/openapi_python_client/resolver/collision_resolver.py +++ b/openapi_python_client/resolver/collision_resolver.py @@ -12,8 +12,7 @@ def __init__(self, root: SchemaData, refs: Dict[str, SchemaData], errors: List[s self._errors: List[str] = errors self._parent = parent self._refs_index: Dict[str, str] = dict() - self._keys_to_replace: Dict[Reference, Tuple[int, SchemaData]] = dict() - self._debug = set() + self._keys_to_replace: Dict[str, Tuple[int, SchemaData, str]] = dict() def _browse_schema(self, attr: Any, root_attr: Any) -> None: if isinstance(attr, dict): @@ -26,14 +25,10 @@ def _browse_schema(self, attr: Any, root_attr: Any) -> None: assert value schema = self._get_from_ref(ref, root_attr) - # if value == '/components/schemas/EeSubscription': - # print(schema) - # print() hashed_schema = self._reference_schema_hash(schema) if value in self._refs_index.keys(): if self._refs_index[value] != hashed_schema: - self._debug.add(ref.pointer.value) if ref.is_local(): self._increment_ref(ref, root_attr, hashed_schema, attr, key) else: @@ -49,11 +44,10 @@ def _browse_schema(self, attr: Any, root_attr: Any) -> None: self._browse_schema(val, root_attr) def _get_from_ref(self, ref: Reference, attr: SchemaData) -> SchemaData: - if ref.is_local(): - cursor = attr - else: + if ref.is_remote(): assert ref.abs_path in self._refs.keys() - cursor = self._refs[ref.abs_path] + attr = self._refs[ref.abs_path] + cursor = attr query = ref.pointer.unescapated_value query_parts = [] @@ -69,15 +63,13 @@ def _get_from_ref(self, ref: Reference, attr: SchemaData) -> SchemaData: if isinstance(cursor, dict) and key in cursor: cursor = cursor[key] else: - print('ERROR') + print("ERROR") - # if list(cursor) == ['$ref']: - # ref = Reference(cursor['$ref'],self._parent) - # if ref.is_remote(): - # print('remote_ref') - # print(ref.value) - # attr = self._refs[ref.abs_path] - # return self._get_from_ref(ref,attr) + if list(cursor) == ["$ref"]: + ref2 = Reference(cursor["$ref"], self._parent) + if ref2.is_remote(): + attr = self._refs[ref2.abs_path] + return self._get_from_ref(ref2, attr) return cursor @@ -90,15 +82,18 @@ def _increment_ref( while incremented_value in self._refs_index.keys(): if self._refs_index[incremented_value] == hashed_schema: - attr[key] = ref.value + "_" + str(i) - return + if ref.value not in self._keys_to_replace.keys(): + break # have to increment target key aswell + else: + attr[key] = ref.value + "_" + str(i) + return else: i = i + 1 incremented_value = value + "_" + str(i) attr[key] = ref.value + "_" + str(i) self._refs_index[incremented_value] = hashed_schema - self._keys_to_replace[ref] = (i, schema) + self._keys_to_replace[ref.value] = (i, schema, ref.pointer.value) def _modify_root_ref_name(self, ref_pointer: str, i: int, attr: SchemaData) -> None: cursor = attr @@ -127,15 +122,11 @@ def _modify_root_ref_name(self, ref_pointer: str, i: int, attr: SchemaData) -> N return def resolve(self) -> None: - debug = set() self._browse_schema(self._root, self._root) for file, schema in self._refs.items(): self._browse_schema(schema, schema) for a, b in self._keys_to_replace.items(): - debug.add(a.pointer.value) - self._modify_root_ref_name(a.pointer.value, b[0], b[1]) - - print(self._debug) + self._modify_root_ref_name(b[2], b[0], b[1]) def _reference_schema_hash(self, schema: Dict[str, Any]) -> str: md5 = hashlib.md5() diff --git a/tests/test_resolver/test_resolver_collision_resolver.py b/tests/test_resolver/test_resolver_collision_resolver.py index 4f0ccf37d..b19541141 100644 --- a/tests/test_resolver/test_resolver_collision_resolver.py +++ b/tests/test_resolver/test_resolver_collision_resolver.py @@ -19,6 +19,7 @@ def test__collision_resolver(): "localref": {"$ref": "#/local_ref"}, "local_ref": {"description": "a local ref"}, "last": {"$ref": "first_instance.yaml#/fourth_instance"}, + "baz": {"$ref": "fifth_instance.yaml#/foo"}, } external_schemas = { @@ -30,11 +31,13 @@ def test__collision_resolver(): "/home/user/second_instance.yaml": {"foo": {"description": "foo_second_description"}}, "/home/user/third_instance.yaml": {"foo": {"description": "foo_third_description"}}, "/home/user/fourth_instance.yaml": {"foo": {"description": "foo_fourth_description"}}, + "/home/user/fifth_instance.yaml": {"foo": {"description": "foo_second_description"}}, } desired_result = { "foobar": {"$ref": "#/foo"}, "barfoo": {"$ref": "#/foo_2"}, + "baz": {"$ref": "#/foo_2"}, "barbarfoo": {"$ref": "#/foo_3"}, "foobarfoo": {"$ref": "#/foo_2"}, "barfoobar": {"$ref": "#/bar/foo"}, From 5fafa0ae271f0ffefdbd88d56b366a0c08104e70 Mon Sep 17 00:00:00 2001 From: avy Date: Mon, 15 Mar 2021 14:33:33 +0100 Subject: [PATCH 30/39] do not crash on retry of key modification --- openapi_python_client/resolver/collision_resolver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openapi_python_client/resolver/collision_resolver.py b/openapi_python_client/resolver/collision_resolver.py index ca4bfdba9..aafbfca21 100644 --- a/openapi_python_client/resolver/collision_resolver.py +++ b/openapi_python_client/resolver/collision_resolver.py @@ -111,8 +111,8 @@ def _modify_root_ref_name(self, ref_pointer: str, i: int, attr: SchemaData) -> N if key == "": continue - if key == last_key: - assert key in cursor + if key == last_key and key + "_" + str(i) not in cursor: + assert key in cursor, "Didnt find %s in %s" % (ref_pointer, attr) cursor[key + "_" + str(i)] = cursor.pop(key) return From 8383bb39aa1dfcab0032ed3f1983031a6e664f46 Mon Sep 17 00:00:00 2001 From: avy Date: Mon, 15 Mar 2021 16:09:59 +0100 Subject: [PATCH 31/39] find collision of 2 same object at different place --- openapi_python_client/resolver/collision_resolver.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/openapi_python_client/resolver/collision_resolver.py b/openapi_python_client/resolver/collision_resolver.py index aafbfca21..a6168eff5 100644 --- a/openapi_python_client/resolver/collision_resolver.py +++ b/openapi_python_client/resolver/collision_resolver.py @@ -12,6 +12,7 @@ def __init__(self, root: SchemaData, refs: Dict[str, SchemaData], errors: List[s self._errors: List[str] = errors self._parent = parent self._refs_index: Dict[str, str] = dict() + self._schema_index: Dict[str, Reference] = dict() self._keys_to_replace: Dict[str, Tuple[int, SchemaData, str]] = dict() def _browse_schema(self, attr: Any, root_attr: Any) -> None: @@ -36,6 +37,17 @@ def _browse_schema(self, attr: Any, root_attr: Any) -> None: self._increment_ref(ref, self._refs[ref.abs_path], hashed_schema, attr, key) else: self._refs_index[value] = hashed_schema + + if hashed_schema in self._schema_index.keys(): + existing_ref = self._schema_index[hashed_schema] + if ( + existing_ref.pointer.value != ref.pointer.value + and ref.pointer.tokens()[-1] == existing_ref.pointer.tokens()[-1] + ): + print("Found same schema for different pointer") + else: + self._schema_index[hashed_schema] = ref + else: self._browse_schema(val, root_attr) From 692bd1aa02173ee2305cfcd4f9b3ae1745f06e69 Mon Sep 17 00:00:00 2001 From: avy Date: Mon, 15 Mar 2021 16:36:03 +0100 Subject: [PATCH 32/39] use tokens instead of splitting paths --- .../resolver/collision_resolver.py | 24 ++++--------------- .../resolver/resolved_schema.py | 21 ++++++++-------- 2 files changed, 15 insertions(+), 30 deletions(-) diff --git a/openapi_python_client/resolver/collision_resolver.py b/openapi_python_client/resolver/collision_resolver.py index a6168eff5..361685c33 100644 --- a/openapi_python_client/resolver/collision_resolver.py +++ b/openapi_python_client/resolver/collision_resolver.py @@ -13,7 +13,7 @@ def __init__(self, root: SchemaData, refs: Dict[str, SchemaData], errors: List[s self._parent = parent self._refs_index: Dict[str, str] = dict() self._schema_index: Dict[str, Reference] = dict() - self._keys_to_replace: Dict[str, Tuple[int, SchemaData, str]] = dict() + self._keys_to_replace: Dict[str, Tuple[int, SchemaData, List[str]]] = dict() def _browse_schema(self, attr: Any, root_attr: Any) -> None: if isinstance(attr, dict): @@ -60,13 +60,7 @@ def _get_from_ref(self, ref: Reference, attr: SchemaData) -> SchemaData: assert ref.abs_path in self._refs.keys() attr = self._refs[ref.abs_path] cursor = attr - query = ref.pointer.unescapated_value - query_parts = [] - - if query.startswith("/paths"): - query_parts = ["paths", query.replace("/paths//", "/").replace("/paths", "")] - else: - query_parts = query.split("/") + query_parts = ref.pointer.tokens() for key in query_parts: if key == "": @@ -105,18 +99,10 @@ def _increment_ref( attr[key] = ref.value + "_" + str(i) self._refs_index[incremented_value] = hashed_schema - self._keys_to_replace[ref.value] = (i, schema, ref.pointer.value) + self._keys_to_replace[ref.value] = (i, schema, ref.pointer.tokens()) - def _modify_root_ref_name(self, ref_pointer: str, i: int, attr: SchemaData) -> None: + def _modify_root_ref_name(self, query_parts: List[str], i: int, attr: SchemaData) -> None: cursor = attr - query = ref_pointer - query_parts = [] - - if query.startswith("/paths"): - query_parts = ["paths", query.replace("/paths//", "/").replace("/paths", "")] - else: - query_parts = query.split("/") - last_key = query_parts[-1] for key in query_parts: @@ -124,7 +110,7 @@ def _modify_root_ref_name(self, ref_pointer: str, i: int, attr: SchemaData) -> N continue if key == last_key and key + "_" + str(i) not in cursor: - assert key in cursor, "Didnt find %s in %s" % (ref_pointer, attr) + assert key in cursor, "Didnt find %s in %s" % (key, attr) cursor[key + "_" + str(i)] = cursor.pop(key) return diff --git a/openapi_python_client/resolver/resolved_schema.py b/openapi_python_client/resolver/resolved_schema.py index 9d0fa9066..3c473f234 100644 --- a/openapi_python_client/resolver/resolved_schema.py +++ b/openapi_python_client/resolver/resolved_schema.py @@ -47,12 +47,13 @@ def _process_remote_paths(self) -> None: remote_path = ref.abs_path path = ref.pointer.unescapated_value + tokens = ref.pointer.tokens() if remote_path not in self._refs: self._errors.append("Failed to resolve remote reference > {0}".format(remote_path)) else: remote_schema = self._refs[remote_path] - remote_value = self._lookup_dict(remote_schema, path) + remote_value = self._lookup_dict(remote_schema, tokens) if not remote_value: self._errors.append("Failed to read remote value {}, in remote ref {}".format(path, remote_path)) else: @@ -87,16 +88,16 @@ def _transform_to_local_components(self, owner: SchemaData, ref: Reference) -> N self._ensure_components_dir_exists(ref) # print('Processing remote component > {0}'.format(ref.value)) - remote_component = self._lookup_dict(owner, ref.pointer.value) + remote_component = self._lookup_dict(owner, ref.pointer.tokens()) pointer_parent = ref.pointer.parent if pointer_parent is not None: - root_components_dir = self._lookup_dict(self._resolved_remotes_components, pointer_parent.value) + root_components_dir = self._lookup_dict(self._resolved_remotes_components, pointer_parent.tokens()) component_name = ref.pointer.value.split("/")[-1] if remote_component is None: print("Weird relookup of >> {0}".format(ref.value)) - assert ref.is_local() and self._lookup_dict(self._resolved_remotes_components, ref.path) + assert ref.is_local() and self._lookup_dict(self._resolved_remotes_components, ref.pointer.tokens()) return if "$ref" in remote_component: @@ -109,10 +110,14 @@ def _transform_to_local_components(self, owner: SchemaData, ref: Reference) -> N if component_name in root_components_dir: if remote_component == root_components_dir[component_name]: return + elif list(remote_component) == ["$ref"]: + pass else: print("FOUND COLLISION IN RESOLVED SCHEMA, SHOULD NOT HAPPEN") # print(component_name) + # print() # print(remote_component) + # print() # print(root_components_dir[component_name]) pass else: @@ -136,14 +141,8 @@ def _ensure_components_dir_exists(self, ref: Reference) -> None: def _transform_to_local_ref(self, owner: Dict[str, Any], ref: Reference) -> None: owner["$ref"] = "#{0}".format(ref.pointer.value) - def _lookup_dict(self, attr: SchemaData, query: str) -> Union[SchemaData, None]: + def _lookup_dict(self, attr: SchemaData, query_parts: List[str]) -> Union[SchemaData, None]: cursor = attr - query_parts = [] - - if query.startswith("/paths"): - query_parts = ["paths", query.replace("/paths//", "/").replace("/paths", "")] - else: - query_parts = query.split("/") for key in query_parts: if key == "": From aa04afecacf750027b234bae67b08a3c160151d7 Mon Sep 17 00:00:00 2001 From: avy Date: Tue, 16 Mar 2021 13:29:12 +0100 Subject: [PATCH 33/39] remove ref key when replacing it --- .../resolver/resolved_schema.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/openapi_python_client/resolver/resolved_schema.py b/openapi_python_client/resolver/resolved_schema.py index 3c473f234..cba452c8b 100644 --- a/openapi_python_client/resolver/resolved_schema.py +++ b/openapi_python_client/resolver/resolved_schema.py @@ -26,6 +26,8 @@ def errors(self) -> List[str]: def _dict_deep_update(self, d: Dict[str, Any], u: Dict[str, Any]) -> Dict[str, Any]: for k, v in u.items(): + if isinstance(d, Dict) and list(d) == ["$ref"]: + d.pop("$ref") if isinstance(v, Dict): d[k] = self._dict_deep_update(d.get(k, {}), v) else: @@ -107,20 +109,7 @@ def _transform_to_local_components(self, owner: SchemaData, ref: Reference) -> N self._process_remote_components(remote_component, parent_path=ref.parent) if root_components_dir is not None: - if component_name in root_components_dir: - if remote_component == root_components_dir[component_name]: - return - elif list(remote_component) == ["$ref"]: - pass - else: - print("FOUND COLLISION IN RESOLVED SCHEMA, SHOULD NOT HAPPEN") - # print(component_name) - # print() - # print(remote_component) - # print() - # print(root_components_dir[component_name]) - pass - else: + if component_name not in root_components_dir: root_components_dir[component_name] = remote_component self._process_remote_components(owner, remote_component, 2, ref.parent) From 6b584b65e5660762afb14c6163cc7b49d237ca80 Mon Sep 17 00:00:00 2001 From: avy Date: Thu, 18 Mar 2021 10:18:00 +0100 Subject: [PATCH 34/39] fix build_schema bug when model points to loaded ref --- openapi_python_client/parser/properties/__init__.py | 7 ++----- openapi_python_client/parser/responses.py | 1 + 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/openapi_python_client/parser/properties/__init__.py b/openapi_python_client/parser/properties/__init__.py index 8ecad0e60..18304058d 100644 --- a/openapi_python_client/parser/properties/__init__.py +++ b/openapi_python_client/parser/properties/__init__.py @@ -811,11 +811,8 @@ def build_schemas(*, components: Dict[str, Union[oai.Reference, oai.Schema]]) -> visited.append(name) if isinstance(data, oai.Reference): - class_name = _reference_model_name(data) - - if not schemas.models.get(class_name) and not schemas.enums.get(class_name): - references_by_name[name] = data - references_to_process.append((name, data)) + references_by_name[name] = data + references_to_process.append((name, data)) continue schemas_or_err = update_schemas_with_data(name, data, schemas, lazy_self_references) diff --git a/openapi_python_client/parser/responses.py b/openapi_python_client/parser/responses.py index 3d01a0eab..00810b12f 100644 --- a/openapi_python_client/parser/responses.py +++ b/openapi_python_client/parser/responses.py @@ -20,6 +20,7 @@ class Response: _SOURCE_BY_CONTENT_TYPE = { "application/json": "response.json()", + "application/problem+json": "response.json()", "application/vnd.api+json": "response.json()", "application/octet-stream": "response.content", "text/html": "response.text", From 0ce86cd08b454592eb6c403874bb9a54d568fe75 Mon Sep 17 00:00:00 2001 From: avy Date: Thu, 18 Mar 2021 10:21:55 +0100 Subject: [PATCH 35/39] add json loading to data_loader --- openapi_python_client/resolver/data_loader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openapi_python_client/resolver/data_loader.py b/openapi_python_client/resolver/data_loader.py index aaa0fd26b..c29b5d6f1 100644 --- a/openapi_python_client/resolver/data_loader.py +++ b/openapi_python_client/resolver/data_loader.py @@ -1,4 +1,5 @@ import yaml +import json from .resolver_types import SchemaData @@ -15,7 +16,7 @@ def load(cls, path: str, data: bytes) -> SchemaData: @classmethod def load_json(cls, data: bytes) -> SchemaData: - raise NotImplementedError() + return json.loads(data) @classmethod def load_yaml(cls, data: bytes) -> SchemaData: From 2b3c125db0127b1d38e03c45682fd5fbb9aa6900 Mon Sep 17 00:00:00 2001 From: avy Date: Thu, 18 Mar 2021 11:21:07 +0100 Subject: [PATCH 36/39] dont use resolved schema in collision resolver --- .../test_resolver_collision_resolver.py | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/tests/test_resolver/test_resolver_collision_resolver.py b/tests/test_resolver/test_resolver_collision_resolver.py index b19541141..ae48a4d8c 100644 --- a/tests/test_resolver/test_resolver_collision_resolver.py +++ b/tests/test_resolver/test_resolver_collision_resolver.py @@ -8,7 +8,6 @@ def test__collision_resolver(): from openapi_python_client.resolver.collision_resolver import CollisionResolver - from openapi_python_client.resolver.resolved_schema import ResolvedSchema root_schema = { "foobar": {"$ref": "first_instance.yaml#/foo"}, @@ -34,28 +33,34 @@ def test__collision_resolver(): "/home/user/fifth_instance.yaml": {"foo": {"description": "foo_second_description"}}, } - desired_result = { - "foobar": {"$ref": "#/foo"}, - "barfoo": {"$ref": "#/foo_2"}, - "baz": {"$ref": "#/foo_2"}, - "barbarfoo": {"$ref": "#/foo_3"}, - "foobarfoo": {"$ref": "#/foo_2"}, - "barfoobar": {"$ref": "#/bar/foo"}, + root_schema_result = { + "foobar": {"$ref": "first_instance.yaml#/foo"}, + "barfoo": {"$ref": "second_instance.yaml#/foo_2"}, + "barbarfoo": {"$ref": "third_instance.yaml#/foo_3"}, + "foobarfoo": {"$ref": "second_instance.yaml#/foo_2"}, + "barfoobar": {"$ref": "first_instance.yaml#/bar/foo"}, "localref": {"$ref": "#/local_ref"}, "local_ref": {"description": "a local ref"}, - "last": {"$ref": "#/fourth_instance"}, - "foo": {"description": "foo_first_description"}, - "foo_2": {"description": "foo_second_description"}, - "foo_3": {"description": "foo_third_description"}, - "bar": {"foo": {"description": "nested foo"}}, - "foo_4": {"description": "foo_fourth_description"}, - "fourth_instance": {"$ref": "#/foo_4"}, + "last": {"$ref": "first_instance.yaml#/fourth_instance"}, + "baz": {"$ref": "fifth_instance.yaml#/foo_2"}, + } + + external_schemas_result = { + "/home/user/first_instance.yaml": { + "foo": {"description": "foo_first_description"}, + "bar": {"foo": {"description": "nested foo"}}, + "fourth_instance": {"$ref": "fourth_instance.yaml#/foo_4"}, + }, + "/home/user/second_instance.yaml": {"foo_2": {"description": "foo_second_description"}}, + "/home/user/third_instance.yaml": {"foo_3": {"description": "foo_third_description"}}, + "/home/user/fourth_instance.yaml": {"foo_4": {"description": "foo_fourth_description"}}, + "/home/user/fifth_instance.yaml": {"foo_2": {"description": "foo_second_description"}}, } + errors = [] CollisionResolver(root_schema, external_schemas, errors, "/home/user").resolve() - resolved_schema = ResolvedSchema(root_schema, external_schemas, errors, "/home/user").schema - print(resolved_schema) assert len(errors) == 0 - assert resolved_schema == desired_result + assert root_schema == root_schema_result + assert external_schemas == external_schemas_result From aae98e747c0a62eea958c82f94e85baf78b7e719 Mon Sep 17 00:00:00 2001 From: avy Date: Fri, 19 Mar 2021 10:44:39 +0100 Subject: [PATCH 37/39] improve collision resolver and resolved schema tests --- .../resolver/collision_resolver.py | 4 +- .../resolver/resolved_schema.py | 5 + .../test_resolver_collision_resolver.py | 101 +++++++++++++++++- .../test_resolver_resolved_schema.py | 97 ++++++++++++++++- 4 files changed, 202 insertions(+), 5 deletions(-) diff --git a/openapi_python_client/resolver/collision_resolver.py b/openapi_python_client/resolver/collision_resolver.py index 361685c33..85e252c66 100644 --- a/openapi_python_client/resolver/collision_resolver.py +++ b/openapi_python_client/resolver/collision_resolver.py @@ -44,7 +44,7 @@ def _browse_schema(self, attr: Any, root_attr: Any) -> None: existing_ref.pointer.value != ref.pointer.value and ref.pointer.tokens()[-1] == existing_ref.pointer.tokens()[-1] ): - print("Found same schema for different pointer") + self._errors.append(f"Found a duplicate schema in {existing_ref.value} and {ref.value}") else: self._schema_index[hashed_schema] = ref @@ -69,7 +69,7 @@ def _get_from_ref(self, ref: Reference, attr: SchemaData) -> SchemaData: if isinstance(cursor, dict) and key in cursor: cursor = cursor[key] else: - print("ERROR") + self._errors.append(f"Did not find data corresponding to the reference {ref.value}") if list(cursor) == ["$ref"]: ref2 = Reference(cursor["$ref"], self._parent) diff --git a/openapi_python_client/resolver/resolved_schema.py b/openapi_python_client/resolver/resolved_schema.py index cba452c8b..2612e0362 100644 --- a/openapi_python_client/resolver/resolved_schema.py +++ b/openapi_python_client/resolver/resolved_schema.py @@ -41,6 +41,7 @@ def _process(self) -> None: def _process_remote_paths(self) -> None: refs_to_replace = [] + refs_to_remove = [] for owner, ref_key, ref_val in self._lookup_schema_references_in(self._root, "paths"): ref = Reference(ref_val, self._parent) @@ -58,6 +59,7 @@ def _process_remote_paths(self) -> None: remote_value = self._lookup_dict(remote_schema, tokens) if not remote_value: self._errors.append("Failed to read remote value {}, in remote ref {}".format(path, remote_path)) + refs_to_remove.append((owner, ref_key)) else: refs_to_replace.append((owner, remote_schema, remote_value)) @@ -65,6 +67,9 @@ def _process_remote_paths(self) -> None: self._process_remote_components(remote_schema, remote_value, 1, self._parent) self._replace_reference_with(owner, remote_value) + for owner, ref_key in refs_to_remove: + owner.pop(ref_key) + def _process_remote_components( self, owner: SchemaData, subpart: Union[SchemaData, None] = None, depth: int = 0, parent_path: str = None ) -> None: diff --git a/tests/test_resolver/test_resolver_collision_resolver.py b/tests/test_resolver/test_resolver_collision_resolver.py index ae48a4d8c..0d9191a1c 100644 --- a/tests/test_resolver/test_resolver_collision_resolver.py +++ b/tests/test_resolver/test_resolver_collision_resolver.py @@ -5,6 +5,44 @@ import pytest +def test__collision_resolver_get_schema_from_ref(): + + from openapi_python_client.resolver.collision_resolver import CollisionResolver + + root_schema = {"foo": {"$ref": "first_instance.yaml#/foo"}} + + external_schemas = {"/home/user/first_instance.yaml": {"food": {"description": "food_first_description"}}} + + errors = [] + + CollisionResolver(root_schema, external_schemas, errors, "/home/user").resolve() + + assert len(errors) == 1 + assert errors == ["Did not find data corresponding to the reference first_instance.yaml#/foo"] + + +def test__collision_resolver_duplicate_schema(): + + from openapi_python_client.resolver.collision_resolver import CollisionResolver + + root_schema = { + "foo": {"$ref": "first_instance.yaml#/foo"}, + "bar": {"$ref": "second_instance.yaml#/bar/foo"}, + } + + external_schemas = { + "/home/user/first_instance.yaml": {"foo": {"description": "foo_first_description"}}, + "/home/user/second_instance.yaml": {"bar": {"foo": {"description": "foo_first_description"}}}, + } + + errors = [] + + CollisionResolver(root_schema, external_schemas, errors, "/home/user").resolve() + + assert len(errors) == 1 + assert errors == ["Found a duplicate schema in first_instance.yaml#/foo and second_instance.yaml#/bar/foo"] + + def test__collision_resolver(): from openapi_python_client.resolver.collision_resolver import CollisionResolver @@ -17,6 +55,7 @@ def test__collision_resolver(): "barfoobar": {"$ref": "first_instance.yaml#/bar/foo"}, "localref": {"$ref": "#/local_ref"}, "local_ref": {"description": "a local ref"}, + "array": ["array_item_one", "array_item_two"], "last": {"$ref": "first_instance.yaml#/fourth_instance"}, "baz": {"$ref": "fifth_instance.yaml#/foo"}, } @@ -27,7 +66,10 @@ def test__collision_resolver(): "bar": {"foo": {"description": "nested foo"}}, "fourth_instance": {"$ref": "fourth_instance.yaml#/foo"}, }, - "/home/user/second_instance.yaml": {"foo": {"description": "foo_second_description"}}, + "/home/user/second_instance.yaml": { + "foo": {"description": "foo_second_description"}, + "another_local_ref": {"$ref": "#/foo"}, + }, "/home/user/third_instance.yaml": {"foo": {"description": "foo_third_description"}}, "/home/user/fourth_instance.yaml": {"foo": {"description": "foo_fourth_description"}}, "/home/user/fifth_instance.yaml": {"foo": {"description": "foo_second_description"}}, @@ -41,6 +83,7 @@ def test__collision_resolver(): "barfoobar": {"$ref": "first_instance.yaml#/bar/foo"}, "localref": {"$ref": "#/local_ref"}, "local_ref": {"description": "a local ref"}, + "array": ["array_item_one", "array_item_two"], "last": {"$ref": "first_instance.yaml#/fourth_instance"}, "baz": {"$ref": "fifth_instance.yaml#/foo_2"}, } @@ -51,7 +94,10 @@ def test__collision_resolver(): "bar": {"foo": {"description": "nested foo"}}, "fourth_instance": {"$ref": "fourth_instance.yaml#/foo_4"}, }, - "/home/user/second_instance.yaml": {"foo_2": {"description": "foo_second_description"}}, + "/home/user/second_instance.yaml": { + "foo_2": {"description": "foo_second_description"}, + "another_local_ref": {"$ref": "#/foo_2"}, + }, "/home/user/third_instance.yaml": {"foo_3": {"description": "foo_third_description"}}, "/home/user/fourth_instance.yaml": {"foo_4": {"description": "foo_fourth_description"}}, "/home/user/fifth_instance.yaml": {"foo_2": {"description": "foo_second_description"}}, @@ -64,3 +110,54 @@ def test__collision_resolver(): assert len(errors) == 0 assert root_schema == root_schema_result assert external_schemas == external_schemas_result + + +def test__collision_resolver_deep_root_keys(): + + from openapi_python_client.resolver.collision_resolver import CollisionResolver + + root_schema = { + "foobar": {"$ref": "first_instance.yaml#/bar/foo"}, + "barfoo": {"$ref": "second_instance.yaml#/bar/foo"}, + "barfoobar": {"$ref": "second_instance.yaml#/barfoobar"}, + } + + external_schemas = { + "/home/user/first_instance.yaml": { + "bar": {"foo": {"description": "foo_first_description"}}, + }, + "/home/user/second_instance.yaml": { + "bar": {"foo": {"description": "foo_second_description"}}, + "barfoobar": { + "type": "object", + "allOf": [{"description": "first_description"}, {"description": "second_description"}], + }, + }, + } + + root_schema_result = { + "foobar": {"$ref": "first_instance.yaml#/bar/foo"}, + "barfoo": {"$ref": "second_instance.yaml#/bar/foo_2"}, + "barfoobar": {"$ref": "second_instance.yaml#/barfoobar"}, + } + + external_schemas_result = { + "/home/user/first_instance.yaml": { + "bar": {"foo": {"description": "foo_first_description"}}, + }, + "/home/user/second_instance.yaml": { + "bar": {"foo_2": {"description": "foo_second_description"}}, + "barfoobar": { + "type": "object", + "allOf": [{"description": "first_description"}, {"description": "second_description"}], + }, + }, + } + + errors = [] + + CollisionResolver(root_schema, external_schemas, errors, "/home/user").resolve() + + assert len(errors) == 0 + assert root_schema == root_schema_result + assert external_schemas == external_schemas_result diff --git a/tests/test_resolver/test_resolver_resolved_schema.py b/tests/test_resolver/test_resolver_resolved_schema.py index 5728a24eb..76529f96b 100644 --- a/tests/test_resolver/test_resolver_resolved_schema.py +++ b/tests/test_resolver/test_resolver_resolved_schema.py @@ -10,7 +10,11 @@ def test__resolved_schema_with_resolved_external_references(): from openapi_python_client.resolver.resolved_schema import ResolvedSchema root_schema = {"foobar": {"$ref": "foobar.yaml#/foo"}} - external_schemas = {"/home/user/foobar.yaml": {"foo": {"description": "foobar_description"}}} + + external_schemas = { + "/home/user/foobar.yaml": {"foo": {"$ref": "/home/user/foobar_2.yaml#/foo"}}, + "/home/user/foobar_2.yaml": {"foo": {"description": "foobar_description"}}, + } errors = [] resolved_schema = ResolvedSchema(root_schema, external_schemas, errors, "/home/user").schema @@ -24,6 +28,97 @@ def test__resolved_schema_with_resolved_external_references(): assert "foobar_description" in resolved_schema["foo"]["description"] +def test__resolved_schema_with_duplicate_ref(): + + from openapi_python_client.resolver.resolved_schema import ResolvedSchema + + root_schema = { + "foo": {"$ref": "foobar.yaml#/foo"}, + "bar": {"$ref": "foobar.yaml#/foo"}, + "list": [{"foobar": {"$ref": "foobar.yaml#/bar"}}, {"barfoo": {"$ref": "foobar.yaml#/bar2/foo"}}], + } + + external_schemas = { + "/home/user/foobar.yaml": { + "foo": {"description": "foo_description"}, + "bar": {"$ref": "#/foo"}, + "bar2": {"foo": {"description": "foo_second_description"}}, + }, + } + + errors = [] + + resolved_schema = ResolvedSchema(root_schema, external_schemas, errors, "/home/user").schema + + assert len(errors) == 0 + + +def test__resolved_schema_with_malformed_schema(): + + from openapi_python_client.resolver.resolved_schema import ResolvedSchema + + root_schema = { + "paths": { + "/foo/bar": {"$ref": "inexistant.yaml#/paths/~1foo~1bar"}, + "/bar": {"$ref": "foobar.yaml#/paths/~1bar"}, + }, + "foo": {"$ref": "inexistant.yaml#/foo"}, + } + + external_schemas = { + "/home/user/foobar.yaml": { + "paths": { + "/foo/bar": {"description": "foobar_description"}, + }, + }, + } + + errors = [] + + resolved_schema = ResolvedSchema(root_schema, external_schemas, errors, "/home/user").schema + + assert len(errors) == 4 + assert errors == [ + "Failed to resolve remote reference > /home/user/inexistant.yaml", + "Failed to read remote value /paths//bar, in remote ref /home/user/foobar.yaml", + "Failed to resolve remote reference > /home/user/inexistant.yaml", + "Failed to resolve remote reference > /home/user/inexistant.yaml", + ] + + +def test__resolved_schema_with_remote_paths(): + + from openapi_python_client.resolver.resolved_schema import ResolvedSchema + + root_schema = { + "paths": { + "/foo/bar": {"$ref": "foobar.yaml#/paths/~1foo~1bar"}, + "/foo/bar2": {"$ref": "#/bar2"}, + }, + "bar2": {"description": "bar2_description"}, + } + + external_schemas = { + "/home/user/foobar.yaml": { + "paths": { + "/foo/bar": {"description": "foobar_description"}, + }, + }, + } + + expected_result = { + "paths": {"/foo/bar": {"description": "foobar_description"}, "/foo/bar2": {"$ref": "#/bar2"}}, + "bar2": {"description": "bar2_description"}, + } + + errors = [] + + resolved_schema = ResolvedSchema(root_schema, external_schemas, errors, "/home/user").schema + + assert len(errors) == 0 + assert resolved_schema == expected_result + + def test__resolved_schema_with_absolute_paths(): from openapi_python_client.resolver.resolved_schema import ResolvedSchema From a624a8818ac74fdb21b7628872bdcfb97da7b039 Mon Sep 17 00:00:00 2001 From: avy Date: Fri, 19 Mar 2021 12:29:13 +0100 Subject: [PATCH 38/39] fix data_load json test --- openapi_python_client/resolver/data_loader.py | 3 ++- tests/test_resolver/test_resolver_data_loader.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/openapi_python_client/resolver/data_loader.py b/openapi_python_client/resolver/data_loader.py index c29b5d6f1..df6677020 100644 --- a/openapi_python_client/resolver/data_loader.py +++ b/openapi_python_client/resolver/data_loader.py @@ -1,6 +1,7 @@ -import yaml import json +import yaml + from .resolver_types import SchemaData diff --git a/tests/test_resolver/test_resolver_data_loader.py b/tests/test_resolver/test_resolver_data_loader.py index ed20dd95f..271067ccd 100644 --- a/tests/test_resolver/test_resolver_data_loader.py +++ b/tests/test_resolver/test_resolver_data_loader.py @@ -45,6 +45,8 @@ def test_load_yaml(mocker): def test_load_json(mocker): from openapi_python_client.resolver.data_loader import DataLoader + json_loads = mocker.patch("json.loads") + content = mocker.MagicMock() - with pytest.raises(NotImplementedError): - DataLoader.load_json(content) + DataLoader.load_json(content) + json_loads.assert_called_once_with(content) From 59fb49b9bd701de30b259a5e24d486cb0e110d63 Mon Sep 17 00:00:00 2001 From: avy Date: Mon, 22 Mar 2021 08:43:41 +0100 Subject: [PATCH 39/39] add new tests for coverage purposes --- .../test_resolver/test_resolver_reference.py | 11 ++++++++++ .../test_resolver_resolved_schema.py | 21 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/tests/test_resolver/test_resolver_reference.py b/tests/test_resolver/test_resolver_reference.py index 6782426f3..bc13266b2 100644 --- a/tests/test_resolver/test_resolver_reference.py +++ b/tests/test_resolver/test_resolver_reference.py @@ -186,6 +186,17 @@ def test_path(): assert ref.path == path +def test_abs_path(): + + from openapi_python_client.resolver.reference import Reference + + ref = Reference("foo.yaml#/foo") + ref_with_parent = Reference("foo.yaml#/foo", "/home/user") + + assert ref.abs_path == "foo.yaml" + assert ref_with_parent.abs_path == "/home/user/foo.yaml" + + def test_is_full_document(): from openapi_python_client.resolver.reference import Reference diff --git a/tests/test_resolver/test_resolver_resolved_schema.py b/tests/test_resolver/test_resolver_resolved_schema.py index 76529f96b..3b3e6b9d8 100644 --- a/tests/test_resolver/test_resolver_resolved_schema.py +++ b/tests/test_resolver/test_resolver_resolved_schema.py @@ -28,6 +28,27 @@ def test__resolved_schema_with_resolved_external_references(): assert "foobar_description" in resolved_schema["foo"]["description"] +def test__resolved_schema_with_depth_refs(): + + from openapi_python_client.resolver.resolved_schema import ResolvedSchema + + root_schema = {"foo": {"$ref": "foo.yaml#/foo"}, "bar": {"$ref": "bar.yaml#/bar"}} + + external_schemas = { + "/home/user/foo.yaml": {"foo": {"$ref": "bar.yaml#/bar"}}, + "/home/user/bar.yaml": {"bar": {"description": "bar"}}, + } + + errors = [] + + expected_result = {"foo": {"$ref": "#/bar"}, "bar": {"description": "bar"}} + + resolved_schema = ResolvedSchema(root_schema, external_schemas, errors, "/home/user").schema + + assert len(errors) == 0 + assert resolved_schema == expected_result + + def test__resolved_schema_with_duplicate_ref(): from openapi_python_client.resolver.resolved_schema import ResolvedSchema