feat(spec2sdk): add URL references resolver

README.md

@@ -10,7 +10,7 @@
 from pathlib import Path
 from spec2sdk.main import generate
 
-generate(input_file=Path("path/to/api.yml"), output_dir=Path("path/to/output-dir/"))
+generate(url=Path("path/to/api.yml").absolute().as_uri(), output_dir=Path("path/to/output-dir/"))
 ```
 
 # Open API specification requirements
@@ -160,7 +160,7 @@ def render_email_field(py_type: EmailPythonType) -> TypeRenderer:
 
 
 if __name__ == "__main__":
-    generate(input_file=Path("api.yml"), output_dir=Path("output"))
+    generate(url=Path("api.yml").absolute().as_uri(), output_dir=Path("output"))
 ```
 
 ## Output

spec2sdk/main.py

@@ -2,6 +2,7 @@
 import shutil
 import subprocess
 from pathlib import Path
+from urllib.parse import urlparse
 
 from openapi_spec_validator import validate
 
@@ -28,10 +29,8 @@ def run_formatter(formatter: str):
     run_formatter("ruff check --fix")
 
 
-def generate(input_file: Path, output_dir: Path):
-    schema = ResolvingParser(base_path=input_file.parent).parse(
-        relative_filepath=input_file.relative_to(input_file.parent),
-    )
+def generate(url: str, output_dir: Path):
+    schema = ResolvingParser().parse(url=url)
     validate(schema)
     spec = parse_spec(schema)
 
@@ -53,9 +52,9 @@ def generate(input_file: Path, output_dir: Path):
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "--input-file",
-        type=Path,
-        help="Path to the OpenAPI specification file in the YAML format",
+        "--input",
+        type=str,
+        help="File path or URL to the OpenAPI specification file in the YAML format",
         required=True,
     )
     parser.add_argument(
@@ -66,7 +65,10 @@ def main():
     )
 
     args = parser.parse_args()
-    generate(input_file=args.input_file, output_dir=args.output_dir)
+    generate(
+        url=Path(args.input).absolute().as_uri() if urlparse(args.input).scheme == "" else args.input,
+        output_dir=args.output_dir,
+    )
 
 
 if __name__ == "__main__":

spec2sdk/parsers/resolver.py

@@ -1,65 +1,72 @@
 from functools import reduce
-from pathlib import Path
-from typing import Any, Sequence
-from urllib.parse import urlparse
+from typing import Any, ClassVar, Sequence
+from urllib.parse import urldefrag, urljoin
+from urllib.request import urlopen
 
 import yaml
+from pydantic import BaseModel, ConfigDict
 
 from spec2sdk.parsers.exceptions import CircularReference
 
 SCHEMA_NAME_FIELD = "x-schema-name"
 
 
+class SchemaLoader(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    _schema_cache: ClassVar[dict] = {}
+    schema_url: str
+
+    def _read_schema(self) -> dict:
+        if self.schema_url not in self._schema_cache:
+            self._schema_cache[self.schema_url] = yaml.safe_load(stream=urlopen(self.schema_url))
+
+        return self._schema_cache[self.schema_url]
+
+    def get_schema_fragment(self, path: str) -> dict:
+        return reduce(
+            lambda acc, key: acc[key],
+            filter(None, path.split("/")),
+            self._read_schema(),
+        )
+
+    def parse(self, reference: str) -> "SchemaLoader":
+        reference_url, _ = urldefrag(reference)
+        new_schema_url = urljoin(self.schema_url, reference_url)
+
+        return self if new_schema_url == self.schema_url else SchemaLoader(schema_url=new_schema_url)
+
+
 class ResolvingParser:
-    def __init__(self, base_path: Path):
-        self._base_path = base_path
-        self._file_cache = {}
+    def __init__(self):
         self._resolved_references = {}
 
-    def _read_schema_from_file(self, relative_filepath: Path) -> dict:
-        absolute_filepath = (self._base_path / relative_filepath).resolve()
-        if absolute_filepath not in self._file_cache:
-            self._file_cache[absolute_filepath] = yaml.safe_load(absolute_filepath.read_text())
-        return self._file_cache[absolute_filepath]
-
-    def _resolve_schema(self, relative_filepath: Path, schema: dict, parent_references: Sequence[str]) -> dict:
+    def _resolve_schema(self, schema_loader: SchemaLoader, schema: dict, parent_reference_ids: Sequence[str]) -> dict:
         def resolve_value(value: Any) -> Any:
             if isinstance(value, dict):
-                return self._resolve_schema(relative_filepath, value, parent_references)
+                return self._resolve_schema(schema_loader, value, parent_reference_ids)
             elif isinstance(value, list):
                 return [resolve_value(item) for item in value]
             else:
                 return value
 
         def resolve_reference(reference: str) -> dict:
-            if reference in parent_references:
+            reference_schema_loader = schema_loader.parse(reference)
+            _, reference_fragment_path = urldefrag(reference)
+            reference_id = f"{reference_schema_loader.schema_url}#{reference_fragment_path}"
+
+            if reference_id in parent_reference_ids:
                 raise CircularReference(f"Circular reference found in {reference}")
 
-            if reference not in self._resolved_references:
-                parsed_ref = urlparse(reference)
-
-                if parsed_ref.scheme == parsed_ref.netloc == "":
-                    ref_relative_filepath = (
-                        relative_filepath.parent / Path(parsed_ref.path) if parsed_ref.path else relative_filepath
-                    )
-                    ref_schema_fragment = reduce(
-                        lambda acc, key: acc[key],
-                        filter(None, parsed_ref.fragment.split("/")),
-                        self._read_schema_from_file(ref_relative_filepath),
-                    )
-                    ref_schema_name = parsed_ref.fragment.rsplit("/", 1)[-1]
-
-                    self._resolved_references[reference] = {
-                        SCHEMA_NAME_FIELD: ref_schema_name,
-                    } | self._resolve_schema(
-                        relative_filepath=ref_relative_filepath,
-                        schema=ref_schema_fragment,
-                        parent_references=(*parent_references, reference),
-                    )
-                else:
-                    raise NotImplementedError(f"Unsupported schema reference: {reference}")
-
-            return self._resolved_references[reference]
+            if reference_id not in self._resolved_references:
+                self._resolved_references[reference_id] = {
+                    SCHEMA_NAME_FIELD: reference_fragment_path.rsplit("/", 1)[-1],
+                } | self._resolve_schema(
+                    schema_loader=reference_schema_loader,
+                    schema=reference_schema_loader.get_schema_fragment(path=reference_fragment_path),
+                    parent_reference_ids=(*parent_reference_ids, reference_id),
+                )
+
+            return self._resolved_references[reference_id]
 
         return {
             new_key: new_value
@@ -69,9 +76,11 @@ def resolve_reference(reference: str) -> dict:
             ).items()
         }
 
-    def parse(self, relative_filepath: Path) -> dict:
+    def parse(self, url: str) -> dict:
+        schema_loader = SchemaLoader(schema_url=url)
+
         return self._resolve_schema(
-            relative_filepath=relative_filepath,
-            schema=self._read_schema_from_file(relative_filepath),
-            parent_references=(),
+            schema_loader=schema_loader,
+            schema=schema_loader.get_schema_fragment(path="/"),
+            parent_reference_ids=(),
         )

tests/parsers/resolver/test_data/schema_cache/api.yml

@@ -0,0 +1,32 @@
+openapi: 3.0.0
+
+info:
+  title: Example API
+  version: '1.0'
+
+paths:
+  /users:
+    get:
+      operationId: getUsers
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: 'definitions/users.yml#/components/schemas/User'
+
+  /userGroups:
+    get:
+      operationId: getUserGroups
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: './definitions/users.yml#/components/schemas/UserGroup'

tests/parsers/resolver/test_data/schema_cache/definitions/users.yml

@@ -0,0 +1,25 @@
+components:
+  schemas:
+    User:
+      type: object
+      properties:
+        name:
+          type: string
+        email:
+          type: string
+
+    UserGroup:
+      type: object
+      properties:
+        name:
+          type: string
+        permissions:
+          type: array
+          items:
+            $ref: 'users.yml#/components/schemas/Permission'
+
+    Permission:
+      type: object
+      properties:
+        name:
+          type: string

tests/parsers/resolver/test_data/url_references/expected_output/schema.json

@@ -0,0 +1,60 @@
+{
+  "info": {
+    "title": "Example API",
+    "version": "1.0"
+  },
+  "openapi": "3.0.0",
+  "paths": {
+    "/comment{commentId}": {
+      "get": {
+        "operationId": "getComment",
+        "parameters": [
+          {
+            "in": "path",
+            "name": "commentId",
+            "required": true,
+            "schema": {
+              "type": "integer"
+            },
+            "x-schema-name": "CommentId"
+          }
+        ],
+        "responses": {
+          "200": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "properties": {
+                    "author": {
+                      "properties": {
+                        "city": {
+                          "maxLength": 50,
+                          "type": "string",
+                          "x-schema-name": "City"
+                        },
+                        "email": {
+                          "type": "string"
+                        },
+                        "name": {
+                          "type": "string"
+                        }
+                      },
+                      "type": "object",
+                      "x-schema-name": "User"
+                    },
+                    "text": {
+                      "type": "string"
+                    }
+                  },
+                  "type": "object",
+                  "x-schema-name": "Comment"
+                }
+              }
+            },
+            "description": "Successful response"
+          }
+        }
+      }
+    }
+  }
+}

tests/parsers/resolver/test_data/url_references/input/api.yml

@@ -0,0 +1,19 @@
+openapi: 3.0.0
+
+info:
+  title: Example API
+  version: '1.0'
+
+paths:
+  /comment{commentId}:
+    get:
+      operationId: getComment
+      parameters:
+        - $ref: 'http://localhost/definitions/parameters.yml#/components/parameters/CommentId'
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                $ref: 'http://localhost/definitions/schemas.yml#/components/schemas/Comment'

tests/parsers/resolver/test_data/url_references/input/definitions/parameters.yml

@@ -0,0 +1,8 @@
+components:
+  parameters:
+    CommentId:
+      name: commentId
+      in: path
+      required: true
+      schema:
+        type: integer

tests/parsers/resolver/test_data/url_references/input/definitions/schemas.yml

@@ -0,0 +1,9 @@
+components:
+  schemas:
+    Comment:
+      type: object
+      properties:
+        text:
+          type: string
+        author:
+          $ref: 'users/user.yml#/components/schemas/User'

tests/parsers/resolver/test_data/url_references/input/definitions/users/location.yml

@@ -0,0 +1,5 @@
+components:
+  schemas:
+    City:
+      type: string
+      maxLength: 50

tests/parsers/resolver/test_data/url_references/input/definitions/users/user.yml

@@ -0,0 +1,11 @@
+components:
+  schemas:
+    User:
+      type: object
+      properties:
+        name:
+          type: string
+        email:
+          type: string
+        city:
+          $ref: './location.yml#/components/schemas/City'