Generate jsonschema from pydantic v2

.gitignore

@@ -1,4 +1,6 @@
 
 .DS_Store
 node_modules/
-.venv
+.venv
+env
+__pycache__

README.md

@@ -112,4 +112,9 @@ _Do you use a different IDE which also supports JSON Schema? Please open a PR wi
 ## Contributing 
 PRs that improve these schemata are welcome! 
 
-Please ensure that JSON keys are sorted by [vscode-sort-json](https://marketplace.visualstudio.com/items?itemName=richie5um2.vscode-sort-json) according to the rules in `.vscode/settings`. 
+These schemas are generated from [pydantic models](https://docs.pydantic.dev/latest/concepts/json_schema/). To make updates, the process is as follows:
+
+1. Create a virtual environment and install the dependencies: `pip install -r requirements.txt`
+2. Make changes to the corresponding pydantic models in `src/latest.py`
+3. Run the generation script: `python3 src/generate.py`
+4. Add tests for valid and invalid JSON files in `tests/latest`

generate.py

@@ -0,0 +1,56 @@
+import json
+from pathlib import Path
+from typing import Any, Dict
+
+from src.latest.dbt_cloud import DbtCloud
+from src.latest.dbt_yml_files import DbtYmlFiles
+from src.latest.dependencies import Dependencies, Packages
+from src.latest.selectors import Selectors
+from pydantic.json_schema import GenerateJsonSchema
+
+
+class RemoveNullsGenerateJsonSchema(GenerateJsonSchema):
+ """A GenerateJsonSchema which removes nullability from types.
+
+ We do not want to include optional values in the json schema because
+ that would inhibit code completion and validation.
+
+ Certain properties (such as freshness overrides) need to be nullable,
+ which can be achieved by setting the $comment value below.
+ """
+
+ def _remove_null(self, json_schema: Dict[str, Any]):
+ if "$comment" in json_schema and json_schema["$comment"] == "truly_nullable":
+ return
+ if "anyOf" in json_schema:
+ json_schema["anyOf"] = [
+ item for item in json_schema["anyOf"] if item != {"type": "null"}
+ ]
+ for v in json_schema.values():
+ if isinstance(v, dict):
+ self._remove_null(v)
+
+ def generate(self, schema, mode="validation"):
+ json_schema = super().generate(schema, mode=mode)
+ json_schema["$schema"] = "http://json-schema.org/draft-07/schema#"
+ self._remove_null(json_schema)
+ return json_schema
+
+
+if __name__ == "__main__":
+ files = {
+ "dbt_yml_files": DbtYmlFiles,
+ "dependencies": Dependencies,
+ "packages": Packages,
+ "selectors": Selectors,
+ "dbt_cloud": DbtCloud,
+ }
+ output_directory = Path("schemas/latest")
+ for file_name, model in files.items():
+ schema_file = output_directory / f"{file_name}-latest.json"
+ schema_file.parent.mkdir(parents=True, exist_ok=True)
+ schema = model.model_json_schema(
+ mode="validation", schema_generator=RemoveNullsGenerateJsonSchema
+ )
+ print("Generating", schema_file)
+ schema_file.write_text(json.dumps(schema, indent=2))

requirements.txt

@@ -0,0 +1 @@
+datamodel-code-generator

schemas/latest-originals/dbt_cloud-latest.json

@@ -0,0 +1,14 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "type": "object",
+ "properties": {
+ "project-id": {
+ "type": "string"
+ },
+ "defer-env-id": {
+ "type": "string"
+ }
+ },
+ "required": ["project-id"],
+ "additionalProperties": false
+}