From e426e013481770f4e6c975cf5798eddc9a9d7a26 Mon Sep 17 00:00:00 2001
From: Timur Bazhirov <timur@exabyte.io>
Date: Sun, 24 Dec 2023 02:37:50 -0800
Subject: [PATCH 01/13] chore: test deploy job x 1

---
 .github/workflows/cicd.yml | 292 ++++++++++++++++++++-----------------
 1 file changed, 160 insertions(+), 132 deletions(-)

diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml
index cbac10f7e..760f66d3f 100644
--- a/.github/workflows/cicd.yml
+++ b/.github/workflows/cicd.yml
@@ -7,140 +7,168 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run-py-linter:
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: [3.8.6]
-
-    steps:
-      - name: Checkout this repository
-        uses: actions/checkout@v4
-        with:
-          lfs: true
-
-      - name: Checkout actions repository
-        uses: actions/checkout@v4
-        with:
-          repository: Exabyte-io/actions
-          token: ${{ secrets.BOT_GITHUB_TOKEN }}
-          path: actions
-
-      - name: Run ruff linter
-        uses: ./actions/py/lint
-        with:
-          python-version: ${{ matrix.python-version }}
-
-  run-py-tests:
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version:
-          - 3.8.6
-          - 3.9.x
-          # Enable after resolving Cython/PyYAML issue https://github.com/yaml/pyyaml/issues/724
-          # - 3.10.x
-          # - 3.11.x
-          # Enable after: AttributeError: module 'pkgutil' has no attribute 'ImpImporter'. Did you mean: 'zipimporter'?
-          # - 3.12.x
-
-    steps:
-      - name: Checkout this repository
-        uses: actions/checkout@v4
-        with:
-          lfs: true
-
-      - name: Checkout actions repository
-        uses: actions/checkout@v4
-        with:
-          repository: Exabyte-io/actions
-          token: ${{ secrets.BOT_GITHUB_TOKEN }}
-          path: actions
-
-      - name: Run python tests
-        uses: ./actions/py/test
-        with:
-          python-version: ${{ matrix.python-version }}
-
-  run-js-tests:
+#   run-py-linter:
+#     runs-on: ubuntu-20.04
+#     strategy:
+#       matrix:
+#         python-version: [3.8.6]
+#
+#     steps:
+#       - name: Checkout this repository
+#         uses: actions/checkout@v4
+#         with:
+#           lfs: true
+#
+#       - name: Checkout actions repository
+#         uses: actions/checkout@v4
+#         with:
+#           repository: Exabyte-io/actions
+#           token: ${{ secrets.BOT_GITHUB_TOKEN }}
+#           path: actions
+#
+#       - name: Run ruff linter
+#         uses: ./actions/py/lint
+#         with:
+#           python-version: ${{ matrix.python-version }}
+#
+#   run-py-tests:
+#     runs-on: ubuntu-20.04
+#     strategy:
+#       matrix:
+#         python-version:
+#           - 3.8.6
+#           - 3.9.x
+#           # Enable after resolving Cython/PyYAML issue https://github.com/yaml/pyyaml/issues/724
+#           # - 3.10.x
+#           # - 3.11.x
+#           # Enable after: AttributeError: module 'pkgutil' has no attribute 'ImpImporter'. Did you mean: 'zipimporter'?
+#           # - 3.12.x
+#
+#     steps:
+#       - name: Checkout this repository
+#         uses: actions/checkout@v4
+#         with:
+#           lfs: true
+#
+#       - name: Checkout actions repository
+#         uses: actions/checkout@v4
+#         with:
+#           repository: Exabyte-io/actions
+#           token: ${{ secrets.BOT_GITHUB_TOKEN }}
+#           path: actions
+#
+#       - name: Run python tests
+#         uses: ./actions/py/test
+#         with:
+#           python-version: ${{ matrix.python-version }}
+#
+#   run-js-tests:
+#     runs-on: ubuntu-latest
+#     strategy:
+#       matrix:
+#         node-version:
+#           - 14.x
+#           - 20.x
+#
+#     steps:
+#       - name: Checkout this repository
+#         uses: actions/checkout@v4
+#         with:
+#           lfs: true
+#
+#       - name: Checkout actions repository
+#         uses: actions/checkout@v4
+#         with:
+#           repository: Exabyte-io/actions
+#           token: ${{ secrets.BOT_GITHUB_TOKEN }}
+#           path: actions
+#
+#       - name: Run JS validate
+#         uses: ./actions/js/validate
+#         with:
+#           node-version: '14.x'
+#
+#       - name: Run JS tests
+#         uses: ./actions/js/test
+#         with:
+#           node-version: ${{ matrix.node-version }}
+
+  deploy:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version:
-          - 14.x
-          - 20.x
-
+    permissions:
+      contents: write
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
     steps:
-      - name: Checkout this repository
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v3
         with:
           lfs: true
 
-      - name: Checkout actions repository
-        uses: actions/checkout@v4
-        with:
-          repository: Exabyte-io/actions
-          token: ${{ secrets.BOT_GITHUB_TOKEN }}
-          path: actions
-
-      - name: Run JS validate
-        uses: ./actions/js/validate
-        with:
-          node-version: '14.x'
-
-      - name: Run JS tests
-        uses: ./actions/js/test
-        with:
-          node-version: ${{ matrix.node-version }}
-
-  publish-js-package:
-    needs: [run-py-linter, run-py-tests, run-js-tests]
-    runs-on: ubuntu-latest
-    if: github.ref_name == 'dev'
-
-    steps:
-      - name: Checkout this repository
-        uses: actions/checkout@v4
-        with:
-          lfs: true
-
-      - name: Checkout actions repository
-        uses: actions/checkout@v4
-        with:
-          repository: Exabyte-io/actions
-          token: ${{ secrets.BOT_GITHUB_TOKEN }}
-          path: actions
-
-      - name: Publish JS release
-        uses: ./actions/js/publish
-        with:
-          npm-token: ${{ secrets.NPM_TOKEN }}
-          github-token: ${{ secrets.BOT_GITHUB_TOKEN }}
-
-
-  publish-py-package:
-    needs: publish-js-package
-    runs-on: ubuntu-latest
-    if: github.ref_name == 'dev'
-
-    steps:
-      - name: Checkout this repository
-        uses: actions/checkout@v4
-        with:
-          lfs: true
-
-      - name: Checkout actions repository
-        uses: actions/checkout@v4
-        with:
-          repository: Exabyte-io/actions
-          token: ${{ secrets.BOT_GITHUB_TOKEN }}
-          path: actions
-
-      - name: Publish python release
-        uses: ./actions/py/publish
-        with:
-          python-version: 3.x
-          github-token: ${{ secrets.BOT_GITHUB_TOKEN }}
-          pypi-username: ${{ secrets.PYPI_USERNAME }}
-          pypi-password: ${{ secrets.PYPI_PASSWORD }}
-          publish-tag: 'false'
+      - name: Build
+        shell: bash -l {0}
+        run: |
+          mkdir ./docs
+            cp -r ./schema ./docs/
+            cp -r ./example ./docs/
+
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v3
+        # If you're changing the branch from main,
+        # also change the `main` in `refs/heads/main`
+        # below accordingly.
+        # if: github.ref == 'refs/heads/main'
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs
+
+#   publish-js-package:
+#     needs: [run-py-linter, run-py-tests, run-js-tests]
+#     runs-on: ubuntu-latest
+#     if: github.ref_name == 'dev'
+#
+#     steps:
+#       - name: Checkout this repository
+#         uses: actions/checkout@v4
+#         with:
+#           lfs: true
+#
+#       - name: Checkout actions repository
+#         uses: actions/checkout@v4
+#         with:
+#           repository: Exabyte-io/actions
+#           token: ${{ secrets.BOT_GITHUB_TOKEN }}
+#           path: actions
+#
+#       - name: Publish JS release
+#         uses: ./actions/js/publish
+#         with:
+#           npm-token: ${{ secrets.NPM_TOKEN }}
+#           github-token: ${{ secrets.BOT_GITHUB_TOKEN }}
+#
+#
+#   publish-py-package:
+#     needs: publish-js-package
+#     runs-on: ubuntu-latest
+#     if: github.ref_name == 'dev'
+#
+#     steps:
+#       - name: Checkout this repository
+#         uses: actions/checkout@v4
+#         with:
+#           lfs: true
+#
+#       - name: Checkout actions repository
+#         uses: actions/checkout@v4
+#         with:
+#           repository: Exabyte-io/actions
+#           token: ${{ secrets.BOT_GITHUB_TOKEN }}
+#           path: actions
+#
+#       - name: Publish python release
+#         uses: ./actions/py/publish
+#         with:
+#           python-version: 3.x
+#           github-token: ${{ secrets.BOT_GITHUB_TOKEN }}
+#           pypi-username: ${{ secrets.PYPI_USERNAME }}
+#           pypi-password: ${{ secrets.PYPI_PASSWORD }}
+#           publish-tag: 'false'

From 417baa36a6f217a7f6f1d32e0502f30833fe2734 Mon Sep 17 00:00:00 2001
From: Timur Bazhirov <timur@exabyte.io>
Date: Sun, 24 Dec 2023 02:38:50 -0800
Subject: [PATCH 02/13] chore: test deploy job x 2

---
 .github/workflows/cicd.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml
index 760f66d3f..70e503a00 100644
--- a/.github/workflows/cicd.yml
+++ b/.github/workflows/cicd.yml
@@ -95,10 +95,6 @@ jobs:
 
   deploy:
     runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}
     steps:
       - uses: actions/checkout@v3
         with:

From 741c20f97353b20f082af6eb9b673fbdd5a39eba Mon Sep 17 00:00:00 2001
From: Timur Bazhirov <timur@exabyte.io>
Date: Sun, 24 Dec 2023 02:49:17 -0800
Subject: [PATCH 03/13] chore: test deploy job x 3

---
 .github/workflows/cicd.yml | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml
index 70e503a00..f02233013 100644
--- a/.github/workflows/cicd.yml
+++ b/.github/workflows/cicd.yml
@@ -104,8 +104,23 @@ jobs:
         shell: bash -l {0}
         run: |
           mkdir ./docs
-            cp -r ./schema ./docs/
-            cp -r ./example ./docs/
+          cp -r ./schema ./docs/
+          cp -r ./example ./docs/
+          # https://stackoverflow.com/questions/3785055/how-can-i-create-a-simple-index-html-file-which-lists-all-files-directories
+          cd ./docs
+          find . -type d -print -exec sh -c 'tree "$0" \
+            -H "." \
+            -L 1 \
+            --noreport \
+            --dirsfirst \
+            --charset utf-8 \
+            -I "index.html" \
+            -T "Custom Title" \
+            --ignore-case \
+            --timefmt "%d-%b-%Y %H:%M" \
+            -s \
+            -D \
+            -o "$0/index.html"' {} \;
 
       - name: Deploy
         uses: peaceiris/actions-gh-pages@v3

From 59f81553316d1a613e8b02ecdbdb7881afd1d6f7 Mon Sep 17 00:00:00 2001
From: Timur Bazhirov <timur@exabyte.io>
Date: Sun, 24 Dec 2023 02:59:57 -0800
Subject: [PATCH 04/13] chore: test deploy job x 4

---
 .github/workflows/cicd.yml | 16 +---------------
 docs/README.md             |  0
 2 files changed, 1 insertion(+), 15 deletions(-)
 create mode 100644 docs/README.md

diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml
index f02233013..d74432009 100644
--- a/.github/workflows/cicd.yml
+++ b/.github/workflows/cicd.yml
@@ -106,21 +106,7 @@ jobs:
           mkdir ./docs
           cp -r ./schema ./docs/
           cp -r ./example ./docs/
-          # https://stackoverflow.com/questions/3785055/how-can-i-create-a-simple-index-html-file-which-lists-all-files-directories
-          cd ./docs
-          find . -type d -print -exec sh -c 'tree "$0" \
-            -H "." \
-            -L 1 \
-            --noreport \
-            --dirsfirst \
-            --charset utf-8 \
-            -I "index.html" \
-            -T "Custom Title" \
-            --ignore-case \
-            --timefmt "%d-%b-%Y %H:%M" \
-            -s \
-            -D \
-            -o "$0/index.html"' {} \;
+          wget https://github.com/Exabyte-io/api-examples/
 
       - name: Deploy
         uses: peaceiris/actions-gh-pages@v3
diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 000000000..e69de29bb

From 647d405d570a81c91370105cff70c2b80989b84d Mon Sep 17 00:00:00 2001
From: Timur Bazhirov <timur@exabyte.io>
Date: Sun, 24 Dec 2023 04:06:47 -0800
Subject: [PATCH 05/13] chore: add a way to build schemas in docs subfolder

---
 build_schemas.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/build_schemas.py b/build_schemas.py
index b3e2d21c6..c9ee27b5a 100644
--- a/build_schemas.py
+++ b/build_schemas.py
@@ -13,7 +13,9 @@
 After that, check for the files `src/py/mat3ra/esse/data/schemas.py` and `src/py/mat3ra/esse/data/examples.py`.
 """
 import os
+import json
 import yaml
+from pathlib import Path
 
 from mat3ra.esse.utils import parse_include_reference_statements_by_dir
 
@@ -39,3 +41,21 @@
         + f"RESULTS = json.loads(json.dumps({RESULTS}))\n"
     )
     f.write(content)
+
+if os.environ.get("BUILD_DOCS") != "true":
+    exit(0)
+
+for schema in SCHEMAS:
+    id_as_path = schema["$id"].replace("-", "_")
+    full_path = os.path.join(TOP_DIR, "docs/py/schema", id_as_path + ".json")
+    Path(full_path).parent.mkdir(parents=True, exist_ok=True)
+    with open(full_path, "w") as f:
+        f.write(json.dumps(schema, sort_keys=True, indent=4, separators=(",", ": ")))
+
+for example in EXAMPLES:
+    id_as_path = example["path"].replace("-", "_")
+    full_path = os.path.join(TOP_DIR, "docs/py/example", id_as_path + ".json")
+    Path(full_path).parent.mkdir(parents=True, exist_ok=True)
+    with open(full_path, "w") as f:
+        f.write(json.dumps(schema, sort_keys=True, indent=4, separators=(",", ": ")))
+

From 3510519aee4f81f1f03a07d6d7e6bcd21a299968 Mon Sep 17 00:00:00 2001
From: Timur Bazhirov <timur@exabyte.io>
Date: Sun, 24 Dec 2023 04:46:18 -0800
Subject: [PATCH 06/13] chore: update JS runtime to have wrapped examples

---
 src/js/esse/index.js |  5 +++--
 src/js/esse/utils.js | 10 +++++++---
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/js/esse/index.js b/src/js/esse/index.js
index 1a4a47c9c..a6f5b1286 100644
--- a/src/js/esse/index.js
+++ b/src/js/esse/index.js
@@ -5,12 +5,13 @@ import { EXAMPLES_DIR, SCHEMAS_DIR } from "./settings";
 import { parseIncludeReferenceStatementsByDir } from "./utils";
 
 const SCHEMAS = parseIncludeReferenceStatementsByDir(SCHEMAS_DIR);
-const EXAMPLES = parseIncludeReferenceStatementsByDir(EXAMPLES_DIR);
+const EXAMPLES = parseIncludeReferenceStatementsByDir(EXAMPLES_DIR, true);
 
 export class ESSE {
     constructor() {
         this.schemas = SCHEMAS;
-        this.examples = EXAMPLES;
+        this.wrappedExamples = EXAMPLES;
+        this.examples = EXAMPLES.map((example) => example.data);
     }
 
     getSchemaById(schemaId) {
diff --git a/src/js/esse/utils.js b/src/js/esse/utils.js
index e9e2e9065..f8c113dd3 100644
--- a/src/js/esse/utils.js
+++ b/src/js/esse/utils.js
@@ -24,14 +24,18 @@ export function parseIncludeReferenceStatements(filePath) {
  * Resolves `include` and `$ref` statements for all the JSON files inside a given directory.
  * @param dirPath {String} directory to parse.
  */
-export function parseIncludeReferenceStatementsByDir(dirPath) {
+export function parseIncludeReferenceStatementsByDir(dirPath, wrapInDataAndPath=false) {
     const data = [];
     file.walkSync(dirPath, (dirPath_, dirs_, files_) => {
         files_.forEach((file_) => {
             const filePath = path.join(dirPath_, file_);
             if (filePath.endsWith(".json")) {
-                // ignore files like .DS_Store
-                data.push(parseIncludeReferenceStatements(filePath));
+                const config = parseIncludeReferenceStatements(filePath);
+                if (wrapInDataAndPath) {
+                    data.push({ data: config, path: path.dirname(filePath) });
+                } else {
+                    data.push(config);
+                }
             }
         });
     });

From 11f2ab7bbc5b6d34f5df8259ce77365ab6a8fb24 Mon Sep 17 00:00:00 2001
From: Timur Bazhirov <timur@exabyte.io>
Date: Sun, 24 Dec 2023 04:49:37 -0800
Subject: [PATCH 07/13] chore: update build schemas script to build docs assets

---
 build_schemas.js | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/build_schemas.js b/build_schemas.js
index 269a0fbf3..5723d29fa 100644
--- a/build_schemas.js
+++ b/build_schemas.js
@@ -4,10 +4,12 @@
  * downstream consumption to avoid FS calls in the browser.
  */
 const fs = require("fs");
+const path = require("path");
+const mergeAllOf = require("json-schema-merge-allof");
 const { ESSE } = require("./lib/js/esse");
 
 const esse = new ESSE();
-const { schemas } = esse;
+const { schemas, wrappedExamples } = esse;
 const schema = esse.buildGlobalSchema();
 
 fs.writeFileSync(
@@ -17,3 +19,30 @@ fs.writeFileSync(
 );
 
 fs.writeFileSync("./schema.js", "module.exports = " + JSON.stringify(schema), "utf8");
+
+if (process.env.BUILD_DOCS !== "true") {
+    process.exit(0);
+}
+schemas.forEach((schema) => {
+    if (!process.env.SKIP_MERGE_ALLOF === "true") {
+        schema = mergeAllOf(schema, {resolvers: {defaultResolver: mergeAllOf.options.resolvers.title}});
+    }
+    id_as_path = schema["$id"].replace("-", "_");
+    full_path = `./docs/js/schema${id_as_path}.json`;
+    fs.mkdirSync(path.dirname(full_path), {recursive: true})
+    fs.writeFileSync(
+        full_path,
+        JSON.stringify(schema, null, 4),
+        "utf8",
+    );
+})
+wrappedExamples.forEach((example) => {
+    id_as_path = example["path"].replace("-", "_");
+    full_path = `./docs/js/example${id_as_path}.json`;
+    fs.mkdirSync(path.dirname(full_path), {recursive: true})
+    fs.writeFileSync(
+        full_path,
+        JSON.stringify(example, null, 4),
+        "utf8",
+    );
+})

From c9d85e96b144f0d06363700964231d7372ce24a7 Mon Sep 17 00:00:00 2001
From: Timur Bazhirov <timur@exabyte.io>
Date: Sun, 24 Dec 2023 04:49:50 -0800
Subject: [PATCH 08/13] chore: update package-json

---
 package.json | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/package.json b/package.json
index 409acd6cf..89787cb93 100644
--- a/package.json
+++ b/package.json
@@ -46,16 +46,17 @@
     },
     "license": "Apache-2.0",
     "dependencies": {
-        "ajv": "^4.1.7",
         "@babel/cli": "7.16.0",
         "@babel/core": "7.16.0",
-        "@babel/register": "7.16.0",
-        "@babel/preset-env": "7.16.4",
         "@babel/eslint-parser": "7.16.3",
+        "@babel/preset-env": "7.16.4",
+        "@babel/register": "7.16.0",
+        "ajv": "^4.1.7",
         "file": "^0.2.2",
+        "js-yaml": "^4.1.0",
         "json-schema-deref-sync": "0.14.0",
-        "lodash": "4.17.21",
-        "js-yaml": "^4.1.0"
+        "json-schema-merge-allof": "^0.8.1",
+        "lodash": "4.17.21"
     },
     "engines": {
         "node": ">=12.0.0"

From c8f306e81f2818ffc70ce3ceee543a8444f2aca5 Mon Sep 17 00:00:00 2001
From: Timur Bazhirov <timur@exabyte.io>
Date: Sun, 24 Dec 2023 04:49:58 -0800
Subject: [PATCH 09/13] chore: update package-lock

---
 package-lock.json | 71 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/package-lock.json b/package-lock.json
index f64ea92c3..ef124e503 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -2158,6 +2158,27 @@
             "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz",
             "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg=="
         },
+        "compute-gcd": {
+            "version": "1.2.1",
+            "resolved": "https://registry.npmjs.org/compute-gcd/-/compute-gcd-1.2.1.tgz",
+            "integrity": "sha512-TwMbxBNz0l71+8Sc4czv13h4kEqnchV9igQZBi6QUaz09dnz13juGnnaWWJTRsP3brxOoxeB4SA2WELLw1hCtg==",
+            "requires": {
+                "validate.io-array": "^1.0.3",
+                "validate.io-function": "^1.0.2",
+                "validate.io-integer-array": "^1.0.0"
+            }
+        },
+        "compute-lcm": {
+            "version": "1.1.2",
+            "resolved": "https://registry.npmjs.org/compute-lcm/-/compute-lcm-1.1.2.tgz",
+            "integrity": "sha512-OFNPdQAXnQhDSKioX8/XYT6sdUlXwpeMjfd6ApxMJfyZ4GxmLR1xvMERctlYhlHwIiz6CSpBc2+qYKjHGZw4TQ==",
+            "requires": {
+                "compute-gcd": "^1.2.1",
+                "validate.io-array": "^1.0.3",
+                "validate.io-function": "^1.0.2",
+                "validate.io-integer-array": "^1.0.0"
+            }
+        },
         "concat-map": {
             "version": "0.0.1",
             "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -3729,6 +3750,14 @@
             "integrity": "sha512-rkbaDZw3IPwd/ZPXob4XqQwVDKN/qeC2Dd7jL8EEGLEHLRmkPJgGAPw6OIIVmnwJrdcDh3vMR83/fc7lR5YpqA==",
             "dev": true
         },
+        "json-schema-compare": {
+            "version": "0.2.2",
+            "resolved": "https://registry.npmjs.org/json-schema-compare/-/json-schema-compare-0.2.2.tgz",
+            "integrity": "sha512-c4WYmDKyJXhs7WWvAWm3uIYnfyWFoIp+JEoX34rctVvEkMYCPGhXtvmFFXiffBbxfZsvQ0RNnV5H7GvDF5HCqQ==",
+            "requires": {
+                "lodash": "^4.17.4"
+            }
+        },
         "json-schema-deref-sync": {
             "version": "0.14.0",
             "resolved": "https://registry.npmjs.org/json-schema-deref-sync/-/json-schema-deref-sync-0.14.0.tgz",
@@ -3744,6 +3773,16 @@
                 "valid-url": "~1.0.9"
             }
         },
+        "json-schema-merge-allof": {
+            "version": "0.8.1",
+            "resolved": "https://registry.npmjs.org/json-schema-merge-allof/-/json-schema-merge-allof-0.8.1.tgz",
+            "integrity": "sha512-CTUKmIlPJbsWfzRRnOXz+0MjIqvnleIXwFTzz+t9T86HnYX/Rozria6ZVGLktAU9e+NygNljveP+yxqtQp/Q4w==",
+            "requires": {
+                "compute-lcm": "^1.1.2",
+                "json-schema-compare": "^0.2.2",
+                "lodash": "^4.17.20"
+            }
+        },
         "json-schema-traverse": {
             "version": "0.4.1",
             "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
@@ -5308,6 +5347,38 @@
             "resolved": "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz",
             "integrity": "sha512-QQDsV8OnSf5Uc30CKSwG9lnhMPe6exHtTXLRYX8uMwKENy640pU+2BgBL0LRbDh/eYRahNCS7aewCx0wf3NYVA=="
         },
+        "validate.io-array": {
+            "version": "1.0.6",
+            "resolved": "https://registry.npmjs.org/validate.io-array/-/validate.io-array-1.0.6.tgz",
+            "integrity": "sha512-DeOy7CnPEziggrOO5CZhVKJw6S3Yi7e9e65R1Nl/RTN1vTQKnzjfvks0/8kQ40FP/dsjRAOd4hxmJ7uLa6vxkg=="
+        },
+        "validate.io-function": {
+            "version": "1.0.2",
+            "resolved": "https://registry.npmjs.org/validate.io-function/-/validate.io-function-1.0.2.tgz",
+            "integrity": "sha512-LlFybRJEriSuBnUhQyG5bwglhh50EpTL2ul23MPIuR1odjO7XaMLFV8vHGwp7AZciFxtYOeiSCT5st+XSPONiQ=="
+        },
+        "validate.io-integer": {
+            "version": "1.0.5",
+            "resolved": "https://registry.npmjs.org/validate.io-integer/-/validate.io-integer-1.0.5.tgz",
+            "integrity": "sha512-22izsYSLojN/P6bppBqhgUDjCkr5RY2jd+N2a3DCAUey8ydvrZ/OkGvFPR7qfOpwR2LC5p4Ngzxz36g5Vgr/hQ==",
+            "requires": {
+                "validate.io-number": "^1.0.3"
+            }
+        },
+        "validate.io-integer-array": {
+            "version": "1.0.0",
+            "resolved": "https://registry.npmjs.org/validate.io-integer-array/-/validate.io-integer-array-1.0.0.tgz",
+            "integrity": "sha512-mTrMk/1ytQHtCY0oNO3dztafHYyGU88KL+jRxWuzfOmQb+4qqnWmI+gykvGp8usKZOM0H7keJHEbRaFiYA0VrA==",
+            "requires": {
+                "validate.io-array": "^1.0.3",
+                "validate.io-integer": "^1.0.4"
+            }
+        },
+        "validate.io-number": {
+            "version": "1.0.3",
+            "resolved": "https://registry.npmjs.org/validate.io-number/-/validate.io-number-1.0.3.tgz",
+            "integrity": "sha512-kRAyotcbNaSYoDnXvb4MHg/0a1egJdLwS6oJ38TJY7aw9n93Fl/3blIXdyYvPOp55CNxywooG/3BcrwNrBpcSg=="
+        },
         "which": {
             "version": "2.0.2",
             "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",

From 0b8c77ac02db0b3166da9602f055597e4d457581 Mon Sep 17 00:00:00 2001
From: Timur Bazhirov <timur@exabyte.io>
Date: Sun, 24 Dec 2023 05:02:32 -0800
Subject: [PATCH 10/13] chore: fix docs paths

---
 build_schemas.js     | 4 ++--
 src/js/esse/utils.js | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/build_schemas.js b/build_schemas.js
index 5723d29fa..3fc22298d 100644
--- a/build_schemas.js
+++ b/build_schemas.js
@@ -28,7 +28,7 @@ schemas.forEach((schema) => {
         schema = mergeAllOf(schema, {resolvers: {defaultResolver: mergeAllOf.options.resolvers.title}});
     }
     id_as_path = schema["$id"].replace("-", "_");
-    full_path = `./docs/js/schema${id_as_path}.json`;
+    full_path = `./docs/js/schema/${id_as_path}.json`;
     fs.mkdirSync(path.dirname(full_path), {recursive: true})
     fs.writeFileSync(
         full_path,
@@ -38,7 +38,7 @@ schemas.forEach((schema) => {
 })
 wrappedExamples.forEach((example) => {
     id_as_path = example["path"].replace("-", "_");
-    full_path = `./docs/js/example${id_as_path}.json`;
+    full_path = `./docs/js/example/${id_as_path}.json`;
     fs.mkdirSync(path.dirname(full_path), {recursive: true})
     fs.writeFileSync(
         full_path,
diff --git a/src/js/esse/utils.js b/src/js/esse/utils.js
index f8c113dd3..cd6e8501d 100644
--- a/src/js/esse/utils.js
+++ b/src/js/esse/utils.js
@@ -26,13 +26,15 @@ export function parseIncludeReferenceStatements(filePath) {
  */
 export function parseIncludeReferenceStatementsByDir(dirPath, wrapInDataAndPath=false) {
     const data = [];
+    const topDir = path.resolve(__dirname, "../../../");
     file.walkSync(dirPath, (dirPath_, dirs_, files_) => {
         files_.forEach((file_) => {
             const filePath = path.join(dirPath_, file_);
             if (filePath.endsWith(".json")) {
                 const config = parseIncludeReferenceStatements(filePath);
                 if (wrapInDataAndPath) {
-                    data.push({ data: config, path: path.dirname(filePath) });
+                    const _path = path.dirname(filePath).replace(path.join(topDir, "example"), "");
+                    data.push({ data: config, path: _path });
                 } else {
                     data.push(config);
                 }

From 353c3b9cc2fdae6fa16e02f4d0364219d852266e Mon Sep 17 00:00:00 2001
From: Timur Bazhirov <timur@exabyte.io>
Date: Sun, 24 Dec 2023 05:17:43 -0800
Subject: [PATCH 11/13] chore: fix docs assets build logic

---
 build_schemas.js     | 2 +-
 src/js/esse/utils.js | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/build_schemas.js b/build_schemas.js
index 3fc22298d..2d96353e5 100644
--- a/build_schemas.js
+++ b/build_schemas.js
@@ -42,7 +42,7 @@ wrappedExamples.forEach((example) => {
     fs.mkdirSync(path.dirname(full_path), {recursive: true})
     fs.writeFileSync(
         full_path,
-        JSON.stringify(example, null, 4),
+        JSON.stringify(example["data"], null, 4),
         "utf8",
     );
 })
diff --git a/src/js/esse/utils.js b/src/js/esse/utils.js
index cd6e8501d..172e57650 100644
--- a/src/js/esse/utils.js
+++ b/src/js/esse/utils.js
@@ -33,7 +33,11 @@ export function parseIncludeReferenceStatementsByDir(dirPath, wrapInDataAndPath=
             if (filePath.endsWith(".json")) {
                 const config = parseIncludeReferenceStatements(filePath);
                 if (wrapInDataAndPath) {
-                    const _path = path.dirname(filePath).replace(path.join(topDir, "example"), "");
+                    const _path = path.join(
+                        // remove leading slashes and "example" from path
+                        path.dirname(filePath).replace(path.join(topDir, "example"), "").replace(/^\/+/, ''),
+                        path.basename(filePath).replace(".json", "")
+                    );
                     data.push({ data: config, path: _path });
                 } else {
                     data.push(config);

From c1bbb2d377c9a717a3098037a7d518cd18202629 Mon Sep 17 00:00:00 2001
From: Timur Bazhirov <timur@exabyte.io>
Date: Sun, 24 Dec 2023 05:23:34 -0800
Subject: [PATCH 12/13] chore: re-enable tests, tmp disable docs build

---
 .github/workflows/cicd.yml | 302 ++++++++++++++++++-------------------
 1 file changed, 151 insertions(+), 151 deletions(-)

diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml
index d74432009..87b5ef5c9 100644
--- a/.github/workflows/cicd.yml
+++ b/.github/workflows/cicd.yml
@@ -7,165 +7,165 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-#   run-py-linter:
-#     runs-on: ubuntu-20.04
-#     strategy:
-#       matrix:
-#         python-version: [3.8.6]
-#
-#     steps:
-#       - name: Checkout this repository
-#         uses: actions/checkout@v4
-#         with:
-#           lfs: true
-#
-#       - name: Checkout actions repository
-#         uses: actions/checkout@v4
-#         with:
-#           repository: Exabyte-io/actions
-#           token: ${{ secrets.BOT_GITHUB_TOKEN }}
-#           path: actions
-#
-#       - name: Run ruff linter
-#         uses: ./actions/py/lint
-#         with:
-#           python-version: ${{ matrix.python-version }}
-#
-#   run-py-tests:
-#     runs-on: ubuntu-20.04
-#     strategy:
-#       matrix:
-#         python-version:
-#           - 3.8.6
-#           - 3.9.x
-#           # Enable after resolving Cython/PyYAML issue https://github.com/yaml/pyyaml/issues/724
-#           # - 3.10.x
-#           # - 3.11.x
-#           # Enable after: AttributeError: module 'pkgutil' has no attribute 'ImpImporter'. Did you mean: 'zipimporter'?
-#           # - 3.12.x
-#
-#     steps:
-#       - name: Checkout this repository
-#         uses: actions/checkout@v4
-#         with:
-#           lfs: true
-#
-#       - name: Checkout actions repository
-#         uses: actions/checkout@v4
-#         with:
-#           repository: Exabyte-io/actions
-#           token: ${{ secrets.BOT_GITHUB_TOKEN }}
-#           path: actions
-#
-#       - name: Run python tests
-#         uses: ./actions/py/test
-#         with:
-#           python-version: ${{ matrix.python-version }}
-#
-#   run-js-tests:
-#     runs-on: ubuntu-latest
-#     strategy:
-#       matrix:
-#         node-version:
-#           - 14.x
-#           - 20.x
-#
-#     steps:
-#       - name: Checkout this repository
-#         uses: actions/checkout@v4
-#         with:
-#           lfs: true
-#
-#       - name: Checkout actions repository
-#         uses: actions/checkout@v4
-#         with:
-#           repository: Exabyte-io/actions
-#           token: ${{ secrets.BOT_GITHUB_TOKEN }}
-#           path: actions
-#
-#       - name: Run JS validate
-#         uses: ./actions/js/validate
-#         with:
-#           node-version: '14.x'
-#
-#       - name: Run JS tests
-#         uses: ./actions/js/test
-#         with:
-#           node-version: ${{ matrix.node-version }}
+  run-py-linter:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: [3.8.6]
+
+    steps:
+      - name: Checkout this repository
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Checkout actions repository
+        uses: actions/checkout@v4
+        with:
+          repository: Exabyte-io/actions
+          token: ${{ secrets.BOT_GITHUB_TOKEN }}
+          path: actions
+
+      - name: Run ruff linter
+        uses: ./actions/py/lint
+        with:
+          python-version: ${{ matrix.python-version }}
 
-  deploy:
+  run-py-tests:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version:
+          - 3.8.6
+          - 3.9.x
+          # Enable after resolving Cython/PyYAML issue https://github.com/yaml/pyyaml/issues/724
+          # - 3.10.x
+          # - 3.11.x
+          # Enable after: AttributeError: module 'pkgutil' has no attribute 'ImpImporter'. Did you mean: 'zipimporter'?
+          # - 3.12.x
+
+    steps:
+      - name: Checkout this repository
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Checkout actions repository
+        uses: actions/checkout@v4
+        with:
+          repository: Exabyte-io/actions
+          token: ${{ secrets.BOT_GITHUB_TOKEN }}
+          path: actions
+
+      - name: Run python tests
+        uses: ./actions/py/test
+        with:
+          python-version: ${{ matrix.python-version }}
+
+  run-js-tests:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version:
+          - 14.x
+          - 20.x
+
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout this repository
+        uses: actions/checkout@v4
         with:
           lfs: true
 
-      - name: Build
-        shell: bash -l {0}
-        run: |
-          mkdir ./docs
-          cp -r ./schema ./docs/
-          cp -r ./example ./docs/
-          wget https://github.com/Exabyte-io/api-examples/
-
-      - name: Deploy
-        uses: peaceiris/actions-gh-pages@v3
-        # If you're changing the branch from main,
-        # also change the `main` in `refs/heads/main`
-        # below accordingly.
-        # if: github.ref == 'refs/heads/main'
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./docs
-
-#   publish-js-package:
-#     needs: [run-py-linter, run-py-tests, run-js-tests]
-#     runs-on: ubuntu-latest
-#     if: github.ref_name == 'dev'
-#
-#     steps:
-#       - name: Checkout this repository
-#         uses: actions/checkout@v4
-#         with:
-#           lfs: true
-#
-#       - name: Checkout actions repository
-#         uses: actions/checkout@v4
-#         with:
-#           repository: Exabyte-io/actions
-#           token: ${{ secrets.BOT_GITHUB_TOKEN }}
-#           path: actions
-#
-#       - name: Publish JS release
-#         uses: ./actions/js/publish
-#         with:
-#           npm-token: ${{ secrets.NPM_TOKEN }}
-#           github-token: ${{ secrets.BOT_GITHUB_TOKEN }}
-#
-#
-#   publish-py-package:
-#     needs: publish-js-package
+      - name: Checkout actions repository
+        uses: actions/checkout@v4
+        with:
+          repository: Exabyte-io/actions
+          token: ${{ secrets.BOT_GITHUB_TOKEN }}
+          path: actions
+
+      - name: Run JS validate
+        uses: ./actions/js/validate
+        with:
+          node-version: '14.x'
+
+      - name: Run JS tests
+        uses: ./actions/js/test
+        with:
+          node-version: ${{ matrix.node-version }}
+
+#   deploy:
 #     runs-on: ubuntu-latest
-#     if: github.ref_name == 'dev'
-#
 #     steps:
-#       - name: Checkout this repository
-#         uses: actions/checkout@v4
+#       - uses: actions/checkout@v3
 #         with:
 #           lfs: true
 #
-#       - name: Checkout actions repository
-#         uses: actions/checkout@v4
-#         with:
-#           repository: Exabyte-io/actions
-#           token: ${{ secrets.BOT_GITHUB_TOKEN }}
-#           path: actions
-#
-#       - name: Publish python release
-#         uses: ./actions/py/publish
-#         with:
-#           python-version: 3.x
-#           github-token: ${{ secrets.BOT_GITHUB_TOKEN }}
-#           pypi-username: ${{ secrets.PYPI_USERNAME }}
-#           pypi-password: ${{ secrets.PYPI_PASSWORD }}
-#           publish-tag: 'false'
+#       - name: Build
+#         shell: bash -l {0}
+#         run: |
+#           mkdir ./docs
+#           cp -r ./schema ./docs/
+#           cp -r ./example ./docs/
+#           wget https://github.com/Exabyte-io/api-examples/
+#
+#       - name: Deploy
+#         uses: peaceiris/actions-gh-pages@v3
+#         # If you're changing the branch from main,
+#         # also change the `main` in `refs/heads/main`
+#         # below accordingly.
+#         # if: github.ref == 'refs/heads/main'
+#         with:
+#           github_token: ${{ secrets.GITHUB_TOKEN }}
+#           publish_dir: ./docs
+
+  publish-js-package:
+    needs: [run-py-linter, run-py-tests, run-js-tests]
+    runs-on: ubuntu-latest
+    if: github.ref_name == 'dev'
+
+    steps:
+      - name: Checkout this repository
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Checkout actions repository
+        uses: actions/checkout@v4
+        with:
+          repository: Exabyte-io/actions
+          token: ${{ secrets.BOT_GITHUB_TOKEN }}
+          path: actions
+
+      - name: Publish JS release
+        uses: ./actions/js/publish
+        with:
+          npm-token: ${{ secrets.NPM_TOKEN }}
+          github-token: ${{ secrets.BOT_GITHUB_TOKEN }}
+
+
+  publish-py-package:
+    needs: publish-js-package
+    runs-on: ubuntu-latest
+    if: github.ref_name == 'dev'
+
+    steps:
+      - name: Checkout this repository
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Checkout actions repository
+        uses: actions/checkout@v4
+        with:
+          repository: Exabyte-io/actions
+          token: ${{ secrets.BOT_GITHUB_TOKEN }}
+          path: actions
+
+      - name: Publish python release
+        uses: ./actions/py/publish
+        with:
+          python-version: 3.x
+          github-token: ${{ secrets.BOT_GITHUB_TOKEN }}
+          pypi-username: ${{ secrets.PYPI_USERNAME }}
+          pypi-password: ${{ secrets.PYPI_PASSWORD }}
+          publish-tag: 'false'

From 64227c58925b7b7a20930ff553ad2490adab8377 Mon Sep 17 00:00:00 2001
From: Timur Bazhirov <timur@exabyte.io>
Date: Sun, 24 Dec 2023 18:32:07 -0800
Subject: [PATCH 13/13] chore: add schema re-build as a pre-commit action

---
 .gitignore        |  6 +++++-
 .husky/pre-commit | 14 ++++++++++++++
 package.json      |  2 ++
 3 files changed, 21 insertions(+), 1 deletion(-)
 create mode 100755 .husky/pre-commit

diff --git a/.gitignore b/.gitignore
index 905afa908..3aaa86a60 100644
--- a/.gitignore
+++ b/.gitignore
@@ -53,7 +53,11 @@ dist/
 .nyc_output
 .eslintcache
 
-.husky
+.husky/*
+!.husky/pre-commit
 
 schemas.js
 schema.js
+
+# Static assets, if any
+site
diff --git a/.husky/pre-commit b/.husky/pre-commit
new file mode 100755
index 000000000..e24dfc9b7
--- /dev/null
+++ b/.husky/pre-commit
@@ -0,0 +1,14 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+SRC_PATTERN="\.json$"
+if git diff --cached --name-only | grep --quiet -E "$SRC_PATTERN"
+then
+  echo "JSON assets changed. Running build scripts."
+  echo "Re-building JS assets."
+  npm run build:assets
+  echo "Re-building Python assets: requires virtual environment and dependencies (pip install .'[tests]')."
+  python build_schemas.py
+fi
+
+npx lint-staged
diff --git a/package.json b/package.json
index 89787cb93..063a12ae5 100644
--- a/package.json
+++ b/package.json
@@ -5,6 +5,8 @@
     "scripts": {
         "prepublishOnly": "rm -rf lib; npm run transpile",
         "transpile": "mkdir -p lib; babel src/js --out-dir lib/js; node build_schemas.js",
+        "build:assets": "node build_schemas.js",
+        "build:assets-with-docs": "BUILD_DOCS=true node build_schemas.js",
         "test": "nyc --reporter=text mocha --bail --require @babel/register tests/js/",
         "lint": "eslint src/js && prettier --write src/js",
         "lint:fix": "eslint --fix --cache src/js && prettier --write src/js",