update/SOF-7183-1: Update JS Runtime, Create "./docs" schemas/examples

.github/workflows/cicd.yml

@@ -93,6 +93,31 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
 
+#   deploy:
+#     runs-on: ubuntu-latest
+#     steps:
+#       - uses: actions/checkout@v3
+#         with:
+#           lfs: true
+#
+#       - name: Build
+#         shell: bash -l {0}
+#         run: |
+#           mkdir ./docs
+#           cp -r ./schema ./docs/
+#           cp -r ./example ./docs/
+#           wget https://github.com/Exabyte-io/api-examples/
+#
+#       - name: Deploy
+#         uses: peaceiris/actions-gh-pages@v3
+#         # If you're changing the branch from main,
+#         # also change the `main` in `refs/heads/main`
+#         # below accordingly.
+#         # if: github.ref == 'refs/heads/main'
+#         with:
+#           github_token: ${{ secrets.GITHUB_TOKEN }}
+#           publish_dir: ./docs
+
   publish-js-package:
     needs: [run-py-linter, run-py-tests, run-js-tests]
     runs-on: ubuntu-latest

.gitignore

@@ -53,7 +53,11 @@ dist/
 .nyc_output
 .eslintcache
 
-.husky
+.husky/*
+!.husky/pre-commit
 
 schemas.js
 schema.js
+
+# Static assets, if any
+site

.husky/pre-commit

@@ -0,0 +1,14 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+SRC_PATTERN="\.json$"
+if git diff --cached --name-only | grep --quiet -E "$SRC_PATTERN"
+then
+  echo "JSON assets changed. Running build scripts."
+  echo "Re-building JS assets."
+  npm run build:assets
+  echo "Re-building Python assets: requires virtual environment and dependencies (pip install .'[tests]')."
+  python build_schemas.py
+fi
+
+npx lint-staged

build_schemas.js

@@ -4,10 +4,12 @@
  * downstream consumption to avoid FS calls in the browser.
  */
 const fs = require("fs");
+const path = require("path");
+const mergeAllOf = require("json-schema-merge-allof");
 const { ESSE } = require("./lib/js/esse");
 
 const esse = new ESSE();
-const { schemas } = esse;
+const { schemas, wrappedExamples } = esse;
 const schema = esse.buildGlobalSchema();
 
 fs.writeFileSync(
@@ -17,3 +19,30 @@ fs.writeFileSync(
 );
 
 fs.writeFileSync("./schema.js", "module.exports = " + JSON.stringify(schema), "utf8");
+
+if (process.env.BUILD_DOCS !== "true") {
+    process.exit(0);
+}
+schemas.forEach((schema) => {
+    if (!process.env.SKIP_MERGE_ALLOF === "true") {
+        schema = mergeAllOf(schema, {resolvers: {defaultResolver: mergeAllOf.options.resolvers.title}});
+    }
+    id_as_path = schema["$id"].replace("-", "_");
+    full_path = `./docs/js/schema/${id_as_path}.json`;
+    fs.mkdirSync(path.dirname(full_path), {recursive: true})
+    fs.writeFileSync(
+        full_path,
+        JSON.stringify(schema, null, 4),
+        "utf8",
+    );
+})
+wrappedExamples.forEach((example) => {
+    id_as_path = example["path"].replace("-", "_");
+    full_path = `./docs/js/example/${id_as_path}.json`;
+    fs.mkdirSync(path.dirname(full_path), {recursive: true})
+    fs.writeFileSync(
+        full_path,
+        JSON.stringify(example["data"], null, 4),
+        "utf8",
+    );
+})

build_schemas.py

@@ -13,7 +13,9 @@
 After that, check for the files `src/py/mat3ra/esse/data/schemas.py` and `src/py/mat3ra/esse/data/examples.py`.
 """
 import os
+import json
 import yaml
+from pathlib import Path
 
 from mat3ra.esse.utils import parse_include_reference_statements_by_dir
 
@@ -39,3 +41,21 @@
         + f"RESULTS = json.loads(json.dumps({RESULTS}))\n"
     )
     f.write(content)
+
+if os.environ.get("BUILD_DOCS") != "true":
+    exit(0)
+
+for schema in SCHEMAS:
+    id_as_path = schema["$id"].replace("-", "_")
+    full_path = os.path.join(TOP_DIR, "docs/py/schema", id_as_path + ".json")
+    Path(full_path).parent.mkdir(parents=True, exist_ok=True)
+    with open(full_path, "w") as f:
+        f.write(json.dumps(schema, sort_keys=True, indent=4, separators=(",", ": ")))
+
+for example in EXAMPLES:
+    id_as_path = example["path"].replace("-", "_")
+    full_path = os.path.join(TOP_DIR, "docs/py/example", id_as_path + ".json")
+    Path(full_path).parent.mkdir(parents=True, exist_ok=True)
+    with open(full_path, "w") as f:
+        f.write(json.dumps(schema, sort_keys=True, indent=4, separators=(",", ": ")))
+

package.json

@@ -5,6 +5,8 @@
     "scripts": {
         "prepublishOnly": "rm -rf lib; npm run transpile",
         "transpile": "mkdir -p lib; babel src/js --out-dir lib/js; node build_schemas.js",
+        "build:assets": "node build_schemas.js",
+        "build:assets-with-docs": "BUILD_DOCS=true node build_schemas.js",
         "test": "nyc --reporter=text mocha --bail --require @babel/register tests/js/",
         "lint": "eslint src/js && prettier --write src/js",
         "lint:fix": "eslint --fix --cache src/js && prettier --write src/js",
@@ -46,16 +48,17 @@
     },
     "license": "Apache-2.0",
     "dependencies": {
-        "ajv": "^4.1.7",
         "@babel/cli": "7.16.0",
         "@babel/core": "7.16.0",
-        "@babel/register": "7.16.0",
-        "@babel/preset-env": "7.16.4",
         "@babel/eslint-parser": "7.16.3",
+        "@babel/preset-env": "7.16.4",
+        "@babel/register": "7.16.0",
+        "ajv": "^4.1.7",
         "file": "^0.2.2",
+        "js-yaml": "^4.1.0",
         "json-schema-deref-sync": "0.14.0",
-        "lodash": "4.17.21",
-        "js-yaml": "^4.1.0"
+        "json-schema-merge-allof": "^0.8.1",
+        "lodash": "4.17.21"
     },
     "engines": {
         "node": ">=12.0.0"

src/js/esse/index.js

@@ -5,12 +5,13 @@ import { EXAMPLES_DIR, SCHEMAS_DIR } from "./settings";
 import { parseIncludeReferenceStatementsByDir } from "./utils";
 
 const SCHEMAS = parseIncludeReferenceStatementsByDir(SCHEMAS_DIR);
-const EXAMPLES = parseIncludeReferenceStatementsByDir(EXAMPLES_DIR);
+const EXAMPLES = parseIncludeReferenceStatementsByDir(EXAMPLES_DIR, true);
 
 export class ESSE {
     constructor() {
         this.schemas = SCHEMAS;
-        this.examples = EXAMPLES;
+        this.wrappedExamples = EXAMPLES;
+        this.examples = EXAMPLES.map((example) => example.data);
     }
 
     getSchemaById(schemaId) {

src/js/esse/utils.js

@@ -24,14 +24,24 @@
  * Resolves `include` and `$ref` statements for all the JSON files inside a given directory.
  * @param dirPath {String} directory to parse.
  */
-export function parseIncludeReferenceStatementsByDir(dirPath) {
+export function parseIncludeReferenceStatementsByDir(dirPath, wrapInDataAndPath=false) {
     const data = [];
+    const topDir = path.resolve(__dirname, "../../../");
     file.walkSync(dirPath, (dirPath_, dirs_, files_) => {
         files_.forEach((file_) => {
             const filePath = path.join(dirPath_, file_);
             if (filePath.endsWith(".json")) {
-                // ignore files like .DS_Store
-                data.push(parseIncludeReferenceStatements(filePath));
+                const config = parseIncludeReferenceStatements(filePath);
+                if (wrapInDataAndPath) {
+                    const _path = path.join(
+                        // remove leading slashes and "example" from path
+                        path.dirname(filePath).replace(path.join(topDir, "example"), "").replace(/^\/+/, ''),
+                        path.basename(filePath).replace(".json", "")
+                    );
+                    data.push({ data: config, path: _path });
+                } else {
+                    data.push(config);
+                }
             }
         });
     });