Add JSON5 parsing capabilities for integration configs (#1732)

* update live mv table name (#1725) Signed-off-by: YANGDB <yang.db.dev@gmail.com> Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> * Integrated JSON5 Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> * json_data_adaptor Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> * Added tests for JSON5 Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> * Updated integration configs to json5 Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> * Revert "update live mv table name (#1725)" This reverts commit 71597f5. Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> * Updated filename Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> * Delete t --count 5c93001..fe64d02 Signed-off-by: Devesh Rahatekar <79015420+devesh-2002@users.noreply.github.com> * Updated some integrations Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> --------- Signed-off-by: YANGDB <yang.db.dev@gmail.com> Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> Signed-off-by: Devesh Rahatekar <79015420+devesh-2002@users.noreply.github.com> Signed-off-by: Simeon Widdis <sawiddis@gmail.com> Co-authored-by: YANGDB <yang.db.dev@gmail.com> Co-authored-by: Simeon Widdis <sawiddis@gmail.com>
opensearch-project · May 2, 2024 · 9532b7d · 9532b7d
1 parent d3ab3cd
commit 9532b7d
Show file tree

Hide file tree

Showing 5 changed files with 90 additions and 159 deletions.
diff --git a/package.json b/package.json
@@ -33,6 +33,7 @@
     "antlr4ts": "^0.5.0-alpha.4",
     "cypress-multi-reporters": "^1.6.3",
     "cypress-parallel": "^0.13.0",
+    "json5": "^2.2.3",
     "mime": "^3.0.0",
     "mocha": "10.1.0",
     "performance-now": "^2.1.0",

diff --git a/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json b/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json
@@ -1,90 +1,52 @@
 {
-  "name": "apache",
-  "version": "1.0.0",
-  "displayName": "Apache Access Logs",
-  "description": "Insights on applications using the standard Apache access log format.",
-  "license": "Apache-2.0",
-  "type": "logs_apache",
-  "labels": ["Observability", "Logs", "Flint S3"],
-  "author": "OpenSearch",
-  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/apache/info",
-  "workflows": [
-    {
-      "name": "queries",
-      "label": "Queries (recommended)",
-      "description": "Tables and pre-written queries for quickly getting insights on your data.",
-      "enabled_by_default": true
-    },
-    {
-      "name": "dashboards",
-      "label": "Dashboards & Visualizations",
-      "description": "Dashboards and indices that enable you to easily visualize important metrics.",
-      "enabled_by_default": false
+  name: "apache",
+  version: "1.0.0",
+  displayName: "Apache Access Logs",
+  description: "Insights on applications using the standard Apache access log format.",
+  license: "Apache-2.0",
+  type: "logs_apache",
+  labels: ["Observability", "Logs", "Flint S3"],
+  author: "OpenSearch",
+  sourceUrl: "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/apache/info",
+  workflows: [
+    {
+      name: "queries",
+      label: "Queries (recommended)",
+      description: "Tables and pre-written queries for quickly getting insights on your data.",
+      enabled_by_default: true
+    },
+    {
+      name: "dashboards",
+      label: "Dashboards & Visualizations",
+      description: "Dashboards and indices that enable you to easily visualize important metrics.",
+      enabled_by_default: false
     }
   ],
-  "statics": {
-    "logo": {
-      "annotation": "Apache Logo",
-      "path": "logo.png"
+  statics: {
+    logo: {
+      annotation: "Apache Logo",
+      path: "logo.png"
     },
-    "gallery": [
+    gallery: [
       {
-        "annotation": "Apache Dashboard",
-        "path": "dashboard1.png"
+        annotation: "Apache Dashboard",
+        path: "dashboard1.png"
       }
     ]
   },
-  "components": [
-    {
-      "name": "communication",
-      "version": "1.0.0"
-    },
-    {
-      "name": "http",
-      "version": "1.0.0"
-    },
-    {
-      "name": "logs_apache",
-      "version": "1.0.0"
-    }
+  components: [
+    { name: "communication", version: "1.0.0" },
+    { name: "http", version: "1.0.0" },
+    { name: "logs_apache", version: "1.0.0" }
   ],
-  "assets": [
-    {
-      "name": "apache",
-      "version": "1.0.0",
-      "extension": "ndjson",
-      "type": "savedObjectBundle",
-      "workflows": ["dashboards"]
-    },
-    {
-      "name": "example_queries",
-      "version": "1.0.0",
-      "extension": "ndjson",
-      "type": "savedObjectBundle",
-      "workflows": ["queries"]
-    },
-    {
-      "name": "create_table",
-      "version": "1.0.0",
-      "extension": "sql",
-      "type": "query"
-    },
-    {
-      "name": "create_skipping_index",
-      "version": "1.0.0",
-      "extension": "sql",
-      "type": "query",
-      "workflows": ["queries"]
-    },
-    {
-      "name": "create_mv",
-      "version": "1.0.0",
-      "extension": "sql",
-      "type": "query",
-      "workflows": ["dashboards"]
-    }
+  assets: [
+    { name: "apache", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["dashboards"] },
+    { name: "example_queries", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["queries"] },
+    { name: "create_table", version: "1.0.0", extension: "sql", type: "query" },
+    { name: "create_skipping_index", version: "1.0.0", extension: "sql", type: "query", workflows: ["queries"] },
+    { name: "create_mv", version: "1.0.0", extension: "sql", type: "query", workflows: ["dashboards"] }
   ],
-  "sampleData": {
-    "path": "sample.json"
+  sampleData: {
+    path: "sample.json"
   }
 }
diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json
@@ -1,94 +1,56 @@
 {
-  "name": "nginx",
-  "version": "1.0.0",
-  "displayName": "Nginx",
-  "description": "Analyze Nginx access logs.",
-  "license": "Apache-2.0",
-  "type": "logs",
-  "labels": ["Observability", "Logs", "Flint S3"],
-  "author": "OpenSearch",
-  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info",
-  "workflows": [
-    {
-      "name": "queries",
-      "label": "Queries (recommended)",
-      "description": "Tables and pre-written queries for quickly getting insights on your data.",
-      "enabled_by_default": true
-    },
-    {
-      "name": "dashboards",
-      "label": "Dashboards & Visualizations",
-      "description": "Dashboards and indices that enable you to easily visualize important metrics.",
-      "enabled_by_default": false
+  name: "nginx",
+  version: "1.0.0",
+  displayName: "Nginx",
+  description: "Analyze Nginx access logs.",
+  license: "Apache-2.0",
+  type: "logs",
+  labels: ["Observability", "Logs", "Flint S3"],
+  author: "OpenSearch",
+  sourceUrl: "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info",
+  workflows: [
+    {
+      name: "queries",
+      label: "Queries (recommended)",
+      description: "Tables and pre-written queries for quickly getting insights on your data.",
+      enabled_by_default: true
+    },
+    {
+      name: "dashboards",
+      label: "Dashboards & Visualizations",
+      description: "Dashboards and indices that enable you to easily visualize important metrics.",
+      enabled_by_default: false
     }
   ],
-  "statics": {
-    "logo": {
-      "annotation": "NginX Logo",
-      "path": "logo.svg"
+  statics: {
+    logo: {
+      annotation: "NginX Logo",
+      path: "logo.svg"
     },
-    "gallery": [
+    gallery: [
       {
-        "annotation": "NginX Dashboard",
-        "path": "dashboard1.png"
+        annotation: "NginX Dashboard",
+        path: "dashboard1.png"
       },
       {
-        "annotation": "NginX Dashboard view",
-        "path": "dashboard2.png"
+        annotation: "NginX Dashboard view",
+        path: "dashboard2.png"
       }
     ]
   },
-  "components": [
-    {
-      "name": "communication",
-      "version": "1.0.0"
-    },
-    {
-      "name": "http",
-      "version": "1.0.0"
-    },
-    {
-      "name": "logs",
-      "version": "1.0.0"
-    }
+  components: [
+    { name: "communication", version: "1.0.0" },
+    { name: "http", version: "1.0.0" },
+    { name: "logs", version: "1.0.0" }
   ],
-  "assets": [
-    {
-      "name": "nginx",
-      "version": "1.0.0",
-      "extension": "ndjson",
-      "type": "savedObjectBundle",
-      "workflows": ["dashboards"]
-    },
-    {
-      "name": "create_table",
-      "version": "1.0.0",
-      "extension": "sql",
-      "type": "query"
-    },
-    {
-      "name": "create_skipping_index",
-      "version": "1.0.0",
-      "extension": "sql",
-      "type": "query",
-      "workflows": ["queries"]
-    },
-    {
-      "name": "create_mv",
-      "version": "1.0.0",
-      "extension": "sql",
-      "type": "query",
-      "workflows": ["dashboards"]
-    },
-    {
-      "name": "example_queries",
-      "version": "1.0.0",
-      "extension": "ndjson",
-      "type": "savedObjectBundle",
-      "workflows": ["queries"]
-    }
+  assets: [
+    { name: "nginx", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["dashboards"] },
+    { name: "create_table", version: "1.0.0", extension: "sql", type: "query" },
+    { name: "create_skipping_index", version: "1.0.0", extension: "sql", type: "query", workflows: ["queries"] },
+    { name: "create_mv", version: "1.0.0", extension: "sql", type: "query", workflows: ["dashboards"] },
+    { name: "example_queries", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["queries"] }
   ],
-  "sampleData": {
-    "path": "sample.json"
+  sampleData: {
+    path: "sample.json"
   }
 }
diff --git a/server/adaptors/integrations/repository/fs_data_adaptor.ts b/server/adaptors/integrations/repository/fs_data_adaptor.ts
@@ -5,6 +5,7 @@
 
 import * as fs from 'fs/promises';
 import path from 'path';
+import JSON5 from 'json5';
 import { CatalogDataAdaptor, IntegrationPart } from './catalog_data_adaptor';
 import { tryParseNDJson } from './utils';
 
@@ -45,7 +46,7 @@ export class FileSystemDataAdaptor implements CatalogDataAdaptor {
     }
     // First try to parse as JSON, then NDJSON, then fail.
     try {
-      const parsed = JSON.parse(content);
+      const parsed = JSON5.parse(content);
       return { ok: true, value: parsed };
     } catch (err) {
       const parsed = await tryParseNDJson(content);

diff --git a/yarn.lock b/yarn.lock
@@ -2007,6 +2007,11 @@ json-stringify-safe@~5.0.1:
   resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
   integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==
 
+json5@^2.2.3:
+  version "2.2.3"
+  resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283"
+  integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==
+
 jsonfile@^6.0.1:
   version "6.1.0"
   resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae"