Add JSON5 parsing capabilities for integration configs (#1732)

* update live mv table name (#1725) Signed-off-by: YANGDB <yang.db.dev@gmail.com> Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> * Integrated JSON5 Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> * json_data_adaptor Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> * Added tests for JSON5 Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> * Updated integration configs to json5 Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> * Revert "update live mv table name (#1725)" This reverts commit 71597f5. Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> * Updated filename Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> * Delete t --count 5c93001..fe64d02 Signed-off-by: Devesh Rahatekar <79015420+devesh-2002@users.noreply.github.com> * Updated some integrations Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> --------- Signed-off-by: YANGDB <yang.db.dev@gmail.com> Signed-off-by: dev241202@gmail.com <dev241202@gmail.com> Signed-off-by: Devesh Rahatekar <79015420+devesh-2002@users.noreply.github.com> Signed-off-by: Simeon Widdis <sawiddis@gmail.com> Co-authored-by: YANGDB <yang.db.dev@gmail.com> Co-authored-by: Simeon Widdis <sawiddis@gmail.com> (cherry picked from commit 9532b7d) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
opensearch-project · May 2, 2024 · 427fdc0 · 427fdc0
1 parent ee5b3f0
commit 427fdc0
Show file tree

Hide file tree

Showing 5 changed files with 90 additions and 159 deletions.
diff --git a/package.json b/package.json
@@ -28,6 +28,7 @@
  "antlr4ts": "^0.5.0-alpha.4",
  "cypress-multi-reporters": "^1.6.3",
  "cypress-parallel": "^0.13.0",
+ "json5": "^2.2.3",
  "mime": "^3.0.0",
  "mocha": "10.1.0",
  "performance-now": "^2.1.0",

diff --git a/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json b/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json
@@ -1,90 +1,52 @@
 {
- "name": "apache",
- "version": "1.0.0",
- "displayName": "Apache Access Logs",
- "description": "Insights on applications using the standard Apache access log format.",
- "license": "Apache-2.0",
- "type": "logs_apache",
- "labels": ["Observability", "Logs", "Flint S3"],
- "author": "OpenSearch",
- "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/apache/info",
- "workflows": [
- {
- "name": "queries",
- "label": "Queries (recommended)",
- "description": "Tables and pre-written queries for quickly getting insights on your data.",
- "enabled_by_default": true
- },
- {
- "name": "dashboards",
- "label": "Dashboards & Visualizations",
- "description": "Dashboards and indices that enable you to easily visualize important metrics.",
- "enabled_by_default": false
+ name: "apache",
+ version: "1.0.0",
+ displayName: "Apache Access Logs",
+ description: "Insights on applications using the standard Apache access log format.",
+ license: "Apache-2.0",
+ type: "logs_apache",
+ labels: ["Observability", "Logs", "Flint S3"],
+ author: "OpenSearch",
+ sourceUrl: "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/apache/info",
+ workflows: [
+ {
+ name: "queries",
+ label: "Queries (recommended)",
+ description: "Tables and pre-written queries for quickly getting insights on your data.",
+ enabled_by_default: true
+ },
+ {
+ name: "dashboards",
+ label: "Dashboards & Visualizations",
+ description: "Dashboards and indices that enable you to easily visualize important metrics.",
+ enabled_by_default: false
  }
  ],
- "statics": {
- "logo": {
- "annotation": "Apache Logo",
- "path": "logo.png"
+ statics: {
+ logo: {
+ annotation: "Apache Logo",
+ path: "logo.png"
  },
- "gallery": [
+ gallery: [
  {
- "annotation": "Apache Dashboard",
- "path": "dashboard1.png"
+ annotation: "Apache Dashboard",
+ path: "dashboard1.png"
  }
  ]
  },
- "components": [
- {
- "name": "communication",
- "version": "1.0.0"
- },
- {
- "name": "http",
- "version": "1.0.0"
- },
- {
- "name": "logs_apache",
- "version": "1.0.0"
- }
+ components: [
+ { name: "communication", version: "1.0.0" },
+ { name: "http", version: "1.0.0" },
+ { name: "logs_apache", version: "1.0.0" }
  ],
- "assets": [
- {
- "name": "apache",
- "version": "1.0.0",
- "extension": "ndjson",
- "type": "savedObjectBundle",
- "workflows": ["dashboards"]
- },
- {
- "name": "example_queries",
- "version": "1.0.0",
- "extension": "ndjson",
- "type": "savedObjectBundle",
- "workflows": ["queries"]
- },
- {
- "name": "create_table",
- "version": "1.0.0",
- "extension": "sql",
- "type": "query"
- },
- {
- "name": "create_skipping_index",
- "version": "1.0.0",
- "extension": "sql",
- "type": "query",
- "workflows": ["queries"]
- },
- {
- "name": "create_mv",
- "version": "1.0.0",
- "extension": "sql",
- "type": "query",
- "workflows": ["dashboards"]
- }
+ assets: [
+ { name: "apache", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["dashboards"] },
+ { name: "example_queries", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["queries"] },
+ { name: "create_table", version: "1.0.0", extension: "sql", type: "query" },
+ { name: "create_skipping_index", version: "1.0.0", extension: "sql", type: "query", workflows: ["queries"] },
+ { name: "create_mv", version: "1.0.0", extension: "sql", type: "query", workflows: ["dashboards"] }
  ],
- "sampleData": {
- "path": "sample.json"
+ sampleData: {
+ path: "sample.json"
  }
 }
diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json
@@ -1,94 +1,56 @@
 {
- "name": "nginx",
- "version": "1.0.0",
- "displayName": "Nginx",
- "description": "Analyze Nginx access logs.",
- "license": "Apache-2.0",
- "type": "logs",
- "labels": ["Observability", "Logs", "Flint S3"],
- "author": "OpenSearch",
- "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info",
- "workflows": [
- {
- "name": "queries",
- "label": "Queries (recommended)",
- "description": "Tables and pre-written queries for quickly getting insights on your data.",
- "enabled_by_default": true
- },
- {
- "name": "dashboards",
- "label": "Dashboards & Visualizations",
- "description": "Dashboards and indices that enable you to easily visualize important metrics.",
- "enabled_by_default": false
+ name: "nginx",
+ version: "1.0.0",
+ displayName: "Nginx",
+ description: "Analyze Nginx access logs.",
+ license: "Apache-2.0",
+ type: "logs",
+ labels: ["Observability", "Logs", "Flint S3"],
+ author: "OpenSearch",
+ sourceUrl: "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info",
+ workflows: [
+ {
+ name: "queries",
+ label: "Queries (recommended)",
+ description: "Tables and pre-written queries for quickly getting insights on your data.",
+ enabled_by_default: true
+ },
+ {
+ name: "dashboards",
+ label: "Dashboards & Visualizations",
+ description: "Dashboards and indices that enable you to easily visualize important metrics.",
+ enabled_by_default: false
  }
  ],
- "statics": {
- "logo": {
- "annotation": "NginX Logo",
- "path": "logo.svg"
+ statics: {
+ logo: {
+ annotation: "NginX Logo",
+ path: "logo.svg"
  },
- "gallery": [
+ gallery: [
  {
- "annotation": "NginX Dashboard",
- "path": "dashboard1.png"
+ annotation: "NginX Dashboard",
+ path: "dashboard1.png"
  },
  {
- "annotation": "NginX Dashboard view",
- "path": "dashboard2.png"
+ annotation: "NginX Dashboard view",
+ path: "dashboard2.png"
  }
  ]
  },
- "components": [
- {
- "name": "communication",
- "version": "1.0.0"
- },
- {
- "name": "http",
- "version": "1.0.0"
- },
- {
- "name": "logs",
- "version": "1.0.0"
- }
+ components: [
+ { name: "communication", version: "1.0.0" },
+ { name: "http", version: "1.0.0" },
+ { name: "logs", version: "1.0.0" }
  ],
- "assets": [
- {
- "name": "nginx",
- "version": "1.0.0",
- "extension": "ndjson",
- "type": "savedObjectBundle",
- "workflows": ["dashboards"]
- },
- {
- "name": "create_table",
- "version": "1.0.0",
- "extension": "sql",
- "type": "query"
- },
- {
- "name": "create_skipping_index",
- "version": "1.0.0",
- "extension": "sql",
- "type": "query",
- "workflows": ["queries"]
- },
- {
- "name": "create_mv",
- "version": "1.0.0",
- "extension": "sql",
- "type": "query",
- "workflows": ["dashboards"]
- },
- {
- "name": "example_queries",
- "version": "1.0.0",
- "extension": "ndjson",
- "type": "savedObjectBundle",
- "workflows": ["queries"]
- }
+ assets: [
+ { name: "nginx", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["dashboards"] },
+ { name: "create_table", version: "1.0.0", extension: "sql", type: "query" },
+ { name: "create_skipping_index", version: "1.0.0", extension: "sql", type: "query", workflows: ["queries"] },
+ { name: "create_mv", version: "1.0.0", extension: "sql", type: "query", workflows: ["dashboards"] },
+ { name: "example_queries", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["queries"] }
  ],
- "sampleData": {
- "path": "sample.json"
+ sampleData: {
+ path: "sample.json"
  }
 }
diff --git a/server/adaptors/integrations/repository/fs_data_adaptor.ts b/server/adaptors/integrations/repository/fs_data_adaptor.ts
@@ -5,6 +5,7 @@
 
 import * as fs from 'fs/promises';
 import path from 'path';
+import JSON5 from 'json5';
 import { CatalogDataAdaptor, IntegrationPart } from './catalog_data_adaptor';
 import { tryParseNDJson } from './utils';
 
@@ -45,7 +46,7 @@ export class FileSystemDataAdaptor implements CatalogDataAdaptor {
  }
  // First try to parse as JSON, then NDJSON, then fail.
  try {
- const parsed = JSON.parse(content);
+ const parsed = JSON5.parse(content);
  return { ok: true, value: parsed };
  } catch (err) {
  const parsed = await tryParseNDJson(content);

diff --git a/yarn.lock b/yarn.lock
@@ -2012,6 +2012,11 @@ json-stringify-safe@~5.0.1:
  resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
  integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==
 
+json5@^2.2.3:
+ version "2.2.3"
+ resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283"
+ integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==
+
 jsonfile@^6.0.1:
  version "6.1.0"
  resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae"