Skip to content

Commit

Permalink
Add JSON5 parsing capabilities for integration configs (#1732)
Browse files Browse the repository at this point in the history
* update live mv table name (#1725)

Signed-off-by: YANGDB <yang.db.dev@gmail.com>
Signed-off-by: dev241202@gmail.com <dev241202@gmail.com>

* Integrated JSON5

Signed-off-by: dev241202@gmail.com <dev241202@gmail.com>

* json_data_adaptor

Signed-off-by: dev241202@gmail.com <dev241202@gmail.com>

* Added tests for JSON5

Signed-off-by: dev241202@gmail.com <dev241202@gmail.com>

* Updated integration configs to json5

Signed-off-by: dev241202@gmail.com <dev241202@gmail.com>

* Revert "update live mv table name (#1725)"

This reverts commit 71597f5.

Signed-off-by: dev241202@gmail.com <dev241202@gmail.com>

* Updated filename

Signed-off-by: dev241202@gmail.com <dev241202@gmail.com>

* Delete t --count 5c93001..fe64d02

Signed-off-by: Devesh Rahatekar <79015420+devesh-2002@users.noreply.github.com>

* Updated some integrations

Signed-off-by: dev241202@gmail.com <dev241202@gmail.com>

---------

Signed-off-by: YANGDB <yang.db.dev@gmail.com>
Signed-off-by: dev241202@gmail.com <dev241202@gmail.com>
Signed-off-by: Devesh Rahatekar <79015420+devesh-2002@users.noreply.github.com>
Signed-off-by: Simeon Widdis <sawiddis@gmail.com>
Co-authored-by: YANGDB <yang.db.dev@gmail.com>
Co-authored-by: Simeon Widdis <sawiddis@gmail.com>
  • Loading branch information
3 people authored May 2, 2024
1 parent d3ab3cd commit 9532b7d
Show file tree
Hide file tree
Showing 5 changed files with 90 additions and 159 deletions.
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
"antlr4ts": "^0.5.0-alpha.4",
"cypress-multi-reporters": "^1.6.3",
"cypress-parallel": "^0.13.0",
"json5": "^2.2.3",
"mime": "^3.0.0",
"mocha": "10.1.0",
"performance-now": "^2.1.0",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,90 +1,52 @@
{
"name": "apache",
"version": "1.0.0",
"displayName": "Apache Access Logs",
"description": "Insights on applications using the standard Apache access log format.",
"license": "Apache-2.0",
"type": "logs_apache",
"labels": ["Observability", "Logs", "Flint S3"],
"author": "OpenSearch",
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/apache/info",
"workflows": [
{
"name": "queries",
"label": "Queries (recommended)",
"description": "Tables and pre-written queries for quickly getting insights on your data.",
"enabled_by_default": true
},
{
"name": "dashboards",
"label": "Dashboards & Visualizations",
"description": "Dashboards and indices that enable you to easily visualize important metrics.",
"enabled_by_default": false
name: "apache",
version: "1.0.0",
displayName: "Apache Access Logs",
description: "Insights on applications using the standard Apache access log format.",
license: "Apache-2.0",
type: "logs_apache",
labels: ["Observability", "Logs", "Flint S3"],
author: "OpenSearch",
sourceUrl: "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/apache/info",
workflows: [
{
name: "queries",
label: "Queries (recommended)",
description: "Tables and pre-written queries for quickly getting insights on your data.",
enabled_by_default: true
},
{
name: "dashboards",
label: "Dashboards & Visualizations",
description: "Dashboards and indices that enable you to easily visualize important metrics.",
enabled_by_default: false
}
],
"statics": {
"logo": {
"annotation": "Apache Logo",
"path": "logo.png"
statics: {
logo: {
annotation: "Apache Logo",
path: "logo.png"
},
"gallery": [
gallery: [
{
"annotation": "Apache Dashboard",
"path": "dashboard1.png"
annotation: "Apache Dashboard",
path: "dashboard1.png"
}
]
},
"components": [
{
"name": "communication",
"version": "1.0.0"
},
{
"name": "http",
"version": "1.0.0"
},
{
"name": "logs_apache",
"version": "1.0.0"
}
components: [
{ name: "communication", version: "1.0.0" },
{ name: "http", version: "1.0.0" },
{ name: "logs_apache", version: "1.0.0" }
],
"assets": [
{
"name": "apache",
"version": "1.0.0",
"extension": "ndjson",
"type": "savedObjectBundle",
"workflows": ["dashboards"]
},
{
"name": "example_queries",
"version": "1.0.0",
"extension": "ndjson",
"type": "savedObjectBundle",
"workflows": ["queries"]
},
{
"name": "create_table",
"version": "1.0.0",
"extension": "sql",
"type": "query"
},
{
"name": "create_skipping_index",
"version": "1.0.0",
"extension": "sql",
"type": "query",
"workflows": ["queries"]
},
{
"name": "create_mv",
"version": "1.0.0",
"extension": "sql",
"type": "query",
"workflows": ["dashboards"]
}
assets: [
{ name: "apache", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["dashboards"] },
{ name: "example_queries", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["queries"] },
{ name: "create_table", version: "1.0.0", extension: "sql", type: "query" },
{ name: "create_skipping_index", version: "1.0.0", extension: "sql", type: "query", workflows: ["queries"] },
{ name: "create_mv", version: "1.0.0", extension: "sql", type: "query", workflows: ["dashboards"] }
],
"sampleData": {
"path": "sample.json"
sampleData: {
path: "sample.json"
}
}
Original file line number Diff line number Diff line change
@@ -1,94 +1,56 @@
{
"name": "nginx",
"version": "1.0.0",
"displayName": "Nginx",
"description": "Analyze Nginx access logs.",
"license": "Apache-2.0",
"type": "logs",
"labels": ["Observability", "Logs", "Flint S3"],
"author": "OpenSearch",
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info",
"workflows": [
{
"name": "queries",
"label": "Queries (recommended)",
"description": "Tables and pre-written queries for quickly getting insights on your data.",
"enabled_by_default": true
},
{
"name": "dashboards",
"label": "Dashboards & Visualizations",
"description": "Dashboards and indices that enable you to easily visualize important metrics.",
"enabled_by_default": false
name: "nginx",
version: "1.0.0",
displayName: "Nginx",
description: "Analyze Nginx access logs.",
license: "Apache-2.0",
type: "logs",
labels: ["Observability", "Logs", "Flint S3"],
author: "OpenSearch",
sourceUrl: "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info",
workflows: [
{
name: "queries",
label: "Queries (recommended)",
description: "Tables and pre-written queries for quickly getting insights on your data.",
enabled_by_default: true
},
{
name: "dashboards",
label: "Dashboards & Visualizations",
description: "Dashboards and indices that enable you to easily visualize important metrics.",
enabled_by_default: false
}
],
"statics": {
"logo": {
"annotation": "NginX Logo",
"path": "logo.svg"
statics: {
logo: {
annotation: "NginX Logo",
path: "logo.svg"
},
"gallery": [
gallery: [
{
"annotation": "NginX Dashboard",
"path": "dashboard1.png"
annotation: "NginX Dashboard",
path: "dashboard1.png"
},
{
"annotation": "NginX Dashboard view",
"path": "dashboard2.png"
annotation: "NginX Dashboard view",
path: "dashboard2.png"
}
]
},
"components": [
{
"name": "communication",
"version": "1.0.0"
},
{
"name": "http",
"version": "1.0.0"
},
{
"name": "logs",
"version": "1.0.0"
}
components: [
{ name: "communication", version: "1.0.0" },
{ name: "http", version: "1.0.0" },
{ name: "logs", version: "1.0.0" }
],
"assets": [
{
"name": "nginx",
"version": "1.0.0",
"extension": "ndjson",
"type": "savedObjectBundle",
"workflows": ["dashboards"]
},
{
"name": "create_table",
"version": "1.0.0",
"extension": "sql",
"type": "query"
},
{
"name": "create_skipping_index",
"version": "1.0.0",
"extension": "sql",
"type": "query",
"workflows": ["queries"]
},
{
"name": "create_mv",
"version": "1.0.0",
"extension": "sql",
"type": "query",
"workflows": ["dashboards"]
},
{
"name": "example_queries",
"version": "1.0.0",
"extension": "ndjson",
"type": "savedObjectBundle",
"workflows": ["queries"]
}
assets: [
{ name: "nginx", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["dashboards"] },
{ name: "create_table", version: "1.0.0", extension: "sql", type: "query" },
{ name: "create_skipping_index", version: "1.0.0", extension: "sql", type: "query", workflows: ["queries"] },
{ name: "create_mv", version: "1.0.0", extension: "sql", type: "query", workflows: ["dashboards"] },
{ name: "example_queries", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["queries"] }
],
"sampleData": {
"path": "sample.json"
sampleData: {
path: "sample.json"
}
}
3 changes: 2 additions & 1 deletion server/adaptors/integrations/repository/fs_data_adaptor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import * as fs from 'fs/promises';
import path from 'path';
import JSON5 from 'json5';
import { CatalogDataAdaptor, IntegrationPart } from './catalog_data_adaptor';
import { tryParseNDJson } from './utils';

Expand Down Expand Up @@ -45,7 +46,7 @@ export class FileSystemDataAdaptor implements CatalogDataAdaptor {
}
// First try to parse as JSON, then NDJSON, then fail.
try {
const parsed = JSON.parse(content);
const parsed = JSON5.parse(content);
return { ok: true, value: parsed };
} catch (err) {
const parsed = await tryParseNDJson(content);
Expand Down
5 changes: 5 additions & 0 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -2007,6 +2007,11 @@ json-stringify-safe@~5.0.1:
resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==

json5@^2.2.3:
version "2.2.3"
resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283"
integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==

jsonfile@^6.0.1:
version "6.1.0"
resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae"
Expand Down

0 comments on commit 9532b7d

Please sign in to comment.