diff --git a/server/adaptors/integrations/__data__/repository/amazon_cloudfront/amazon_cloudfront-1.0.0.json b/server/adaptors/integrations/__data__/repository/amazon_cloudfront/amazon_cloudfront-1.1.0.json similarity index 91% rename from server/adaptors/integrations/__data__/repository/amazon_cloudfront/amazon_cloudfront-1.0.0.json rename to server/adaptors/integrations/__data__/repository/amazon_cloudfront/amazon_cloudfront-1.1.0.json index 16ae0900c..4c8c5f99b 100644 --- a/server/adaptors/integrations/__data__/repository/amazon_cloudfront/amazon_cloudfront-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/amazon_cloudfront/amazon_cloudfront-1.1.0.json @@ -1,13 +1,13 @@ { "name": "amazon_cloudfront", - "version": "1.0.0", + "version": "1.1.0", "displayName": "Amazon CloudFront", "description": "Analyze access logs for Amazon CloudFront.", "license": "Apache-2.0", "type": "logs-aws_cloudfront", "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"], "author": "OpenSearch", - "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudfront/info", + "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/amazon_cloudfront-1.0.0", "statics": { "logo": { "annotation": "CloudFront Logo", diff --git a/server/adaptors/integrations/__data__/repository/amazon_cloudfront/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/amazon_cloudfront/assets/create_table-1.0.0.sql index d6345312c..a217b502b 100644 --- a/server/adaptors/integrations/__data__/repository/amazon_cloudfront/assets/create_table-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/amazon_cloudfront/assets/create_table-1.0.0.sql @@ -35,5 +35,8 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( ) USING csv LOCATION '{s3_bucket_location}' -OPTIONS ( sep='\t' ) +OPTIONS ( + sep='\t', + recursiveFileLookup='true' +) TBLPROPERTIES ( 'skip.header.line.count'='2' ) diff --git a/server/adaptors/integrations/__data__/repository/amazon_elb/amazon_elb-1.0.0.json b/server/adaptors/integrations/__data__/repository/amazon_elb/amazon_elb-1.0.0.json index 8be2d5919..d1f9aa87b 100644 --- a/server/adaptors/integrations/__data__/repository/amazon_elb/amazon_elb-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/amazon_elb/amazon_elb-1.0.0.json @@ -7,7 +7,7 @@ "type": "logs_elb", "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"], "author": "OpenSearch", - "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_elb/info", + "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/amazon_elb-1.0.0", "workflows": [ { "name": "queries", diff --git a/server/adaptors/integrations/__data__/repository/amazon_s3/amazon_s3-1.0.0.json b/server/adaptors/integrations/__data__/repository/amazon_s3/amazon_s3-1.1.0.json similarity index 91% rename from server/adaptors/integrations/__data__/repository/amazon_s3/amazon_s3-1.0.0.json rename to server/adaptors/integrations/__data__/repository/amazon_s3/amazon_s3-1.1.0.json index 4e1673420..7f1e58a5c 100644 --- a/server/adaptors/integrations/__data__/repository/amazon_s3/amazon_s3-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/amazon_s3/amazon_s3-1.1.0.json @@ -1,13 +1,13 @@ { "name": "amazon_s3", - "version": "1.0.0", + "version": "1.1.0", "displayName": "Amazon S3", "description": "Analyze Amazon S3 server logs.", "license": "Apache-2.0", "type": "logs_s3", "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"], "author": "OpenSearch", - "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_s3/info", + "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/amazon_s3_access-logs-1.1.0", "workflows": [ { "name": "queries", diff --git a/server/adaptors/integrations/__data__/repository/amazon_s3/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/amazon_s3/assets/create_table-1.0.0.sql index d2fcd7ba2..455fd3bc9 100644 --- a/server/adaptors/integrations/__data__/repository/amazon_s3/assets/create_table-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/amazon_s3/assets/create_table-1.0.0.sql @@ -27,6 +27,7 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( ) USING csv OPTIONS ( - sep=' ' + sep=' ', + recursiveFileLookup='true' ) LOCATION '{s3_bucket_location}' diff --git a/server/adaptors/integrations/__data__/repository/amazon_vpc_flow/amazon_vpc_flow-1.0.0.json b/server/adaptors/integrations/__data__/repository/amazon_vpc_flow/amazon_vpc_flow-1.1.0.json similarity index 93% rename from server/adaptors/integrations/__data__/repository/amazon_vpc_flow/amazon_vpc_flow-1.0.0.json rename to server/adaptors/integrations/__data__/repository/amazon_vpc_flow/amazon_vpc_flow-1.1.0.json index 577159bb5..868b7e17b 100644 --- a/server/adaptors/integrations/__data__/repository/amazon_vpc_flow/amazon_vpc_flow-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/amazon_vpc_flow/amazon_vpc_flow-1.1.0.json @@ -1,13 +1,13 @@ { "name": "amazon_vpc_flow", - "version": "1.0.0", + "version": "1.1.0", "displayName": "Amazon VPC", "description": "Monitor IP traffic with VPC Flow Logs.", "license": "Apache-2.0", "type": "logs_vpc", "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"], "author": "OpenSearch", - "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info", + "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/amazon_vpc_flow_1.1.0", "workflows": [ { "name": "queries", diff --git a/server/adaptors/integrations/__data__/repository/amazon_vpc_flow/assets/create_table_vpc_schema-1.0.0.sql b/server/adaptors/integrations/__data__/repository/amazon_vpc_flow/assets/create_table_vpc_schema-1.0.0.sql index 61fd60a4f..bab97cc53 100644 --- a/server/adaptors/integrations/__data__/repository/amazon_vpc_flow/assets/create_table_vpc_schema-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/amazon_vpc_flow/assets/create_table_vpc_schema-1.0.0.sql @@ -28,5 +28,6 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( pkt_dst_aws_service string, flow_direction string, traffic_path int -) USING parquet +) +USING parquet LOCATION '{s3_bucket_location}' diff --git a/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json b/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json index be251cab4..36186d924 100644 --- a/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json @@ -1,52 +1,52 @@ { - name: "apache", - version: "1.0.0", - displayName: "Apache Access Logs", - description: "Insights on applications using the standard Apache access log format.", - license: "Apache-2.0", - type: "logs_apache", - labels: ["Observability", "Logs", "Flint S3"], - author: "OpenSearch", - sourceUrl: "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/apache/info", - workflows: [ + "name": "apache", + "version": "1.0.0", + "displayName": "Apache Access Logs", + "description": "Insights on applications using the standard Apache access log format.", + "license": "Apache-2.0", + "type": "logs_apache", + "labels": ["Observability", "Logs", "Flint S3"], + "author": "OpenSearch", + "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/apache-1.0.0", + "workflows": [ { - name: "queries", - label: "Queries (recommended)", - description: "Tables and pre-written queries for quickly getting insights on your data.", - enabled_by_default: true + "name": "queries", + "label": "Queries (recommended)", + "description": "Tables and pre-written queries for quickly getting insights on your data.", + "enabled_by_default": true }, { - name: "dashboards", - label: "Dashboards & Visualizations", - description: "Dashboards and indices that enable you to easily visualize important metrics.", - enabled_by_default: false + "name": "dashboards", + "label": "Dashboards & Visualizations", + "description": "Dashboards and indices that enable you to easily visualize important metrics.", + "enabled_by_default": false } ], - statics: { - logo: { - annotation: "Apache Logo", - path: "logo.png" + "statics": { + "logo": { + "annotation": "Apache Logo", + "path": "logo.png" }, - gallery: [ + "gallery": [ { - annotation: "Apache Dashboard", - path: "dashboard1.png" + "annotation": "Apache Dashboard", + "path": "dashboard1.png" } ] }, - components: [ - { name: "communication", version: "1.0.0" }, - { name: "http", version: "1.0.0" }, - { name: "logs_apache", version: "1.0.0" } + "components": [ + { "name": "communication", "version": "1.0.0" }, + { "name": "http", "version": "1.0.0" }, + { "name": "logs_apache", "version": "1.0.0" } ], - assets: [ - { name: "apache", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["dashboards"] }, - { name: "example_queries", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["queries"] }, - { name: "create_table", version: "1.0.0", extension: "sql", type: "query" }, - { name: "create_skipping_index", version: "1.0.0", extension: "sql", type: "query", workflows: ["queries"] }, - { name: "create_mv", version: "1.0.0", extension: "sql", type: "query", workflows: ["dashboards"] } + "assets": [ + { "name": "apache", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["dashboards"] }, + { "name": "example_queries", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["queries"] }, + { "name": "create_table", "version": "1.0.0", "extension": "sql", "type": "query" }, + { "name": "create_skipping_index", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["queries"] }, + { "name": "create_mv", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["dashboards"] } ], - sampleData: { - path: "sample.json" + "sampleData": { + "path": "sample.json" } } diff --git a/server/adaptors/integrations/__data__/repository/apache/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/apache/assets/create_table-1.0.0.sql index 5f6ab8e7a..25f4a1213 100644 --- a/server/adaptors/integrations/__data__/repository/apache/assets/create_table-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/apache/assets/create_table-1.0.0.sql @@ -9,9 +9,11 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( body_bytes_sent INT, http_referer STRING, http_user_agent STRING -) USING csv -LOCATION '{s3_bucket_location}' +) +USING csv OPTIONS ( sep=' ', - nullValue='-' + nullValue='-', + recursiveFileLookup='true' ) +LOCATION '{s3_bucket_location}' diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.1.0.json similarity index 94% rename from server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json rename to server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.1.0.json index 173c0a9cb..150c18ebe 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.1.0.json @@ -1,13 +1,13 @@ { "name": "aws_cloudtrail", - "version": "1.0.0", + "version": "1.1.0", "displayName": "AWS CloudTrail", "description": "Analyze CloudTrail events.", "license": "Apache-2.0", "type": "logs-aws_cloudtrail", "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"], "author": "OpenSearch", - "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudtrail/info", + "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/aws_cloudtrail-1.1.0", "workflows": [ { "name": "dashboards", diff --git a/server/adaptors/integrations/__data__/repository/aws_waf/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_waf/assets/create_table-1.0.0.sql index f0178c814..56d4b814b 100644 --- a/server/adaptors/integrations/__data__/repository/aws_waf/assets/create_table-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_waf/assets/create_table-1.0.0.sql @@ -125,4 +125,7 @@ CREATE EXTERNAL TABLE {table_name} ( `ja3Fingerprint` string ) USING json +OPTIONS ( + recursiveFileLookup='true' +) LOCATION '{s3_bucket_location}' diff --git a/server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.1.0.json similarity index 92% rename from server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json rename to server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.1.0.json index f6f6eb6f6..db0787510 100644 --- a/server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.1.0.json @@ -1,13 +1,13 @@ { "name": "aws_waf", - "version": "1.0.0", + "version": "1.1.0", "displayName": "AWS Web Application Firewall", "description": "Analyze AWS WAF web ACL traffic.", "license": "Apache-2.0", "type": "logs_waf", "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"], "author": "OpenSearch", - "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_waf/info", + "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/aws_waf-1.0.0", "workflows": [ { "name": "queries", diff --git a/server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql index 37ba164bf..f03fa8f4d 100644 --- a/server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql @@ -1,7 +1,9 @@ CREATE EXTERNAL TABLE {table_name} ( record STRING -) USING csv -LOCATION '{s3_bucket_location}' +) +USING csv OPTIONS ( - sep='\0x1E' + sep='\0x1E', + recursiveFileLookup='true' ) +LOCATION '{s3_bucket_location}' diff --git a/server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.0.0.json b/server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.1.0.json similarity index 90% rename from server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.0.0.json rename to server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.1.0.json index 0ee1e9c9a..b081ac38d 100644 --- a/server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.1.0.json @@ -1,13 +1,13 @@ { "name": "haproxy", - "version": "1.0.0", + "version": "1.1.0", "displayName": "HAProxy", "description": "Analyze HAProxy access logs.", "license": "Apache-2.0", "type": "logs", "labels": ["Observability", "Logs", "Flint S3"], "author": "OpenSearch", - "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/haproxy/info", + "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/haproxy-1.0.0", "workflows": [ { "name": "queries", diff --git a/server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql index 38f719421..c3c1e0c42 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql @@ -10,9 +10,11 @@ CREATE EXTERNAL TABLE {table_name} ( http_referer STRING, http_user_agent STRING, gzip_ratio STRING -) USING csv -LOCATION '{s3_bucket_location}' +) +USING csv OPTIONS ( sep=' ', - nullValue='-' + nullValue='-', + recursiveFileLookup='true' ) +LOCATION '{s3_bucket_location}' diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json index f539bd3a0..c64a83b68 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json @@ -1,56 +1,56 @@ { - name: "nginx", - version: "1.0.0", - displayName: "Nginx", - description: "Analyze Nginx access logs.", - license: "Apache-2.0", - type: "logs", - labels: ["Observability", "Logs", "Flint S3"], - author: "OpenSearch", - sourceUrl: "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info", - workflows: [ + "name": "nginx", + "version": "1.0.0", + "displayName": "Nginx", + "description": "Analyze Nginx access logs.", + "license": "Apache-2.0", + "type": "logs", + "labels": ["Observability", "Logs", "Flint S3"], + "author": "OpenSearch", + "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/nginx-1.0.0", + "workflows": [ { - name: "queries", - label: "Queries (recommended)", - description: "Tables and pre-written queries for quickly getting insights on your data.", - enabled_by_default: true + "name": "queries", + "label": "Queries (recommended)", + "description": "Tables and pre-written queries for quickly getting insights on your data.", + "enabled_by_default": true }, { - name: "dashboards", - label: "Dashboards & Visualizations", - description: "Dashboards and indices that enable you to easily visualize important metrics.", - enabled_by_default: false + "name": "dashboards", + "label": "Dashboards & Visualizations", + "description": "Dashboards and indices that enable you to easily visualize important metrics.", + "enabled_by_default": false } ], - statics: { - logo: { - annotation: "NginX Logo", - path: "logo.svg" + "statics": { + "logo": { + "annotation": "NginX Logo", + "path": "logo.svg" }, - gallery: [ + "gallery": [ { - annotation: "NginX Dashboard", - path: "dashboard1.png" + "annotation": "NginX Dashboard", + "path": "dashboard1.png" }, { - annotation: "NginX Dashboard view", - path: "dashboard2.png" + "annotation": "NginX Dashboard view", + "path": "dashboard2.png" } ] }, - components: [ - { name: "communication", version: "1.0.0" }, - { name: "http", version: "1.0.0" }, - { name: "logs", version: "1.0.0" } + "components": [ + { "name": "communication", "version": "1.0.0" }, + { "name": "http", "version": "1.0.0" }, + { "name": "logs", "version": "1.0.0" } ], - assets: [ - { name: "nginx", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["dashboards"] }, - { name: "create_table", version: "1.0.0", extension: "sql", type: "query" }, - { name: "create_skipping_index", version: "1.0.0", extension: "sql", type: "query", workflows: ["queries"] }, - { name: "create_mv", version: "1.0.0", extension: "sql", type: "query", workflows: ["dashboards"] }, - { name: "example_queries", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["queries"] } + "assets": [ + { "name": "nginx", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["dashboards"] }, + { "name": "create_table", "version": "1.0.0", "extension": "sql", "type": "query" }, + { "name": "create_skipping_index", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["queries"] }, + { "name": "create_mv", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["dashboards"] }, + { "name": "example_queries", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["queries"] } ], - sampleData: { - path: "sample.json" + "sampleData": { + "path": "sample.json" } }