[Integration] Integration enhancements

server/adaptors/integrations/__data__/repository/amazon_cloudfront/amazon_cloudfront-1.0.0.json → server/adaptors/integrations/__data__/repository/amazon_cloudfront/amazon_cloudfront-1.1.0.json

@@ -1,13 +1,13 @@
 {
  "name": "amazon_cloudfront",
- "version": "1.0.0",
+ "version": "1.1.0",
  "displayName": "Amazon CloudFront",
  "description": "Analyze access logs for Amazon CloudFront.",
  "license": "Apache-2.0",
  "type": "logs-aws_cloudfront",
  "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
  "author": "OpenSearch",
- "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudfront/info",
+ "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/amazon_cloudfront-1.0.0",
  "statics": {
  "logo": {
  "annotation": "CloudFront Logo",

server/adaptors/integrations/__data__/repository/amazon_cloudfront/assets/create_table-1.0.0.sql

@@ -35,5 +35,8 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
 )
 USING csv
 LOCATION '{s3_bucket_location}'
-OPTIONS ( sep='\t' )
+OPTIONS (
+ sep='\t',
+ recursiveFileLookup='true'
+)
 TBLPROPERTIES ( 'skip.header.line.count'='2' )

server/adaptors/integrations/__data__/repository/amazon_elb/amazon_elb-1.0.0.json

@@ -7,7 +7,7 @@
  "type": "logs_elb",
  "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"],
  "author": "OpenSearch",
- "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_elb/info",
+ "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/amazon_elb-1.0.0",
  "workflows": [
  {
  "name": "queries",

server/adaptors/integrations/__data__/repository/amazon_s3/amazon_s3-1.0.0.json → server/adaptors/integrations/__data__/repository/amazon_s3/amazon_s3-1.1.0.json

@@ -1,13 +1,13 @@
 {
  "name": "amazon_s3",
- "version": "1.0.0",
+ "version": "1.1.0",
  "displayName": "Amazon S3",
  "description": "Analyze Amazon S3 server logs.",
  "license": "Apache-2.0",
  "type": "logs_s3",
  "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
  "author": "OpenSearch",
- "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_s3/info",
+ "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/amazon_s3_access-logs-1.1.0",
  "workflows": [
  {
  "name": "queries",

server/adaptors/integrations/__data__/repository/amazon_s3/assets/create_table-1.0.0.sql

@@ -27,6 +27,7 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
 )
 USING csv
 OPTIONS (
- sep=' '
+ sep=' ',
+ recursiveFileLookup='true'
 )
 LOCATION '{s3_bucket_location}'

server/adaptors/integrations/__data__/repository/amazon_vpc_flow/amazon_vpc_flow-1.0.0.json → server/adaptors/integrations/__data__/repository/amazon_vpc_flow/amazon_vpc_flow-1.1.0.json

@@ -1,13 +1,13 @@
 {
  "name": "amazon_vpc_flow",
- "version": "1.0.0",
+ "version": "1.1.0",
  "displayName": "Amazon VPC",
  "description": "Monitor IP traffic with VPC Flow Logs.",
  "license": "Apache-2.0",
  "type": "logs_vpc",
  "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
  "author": "OpenSearch",
- "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info",
+ "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/amazon_vpc_flow_1.1.0",
  "workflows": [
  {
  "name": "queries",

server/adaptors/integrations/__data__/repository/amazon_vpc_flow/assets/create_table_vpc_schema-1.0.0.sql

@@ -28,5 +28,6 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
  pkt_dst_aws_service string,
  flow_direction string,
  traffic_path int
-) USING parquet
+)
+USING parquet
 LOCATION '{s3_bucket_location}'

server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json

@@ -1,52 +1,52 @@
 {
- name: "apache",
- version: "1.0.0",
- displayName: "Apache Access Logs",
- description: "Insights on applications using the standard Apache access log format.",
- license: "Apache-2.0",
- type: "logs_apache",
- labels: ["Observability", "Logs", "Flint S3"],
- author: "OpenSearch",
- sourceUrl: "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/apache/info",
- workflows: [
+ "name": "apache",
+ "version": "1.0.0",
+ "displayName": "Apache Access Logs",
+ "description": "Insights on applications using the standard Apache access log format.",
+ "license": "Apache-2.0",
+ "type": "logs_apache",
+ "labels": ["Observability", "Logs", "Flint S3"],
+ "author": "OpenSearch",
+ "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/apache-1.0.0",
+ "workflows": [
  {
- name: "queries",
- label: "Queries (recommended)",
- description: "Tables and pre-written queries for quickly getting insights on your data.",
- enabled_by_default: true
+ "name": "queries",
+ "label": "Queries (recommended)",
+ "description": "Tables and pre-written queries for quickly getting insights on your data.",
+ "enabled_by_default": true
  },
  {
- name: "dashboards",
- label: "Dashboards & Visualizations",
- description: "Dashboards and indices that enable you to easily visualize important metrics.",
- enabled_by_default: false
+ "name": "dashboards",
+ "label": "Dashboards & Visualizations",
+ "description": "Dashboards and indices that enable you to easily visualize important metrics.",
+ "enabled_by_default": false
  }
  ],
- statics: {
- logo: {
- annotation: "Apache Logo",
- path: "logo.png"
+ "statics": {
+ "logo": {
+ "annotation": "Apache Logo",
+ "path": "logo.png"
  },
- gallery: [
+ "gallery": [
  {
- annotation: "Apache Dashboard",
- path: "dashboard1.png"
+ "annotation": "Apache Dashboard",
+ "path": "dashboard1.png"
  }
  ]
  },
- components: [
- { name: "communication", version: "1.0.0" },
- { name: "http", version: "1.0.0" },
- { name: "logs_apache", version: "1.0.0" }
+ "components": [
+ { "name": "communication", "version": "1.0.0" },
+ { "name": "http", "version": "1.0.0" },
+ { "name": "logs_apache", "version": "1.0.0" }
  ],
- assets: [
- { name: "apache", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["dashboards"] },
- { name: "example_queries", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["queries"] },
- { name: "create_table", version: "1.0.0", extension: "sql", type: "query" },
- { name: "create_skipping_index", version: "1.0.0", extension: "sql", type: "query", workflows: ["queries"] },
- { name: "create_mv", version: "1.0.0", extension: "sql", type: "query", workflows: ["dashboards"] }
+ "assets": [
+ { "name": "apache", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["dashboards"] },
+ { "name": "example_queries", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["queries"] },
+ { "name": "create_table", "version": "1.0.0", "extension": "sql", "type": "query" },
+ { "name": "create_skipping_index", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["queries"] },
+ { "name": "create_mv", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["dashboards"] }
  ],
- sampleData: {
- path: "sample.json"
+ "sampleData": {
+ "path": "sample.json"
  }
 }

server/adaptors/integrations/__data__/repository/apache/assets/create_table-1.0.0.sql

@@ -9,9 +9,11 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
  body_bytes_sent INT,
  http_referer STRING,
  http_user_agent STRING
-) USING csv
-LOCATION '{s3_bucket_location}'
+)
+USING csv
 OPTIONS (
  sep=' ',
- nullValue='-'
+ nullValue='-',
+ recursiveFileLookup='true'
 )
+LOCATION '{s3_bucket_location}'

server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json → server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.1.0.json

@@ -1,13 +1,13 @@
 {
  "name": "aws_cloudtrail",
- "version": "1.0.0",
+ "version": "1.1.0",
  "displayName": "AWS CloudTrail",
  "description": "Analyze CloudTrail events.",
  "license": "Apache-2.0",
  "type": "logs-aws_cloudtrail",
  "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
  "author": "OpenSearch",
- "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudtrail/info",
+ "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/aws_cloudtrail-1.1.0",
  "workflows": [
  {
  "name": "dashboards",

server/adaptors/integrations/__data__/repository/aws_waf/assets/create_table-1.0.0.sql

@@ -125,4 +125,7 @@ CREATE EXTERNAL TABLE {table_name} (
  `ja3Fingerprint` string
 )
 USING json
+OPTIONS (
+ recursiveFileLookup='true'
+)
 LOCATION '{s3_bucket_location}'

server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json → server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.1.0.json

@@ -1,13 +1,13 @@
 {
  "name": "aws_waf",
- "version": "1.0.0",
+ "version": "1.1.0",
  "displayName": "AWS Web Application Firewall",
  "description": "Analyze AWS WAF web ACL traffic.",
  "license": "Apache-2.0",
  "type": "logs_waf",
  "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
  "author": "OpenSearch",
- "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_waf/info",
+ "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/aws_waf-1.0.0",
  "workflows": [
  {
  "name": "queries",

server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql

@@ -1,7 +1,9 @@
 CREATE EXTERNAL TABLE {table_name} (
  record STRING
-) USING csv
-LOCATION '{s3_bucket_location}'
+)
+USING csv
 OPTIONS (
- sep='\0x1E'
+ sep='\0x1E',
+ recursiveFileLookup='true'
 )
+LOCATION '{s3_bucket_location}'

server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.0.0.json → server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.1.0.json

@@ -1,13 +1,13 @@
 {
  "name": "haproxy",
- "version": "1.0.0",
+ "version": "1.1.0",
  "displayName": "HAProxy",
  "description": "Analyze HAProxy access logs.",
  "license": "Apache-2.0",
  "type": "logs",
  "labels": ["Observability", "Logs", "Flint S3"],
  "author": "OpenSearch",
- "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/haproxy/info",
+ "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/haproxy-1.0.0",
  "workflows": [
  {
  "name": "queries",

server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql

@@ -10,9 +10,11 @@ CREATE EXTERNAL TABLE {table_name} (
  http_referer STRING,
  http_user_agent STRING,
  gzip_ratio STRING
-) USING csv
-LOCATION '{s3_bucket_location}'
+)
+USING csv
 OPTIONS (
  sep=' ',
- nullValue='-'
+ nullValue='-',
+ recursiveFileLookup='true'
 )
+LOCATION '{s3_bucket_location}'

server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json

@@ -1,56 +1,56 @@
 {
- name: "nginx",
- version: "1.0.0",
- displayName: "Nginx",
- description: "Analyze Nginx access logs.",
- license: "Apache-2.0",
- type: "logs",
- labels: ["Observability", "Logs", "Flint S3"],
- author: "OpenSearch",
- sourceUrl: "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info",
- workflows: [
+ "name": "nginx",
+ "version": "1.0.0",
+ "displayName": "Nginx",
+ "description": "Analyze Nginx access logs.",
+ "license": "Apache-2.0",
+ "type": "logs",
+ "labels": ["Observability", "Logs", "Flint S3"],
+ "author": "OpenSearch",
+ "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/nginx-1.0.0",
+ "workflows": [
  {
- name: "queries",
- label: "Queries (recommended)",
- description: "Tables and pre-written queries for quickly getting insights on your data.",
- enabled_by_default: true
+ "name": "queries",
+ "label": "Queries (recommended)",
+ "description": "Tables and pre-written queries for quickly getting insights on your data.",
+ "enabled_by_default": true
  },
  {
- name: "dashboards",
- label: "Dashboards & Visualizations",
- description: "Dashboards and indices that enable you to easily visualize important metrics.",
- enabled_by_default: false
+ "name": "dashboards",
+ "label": "Dashboards & Visualizations",
+ "description": "Dashboards and indices that enable you to easily visualize important metrics.",
+ "enabled_by_default": false
  }
  ],
- statics: {
- logo: {
- annotation: "NginX Logo",
- path: "logo.svg"
+ "statics": {
+ "logo": {
+ "annotation": "NginX Logo",
+ "path": "logo.svg"
  },
- gallery: [
+ "gallery": [
  {
- annotation: "NginX Dashboard",
- path: "dashboard1.png"
+ "annotation": "NginX Dashboard",
+ "path": "dashboard1.png"
  },
  {
- annotation: "NginX Dashboard view",
- path: "dashboard2.png"
+ "annotation": "NginX Dashboard view",
+ "path": "dashboard2.png"
  }
  ]
  },
- components: [
- { name: "communication", version: "1.0.0" },
- { name: "http", version: "1.0.0" },
- { name: "logs", version: "1.0.0" }
+ "components": [
+ { "name": "communication", "version": "1.0.0" },
+ { "name": "http", "version": "1.0.0" },
+ { "name": "logs", "version": "1.0.0" }
  ],
- assets: [
- { name: "nginx", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["dashboards"] },
- { name: "create_table", version: "1.0.0", extension: "sql", type: "query" },
- { name: "create_skipping_index", version: "1.0.0", extension: "sql", type: "query", workflows: ["queries"] },
- { name: "create_mv", version: "1.0.0", extension: "sql", type: "query", workflows: ["dashboards"] },
- { name: "example_queries", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["queries"] }
+ "assets": [
+ { "name": "nginx", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["dashboards"] },
+ { "name": "create_table", "version": "1.0.0", "extension": "sql", "type": "query" },
+ { "name": "create_skipping_index", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["queries"] },
+ { "name": "create_mv", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["dashboards"] },
+ { "name": "example_queries", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["queries"] }
  ],
- sampleData: {
- path: "sample.json"
+ "sampleData": {
+ "path": "sample.json"
  }
 }