[Backport 2.x] Integration enhancements

server/adaptors/integrations/__data__/repository/amazon_cloudfront/amazon_cloudfront-1.0.0.json → server/adaptors/integrations/__data__/repository/amazon_cloudfront/amazon_cloudfront-1.1.0.json

@@ -1,13 +1,13 @@
 {
   "name": "amazon_cloudfront",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "displayName": "Amazon CloudFront",
   "description": "Analyze access logs for Amazon CloudFront.",
   "license": "Apache-2.0",
   "type": "logs-aws_cloudfront",
   "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
   "author": "OpenSearch",
-  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudfront/info",
+  "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/amazon_cloudfront-1.0.0",
   "statics": {
     "logo": {
       "annotation": "CloudFront Logo",

server/adaptors/integrations/__data__/repository/amazon_cloudfront/assets/create_table-1.0.0.sql

@@ -35,5 +35,8 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
 )
 USING csv
 LOCATION '{s3_bucket_location}'
-OPTIONS ( sep='\t' )
+OPTIONS (
+  sep='\t',
+  recursiveFileLookup='true'
+)
 TBLPROPERTIES ( 'skip.header.line.count'='2' )

server/adaptors/integrations/__data__/repository/amazon_elb/amazon_elb-1.0.0.json

@@ -7,7 +7,7 @@
   "type": "logs_elb",
   "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"],
   "author": "OpenSearch",
-  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_elb/info",
+  "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/amazon_elb-1.0.0",
   "workflows": [
     {
       "name": "queries",

server/adaptors/integrations/__data__/repository/amazon_s3/amazon_s3-1.0.0.json → server/adaptors/integrations/__data__/repository/amazon_s3/amazon_s3-1.1.0.json

@@ -1,13 +1,13 @@
 {
   "name": "amazon_s3",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "displayName": "Amazon S3",
   "description": "Analyze Amazon S3 server logs.",
   "license": "Apache-2.0",
   "type": "logs_s3",
   "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
   "author": "OpenSearch",
-  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_s3/info",
+  "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/amazon_s3_access-logs-1.1.0",
   "workflows": [
     {
       "name": "queries",

server/adaptors/integrations/__data__/repository/amazon_s3/assets/create_table-1.0.0.sql

@@ -27,6 +27,7 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
 )
 USING csv
 OPTIONS (
-  sep=' '
+  sep=' ',
+  recursiveFileLookup='true'
 )
 LOCATION '{s3_bucket_location}'

server/adaptors/integrations/__data__/repository/amazon_vpc_flow/amazon_vpc_flow-1.0.0.json → server/adaptors/integrations/__data__/repository/amazon_vpc_flow/amazon_vpc_flow-1.1.0.json

@@ -1,13 +1,13 @@
 {
   "name": "amazon_vpc_flow",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "displayName": "Amazon VPC",
   "description": "Monitor IP traffic with VPC Flow Logs.",
   "license": "Apache-2.0",
   "type": "logs_vpc",
   "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
   "author": "OpenSearch",
-  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info",
+  "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/amazon_vpc_flow_1.1.0",
   "workflows": [
     {
       "name": "queries",

server/adaptors/integrations/__data__/repository/amazon_vpc_flow/assets/create_table_vpc_schema-1.0.0.sql

@@ -28,5 +28,6 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
     pkt_dst_aws_service string,
     flow_direction string,
     traffic_path int
-) USING parquet
+)
+USING parquet
 LOCATION '{s3_bucket_location}'

server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json

@@ -1,52 +1,52 @@
 {
-  name: "apache",
-  version: "1.0.0",
-  displayName: "Apache Access Logs",
-  description: "Insights on applications using the standard Apache access log format.",
-  license: "Apache-2.0",
-  type: "logs_apache",
-  labels: ["Observability", "Logs", "Flint S3"],
-  author: "OpenSearch",
-  sourceUrl: "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/apache/info",
-  workflows: [
+  "name": "apache",
+  "version": "1.0.0",
+  "displayName": "Apache Access Logs",
+  "description": "Insights on applications using the standard Apache access log format.",
+  "license": "Apache-2.0",
+  "type": "logs_apache",
+  "labels": ["Observability", "Logs", "Flint S3"],
+  "author": "OpenSearch",
+  "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/apache-1.0.0",
+  "workflows": [
     {
-      name: "queries",
-      label: "Queries (recommended)",
-      description: "Tables and pre-written queries for quickly getting insights on your data.",
-      enabled_by_default: true
+      "name": "queries",
+      "label": "Queries (recommended)",
+      "description": "Tables and pre-written queries for quickly getting insights on your data.",
+      "enabled_by_default": true
     },
     {
-      name: "dashboards",
-      label: "Dashboards & Visualizations",
-      description: "Dashboards and indices that enable you to easily visualize important metrics.",
-      enabled_by_default: false
+      "name": "dashboards",
+      "label": "Dashboards & Visualizations",
+      "description": "Dashboards and indices that enable you to easily visualize important metrics.",
+      "enabled_by_default": false
     }
   ],
-  statics: {
-    logo: {
-      annotation: "Apache Logo",
-      path: "logo.png"
+  "statics": {
+    "logo": {
+      "annotation": "Apache Logo",
+      "path": "logo.png"
     },
-    gallery: [
+    "gallery": [
       {
-        annotation: "Apache Dashboard",
-        path: "dashboard1.png"
+        "annotation": "Apache Dashboard",
+        "path": "dashboard1.png"
       }
     ]
   },
-  components: [
-    { name: "communication", version: "1.0.0" },
-    { name: "http", version: "1.0.0" },
-    { name: "logs_apache", version: "1.0.0" }
+  "components": [
+    { "name": "communication", "version": "1.0.0" },
+    { "name": "http", "version": "1.0.0" },
+    { "name": "logs_apache", "version": "1.0.0" }
   ],
-  assets: [
-    { name: "apache", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["dashboards"] },
-    { name: "example_queries", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["queries"] },
-    { name: "create_table", version: "1.0.0", extension: "sql", type: "query" },
-    { name: "create_skipping_index", version: "1.0.0", extension: "sql", type: "query", workflows: ["queries"] },
-    { name: "create_mv", version: "1.0.0", extension: "sql", type: "query", workflows: ["dashboards"] }
+  "assets": [
+    { "name": "apache", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["dashboards"] },
+    { "name": "example_queries", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["queries"] },
+    { "name": "create_table", "version": "1.0.0", "extension": "sql", "type": "query" },
+    { "name": "create_skipping_index", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["queries"] },
+    { "name": "create_mv", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["dashboards"] }
   ],
-  sampleData: {
-    path: "sample.json"
+  "sampleData": {
+    "path": "sample.json"
   }
 }

server/adaptors/integrations/__data__/repository/apache/assets/create_table-1.0.0.sql

@@ -9,9 +9,11 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
   body_bytes_sent INT,
   http_referer STRING,
   http_user_agent STRING
-) USING csv
-LOCATION '{s3_bucket_location}'
+)
+USING csv
 OPTIONS (
   sep=' ',
-  nullValue='-'
+  nullValue='-',
+  recursiveFileLookup='true'
 )
+LOCATION '{s3_bucket_location}'

server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json → server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.1.0.json

@@ -1,13 +1,13 @@
 {
   "name": "aws_cloudtrail",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "displayName": "AWS CloudTrail",
   "description": "Analyze CloudTrail events.",
   "license": "Apache-2.0",
   "type": "logs-aws_cloudtrail",
   "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
   "author": "OpenSearch",
-  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudtrail/info",
+  "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/aws_cloudtrail-1.1.0",
   "workflows": [
     {
       "name": "dashboards",

server/adaptors/integrations/__data__/repository/aws_waf/assets/create_table-1.0.0.sql

@@ -125,4 +125,7 @@ CREATE EXTERNAL TABLE {table_name} (
   `ja3Fingerprint` string
 )
 USING json
+OPTIONS (
+  recursiveFileLookup='true'
+)
 LOCATION '{s3_bucket_location}'

server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json → server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.1.0.json

@@ -1,13 +1,13 @@
 {
   "name": "aws_waf",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "displayName": "AWS Web Application Firewall",
   "description": "Analyze AWS WAF web ACL traffic.",
   "license": "Apache-2.0",
   "type": "logs_waf",
   "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
   "author": "OpenSearch",
-  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_waf/info",
+  "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/aws_waf-1.0.0",
   "workflows": [
     {
       "name": "queries",

server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql

@@ -1,7 +1,9 @@
 CREATE EXTERNAL TABLE {table_name} (
   record STRING
-) USING csv
-LOCATION '{s3_bucket_location}'
+)
+USING csv
 OPTIONS (
-  sep='\0x1E'
+  sep='\0x1E',
+  recursiveFileLookup='true'
 )
+LOCATION '{s3_bucket_location}'

server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.0.0.json → server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.1.0.json

@@ -1,13 +1,13 @@
 {
   "name": "haproxy",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "displayName": "HAProxy",
   "description": "Analyze HAProxy access logs.",
   "license": "Apache-2.0",
   "type": "logs",
   "labels": ["Observability", "Logs", "Flint S3"],
   "author": "OpenSearch",
-  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/haproxy/info",
+  "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/haproxy-1.0.0",
   "workflows": [
     {
       "name": "queries",

server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql

@@ -10,9 +10,11 @@ CREATE EXTERNAL TABLE {table_name} (
   http_referer STRING,
   http_user_agent STRING,
   gzip_ratio STRING
-) USING csv
-LOCATION '{s3_bucket_location}'
+)
+USING csv
 OPTIONS (
   sep=' ',
-  nullValue='-'
+  nullValue='-',
+  recursiveFileLookup='true'
 )
+LOCATION '{s3_bucket_location}'

server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json

@@ -1,56 +1,56 @@
 {
-  name: "nginx",
-  version: "1.0.0",
-  displayName: "Nginx",
-  description: "Analyze Nginx access logs.",
-  license: "Apache-2.0",
-  type: "logs",
-  labels: ["Observability", "Logs", "Flint S3"],
-  author: "OpenSearch",
-  sourceUrl: "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info",
-  workflows: [
+  "name": "nginx",
+  "version": "1.0.0",
+  "displayName": "Nginx",
+  "description": "Analyze Nginx access logs.",
+  "license": "Apache-2.0",
+  "type": "logs",
+  "labels": ["Observability", "Logs", "Flint S3"],
+  "author": "OpenSearch",
+  "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/nginx-1.0.0",
+  "workflows": [
     {
-      name: "queries",
-      label: "Queries (recommended)",
-      description: "Tables and pre-written queries for quickly getting insights on your data.",
-      enabled_by_default: true
+      "name": "queries",
+      "label": "Queries (recommended)",
+      "description": "Tables and pre-written queries for quickly getting insights on your data.",
+      "enabled_by_default": true
     },
     {
-      name: "dashboards",
-      label: "Dashboards & Visualizations",
-      description: "Dashboards and indices that enable you to easily visualize important metrics.",
-      enabled_by_default: false
+      "name": "dashboards",
+      "label": "Dashboards & Visualizations",
+      "description": "Dashboards and indices that enable you to easily visualize important metrics.",
+      "enabled_by_default": false
     }
   ],
-  statics: {
-    logo: {
-      annotation: "NginX Logo",
-      path: "logo.svg"
+  "statics": {
+    "logo": {
+      "annotation": "NginX Logo",
+      "path": "logo.svg"
     },
-    gallery: [
+    "gallery": [
       {
-        annotation: "NginX Dashboard",
-        path: "dashboard1.png"
+        "annotation": "NginX Dashboard",
+        "path": "dashboard1.png"
       },
       {
-        annotation: "NginX Dashboard view",
-        path: "dashboard2.png"
+        "annotation": "NginX Dashboard view",
+        "path": "dashboard2.png"
       }
     ]
   },
-  components: [
-    { name: "communication", version: "1.0.0" },
-    { name: "http", version: "1.0.0" },
-    { name: "logs", version: "1.0.0" }
+  "components": [
+    { "name": "communication", "version": "1.0.0" },
+    { "name": "http", "version": "1.0.0" },
+    { "name": "logs", "version": "1.0.0" }
   ],
-  assets: [
-    { name: "nginx", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["dashboards"] },
-    { name: "create_table", version: "1.0.0", extension: "sql", type: "query" },
-    { name: "create_skipping_index", version: "1.0.0", extension: "sql", type: "query", workflows: ["queries"] },
-    { name: "create_mv", version: "1.0.0", extension: "sql", type: "query", workflows: ["dashboards"] },
-    { name: "example_queries", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["queries"] }
+  "assets": [
+    { "name": "nginx", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["dashboards"] },
+    { "name": "create_table", "version": "1.0.0", "extension": "sql", "type": "query" },
+    { "name": "create_skipping_index", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["queries"] },
+    { "name": "create_mv", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["dashboards"] },
+    { "name": "example_queries", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["queries"] }
   ],
-  sampleData: {
-    path: "sample.json"
+  "sampleData": {
+    "path": "sample.json"
   }
 }