Merge branch 'main' into aws_elb_with_s3

integrations/observability/aws_vpc_flow/assets/tables/create_mv_vpc-1.0.0.sql

@@ -0,0 +1,36 @@
+CREATE MATERIALIZED VIEW {table_name}_mview AS
+ SELECT
+ version as `aws.vpc.version`,
+ account_id as `aws.vpc.account-id`,
+ interface_id as `aws.vpc.interface-id`,
+ srcaddr as `aws.vpc.srcaddr`,
+ dstaddr as `aws.vpc.dstaddr`,
+ CAST(srcport AS LONG) as `aws.vpc.srcport`,
+ CAST(dstport AS LONG) as `aws.vpc.dstport`,
+ protocol as `aws.vpc.protocol`,
+ CAST(packets AS LONG) as `aws.vpc.packets`,
+ CAST(bytes AS LONG) as `aws.vpc.bytes`,
+ FROM_UNIXTIME(start) as `@timestamp`,
+ FROM_UNIXTIME(end) as `aws.vpc.end`,
+ action as `aws.vpc.action`,
+ log_status as `aws.vpc.log-status`,
+ vpc_id as `aws.vpc.vpc-id`,
+ subnet_id as `aws.vpc.subnet-id`,
+ instance_id as `aws.vpc.instance-id`,
+ region as `aws.vpc.region`,
+ az_id as `aws.vpc.az-id`,
+ sublocation_type as `aws.vpc.sublocation-type`,
+ sublocation_id as `aws.vpc.sublocation-id`,
+ tcp_flags as `aws.vpc.tcp-flags`,
+ pkt_srcaddr as `aws.vpc.pkt-srcaddr`,
+ pkt_dstaddr as `aws.vpc.pkt-dstaddr`,
+ pkt_src_aws_service as `aws.vpc.pkt-src-aws-service`,
+ pkt_dst_aws_service as `aws.vpc.pkt-dst-aws-service`,
+ traffic_path as `aws.vpc.traffic-path`,
+ CASE
+ WHEN regexp(dstaddr, '(10\\..*)|(192\\.168\\..*)|(172\\.1[6-9]\\..*)|(172\\.2[0-9]\\..*)|(172\\.3[0-1]\\.*)')
+ THEN 'ingress'
+ ELSE 'egress'
+ END AS `aws.vpc.flow-direction`
+FROM
+ {table_name};

integrations/observability/aws_vpc_flow/assets/tables/create_table_vpc-1.0.0.sql

@@ -0,0 +1,36 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
+ version INT,
+ srcaddr STRING,
+ dstaddr STRING,
+ srcport INT,
+ dstport INT,
+ protocol INT,
+ start BIGINT,
+ end BIGINT,
+ type STRING,
+ packets INT,
+ bytes BIGINT,
+ account_id STRING,
+ vpc_id STRING,
+ subnet_id STRING,
+ instance_id STRING,
+ interface_id STRING,
+ region STRING,
+ az_id STRING,
+ sublocation_type STRING,
+ sublocation_id STRING,
+ action STRING,
+ tcp_flags STRING,
+ pkt_srcaddr STRING,
+ pkt_dstaddr STRING,
+ pkt_src_aws_service STRING,
+ pkt_dst_aws_service STRING,
+ traffic_path STRING,
+ flow_direction STRING,
+ log_status STRING
+
+)USING csv
+LOCATION '{s3_bucket_location}'
+OPTIONS (
+ sep=' '
+);

integrations/observability/aws_vpc_flow/assets/tables/refresh_mv_vpc-1.0.0.sql

@@ -0,0 +1 @@
+REFRESH MATERIALIZED VIEW {table_name}_mview;

integrations/observability/aws_vpc_flow/aws_vpc_flow-1.0.0.json

@@ -6,7 +6,7 @@
  "license": "Apache-2.0",
  "type": "logs",
  "type-alias": "logs-vpc",
- "labels": ["log", "aws", "s3", "cloud", "communication", "vpc"],
+ "labels": ["log", "aws", "s3", "cloud", "communication", "vpc", "Flint S3"],
  "author": "Haidong Wang",
  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info",
  "statics": {
@@ -44,6 +44,23 @@
  }
  ],
  "assets": {
+ "queries": [
+ {
+ "name": "create_table_vpc",
+ "version": "1.0.0",
+ "language": "sql"
+ },
+ {
+ "name": "create_mv_vpc",
+ "version": "1.0.0",
+ "language": "sql"
+ },
+ {
+ "name": "refresh_mv_vpc",
+ "version": "1.0.0",
+ "language": "sql"
+ }
+ ],
  "savedObjects": {
  "name": "aws_vpc_flow",
  "version": "1.0.0"