Merge pull request #83 from YANG-DB/vpc-flow-flint-s3

update the vpc flow logs with flint-s3 DDL statements

integrations/observability/aws_vpc_flow/assets/tables/create_mv_vpc-1.0.0.sql

@@ -0,0 +1,36 @@
+CREATE MATERIALIZED VIEW {table_name}_mview AS
+    SELECT
+        version as `aws.vpc.version`,
+        account_id as `aws.vpc.account-id`,
+        interface_id as `aws.vpc.interface-id`,
+        srcaddr as `aws.vpc.srcaddr`,
+        dstaddr as `aws.vpc.dstaddr`,
+        CAST(srcport AS LONG) as `aws.vpc.srcport`,
+        CAST(dstport AS LONG) as `aws.vpc.dstport`,
+        protocol as `aws.vpc.protocol`,
+        CAST(packets AS LONG) as `aws.vpc.packets`,
+        CAST(bytes AS LONG) as `aws.vpc.bytes`,
+        FROM_UNIXTIME(start) as `@timestamp`,
+        FROM_UNIXTIME(end) as `aws.vpc.end`,
+        action as `aws.vpc.action`,
+        log_status as `aws.vpc.log-status`,
+        vpc_id as `aws.vpc.vpc-id`,
+        subnet_id as `aws.vpc.subnet-id`,
+        instance_id as `aws.vpc.instance-id`,
+        region as `aws.vpc.region`,
+        az_id as `aws.vpc.az-id`,
+        sublocation_type as `aws.vpc.sublocation-type`,
+        sublocation_id as `aws.vpc.sublocation-id`,
+        tcp_flags as `aws.vpc.tcp-flags`,
+        pkt_srcaddr as `aws.vpc.pkt-srcaddr`,
+        pkt_dstaddr as `aws.vpc.pkt-dstaddr`,
+        pkt_src_aws_service as `aws.vpc.pkt-src-aws-service`,
+        pkt_dst_aws_service as `aws.vpc.pkt-dst-aws-service`,
+        traffic_path as `aws.vpc.traffic-path`,
+        CASE
+            WHEN regexp(dstaddr, '(10\\..*)|(192\\.168\\..*)|(172\\.1[6-9]\\..*)|(172\\.2[0-9]\\..*)|(172\\.3[0-1]\\.*)')
+            THEN 'ingress'
+            ELSE 'egress'
+        END AS `aws.vpc.flow-direction`
+FROM
+    {table_name};

integrations/observability/aws_vpc_flow/assets/tables/create_table_vpc-1.0.0.sql

@@ -0,0 +1,36 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
+    version INT,
+    srcaddr STRING,
+    dstaddr STRING,
+    srcport INT,
+    dstport INT,
+    protocol INT,
+    start BIGINT,
+    end BIGINT,
+    type STRING,
+    packets INT,
+    bytes BIGINT,
+    account_id STRING,
+    vpc_id STRING,
+    subnet_id STRING,
+    instance_id STRING,
+    interface_id STRING,
+    region STRING,
+    az_id STRING,
+    sublocation_type STRING,
+    sublocation_id STRING,
+    action STRING,
+    tcp_flags STRING,
+    pkt_srcaddr STRING,
+    pkt_dstaddr STRING,
+    pkt_src_aws_service STRING,
+    pkt_dst_aws_service STRING,
+    traffic_path STRING,
+    flow_direction STRING,
+    log_status STRING
+
+)USING csv
+LOCATION '{s3_bucket_location}'
+OPTIONS (
+  sep=' '
+);

integrations/observability/aws_vpc_flow/assets/tables/refresh_mv_vpc-1.0.0.sql

@@ -0,0 +1 @@
+REFRESH MATERIALIZED VIEW {table_name}_mview;

integrations/observability/aws_vpc_flow/aws_vpc_flow-1.0.0.json

@@ -6,7 +6,7 @@
   "license": "Apache-2.0",
   "type": "logs",
   "type-alias": "logs-vpc",
-  "labels": ["log", "aws", "s3", "cloud", "communication", "vpc"],
+  "labels": ["log", "aws", "s3", "cloud", "communication", "vpc", "Flint S3"],
   "author": "Haidong Wang",
   "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info",
   "statics": {
@@ -44,6 +44,23 @@
     }
   ],
   "assets": {
+    "queries": [
+      {
+        "name": "create_table_vpc",
+        "version": "1.0.0",
+        "language": "sql"
+      },
+      {
+        "name": "create_mv_vpc",
+        "version": "1.0.0",
+        "language": "sql"
+      },
+      {
+        "name": "refresh_mv_vpc",
+        "version": "1.0.0",
+        "language": "sql"
+      }
+    ],
     "savedObjects": {
       "name": "aws_vpc_flow",
       "version": "1.0.0"