From bbb7385140271a9940a91cbdc247363f25bafe6f Mon Sep 17 00:00:00 2001 From: YANGDB Date: Mon, 30 Oct 2023 11:58:06 -0700 Subject: [PATCH 1/2] update vpc flow with flint-s3 based DDL assets Signed-off-by: YANGDB --- ...rds-observability.release-notes-2.9.0.0.md | 2 +- .../assets/create_mv_vpc-1.0.0.sql | 18 +++++++++++++++++ .../assets/create_table_vpc-1.0.0.sql | 20 +++++++++++++++++++ .../assets/refresh_mv_vpc-1.0.0.sql | 1 + 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql create mode 100644 server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_table_vpc-1.0.0.sql create mode 100644 server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql diff --git a/release-notes/dashboards-observability.release-notes-2.9.0.0.md b/release-notes/dashboards-observability.release-notes-2.9.0.0.md index ed1472781..91d295360 100644 --- a/release-notes/dashboards-observability.release-notes-2.9.0.0.md +++ b/release-notes/dashboards-observability.release-notes-2.9.0.0.md @@ -15,4 +15,4 @@ Compatible with OpenSearch and OpenSearch Dashboards Version 2.9.0 - Remove deprecated layout editor ([#646](https://github.com/opensearch-project/dashboards-observability/pull/646)) ### Documentation -- Integraions Plugin Design RFC ([#644](https://github.com/opensearch-project/dashboards-observability/issues/644)) +- Integrations Plugin Design RFC ([#644](https://github.com/opensearch-project/dashboards-observability/issues/644)) diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql new file mode 100644 index 000000000..e06e4fdea --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql @@ -0,0 +1,18 @@ +CREATE MATERIALIZED VIEW {table_name}_mview AS + SELECT + version as `aws.vpc.version`, + account_id as `aws.vpc.account-id`, + interface_id as `aws.vpc.interface-id`, + srcaddr as `aws.vpc.srcaddr`, + dstaddr as `aws.vpc.dstaddr`, + CAST(srcport AS LONG) as `aws.vpc.srcport`, + CAST(dstport AS LONG) as `aws.vpc.dstport`, + protocol as `aws.vpc.protocol`, + CAST(packets AS LONG) as `aws.vpc.packets`, + CAST(bytes AS LONG) as `aws.vpc.bytes`, + FROM_UNIXTIME(start) as `aws.vpc.start`, + FROM_UNIXTIME(end) as `aws.vpc.end`, + action as `aws.vpc.action`, + log_status as `aws.vpc.log-status` +FROM + {table_name}; \ No newline at end of file diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_table_vpc-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_table_vpc-1.0.0.sql new file mode 100644 index 000000000..7b9b4b3a7 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_table_vpc-1.0.0.sql @@ -0,0 +1,20 @@ +CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( + version INT, + account_id STRING, + interface_id STRING, + srcaddr STRING, + dstaddr STRING, + srcport STRING, + dstport STRING, + protocol STRING, + packets STRING, + bytes STRING, + start BIGINT, + end BIGINT, + action STRING, + log_status STRING +)USING csv +LOCATION '{s3_bucket_location}' +OPTIONS ( + sep=' ' +); \ No newline at end of file diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql new file mode 100644 index 000000000..0f6ba9896 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql @@ -0,0 +1 @@ +REFRESH MATERIALIZED VIEW {table_name}_mview; From fdcfdfa2ca68390ce4b50f84a8a8a6dcf6cd0ce8 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Mon, 30 Oct 2023 16:11:39 -0700 Subject: [PATCH 2/2] update vpc flow with flint-s3 based DDL assets Signed-off-by: YANGDB --- .../repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql index e06e4fdea..7b6954e17 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql @@ -10,9 +10,9 @@ CREATE MATERIALIZED VIEW {table_name}_mview AS protocol as `aws.vpc.protocol`, CAST(packets AS LONG) as `aws.vpc.packets`, CAST(bytes AS LONG) as `aws.vpc.bytes`, - FROM_UNIXTIME(start) as `aws.vpc.start`, + FROM_UNIXTIME(start) as `@timestamp`, FROM_UNIXTIME(end) as `aws.vpc.end`, action as `aws.vpc.action`, log_status as `aws.vpc.log-status` FROM - {table_name}; \ No newline at end of file + {table_name};