diff --git a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_mv_index-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_mv_index-1.0.0.sql deleted file mode 100644 index 884e05c..0000000 --- a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_mv_index-1.0.0.sql +++ /dev/null @@ -1,41 +0,0 @@ -CREATE VIEW AS - SELECT - type as `aws.elb.elb_type`, - time as `@timestamp`, - elb as `aws.elb.elb_name`, - client_ip as `aws.elb.client.ip`, - client_port as `aws.elb.client.port`, - target_ip as `aws.elb.target_ip`, - target_port as `aws.elb.target_port`, - request_processing_time as `aws.elb.request_processing_time`, - target_processing_time as `aws.elb.target_processing_time`, - response_processing_time as `aws.elb.response_processing_time`, - elb_status_code as `aws.elb.elb_status_code`, - target_status_code as `aws.elb.target_status_code`, - received_bytes as `aws.elb.received_bytes`, - sent_bytes as `aws.elb.sent_bytes`, - request_verb as `http.request.method`, - request_url as `url.full`, - request_proto as `url.schema`, - user_agent as `http.user_agent.name`, - ssl_cipher as `aws.elb.ssl_cipher`, - ssl_protocol as `aws.elb.ssl_protocol`, - target_group_arn as `aws.elb.target_group_arn`, - trace_id as `traceId`, - domain_name as `url.domain`, - chosen_cert_arn as `aws.elb.chosen_cert_arn`, - matched_rule_priority as `aws.elb.matched_rule_priority`, - request_creation_time as `aws.elb.request_creation_time`, - actions_executed as `aws.elb.actions_executed`, - redirect_url as `aws.elb.redirect_url`, - lambda_error_reason as `aws.elb.lambda_error_reason`, - target_port_list as `aws.elb.target_port_list`, - target_status_code_list as `aws.elb.target_status_code_list`, - classification as `aws.elb.classification`, - classification_reason as `aws.elb.classification_reason` - FROM mys3.default.elb_logs_regex -WITH ( - auto_refresh = true, - refresh_interval = '5 Seconds' - ) - \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql deleted file mode 100644 index be089ad..0000000 --- a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql +++ /dev/null @@ -1,40 +0,0 @@ -CREATE EXTERNAL TABLE IF NOT EXISTS ${datasource}.${catalog}.${table} ( - type string, - time string, - elb string, - client_ip string, - client_port int, - target_ip string, - target_port int, - request_processing_time double, - target_processing_time double, - response_processing_time double, - elb_status_code int, - target_status_code string, - received_bytes bigint, - sent_bytes bigint, - request_verb string, - request_url string, - request_proto string, - user_agent string, - ssl_cipher string, - ssl_protocol string, - target_group_arn string, - trace_id string, - domain_name string, - chosen_cert_arn string, - matched_rule_priority string, - request_creation_time string, - actions_executed string, - redirect_url string, - lambda_error_reason string, - target_port_list string, - target_status_code_list string, - classification string, - classification_reason string -) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe' -WITH - SERDEPROPERTIES ( - 'serialization.format' = '1', - 'input.regex' = '([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) (.*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\s]+?)\" \"([^\s]+)\" \"([^ ]*)\" \"([^ ]*)\"' - ) LOCATION ${s3_bucket}; \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/tables/create_mv_elb-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/create_mv_elb-1.0.0.sql new file mode 100644 index 0000000..3c7bbe3 --- /dev/null +++ b/integrations/observability/aws_elb/assets/tables/create_mv_elb-1.0.0.sql @@ -0,0 +1,41 @@ +CREATE MATERIALIZED VIEW {table_name}_mview AS +SELECT + type as `aws.elb.elb_type`, + time as `@timestamp`, + elb as `aws.elb.elb_name`, + split_part (client_ip, ':', 1) as `communication.source.ip`, + split_part (client_ip, ':', 2) as `communication.source.port`, + split_part (target_ip, ':', 1) as `communication.destination.ip`, + split_part (target_ip, ':', 2) as `communication.destination.port`, + request_processing_time as `aws.elb.request_processing_time`, + target_processing_time as `aws.elb.target_processing_time`, + response_processing_time as `aws.elb.response_processing_time`, + elb_status_code as `http.response.status_code`, + target_status_code as `aws.elb.target_status_code`, + received_bytes as `aws.elb.received_bytes`, + sent_bytes as `aws.elb.sent_bytes`, + split_part (request, ' ', 1) as `http.request.method`, + split_part (request, ' ', 2) as `url.full`, + parse_url (split_part (request, ' ', 2), 'HOST') as `url.domain`, + parse_url (split_part (request, ' ', 2), 'PATH') as `url.path`, + split_part (request, ' ', 3) as `url.schema`, + request AS `http.request.body.content`, + user_agent as `http.user_agent.original`, + user_agent as `http.user_agent.name`, + ssl_cipher as `aws.elb.ssl_cipher`, + ssl_protocol as `aws.elb.ssl_protocol`, + split_part (target_group_arn, ':', 4) as `cloud.region`, + split_part (target_group_arn, ':', 5) as `cloud.account.id`, + trace_id as `traceId`, + chosen_cert_arn as `aws.elb.chosen_cert_arn`, + matched_rule_priority as `aws.elb.matched_rule_priority`, + request_creation_time as `aws.elb.request_creation_time`, + actions_executed as `aws.elb.actions_executed`, + redirect_url as `aws.elb.redirect_url`, + lambda_error_reason as `aws.elb.lambda_error_reason`, + target_port_list as `aws.elb.target_port_list`, + target_status_code_list as `aws.elb.target_status_code_list`, + classification as `aws.elb.classification`, + classification_reason as `aws.elb.classification_reason` +FROM + {table_name}; \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/tables/create_table_elb-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/create_table_elb-1.0.0.sql new file mode 100644 index 0000000..f9409d3 --- /dev/null +++ b/integrations/observability/aws_elb/assets/tables/create_table_elb-1.0.0.sql @@ -0,0 +1,36 @@ +CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( + type string, + time timestamp, + elb string, + client_ip string, + target_ip string, + request_processing_time double, + target_processing_time double, + response_processing_time double, + elb_status_code int, + target_status_code string, + received_bytes bigint, + sent_bytes bigint, + request string, + user_agent string, + ssl_cipher string, + ssl_protocol string, + target_group_arn string, + trace_id string, + domain_name string, + chosen_cert_arn string, + matched_rule_priority string, + request_creation_time timestamp, + actions_executed string, + redirect_url string, + lambda_error_reason string, + target_port_list string, + target_status_code_list string, + classification string, + classification_reason string +) +USING csv +LOCATION '{s3_bucket_location}' +OPTIONS ( + sep=' ' +); \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/tables/refresh_mv_elb-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/refresh_mv_elb-1.0.0.sql new file mode 100644 index 0000000..9e68c57 --- /dev/null +++ b/integrations/observability/aws_elb/assets/tables/refresh_mv_elb-1.0.0.sql @@ -0,0 +1 @@ +REFRESH MATERIALIZED VIEW {table_name}_mview; \ No newline at end of file diff --git a/integrations/observability/aws_elb/aws_elb-1.0.0.json b/integrations/observability/aws_elb/aws_elb-1.0.0.json index 28b7269..e54a509 100644 --- a/integrations/observability/aws_elb/aws_elb-1.0.0.json +++ b/integrations/observability/aws_elb/aws_elb-1.0.0.json @@ -5,17 +5,7 @@ "description": "AWS Elastic Load Balancer collector", "license": "Apache-2.0", "type": "logs", - "type-alias": "logs-elb", - "labels": [ - "log", - "aws", - "communication", - "http", - "cloud", - "elb", - "url", - "s3-datasource" - ], + "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_elb/info", "statics": { @@ -59,12 +49,19 @@ "assets": { "tables": [ { - "name": "aws_elb_s3_table", - "version": "1.0.0" + "name": "create_table_elb", + "version": "1.0.0", + "language": "sql" + }, + { + "name": "create_mv_elb", + "version": "1.0.0", + "language": "sql" }, { - "name": "aws_elb_s3_mv_index", - "version": "1.0.0" + "name": "refresh_mv_elb", + "version": "1.0.0", + "language": "sql" } ], "savedObjects": { @@ -75,4 +72,4 @@ "sampleData": { "path": "sample.json" } -} +} \ No newline at end of file diff --git a/integrations/observability/aws_elb/info/ELB_S3_DS_Intgration.md b/integrations/observability/aws_elb/info/ELB_S3_DS_Intgration.md index 28969e6..7c13a3c 100644 --- a/integrations/observability/aws_elb/info/ELB_S3_DS_Intgration.md +++ b/integrations/observability/aws_elb/info/ELB_S3_DS_Intgration.md @@ -11,7 +11,7 @@ This is a brief overview of a sample ingestion flow for the AWS ELB integration ## S3 Table Definition Using S3 datasource as the raw data for this integration requires the following assets to be present: - - S3-ELB [Table definition](../assets/tables/aws_elb_s3_table-1.0.0.sql) this table definition is used by the Spark/EMR catalog + - S3-ELB [Table definition](../assets/tables/create_table_elb-1.0.0.sql) this table definition is used by the Spark/EMR catalog - S3-ELB [Acceleration table refresh command](../assets/tables/aws_elb_s3_refresh_covering_index-1.0.0.sql) this command will initiate the flint job processing that will populate the secondary index according to the specified fields in the mapping metadata section. - S3 [opensearch acceleration index template definition ](../assets/indices/aws_elb_covering_index-1.0.0.mapping) @@ -62,7 +62,7 @@ The next section describes the integration responsibilities for creating the req Assuming all the prerequisites mentioned above are resolved, the first step would be to create the ELB logical table on the catalog ([Glue](https://aws.amazon.com/glue/)/[Hive](https://hive.apache.org/)) -- [The ELB table definition](../assets/tables/aws_elb_s3_table-1.0.0.sql) this table definition is used by the Spark/EMR catalog) +- [The ELB table definition](../assets/tables/create_table_elb-1.0.0.sql) this table definition is used by the Spark/EMR catalog) Once the table is created the next phase will be to generate the index template for the ELB log based on the simple schema for Observability index standard. This index template will be augmented with the [covering index component template](../assets/indices/aws_elb_covering_index-1.0.0.mapping) (In addition to the other component templates)