From c006c49ed540e8be9bb679349d035641670e9f11 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Thu, 28 Sep 2023 16:46:41 -0700 Subject: [PATCH 01/10] add elb s3 flint based integration assets into the existing ELB integration - table definition - skipping table definition - opensearch skipping table index definition Signed-off-by: YANGDB --- ...arch_skipping_index_template-1.0.0.mapping | 34 ++++++++++++++++++ .../aws_elb_s3_skipping_index-1.0.0.sql | 22 ++++++++++++ .../aws_elb/assets/aws_elb_s3_table-1.0.0.sql | 36 +++++++++++++++++++ .../observability/aws_elb/aws_elb-1.0.0.json | 20 ++++++++++- .../aws_elb/info/S3_INGESTION.md | 10 ++++++ 5 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 integrations/observability/aws_elb/assets/aws_elb_s3_opensearch_skipping_index_template-1.0.0.mapping create mode 100644 integrations/observability/aws_elb/assets/aws_elb_s3_skipping_index-1.0.0.sql create mode 100644 integrations/observability/aws_elb/assets/aws_elb_s3_table-1.0.0.sql create mode 100644 integrations/observability/aws_elb/info/S3_INGESTION.md diff --git a/integrations/observability/aws_elb/assets/aws_elb_s3_opensearch_skipping_index_template-1.0.0.mapping b/integrations/observability/aws_elb/assets/aws_elb_s3_opensearch_skipping_index_template-1.0.0.mapping new file mode 100644 index 0000000..b8bea58 --- /dev/null +++ b/integrations/observability/aws_elb/assets/aws_elb_s3_opensearch_skipping_index_template-1.0.0.mapping @@ -0,0 +1,34 @@ +{ + "_meta": { + "version": "0.1", + "indexConfig": { + "kind": "skipping", + "properties": { + "indexedColumns": [ + { + "kind": "Partition", + "columnName": "year", + "columnType": "int" + }, + { + "kind": "ValuesSet", + "columnName": "elb_status_code", + "columnType": "int" + } + ] + } + }, + "source": "alb_logs" + }, + "properties": { + "year": { + "type": "integer" + }, + "elb_status_code": { + "type": "integer" + }, + "file_path": { + "type": "keyword" + } + } +} \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/aws_elb_s3_skipping_index-1.0.0.sql b/integrations/observability/aws_elb/assets/aws_elb_s3_skipping_index-1.0.0.sql new file mode 100644 index 0000000..6d5d45b --- /dev/null +++ b/integrations/observability/aws_elb/assets/aws_elb_s3_skipping_index-1.0.0.sql @@ -0,0 +1,22 @@ +CREATE SKIPPING INDEX ON alb_logs ( + type PARTITION, + time PARTITION, + elb string, + client_ip VALUE_SET, + client_port VALUE_SET, + target_ip VALUE_SET, + target_port VALUE_SET, + request_processing_time PARTITION, + target_processing_time PARTITION, + response_processing_time PARTITION, + elb_status_code VALUE_SET, + target_status_code VALUE_SET, + received_bytes PARTITION, + sent_bytes PARTITION, + request_verb PARTITION, + request_url PARTITION) +WITH ( + auto_refresh = true, + refresh_interval = '1 day', + checkpoint_location = 's3://test/' +) diff --git a/integrations/observability/aws_elb/assets/aws_elb_s3_table-1.0.0.sql b/integrations/observability/aws_elb/assets/aws_elb_s3_table-1.0.0.sql new file mode 100644 index 0000000..ecf941b --- /dev/null +++ b/integrations/observability/aws_elb/assets/aws_elb_s3_table-1.0.0.sql @@ -0,0 +1,36 @@ +CREATE EXTERNAL TABLE IF NOT EXISTS alb_logs ( + type string, + time string, + elb string, + client_ip string, + client_port int, + target_ip string, + target_port int, + request_processing_time double, + target_processing_time double, + response_processing_time double, + elb_status_code int, + target_status_code string, + received_bytes bigint, + sent_bytes bigint, + request_verb string, + request_url string, + request_proto string, + user_agent string, + ssl_cipher string, + ssl_protocol string, + target_group_arn string, + trace_id string, + domain_name string, + chosen_cert_arn string, + matched_rule_priority string, + request_creation_time string, + actions_executed string, + redirect_url string, + lambda_error_reason string, + target_port_list string, + target_status_code_list string, + classification string, + classification_reason string + ) +LOCATION 's3://your-alb-logs-directory/AWSLogs//elasticloadbalancing//' diff --git a/integrations/observability/aws_elb/aws_elb-1.0.0.json b/integrations/observability/aws_elb/aws_elb-1.0.0.json index e687c7c..d1f46fa 100644 --- a/integrations/observability/aws_elb/aws_elb-1.0.0.json +++ b/integrations/observability/aws_elb/aws_elb-1.0.0.json @@ -6,7 +6,7 @@ "license": "Apache-2.0", "type": "logs", "type-alias": "logs-elb", - "labels": ["log", "aws", "communication", "http", "cloud", "elb", "url"], + "labels": ["log", "aws", "communication", "http", "cloud", "elb", "url", "s3-datasource"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_elb/info", "statics": { @@ -48,6 +48,24 @@ } ], "assets": { + "tables": [ + { + "name": "aws_elb_s3_table", + "version": "1.0.0" + }, + { + "name": "aws_elb_s3_skipping_index", + "version": "1.0.0" + } + ] , + "indices": { + "name": "aws_elb_s3_opensearch_skipping_index_template", + "version": "1.0.0" + }, + "queries": { + "name": "", + "version": "1.0.0" + }, "savedObjects": { "name": "aws_elb", "version": "1.0.0" diff --git a/integrations/observability/aws_elb/info/S3_INGESTION.md b/integrations/observability/aws_elb/info/S3_INGESTION.md new file mode 100644 index 0000000..9abbfaa --- /dev/null +++ b/integrations/observability/aws_elb/info/S3_INGESTION.md @@ -0,0 +1,10 @@ +# S3 Based Ingestion Flow + +This is a brief overview of a sample ingestion flow for the AWS ELB integration which is S3 based. + +## List of Prerequisites + +- An OpenSearch domain running through Docker +- A Spark agent running cluster with [Flint Opensearch Extension](https://github.com/opensearch-project/opensearch-spark) +- An ELB instance generating logs into S3 [setup info](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/enable-access-logs.html) + From c0905bfe5de7890b34ec5c468f7138d0032c7df5 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Thu, 28 Sep 2023 23:02:48 -0700 Subject: [PATCH 02/10] add sql saved search to the integration assets Signed-off-by: YANGDB --- .../average_amount_of_data_query-1.0.0.sql | 2 + .../aws_elb/assets/aws_elb_s3_table-1.0.0.sql | 12 +++- .../aws_elb/assets/clients_by_data-1.0.0.sql | 5 ++ .../assets/clients_by_visiting_urls-1.0.0.sql | 4 ++ .../aws_elb/assets/count_http_by_ip-1.0.0.sql | 6 ++ .../distinct_client_count_query-1.0.0.sql | 2 + .../largest_latency_by_date_range-1.0.0.sql | 6 ++ .../most_frequent_user_agents-1.0.0.sql | 6 ++ .../assets/target_traffic_imbalance-1.0.0.sql | 6 ++ .../observability/aws_elb/aws_elb-1.0.0.json | 60 +++++++++++++++++-- 10 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 integrations/observability/aws_elb/assets/average_amount_of_data_query-1.0.0.sql create mode 100644 integrations/observability/aws_elb/assets/clients_by_data-1.0.0.sql create mode 100644 integrations/observability/aws_elb/assets/clients_by_visiting_urls-1.0.0.sql create mode 100644 integrations/observability/aws_elb/assets/count_http_by_ip-1.0.0.sql create mode 100644 integrations/observability/aws_elb/assets/distinct_client_count_query-1.0.0.sql create mode 100644 integrations/observability/aws_elb/assets/largest_latency_by_date_range-1.0.0.sql create mode 100644 integrations/observability/aws_elb/assets/most_frequent_user_agents-1.0.0.sql create mode 100644 integrations/observability/aws_elb/assets/target_traffic_imbalance-1.0.0.sql diff --git a/integrations/observability/aws_elb/assets/average_amount_of_data_query-1.0.0.sql b/integrations/observability/aws_elb/assets/average_amount_of_data_query-1.0.0.sql new file mode 100644 index 0000000..c1f9d6f --- /dev/null +++ b/integrations/observability/aws_elb/assets/average_amount_of_data_query-1.0.0.sql @@ -0,0 +1,2 @@ +-- # List the average amount of data (in kilobytes) that's passing through the Application Load Balancer in request or response pairs +SELECT (avg(sent_bytes)/1000.0 + avg(received_bytes)/1000.0) as prewarm_kilobytes from alb_logs; \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/aws_elb_s3_table-1.0.0.sql b/integrations/observability/aws_elb/assets/aws_elb_s3_table-1.0.0.sql index ecf941b..cf16c00 100644 --- a/integrations/observability/aws_elb/assets/aws_elb_s3_table-1.0.0.sql +++ b/integrations/observability/aws_elb/assets/aws_elb_s3_table-1.0.0.sql @@ -32,5 +32,13 @@ CREATE EXTERNAL TABLE IF NOT EXISTS alb_logs ( target_status_code_list string, classification string, classification_reason string - ) -LOCATION 's3://your-alb-logs-directory/AWSLogs//elasticloadbalancing//' + )PARTITIONED BY ( year STRING, month STRING, day STRING ) + ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe' + WITH SERDEPROPERTIES ( + 'serialization.format' = '1', + 'input.regex' = + '([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) (.*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\s]+?)\" \"([^\s]+)\" \"([^ ]*)\" \"([^ ]*)\"') + LOCATION '/Users/daichen/Temp/us-west-2/'; + +-- # Add partition individually following Hive convention like year=2022/month=04/day=01 +ALTER TABLE alb_logs ADD PARTITION (year='?',month='?', day='?') location '?'; \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/clients_by_data-1.0.0.sql b/integrations/observability/aws_elb/assets/clients_by_data-1.0.0.sql new file mode 100644 index 0000000..dbdfca0 --- /dev/null +++ b/integrations/observability/aws_elb/assets/clients_by_data-1.0.0.sql @@ -0,0 +1,5 @@ +-- # List clients, in descending order, by the amount of data (in megabytes) that each client sent in their requests to the Application Load Balancer +SELECT client_ip, sum(received_bytes/1000000.0) as client_datareceived_megabytes +FROM alb_logs +GROUP by client_ip +ORDER by client_datareceived_megabytes DESC; \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/clients_by_visiting_urls-1.0.0.sql b/integrations/observability/aws_elb/assets/clients_by_visiting_urls-1.0.0.sql new file mode 100644 index 0000000..b440d88 --- /dev/null +++ b/integrations/observability/aws_elb/assets/clients_by_visiting_urls-1.0.0.sql @@ -0,0 +1,4 @@ +-- # List clients, in descending order, by the number of times that each client visited a specified URL +SELECT client_ip, elb, request_url, count(*) as count from alb_logs +GROUP by client_ip, elb, request_url +ORDER by count DESC; \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/count_http_by_ip-1.0.0.sql b/integrations/observability/aws_elb/assets/count_http_by_ip-1.0.0.sql new file mode 100644 index 0000000..b9f6ad9 --- /dev/null +++ b/integrations/observability/aws_elb/assets/count_http_by_ip-1.0.0.sql @@ -0,0 +1,6 @@ +-- # Count the number of HTTP GET requests received by the load balancer grouped by the client IP address +SELECT COUNT(request_verb) + AS count, request_verb, client_ip +FROM alb_logs_partitioned +WHERE day = '?' +GROUP by request_verb, client_ip; \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/distinct_client_count_query-1.0.0.sql b/integrations/observability/aws_elb/assets/distinct_client_count_query-1.0.0.sql new file mode 100644 index 0000000..14ffac5 --- /dev/null +++ b/integrations/observability/aws_elb/assets/distinct_client_count_query-1.0.0.sql @@ -0,0 +1,2 @@ +-- # List all client IP addresses that accessed the Application Load Balancer, and how many times they accessed the Application Load Balancer +SELECT distinct client_ip, count() as count from alb_logs GROUP by client_ip ORDER by count() DESC; \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/largest_latency_by_date_range-1.0.0.sql b/integrations/observability/aws_elb/assets/largest_latency_by_date_range-1.0.0.sql new file mode 100644 index 0000000..2fcb16b --- /dev/null +++ b/integrations/observability/aws_elb/assets/largest_latency_by_date_range-1.0.0.sql @@ -0,0 +1,6 @@ +-- # List each time in a specified date range when the target processing time was more than ? (5) seconds +SELECT * from alb_logs +WHERE (parse_datetime(time,'yyyy-MM-dd''T''HH:mm:ss.SSSSSS''Z') + BETWEEN parse_datetime('?','?') + AND parse_datetime('?','?')) + AND (target_processing_time >= ?); \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/most_frequent_user_agents-1.0.0.sql b/integrations/observability/aws_elb/assets/most_frequent_user_agents-1.0.0.sql new file mode 100644 index 0000000..edef8bd --- /dev/null +++ b/integrations/observability/aws_elb/assets/most_frequent_user_agents-1.0.0.sql @@ -0,0 +1,6 @@ +-- # List the ? (10) URLs that ? (Firefox) users accessed most frequently, in descending order +SELECT request_url, user_agent, count(*) as count +FROM alb_logs +WHERE user_agent LIKE '%?%' +GROUP by request_url, user_agent +ORDER by count(*) DESC LIMIT ?; \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/target_traffic_imbalance-1.0.0.sql b/integrations/observability/aws_elb/assets/target_traffic_imbalance-1.0.0.sql new file mode 100644 index 0000000..be91bd4 --- /dev/null +++ b/integrations/observability/aws_elb/assets/target_traffic_imbalance-1.0.0.sql @@ -0,0 +1,6 @@ +-- # List all targets that the Application Load Balancer routes traffic to and the number of routed requests per target, by percentage distribution +SELECT target_ip, (Count(target_ip)* 100.0 / (Select Count(*) From alb_logs)) + as backend_traffic_percentage +FROM alb_logs +GROUP by target_ip< +ORDER By count() DESC; \ No newline at end of file diff --git a/integrations/observability/aws_elb/aws_elb-1.0.0.json b/integrations/observability/aws_elb/aws_elb-1.0.0.json index d1f46fa..5927b67 100644 --- a/integrations/observability/aws_elb/aws_elb-1.0.0.json +++ b/integrations/observability/aws_elb/aws_elb-1.0.0.json @@ -6,7 +6,16 @@ "license": "Apache-2.0", "type": "logs", "type-alias": "logs-elb", - "labels": ["log", "aws", "communication", "http", "cloud", "elb", "url", "s3-datasource"], + "labels": [ + "log", + "aws", + "communication", + "http", + "cloud", + "elb", + "url", + "s3-datasource" + ], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_elb/info", "statics": { @@ -57,15 +66,54 @@ "name": "aws_elb_s3_skipping_index", "version": "1.0.0" } - ] , + ], "indices": { "name": "aws_elb_s3_opensearch_skipping_index_template", "version": "1.0.0" }, - "queries": { - "name": "", - "version": "1.0.0" - }, + "queries": [ + { + "name": "average_amount_of_data_query", + "description": " List the average amount of data (in kilobytes) that's passing through the Application Load Balancer in request or response pairs", + "version": "1.0.0" + }, + { + "name": "distinct_client_count_query", + "description": "List all client IP addresses that accessed the Application Load Balancer, and how many times they accessed the Application Load Balancer", + "version": "1.0.0" + }, + { + "name": "clients_by_visiting_urls", + "description": "List clients, in descending order, by the number of times that each client visited a specified URL", + "version": "1.0.0" + }, + { + "name": "clients_by_data", + "description": "List clients, in descending order, by the amount of data (in megabytes) that each client sent in their requests to the Application Load Balancer", + "version": "1.0.0" + }, + { + "name": "target_traffic_imbalance", + "description": "List all targets that the Application Load Balancer routes traffic to and the number of routed requests per target, by percentage distribution", + "version": "1.0.0" + }, + { + "name": "count_http_by_ip", + "description": "Count the number of HTTP GET requests received by the load balancer grouped by the client IP address", + "version": "1.0.0" + }, + }, + { + "name": "largest_latency_by_date_range", + "description": "List each time in a specified date range when the target processing time was more than ? (5) seconds", + "version": "1.0.0" + }, + { + "name": "most_frequent_user_agents", + "description": "List the ? (10) URLs that ? (Firefox) users accessed most frequently, in descending order", + "version": "1.0.0" + } + ], "savedObjects": { "name": "aws_elb", "version": "1.0.0" From 7d77ea0d0afb949ea5e3a27d961290928df5cf72 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Thu, 28 Sep 2023 23:03:14 -0700 Subject: [PATCH 03/10] add sql saved search to the integration assets Signed-off-by: YANGDB --- integrations/observability/aws_elb/aws_elb-1.0.0.json | 1 - 1 file changed, 1 deletion(-) diff --git a/integrations/observability/aws_elb/aws_elb-1.0.0.json b/integrations/observability/aws_elb/aws_elb-1.0.0.json index 5927b67..1b0b1d9 100644 --- a/integrations/observability/aws_elb/aws_elb-1.0.0.json +++ b/integrations/observability/aws_elb/aws_elb-1.0.0.json @@ -102,7 +102,6 @@ "description": "Count the number of HTTP GET requests received by the load balancer grouped by the client IP address", "version": "1.0.0" }, - }, { "name": "largest_latency_by_date_range", "description": "List each time in a specified date range when the target processing time was more than ? (5) seconds", From c75bf469be48e267a8a5273c2f591dc9e267c89e Mon Sep 17 00:00:00 2001 From: YANGDB Date: Mon, 2 Oct 2023 17:44:53 -0700 Subject: [PATCH 04/10] update elb-s3 integration assets tree structure Signed-off-by: YANGDB --- .../assets/aws_elb_s3_notebook-1.0.0.json | 105 ++++++++++++++++++ ...arch_skipping_index_template-1.0.0.mapping | 34 ------ .../aws_elb_s3_skipping_index-1.0.0.sql | 22 ---- .../aws_elb/assets/aws_elb_s3_table-1.0.0.sql | 44 -------- ...arch_covering_index_template-1.0.0.mapping | 27 +++++ ...arch_skipping_index_template-1.0.0.mapping | 27 +++++ .../average_amount_of_data_query-1.0.0.sql | 0 .../{ => queries}/clients_by_data-1.0.0.sql | 5 +- .../clients_by_visiting_urls-1.0.0.sql | 0 .../{ => queries}/count_http_by_ip-1.0.0.sql | 0 .../distinct_client_count_query-1.0.0.sql | 0 .../largest_latency_by_date_range-1.0.0.sql | 0 .../most_frequent_user_agents-1.0.0.sql | 0 .../target_traffic_imbalance-1.0.0.sql | 0 .../aws_elb_s3_covering_index-1.0.0.sql | 6 + .../assets/tables/aws_elb_s3_table-1.0.0.sql | 44 ++++++++ .../aws_elb/info/S3_INGESTION.md | 49 ++++++++ 17 files changed, 259 insertions(+), 104 deletions(-) create mode 100644 integrations/observability/aws_elb/assets/aws_elb_s3_notebook-1.0.0.json delete mode 100644 integrations/observability/aws_elb/assets/aws_elb_s3_opensearch_skipping_index_template-1.0.0.mapping delete mode 100644 integrations/observability/aws_elb/assets/aws_elb_s3_skipping_index-1.0.0.sql delete mode 100644 integrations/observability/aws_elb/assets/aws_elb_s3_table-1.0.0.sql create mode 100644 integrations/observability/aws_elb/assets/indices/aws_elb_s3_opensearch_covering_index_template-1.0.0.mapping create mode 100644 integrations/observability/aws_elb/assets/indices/aws_elb_s3_opensearch_skipping_index_template-1.0.0.mapping rename integrations/observability/aws_elb/assets/{ => queries}/average_amount_of_data_query-1.0.0.sql (100%) rename integrations/observability/aws_elb/assets/{ => queries}/clients_by_data-1.0.0.sql (69%) rename integrations/observability/aws_elb/assets/{ => queries}/clients_by_visiting_urls-1.0.0.sql (100%) rename integrations/observability/aws_elb/assets/{ => queries}/count_http_by_ip-1.0.0.sql (100%) rename integrations/observability/aws_elb/assets/{ => queries}/distinct_client_count_query-1.0.0.sql (100%) rename integrations/observability/aws_elb/assets/{ => queries}/largest_latency_by_date_range-1.0.0.sql (100%) rename integrations/observability/aws_elb/assets/{ => queries}/most_frequent_user_agents-1.0.0.sql (100%) rename integrations/observability/aws_elb/assets/{ => queries}/target_traffic_imbalance-1.0.0.sql (100%) create mode 100644 integrations/observability/aws_elb/assets/tables/aws_elb_s3_covering_index-1.0.0.sql create mode 100644 integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql diff --git a/integrations/observability/aws_elb/assets/aws_elb_s3_notebook-1.0.0.json b/integrations/observability/aws_elb/assets/aws_elb_s3_notebook-1.0.0.json new file mode 100644 index 0000000..6db7642 --- /dev/null +++ b/integrations/observability/aws_elb/assets/aws_elb_s3_notebook-1.0.0.json @@ -0,0 +1,105 @@ +{ + "path": "aws_elb_queries", + "name": "[AWS ELB Logs Queries] AWS ELB Queries based on S3", + "dateCreated": "2023-09-30T00:23:39.602Z", + "dateModified": "2023-10-02T17:06:20.090Z", + "backend": "Default", + "paragraphs": [ + { + "output": [ + { + "result": "## ELB Queries\nThe next notebook shows a variety of monitoring and health check queries both in SQL / PPL on top of ELB log index / table\n\n### Queries Details:\n1) Lists the average amount of data (in kilobytes) that's passing through the Application Load Balancer in request or response pairs\n\n2) List clients, in descending order, by the amount of data (in megabytes) that each client sent in their requests to the Application Load Balancer\n\n3) List clients, in descending order, by the number of times that each client visited a specified URL\n\n4) Count the number of HTTP GET requests received by the load balancer grouped by the client IP address\n\n5) List each time in a specified date range when the target processing time was more than ? (5) seconds\n\n6) List the ? (10) URLs that ? (Firefox) users accessed most frequently, in descending order\n\n\n", + "outputType": "MARKDOWN", + "execution_time": "0.599 ms" + } + ], + "input": { + "inputText": "%md\n## ELB Queries\nThe next notebook shows a variety of monitoring and health check queries both in SQL / PPL on top of ELB log index / table\n\n### Queries Details:\n1) Lists the average amount of data (in kilobytes) that's passing through the Application Load Balancer in request or response pairs\n\n2) List clients, in descending order, by the amount of data (in megabytes) that each client sent in their requests to the Application Load Balancer\n\n3) List clients, in descending order, by the number of times that each client visited a specified URL\n\n4) Count the number of HTTP GET requests received by the load balancer grouped by the client IP address\n\n5) List each time in a specified date range when the target processing time was more than ? (5) seconds\n\n6) List the ? (10) URLs that ? (Firefox) users accessed most frequently, in descending order\n\n\n", + "inputType": "MARKDOWN" + }, + "dateCreated": "2023-09-30T00:23:42.373Z", + "dateModified": "2023-10-02T17:06:20.090Z", + "id": "paragraph_bcac0759-a5dc-454b-85e7-507ec5b79afe" + }, + { + "output": [ + { + "result": "\nSELECT (avg(aws.elb.sent_bytes)/1000.0 + avg(aws.elb.received_bytes)/1000.0) as prewarm_kilobytes from ss4o_logs_elb-aws_elb-sample-sample;", + "outputType": "QUERY", + "execution_time": "0.010 ms" + } + ], + "input": { + "inputText": "%sql\nSELECT (avg(aws.elb.sent_bytes)/1000.0 + avg(aws.elb.received_bytes)/1000.0) as prewarm_kilobytes from ss4o_logs_elb-aws_elb-sample-sample;", + "inputType": "MARKDOWN" + }, + "dateCreated": "2023-09-30T00:26:29.468Z", + "dateModified": "2023-09-30T08:03:36.310Z", + "id": "paragraph_10056fac-9d0a-436e-93b5-1c77c207677d" + }, + { + "output": [ + { + "result": "\nSELECT aws.elb.client.ip, sum(aws.elb.received_bytes/1000000.0) as client_datareceived_megabytes FROM ss4o_logs_elb-aws_elb-sample-sample GROUP by aws.elb.client.ip ORDER by client_datareceived_megabytes DESC;", + "outputType": "QUERY", + "execution_time": "0.006 ms" + } + ], + "input": { + "inputText": "%sql\nSELECT aws.elb.client.ip, sum(aws.elb.received_bytes/1000000.0) as client_datareceived_megabytes FROM ss4o_logs_elb-aws_elb-sample-sample GROUP by aws.elb.client.ip ORDER by client_datareceived_megabytes DESC;", + "inputType": "MARKDOWN" + }, + "dateCreated": "2023-09-30T00:31:26.288Z", + "dateModified": "2023-09-30T08:04:46.029Z", + "id": "paragraph_7cadb639-8571-41a0-84b6-917b4f1ed88a" + }, + { + "output": [ + { + "result": "\nSELECT aws.elb.client.ip, url.domain, count(*) as count from ss4o_logs_elb-aws_elb-sample-sample\nGROUP by aws.elb.client.ip, url.domain\nORDER by count DESC;", + "outputType": "QUERY", + "execution_time": "0.006 ms" + } + ], + "input": { + "inputText": "%sql\nSELECT aws.elb.client.ip, url.domain, count(*) as count from ss4o_logs_elb-aws_elb-sample-sample\nGROUP by aws.elb.client.ip, url.domain\nORDER by count DESC;", + "inputType": "MARKDOWN" + }, + "dateCreated": "2023-09-30T00:31:44.191Z", + "dateModified": "2023-09-30T08:12:50.840Z", + "id": "paragraph_dd1fa597-3db2-40d3-a163-e0d248300a53" + }, + { + "output": [ + { + "result": "\nSELECT COUNT(http.request.method)\n AS count, http.request.method, aws.elb.client.ip\nFROM ss4o_logs_elb-aws_elb-sample-sample \nGROUP by http.request.method, aws.elb.client.ip;", + "outputType": "QUERY", + "execution_time": "0.006 ms" + } + ], + "input": { + "inputText": "%sql\nSELECT COUNT(http.request.method)\n AS count, http.request.method, aws.elb.client.ip\nFROM ss4o_logs_elb-aws_elb-sample-sample \nGROUP by http.request.method, aws.elb.client.ip;", + "inputType": "MARKDOWN" + }, + "dateCreated": "2023-09-30T00:31:57.554Z", + "dateModified": "2023-09-30T08:18:53.232Z", + "id": "paragraph_8ae6e362-ebfe-455b-8d43-ebb6db25ecea" + }, + { + "output": [ + { + "result": "\nSELECT url.domain, http.user_agent.name, count(*) as count\nFROM ss4o_logs_elb-aws_elb-sample-sample\nWHERE http.user_agent.name LIKE '%Chrome%'\nGROUP by url.domain, http.user_agent.name\nORDER by count(*) DESC;", + "outputType": "QUERY", + "execution_time": "0.007 ms" + } + ], + "input": { + "inputText": "%sql\nSELECT url.domain, http.user_agent.name, count(*) as count\nFROM ss4o_logs_elb-aws_elb-sample-sample\nWHERE http.user_agent.name LIKE '%Chrome%'\nGROUP by url.domain, http.user_agent.name\nORDER by count(*) DESC;", + "inputType": "MARKDOWN" + }, + "dateCreated": "2023-09-30T00:38:32.321Z", + "dateModified": "2023-09-30T08:29:31.202Z", + "id": "paragraph_7fde081a-6dac-4bb7-ae49-a86695901b1a" + } + ] +} \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/aws_elb_s3_opensearch_skipping_index_template-1.0.0.mapping b/integrations/observability/aws_elb/assets/aws_elb_s3_opensearch_skipping_index_template-1.0.0.mapping deleted file mode 100644 index b8bea58..0000000 --- a/integrations/observability/aws_elb/assets/aws_elb_s3_opensearch_skipping_index_template-1.0.0.mapping +++ /dev/null @@ -1,34 +0,0 @@ -{ - "_meta": { - "version": "0.1", - "indexConfig": { - "kind": "skipping", - "properties": { - "indexedColumns": [ - { - "kind": "Partition", - "columnName": "year", - "columnType": "int" - }, - { - "kind": "ValuesSet", - "columnName": "elb_status_code", - "columnType": "int" - } - ] - } - }, - "source": "alb_logs" - }, - "properties": { - "year": { - "type": "integer" - }, - "elb_status_code": { - "type": "integer" - }, - "file_path": { - "type": "keyword" - } - } -} \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/aws_elb_s3_skipping_index-1.0.0.sql b/integrations/observability/aws_elb/assets/aws_elb_s3_skipping_index-1.0.0.sql deleted file mode 100644 index 6d5d45b..0000000 --- a/integrations/observability/aws_elb/assets/aws_elb_s3_skipping_index-1.0.0.sql +++ /dev/null @@ -1,22 +0,0 @@ -CREATE SKIPPING INDEX ON alb_logs ( - type PARTITION, - time PARTITION, - elb string, - client_ip VALUE_SET, - client_port VALUE_SET, - target_ip VALUE_SET, - target_port VALUE_SET, - request_processing_time PARTITION, - target_processing_time PARTITION, - response_processing_time PARTITION, - elb_status_code VALUE_SET, - target_status_code VALUE_SET, - received_bytes PARTITION, - sent_bytes PARTITION, - request_verb PARTITION, - request_url PARTITION) -WITH ( - auto_refresh = true, - refresh_interval = '1 day', - checkpoint_location = 's3://test/' -) diff --git a/integrations/observability/aws_elb/assets/aws_elb_s3_table-1.0.0.sql b/integrations/observability/aws_elb/assets/aws_elb_s3_table-1.0.0.sql deleted file mode 100644 index cf16c00..0000000 --- a/integrations/observability/aws_elb/assets/aws_elb_s3_table-1.0.0.sql +++ /dev/null @@ -1,44 +0,0 @@ -CREATE EXTERNAL TABLE IF NOT EXISTS alb_logs ( - type string, - time string, - elb string, - client_ip string, - client_port int, - target_ip string, - target_port int, - request_processing_time double, - target_processing_time double, - response_processing_time double, - elb_status_code int, - target_status_code string, - received_bytes bigint, - sent_bytes bigint, - request_verb string, - request_url string, - request_proto string, - user_agent string, - ssl_cipher string, - ssl_protocol string, - target_group_arn string, - trace_id string, - domain_name string, - chosen_cert_arn string, - matched_rule_priority string, - request_creation_time string, - actions_executed string, - redirect_url string, - lambda_error_reason string, - target_port_list string, - target_status_code_list string, - classification string, - classification_reason string - )PARTITIONED BY ( year STRING, month STRING, day STRING ) - ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe' - WITH SERDEPROPERTIES ( - 'serialization.format' = '1', - 'input.regex' = - '([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) (.*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\s]+?)\" \"([^\s]+)\" \"([^ ]*)\" \"([^ ]*)\"') - LOCATION '/Users/daichen/Temp/us-west-2/'; - --- # Add partition individually following Hive convention like year=2022/month=04/day=01 -ALTER TABLE alb_logs ADD PARTITION (year='?',month='?', day='?') location '?'; \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/indices/aws_elb_s3_opensearch_covering_index_template-1.0.0.mapping b/integrations/observability/aws_elb/assets/indices/aws_elb_s3_opensearch_covering_index_template-1.0.0.mapping new file mode 100644 index 0000000..ea80067 --- /dev/null +++ b/integrations/observability/aws_elb/assets/indices/aws_elb_s3_opensearch_covering_index_template-1.0.0.mapping @@ -0,0 +1,27 @@ +{ + "template": { + "mappings": { + "_meta": { + "version": "0.1", + "indexConfig": { + "kind": "covering", + "properties": { + "indexedColumns": [ + { + "kind": "Partition", + "columnName": "year", + "columnType": "int" + }, + { + "kind": "ValuesSet", + "columnName": "elb_status_code", + "columnType": "int" + } + ] + } + }, + "source": "alb_logs" + } + } + } +} \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/indices/aws_elb_s3_opensearch_skipping_index_template-1.0.0.mapping b/integrations/observability/aws_elb/assets/indices/aws_elb_s3_opensearch_skipping_index_template-1.0.0.mapping new file mode 100644 index 0000000..ba01adb --- /dev/null +++ b/integrations/observability/aws_elb/assets/indices/aws_elb_s3_opensearch_skipping_index_template-1.0.0.mapping @@ -0,0 +1,27 @@ +{ + "template": { + "mappings": { + "_meta": { + "version": "0.1", + "indexConfig": { + "kind": "skipping", + "properties": { + "indexedColumns": [ + { + "kind": "Partition", + "columnName": "year", + "columnType": "int" + }, + { + "kind": "ValuesSet", + "columnName": "elb_status_code", + "columnType": "int" + } + ] + } + }, + "source": "alb_logs" + } + } + } +} \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/average_amount_of_data_query-1.0.0.sql b/integrations/observability/aws_elb/assets/queries/average_amount_of_data_query-1.0.0.sql similarity index 100% rename from integrations/observability/aws_elb/assets/average_amount_of_data_query-1.0.0.sql rename to integrations/observability/aws_elb/assets/queries/average_amount_of_data_query-1.0.0.sql diff --git a/integrations/observability/aws_elb/assets/clients_by_data-1.0.0.sql b/integrations/observability/aws_elb/assets/queries/clients_by_data-1.0.0.sql similarity index 69% rename from integrations/observability/aws_elb/assets/clients_by_data-1.0.0.sql rename to integrations/observability/aws_elb/assets/queries/clients_by_data-1.0.0.sql index dbdfca0..82edd20 100644 --- a/integrations/observability/aws_elb/assets/clients_by_data-1.0.0.sql +++ b/integrations/observability/aws_elb/assets/queries/clients_by_data-1.0.0.sql @@ -1,5 +1,2 @@ -- # List clients, in descending order, by the amount of data (in megabytes) that each client sent in their requests to the Application Load Balancer -SELECT client_ip, sum(received_bytes/1000000.0) as client_datareceived_megabytes -FROM alb_logs -GROUP by client_ip -ORDER by client_datareceived_megabytes DESC; \ No newline at end of file +SELECT client_ip, sum(received_bytes/1000000.0) as client_datareceived_megabytes FROM alb_logs GROUP by client_ip ORDER by client_datareceived_megabytes DESC; \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/clients_by_visiting_urls-1.0.0.sql b/integrations/observability/aws_elb/assets/queries/clients_by_visiting_urls-1.0.0.sql similarity index 100% rename from integrations/observability/aws_elb/assets/clients_by_visiting_urls-1.0.0.sql rename to integrations/observability/aws_elb/assets/queries/clients_by_visiting_urls-1.0.0.sql diff --git a/integrations/observability/aws_elb/assets/count_http_by_ip-1.0.0.sql b/integrations/observability/aws_elb/assets/queries/count_http_by_ip-1.0.0.sql similarity index 100% rename from integrations/observability/aws_elb/assets/count_http_by_ip-1.0.0.sql rename to integrations/observability/aws_elb/assets/queries/count_http_by_ip-1.0.0.sql diff --git a/integrations/observability/aws_elb/assets/distinct_client_count_query-1.0.0.sql b/integrations/observability/aws_elb/assets/queries/distinct_client_count_query-1.0.0.sql similarity index 100% rename from integrations/observability/aws_elb/assets/distinct_client_count_query-1.0.0.sql rename to integrations/observability/aws_elb/assets/queries/distinct_client_count_query-1.0.0.sql diff --git a/integrations/observability/aws_elb/assets/largest_latency_by_date_range-1.0.0.sql b/integrations/observability/aws_elb/assets/queries/largest_latency_by_date_range-1.0.0.sql similarity index 100% rename from integrations/observability/aws_elb/assets/largest_latency_by_date_range-1.0.0.sql rename to integrations/observability/aws_elb/assets/queries/largest_latency_by_date_range-1.0.0.sql diff --git a/integrations/observability/aws_elb/assets/most_frequent_user_agents-1.0.0.sql b/integrations/observability/aws_elb/assets/queries/most_frequent_user_agents-1.0.0.sql similarity index 100% rename from integrations/observability/aws_elb/assets/most_frequent_user_agents-1.0.0.sql rename to integrations/observability/aws_elb/assets/queries/most_frequent_user_agents-1.0.0.sql diff --git a/integrations/observability/aws_elb/assets/target_traffic_imbalance-1.0.0.sql b/integrations/observability/aws_elb/assets/queries/target_traffic_imbalance-1.0.0.sql similarity index 100% rename from integrations/observability/aws_elb/assets/target_traffic_imbalance-1.0.0.sql rename to integrations/observability/aws_elb/assets/queries/target_traffic_imbalance-1.0.0.sql diff --git a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_covering_index-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_covering_index-1.0.0.sql new file mode 100644 index 0000000..735a52d --- /dev/null +++ b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_covering_index-1.0.0.sql @@ -0,0 +1,6 @@ +CREATE COVERING INDEX ON "ss4o_logs-aws_elb" +WITH ( + auto_refresh = true, + refresh_interval = '1 day', + checkpoint_location = 's3://test/' +) \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql new file mode 100644 index 0000000..320097e --- /dev/null +++ b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql @@ -0,0 +1,44 @@ +CREATE EXTERNAL TABLE IF NOT EXISTS alb_logs ( + type string as aws.elb.elb_type, + time string as @timestamp, + elb string as aws.elb.elb_name, + client_ip string as aws.elb.client.ip, + client_port int as aws.elb.client.port, + target_ip string as aws.elb.target_ip, + target_port int as aws.elb.target_port, + request_processing_time double as aws.elb.request_processing_time, + target_processing_time double as aws.elb.target_processing_time, + response_processing_time double as aws.elb.response_processing_time, + elb_status_code int as aws.elb.elb_status_code, + target_status_code string as aws.elb.target_status_code, + received_bytes bigint as aws.elb.received_bytes, + sent_bytes bigint as aws.elb.sent_bytes, + request_verb string as http.request.method, + request_url string as url.full, + request_proto string as url.schema, + user_agent string as http.user_agent.name, + ssl_cipher string as aws.elb.ssl_cipher, + ssl_protocol string as aws.elb.ssl_protocol , + target_group_arn string as aws.elb.target_group_arn, + trace_id string as traceId, + domain_name string as url.domain, + chosen_cert_arn string as aws.elb.chosen_cert_arn, + matched_rule_priority string as aws.elb.matched_rule_priority, + request_creation_time string as aws.elb.request_creation_time, + actions_executed string as aws.elb.actions_executed, + redirect_url string as aws.elb.redirect_url, + lambda_error_reason string as aws.elb.lambda_error_reason, + target_port_list string as aws.elb.target_port_list, + target_status_code_list string as aws.elb.target_status_code_list, + classification string as aws.elb.classification , + classification_reason string as aws.elb.classification_reason + )PARTITIONED BY ( year STRING, month STRING, day STRING ) + ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe' + WITH SERDEPROPERTIES ( + 'serialization.format' = '1', + 'input.regex' = + '([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) (.*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\s]+?)\" \"([^\s]+)\" \"([^ ]*)\" \"([^ ]*)\"') + LOCATION {location}; + +-- # Add partition individually following Hive convention like year=2022/month=04/day=01 +ALTER TABLE alb_logs ADD PARTITION (year='?',month='?', day='?') location '?'; \ No newline at end of file diff --git a/integrations/observability/aws_elb/info/S3_INGESTION.md b/integrations/observability/aws_elb/info/S3_INGESTION.md index 9abbfaa..6200e04 100644 --- a/integrations/observability/aws_elb/info/S3_INGESTION.md +++ b/integrations/observability/aws_elb/info/S3_INGESTION.md @@ -8,3 +8,52 @@ This is a brief overview of a sample ingestion flow for the AWS ELB integration - A Spark agent running cluster with [Flint Opensearch Extension](https://github.com/opensearch-project/opensearch-spark) - An ELB instance generating logs into S3 [setup info](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/enable-access-logs.html) +## S3 Table Definition +Using S3 datasource as the raw data for this integration requires the following assets to be present: + + - S3-ELB [Table definition](../assets/aws_elb_s3_table-1.0.0.sql) this table definition is used by the Spark/EMR catalog + - S3-ELB [Acceleration table definition](../assets/aws_elb_s3_skipping_index-1.0.0.sql) this table is used by opensearch flint-spark + - S3 opensearch acceleration index template definition + - Covering Index for accelerating general SQL/PPL queries targeted for S3 + - Materialized view Index for accelerating the ELB dashboards based on OpenSearch indices + + +#### ELB table mapping +The next columns mapping between the S3-ELB table definition and the ELB schema index mapping: + +| Field Source Name | Field Target Name | Type | +|-----------------------------|---------------------------------|----------| +| type | aws.elb.elb_type | string | +| time | @timestamp | string | +| elb | aws.elb.elb_name | string | +| client_ip | aws.elb.client.ip | string | +| client_port | aws.elb.client.port | int | +| target_ip | aws.elb.target_ip | string | +| target_port | aws.elb.target_port | int | +| request_processing_time | aws.elb.request_processing_time | double | +| target_processing_time | aws.elb.target_processing_time | double | +| response_processing_time | aws.elb.response_processing_time| double | +| elb_status_code | aws.elb.elb_status_code | int | +| target_status_code | aws.elb.target_status_code | string | +| received_bytes | aws.elb.received_bytes | bigint | +| sent_bytes | aws.elb.sent_bytes | bigint | +| request_verb | http.request.method | string | +| request_url | url.full | string | +| request_proto | url.schema | string | +| user_agent | http.user_agent.name | string | +| ssl_cipher | aws.elb.ssl_cipher | string | +| ssl_protocol | aws.elb.ssl_protocol | string | +| target_group_arn | aws.elb.target_group_arn | string | +| trace_id | traceId | string | +| domain_name | url.domain | string | +| chosen_cert_arn | aws.elb.chosen_cert_arn | string | +| matched_rule_priority | aws.elb.matched_rule_priority | string | +| request_creation_time | aws.elb.request_creation_time | string | +| actions_executed | aws.elb.actions_executed | string | +| redirect_url | aws.elb.redirect_url | string | +| lambda_error_reason | aws.elb.lambda_error_reason | string | +| target_port_list | aws.elb.target_port_list | string | +| target_status_code_list | aws.elb.target_status_code_list | string | +| classification | aws.elb.classification | string | +| classification_reason | aws.elb.classification_reason | string | + From c6ba0ced14315e1cdc5113ebd5d7e1db0ee2f5f9 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Tue, 3 Oct 2023 22:16:41 -0700 Subject: [PATCH 05/10] update elb-s3 integration assets tree structure and assets Signed-off-by: YANGDB --- .../aws_elb_covering_index-1.0.0.mapping | 118 ++++++++++++++++++ ...arch_covering_index_template-1.0.0.mapping | 27 ---- ...arch_skipping_index_template-1.0.0.mapping | 27 ---- .../aws_elb_s3_covering_index-1.0.0.sql | 6 - ...ws_elb_s3_refresh_covering_index-1.0.0.sql | 1 + .../assets/tables/aws_elb_s3_table-1.0.0.sql | 12 +- ...3_INGESTION.md => ELB_S3_DS_Intgration.md} | 31 ++++- 7 files changed, 149 insertions(+), 73 deletions(-) create mode 100644 integrations/observability/aws_elb/assets/indices/aws_elb_covering_index-1.0.0.mapping delete mode 100644 integrations/observability/aws_elb/assets/indices/aws_elb_s3_opensearch_covering_index_template-1.0.0.mapping delete mode 100644 integrations/observability/aws_elb/assets/indices/aws_elb_s3_opensearch_skipping_index_template-1.0.0.mapping delete mode 100644 integrations/observability/aws_elb/assets/tables/aws_elb_s3_covering_index-1.0.0.sql create mode 100644 integrations/observability/aws_elb/assets/tables/aws_elb_s3_refresh_covering_index-1.0.0.sql rename integrations/observability/aws_elb/info/{S3_INGESTION.md => ELB_S3_DS_Intgration.md} (58%) diff --git a/integrations/observability/aws_elb/assets/indices/aws_elb_covering_index-1.0.0.mapping b/integrations/observability/aws_elb/assets/indices/aws_elb_covering_index-1.0.0.mapping new file mode 100644 index 0000000..001561c --- /dev/null +++ b/integrations/observability/aws_elb/assets/indices/aws_elb_covering_index-1.0.0.mapping @@ -0,0 +1,118 @@ +{ + "template": { + "aliases": { + "flint_{object_name}_index": {} + }, + "mappings": { + "_meta": { + "version": "1.0.0", + "name": "{object_name}", + "kind": "covering", + "indexedColumns": [ + { + "columnName": "aws.elb.backend.ip", + "columnType": "ip" + }, + { + "columnName": "aws.elb.backend.port", + "columnType": "integer" + }, + { + "columnName": "aws.elb.backend.processing_time", + "columnType": "half_float" + }, + { + "columnName": "aws.elb.backend.status_code", + "columnType": "short" + }, + { + "columnName": "aws.elb.client.ip", + "columnType": "ip" + }, + { + "columnName": "aws.elb.client.port", + "columnType": "integer" + }, + { + "columnName": "aws.elb.connection_time", + "columnType": "integer" + }, + { + "columnName": "aws.elb.destination.ip", + "columnType": "ip" + }, + { + "columnName": "aws.elb.destination.port", + "columnType": "integer" + }, + { + "columnName": "aws.elb.elb_status_code", + "columnType": "short" + }, + { + "columnName": "aws.elb.http.port", + "columnType": "integer" + }, + { + "columnName": "aws.elb.http.version", + "columnType": "keyword" + }, + { + "columnName": "aws.elb.matched_rule_priority", + "columnType": "integer" + }, + { + "columnName": "aws.elb.received_bytes", + "columnType": "integer" + }, + { + "columnName": "aws.elb.request_creation_time", + "columnType": "date" + }, + { + "columnName": "aws.elb.request_processing_time", + "columnType": "half_float" + }, + { + "columnName": "aws.elb.response_processing_time", + "columnType": "half_float" + }, + { + "columnName": "aws.elb.sent_bytes", + "columnType": "integer" + }, + { + "columnName": "aws.elb.ssl_protocol", + "columnType": "keyword" + }, + { + "columnName": "aws.elb.ssl_cipher", + "columnType": "keyword" + }, + { + "columnName": "aws.elb.target_ip", + "columnType": "ip" + }, + { + "columnName": "aws.elb.target_port", + "columnType": "integer" + }, + { + "columnName": "aws.elb.target_processing_time", + "columnType": "half_float" + }, + { + "columnName": "aws.elb.target_status_code", + "columnType": "short" + }, + { + "columnName": "aws.elb.timestamp", + "columnType": "date" + } + ], + "source": "{table_name}", + "options": {} + } + } + } +} \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/indices/aws_elb_s3_opensearch_covering_index_template-1.0.0.mapping b/integrations/observability/aws_elb/assets/indices/aws_elb_s3_opensearch_covering_index_template-1.0.0.mapping deleted file mode 100644 index ea80067..0000000 --- a/integrations/observability/aws_elb/assets/indices/aws_elb_s3_opensearch_covering_index_template-1.0.0.mapping +++ /dev/null @@ -1,27 +0,0 @@ -{ - "template": { - "mappings": { - "_meta": { - "version": "0.1", - "indexConfig": { - "kind": "covering", - "properties": { - "indexedColumns": [ - { - "kind": "Partition", - "columnName": "year", - "columnType": "int" - }, - { - "kind": "ValuesSet", - "columnName": "elb_status_code", - "columnType": "int" - } - ] - } - }, - "source": "alb_logs" - } - } - } -} \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/indices/aws_elb_s3_opensearch_skipping_index_template-1.0.0.mapping b/integrations/observability/aws_elb/assets/indices/aws_elb_s3_opensearch_skipping_index_template-1.0.0.mapping deleted file mode 100644 index ba01adb..0000000 --- a/integrations/observability/aws_elb/assets/indices/aws_elb_s3_opensearch_skipping_index_template-1.0.0.mapping +++ /dev/null @@ -1,27 +0,0 @@ -{ - "template": { - "mappings": { - "_meta": { - "version": "0.1", - "indexConfig": { - "kind": "skipping", - "properties": { - "indexedColumns": [ - { - "kind": "Partition", - "columnName": "year", - "columnType": "int" - }, - { - "kind": "ValuesSet", - "columnName": "elb_status_code", - "columnType": "int" - } - ] - } - }, - "source": "alb_logs" - } - } - } -} \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_covering_index-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_covering_index-1.0.0.sql deleted file mode 100644 index 735a52d..0000000 --- a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_covering_index-1.0.0.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE COVERING INDEX ON "ss4o_logs-aws_elb" -WITH ( - auto_refresh = true, - refresh_interval = '1 day', - checkpoint_location = 's3://test/' -) \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_refresh_covering_index-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_refresh_covering_index-1.0.0.sql new file mode 100644 index 0000000..8b26c91 --- /dev/null +++ b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_refresh_covering_index-1.0.0.sql @@ -0,0 +1 @@ +REFRESH INDEX name ON {index_name} \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql index 320097e..b8a1374 100644 --- a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql +++ b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql @@ -1,4 +1,4 @@ -CREATE EXTERNAL TABLE IF NOT EXISTS alb_logs ( +CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( type string as aws.elb.elb_type, time string as @timestamp, elb string as aws.elb.elb_name, @@ -32,13 +32,9 @@ CREATE EXTERNAL TABLE IF NOT EXISTS alb_logs ( target_status_code_list string as aws.elb.target_status_code_list, classification string as aws.elb.classification , classification_reason string as aws.elb.classification_reason - )PARTITIONED BY ( year STRING, month STRING, day STRING ) + ) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe' WITH SERDEPROPERTIES ( 'serialization.format' = '1', - 'input.regex' = - '([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) (.*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\s]+?)\" \"([^\s]+)\" \"([^ ]*)\" \"([^ ]*)\"') - LOCATION {location}; - --- # Add partition individually following Hive convention like year=2022/month=04/day=01 -ALTER TABLE alb_logs ADD PARTITION (year='?',month='?', day='?') location '?'; \ No newline at end of file + 'input.regex' = '([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) (.*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\s]+?)\" \"([^\s]+)\" \"([^ ]*)\" \"([^ ]*)\"') + LOCATION {s3_bucket_location} \ No newline at end of file diff --git a/integrations/observability/aws_elb/info/S3_INGESTION.md b/integrations/observability/aws_elb/info/ELB_S3_DS_Intgration.md similarity index 58% rename from integrations/observability/aws_elb/info/S3_INGESTION.md rename to integrations/observability/aws_elb/info/ELB_S3_DS_Intgration.md index 6200e04..d0d3b27 100644 --- a/integrations/observability/aws_elb/info/S3_INGESTION.md +++ b/integrations/observability/aws_elb/info/ELB_S3_DS_Intgration.md @@ -11,11 +11,11 @@ This is a brief overview of a sample ingestion flow for the AWS ELB integration ## S3 Table Definition Using S3 datasource as the raw data for this integration requires the following assets to be present: - - S3-ELB [Table definition](../assets/aws_elb_s3_table-1.0.0.sql) this table definition is used by the Spark/EMR catalog - - S3-ELB [Acceleration table definition](../assets/aws_elb_s3_skipping_index-1.0.0.sql) this table is used by opensearch flint-spark - - S3 opensearch acceleration index template definition - - Covering Index for accelerating general SQL/PPL queries targeted for S3 - - Materialized view Index for accelerating the ELB dashboards based on OpenSearch indices + - S3-ELB [Table definition](../assets/tables/aws_elb_s3_table-1.0.0.sql) this table definition is used by the Spark/EMR catalog + - S3-ELB [Acceleration table refresh command](../assets/tables/aws_elb_s3_refresh_covering_index-1.0.0.sql) this command will initiate the flint job processing that + will populate the secondary index according to the specified fields in the mapping metadata section. + - S3 [opensearch acceleration index template definition ](../assets/indices/aws_elb_covering_index-1.0.0.mapping) + - Covering Index for accelerating general SQL/PPL queries targeted for S3 and cached in OpenSearch secondery index - see [covering index acceleration process](https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md#covering-index). #### ELB table mapping @@ -57,3 +57,24 @@ The next columns mapping between the S3-ELB table definition and the ELB schema | classification | aws.elb.classification | string | | classification_reason | aws.elb.classification_reason | string | +### Integration Flow +The next section describes the integration responsibilities for creating the required assets to project ELB s3 based tables into OpenSearch ELB dashboards. + +Assuming all the prerequisites mentioned above are resolved, the first step would be to create the ELB logical table on the catalog ([Glue](https://aws.amazon.com/glue/)/[Hive](https://hive.apache.org/)) + +- [The ELB table definition](../assets/tables/aws_elb_s3_table-1.0.0.sql) this table definition is used by the Spark/EMR catalog) + +Once the table is created the next phase will be to generate the index template for the ELB log based on the simple schema for Observability index standard. +This index template will be augmented with the [covering index component template](../assets/indices/aws_elb_covering_index-1.0.0.mapping) (In addition to the other component templates) +So that the flint data loading process will have a valid index to load into. + +- Once this is done, the next phase will be to initiate the s3 based data loading into the ELB index by calling the [`refresh` command](../assets/tables/aws_elb_s3_refresh_covering_index-1.0.0.sql) + +The last part would be loading the visual assets including the dashboard that will show the ELB status according to the covering index data. + +### User Custom Parameters +The user has the next custom parameter which can be used to dictate the names of the indices and tables: + +- {table_name} - the table (FQN) name used to create the catalog table, example:`glue.default.elb-logs` +- {s3_bucket_location} - S3 bucket location, example -`'s3://your-alb-logs-directory/AWSLogs//elasticloadbalancing//'` +- {object_name} - the actual object name used to refer to by the index name , example -`elb_logs` \ No newline at end of file From b87e51b2246a126d41c90c4e8c32ccf7b704bf14 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Tue, 3 Oct 2023 22:28:24 -0700 Subject: [PATCH 06/10] cleanup elb metadata assets list Signed-off-by: YANGDB --- .../observability/aws_elb/aws_elb-1.0.0.json | 52 ++----------------- .../aws_elb/info/ELB_S3_DS_Intgration.md | 3 +- 2 files changed, 7 insertions(+), 48 deletions(-) diff --git a/integrations/observability/aws_elb/aws_elb-1.0.0.json b/integrations/observability/aws_elb/aws_elb-1.0.0.json index 1b0b1d9..e78f60c 100644 --- a/integrations/observability/aws_elb/aws_elb-1.0.0.json +++ b/integrations/observability/aws_elb/aws_elb-1.0.0.json @@ -47,6 +47,10 @@ "name": "aws_elb", "version": "1.0.0" }, + { + "name": "aws_elb_covering_index", + "version": "1.0.0" + }, { "name": "url", "version": "1.0.0" @@ -63,53 +67,7 @@ "version": "1.0.0" }, { - "name": "aws_elb_s3_skipping_index", - "version": "1.0.0" - } - ], - "indices": { - "name": "aws_elb_s3_opensearch_skipping_index_template", - "version": "1.0.0" - }, - "queries": [ - { - "name": "average_amount_of_data_query", - "description": " List the average amount of data (in kilobytes) that's passing through the Application Load Balancer in request or response pairs", - "version": "1.0.0" - }, - { - "name": "distinct_client_count_query", - "description": "List all client IP addresses that accessed the Application Load Balancer, and how many times they accessed the Application Load Balancer", - "version": "1.0.0" - }, - { - "name": "clients_by_visiting_urls", - "description": "List clients, in descending order, by the number of times that each client visited a specified URL", - "version": "1.0.0" - }, - { - "name": "clients_by_data", - "description": "List clients, in descending order, by the amount of data (in megabytes) that each client sent in their requests to the Application Load Balancer", - "version": "1.0.0" - }, - { - "name": "target_traffic_imbalance", - "description": "List all targets that the Application Load Balancer routes traffic to and the number of routed requests per target, by percentage distribution", - "version": "1.0.0" - }, - { - "name": "count_http_by_ip", - "description": "Count the number of HTTP GET requests received by the load balancer grouped by the client IP address", - "version": "1.0.0" - }, - { - "name": "largest_latency_by_date_range", - "description": "List each time in a specified date range when the target processing time was more than ? (5) seconds", - "version": "1.0.0" - }, - { - "name": "most_frequent_user_agents", - "description": "List the ? (10) URLs that ? (Firefox) users accessed most frequently, in descending order", + "name": "aws_elb_s3_refresh_covering_index", "version": "1.0.0" } ], diff --git a/integrations/observability/aws_elb/info/ELB_S3_DS_Intgration.md b/integrations/observability/aws_elb/info/ELB_S3_DS_Intgration.md index d0d3b27..28969e6 100644 --- a/integrations/observability/aws_elb/info/ELB_S3_DS_Intgration.md +++ b/integrations/observability/aws_elb/info/ELB_S3_DS_Intgration.md @@ -63,14 +63,15 @@ The next section describes the integration responsibilities for creating the req Assuming all the prerequisites mentioned above are resolved, the first step would be to create the ELB logical table on the catalog ([Glue](https://aws.amazon.com/glue/)/[Hive](https://hive.apache.org/)) - [The ELB table definition](../assets/tables/aws_elb_s3_table-1.0.0.sql) this table definition is used by the Spark/EMR catalog) - Once the table is created the next phase will be to generate the index template for the ELB log based on the simple schema for Observability index standard. + This index template will be augmented with the [covering index component template](../assets/indices/aws_elb_covering_index-1.0.0.mapping) (In addition to the other component templates) So that the flint data loading process will have a valid index to load into. - Once this is done, the next phase will be to initiate the s3 based data loading into the ELB index by calling the [`refresh` command](../assets/tables/aws_elb_s3_refresh_covering_index-1.0.0.sql) The last part would be loading the visual assets including the dashboard that will show the ELB status according to the covering index data. +- [Dashboards & visualization loading](../assets/aws_elb-1.0.0.ndjson) ### User Custom Parameters The user has the next custom parameter which can be used to dictate the names of the indices and tables: From ddd00dec7d66123693a7b04b16d44788035343ae Mon Sep 17 00:00:00 2001 From: YANGDB Date: Fri, 20 Oct 2023 16:00:54 -0700 Subject: [PATCH 07/10] update the view statement creation Signed-off-by: YANGDB --- .../assets/tables/aws_elb_s3_mv_index.sql | 41 ++++++++++ ...ws_elb_s3_refresh_covering_index-1.0.0.sql | 1 - .../assets/tables/aws_elb_s3_table-1.0.0.sql | 80 +++++++++---------- 3 files changed, 81 insertions(+), 41 deletions(-) create mode 100644 integrations/observability/aws_elb/assets/tables/aws_elb_s3_mv_index.sql delete mode 100644 integrations/observability/aws_elb/assets/tables/aws_elb_s3_refresh_covering_index-1.0.0.sql diff --git a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_mv_index.sql b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_mv_index.sql new file mode 100644 index 0000000..884e05c --- /dev/null +++ b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_mv_index.sql @@ -0,0 +1,41 @@ +CREATE VIEW AS + SELECT + type as `aws.elb.elb_type`, + time as `@timestamp`, + elb as `aws.elb.elb_name`, + client_ip as `aws.elb.client.ip`, + client_port as `aws.elb.client.port`, + target_ip as `aws.elb.target_ip`, + target_port as `aws.elb.target_port`, + request_processing_time as `aws.elb.request_processing_time`, + target_processing_time as `aws.elb.target_processing_time`, + response_processing_time as `aws.elb.response_processing_time`, + elb_status_code as `aws.elb.elb_status_code`, + target_status_code as `aws.elb.target_status_code`, + received_bytes as `aws.elb.received_bytes`, + sent_bytes as `aws.elb.sent_bytes`, + request_verb as `http.request.method`, + request_url as `url.full`, + request_proto as `url.schema`, + user_agent as `http.user_agent.name`, + ssl_cipher as `aws.elb.ssl_cipher`, + ssl_protocol as `aws.elb.ssl_protocol`, + target_group_arn as `aws.elb.target_group_arn`, + trace_id as `traceId`, + domain_name as `url.domain`, + chosen_cert_arn as `aws.elb.chosen_cert_arn`, + matched_rule_priority as `aws.elb.matched_rule_priority`, + request_creation_time as `aws.elb.request_creation_time`, + actions_executed as `aws.elb.actions_executed`, + redirect_url as `aws.elb.redirect_url`, + lambda_error_reason as `aws.elb.lambda_error_reason`, + target_port_list as `aws.elb.target_port_list`, + target_status_code_list as `aws.elb.target_status_code_list`, + classification as `aws.elb.classification`, + classification_reason as `aws.elb.classification_reason` + FROM mys3.default.elb_logs_regex +WITH ( + auto_refresh = true, + refresh_interval = '5 Seconds' + ) + \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_refresh_covering_index-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_refresh_covering_index-1.0.0.sql deleted file mode 100644 index 8b26c91..0000000 --- a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_refresh_covering_index-1.0.0.sql +++ /dev/null @@ -1 +0,0 @@ -REFRESH INDEX name ON {index_name} \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql index b8a1374..be089ad 100644 --- a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql +++ b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql @@ -1,40 +1,40 @@ -CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( - type string as aws.elb.elb_type, - time string as @timestamp, - elb string as aws.elb.elb_name, - client_ip string as aws.elb.client.ip, - client_port int as aws.elb.client.port, - target_ip string as aws.elb.target_ip, - target_port int as aws.elb.target_port, - request_processing_time double as aws.elb.request_processing_time, - target_processing_time double as aws.elb.target_processing_time, - response_processing_time double as aws.elb.response_processing_time, - elb_status_code int as aws.elb.elb_status_code, - target_status_code string as aws.elb.target_status_code, - received_bytes bigint as aws.elb.received_bytes, - sent_bytes bigint as aws.elb.sent_bytes, - request_verb string as http.request.method, - request_url string as url.full, - request_proto string as url.schema, - user_agent string as http.user_agent.name, - ssl_cipher string as aws.elb.ssl_cipher, - ssl_protocol string as aws.elb.ssl_protocol , - target_group_arn string as aws.elb.target_group_arn, - trace_id string as traceId, - domain_name string as url.domain, - chosen_cert_arn string as aws.elb.chosen_cert_arn, - matched_rule_priority string as aws.elb.matched_rule_priority, - request_creation_time string as aws.elb.request_creation_time, - actions_executed string as aws.elb.actions_executed, - redirect_url string as aws.elb.redirect_url, - lambda_error_reason string as aws.elb.lambda_error_reason, - target_port_list string as aws.elb.target_port_list, - target_status_code_list string as aws.elb.target_status_code_list, - classification string as aws.elb.classification , - classification_reason string as aws.elb.classification_reason - ) - ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe' - WITH SERDEPROPERTIES ( - 'serialization.format' = '1', - 'input.regex' = '([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) (.*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\s]+?)\" \"([^\s]+)\" \"([^ ]*)\" \"([^ ]*)\"') - LOCATION {s3_bucket_location} \ No newline at end of file +CREATE EXTERNAL TABLE IF NOT EXISTS ${datasource}.${catalog}.${table} ( + type string, + time string, + elb string, + client_ip string, + client_port int, + target_ip string, + target_port int, + request_processing_time double, + target_processing_time double, + response_processing_time double, + elb_status_code int, + target_status_code string, + received_bytes bigint, + sent_bytes bigint, + request_verb string, + request_url string, + request_proto string, + user_agent string, + ssl_cipher string, + ssl_protocol string, + target_group_arn string, + trace_id string, + domain_name string, + chosen_cert_arn string, + matched_rule_priority string, + request_creation_time string, + actions_executed string, + redirect_url string, + lambda_error_reason string, + target_port_list string, + target_status_code_list string, + classification string, + classification_reason string +) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe' +WITH + SERDEPROPERTIES ( + 'serialization.format' = '1', + 'input.regex' = '([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) (.*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\s]+?)\" \"([^\s]+)\" \"([^ ]*)\" \"([^ ]*)\"' + ) LOCATION ${s3_bucket}; \ No newline at end of file From 66298bbe2d50aea03ffb8d1d686e292b25127b18 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Fri, 20 Oct 2023 16:03:21 -0700 Subject: [PATCH 08/10] update the elb s3 based assets Signed-off-by: YANGDB --- .../aws_elb_covering_index-1.0.0.mapping | 118 ------------------ ...ndex.sql => aws_elb_s3_mv_index-1.0.0.sql} | 0 .../observability/aws_elb/aws_elb-1.0.0.json | 6 +- 3 files changed, 1 insertion(+), 123 deletions(-) delete mode 100644 integrations/observability/aws_elb/assets/indices/aws_elb_covering_index-1.0.0.mapping rename integrations/observability/aws_elb/assets/tables/{aws_elb_s3_mv_index.sql => aws_elb_s3_mv_index-1.0.0.sql} (100%) diff --git a/integrations/observability/aws_elb/assets/indices/aws_elb_covering_index-1.0.0.mapping b/integrations/observability/aws_elb/assets/indices/aws_elb_covering_index-1.0.0.mapping deleted file mode 100644 index 001561c..0000000 --- a/integrations/observability/aws_elb/assets/indices/aws_elb_covering_index-1.0.0.mapping +++ /dev/null @@ -1,118 +0,0 @@ -{ - "template": { - "aliases": { - "flint_{object_name}_index": {} - }, - "mappings": { - "_meta": { - "version": "1.0.0", - "name": "{object_name}", - "kind": "covering", - "indexedColumns": [ - { - "columnName": "aws.elb.backend.ip", - "columnType": "ip" - }, - { - "columnName": "aws.elb.backend.port", - "columnType": "integer" - }, - { - "columnName": "aws.elb.backend.processing_time", - "columnType": "half_float" - }, - { - "columnName": "aws.elb.backend.status_code", - "columnType": "short" - }, - { - "columnName": "aws.elb.client.ip", - "columnType": "ip" - }, - { - "columnName": "aws.elb.client.port", - "columnType": "integer" - }, - { - "columnName": "aws.elb.connection_time", - "columnType": "integer" - }, - { - "columnName": "aws.elb.destination.ip", - "columnType": "ip" - }, - { - "columnName": "aws.elb.destination.port", - "columnType": "integer" - }, - { - "columnName": "aws.elb.elb_status_code", - "columnType": "short" - }, - { - "columnName": "aws.elb.http.port", - "columnType": "integer" - }, - { - "columnName": "aws.elb.http.version", - "columnType": "keyword" - }, - { - "columnName": "aws.elb.matched_rule_priority", - "columnType": "integer" - }, - { - "columnName": "aws.elb.received_bytes", - "columnType": "integer" - }, - { - "columnName": "aws.elb.request_creation_time", - "columnType": "date" - }, - { - "columnName": "aws.elb.request_processing_time", - "columnType": "half_float" - }, - { - "columnName": "aws.elb.response_processing_time", - "columnType": "half_float" - }, - { - "columnName": "aws.elb.sent_bytes", - "columnType": "integer" - }, - { - "columnName": "aws.elb.ssl_protocol", - "columnType": "keyword" - }, - { - "columnName": "aws.elb.ssl_cipher", - "columnType": "keyword" - }, - { - "columnName": "aws.elb.target_ip", - "columnType": "ip" - }, - { - "columnName": "aws.elb.target_port", - "columnType": "integer" - }, - { - "columnName": "aws.elb.target_processing_time", - "columnType": "half_float" - }, - { - "columnName": "aws.elb.target_status_code", - "columnType": "short" - }, - { - "columnName": "aws.elb.timestamp", - "columnType": "date" - } - ], - "source": "{table_name}", - "options": {} - } - } - } -} \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_mv_index.sql b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_mv_index-1.0.0.sql similarity index 100% rename from integrations/observability/aws_elb/assets/tables/aws_elb_s3_mv_index.sql rename to integrations/observability/aws_elb/assets/tables/aws_elb_s3_mv_index-1.0.0.sql diff --git a/integrations/observability/aws_elb/aws_elb-1.0.0.json b/integrations/observability/aws_elb/aws_elb-1.0.0.json index e78f60c..28b7269 100644 --- a/integrations/observability/aws_elb/aws_elb-1.0.0.json +++ b/integrations/observability/aws_elb/aws_elb-1.0.0.json @@ -47,10 +47,6 @@ "name": "aws_elb", "version": "1.0.0" }, - { - "name": "aws_elb_covering_index", - "version": "1.0.0" - }, { "name": "url", "version": "1.0.0" @@ -67,7 +63,7 @@ "version": "1.0.0" }, { - "name": "aws_elb_s3_refresh_covering_index", + "name": "aws_elb_s3_mv_index", "version": "1.0.0" } ], From c2bfe98c88c89ed847fab148cffd44724612d2e7 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Fri, 20 Oct 2023 16:09:49 -0700 Subject: [PATCH 09/10] update the elb raw logs Signed-off-by: YANGDB --- .../observability/aws_elb/data/raw-sample.txt | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 integrations/observability/aws_elb/data/raw-sample.txt diff --git a/integrations/observability/aws_elb/data/raw-sample.txt b/integrations/observability/aws_elb/data/raw-sample.txt new file mode 100644 index 0000000..44463af --- /dev/null +++ b/integrations/observability/aws_elb/data/raw-sample.txt @@ -0,0 +1,63 @@ +http 2022-07-05T04:47:50.273035Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:65301 192.168.84.111:80 0.001 0.000 0.000 404 404 472 696 "GET http://54.186.209.42:80//_profiler/phpinfo HTTP/1.1" "Mozlila/5.0 (Linux; Android 7.0; SM-G892A Bulid/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Moblie Safari/537.36" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c276-7ebec4d5497565d773421350" "-" "-" 1 2022-07-05T04:47:50.271000Z "forward" "-" "-" "192.168.84.111:80" "404" "-" "-" +http 2022-07-05T04:47:50.887193Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:49331 192.168.81.116:80 0.004 0.002 0.000 404 404 462 696 "GET http://54.186.209.42:80//phpinfo HTTP/1.1" "Mozlila/5.0 (Linux; Android 7.0; SM-G892A Bulid/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Moblie Safari/537.36" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c276-1e32ac363382765a21447e3c" "-" "-" 1 2022-07-05T04:47:50.881000Z "forward" "-" "-" "192.168.81.116:80" "404" "-" "-" +http 2022-07-05T04:47:51.660903Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:65252 192.168.88.185:80 0.001 0.000 0.000 404 404 463 696 "GET http://54.186.209.42:80//info.php HTTP/1.1" "Mozlila/5.0 (Linux; Android 7.0; SM-G892A Bulid/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Moblie Safari/537.36" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c277-400f73fd3c1157171f438d01" "-" "-" 1 2022-07-05T04:47:51.659000Z "forward" "-" "-" "192.168.88.185:80" "404" "-" "-" +http 2022-07-05T04:47:51.888411Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63971 192.168.81.46:80 0.001 0.000 0.000 404 404 471 696 "GET http://54.186.209.42:80//.aws/credentials HTTP/1.1" "Mozlila/5.0 (Linux; Android 7.0; SM-G892A Bulid/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Moblie Safari/537.36" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c277-789e17172847ec3d4c63dc2b" "-" "-" 1 2022-07-05T04:47:51.887000Z "forward" "-" "-" "192.168.81.46:80" "404" "-" "-" +http 2022-07-05T04:47:52.144849Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:51670 192.168.83.3:80 0.000 0.000 0.000 200 200 455 4220 "GET http://54.186.209.42:80// HTTP/1.1" "Mozlila/5.0 (Linux; Android 7.0; SM-G892A Bulid/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Moblie Safari/537.36" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c278-7ac99ada19b1b4813f599da0" "-" "-" 1 2022-07-05T04:47:52.143000Z "forward" "-" "-" "192.168.83.3:80" "200" "-" "-" +http 2022-07-05T04:47:52.988671Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64281 192.168.84.111:80 0.000 0.001 0.000 200 200 541 4215 "GET http://54.186.209.42:80// HTTP/1.1" "Mozlila/5.0 (Linux; Android 7.0; SM-G892A Bulid/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Moblie Safari/537.36" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c278-4e187f596d0856814c36744d" "-" "-" 1 2022-07-05T04:47:52.889000Z "forward" "-" "-" "192.168.84.111:80" "200" "Ambiguous" "UndefinedContentLengthSemantics" +http 2022-07-05T04:47:53.328066Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64196 192.168.81.116:80 0.000 0.000 0.000 404 404 176 294 "GET http://54.186.209.42:80//debug/default/view?panel=config HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c279-4a73188c556172162e54eb56" "-" "-" 1 2022-07-05T04:47:53.327000Z "forward" "-" "-" "192.168.81.116:80" "404" "-" "-" +http 2022-07-05T04:47:53.632439Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64540 192.168.88.185:80 0.000 0.001 0.000 404 404 152 294 "GET http://54.186.209.42:80//php.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c279-6dcb85ce383efab263d9f4e8" "-" "-" 1 2022-07-05T04:47:53.631000Z "forward" "-" "-" "192.168.88.185:80" "404" "-" "-" +http 2022-07-05T04:47:54.139219Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63576 192.168.81.46:80 0.000 0.001 0.000 404 404 153 294 "GET http://54.186.209.42:80//info.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27a-0ffcc2336d6100d53330cd28" "-" "-" 1 2022-07-05T04:47:54.138000Z "forward" "-" "-" "192.168.81.46:80" "404" "-" "-" +http 2022-07-05T04:47:54.362694Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63643 192.168.83.3:80 0.000 0.000 0.000 404 404 156 294 "GET http://54.186.209.42:80//infophp.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27a-4c2550372835b9742cb12b2f" "-" "-" 1 2022-07-05T04:47:54.361000Z "forward" "-" "-" "192.168.83.3:80" "404" "-" "-" +http 2022-07-05T04:47:55.056685Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64548 192.168.84.111:80 0.000 0.000 0.000 404 404 150 294 "GET http://54.186.209.42:80//i.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27b-7c2675e1003850bd5992d0c8" "-" "-" 1 2022-07-05T04:47:55.055000Z "forward" "-" "-" "192.168.84.111:80" "404" "-" "-" +http 2022-07-05T04:47:55.587190Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64213 192.168.81.116:80 0.000 0.001 0.000 404 404 154 294 "GET http://54.186.209.42:80//pinfo.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27b-51211ba3045e7fde546414a2" "-" "-" 1 2022-07-05T04:47:55.586000Z "forward" "-" "-" "192.168.81.116:80" "404" "-" "-" +http 2022-07-05T04:47:55.852966Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63424 192.168.88.185:80 0.000 0.001 0.000 404 404 159 294 "GET http://54.186.209.42:80//phpversion.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27b-20034713456c186d57488059" "-" "-" 1 2022-07-05T04:47:55.852000Z "forward" "-" "-" "192.168.88.185:80" "404" "-" "-" +http 2022-07-05T04:47:56.080628Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63843 192.168.81.46:80 0.000 0.000 0.000 404 404 153 294 "GET http://54.186.209.42:80//time.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27c-66f29e2609e1d60031bb4005" "-" "-" 1 2022-07-05T04:47:56.079000Z "forward" "-" "-" "192.168.81.46:80" "404" "-" "-" +http 2022-07-05T04:47:56.318154Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64174 192.168.83.3:80 0.000 0.000 0.000 404 404 154 294 "GET http://54.186.209.42:80//index.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27c-07ef6bf80d1fcf57053a3a5f" "-" "-" 1 2022-07-05T04:47:56.317000Z "forward" "-" "-" "192.168.83.3:80" "404" "-" "-" +http 2022-07-05T04:47:56.601287Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64519 192.168.84.111:80 0.000 0.000 0.000 404 404 153 294 "GET http://54.186.209.42:80//temp.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27c-458028a77e901e8c6e776ac6" "-" "-" 1 2022-07-05T04:47:56.600000Z "forward" "-" "-" "192.168.84.111:80" "404" "-" "-" +http 2022-07-05T04:47:57.932931Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64041 192.168.81.116:80 0.000 0.000 0.000 404 404 166 294 "GET http://54.186.209.42:80//dashboard/phpinfo.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27d-63a567f0000accd71761a239" "-" "-" 1 2022-07-05T04:47:57.932000Z "forward" "-" "-" "192.168.81.116:80" "404" "-" "-" +http 2022-07-05T04:47:58.748196Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63834 192.168.88.185:80 0.000 0.000 0.000 404 404 175 294 "GET http://54.186.209.42:80//adminphp.php/configuration.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27e-47197fe0790d4aec01457e91" "-" "-" 1 2022-07-05T04:47:58.747000Z "forward" "-" "-" "192.168.88.185:80" "404" "-" "-" +http 2022-07-05T04:47:58.964181Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63578 192.168.81.46:80 0.000 0.000 0.000 404 404 152 294 "GET http://54.186.209.42:80//123.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27e-6d949dba3804a2c424e0671a" "-" "-" 1 2022-07-05T04:47:58.963000Z "forward" "-" "-" "192.168.81.46:80" "404" "-" "-" +http 2022-07-05T04:47:59.259037Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63436 192.168.83.3:80 0.000 0.001 0.000 404 404 156 294 "GET http://54.186.209.42:80//infophp.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27f-79c806842cea7a0f3e791b8b" "-" "-" 1 2022-07-05T04:47:59.258000Z "forward" "-" "-" "192.168.83.3:80" "404" "-" "-" +http 2022-07-05T04:47:59.570104Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64539 192.168.84.111:80 0.000 0.000 0.000 404 404 156 294 "GET http://54.186.209.42:80//isadmin.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27f-7ee8b0ab077163947a2c1df5" "-" "-" 1 2022-07-05T04:47:59.569000Z "forward" "-" "-" "192.168.84.111:80" "404" "-" "-" +http 2022-07-05T04:48:02.777908Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64566 192.168.81.116:80 0.000 0.001 0.000 404 404 151 294 "GET http://54.186.209.42:80//qq.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c282-5be4a529345db70f729b6575" "-" "-" 1 2022-07-05T04:48:02.777000Z "forward" "-" "-" "192.168.81.116:80" "404" "-" "-" +http 2022-07-05T04:48:03.004093Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:62792 192.168.88.185:80 0.000 0.001 0.000 404 404 154 294 "GET http://54.186.209.42:80//test0.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c283-286e7dab5baad0c205e62321" "-" "-" 1 2022-07-05T04:48:03.003000Z "forward" "-" "-" "192.168.88.185:80" "404" "-" "-" +http 2022-07-05T04:48:03.605639Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:62536 192.168.81.46:80 0.000 0.001 0.000 404 404 154 294 "GET http://54.186.209.42:80//test1.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c283-722814cb4ea220c4528a3b87" "-" "-" 1 2022-07-05T04:48:03.604000Z "forward" "-" "-" "192.168.81.46:80" "404" "-" "-" +http 2022-07-05T04:48:03.956331Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:62895 192.168.83.3:80 0.000 0.001 0.000 404 404 156 294 "GET http://54.186.209.42:80//test123.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c283-4b8f5157487cc581524f9cf7" "-" "-" 1 2022-07-05T04:48:03.955000Z "forward" "-" "-" "192.168.83.3:80" "404" "-" "-" +http 2022-07-05T04:48:04.229161Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:61653 192.168.84.111:80 0.000 0.000 0.000 404 404 150 294 "GET http://54.186.209.42:80//test2 HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c284-26718a804eb91a1925288da6" "-" "-" 1 2022-07-05T04:48:04.228000Z "forward" "-" "-" "192.168.84.111:80" "404" "-" "-" +http 2022-07-05T04:48:05.081631Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63162 192.168.81.116:80 0.000 0.001 0.000 404 404 154 294 "GET http://54.186.209.42:80//test4.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c285-64aca78167d0e3b46fc83be5" "-" "-" 1 2022-07-05T04:48:05.080000Z "forward" "-" "-" "192.168.81.116:80" "404" "-" "-" +http 2022-07-05T04:48:05.753240Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63304 192.168.88.185:80 0.000 0.001 0.000 404 404 154 294 "GET http://54.186.209.42:80//test6.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c285-7cc1842b2fc36e7c13172c86" "-" "-" 1 2022-07-05T04:48:05.751000Z "forward" "-" "-" "192.168.88.185:80" "404" "-" "-" +http 2022-07-05T04:48:05.999782Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:62869 192.168.81.46:80 0.000 0.000 0.000 404 404 154 294 "GET http://54.186.209.42:80//test7.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c285-50b45d611928095f52b60eec" "-" "-" 1 2022-07-05T04:48:05.999000Z "forward" "-" "-" "192.168.81.46:80" "404" "-" "-" +http 2022-07-05T04:48:06.501199Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:62950 192.168.83.3:80 0.000 0.001 0.000 404 404 154 294 "GET http://54.186.209.42:80//test9.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c286-4967a5a63c78ac5c73ee7012" "-" "-" 1 2022-07-05T04:48:06.500000Z "forward" "-" "-" "192.168.83.3:80" "404" "-" "-" +http 2022-07-05T04:47:50.575546Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64944 192.168.81.116:80 0.001 0.000 0.000 404 404 466 696 "GET http://54.186.209.42:80//phpinfo.php HTTP/1.1" "Mozlila/5.0 (Linux; Android 7.0; SM-G892A Bulid/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Moblie Safari/537.36" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c276-789963023ea8f6877335ce89" "-" "-" 1 2022-07-05T04:47:50.574000Z "forward" "-" "-" "192.168.81.116:80" "404" "-" "-" +http 2022-07-05T04:47:51.146106Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64770 192.168.88.185:80 0.001 0.000 0.000 404 404 462 696 "GET http://54.186.209.42:80//aws.yml HTTP/1.1" "Mozlila/5.0 (Linux; Android 7.0; SM-G892A Bulid/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Moblie Safari/537.36" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c277-169f81d8416ea99c6a6c2940" "-" "-" 1 2022-07-05T04:47:51.145000Z "forward" "-" "-" "192.168.88.185:80" "404" "-" "-" +http 2022-07-05T04:47:51.374334Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64608 192.168.81.46:80 0.001 0.000 0.000 404 404 463 696 "GET http://54.186.209.42:80//.env.bak HTTP/1.1" "Mozlila/5.0 (Linux; Android 7.0; SM-G892A Bulid/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Moblie Safari/537.36" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c277-674c59622f71a7622dbde22d" "-" "-" 1 2022-07-05T04:47:51.373000Z "forward" "-" "-" "192.168.81.46:80" "404" "-" "-" +http 2022-07-05T04:47:52.368462Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64823 192.168.83.3:80 0.001 0.000 0.000 404 404 473 696 "GET http://54.186.209.42:80//frontend_dev.php/$ HTTP/1.1" "Mozlila/5.0 (Linux; Android 7.0; SM-G892A Bulid/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Moblie Safari/537.36" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c278-42b0afd354711f3a62546d39" "-" "-" 1 2022-07-05T04:47:52.367000Z "forward" "-" "-" "192.168.83.3:80" "404" "-" "-" +http 2022-07-05T04:47:53.111022Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64005 192.168.84.111:80 0.001 0.001 0.000 404 404 464 696 "GET http://54.186.209.42:80//config.js HTTP/1.1" "Mozlila/5.0 (Linux; Android 7.0; SM-G892A Bulid/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Moblie Safari/537.36" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c279-022b33b3671315255073ec02" "-" "-" 1 2022-07-05T04:47:53.109000Z "forward" "-" "-" "192.168.84.111:80" "404" "-" "-" +http 2022-07-05T04:47:53.860222Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63362 192.168.81.116:80 0.000 0.001 0.000 404 404 156 294 "GET http://54.186.209.42:80//phpinfo.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c279-41b589fd60c0064b39de8838" "-" "-" 1 2022-07-05T04:47:53.859000Z "forward" "-" "-" "192.168.81.116:80" "404" "-" "-" +http 2022-07-05T04:47:54.624882Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64619 192.168.88.185:80 0.000 0.000 0.000 404 404 157 294 "GET http://54.186.209.42:80//php_info.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27a-2c49ea453d67cf3b6c272efd" "-" "-" 1 2022-07-05T04:47:54.624000Z "forward" "-" "-" "192.168.88.185:80" "404" "-" "-" +http 2022-07-05T04:47:54.841404Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63961 192.168.81.46:80 0.000 0.001 0.000 404 404 153 294 "GET http://54.186.209.42:80//test.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27a-1649d8fb0ed3daee40c9d852" "-" "-" 1 2022-07-05T04:47:54.840000Z "forward" "-" "-" "192.168.81.46:80" "404" "-" "-" +http 2022-07-05T04:47:55.297015Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64161 192.168.83.3:80 0.000 0.000 0.000 404 404 153 294 "GET http://54.186.209.42:80//asdf.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27b-2a272b865ed95aee4eabdcfc" "-" "-" 1 2022-07-05T04:47:55.296000Z "forward" "-" "-" "192.168.83.3:80" "404" "-" "-" +http 2022-07-05T04:47:56.868338Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63450 192.168.84.111:80 0.000 0.000 0.000 404 404 160 294 "GET http://54.186.209.42:80//old_phpinfo.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27c-2be5860f5600a7030016581c" "-" "-" 1 2022-07-05T04:47:56.867000Z "forward" "-" "-" "192.168.84.111:80" "404" "-" "-" +http 2022-07-05T04:47:57.119160Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:62453 192.168.81.116:80 0.000 0.000 0.000 404 404 154 294 "GET http://54.186.209.42:80//infos.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27d-796b48ef4d0f70da2c4bd692" "-" "-" 1 2022-07-05T04:47:57.118000Z "forward" "-" "-" "192.168.81.116:80" "404" "-" "-" +http 2022-07-05T04:47:57.404884Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63761 192.168.88.185:80 0.000 0.001 0.000 404 404 167 294 "GET http://54.186.209.42:80//linusadmin-phpinfo.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27d-46bae46c5668d23a33bf5258" "-" "-" 1 2022-07-05T04:47:57.404000Z "forward" "-" "-" "192.168.88.185:80" "404" "-" "-" +http 2022-07-05T04:47:57.652526Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64131 192.168.81.46:80 0.000 0.000 0.000 404 404 157 294 "GET http://54.186.209.42:80//php-info.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27d-40421c8c601005b567d9e0ee" "-" "-" 1 2022-07-05T04:47:57.651000Z "forward" "-" "-" "192.168.81.46:80" "404" "-" "-" +http 2022-07-05T04:47:58.205925Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64648 192.168.83.3:80 0.000 0.001 0.000 404 404 150 294 "GET http://54.186.209.42:80//a.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27e-351af93d6ddc69261fb0c8b3" "-" "-" 1 2022-07-05T04:47:58.205000Z "forward" "-" "-" "192.168.83.3:80" "404" "-" "-" +http 2022-07-05T04:47:58.522863Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64212 192.168.84.111:80 0.000 0.001 0.000 404 404 153 294 "GET http://54.186.209.42:80//php1.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27e-46e500a64af2f94d1ec26561" "-" "-" 1 2022-07-05T04:47:58.522000Z "forward" "-" "-" "192.168.84.111:80" "404" "-" "-" +http 2022-07-05T04:47:59.901097Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63379 192.168.81.116:80 0.000 0.000 0.000 404 404 151 294 "GET http://54.186.209.42:80//jo.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c27f-0bcbe17d12d3d59e232416af" "-" "-" 1 2022-07-05T04:47:59.900000Z "forward" "-" "-" "192.168.81.116:80" "404" "-" "-" +http 2022-07-05T04:48:00.107933Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63243 192.168.88.185:80 0.000 0.001 0.000 404 404 155 294 "GET http://54.186.209.42:80//lindex.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c280-1d033abb3b740758494c6ad8" "-" "-" 1 2022-07-05T04:48:00.107000Z "forward" "-" "-" "192.168.88.185:80" "404" "-" "-" +http 2022-07-05T04:48:00.407959Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63466 192.168.81.46:80 0.000 0.001 0.000 404 404 152 294 "GET http://54.186.209.42:80//new.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c280-1e7ea4696e5ebe7455cdb459" "-" "-" 1 2022-07-05T04:48:00.407000Z "forward" "-" "-" "192.168.81.46:80" "404" "-" "-" +http 2022-07-05T04:48:00.699160Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64297 192.168.83.3:80 0.000 0.000 0.000 404 404 152 294 "GET http://54.186.209.42:80//phpinfo HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c280-113677ce2d8451af6045df15" "-" "-" 1 2022-07-05T04:48:00.698000Z "forward" "-" "-" "192.168.83.3:80" "404" "-" "-" +http 2022-07-05T04:48:00.947750Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63471 192.168.84.111:80 0.000 0.000 0.000 404 404 157 294 "GET http://54.186.209.42:80//phpinfo.php3 HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c280-7504ccbc2beaa0006f912ec9" "-" "-" 1 2022-07-05T04:48:00.947000Z "forward" "-" "-" "192.168.84.111:80" "404" "-" "-" +http 2022-07-05T04:48:01.223652Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63562 192.168.81.116:80 0.000 0.003 0.000 404 404 157 294 "GET http://54.186.209.42:80//phpinfo.php5 HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c281-403774f55a0bebbf7ec62d36" "-" "-" 1 2022-07-05T04:48:01.220000Z "forward" "-" "-" "192.168.81.116:80" "404" "-" "-" +http 2022-07-05T04:48:01.617029Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:62558 192.168.88.185:80 0.000 0.003 0.000 404 404 157 294 "GET http://54.186.209.42:80//phpinfos.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c281-5f23b54b7f0bad994d8c2532" "-" "-" 1 2022-07-05T04:48:01.613000Z "forward" "-" "-" "192.168.88.185:80" "404" "-" "-" +http 2022-07-05T04:48:01.865066Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:62834 192.168.81.46:80 0.000 0.000 0.000 404 404 156 294 "GET http://54.186.209.42:80//phptest.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c281-2b9939284545f574766182ff" "-" "-" 1 2022-07-05T04:48:01.864000Z "forward" "-" "-" "192.168.81.46:80" "404" "-" "-" +http 2022-07-05T04:48:02.143039Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63986 192.168.83.3:80 0.000 0.001 0.000 404 404 151 294 "GET http://54.186.209.42:80//pi.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c282-701af202496d073a197726bf" "-" "-" 1 2022-07-05T04:48:02.142000Z "forward" "-" "-" "192.168.83.3:80" "404" "-" "-" +http 2022-07-05T04:48:02.439951Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:64296 192.168.84.111:80 0.000 0.001 0.000 404 404 152 294 "GET http://54.186.209.42:80//pi.php5 HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c282-6d04bdd26ccf433d2e283188" "-" "-" 1 2022-07-05T04:48:02.438000Z "forward" "-" "-" "192.168.84.111:80" "404" "-" "-" +http 2022-07-05T04:48:03.271107Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:62394 192.168.81.116:80 0.000 0.001 0.000 404 404 150 294 "GET http://54.186.209.42:80//test1 HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c283-10af13f1483bb2e0535e9c6e" "-" "-" 1 2022-07-05T04:48:03.269000Z "forward" "-" "-" "192.168.81.116:80" "404" "-" "-" +http 2022-07-05T04:48:04.493817Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:62482 192.168.88.185:80 0.000 0.001 0.000 404 404 154 294 "GET http://54.186.209.42:80//test2.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c284-3cdb7059688d10ab70354765" "-" "-" 1 2022-07-05T04:48:04.492000Z "forward" "-" "-" "192.168.88.185:80" "404" "-" "-" +http 2022-07-05T04:48:04.805237Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:62752 192.168.81.46:80 0.000 0.001 0.000 404 404 154 294 "GET http://54.186.209.42:80//test3.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c284-6f3e45af1758f2b71feaae06" "-" "-" 1 2022-07-05T04:48:04.803000Z "forward" "-" "-" "192.168.81.46:80" "404" "-" "-" +http 2022-07-05T04:48:05.368207Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:62980 192.168.83.3:80 0.000 0.001 0.000 404 404 154 294 "GET http://54.186.209.42:80//test5.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c285-2f6c55d25896beab0cc59aeb" "-" "-" 1 2022-07-05T04:48:05.367000Z "forward" "-" "-" "192.168.83.3:80" "404" "-" "-" +http 2022-07-05T04:48:06.239185Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:61583 192.168.84.111:80 0.000 0.001 0.000 404 404 154 294 "GET http://54.186.209.42:80//test8.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c286-63b0a9d15a6720b75920d62d" "-" "-" 1 2022-07-05T04:48:06.238000Z "forward" "-" "-" "192.168.84.111:80" "404" "-" "-" +http 2022-07-05T04:48:06.820350Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:62836 192.168.81.116:80 0.000 0.002 0.000 404 404 151 294 "GET http://54.186.209.42:80//up.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c286-251862e47ba682693b9c41ed" "-" "-" 1 2022-07-05T04:48:06.817000Z "forward" "-" "-" "192.168.81.116:80" "404" "-" "-" +http 2022-07-05T04:48:07.115557Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:63091 192.168.88.185:80 0.000 0.001 0.000 404 404 150 294 "GET http://54.186.209.42:80//w.php HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c287-677dff0e52fa6bb9057c10f4" "-" "-" 1 2022-07-05T04:48:07.114000Z "forward" "-" "-" "192.168.88.185:80" "404" "-" "-" +http 2022-07-05T04:48:07.456254Z app/k8s-game2048-ingress2-56f7898e10/2b395c9f7d176e32 20.78.123.34:61632 192.168.81.46:80 0.000 0.001 0.000 404 404 152 294 "GET http://54.186.209.42:80//php.ini HTTP/1.1" "python-requests/2.28.0" - - arn:aws:elasticloadbalancing:us-west-2:347283850106:targetgroup/k8s-game2048-service2-d1b9eaccf8/c979ac34eb74c5c5 "Root=1-62c3c287-6182d3d42e04e9691954ac98" "-" "-" 1 2022-07-05T04:48:07.455000Z "forward" "-" "-" "192.168.81.46:80" "404" "-" "-" \ No newline at end of file From 20e5b05b0a06af0c1eb7b7ea1adf34a69d0df62b Mon Sep 17 00:00:00 2001 From: YANGDB Date: Tue, 31 Oct 2023 09:38:45 -0700 Subject: [PATCH 10/10] update assets name & content Signed-off-by: YANGDB --- .../tables/aws_elb_s3_mv_index-1.0.0.sql | 41 ------------------- .../assets/tables/aws_elb_s3_table-1.0.0.sql | 40 ------------------ .../assets/tables/create_mv_elb-1.0.0.sql | 41 +++++++++++++++++++ .../assets/tables/create_table_elb-1.0.0.sql | 36 ++++++++++++++++ .../assets/tables/refresh_mv_elb-1.0.0.sql | 1 + .../observability/aws_elb/aws_elb-1.0.0.json | 29 ++++++------- .../aws_elb/info/ELB_S3_DS_Intgration.md | 4 +- 7 files changed, 93 insertions(+), 99 deletions(-) delete mode 100644 integrations/observability/aws_elb/assets/tables/aws_elb_s3_mv_index-1.0.0.sql delete mode 100644 integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql create mode 100644 integrations/observability/aws_elb/assets/tables/create_mv_elb-1.0.0.sql create mode 100644 integrations/observability/aws_elb/assets/tables/create_table_elb-1.0.0.sql create mode 100644 integrations/observability/aws_elb/assets/tables/refresh_mv_elb-1.0.0.sql diff --git a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_mv_index-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_mv_index-1.0.0.sql deleted file mode 100644 index 884e05c..0000000 --- a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_mv_index-1.0.0.sql +++ /dev/null @@ -1,41 +0,0 @@ -CREATE VIEW AS - SELECT - type as `aws.elb.elb_type`, - time as `@timestamp`, - elb as `aws.elb.elb_name`, - client_ip as `aws.elb.client.ip`, - client_port as `aws.elb.client.port`, - target_ip as `aws.elb.target_ip`, - target_port as `aws.elb.target_port`, - request_processing_time as `aws.elb.request_processing_time`, - target_processing_time as `aws.elb.target_processing_time`, - response_processing_time as `aws.elb.response_processing_time`, - elb_status_code as `aws.elb.elb_status_code`, - target_status_code as `aws.elb.target_status_code`, - received_bytes as `aws.elb.received_bytes`, - sent_bytes as `aws.elb.sent_bytes`, - request_verb as `http.request.method`, - request_url as `url.full`, - request_proto as `url.schema`, - user_agent as `http.user_agent.name`, - ssl_cipher as `aws.elb.ssl_cipher`, - ssl_protocol as `aws.elb.ssl_protocol`, - target_group_arn as `aws.elb.target_group_arn`, - trace_id as `traceId`, - domain_name as `url.domain`, - chosen_cert_arn as `aws.elb.chosen_cert_arn`, - matched_rule_priority as `aws.elb.matched_rule_priority`, - request_creation_time as `aws.elb.request_creation_time`, - actions_executed as `aws.elb.actions_executed`, - redirect_url as `aws.elb.redirect_url`, - lambda_error_reason as `aws.elb.lambda_error_reason`, - target_port_list as `aws.elb.target_port_list`, - target_status_code_list as `aws.elb.target_status_code_list`, - classification as `aws.elb.classification`, - classification_reason as `aws.elb.classification_reason` - FROM mys3.default.elb_logs_regex -WITH ( - auto_refresh = true, - refresh_interval = '5 Seconds' - ) - \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql deleted file mode 100644 index be089ad..0000000 --- a/integrations/observability/aws_elb/assets/tables/aws_elb_s3_table-1.0.0.sql +++ /dev/null @@ -1,40 +0,0 @@ -CREATE EXTERNAL TABLE IF NOT EXISTS ${datasource}.${catalog}.${table} ( - type string, - time string, - elb string, - client_ip string, - client_port int, - target_ip string, - target_port int, - request_processing_time double, - target_processing_time double, - response_processing_time double, - elb_status_code int, - target_status_code string, - received_bytes bigint, - sent_bytes bigint, - request_verb string, - request_url string, - request_proto string, - user_agent string, - ssl_cipher string, - ssl_protocol string, - target_group_arn string, - trace_id string, - domain_name string, - chosen_cert_arn string, - matched_rule_priority string, - request_creation_time string, - actions_executed string, - redirect_url string, - lambda_error_reason string, - target_port_list string, - target_status_code_list string, - classification string, - classification_reason string -) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe' -WITH - SERDEPROPERTIES ( - 'serialization.format' = '1', - 'input.regex' = '([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) (.*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\s]+?)\" \"([^\s]+)\" \"([^ ]*)\" \"([^ ]*)\"' - ) LOCATION ${s3_bucket}; \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/tables/create_mv_elb-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/create_mv_elb-1.0.0.sql new file mode 100644 index 0000000..3c7bbe3 --- /dev/null +++ b/integrations/observability/aws_elb/assets/tables/create_mv_elb-1.0.0.sql @@ -0,0 +1,41 @@ +CREATE MATERIALIZED VIEW {table_name}_mview AS +SELECT + type as `aws.elb.elb_type`, + time as `@timestamp`, + elb as `aws.elb.elb_name`, + split_part (client_ip, ':', 1) as `communication.source.ip`, + split_part (client_ip, ':', 2) as `communication.source.port`, + split_part (target_ip, ':', 1) as `communication.destination.ip`, + split_part (target_ip, ':', 2) as `communication.destination.port`, + request_processing_time as `aws.elb.request_processing_time`, + target_processing_time as `aws.elb.target_processing_time`, + response_processing_time as `aws.elb.response_processing_time`, + elb_status_code as `http.response.status_code`, + target_status_code as `aws.elb.target_status_code`, + received_bytes as `aws.elb.received_bytes`, + sent_bytes as `aws.elb.sent_bytes`, + split_part (request, ' ', 1) as `http.request.method`, + split_part (request, ' ', 2) as `url.full`, + parse_url (split_part (request, ' ', 2), 'HOST') as `url.domain`, + parse_url (split_part (request, ' ', 2), 'PATH') as `url.path`, + split_part (request, ' ', 3) as `url.schema`, + request AS `http.request.body.content`, + user_agent as `http.user_agent.original`, + user_agent as `http.user_agent.name`, + ssl_cipher as `aws.elb.ssl_cipher`, + ssl_protocol as `aws.elb.ssl_protocol`, + split_part (target_group_arn, ':', 4) as `cloud.region`, + split_part (target_group_arn, ':', 5) as `cloud.account.id`, + trace_id as `traceId`, + chosen_cert_arn as `aws.elb.chosen_cert_arn`, + matched_rule_priority as `aws.elb.matched_rule_priority`, + request_creation_time as `aws.elb.request_creation_time`, + actions_executed as `aws.elb.actions_executed`, + redirect_url as `aws.elb.redirect_url`, + lambda_error_reason as `aws.elb.lambda_error_reason`, + target_port_list as `aws.elb.target_port_list`, + target_status_code_list as `aws.elb.target_status_code_list`, + classification as `aws.elb.classification`, + classification_reason as `aws.elb.classification_reason` +FROM + {table_name}; \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/tables/create_table_elb-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/create_table_elb-1.0.0.sql new file mode 100644 index 0000000..f9409d3 --- /dev/null +++ b/integrations/observability/aws_elb/assets/tables/create_table_elb-1.0.0.sql @@ -0,0 +1,36 @@ +CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( + type string, + time timestamp, + elb string, + client_ip string, + target_ip string, + request_processing_time double, + target_processing_time double, + response_processing_time double, + elb_status_code int, + target_status_code string, + received_bytes bigint, + sent_bytes bigint, + request string, + user_agent string, + ssl_cipher string, + ssl_protocol string, + target_group_arn string, + trace_id string, + domain_name string, + chosen_cert_arn string, + matched_rule_priority string, + request_creation_time timestamp, + actions_executed string, + redirect_url string, + lambda_error_reason string, + target_port_list string, + target_status_code_list string, + classification string, + classification_reason string +) +USING csv +LOCATION '{s3_bucket_location}' +OPTIONS ( + sep=' ' +); \ No newline at end of file diff --git a/integrations/observability/aws_elb/assets/tables/refresh_mv_elb-1.0.0.sql b/integrations/observability/aws_elb/assets/tables/refresh_mv_elb-1.0.0.sql new file mode 100644 index 0000000..9e68c57 --- /dev/null +++ b/integrations/observability/aws_elb/assets/tables/refresh_mv_elb-1.0.0.sql @@ -0,0 +1 @@ +REFRESH MATERIALIZED VIEW {table_name}_mview; \ No newline at end of file diff --git a/integrations/observability/aws_elb/aws_elb-1.0.0.json b/integrations/observability/aws_elb/aws_elb-1.0.0.json index 28b7269..e54a509 100644 --- a/integrations/observability/aws_elb/aws_elb-1.0.0.json +++ b/integrations/observability/aws_elb/aws_elb-1.0.0.json @@ -5,17 +5,7 @@ "description": "AWS Elastic Load Balancer collector", "license": "Apache-2.0", "type": "logs", - "type-alias": "logs-elb", - "labels": [ - "log", - "aws", - "communication", - "http", - "cloud", - "elb", - "url", - "s3-datasource" - ], + "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_elb/info", "statics": { @@ -59,12 +49,19 @@ "assets": { "tables": [ { - "name": "aws_elb_s3_table", - "version": "1.0.0" + "name": "create_table_elb", + "version": "1.0.0", + "language": "sql" + }, + { + "name": "create_mv_elb", + "version": "1.0.0", + "language": "sql" }, { - "name": "aws_elb_s3_mv_index", - "version": "1.0.0" + "name": "refresh_mv_elb", + "version": "1.0.0", + "language": "sql" } ], "savedObjects": { @@ -75,4 +72,4 @@ "sampleData": { "path": "sample.json" } -} +} \ No newline at end of file diff --git a/integrations/observability/aws_elb/info/ELB_S3_DS_Intgration.md b/integrations/observability/aws_elb/info/ELB_S3_DS_Intgration.md index 28969e6..7c13a3c 100644 --- a/integrations/observability/aws_elb/info/ELB_S3_DS_Intgration.md +++ b/integrations/observability/aws_elb/info/ELB_S3_DS_Intgration.md @@ -11,7 +11,7 @@ This is a brief overview of a sample ingestion flow for the AWS ELB integration ## S3 Table Definition Using S3 datasource as the raw data for this integration requires the following assets to be present: - - S3-ELB [Table definition](../assets/tables/aws_elb_s3_table-1.0.0.sql) this table definition is used by the Spark/EMR catalog + - S3-ELB [Table definition](../assets/tables/create_table_elb-1.0.0.sql) this table definition is used by the Spark/EMR catalog - S3-ELB [Acceleration table refresh command](../assets/tables/aws_elb_s3_refresh_covering_index-1.0.0.sql) this command will initiate the flint job processing that will populate the secondary index according to the specified fields in the mapping metadata section. - S3 [opensearch acceleration index template definition ](../assets/indices/aws_elb_covering_index-1.0.0.mapping) @@ -62,7 +62,7 @@ The next section describes the integration responsibilities for creating the req Assuming all the prerequisites mentioned above are resolved, the first step would be to create the ELB logical table on the catalog ([Glue](https://aws.amazon.com/glue/)/[Hive](https://hive.apache.org/)) -- [The ELB table definition](../assets/tables/aws_elb_s3_table-1.0.0.sql) this table definition is used by the Spark/EMR catalog) +- [The ELB table definition](../assets/tables/create_table_elb-1.0.0.sql) this table definition is used by the Spark/EMR catalog) Once the table is created the next phase will be to generate the index template for the ELB log based on the simple schema for Observability index standard. This index template will be augmented with the [covering index component template](../assets/indices/aws_elb_covering_index-1.0.0.mapping) (In addition to the other component templates)