Merge pull request #79 from YANG-DB/aws_elb_with_s3

add elb s3 flint based integration

integrations/observability/aws_elb/assets/aws_elb_s3_notebook-1.0.0.json

@@ -0,0 +1,105 @@
+{
+ "path": "aws_elb_queries",
+ "name": "[AWS ELB Logs Queries] AWS ELB Queries based on S3",
+ "dateCreated": "2023-09-30T00:23:39.602Z",
+ "dateModified": "2023-10-02T17:06:20.090Z",
+ "backend": "Default",
+ "paragraphs": [
+ {
+ "output": [
+ {
+ "result": "## ELB Queries\nThe next notebook shows a variety of monitoring and health check queries both in SQL / PPL on top of ELB log index / table\n\n### Queries Details:\n1) Lists the average amount of data (in kilobytes) that's passing through the Application Load Balancer in request or response pairs\n\n2) List clients, in descending order, by the amount of data (in megabytes) that each client sent in their requests to the Application Load Balancer\n\n3) List clients, in descending order, by the number of times that each client visited a specified URL\n\n4) Count the number of HTTP GET requests received by the load balancer grouped by the client IP address\n\n5) List each time in a specified date range when the target processing time was more than ? (5) seconds\n\n6) List the ? (10) URLs that ? (Firefox) users accessed most frequently, in descending order\n\n\n",
+ "outputType": "MARKDOWN",
+ "execution_time": "0.599 ms"
+ }
+ ],
+ "input": {
+ "inputText": "%md\n## ELB Queries\nThe next notebook shows a variety of monitoring and health check queries both in SQL / PPL on top of ELB log index / table\n\n### Queries Details:\n1) Lists the average amount of data (in kilobytes) that's passing through the Application Load Balancer in request or response pairs\n\n2) List clients, in descending order, by the amount of data (in megabytes) that each client sent in their requests to the Application Load Balancer\n\n3) List clients, in descending order, by the number of times that each client visited a specified URL\n\n4) Count the number of HTTP GET requests received by the load balancer grouped by the client IP address\n\n5) List each time in a specified date range when the target processing time was more than ? (5) seconds\n\n6) List the ? (10) URLs that ? (Firefox) users accessed most frequently, in descending order\n\n\n",
+ "inputType": "MARKDOWN"
+ },
+ "dateCreated": "2023-09-30T00:23:42.373Z",
+ "dateModified": "2023-10-02T17:06:20.090Z",
+ "id": "paragraph_bcac0759-a5dc-454b-85e7-507ec5b79afe"
+ },
+ {
+ "output": [
+ {
+ "result": "\nSELECT (avg(aws.elb.sent_bytes)/1000.0 + avg(aws.elb.received_bytes)/1000.0) as prewarm_kilobytes from ss4o_logs_elb-aws_elb-sample-sample;",
+ "outputType": "QUERY",
+ "execution_time": "0.010 ms"
+ }
+ ],
+ "input": {
+ "inputText": "%sql\nSELECT (avg(aws.elb.sent_bytes)/1000.0 + avg(aws.elb.received_bytes)/1000.0) as prewarm_kilobytes from ss4o_logs_elb-aws_elb-sample-sample;",
+ "inputType": "MARKDOWN"
+ },
+ "dateCreated": "2023-09-30T00:26:29.468Z",
+ "dateModified": "2023-09-30T08:03:36.310Z",
+ "id": "paragraph_10056fac-9d0a-436e-93b5-1c77c207677d"
+ },
+ {
+ "output": [
+ {
+ "result": "\nSELECT aws.elb.client.ip, sum(aws.elb.received_bytes/1000000.0) as client_datareceived_megabytes FROM ss4o_logs_elb-aws_elb-sample-sample GROUP by aws.elb.client.ip ORDER by client_datareceived_megabytes DESC;",
+ "outputType": "QUERY",
+ "execution_time": "0.006 ms"
+ }
+ ],
+ "input": {
+ "inputText": "%sql\nSELECT aws.elb.client.ip, sum(aws.elb.received_bytes/1000000.0) as client_datareceived_megabytes FROM ss4o_logs_elb-aws_elb-sample-sample GROUP by aws.elb.client.ip ORDER by client_datareceived_megabytes DESC;",
+ "inputType": "MARKDOWN"
+ },
+ "dateCreated": "2023-09-30T00:31:26.288Z",
+ "dateModified": "2023-09-30T08:04:46.029Z",
+ "id": "paragraph_7cadb639-8571-41a0-84b6-917b4f1ed88a"
+ },
+ {
+ "output": [
+ {
+ "result": "\nSELECT aws.elb.client.ip, url.domain, count(*) as count from ss4o_logs_elb-aws_elb-sample-sample\nGROUP by aws.elb.client.ip, url.domain\nORDER by count DESC;",
+ "outputType": "QUERY",
+ "execution_time": "0.006 ms"
+ }
+ ],
+ "input": {
+ "inputText": "%sql\nSELECT aws.elb.client.ip, url.domain, count(*) as count from ss4o_logs_elb-aws_elb-sample-sample\nGROUP by aws.elb.client.ip, url.domain\nORDER by count DESC;",
+ "inputType": "MARKDOWN"
+ },
+ "dateCreated": "2023-09-30T00:31:44.191Z",
+ "dateModified": "2023-09-30T08:12:50.840Z",
+ "id": "paragraph_dd1fa597-3db2-40d3-a163-e0d248300a53"
+ },
+ {
+ "output": [
+ {
+ "result": "\nSELECT COUNT(http.request.method)\n AS count, http.request.method, aws.elb.client.ip\nFROM ss4o_logs_elb-aws_elb-sample-sample \nGROUP by http.request.method, aws.elb.client.ip;",
+ "outputType": "QUERY",
+ "execution_time": "0.006 ms"
+ }
+ ],
+ "input": {
+ "inputText": "%sql\nSELECT COUNT(http.request.method)\n AS count, http.request.method, aws.elb.client.ip\nFROM ss4o_logs_elb-aws_elb-sample-sample \nGROUP by http.request.method, aws.elb.client.ip;",
+ "inputType": "MARKDOWN"
+ },
+ "dateCreated": "2023-09-30T00:31:57.554Z",
+ "dateModified": "2023-09-30T08:18:53.232Z",
+ "id": "paragraph_8ae6e362-ebfe-455b-8d43-ebb6db25ecea"
+ },
+ {
+ "output": [
+ {
+ "result": "\nSELECT url.domain, http.user_agent.name, count(*) as count\nFROM ss4o_logs_elb-aws_elb-sample-sample\nWHERE http.user_agent.name LIKE '%Chrome%'\nGROUP by url.domain, http.user_agent.name\nORDER by count(*) DESC;",
+ "outputType": "QUERY",
+ "execution_time": "0.007 ms"
+ }
+ ],
+ "input": {
+ "inputText": "%sql\nSELECT url.domain, http.user_agent.name, count(*) as count\nFROM ss4o_logs_elb-aws_elb-sample-sample\nWHERE http.user_agent.name LIKE '%Chrome%'\nGROUP by url.domain, http.user_agent.name\nORDER by count(*) DESC;",
+ "inputType": "MARKDOWN"
+ },
+ "dateCreated": "2023-09-30T00:38:32.321Z",
+ "dateModified": "2023-09-30T08:29:31.202Z",
+ "id": "paragraph_7fde081a-6dac-4bb7-ae49-a86695901b1a"
+ }
+ ]
+}

integrations/observability/aws_elb/assets/queries/average_amount_of_data_query-1.0.0.sql

@@ -0,0 +1,2 @@
+-- # List the average amount of data (in kilobytes) that's passing through the Application Load Balancer in request or response pairs
+SELECT (avg(sent_bytes)/1000.0 + avg(received_bytes)/1000.0) as prewarm_kilobytes from alb_logs;

integrations/observability/aws_elb/assets/queries/clients_by_data-1.0.0.sql

@@ -0,0 +1,2 @@
+-- # List clients, in descending order, by the amount of data (in megabytes) that each client sent in their requests to the Application Load Balancer
+SELECT client_ip, sum(received_bytes/1000000.0) as client_datareceived_megabytes FROM alb_logs GROUP by client_ip ORDER by client_datareceived_megabytes DESC;

integrations/observability/aws_elb/assets/queries/clients_by_visiting_urls-1.0.0.sql

@@ -0,0 +1,4 @@
+-- # List clients, in descending order, by the number of times that each client visited a specified URL
+SELECT client_ip, elb, request_url, count(*) as count from alb_logs
+GROUP by client_ip, elb, request_url
+ORDER by count DESC;

integrations/observability/aws_elb/assets/queries/count_http_by_ip-1.0.0.sql

@@ -0,0 +1,6 @@
+-- # Count the number of HTTP GET requests received by the load balancer grouped by the client IP address
+SELECT COUNT(request_verb)
+ AS count, request_verb, client_ip
+FROM alb_logs_partitioned
+WHERE day = '?'
+GROUP by request_verb, client_ip;

integrations/observability/aws_elb/assets/queries/distinct_client_count_query-1.0.0.sql

@@ -0,0 +1,2 @@
+-- # List all client IP addresses that accessed the Application Load Balancer, and how many times they accessed the Application Load Balancer 
+SELECT distinct client_ip, count() as count from alb_logs GROUP by client_ip ORDER by count() DESC;

integrations/observability/aws_elb/assets/queries/largest_latency_by_date_range-1.0.0.sql

@@ -0,0 +1,6 @@
+-- # List each time in a specified date range when the target processing time was more than ? (5) seconds
+SELECT * from alb_logs
+WHERE (parse_datetime(time,'yyyy-MM-dd''T''HH:mm:ss.SSSSSS''Z')
+ BETWEEN parse_datetime('?','?')
+ AND parse_datetime('?','?'))
+ AND (target_processing_time >= ?);

integrations/observability/aws_elb/assets/queries/most_frequent_user_agents-1.0.0.sql

@@ -0,0 +1,6 @@
+-- # List the ? (10) URLs that ? (Firefox) users accessed most frequently, in descending order
+SELECT request_url, user_agent, count(*) as count
+FROM alb_logs
+WHERE user_agent LIKE '%?%'
+GROUP by request_url, user_agent
+ORDER by count(*) DESC LIMIT ?;

integrations/observability/aws_elb/assets/queries/target_traffic_imbalance-1.0.0.sql

@@ -0,0 +1,6 @@
+-- # List all targets that the Application Load Balancer routes traffic to and the number of routed requests per target, by percentage distribution 
+SELECT target_ip, (Count(target_ip)* 100.0 / (Select Count(*) From alb_logs))
+ as backend_traffic_percentage
+FROM alb_logs
+GROUP by target_ip<
+ORDER By count() DESC;

integrations/observability/aws_elb/assets/tables/create_mv_elb-1.0.0.sql

@@ -0,0 +1,41 @@
+CREATE MATERIALIZED VIEW {table_name}_mview AS
+SELECT
+ type as `aws.elb.elb_type`,
+ time as `@timestamp`,
+ elb as `aws.elb.elb_name`,
+ split_part (client_ip, ':', 1) as `communication.source.ip`,
+ split_part (client_ip, ':', 2) as `communication.source.port`,
+ split_part (target_ip, ':', 1) as `communication.destination.ip`,
+ split_part (target_ip, ':', 2) as `communication.destination.port`,
+ request_processing_time as `aws.elb.request_processing_time`,
+ target_processing_time as `aws.elb.target_processing_time`,
+ response_processing_time as `aws.elb.response_processing_time`,
+ elb_status_code as `http.response.status_code`,
+ target_status_code as `aws.elb.target_status_code`,
+ received_bytes as `aws.elb.received_bytes`,
+ sent_bytes as `aws.elb.sent_bytes`,
+ split_part (request, ' ', 1) as `http.request.method`,
+ split_part (request, ' ', 2) as `url.full`,
+ parse_url (split_part (request, ' ', 2), 'HOST') as `url.domain`,
+ parse_url (split_part (request, ' ', 2), 'PATH') as `url.path`,
+ split_part (request, ' ', 3) as `url.schema`,
+ request AS `http.request.body.content`,
+ user_agent as `http.user_agent.original`,
+ user_agent as `http.user_agent.name`,
+ ssl_cipher as `aws.elb.ssl_cipher`,
+ ssl_protocol as `aws.elb.ssl_protocol`,
+ split_part (target_group_arn, ':', 4) as `cloud.region`,
+ split_part (target_group_arn, ':', 5) as `cloud.account.id`,
+ trace_id as `traceId`,
+ chosen_cert_arn as `aws.elb.chosen_cert_arn`,
+ matched_rule_priority as `aws.elb.matched_rule_priority`,
+ request_creation_time as `aws.elb.request_creation_time`,
+ actions_executed as `aws.elb.actions_executed`,
+ redirect_url as `aws.elb.redirect_url`,
+ lambda_error_reason as `aws.elb.lambda_error_reason`,
+ target_port_list as `aws.elb.target_port_list`,
+ target_status_code_list as `aws.elb.target_status_code_list`,
+ classification as `aws.elb.classification`,
+ classification_reason as `aws.elb.classification_reason`
+FROM
+ {table_name};

integrations/observability/aws_elb/assets/tables/create_table_elb-1.0.0.sql

@@ -0,0 +1,36 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
+ type string,
+ time timestamp,
+ elb string,
+ client_ip string,
+ target_ip string,
+ request_processing_time double,
+ target_processing_time double,
+ response_processing_time double,
+ elb_status_code int,
+ target_status_code string,
+ received_bytes bigint,
+ sent_bytes bigint,
+ request string,
+ user_agent string,
+ ssl_cipher string,
+ ssl_protocol string,
+ target_group_arn string,
+ trace_id string,
+ domain_name string,
+ chosen_cert_arn string,
+ matched_rule_priority string,
+ request_creation_time timestamp,
+ actions_executed string,
+ redirect_url string,
+ lambda_error_reason string,
+ target_port_list string,
+ target_status_code_list string,
+ classification string,
+ classification_reason string
+)
+USING csv
+LOCATION '{s3_bucket_location}'
+OPTIONS (
+ sep=' '
+);

integrations/observability/aws_elb/assets/tables/refresh_mv_elb-1.0.0.sql

@@ -0,0 +1 @@
+REFRESH MATERIALIZED VIEW {table_name}_mview;

integrations/observability/aws_elb/aws_elb-1.0.0.json

@@ -5,8 +5,7 @@
  "description": "AWS Elastic Load Balancer collector",
  "license": "Apache-2.0",
  "type": "logs",
- "type-alias": "logs-elb",
- "labels": ["log", "aws", "communication", "http", "cloud", "elb", "url"],
+ "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"],
  "author": "OpenSearch",
  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_elb/info",
  "statics": {
@@ -48,6 +47,23 @@
  }
  ],
  "assets": {
+ "tables": [
+ {
+ "name": "create_table_elb",
+ "version": "1.0.0",
+ "language": "sql"
+ },
+ {
+ "name": "create_mv_elb",
+ "version": "1.0.0",
+ "language": "sql"
+ },
+ {
+ "name": "refresh_mv_elb",
+ "version": "1.0.0",
+ "language": "sql"
+ }
+ ],
  "savedObjects": {
  "name": "aws_elb",
  "version": "1.0.0"
@@ -56,4 +72,4 @@
  "sampleData": {
  "path": "sample.json"
  }
-}
+}