From 7f12d2969fd831663927eb9f19b6c03bbbbf3236 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Wed, 26 Jun 2024 14:15:45 -0700 Subject: [PATCH 01/18] add Nginx getting-started catalog component for setting up fluent-bit agent for collecting, transforming and shipping Nginx access logs into opensearch Signed-off-by: YANGDB --- getting-started/observability/nginx/.env | 18 + getting-started/observability/nginx/README.md | 29 ++ .../nginx/fluent-bit/fluent-bit.conf | 52 +++ .../nginx/fluent-bit/otel-converter.lua | 86 ++++ .../nginx/fluent-bit/parsers.conf | 13 + .../nginx/nginx-getting-started-1.0.0.json | 58 +++ .../observability/nginx/nginx-node.yml | 42 ++ .../observability/nginx/nginx-sample.demo | 0 .../nginx/opensearch_dashboards.yml | 368 ++++++++++++++++++ .../observability/nginx/os-cluster.yml | 49 +++ 10 files changed, 715 insertions(+) create mode 100644 getting-started/observability/nginx/.env create mode 100644 getting-started/observability/nginx/README.md create mode 100644 getting-started/observability/nginx/fluent-bit/fluent-bit.conf create mode 100644 getting-started/observability/nginx/fluent-bit/otel-converter.lua create mode 100644 getting-started/observability/nginx/fluent-bit/parsers.conf create mode 100644 getting-started/observability/nginx/nginx-getting-started-1.0.0.json create mode 100644 getting-started/observability/nginx/nginx-node.yml create mode 100644 getting-started/observability/nginx/nginx-sample.demo create mode 100644 getting-started/observability/nginx/opensearch_dashboards.yml create mode 100644 getting-started/observability/nginx/os-cluster.yml diff --git a/getting-started/observability/nginx/.env b/getting-started/observability/nginx/.env new file mode 100644 index 0000000..60ee804 --- /dev/null +++ b/getting-started/observability/nginx/.env @@ -0,0 +1,18 @@ +# Nginx Proxy +NGINX_PORT=90 +NGINX_ADDR=nginx:${NGINX_PORT} + +# OpenSearch version +OPENSEARCH_VERSION=2.15.0 +OPENSEARCH_ADMIN_PASSWORD=my_%New%_passW0rd!@# +OPENSEARCH_INITIAL_ADMIN_PASSWORD=my_%New%_passW0rd!@# + +# OpenSearch Node1 +OPENSEARCH_PORT=9200 +OPENSEARCH_HOST=opensearch +OPENSEARCH_ADDR=${OPENSEARCH_HOST}:${OPENSEARCH_PORT} + +# OpenSearch Dashboard +OPENSEARCH_DASHBOARD_PORT=5601 +OPENSEARCH_DASHBOARD_HOST=opensearch-dashboards +OPENSEARCH_DASHBOARD_ADDR=${OPENSEARCH_DASHBOARD_HOST}:${OPENSEARCH_DASHBOARD_PORT} diff --git a/getting-started/observability/nginx/README.md b/getting-started/observability/nginx/README.md new file mode 100644 index 0000000..551fad4 --- /dev/null +++ b/getting-started/observability/nginx/README.md @@ -0,0 +1,29 @@ +# Running OpenSearch Joining Existing Cluster +### Create the External Network +Before running any docker-compose files, create the opensearch-net network: +`docker network create opensearch-net` + +### Run the Main OpenSearch Cluster +Run the initial opensearch cluster : +`docker-compose -f os-cluster.yml up -d` + +Check cluster status and health: +``` +curl -X GET "https://admin:my_%25New%25_passW0rd%21%40%23@localhost:9200/_cluster/health?pretty" --insecure +curl -X GET "https://admin:my_%25New%25_passW0rd%21%40%23@localhost:9200/_cat/nodes?v" --insecure +``` + +Connect to the dashboard: +`http://localhost:5601` + - User:`admin` + - Password:`my_%New%_passW0rd!@#` + + +Next run the joining node: +`docker-compose -f nginx-node.yml up -d` + +Check cluster status and health: +``` +curl -X GET "https://admin:my_%25New%25_passW0rd%21%40%23@localhost:9200/_cluster/health?pretty" --insecure +curl -X GET "https://admin:my_%25New%25_passW0rd%21%40%23@localhost:9200/_cat/nodes?v" --insecure +``` diff --git a/getting-started/observability/nginx/fluent-bit/fluent-bit.conf b/getting-started/observability/nginx/fluent-bit/fluent-bit.conf new file mode 100644 index 0000000..af0509d --- /dev/null +++ b/getting-started/observability/nginx/fluent-bit/fluent-bit.conf @@ -0,0 +1,52 @@ +[SERVICE] + Parsers_File parsers.conf + +[INPUT] + Name forward + Port 24224 + +[FILTER] + Name parser + Match nginx.access + Key_Name log + Parser nginx + +[FILTER] + Name parser + Match apache.access + Key_Name log + Parser apache + +[Filter] + Name lua + Match * + Script otel-converter.lua + call convert_to_otel + +[OUTPUT] + Name opensearch + Match nginx.access + Host opensearch-node1 + Port 9200 + Index ss4o_logs-nginx-prod + Suppress_Type_Name On + tls On + tls.verify Off + HTTP_User admin + HTTP_Passwd my_%New%_passW0rd!@# + +[OUTPUT] + Name opensearch + Match apache.access + Host opensearch-node1 + Port 9200 + Index ss4o_logs-apache-prod + Suppress_Type_Name On + tls On + tls.verify Off + HTTP_User admin + HTTP_Passwd my_%New%_passW0rd!@# + +[OUTPUT] + Name stdout + Match nginx.access \ No newline at end of file diff --git a/getting-started/observability/nginx/fluent-bit/otel-converter.lua b/getting-started/observability/nginx/fluent-bit/otel-converter.lua new file mode 100644 index 0000000..8ab4296 --- /dev/null +++ b/getting-started/observability/nginx/fluent-bit/otel-converter.lua @@ -0,0 +1,86 @@ +local hexCharset = "0123456789abcdef" +local function randHex(length) + if length > 0 then + local index = math.random(1, #hexCharset) + return randHex(length - 1) .. hexCharset:sub(index, index) + else + return "" + end +end + +local function format_apache(c) + return string.format( + "%s - %s [%s] \"%s %s HTTP/1.1\" %s %s", + c.host, + c.user, + os.date("%d/%b/%Y:%H:%M:%S %z"), + c.method, + c.path, + c.code, + c.size + ) +end + +local function format_nginx(c) + return string.format( + "%s %s %s [%s] \"%s %s HTTP/1.1\" %s %s \"%s\" \"%s\"", + c.remote, + c.host, + c.user, + os.date("%d/%b/%Y:%H:%M:%S %z"), + c.method, + c.path, + c.code, + c.size, + c.referer, + c.agent + ) +end + +local formats = { + ["apache.access"] = format_apache, + ["nginx.access"] = format_nginx +} + +function convert_to_otel(tag, timestamp, record) + local data = { + traceId=randHex(32), + spanId=randHex(16), + ["@timestamp"]=os.date("!%Y-%m-%dT%H:%M:%S.000Z"), + observedTimestamp=os.date("!%Y-%m-%dT%H:%M:%S.000Z"), + body=formats[tag](record), + attributes={ + data_stream={ + dataset=tag, + namespace="production", + type="logs" + } + }, + event={ + category="web", + name="access", + domain=tag, + kind="event", + result="success", + type="access" + }, + http={ + request={ + method=record.method + }, + response={ + bytes=tonumber(record.size), + status_code=tonumber(record.code) + }, + flavor="1.1", + url=record.path + }, + communication={ + source={ + address="127.0.0.1", + ip=record.remote + } + } + } + return 1, timestamp, data +end diff --git a/getting-started/observability/nginx/fluent-bit/parsers.conf b/getting-started/observability/nginx/fluent-bit/parsers.conf new file mode 100644 index 0000000..afff812 --- /dev/null +++ b/getting-started/observability/nginx/fluent-bit/parsers.conf @@ -0,0 +1,13 @@ +[PARSER] + Name apache + Format regex + Regex ^(?[^ ]*) [^ ]* (?[^ ]*) \[(?