add Nginx getting-started catalog component for setting up fluent-bit…

integrations/observability/nginx/getting-started/.env

@@ -0,0 +1,18 @@
+# Nginx Proxy
+NGINX_PORT=90
+NGINX_ADDR=nginx:${NGINX_PORT}
+
+# OpenSearch version
+OPENSEARCH_VERSION=2.15.0
+OPENSEARCH_ADMIN_PASSWORD=my_%New%_passW0rd!@#
+OPENSEARCH_INITIAL_ADMIN_PASSWORD=my_%New%_passW0rd!@#
+
+# OpenSearch Node1
+OPENSEARCH_PORT=9200
+OPENSEARCH_HOST=opensearch
+OPENSEARCH_ADDR=${OPENSEARCH_HOST}:${OPENSEARCH_PORT}
+
+# OpenSearch Dashboard
+OPENSEARCH_DASHBOARD_PORT=5601
+OPENSEARCH_DASHBOARD_HOST=opensearch-dashboards
+OPENSEARCH_DASHBOARD_ADDR=${OPENSEARCH_DASHBOARD_HOST}:${OPENSEARCH_DASHBOARD_PORT}

integrations/observability/nginx/getting-started/fluent-bit/fluent-bit.conf

@@ -0,0 +1,52 @@
+[SERVICE]
+    Parsers_File parsers.conf
+
+[INPUT]
+    Name forward
+    Port 24224
+
+[FILTER]
+    Name parser
+    Match nginx.access
+    Key_Name log
+    Parser nginx
+
+[FILTER]
+    Name parser
+    Match apache.access
+    Key_Name log
+    Parser apache
+
+[Filter]
+    Name    lua
+    Match   *
+    Script  otel-converter.lua
+    call    convert_to_otel
+
+[OUTPUT]
+    Name  opensearch
+    Match nginx.access
+    Host  opensearch-node1
+    Port  9200
+    Index ss4o_logs-nginx-prod
+    Suppress_Type_Name On
+    tls             On
+    tls.verify      Off
+    HTTP_User       admin
+    HTTP_Passwd     my_%New%_passW0rd!@#
+
+[OUTPUT]
+    Name  opensearch
+    Match apache.access
+    Host  opensearch-node1
+    Port  9200
+    Index ss4o_logs-apache-prod
+    Suppress_Type_Name On
+    tls             On
+    tls.verify      Off
+    HTTP_User       admin
+    HTTP_Passwd     my_%New%_passW0rd!@#
+
+[OUTPUT]
+    Name stdout
+    Match nginx.access

integrations/observability/nginx/getting-started/fluent-bit/otel-converter.lua

@@ -0,0 +1,86 @@
+local hexCharset = "0123456789abcdef"
+local function randHex(length)
+    if length > 0 then
+        local index = math.random(1, #hexCharset)
+        return randHex(length - 1) .. hexCharset:sub(index, index)
+    else
+        return ""
+    end
+end
+
+local function format_apache(c)
+    return string.format(
+        "%s - %s [%s] \"%s %s HTTP/1.1\" %s %s",
+        c.host,
+        c.user,
+        os.date("%d/%b/%Y:%H:%M:%S %z"),
+        c.method,
+        c.path,
+        c.code,
+        c.size
+    )
+end
+
+local function format_nginx(c)
+    return string.format(
+        "%s %s %s [%s] \"%s %s HTTP/1.1\" %s %s \"%s\" \"%s\"",
+        c.remote,
+        c.host,
+        c.user,
+        os.date("%d/%b/%Y:%H:%M:%S %z"),
+        c.method,
+        c.path,
+        c.code,
+        c.size,
+        c.referer,
+        c.agent
+    )
+end
+
+local formats = {
+    ["apache.access"] = format_apache,
+    ["nginx.access"] = format_nginx
+}
+
+function convert_to_otel(tag, timestamp, record)
+    local data = {
+        traceId=randHex(32),
+        spanId=randHex(16),
+        ["@timestamp"]=os.date("!%Y-%m-%dT%H:%M:%S.000Z"),
+        observedTimestamp=os.date("!%Y-%m-%dT%H:%M:%S.000Z"),
+        body=formats[tag](record),
+        attributes={
+            data_stream={
+                dataset=tag,
+                namespace="production",
+                type="logs"
+            }
+        },
+        event={
+            category="web",
+            name="access",
+            domain=tag,
+            kind="event",
+            result="success",
+            type="access"
+        },
+        http={
+            request={
+                method=record.method
+            },
+            response={
+                bytes=tonumber(record.size),
+                status_code=tonumber(record.code)
+            },
+            flavor="1.1",
+            url=record.path
+        },
+        communication={
+            source={
+                address="127.0.0.1",
+                ip=record.remote
+            }
+        }
+    }
+    return 1, timestamp, data
+end

integrations/observability/nginx/getting-started/fluent-bit/parsers.conf

@@ -0,0 +1,13 @@
+[PARSER]
+    Name   apache
+    Format regex
+    Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+    Time_Key time
+    Time_Format %d/%b/%Y:%H:%M:%S %z
+
+[PARSER]
+    Name   nginx
+    Format regex
+    Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")
+    Time_Key time
+    Time_Format %d/%b/%Y:%H:%M:%S %z

integrations/observability/nginx/getting-started/nginx-getting-started-1.0.0.json

@@ -0,0 +1,58 @@
+{
+  "name": "nginx",
+  "version": "1.0.0",
+  "displayName": "Nginx-Logs",
+  "description": "Getting Started With Nginx access logs.",
+  "license": "Apache-2.0",
+  "type": "logs",
+  "labels": ["Observability", "Logs", "Fluent-bit"],
+  "author": "OpenSearch",
+  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info",
+  "workflow": [
+    {
+      "name": "Fluent-Bit Parser",
+      "label": "Log Parsing",
+      "info": "<info URL>",
+      "description": "Setup Fluent-Bit parser config file parsing Nginx access log fields",
+      "content": "[PARSER]\n    Name   apache\n    Format regex\n    Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \\[(?<time>[^\\]]*)\\] \"(?<method>\\S+)(?: +(?<path>[^\\\"]*?)(?: +\\S*)?)?\" (?<code>[^ ]*) (?<size>[^ ]*)(?: \"(?<referer>[^\\\"]*)\" \"(?<agent>[^\\\"]*)\")?$\n    Time_Key time\n    Time_Format %d/%b/%Y:%H:%M:%S %z\n\n[PARSER]\n    Name   nginx\n    Format regex\n    Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \\[(?<time>[^\\]]*)\\] \"(?<method>\\S+)(?: +(?<path>[^\\\"]*?)(?: +\\S*)?)?\" (?<code>[^ ]*) (?<size>[^ ]*)(?: \"(?<referer>[^\\\"]*)\" \"(?<agent>[^\\\"]*)\")\n    Time_Key time\n    Time_Format %d/%b/%Y:%H:%M:%S %z\n"
+    },
+    {
+      "name": "Fluent-Bit Log Converter",
+      "label": "Log Parsing",
+      "info": "<info URL>",
+      "description": "Setup Fluent-Bit logs converter lua script config file converting Nginx access log into Simple schema format",
+      "content": "local hexCharset = \"0123456789abcdef\"\nlocal function randHex(length)\n    if length > 0 then\n        local index = math.random(1, #hexCharset)\n        return randHex(length - 1) .. hexCharset:sub(index, index)\n    else\n        return \"\"\n    end\nend\n\nlocal function format_apache(c)\n    return string.format(\n        \"%s - %s [%s] \\\"%s %s HTTP/1.1\\\" %s %s\",\n        c.host,\n        c.user,\n        os.date(\"%d/%b/%Y:%H:%M:%S %z\"),\n        c.method,\n        c.path,\n        c.code,\n        c.size\n    )\nend\n\nlocal function format_nginx(c)\n    return string.format(\n        \"%s %s %s [%s] \\\"%s %s HTTP/1.1\\\" %s %s \\\"%s\\\" \\\"%s\\\"\",\n        c.remote,\n        c.host,\n        c.user,\n        os.date(\"%d/%b/%Y:%H:%M:%S %z\"),\n        c.method,\n        c.path,\n        c.code,\n        c.size,\n        c.referer,\n        c.agent\n    )\nend\n\nlocal formats = {\n    [\"apache.access\"] = format_apache,\n    [\"nginx.access\"] = format_nginx\n}\n\nfunction convert_to_otel(tag, timestamp, record)\n    local data = {\n        traceId=randHex(32),\n        spanId=randHex(16),\n        [\"@timestamp\"]=os.date(\"!%Y-%m-%dT%H:%M:%S.000Z\"),\n        observedTimestamp=os.date(\"!%Y-%m-%dT%H:%M:%S.000Z\"),\n        body=formats[tag](record),\n        attributes={\n            data_stream={\n                dataset=tag,\n                namespace=\"production\",\n                type=\"logs\"\n            }\n        },\n        event={\n            category=\"web\",\n            name=\"access\",\n            domain=tag,\n            kind=\"event\",\n            result=\"success\",\n            type=\"access\"\n        },\n        http={\n            request={\n                method=record.method\n            },\n            response={\n                bytes=tonumber(record.size),\n                status_code=tonumber(record.code)\n            },\n            flavor=\"1.1\",\n            url=record.path\n        },\n        communication={\n            source={\n                address=\"127.0.0.1\",\n                ip=record.remote\n            }\n        }\n    }\n    return 1, timestamp, data\nend\n"
+    },
+    {
+      "name": "Fluent-Bit Setup",
+      "label": "Agent Set-Up",
+      "info": "<info URL>",
+      "description": "Setup Fluent-Bit conf file including logs parsing and OpenSearch access",
+      "content": "[SERVICE]\n    Parsers_File parsers.conf\n\n[INPUT]\n    Name forward\n    Port 24224\n\n[FILTER]\n    Name parser\n    Match nginx.access\n    Key_Name log\n    Parser nginx\n\n[FILTER]\n    Name parser\n    Match apache.access\n    Key_Name log\n    Parser apache\n\n[Filter]\n    Name    lua\n    Match   *\n    Script  otel-converter.lua\n    call    convert_to_otel\n\n[OUTPUT]\n    Name  opensearch\n    Match nginx.access\n    Host  opensearch-node1\n    Port  9200\n    Index ss4o_logs-nginx-prod\n    Suppress_Type_Name On\n    tls             On\n    tls.verify      Off\n    HTTP_User       admin\n    HTTP_Passwd     my_%New%_passW0rd!@#\n\n[OUTPUT]\n    Name  opensearch\n    Match apache.access\n    Host  opensearch-node1\n    Port  9200\n    Index ss4o_logs-apache-prod\n    Suppress_Type_Name On\n    tls             On\n    tls.verify      Off\n    HTTP_User       admin\n    HTTP_Passwd     my_%New%_passW0rd!@#\n\n[OUTPUT]\n    Name stdout\n    Match nginx.access"
+    }
+  ],
+  "statics": {
+    "logo": [{
+      "annotation": "Fluent-Bit Logo",
+      "path": "logo.svg"
+    },
+      {
+        "annotation": "Nginx Logo",
+        "path": "logo.svg"
+      }]
+  },
+  "relatedAssets": [
+    {
+      "name": "nginx-integration",
+      "type": "integration",
+      "version": "1.0.0",
+      "displayName": "Nginx-Integration",
+      "catalog": "Observability",
+      "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info"
+    }
+  ],
+  "liveDemo": {
+    "command": "docker-compose up -d",
+    "path": "<docker-compose-url>"
+  }
+}

integrations/observability/nginx/getting-started/nginx-node.yml

@@ -0,0 +1,42 @@
+version: '3'
+
+services:
+  nginx-node:
+    image: kscarlett/nginx-log-generator:latest
+    networks:
+      - opensearch-net
+    links:
+      - fluentbit
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: 127.0.0.1:24224
+        tag: nginx.access
+        fluentd-async: "true"
+  apache:
+    image: mingrammer/flog
+    command: "--loop -d 1s"
+    networks:
+      - opensearch-net
+    links:
+      - fluentbit
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: 127.0.0.1:24224
+        tag: apache.access
+        fluentd-async: "true"
+  fluentbit:
+    container_name: fluentbit
+    image: fluent/fluent-bit:latest
+    volumes:
+      - ./fluent-bit:/fluent-bit/etc
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"
+    networks:
+      - opensearch-net
+
+networks:
+  opensearch-net:
+    external: true