From 76c8a89952de631c6354fc919825e9fa45bfb6c6 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Tue, 16 Jul 2024 17:46:20 -0700 Subject: [PATCH 1/3] add csv upload file integration Signed-off-by: YANGDB --- integrations/observability/csv_file/README.md | 38 +++ .../observability/csv_file/csv-1.0.0.json | 56 ++++ .../observability/csv_file/data/logs.csv | 10 + .../csv_file/getting-started/fluent-bit/.env | 18 ++ .../fluent-bit/data/fluent-bit.conf | 28 ++ .../getting-started/fluent-bit/data/logs.csv | 10 + .../fluent-bit/data/parsers.conf | 6 + .../fluent-bit/docker-complete.yml | 58 ++++ .../fluent-bit/docker-compose.yml | 18 ++ .../fluent-bit/opensearch_dashboards.yml | 10 + .../csv_file/info/Getting-Started.md | 287 ++++++++++++++++++ .../csv_file/static/data-pepper.png | Bin 0 -> 6990 bytes .../csv_file/static/fluentbit.png | Bin 0 -> 6693 bytes .../observability/csv_file/static/logo.png | Bin 0 -> 17473 bytes 14 files changed, 539 insertions(+) create mode 100644 integrations/observability/csv_file/README.md create mode 100644 integrations/observability/csv_file/csv-1.0.0.json create mode 100644 integrations/observability/csv_file/data/logs.csv create mode 100644 integrations/observability/csv_file/getting-started/fluent-bit/.env create mode 100644 integrations/observability/csv_file/getting-started/fluent-bit/data/fluent-bit.conf create mode 100644 integrations/observability/csv_file/getting-started/fluent-bit/data/logs.csv create mode 100644 integrations/observability/csv_file/getting-started/fluent-bit/data/parsers.conf create mode 100644 integrations/observability/csv_file/getting-started/fluent-bit/docker-complete.yml create mode 100644 integrations/observability/csv_file/getting-started/fluent-bit/docker-compose.yml create mode 100644 integrations/observability/csv_file/getting-started/fluent-bit/opensearch_dashboards.yml create mode 100644 integrations/observability/csv_file/info/Getting-Started.md create mode 100644 integrations/observability/csv_file/static/data-pepper.png create mode 100644 integrations/observability/csv_file/static/fluentbit.png create mode 100644 integrations/observability/csv_file/static/logo.png diff --git a/integrations/observability/csv_file/README.md b/integrations/observability/csv_file/README.md new file mode 100644 index 0000000..d20db83 --- /dev/null +++ b/integrations/observability/csv_file/README.md @@ -0,0 +1,38 @@ +# CSV Upload Integration + +> CSV File based Upload Integration + +## What is CSV Upload ? + +CSV upload is an example of parsing and loading a CSV file into opensearch index using an agent + +## What is CSV Integration ? + +An integration is a bundle of pre-canned assets which are bundled togather in a meaningful manner. + +**_CSV Upload_** integration includes docker live example including getting started instruction of using data-prepper or fluent-bit for +uploading the csv file into a dedicated index using a parser to transform the csv into json + +## Ingesting CVS Using Data-Prepper + ... + +## Ingesting CVS Using Flunet-Bit + +... + +--- +## Loading Integrations via DashboardManagement + +To update an integration template navigate to the DashboardManagement and select [savedObjects](https://localhost:5601/_dashboards/app/management/opensearch-dashboards/objects) and import the new artifact: + +1) Download the `nginx-1.0.0.ndjson` artifact from the [catalog release page](https://github.com/opensearch-project/opensearch-catalog/releases/edit/nginx-1.0.0) + +2) Go to the [DashboardManagement -> savedObjects ](https://localhost:5601/_dashboards/app/management/opensearch-dashboards/objects) + +![](https://github.com/opensearch-project/opensearch-catalog/assets/48943349/d96e9a78-e3de-4cce-ba66-23f7c084778d) + +![](https://github.com/opensearch-project/opensearch-catalog/assets/48943349/a63ae102-706a-4980-b758-fff7f6b24a94) + +3) Once there select import to load the recently downloaded integration artifact (`nginx-1.0.0.ndjson` suffix) + +4) Open the [nginx integration](https://localhost:5601/app/integrations#/available/nginx) and install diff --git a/integrations/observability/csv_file/csv-1.0.0.json b/integrations/observability/csv_file/csv-1.0.0.json new file mode 100644 index 0000000..784c54a --- /dev/null +++ b/integrations/observability/csv_file/csv-1.0.0.json @@ -0,0 +1,56 @@ +{ + "name": "nginx", + "version": "1.0.0", + "displayName": "Nginx", + "description": "Analyze Nginx access logs.", + "license": "Apache-2.0", + "type": "logs", + "labels": ["Observability", "Logs", "Flint S3"], + "author": "OpenSearch", + "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info", + "workflows": [ + { + "name": "queries", + "label": "Queries (recommended)", + "description": "Tables and pre-written queries for quickly getting insights on your data.", + "enabled_by_default": true + }, + { + "name": "dashboards", + "label": "Dashboards & Visualizations", + "description": "Dashboards and indices that enable you to easily visualize important metrics.", + "enabled_by_default": false + } + ], + "statics": { + "logo": { + "annotation": "NginX Logo", + "path": "logo.svg" + }, + "gallery": [ + { + "annotation": "NginX Dashboard", + "path": "dashboard1.png" + }, + { + "annotation": "NginX Dashboard view", + "path": "dashboard2.png" + } + ] + }, + "components": [ + { "name": "communication", "version": "1.0.0" }, + { "name": "http", "version": "1.0.0" }, + { "name": "logs", "version": "1.0.0" } + ], + "assets": [ + { "name": "nginx", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["dashboards"] }, + { "name": "create_table", "version": "1.0.0", "extension": "sql", "type": "query" }, + { "name": "create_skipping_index", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["queries"] }, + { "name": "create_mv", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["dashboards"] }, + { "name": "example_queries", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["queries"] } + ], + "sampleData": { + "path": "sample.json" + } +} \ No newline at end of file diff --git a/integrations/observability/csv_file/data/logs.csv b/integrations/observability/csv_file/data/logs.csv new file mode 100644 index 0000000..d0136c7 --- /dev/null +++ b/integrations/observability/csv_file/data/logs.csv @@ -0,0 +1,10 @@ +2024-07-16 12:00:00,INFO,Application started successfully,App1,host1 +2024-07-16 12:01:00,DEBUG,User logged in,App1,host1 +2024-07-16 12:01:05,ERROR,Failed to load resource,App1,host1 +2024-07-16 12:02:00,WARN,Deprecated API used,App1,host1 +2024-07-16 12:03:00,INFO,Background job executed,App1,host1 +2024-07-16 12:04:00,DEBUG,Cache cleared,App1,host1 +2024-07-16 12:05:00,INFO,User logged out,App1,host1 +2024-07-16 12:06:00,ERROR,Database connection failed,App1,host1 +2024-07-16 12:07:00,INFO,Application shutdown initiated,App1,host1 +2024-07-16 12:08:00,INFO,Application shutdown completed,App1,host1 diff --git a/integrations/observability/csv_file/getting-started/fluent-bit/.env b/integrations/observability/csv_file/getting-started/fluent-bit/.env new file mode 100644 index 0000000..60ee804 --- /dev/null +++ b/integrations/observability/csv_file/getting-started/fluent-bit/.env @@ -0,0 +1,18 @@ +# Nginx Proxy +NGINX_PORT=90 +NGINX_ADDR=nginx:${NGINX_PORT} + +# OpenSearch version +OPENSEARCH_VERSION=2.15.0 +OPENSEARCH_ADMIN_PASSWORD=my_%New%_passW0rd!@# +OPENSEARCH_INITIAL_ADMIN_PASSWORD=my_%New%_passW0rd!@# + +# OpenSearch Node1 +OPENSEARCH_PORT=9200 +OPENSEARCH_HOST=opensearch +OPENSEARCH_ADDR=${OPENSEARCH_HOST}:${OPENSEARCH_PORT} + +# OpenSearch Dashboard +OPENSEARCH_DASHBOARD_PORT=5601 +OPENSEARCH_DASHBOARD_HOST=opensearch-dashboards +OPENSEARCH_DASHBOARD_ADDR=${OPENSEARCH_DASHBOARD_HOST}:${OPENSEARCH_DASHBOARD_PORT} diff --git a/integrations/observability/csv_file/getting-started/fluent-bit/data/fluent-bit.conf b/integrations/observability/csv_file/getting-started/fluent-bit/data/fluent-bit.conf new file mode 100644 index 0000000..5be828a --- /dev/null +++ b/integrations/observability/csv_file/getting-started/fluent-bit/data/fluent-bit.conf @@ -0,0 +1,28 @@ +[SERVICE] + Flush 1 + Log_Level info + Parsers_File parsers.conf + +[INPUT] + Name tail + Path /fluent-bit/data/*.csv + Parser csv + Tag csv + +[INPUT] + Name dummy + Dummy {"timestamp":"2024-07-16 12:09:00", "log_level":"INFO", "message":"Dummy log message", "application":"App2", "host":"host2"} + Tag dummy + +[OUTPUT] + Name opensearch + Host opensearch-node1 + Match * + Port 9200 + Type _doc + Index logs-index + tls On + tls.verify Off + Suppress_Type_Name On + HTTP_User admin + HTTP_Passwd my_%New%_passW0rd!@# \ No newline at end of file diff --git a/integrations/observability/csv_file/getting-started/fluent-bit/data/logs.csv b/integrations/observability/csv_file/getting-started/fluent-bit/data/logs.csv new file mode 100644 index 0000000..d0136c7 --- /dev/null +++ b/integrations/observability/csv_file/getting-started/fluent-bit/data/logs.csv @@ -0,0 +1,10 @@ +2024-07-16 12:00:00,INFO,Application started successfully,App1,host1 +2024-07-16 12:01:00,DEBUG,User logged in,App1,host1 +2024-07-16 12:01:05,ERROR,Failed to load resource,App1,host1 +2024-07-16 12:02:00,WARN,Deprecated API used,App1,host1 +2024-07-16 12:03:00,INFO,Background job executed,App1,host1 +2024-07-16 12:04:00,DEBUG,Cache cleared,App1,host1 +2024-07-16 12:05:00,INFO,User logged out,App1,host1 +2024-07-16 12:06:00,ERROR,Database connection failed,App1,host1 +2024-07-16 12:07:00,INFO,Application shutdown initiated,App1,host1 +2024-07-16 12:08:00,INFO,Application shutdown completed,App1,host1 diff --git a/integrations/observability/csv_file/getting-started/fluent-bit/data/parsers.conf b/integrations/observability/csv_file/getting-started/fluent-bit/data/parsers.conf new file mode 100644 index 0000000..056a359 --- /dev/null +++ b/integrations/observability/csv_file/getting-started/fluent-bit/data/parsers.conf @@ -0,0 +1,6 @@ +[PARSER] + Name csv + Format regex + Regex ^(?[^,]+),(?[^,]+),(?[^,]+),(?[^,]+),(?[^,]+)$ + Time_Key timestamp + Time_Format %Y-%m-%d %H:%M:%S diff --git a/integrations/observability/csv_file/getting-started/fluent-bit/docker-complete.yml b/integrations/observability/csv_file/getting-started/fluent-bit/docker-complete.yml new file mode 100644 index 0000000..64d0525 --- /dev/null +++ b/integrations/observability/csv_file/getting-started/fluent-bit/docker-complete.yml @@ -0,0 +1,58 @@ +version: '3.8' + +services: + opensearch-dashboards: + image: opensearchproject/opensearch-dashboards:${OPENSEARCH_VERSION} + container_name: opensearch-dashboards + ports: + - 5601:5601 + environment: + OPENSEARCH_HOSTS: '["https://opensearch-node1:9200"]' + depends_on: + - opensearch-node1 + volumes: + - ./opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml + networks: + - opensearch-net + + opensearch-node1: + image: opensearchproject/opensearch:${OPENSEARCH_VERSION} + container_name: opensearch-node1 + environment: + - cluster.name=my-cluster + - node.name=opensearch-node1 + - discovery.seed_hosts=opensearch-node1 + - cluster.initial_master_nodes=opensearch-node1 + - bootstrap.memory_lock=true + - plugins.query.datasources.encryption.masterkey=8e3f206ea7c07cc1bfc5cf40 + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD}" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - opensearch-data:/usr/share/opensearch/data + ports: + - 9200:9200 + - 9600:9600 + networks: + - opensearch-net + + fluent-bit: + image: fluent/fluent-bit:latest + container_name: fluent-bit + volumes: + - ./data:/fluent-bit/data + command: ["/fluent-bit/bin/fluent-bit", "-c", "/fluent-bit/data/fluent-bit.conf"] + depends_on: + - opensearch-node1 + networks: + - opensearch-net + +volumes: + opensearch-data: + +networks: + opensearch-net: + driver: bridge diff --git a/integrations/observability/csv_file/getting-started/fluent-bit/docker-compose.yml b/integrations/observability/csv_file/getting-started/fluent-bit/docker-compose.yml new file mode 100644 index 0000000..1897c8b --- /dev/null +++ b/integrations/observability/csv_file/getting-started/fluent-bit/docker-compose.yml @@ -0,0 +1,18 @@ +version: '3.8' + +services: + fluent-bit: + image: fluent/fluent-bit:latest + container_name: fluent-bit + volumes: + - ./data:/fluent-bit/data + command: ["/fluent-bit/bin/fluent-bit", "-c", "/fluent-bit/data/fluent-bit.conf"] + networks: + - opensearch-net + +volumes: + opensearch-data: + +networks: + opensearch-net: + driver: bridge diff --git a/integrations/observability/csv_file/getting-started/fluent-bit/opensearch_dashboards.yml b/integrations/observability/csv_file/getting-started/fluent-bit/opensearch_dashboards.yml new file mode 100644 index 0000000..e386ddb --- /dev/null +++ b/integrations/observability/csv_file/getting-started/fluent-bit/opensearch_dashboards.yml @@ -0,0 +1,10 @@ +opensearch.hosts: ["https://opensearch-node1:9200"] +server.host: 0.0.0.0 +opensearch.ssl.verificationMode: none +opensearch.username: "admin" +opensearch.password: "my_%New%_passW0rd!@#" +opensearch.requestHeadersWhitelist: [ authorization,securitytenant ] +opensearch_security.multitenancy.enabled: false +opensearch_security.multitenancy.tenants.preferred: ["Private", "Global"] +opensearch_security.readonly_mode.roles: ["kibana_read_only"] +vis_type_vega.enableExternalUrls: true diff --git a/integrations/observability/csv_file/info/Getting-Started.md b/integrations/observability/csv_file/info/Getting-Started.md new file mode 100644 index 0000000..a1407bf --- /dev/null +++ b/integrations/observability/csv_file/info/Getting-Started.md @@ -0,0 +1,287 @@ +# Getting Started with Nginx Ingestion using Fluent Bit + +This tutorial covers two different setups processes for getting started with Nginx ingestion: a live example using Docker and a self-managed setup with code snippets. + +## Live Example using Docker + +### Step 1: Create Docker Network +Before running any Docker Compose files, create the Docker network. +```sh +docker network create opensearch-net +``` + +**Description**: +Create a Docker network named opensearch-net for the OpenSearch and fluent-bit containers to communicate. +Use this specific command if your existing `opensearch` & `opensearch-dashboards` are already running within a docker-compose container. + +If `opensearch` & `opensearch-dashboards` are running outside of a container scope - for example in your `localhost`, change the original docker network definition Into the following: +```yaml + network_mode: host +``` + +### Step 2: Setup Docker `.env` File +Download and set up the environment variables. +```sh +wget https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/nginx/getting-started/.env +``` + +**Description**: +The .env file contains environment variables required for Docker Compose to configure the OpenSearch and Fluent-Bit containers. + +Update the following parameters: + +```yaml +# OpenSearch Node1 +OPENSEARCH_PORT=9200 +OPENSEARCH_HOST=opensearch +OPENSEARCH_ADDR=${OPENSEARCH_HOST}:${OPENSEARCH_PORT} + +# OpenSearch Dashboard +OPENSEARCH_DASHBOARD_PORT=5601 +OPENSEARCH_DASHBOARD_HOST=opensearch-dashboards +OPENSEARCH_DASHBOARD_ADDR=${OPENSEARCH_DASHBOARD_HOST}:${OPENSEARCH_DASHBOARD_PORT} +``` + +If running `opensearch` & `opensearch-dashboards` are running outside of a container scope - also update the host names `OPENSEARCH_HOST`, `OPENSEARCH_DASHBOARD_HOST` appearing in the .env file to be able to recognize your local running services. + +### Step 3: Setup Fluent Bit Folder +Download the Fluent Bit configuration files. +```sh +wget https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/nginx/getting-started/fluent-bit/fluent-bit.conf \ + https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/nginx/getting-started/fluent-bit/otel-converter.lua \ + https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/nginx/getting-started/fluent-bit/parsers.conf +``` + +**Description**: +Get the local fluent-bit relevant config files. +- Update the `Host` field to match the `opensearch` location - in case its not a part of a docker-compose service, or host name as defined by the docker-compose running your server +- Update the `Index` field to match the index naming specification as defined by the [simple schema for observability](https://github.com/opensearch-project/opensearch-catalog/blob/main/docs/schema/observability/Naming-convention.md). + +```yaml + +[OUTPUT] + Name opensearch + Match nginx.access + Host opensearch-node1 + Port 9200 + Index ss4o_logs-nginx-prod + Suppress_Type_Name On + tls On + tls.verify Off + HTTP_User admin + HTTP_Passwd my_%New%_passW0rd!@# + +[OUTPUT] + Name opensearch + Match apache.access + Host opensearch-node1 + Port 9200 + Index ss4o_logs-apache-prod + Suppress_Type_Name On + tls On + tls.verify Off + HTTP_User admin + HTTP_Passwd my_%New%_passW0rd!@# + +``` + + +### Step 4: Run Docker Compose +Download and run the Docker Compose file for the Nginx live example. + +```sh +wget -O nginx-node.yml https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/nginx/getting-started/nginx-node.yml + +docker-compose -f nginx-node.yml up -d +``` +**Description**: +Run the nginx-node docker compose after updating the `networks` definition to accommodate your existing `opensearch` service. +```yaml +networks: + opensearch-net: + external: true +``` + +## Self-Managed Setup + +The next part describe the details for manually updating the `fluent-bit` agent for running along-side the `nginx` service and transforming its logs +into `simple schema for observability ` compliant json to be ingested into opensearch. +> All the files are present in the `getting-started` folder of this integration. + +### Step 1: Fluent Bit Parser +Set up the Fluent Bit parser configuration to parse Nginx access log fields. + +**parsers.conf** +```ini +[PARSER] + Name apache + Format regex + Regex ^(?[^ ]*) [^ ]* (?[^ ]*) \[(?