diff --git a/integrations/observability/csv_file/getting-started/fluent-bit/.env b/integrations/observability/csv_file/getting-started/.env similarity index 100% rename from integrations/observability/csv_file/getting-started/fluent-bit/.env rename to integrations/observability/csv_file/getting-started/.env diff --git a/integrations/observability/csv_file/getting-started/Getting-Started.md b/integrations/observability/csv_file/getting-started/Getting-Started.md index 9ac46bc..0f5bf04 100644 --- a/integrations/observability/csv_file/getting-started/Getting-Started.md +++ b/integrations/observability/csv_file/getting-started/Getting-Started.md @@ -113,14 +113,14 @@ If you prefer to test this setup using Docker Compose, you can use the following Under the `getting-started` section you can examine a live docker-compose sample: ```yaml /csv_file/getting-started/fluent-bit -|-- docker-complete.yml +|-- docker-compose.yml |-- data/ |-- fluent-bit.conf |-- parsers.conf |-- logs.csv ``` -Use the [docker-compose](fluent-bit/docker-complete.yml) you can find a complete: +Use the [docker-compose](docker-compose.yml) you can find a complete: `docker compose -f docker-complete.yml up -d` would instantiate the services and start sending the csv sample logs into an index. diff --git a/integrations/observability/csv_file/getting-started/fluent-bit/docker-complete.yml b/integrations/observability/csv_file/getting-started/docker-compose.yml similarity index 85% rename from integrations/observability/csv_file/getting-started/fluent-bit/docker-complete.yml rename to integrations/observability/csv_file/getting-started/docker-compose.yml index 64d0525..dc980e5 100644 --- a/integrations/observability/csv_file/getting-started/fluent-bit/docker-complete.yml +++ b/integrations/observability/csv_file/getting-started/docker-compose.yml @@ -31,22 +31,20 @@ services: memlock: soft: -1 hard: -1 - volumes: - - opensearch-data:/usr/share/opensearch/data ports: - 9200:9200 - 9600:9600 networks: - opensearch-net - fluent-bit: - image: fluent/fluent-bit:latest container_name: fluent-bit + image: fluent/fluent-bit:latest volumes: - - ./data:/fluent-bit/data - command: ["/fluent-bit/bin/fluent-bit", "-c", "/fluent-bit/data/fluent-bit.conf"] - depends_on: - - opensearch-node1 + - ./logs:/logs + - ./fluent-bit:/fluent-bit/etc + ports: + - "24224:24224" + - "24224:24224/udp" networks: - opensearch-net diff --git a/integrations/observability/csv_file/getting-started/fluent-bit/data/fluent-bit.conf b/integrations/observability/csv_file/getting-started/fluent-bit/fluent-bit.conf similarity index 68% rename from integrations/observability/csv_file/getting-started/fluent-bit/data/fluent-bit.conf rename to integrations/observability/csv_file/getting-started/fluent-bit/fluent-bit.conf index 5be828a..b10fdd4 100644 --- a/integrations/observability/csv_file/getting-started/fluent-bit/data/fluent-bit.conf +++ b/integrations/observability/csv_file/getting-started/fluent-bit/fluent-bit.conf @@ -1,13 +1,13 @@ [SERVICE] Flush 1 - Log_Level info + Log_Level debug Parsers_File parsers.conf [INPUT] - Name tail - Path /fluent-bit/data/*.csv - Parser csv - Tag csv + Name tail + Path /logs/*.csv + Parser csv_log + Tag csv_logs [INPUT] Name dummy @@ -25,4 +25,10 @@ tls.verify Off Suppress_Type_Name On HTTP_User admin - HTTP_Passwd my_%New%_passW0rd!@# \ No newline at end of file + HTTP_Passwd my_%New%_passW0rd!@# + Trace_Output On + Trace_Error On + +[OUTPUT] + Name stdout + Match * \ No newline at end of file diff --git a/integrations/observability/csv_file/getting-started/fluent-bit/fluent-bit.yml b/integrations/observability/csv_file/getting-started/fluent-bit/fluent-bit.yml deleted file mode 100644 index 1897c8b..0000000 --- a/integrations/observability/csv_file/getting-started/fluent-bit/fluent-bit.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: '3.8' - -services: - fluent-bit: - image: fluent/fluent-bit:latest - container_name: fluent-bit - volumes: - - ./data:/fluent-bit/data - command: ["/fluent-bit/bin/fluent-bit", "-c", "/fluent-bit/data/fluent-bit.conf"] - networks: - - opensearch-net - -volumes: - opensearch-data: - -networks: - opensearch-net: - driver: bridge diff --git a/integrations/observability/csv_file/getting-started/fluent-bit/data/parsers.conf b/integrations/observability/csv_file/getting-started/fluent-bit/parsers.conf similarity index 54% rename from integrations/observability/csv_file/getting-started/fluent-bit/data/parsers.conf rename to integrations/observability/csv_file/getting-started/fluent-bit/parsers.conf index 056a359..1f1304c 100644 --- a/integrations/observability/csv_file/getting-started/fluent-bit/data/parsers.conf +++ b/integrations/observability/csv_file/getting-started/fluent-bit/parsers.conf @@ -1,6 +1,6 @@ [PARSER] - Name csv + Name csv_log Format regex - Regex ^(?[^,]+),(?[^,]+),(?[^,]+),(?[^,]+),(?[^,]+)$ + Regex ^(?[^,]+),(?[^,]+),(?.*),(?[^,]+),(?[^,]+)$ Time_Key timestamp - Time_Format %Y-%m-%d %H:%M:%S + Time_Format %Y-%m-%d %H:%M:%S \ No newline at end of file diff --git a/integrations/observability/csv_file/getting-started/fluent-bit/data/logs.csv b/integrations/observability/csv_file/getting-started/logs/logs.csv similarity index 100% rename from integrations/observability/csv_file/getting-started/fluent-bit/data/logs.csv rename to integrations/observability/csv_file/getting-started/logs/logs.csv diff --git a/integrations/observability/csv_file/getting-started/fluent-bit/opensearch_dashboards.yml b/integrations/observability/csv_file/getting-started/opensearch_dashboards.yml similarity index 100% rename from integrations/observability/csv_file/getting-started/fluent-bit/opensearch_dashboards.yml rename to integrations/observability/csv_file/getting-started/opensearch_dashboards.yml diff --git a/integrations/observability/golang_client/getting-started/fluent-bit/fluent-bit.conf b/integrations/observability/golang_client/getting-started/fluent-bit/fluent-bit.conf index a08b404..4179ce5 100644 --- a/integrations/observability/golang_client/getting-started/fluent-bit/fluent-bit.conf +++ b/integrations/observability/golang_client/getting-started/fluent-bit/fluent-bit.conf @@ -15,7 +15,7 @@ Match * Port 9200 Type _doc - Index applicative_logs + Index applicative_logs-golang tls On tls.verify Off Suppress_Type_Name On diff --git a/integrations/observability/java_client/getting-started/fluent-bit/fluent-bit.conf b/integrations/observability/java_client/getting-started/fluent-bit/fluent-bit.conf index a08b404..6635521 100644 --- a/integrations/observability/java_client/getting-started/fluent-bit/fluent-bit.conf +++ b/integrations/observability/java_client/getting-started/fluent-bit/fluent-bit.conf @@ -15,7 +15,7 @@ Match * Port 9200 Type _doc - Index applicative_logs + Index applicative_logs-java tls On tls.verify Off Suppress_Type_Name On diff --git a/integrations/observability/python_client/getting-started/fluent-bit/fluent-bit.conf b/integrations/observability/python_client/getting-started/fluent-bit/fluent-bit.conf index a08b404..8bfb8b0 100644 --- a/integrations/observability/python_client/getting-started/fluent-bit/fluent-bit.conf +++ b/integrations/observability/python_client/getting-started/fluent-bit/fluent-bit.conf @@ -15,7 +15,7 @@ Match * Port 9200 Type _doc - Index applicative_logs + Index applicative_logs-python tls On tls.verify Off Suppress_Type_Name On