Merge pull request #34 from Swiddis/visualization-catalog

Stub visualization catalog with current Nginx visualizations

visualizations/README.md

@@ -0,0 +1,30 @@
+# Visualization Catalog
+
+The Visualization Catalog is a central repository for organizing and sharing visualizations used in integrations developed using the standard SS4O framework.
+This catalog aims to eliminate duplication of effort by providing a comprehensive collection of commonly applicable visualizations across multiple contexts.
+
+## Problem
+
+As we continue to develop more integrations using standard SS4O, we often encounter overlapping visualization functionality.
+For example, many integrations involving an http component may benefit from a "Status codes over time" visualization.
+However, without a centralized location for visualizations, developers end up reinventing the wheel, resulting in redundancy and inefficiency.
+
+## Solution
+
+The solution to this problem is the Visualization Catalog, which serves as a directory within this repository.
+It allows integration developers to easily find and utilize visualizations that are relevant to their data.
+By sharing visualizations in one place, we promote collaboration and reduce duplication of effort.
+
+## Tasks
+
+- [X] Organize Existing Visualizations: Create a new folder within the repository dedicated to the Visualization Catalog. Optionally, structure the visualizations by component to facilitate searchability.
+- [ ] Documentation and Guidelines: Provide comprehensive documentation on developing visualizations, explaining the major types and how different fields work. This will serve as a guide for integration developers looking to contribute their own visualizations to the catalog.
+- [ ] Enhance the CLI Tool: Extend the CLI tool introduced in #31 (implemented via #32) to analyze visualizations and determine the types of data they can work with. This feature will enable individual visualizations to be checked for compatibility with the standard schema.
+- [ ] Search Functionality: Improve the CLI tool by adding search functionality. This enhancement will enable users to quickly find visualizations based on their specific requirements.
+- [ ] Automated Dashboard Compilation: Enhance the CLI tool with the capability to automatically compile a list of visualizations into a dashboard. While this feature may be complex, its core objective is to generate a Saved Object file that can be imported into OpenSearch Dashboards.
+- [ ] Advanced Catalog API: Consider extending the functionality of the Visualization Catalog to an API with more advanced features. However, this step is optional and should be evaluated based on project demand and feasibility.
+
+## Current Visualizations
+
+The list of visualizations is maintained in [index.yml](index.yml).
+The format is still experimental, and may be subject to change in the future.

visualizations/index.yml

@@ -0,0 +1,21 @@
+visualization_catalog:
+  observability:
+    http:
+      parent: logs
+      visualizations:
+      - path: observability/http/data_volume.json
+        name: Data Volume
+        description: Get the rate of data (in bytes) returned by a log source.
+        screenshot: observability/http/data_volume.png
+      - path: observability/http/errors_over_time.json
+        name: Errors over Time
+        description: Get the rate of errors returned by a log source.
+        screenshot: observability/http/errors_over_time.png
+      - path: observability/http/status_codes_over_time.json
+        name: Status codes over Time
+        description: Get the proportions of different status codes returned by a log source.
+        screenshot: observability/http/status_codes_over_time.png
+      - path: observability/http/top_urls.json
+        name: Top URLs
+        description: Get the most frequent URLs visited by a log source.
+        screenshot: observability/http/top_urls.png

visualizations/observability/http/data_volume.json

@@ -0,0 +1,27 @@
+{
+    "attributes": {
+        "description": "",
+        "kibanaSavedObjectMeta": {
+            "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
+        },
+        "savedSearchRefName": "search_0",
+        "title": "Data Volume",
+        "uiStateJSON": "{}",
+        "version": 1,
+        "visState": "{\"title\":\"Data Volume\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"http.response.bytes\",\"customLabel\":\"Response Bytes\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"observerTime\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Response Bytes\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Response Bytes\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{}}}"
+    },
+    "id": "7b1dd46f-2c27-4933-a840-d4fc88f46d53",
+    "migrationVersion": {
+        "visualization": "7.10.0"
+    },
+    "references": [
+        {
+            "id": "aaa33c4e-794a-4923-b853-1abd7359d807",
+            "name": "search_0",
+            "type": "search"
+        }
+    ],
+    "type": "visualization",
+    "updated_at": "2023-06-30T23:54:38.187Z",
+    "version": "WzcyNCwzXQ=="
+}

visualizations/observability/http/errors_over_time.json

@@ -0,0 +1,27 @@
+{
+    "attributes": {
+        "description": "",
+        "kibanaSavedObjectMeta": {
+            "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
+        },
+        "savedSearchRefName": "search_0",
+        "title": "HTTP Errors over time",
+        "uiStateJSON": "{}",
+        "version": 1,
+        "visState": "{\"title\":\"HTTP Errors over time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"
+    },
+    "id": "0cee1420-9fe6-4135-8def-b053007c2409",
+    "migrationVersion": {
+        "visualization": "7.10.0"
+    },
+    "references": [
+        {
+            "id": "32bc8b8a-3b0d-4687-b4b1-97f2336a13e9",
+            "name": "search_0",
+            "type": "search"
+        }
+    ],
+    "type": "visualization",
+    "updated_at": "2023-06-30T23:54:38.187Z",
+    "version": "WzcyMywzXQ=="
+}

visualizations/observability/http/status_codes_over_time.json

@@ -0,0 +1,27 @@
+{
+    "attributes": {
+        "description": "",
+        "kibanaSavedObjectMeta": {
+            "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
+        },
+        "savedSearchRefName": "search_0",
+        "title": "HTTP Status Codes over Time",
+        "uiStateJSON": "{}",
+        "version": 1,
+        "visState": "{\"title\":\"HTTP Status Codes over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"filters\",\"params\":{\"filters\":[{\"input\":{\"query\":\"http.response.status_code:[200 TO 299]\",\"language\":\"lucene\"},\"label\":\"200s\"},{\"input\":{\"query\":\"http.response.status_code:[300 TO 399]\",\"language\":\"lucene\"},\"label\":\"300s\"},{\"input\":{\"query\":\"http.response.status_code:[400 TO 499]\",\"language\":\"lucene\"},\"label\":\"400s\"},{\"input\":{\"query\":\"http.response.status_code:[500 TO 599]\",\"language\":\"lucene\"},\"label\":\"500s\"},{\"input\":{\"query\":\"http.response.status_code:0\",\"language\":\"lucene\"},\"label\":\"0\"}]},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"
+    },
+    "id": "fbd6622e-4f93-4764-be8c-3cbfefe9592c",
+    "migrationVersion": {
+        "visualization": "7.10.0"
+    },
+    "references": [
+        {
+            "id": "aaa33c4e-794a-4923-b853-1abd7359d807",
+            "name": "search_0",
+            "type": "search"
+        }
+    ],
+    "type": "visualization",
+    "updated_at": "2023-06-30T23:54:38.187Z",
+    "version": "WzcyMSwzXQ=="
+}

visualizations/observability/http/top_urls.json

@@ -0,0 +1,26 @@
+{
+    "attributes": {
+        "description": "",
+        "kibanaSavedObjectMeta": {
+            "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"
+        },
+        "title": "HTTP Top URLs",
+        "uiStateJSON": "{}",
+        "version": 1,
+        "visState": "{\"title\":\"HTTP Top URLs\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"http.url\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}"
+    },
+    "id": "a59b6730-17a2-11ee-b5d7-d9b73d55db70",
+    "migrationVersion": {
+        "visualization": "7.10.0"
+    },
+    "references": [
+        {
+            "id": "d6af4161-6be5-48d0-9897-d1ffba8ab9b6",
+            "name": "kibanaSavedObjectMeta.searchSourceJSON.index",
+            "type": "index-pattern"
+        }
+    ],
+    "type": "visualization",
+    "updated_at": "2023-07-01T00:03:02.563Z",
+    "version": "WzczMiwzXQ=="
+}