diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml new file mode 100644 index 000000000..3cf1c8eb0 --- /dev/null +++ b/.github/workflows/python-ci.yml @@ -0,0 +1,40 @@ +name: Integrations CLI Test and Build + +on: + workflow_dispatch: + pull_request: + push: + branches-ignore: + - 'dependabot/**' + paths: + - 'scripts/integrations-cli' + - '.github/workflows/python-ci.yml' + +jobs: + build: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./scripts/integrations-cli + strategy: + matrix: + python-version: [3.8] + opensearch-version: [ latest ] + + steps: + - name: Checkout Integrations CLI + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Python Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install setuptools wheel + + - name: Run Tests + run: pytest diff --git a/scripts/integrations-cli/.gitignore b/scripts/integrations-cli/.gitignore new file mode 100644 index 000000000..1dbd8b5a9 --- /dev/null +++ b/scripts/integrations-cli/.gitignore @@ -0,0 +1,180 @@ +# Directories used by CLI +artifacts +integrations + +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +# End of https://www.toptal.com/developers/gitignore/api/python diff --git a/scripts/integrations-cli/README.md b/scripts/integrations-cli/README.md new file mode 100644 index 000000000..d918f5f3b --- /dev/null +++ b/scripts/integrations-cli/README.md @@ -0,0 +1,143 @@ +# Integrations CLI + +## Description + +This is a CLI tool for creating, validating, and maintaining OpenSearch Dashboard Integrations. +The tool has three main functions: + +- `create` a new integration with a specified configuration. +- `check` an integration for validity +- `package` an integration to be upload to an OpenSearch Integrations instance + +Further information is provided in the `Usage` section. + +## Installation + +The project requires Python 3.7 or better. +As the project is not yet a full Python package, it must be installed manually. +To manually install, create a virtual environment and install the requirements. + +On Unix: + +```bash +$ python3 -m venv venv +$ source ./venv/bin/activate +$ pip install -r requirements.txt +``` + +On Windows: + +```ps1 +> python -m venv venv +> ./venv/Scripts/activate.ps1 +> pip install -r requirements.txt +``` + +On Unix, the script `integ-cli` that will automatically install necessary dependencies. + +```bash +$ ./integ-cli +``` + +## Usage + +On Unix, the CLI is callable with the included `integ-cli` script. +The simplest way to start making an integration is via `create`. + +```bash +$ ./integ-cli --help +``` + +Further development instructions are provided below. + +### What's in an Integration? + +An integration typically consists of three components: + +- A Config file +- Schemas +- Assets + +The Config file `config.json` defines parameters for the integration. +Most notably, the versions of all involved dependencies. +It also has information like a description, +which is used when presenting the integration in the catalog. + +The Schemas `/schema` are the rules for the data, and define the permitted fields and types. +During integration creation, they are provided based on the selected components. +These are typically not meant to be changed. + +The Assets `/assets` are the "heart" of the integration. +They contain the bulk of what you develop. +All developed Assets should be compatible with the provided components. +Saved Objects, such as schemas and dashboards, are saved under `/assets/display`. + +### How to Install an Integration + +First, generate a template with desired defaults. + +```bash +$ ./integ-cli create [NAME] +``` + +To begin using the integration, first load the templates in `schema`. +The loading order generally depends on dependencies, look for a `composed_of` field. +Dependencies are loaded by uploading them to a running OpenSearch cluster. +For example, to load the `http` component under an `nginx` integration: + +```bash +$ curl -XPUT localhost:9200/_component_template/http_template -H "Content-Type: application/json" --data-binary "@integrations/nginx/schema/http.mapping" +``` + +If the component contains an `index_mapping`, +then instead post it as an index template, e.g. with `logs`: + +```bash +$ curl -XPUT localhost:9200/_index_template/logs -H "Content-Type: application/json" --data-binary "@integrations/nginx/schema/logs.mapping" +``` + +Now we turn to OpenSearch Dashboards. +First is an index pattern, which defines where to find the data. +This should generally match the pattern `ss4o_*`. + +```bash +$ curl -XPOST localhost:5601/api/saved_objects/index-pattern/sso_logs -H 'osd-xsrf: true' -H 'Content-Type: application/json' -d '{ "attributes": { "title": "sso_logs-*-*", "timeFieldName": "@timestamp" } }' +``` + +Finally, load the saved objects under `/assets/display`: + +```bash +curl -XPOST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "osd-xsrf: true" --form file=@integrations/nginx/assets/display/http.ndjson +``` + +We are aware that this is a complicated process. +We're working on it. + +### How to Develop an Integration + +Once the saved objects have been loaded, +you can freely change the relevant dashboards within the OpenSearch Dashboards UI. +To update the Display Assets, +export the new dashboards and replace the `.ndjson` files as desired. + +If you need to add another `schema`, +add it to the `config.json` and add the mapping to the `/schema` directory. +This is also going to be automated in the future. + +## Development + +Running the tests: + +```bash +$ python -m unittest +``` + +Formatting: + +```bash +$ black . +``` + +## License + +This project, like the rest of the OpenSearch projects, is licensed under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0.html). diff --git a/scripts/integrations-cli/integ-cli b/scripts/integrations-cli/integ-cli new file mode 100644 index 000000000..80460a679 --- /dev/null +++ b/scripts/integrations-cli/integ-cli @@ -0,0 +1,2 @@ +#!/usr/bin/bash +python3 integrations_cli/main.py "$@" diff --git a/scripts/integrations-cli/integrations_cli/helpers/__init__.py b/scripts/integrations-cli/integrations_cli/helpers/__init__.py new file mode 100644 index 000000000..56c892129 --- /dev/null +++ b/scripts/integrations-cli/integrations_cli/helpers/__init__.py @@ -0,0 +1,3 @@ +from . import constants, validate +from .catalog import CatalogManager +from .create import IntegrationBuilder diff --git a/scripts/integrations-cli/integrations_cli/helpers/catalog.py b/scripts/integrations-cli/integrations_cli/helpers/catalog.py new file mode 100644 index 000000000..8ef6ed68c --- /dev/null +++ b/scripts/integrations-cli/integrations_cli/helpers/catalog.py @@ -0,0 +1,22 @@ +import json +import os +from functools import lru_cache + +from returns.io import impure_safe + +from . import constants, validate + + +@impure_safe +def _load_catalog_file() -> dict: + catalog_path = os.path.join(constants.SCHEMA_ROOT, "observability/catalog.json") + with open(catalog_path, "r", encoding="utf-8") as catalog_file: + return json.load(catalog_file) + + +class CatalogManager: + @lru_cache + def __init__(self): + catalog = _load_catalog_file().bind_result(validate.validate_catalog) + # For now just re-throw exceptions on catalog load failure + self.catalog = catalog.unwrap()._inner_value diff --git a/scripts/integrations-cli/integrations_cli/helpers/constants.py b/scripts/integrations-cli/integrations_cli/helpers/constants.py new file mode 100644 index 000000000..b6445560c --- /dev/null +++ b/scripts/integrations-cli/integrations_cli/helpers/constants.py @@ -0,0 +1,24 @@ +import glob +import json +import os + +SCHEMA_ROOT = "../../src/main/resources/schema" + +DEFAULT_CONFIG = { + "template-name": "default", + "version": {"integration": "0.1.0", "schema": "1.0.0", "resource": "^1.23.0"}, + "description": "", + "identification": "", + "components": [], + "collection": [], + "repository": {"url": ""}, +} + +SCHEMAS = {} + +# For now, assume we're running in the current relative directory +if os.path.isdir(SCHEMA_ROOT): + for filename in glob.glob(os.path.join(SCHEMA_ROOT, "**/*.schema"), recursive=True): + schema_name = os.path.split(filename)[1] + with open(filename, "r", encoding="utf-8") as schema_file: + SCHEMAS[schema_name] = json.load(schema_file) diff --git a/scripts/integrations-cli/integrations_cli/helpers/create.py b/scripts/integrations-cli/integrations_cli/helpers/create.py new file mode 100644 index 000000000..df1fc2f62 --- /dev/null +++ b/scripts/integrations-cli/integrations_cli/helpers/create.py @@ -0,0 +1,96 @@ +import json +import os +import re +from copy import deepcopy +from urllib import parse +import click +import requests +from termcolor import colored + +from .constants import DEFAULT_CONFIG +from .validate import validate_config + + +class IntegrationBuilder: + def __init__(self): + self.path = None + self.config = deepcopy(DEFAULT_CONFIG) + self.dashboards = [] + self.mappings = dict() + + def with_name(self, name: str) -> "IntegrationBuilder": + self.config["template-name"] = name + return self + + def with_path(self, path: str) -> "IntegrationBuilder": + self.path = path + return self + + def with_schema_version(self, version: str) -> "IntegrationBuilder": + if not re.match(r"^\d+\.\d+\.\d+", version): + raise ValueError("Invalid version") + self.config["version"]["schema"] = version + return self + + def with_resource_version(self, version: str) -> "IntegrationBuilder": + if not re.match(r"^\^?\d+\.\d+\.\d+", version): + raise ValueError("Invalid version") + self.config["version"]["resource"] = version + return self + + def with_description(self, desc: str) -> "IntegrationBuilder": + self.config["description"] = desc + return self + + def with_repository(self, repo_url: str) -> "IntegrationBuilder": + if repo_url.strip() == "": + return self + if not parse.urlparse(repo_url, allow_fragments=False): + raise ValueError("Invalid URL") + self.config["repository"]["url"] = repo_url + return self + + def with_component(self, component: dict) -> "IntegrationBuilder": + ## TODO make robust + self.config["components"] = sorted( + set(self.config["components"]) | {component["component"]} + ) + url = ( + component["url"] + .replace("github.com", "raw.githubusercontent.com") + .replace("/tree", "") + + ".mapping" + ) + try: + mapping = requests.get(url).json() + self.mappings[url.split("/")[-1]] = mapping + except requests.exceptions.ConnectionError: + click.echo( + colored( + "Network error while pulling component mapping, manual install needed", + "red", + ) + ) + return self + + def with_dashboard(self, component: str) -> "IntegrationBuilder": + ## TODO make robust + self.dashboards.append(component) + return self + + def build(self): + assert self.path is not None + os.makedirs(self.path, exist_ok=True) + files = {"config.json": validate_config(self.config).unwrap()} + for fname, mapping in self.mappings.items(): + files[f"schema/{fname}"] = mapping # TODO validate + for filename, data in files.items(): + with open(os.path.join(self.path, filename), "w", encoding="utf-8") as file: + json.dump(data, file, ensure_ascii=False, indent=2) + for dashboard in self.dashboards: + with open(dashboard, "r") as fin: + with open( + os.path.join(self.path, "assets/display", dashboard.split("/")[-1]), + "w", + ) as fout: + fout.write(fin.read()) diff --git a/scripts/integrations-cli/integrations_cli/helpers/validate.py b/scripts/integrations-cli/integrations_cli/helpers/validate.py new file mode 100644 index 000000000..ee8b085b0 --- /dev/null +++ b/scripts/integrations-cli/integrations_cli/helpers/validate.py @@ -0,0 +1,29 @@ +import jsonschema +from returns.result import Failure, Result, safe + +from .constants import SCHEMAS + + +@safe( + exceptions=( + jsonschema.exceptions.ValidationError, + jsonschema.exceptions.SchemaError, + ) +) +def validate_instance(instance: dict, schema: dict) -> dict: + jsonschema.validate(instance, schema) + return instance + + +def validate_config(config: dict) -> Result[dict, Exception]: + try: + return validate_instance(config, SCHEMAS["integration.schema"]) + except KeyError as err: + return Failure(err) + + +def validate_catalog(catalog: dict) -> Result[dict, Exception]: + try: + return validate_instance(catalog, SCHEMAS["catalog.schema"]) + except KeyError as err: + return Failure(err) diff --git a/scripts/integrations-cli/integrations_cli/main.py b/scripts/integrations-cli/integrations_cli/main.py new file mode 100644 index 000000000..777ac57fb --- /dev/null +++ b/scripts/integrations-cli/integrations_cli/main.py @@ -0,0 +1,145 @@ +#!./venv/bin/python3 +import json +import os +import zipfile + +import click +from returns.pipeline import is_successful +from returns.result import Result +from termcolor import colored + +import helpers + +available_templates = {"log": "template-dashboards/logs.ndjson"} + + +@click.group() +def integrations_cli(): + """Create and maintain Integrations for the OpenSearch Integrations Plugin.""" + + +def do_add_component(builder: helpers.IntegrationBuilder) -> bool: + """Start an interactive session for""" + manager = helpers.CatalogManager() + choices = {} + for category in manager.catalog["categories"]: + click.echo(f"- {colored(category['category'], color='light_blue')}") + for component in category["components"]: + desc = component["description"] + desc = desc if len(desc) < 50 else desc[:47] + "..." + click.echo(f" - {colored(component['component'], 'light_yellow')}: {desc}") + choices[component["component"]] = component + component = choices[ + click.prompt( + "Select component", type=click.Choice(list(choices)), show_choices=False + ) + ] + builder = builder.with_component(component) + for dep in component["dependencies"] if "dependencies" in component else []: + builder = builder.with_component(choices[dep]) + if component["component"] in available_templates: + if ( + click.prompt( + f"A pre-made template dashboard for component `{component['component']}` was detected. Include it? (y/n)", + type=click.Choice(["y", "n"], False), + show_choices=False, + ) + == "y" + ): + builder.with_dashboard(available_templates[component["component"]]) + return ( + click.prompt( + "Configure another component? (y/n)", + type=click.Choice(["y", "n"], False), + show_choices=False, + ) + == "y" + ) + + +@integrations_cli.command() +@click.argument("name") +def create(name: str): + """Create a new Integration from a specified template.""" + click.echo(f"Creating new integration '{name}'") + integration_path = os.path.join(os.getcwd(), "integrations", name) + if os.path.exists(integration_path) and len(os.listdir(integration_path)) != 0: + raise click.ClickException( + f"destination path '{integration_path}' exists and is non-empty" + ) + for subdir in ["assets", "schema", "assets/display"]: + os.makedirs(os.path.join(integration_path, subdir)) + builder = helpers.IntegrationBuilder().with_name(name).with_path(integration_path) + click.prompt("Schema version", default="1.0.0", type=builder.with_schema_version) + click.prompt("Resource version", type=builder.with_resource_version) + click.prompt("Integration description", default="", type=builder.with_description) + click.prompt( + "Integration repository URL", + default="", + type=builder.with_repository, + ) + add_component = ( + click.prompt( + "Would you like to configure components interactively? (y/n)", + type=click.Choice(["y", "n"], False), + show_choices=False, + ) + == "y" + ) + while add_component: + add_component = do_add_component(builder) + builder.build() + click.echo(colored(f"Integration created at '{integration_path}'", "green")) + + +def validate_component(path: str, validator) -> Result[dict, Exception]: + with open(path, "r", encoding="utf-8") as item_data: + loaded = json.load(item_data) + result = validator(loaded) + if not is_successful(result): + msg = str(result.failure) + msg = ("> " + msg).replace("\n", "\n> ") + click.echo(colored(msg, "red"), err=True) + return result + + +def full_integration_is_valid(name: str) -> bool: + integration_path = os.path.join(os.getcwd(), "integrations", name) + integration_parts = {"config.json": helpers.validate.validate_config} + encountered_errors = False + for item, validator in integration_parts.items(): + item_path = os.path.join(integration_path, item) + if not is_successful(validate_component(item_path, validator)): + encountered_errors = True + return not encountered_errors + + +@integrations_cli.command() +@click.argument("name") +def check(name: str): + """Analyze the current Integration and report errors.""" + click.echo(f"Checking integration '{name}'") + if full_integration_is_valid(name): + click.echo(colored("Integration is valid", "green")) + + +@integrations_cli.command() +@click.argument("name") +def package(name: str): + """Package the current integration for use in OpenSearch.""" + click.echo(f"Checking integration '{name}'") + if not full_integration_is_valid(name): + return + click.echo(f"Packaging integration '{name}'") + os.makedirs("artifacts", exist_ok=True) + integration_path = os.path.join(os.getcwd(), "integrations", name) + artifact_path = os.path.join("artifacts", f"{name}.zip") + with zipfile.ZipFile(artifact_path, "w") as zf: + for dirpath, dirnames, filenames in os.walk(integration_path): + for item in dirnames + filenames: + zf.write(os.path.join(integration_path, dirpath, item), arcname=os.path.join(dirpath, item)) + click.echo(colored(f"Packaged integration as '{artifact_path}'", "green")) + + +if __name__ == "__main__": + integrations_cli() diff --git a/scripts/integrations-cli/requirements.txt b/scripts/integrations-cli/requirements.txt new file mode 100644 index 000000000..22fe26ab1 Binary files /dev/null and b/scripts/integrations-cli/requirements.txt differ diff --git a/scripts/integrations-cli/scripts.py b/scripts/integrations-cli/scripts.py new file mode 100644 index 000000000..f13d3d394 --- /dev/null +++ b/scripts/integrations-cli/scripts.py @@ -0,0 +1,17 @@ +import subprocess + + +def test(): + """ + Run all unittests. Equivalent to: + `poetry run python -u -m unittest discover` + """ + subprocess.run(["python", "-u", "-m", "unittest", "discover"]) + + +def format(): + """ + Reformat all code. Equivalent to: + `poetry run black .` + """ + subprocess.run(["black", "."]) diff --git a/scripts/integrations-cli/template-dashboards/logs.ndjson b/scripts/integrations-cli/template-dashboards/logs.ndjson new file mode 100644 index 000000000..07f561a4a --- /dev/null +++ b/scripts/integrations-cli/template-dashboards/logs.ndjson @@ -0,0 +1,10 @@ +{"attributes":{"fields":"[{\"count\":0,\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_score\",\"type\":\"number\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_type\",\"type\":\"string\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"attributes.data_stream.dataset\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"attributes.data_stream.dataset.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"attributes.data_stream.dataset\"}}},{\"count\":0,\"name\":\"attributes.data_stream.namespace\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"attributes.data_stream.namespace.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"attributes.data_stream.namespace\"}}},{\"count\":0,\"name\":\"attributes.data_stream.type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"attributes.data_stream.type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"attributes.data_stream.type\"}}},{\"count\":0,\"name\":\"body\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"body.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"body\"}}},{\"count\":0,\"name\":\"communication.source.address\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"communication.source.address.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"communication.source.address\"}}},{\"count\":0,\"name\":\"communication.source.ip\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"communication.source.ip.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"communication.source.ip\"}}},{\"count\":0,\"name\":\"event.category\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"event.category.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"event.category\"}}},{\"count\":0,\"name\":\"event.domain\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"event.domain.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"event.domain\"}}},{\"count\":0,\"name\":\"event.kind\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"event.kind.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"event.kind\"}}},{\"count\":0,\"name\":\"event.name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"event.name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"event.name\"}}},{\"count\":0,\"name\":\"event.result\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"event.result.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"event.result\"}}},{\"count\":0,\"name\":\"event.type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"event.type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"event.type\"}}},{\"count\":0,\"name\":\"http.flavor\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"http.flavor.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"http.flavor\"}}},{\"count\":1,\"name\":\"http.request.method\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"http.request.method.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"http.request.method\"}}},{\"count\":0,\"name\":\"http.response.bytes\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"http.response.status_code\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"http.response.status_code.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"http.response.status_code\"}}},{\"count\":0,\"name\":\"http.url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"http.url.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"http.url\"}}},{\"count\":0,\"name\":\"observerTime\",\"type\":\"date\",\"esTypes\":[\"date\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"span_id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"span_id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"span_id\"}}},{\"count\":0,\"name\":\"trace_id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"trace_id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"trace_id\"}}}]","timeFieldName":"@timestamp","title":"sso_logs-*-*"},"id":"47892350-b495-11ed-af0a-cf5c93b5a3b6","migrationVersion":{"index-pattern":"7.6.0"},"references":[],"type":"index-pattern","updated_at":"2023-04-14T23:22:53.971Z","version":"Wzk4LDJd"} +{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"All Logs\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"exists\",\"key\":\"http.url\",\"value\":\"exists\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"exists\":{\"field\":\"http.url\"},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"[Http Logs 1.0] All HTTP Logs","version":1},"id":"d80e05b2-518c-4c3d-9651-4c9d8632dce4","migrationVersion":{"search":"7.9.3"},"references":[{"id":"47892350-b495-11ed-af0a-cf5c93b5a3b6","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"47892350-b495-11ed-af0a-cf5c93b5a3b6","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2023-04-14T23:32:16.431Z","version":"WzEwNCwyXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"},"savedSearchRefName":"search_0","title":"[Http Logs 1.0] Response codes over time","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"[Http Logs 1.0] Response codes over time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"filters\",\"params\":{\"filters\":[{\"input\":{\"query\":\"http.response.status_code:[200 TO 299]\",\"language\":\"lucene\"},\"label\":\"200s\"},{\"input\":{\"query\":\"http.response.status_code:[300 TO 399]\",\"language\":\"lucene\"},\"label\":\"300s\"},{\"input\":{\"query\":\"http.response.status_code:[400 TO 499]\",\"language\":\"lucene\"},\"label\":\"400s\"},{\"input\":{\"query\":\"http.response.status_code:[500 TO 599]\",\"language\":\"lucene\"},\"label\":\"500s\"},{\"input\":{\"query\":\"http.response.status_code:0\",\"language\":\"lucene\"},\"label\":\"0\"}]},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"3b49a65d-54d8-483d-a8f0-3d7c855e1ecf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"d80e05b2-518c-4c3d-9651-4c9d8632dce4","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2023-04-14T23:22:03.376Z","version":"Wzg5LDJd"} +{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"query\":{\"query\":\"http.response.status_code >= 300\",\"language\":\"kuery\"},\"version\":true,\"highlight\":{\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"pre_tags\":[\"@kibana-highlighted-field@\"],\"require_field_match\":false,\"fragment_size\":2147483647},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"title":"[Http Logs 1.0] Http Error Logs","version":1},"id":"9f820fbe-ddde-43a2-9402-30bd295c97f6","migrationVersion":{"search":"7.9.3"},"references":[{"id":"47892350-b495-11ed-af0a-cf5c93b5a3b6","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2023-04-14T23:24:54.247Z","version":"WzEwMiwyXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"},"savedSearchRefName":"search_0","title":"[Http Logs 1.0] Errors over time","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"[Http Logs 1.0] Errors over time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"865e577b-634b-4a65-b9d6-7e324c395d18","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"9f820fbe-ddde-43a2-9402-30bd295c97f6","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2023-04-14T23:22:03.376Z","version":"WzkxLDJd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Top Paths","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Top Paths\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"http.url.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}"},"id":"dc1803f0-b478-11ed-9063-ebe46f9ac203","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"d80e05b2-518c-4c3d-9651-4c9d8632dce4","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2023-04-14T23:22:03.376Z","version":"WzkyLDJd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Data Volume","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Data Volume\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"http.response.bytes\",\"customLabel\":\"Response Bytes\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"observerTime\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Response Bytes\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Response Bytes\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{}}}"},"id":"99acc580-b47a-11ed-9063-ebe46f9ac203","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"d80e05b2-518c-4c3d-9651-4c9d8632dce4","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2023-04-14T23:22:03.376Z","version":"WzkzLDJd"} +{"attributes":{"description":"requests per minute aggregation","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Req-per-min","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Req-per-min\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"moving_avg\",\"params\":{\"metricAgg\":\"custom\",\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"count\",\"params\":{}},\"window\":5,\"script\":\"MovingFunctions.unweightedAvg(values)\",\"customLabel\":\"Requests (Moving Avg.)\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"m\",\"drop_partials\":false,\"min_doc_count\":0,\"extended_bounds\":{},\"customLabel\":\"Req/Min\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}"},"id":"01ea64d0-b62f-11ed-a677-43d7aa86763b","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"47892350-b495-11ed-af0a-cf5c93b5a3b6","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-14T23:22:03.376Z","version":"Wzk0LDJd"} +{"attributes":{"description":"Basic template for HTTP-based observability dashboards","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"embeddableConfig\":{\"hidePanelTitles\":false,\"title\":\"[HTTP Logs 1.0] Response codes over time\"},\"gridData\":{\"h\":8,\"i\":\"1f31e50b-06e3-41e6-972e-e4e5fe1a9872\",\"w\":48,\"x\":0,\"y\":0},\"panelIndex\":\"1f31e50b-06e3-41e6-972e-e4e5fe1a9872\",\"title\":\"[HTTP Logs 1.0] Response codes over time\",\"version\":\"2.5.0\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{\"hidePanelTitles\":false,\"title\":\"[HTTP Logs 1.0] Errors over time\"},\"gridData\":{\"h\":9,\"i\":\"d91a8da4-b34b-470a-aca6-9c76b47cd6fb\",\"w\":24,\"x\":0,\"y\":8},\"panelIndex\":\"d91a8da4-b34b-470a-aca6-9c76b47cd6fb\",\"title\":\"[HTTP Logs 1.0] Errors over time\",\"version\":\"2.5.0\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{\"hidePanelTitles\":false,\"title\":\"[HTTP Logs 1.0] Top Paths\"},\"gridData\":{\"h\":15,\"i\":\"27149e5a-3a77-4f3c-800e-8a160c3765f4\",\"w\":24,\"x\":24,\"y\":8},\"panelIndex\":\"27149e5a-3a77-4f3c-800e-8a160c3765f4\",\"title\":\"[HTTP Logs 1.0] Top Paths\",\"version\":\"2.5.0\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{\"hidePanelTitles\":false,\"title\":\"[HTTP Logs 1.0] Data Volume\"},\"gridData\":{\"h\":15,\"i\":\"4d8c2aa7-159c-4a1a-80ff-00a9299056ce\",\"w\":24,\"x\":0,\"y\":17},\"panelIndex\":\"4d8c2aa7-159c-4a1a-80ff-00a9299056ce\",\"title\":\"[HTTP Logs 1.0] Data Volume\",\"version\":\"2.5.0\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{\"hidePanelTitles\":false,\"title\":\"[HTTP Logs 1.0] Requests per Minute\"},\"gridData\":{\"h\":15,\"i\":\"800b7f19-f50c-417f-8987-21b930531cbe\",\"w\":24,\"x\":24,\"y\":23},\"panelIndex\":\"800b7f19-f50c-417f-8987-21b930531cbe\",\"title\":\"[HTTP Logs 1.0] Requests per Minute\",\"version\":\"2.5.0\",\"panelRefName\":\"panel_4\"}]","refreshInterval":{"pause":true,"value":0},"timeFrom":"now-10m","timeRestore":true,"timeTo":"now","title":"[HTTP Logs 1.0] Overview","version":1},"id":"96847220-5261-44d0-89b4-65f3a659f13a","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"3b49a65d-54d8-483d-a8f0-3d7c855e1ecf","name":"panel_0","type":"visualization"},{"id":"865e577b-634b-4a65-b9d6-7e324c395d18","name":"panel_1","type":"visualization"},{"id":"dc1803f0-b478-11ed-9063-ebe46f9ac203","name":"panel_2","type":"visualization"},{"id":"99acc580-b47a-11ed-9063-ebe46f9ac203","name":"panel_3","type":"visualization"},{"id":"01ea64d0-b62f-11ed-a677-43d7aa86763b","name":"panel_4","type":"visualization"}],"type":"dashboard","updated_at":"2023-04-14T23:33:16.755Z","version":"WzEwNSwyXQ=="} +{"exportedCount":9,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/scripts/integrations-cli/tests/__init__.py b/scripts/integrations-cli/tests/__init__.py new file mode 100644 index 000000000..70119999d --- /dev/null +++ b/scripts/integrations-cli/tests/__init__.py @@ -0,0 +1 @@ +# Empty init file required for test discovery diff --git a/scripts/integrations-cli/tests/test_validate.py b/scripts/integrations-cli/tests/test_validate.py new file mode 100644 index 000000000..4852f490c --- /dev/null +++ b/scripts/integrations-cli/tests/test_validate.py @@ -0,0 +1,37 @@ +import unittest +from copy import deepcopy + +import jsonschema +from returns.pipeline import is_successful + +import integrations_cli.helpers.constants as constants +import integrations_cli.helpers.validate as validate +from integrations_cli.helpers.catalog import _load_catalog_file + + +class TestSchemas: + def test_config_schema_is_valid(self): + config_schema = constants.SCHEMAS["integration.schema"] + jsonschema.Draft6Validator.check_schema(config_schema) + + +class TestConfigValidations: + def test_default_config_is_valid(self): + config = deepcopy(constants.DEFAULT_CONFIG) + assert is_successful(validate.validate_config(config)) + + def test_default_with_no_name_is_invalid(self): + config = deepcopy(constants.DEFAULT_CONFIG) + del config["template-name"] + assert not is_successful(validate.validate_config(config)) + + def test_default_with_integer_description_is_invalid(self): + config = deepcopy(constants.DEFAULT_CONFIG) + config["description"] = 0 + assert not is_successful(validate.validate_config(config)) + + +class TestCatalogValidations: + def test_catalog_is_valid(self): + catalog = _load_catalog_file().bind_result(validate.validate_catalog) + assert is_successful(catalog) diff --git a/scripts/integrations-cli/upload b/scripts/integrations-cli/upload new file mode 100644 index 000000000..29d0ff005 --- /dev/null +++ b/scripts/integrations-cli/upload @@ -0,0 +1,43 @@ +#!/usr/bin/bash + +# Demo script for uploading a generated integration to OSD + +bash integ-cli check $1 +FILE=integrations/$1 + +echo "Loading $1 integration" + +# First upload component templates +for f in $FILE/schema/*.mapping; do + if ! grep -q "\"index_mapping\"" "$f"; then + # Get filename without extension + filename=$(basename -- "$f") + filename="${filename%.*}" + curl -XPUT "localhost:9200/_component_template/${filename}_template" -H "Content-Type: application/json" --data-binary "@$f" + echo -e "\n" + fi +done + +# Now index templates +for f in $FILE/schema/*.mapping; do + if grep -q "\"index_mapping\"" "$f"; then + # Get filename without extension + filename=$(basename -- "$f") + filename="${filename%.*}" + curl -XPUT "localhost:9200/_index_template/${filename}" -H "Content-Type: application/json" --data-binary "@$f" + echo -e "\n" + fi +done + +# Add an sso_logs index pattern +# Other categories currently not supported +curl -XPOST localhost:5601/api/saved_objects/index-pattern/sso_logs -H 'osd-xsrf: true' -H 'Content-Type: application/json' -d '{ "attributes": { "title": "sso_logs-*-*", "timeFieldName": "@timestamp" } }' +echo -e "\n" + +# Finally, upload the saved objects +for f in $FILE/assets/display/*.ndjson; do + curl -XPOST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "osd-xsrf: true" --form file=@$f + echo -e "\n" +done + +echo -e "\nLoaded" diff --git a/src/main/resources/schema/observability/catalog.json b/src/main/resources/schema/observability/catalog.json index 66f52e705..32999a2ae 100644 --- a/src/main/resources/schema/observability/catalog.json +++ b/src/main/resources/schema/observability/catalog.json @@ -16,7 +16,11 @@ "version": "1.0", "url": "https://github.com/opensearch-project/observability/tree/2.x/schema/observability/logs/logs", "container": true, - "tags": [] + "tags": [], + "dependencies": [ + "http", + "communication" + ] }, { "component": "http", diff --git a/src/main/resources/schema/system/catalog.schema b/src/main/resources/schema/system/catalog.schema index 2c8a8c7f9..42b555f26 100644 --- a/src/main/resources/schema/system/catalog.schema +++ b/src/main/resources/schema/system/catalog.schema @@ -4,7 +4,7 @@ "definitions": { "Catalog": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "catalog": { "type": "string" @@ -36,7 +36,7 @@ }, "Category": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "category": { "type": "string" @@ -68,7 +68,7 @@ }, "Component": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "component": { "type": "string" @@ -89,6 +89,12 @@ "tags": { "type": "array", "items": {} + }, + "dependencies": { + "type": "array", + "items": { + "type": "string" + } } }, "required": [ diff --git a/src/main/resources/schema/system/integration.schema b/src/main/resources/schema/system/integration.schema index 481b9f516..45723eee9 100644 --- a/src/main/resources/schema/system/integration.schema +++ b/src/main/resources/schema/system/integration.schema @@ -15,9 +15,6 @@ "description": { "type": "string" }, - "catalog": { - "type": "string" - }, "identification": { "type": "string" }, @@ -38,7 +35,6 @@ } }, "required": [ - "catalog", "components", "collection", "description", @@ -147,9 +143,7 @@ ] } }, - "required": [ - "url" - ], + "required": [], "title": "repository" }, "Version": {