From b9e0d98ed133241e5550a782191d25b8a09bf965 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 26 Jun 2023 16:22:57 -0700 Subject: [PATCH 01/24] Stub diff CLI Signed-off-by: Simeon Widdis --- cli/.gitignore | 176 +++++++++++++++++++++++++++++++++++++++++ cli/osints/__init__.py | 0 cli/osints/main.py | 14 ++++ cli/setup.py | 16 ++++ 4 files changed, 206 insertions(+) create mode 100644 cli/.gitignore create mode 100644 cli/osints/__init__.py create mode 100644 cli/osints/main.py create mode 100644 cli/setup.py diff --git a/cli/.gitignore b/cli/.gitignore new file mode 100644 index 0000000..ad4a1f1 --- /dev/null +++ b/cli/.gitignore @@ -0,0 +1,176 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +# End of https://www.toptal.com/developers/gitignore/api/python diff --git a/cli/osints/__init__.py b/cli/osints/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cli/osints/main.py b/cli/osints/main.py new file mode 100644 index 0000000..78a5988 --- /dev/null +++ b/cli/osints/main.py @@ -0,0 +1,14 @@ +import click + +@click.group() +def cli(): + pass + +@click.command() +@click.option('--schema', type=click.Path(exists=True), help='The mapping for the format the data should have') +@click.option('--data', type=click.Path(exists=True), help='The location of data to validate') +def diff(schema, data): + """Diff between a mapping and some data. Experimental.""" + print(schema, data) + +cli.add_command(diff) diff --git a/cli/setup.py b/cli/setup.py new file mode 100644 index 0000000..87b9512 --- /dev/null +++ b/cli/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup, find_packages + +setup( + name='osints', + version='0.1.0', + packages=find_packages(), + include_package_data=True, + install_requires=[ + 'click', + ], + entry_points={ + 'console_scripts': [ + 'osints = osints.main:cli' + ] + } +) From 3c8b433059455d51230acf0a70d586dc1ec9d425 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 26 Jun 2023 16:28:32 -0700 Subject: [PATCH 02/24] Move diff to own file Signed-off-by: Simeon Widdis --- cli/osints/diff.py | 8 ++++++++ cli/osints/main.py | 8 +------- 2 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 cli/osints/diff.py diff --git a/cli/osints/diff.py b/cli/osints/diff.py new file mode 100644 index 0000000..d12a58c --- /dev/null +++ b/cli/osints/diff.py @@ -0,0 +1,8 @@ +import click + +@click.command() +@click.option('--mapping', type=click.Path(exists=True), help='The mapping for the format the data should have') +@click.option('--data', type=click.Path(exists=True), help='The location of data to validate') +def diff(mapping, data): + """Diff between a mapping and some data. Experimental.""" + print(mapping, data) diff --git a/cli/osints/main.py b/cli/osints/main.py index 78a5988..c1c1dee 100644 --- a/cli/osints/main.py +++ b/cli/osints/main.py @@ -1,14 +1,8 @@ import click +from diff import diff @click.group() def cli(): pass -@click.command() -@click.option('--schema', type=click.Path(exists=True), help='The mapping for the format the data should have') -@click.option('--data', type=click.Path(exists=True), help='The location of data to validate') -def diff(schema, data): - """Diff between a mapping and some data. Experimental.""" - print(schema, data) - cli.add_command(diff) From 1151899465f8c75f44068d0084076acaf9edcc7d Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 26 Jun 2023 17:10:01 -0700 Subject: [PATCH 03/24] Add mapping loading to diff Signed-off-by: Simeon Widdis --- cli/osints/diff.py | 43 ++++++++++++++++++++++++++++++++++++++++--- cli/osints/main.py | 4 +++- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/cli/osints/diff.py b/cli/osints/diff.py index d12a58c..bac077b 100644 --- a/cli/osints/diff.py +++ b/cli/osints/diff.py @@ -1,8 +1,45 @@ import click +import json +import os.path +import glob + + +def load_mapping(mapping: str): + with open(mapping, "r") as mapping_file: + data = json.load(mapping_file) + properties = data.get("template", {}).get("mappings", {}).get("properties") + if properties is None: + return {} + composed_of = data.get("composed_of", []) + curr_dir = os.path.dirname(mapping) + for item in composed_of: + item_glob = glob.glob(os.path.join(curr_dir, f"{item}*")) + if len(item_glob) == 0: + click.echo(f"ERROR: mapping file {mapping} references component {item}, which does not exist.", err=True, fg='red') + raise click.Abort() + if properties.get(item) is not None: + click.echo(f"mapping file {mapping} references component {item} and defines conflicting key '{item}'", err=True, fg='red') + raise click.Abort() + # Greedily take any mapping that matches the name for now. + # Later, configuration will need to be implemented. + if len(item_glob) > 1: + click.echo(f"WARNING: Found more than one mapping for component {item}. Assuming {item_glob[0]}.", err=True, fg='yellow') + properties[item] = load_mapping(item_glob[0]) + return properties + @click.command() -@click.option('--mapping', type=click.Path(exists=True), help='The mapping for the format the data should have') -@click.option('--data', type=click.Path(exists=True), help='The location of data to validate') +@click.option( + "--mapping", + type=click.Path(exists=True, readable=True), + help="The mapping for the format the data should have", +) +@click.option( + "--data", + type=click.Path(exists=True, readable=True), + help="The location of data to validate", +) def diff(mapping, data): """Diff between a mapping and some data. Experimental.""" - print(mapping, data) + properties = load_mapping(mapping) + click.echo(json.dumps(properties, indent=4)) diff --git a/cli/osints/main.py b/cli/osints/main.py index c1c1dee..1874619 100644 --- a/cli/osints/main.py +++ b/cli/osints/main.py @@ -1,8 +1,10 @@ import click -from diff import diff +from .diff import diff + @click.group() def cli(): pass + cli.add_command(diff) From 7b267332e25570f87cc9d9162b837fb7c057b5c3 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 26 Jun 2023 17:12:28 -0700 Subject: [PATCH 04/24] Add README Signed-off-by: Simeon Widdis --- cli/README.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 cli/README.md diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 0000000..d718619 --- /dev/null +++ b/cli/README.md @@ -0,0 +1,4 @@ +# OsInts: the OpenSearch Integrations CLI + +Under construction. +Installation: `pip install .` From 242a7cc1546d24a7e2407f4dc70b4a78be7c8635 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 26 Jun 2023 17:15:51 -0700 Subject: [PATCH 05/24] Replace set with update Signed-off-by: Simeon Widdis --- cli/osints/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/osints/diff.py b/cli/osints/diff.py index bac077b..1ddd467 100644 --- a/cli/osints/diff.py +++ b/cli/osints/diff.py @@ -24,7 +24,7 @@ def load_mapping(mapping: str): # Later, configuration will need to be implemented. if len(item_glob) > 1: click.echo(f"WARNING: Found more than one mapping for component {item}. Assuming {item_glob[0]}.", err=True, fg='yellow') - properties[item] = load_mapping(item_glob[0]) + properties.update(load_mapping(item_glob[0])) return properties From 1e03dcf6e42b846af5afca2a3f2627cbd82527bb Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 27 Jun 2023 15:04:48 -0700 Subject: [PATCH 06/24] Add basic type checking system Signed-off-by: Simeon Widdis --- cli/osints/diff.py | 73 ++++++++++++++++++++++++++++++++++++++++++---- cli/setup.py | 1 + 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/cli/osints/diff.py b/cli/osints/diff.py index 1ddd467..2db08dc 100644 --- a/cli/osints/diff.py +++ b/cli/osints/diff.py @@ -1,10 +1,12 @@ +from beartype import beartype import click import json import os.path import glob -def load_mapping(mapping: str): +@beartype +def load_mapping(mapping: str) -> dict[str, dict]: with open(mapping, "r") as mapping_file: data = json.load(mapping_file) properties = data.get("template", {}).get("mappings", {}).get("properties") @@ -15,19 +17,68 @@ def load_mapping(mapping: str): for item in composed_of: item_glob = glob.glob(os.path.join(curr_dir, f"{item}*")) if len(item_glob) == 0: - click.echo(f"ERROR: mapping file {mapping} references component {item}, which does not exist.", err=True, fg='red') + click.echo(f"ERROR: mapping file {mapping} references component {item}, which does not exist.", err=True, color='red') raise click.Abort() if properties.get(item) is not None: - click.echo(f"mapping file {mapping} references component {item} and defines conflicting key '{item}'", err=True, fg='red') + click.echo(f"ERROR: mapping file {mapping} references component {item} and defines conflicting key '{item}'", err=True, color='red') raise click.Abort() # Greedily take any mapping that matches the name for now. # Later, configuration will need to be implemented. if len(item_glob) > 1: - click.echo(f"WARNING: Found more than one mapping for component {item}. Assuming {item_glob[0]}.", err=True, fg='yellow') + click.echo(f"WARNING: found more than one mapping for component {item}. Assuming {item_glob[0]}.", err=True, color='yellow') properties.update(load_mapping(item_glob[0])) return properties +@beartype +def flat_type_check(expect: str, actual: object) -> dict[str, dict]: + match expect: + case "text" | "keyword": + if not isinstance(actual, str): + return { "expected": expect, "actual": actual } + case "long": + if not isinstance(actual, int): + return { "expected": expect, "actual": actual } + case "alias": + # We assume aliases were already unwrapped by the caller and ignore them. + return {} + case "date": + if not isinstance(actual, str) and not isinstance(actual, int): + return { "expected": expect, "actual": actual } + case _: + click.echo(f"WARNING: unknown type '{expect}'", err=True, color='yellow') + return {} + + +@beartype +def do_check(mapping: dict[str, dict], data: dict[str, object]) -> dict[str, dict]: + result = {} + for key, value in mapping.items(): + if key not in data: + continue + elif "properties" in value and isinstance(data[key], dict): + check = do_check(value, data[key]) + if check != {}: + result[key] = check + elif value.get("type") == "alias": + # Unwrap aliases split by '.' + value_path = value["path"].split('.') + curr_data = data + for step in value_path[:-1]: + if step not in curr_data: + curr_data[step] = {} + curr_data = curr_data[step] + curr_data[value_path[-1]] = data[key] + elif "type" in value: + check = flat_type_check(value["type"], data[key]) + if check != {}: + result[key] = check + for key, value in data.items(): + if key not in mapping: + result[key] = { "expected": None, "actual": value } + return result + + @click.command() @click.option( "--mapping", @@ -39,7 +90,17 @@ def load_mapping(mapping: str): type=click.Path(exists=True, readable=True), help="The location of data to validate", ) -def diff(mapping, data): +@click.option( + "--json", + 'output_json', + is_flag=True, + help="Output machine-readable JSON instead of the default diff format." +) +def diff(mapping, data, output_json): """Diff between a mapping and some data. Experimental.""" properties = load_mapping(mapping) - click.echo(json.dumps(properties, indent=4)) + with open(data, "r") as data_file: + data_json = json.load(data_file) + check = do_check(properties, data_json) + if output_json: + click.echo(json.dumps(check, indent=4, sort_keys=True)) diff --git a/cli/setup.py b/cli/setup.py index 87b9512..cca80cf 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -6,6 +6,7 @@ packages=find_packages(), include_package_data=True, install_requires=[ + 'beartype', 'click', ], entry_points={ From f1bb8b17cdaafc75614d0ddc7b80aaef59c1b02d Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 27 Jun 2023 15:26:30 -0700 Subject: [PATCH 07/24] Add diff output Signed-off-by: Simeon Widdis --- cli/osints/diff.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cli/osints/diff.py b/cli/osints/diff.py index 2db08dc..d1b6e60 100644 --- a/cli/osints/diff.py +++ b/cli/osints/diff.py @@ -57,7 +57,7 @@ def do_check(mapping: dict[str, dict], data: dict[str, object]) -> dict[str, dic if key not in data: continue elif "properties" in value and isinstance(data[key], dict): - check = do_check(value, data[key]) + check = do_check(value["properties"], data[key]) if check != {}: result[key] = check elif value.get("type") == "alias": @@ -79,6 +79,18 @@ def do_check(mapping: dict[str, dict], data: dict[str, object]) -> dict[str, dic return result +@beartype +def output_diff(difference: dict[str, object], prefix: str = '') -> None: + for key, value in sorted(difference.items()): + out_key = prefix + key + if "expected" not in value and "actual" not in value: + output_diff(value, f"{prefix}{key}.") + if value.get("actual") is not None: + click.echo(f"- {out_key}: {json.dumps(value.get('actual'))}") + if value.get("expected") is not None: + click.echo(f"+ {out_key}: {json.dumps(value.get('expected'))}") + + @click.command() @click.option( "--mapping", @@ -104,3 +116,5 @@ def diff(mapping, data, output_json): check = do_check(properties, data_json) if output_json: click.echo(json.dumps(check, indent=4, sort_keys=True)) + else: + output_diff(check) From fad5f61fc23ae2a95d58b82f1df6da9eaa699381 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 27 Jun 2023 15:27:16 -0700 Subject: [PATCH 08/24] Add integer type checking Signed-off-by: Simeon Widdis --- cli/osints/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/osints/diff.py b/cli/osints/diff.py index d1b6e60..4d5717c 100644 --- a/cli/osints/diff.py +++ b/cli/osints/diff.py @@ -36,7 +36,7 @@ def flat_type_check(expect: str, actual: object) -> dict[str, dict]: case "text" | "keyword": if not isinstance(actual, str): return { "expected": expect, "actual": actual } - case "long": + case "long" | "integer": if not isinstance(actual, int): return { "expected": expect, "actual": actual } case "alias": From 892db6572e615a62c01c3ece9f71f1518097f3d1 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 27 Jun 2023 15:42:34 -0700 Subject: [PATCH 09/24] Add ability to show empty mapping fields Signed-off-by: Simeon Widdis --- cli/osints/diff.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/cli/osints/diff.py b/cli/osints/diff.py index 4d5717c..747b77b 100644 --- a/cli/osints/diff.py +++ b/cli/osints/diff.py @@ -51,10 +51,22 @@ def flat_type_check(expect: str, actual: object) -> dict[str, dict]: @beartype -def do_check(mapping: dict[str, dict], data: dict[str, object]) -> dict[str, dict]: +def get_type(mapping: dict) -> str | dict: + if mapping.get("properties"): + return { + key: get_type(value) + for key, value in mapping.get("properties").items() + } + return mapping.get("type", {}) + + +@beartype +def do_check(mapping: dict[str, dict], data: dict[str, object], no_optional: bool = False) -> dict[str, dict]: result = {} for key, value in mapping.items(): if key not in data: + if no_optional: + result[key] = { "expected": get_type(value), "actual": None } continue elif "properties" in value and isinstance(data[key], dict): check = do_check(value["properties"], data[key]) @@ -106,14 +118,20 @@ def output_diff(difference: dict[str, object], prefix: str = '') -> None: "--json", 'output_json', is_flag=True, - help="Output machine-readable JSON instead of the default diff format." + help="Output machine-readable JSON instead of the default diff format" +) +@click.option( + "--no-optional", + "no_optional", + is_flag=True, + help="Treat optional as required: Output fields that are expected in the mappings but missing in the data" ) -def diff(mapping, data, output_json): +def diff(mapping, data, output_json, no_optional): """Diff between a mapping and some data. Experimental.""" properties = load_mapping(mapping) with open(data, "r") as data_file: data_json = json.load(data_file) - check = do_check(properties, data_json) + check = do_check(properties, data_json, no_optional) if output_json: click.echo(json.dumps(check, indent=4, sort_keys=True)) else: From aab73eaccc3a4e8fa4bc3e581fe177deb2e76e0a Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 27 Jun 2023 15:44:07 -0700 Subject: [PATCH 10/24] Add better handling for missing type field Signed-off-by: Simeon Widdis --- cli/osints/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/osints/diff.py b/cli/osints/diff.py index 747b77b..31b6703 100644 --- a/cli/osints/diff.py +++ b/cli/osints/diff.py @@ -57,7 +57,7 @@ def get_type(mapping: dict) -> str | dict: key: get_type(value) for key, value in mapping.get("properties").items() } - return mapping.get("type", {}) + return mapping.get("type", "unknown") @beartype From 96ddd57ca27558e4fa7307d35b560ffe39eb521f Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 27 Jun 2023 15:47:44 -0700 Subject: [PATCH 11/24] Ignore aliases in no-optional mode Signed-off-by: Simeon Widdis --- cli/osints/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/osints/diff.py b/cli/osints/diff.py index 31b6703..3632ec3 100644 --- a/cli/osints/diff.py +++ b/cli/osints/diff.py @@ -65,7 +65,7 @@ def do_check(mapping: dict[str, dict], data: dict[str, object], no_optional: boo result = {} for key, value in mapping.items(): if key not in data: - if no_optional: + if no_optional and value.get("type") != "alias": result[key] = { "expected": get_type(value), "actual": None } continue elif "properties" in value and isinstance(data[key], dict): From 702ecd1fa7431bd9413cfe8deb4731fb272a6c1c Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 27 Jun 2023 16:00:33 -0700 Subject: [PATCH 12/24] Fix error and warning colors Signed-off-by: Simeon Widdis --- cli/.DS_Store | Bin 0 -> 6148 bytes cli/osints/diff.py | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 cli/.DS_Store diff --git a/cli/.DS_Store b/cli/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..d2c0e5d579c498aed50f5f6d54682585069ba7c5 GIT binary patch literal 6148 zcmeHKPfNov6i>G4QiiaDg5CmN2Tt84;-%F21+3^nWwvf;u{LAv++hrQ*DvH3@$>jz zl8R#rorhQG)F?ww!}>|4q*_G$mZScTww4To`*X3gdsFO@4Rt5s1IHF53VtBIcl**Nur z(It+~l#0WmAB5-8VA89vAFCt_qGT}A2~mFlAy*es(pM8tjgx+=bA96wj&OSQt?9JY zX}22kpxK@^)3)Rt(?v{Yh%82A|jcs@u_MAu?&P#+!8;Pnyx6+{&1<68nz zT68Vu2EhZuO)8*C<@SleO*+`6jdLyL22DESdS>{JotfJg3fHrPUFvYgU4zsT1H`~Q z17$O;WBot*{{26nL?dE=82DEV@JiS3daxv0TNf6GwN`@OfTCbrZtx=oI=U1?ESBO; bP%U7W*a37c<_5t6LKgu|12x3JuQKosnl4Q) literal 0 HcmV?d00001 diff --git a/cli/osints/diff.py b/cli/osints/diff.py index 3632ec3..18e5485 100644 --- a/cli/osints/diff.py +++ b/cli/osints/diff.py @@ -17,15 +17,15 @@ def load_mapping(mapping: str) -> dict[str, dict]: for item in composed_of: item_glob = glob.glob(os.path.join(curr_dir, f"{item}*")) if len(item_glob) == 0: - click.echo(f"ERROR: mapping file {mapping} references component {item}, which does not exist.", err=True, color='red') + click.secho(f"ERROR: mapping file {mapping} references component {item}, which does not exist.", err=True, fg='red') raise click.Abort() if properties.get(item) is not None: - click.echo(f"ERROR: mapping file {mapping} references component {item} and defines conflicting key '{item}'", err=True, color='red') + click.secho(f"ERROR: mapping file {mapping} references component {item} and defines conflicting key '{item}'", err=True, fg='red') raise click.Abort() # Greedily take any mapping that matches the name for now. # Later, configuration will need to be implemented. if len(item_glob) > 1: - click.echo(f"WARNING: found more than one mapping for component {item}. Assuming {item_glob[0]}.", err=True, color='yellow') + click.secho(f"WARNING: found more than one mapping for component {item}. Assuming {item_glob[0]}.", err=True, fg='yellow') properties.update(load_mapping(item_glob[0])) return properties @@ -46,7 +46,7 @@ def flat_type_check(expect: str, actual: object) -> dict[str, dict]: if not isinstance(actual, str) and not isinstance(actual, int): return { "expected": expect, "actual": actual } case _: - click.echo(f"WARNING: unknown type '{expect}'", err=True, color='yellow') + click.secho(f"WARNING: unknown type '{expect}'", err=True, fg='yellow') return {} From 2b24d58aba9a2461d460e1b2c5123a7ce7fdec85 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 27 Jun 2023 16:00:55 -0700 Subject: [PATCH 13/24] Remove accidental .DS_Store Signed-off-by: Simeon Widdis --- cli/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 cli/.DS_Store diff --git a/cli/.DS_Store b/cli/.DS_Store deleted file mode 100644 index d2c0e5d579c498aed50f5f6d54682585069ba7c5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKPfNov6i>G4QiiaDg5CmN2Tt84;-%F21+3^nWwvf;u{LAv++hrQ*DvH3@$>jz zl8R#rorhQG)F?ww!}>|4q*_G$mZScTww4To`*X3gdsFO@4Rt5s1IHF53VtBIcl**Nur z(It+~l#0WmAB5-8VA89vAFCt_qGT}A2~mFlAy*es(pM8tjgx+=bA96wj&OSQt?9JY zX}22kpxK@^)3)Rt(?v{Yh%82A|jcs@u_MAu?&P#+!8;Pnyx6+{&1<68nz zT68Vu2EhZuO)8*C<@SleO*+`6jdLyL22DESdS>{JotfJg3fHrPUFvYgU4zsT1H`~Q z17$O;WBot*{{26nL?dE=82DEV@JiS3daxv0TNf6GwN`@OfTCbrZtx=oI=U1?ESBO; bP%U7W*a37c<_5t6LKgu|12x3JuQKosnl4Q) From 7b099effcd38cbce9a3af18a92f85de3be8861ce Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 27 Jun 2023 16:07:23 -0700 Subject: [PATCH 14/24] Rename no-optional Signed-off-by: Simeon Widdis --- cli/osints/diff.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cli/osints/diff.py b/cli/osints/diff.py index 18e5485..56215af 100644 --- a/cli/osints/diff.py +++ b/cli/osints/diff.py @@ -61,11 +61,11 @@ def get_type(mapping: dict) -> str | dict: @beartype -def do_check(mapping: dict[str, dict], data: dict[str, object], no_optional: bool = False) -> dict[str, dict]: +def do_check(mapping: dict[str, dict], data: dict[str, object], show_missing: bool = False) -> dict[str, dict]: result = {} for key, value in mapping.items(): if key not in data: - if no_optional and value.get("type") != "alias": + if show_missing and value.get("type") != "alias": result[key] = { "expected": get_type(value), "actual": None } continue elif "properties" in value and isinstance(data[key], dict): @@ -121,17 +121,17 @@ def output_diff(difference: dict[str, object], prefix: str = '') -> None: help="Output machine-readable JSON instead of the default diff format" ) @click.option( - "--no-optional", - "no_optional", + "--show-missing", + "show_missing", is_flag=True, - help="Treat optional as required: Output fields that are expected in the mappings but missing in the data" + help="Output fields that are expected in the mappings but missing in the data" ) -def diff(mapping, data, output_json, no_optional): +def diff(mapping, data, output_json, show_missing): """Diff between a mapping and some data. Experimental.""" properties = load_mapping(mapping) with open(data, "r") as data_file: data_json = json.load(data_file) - check = do_check(properties, data_json, no_optional) + check = do_check(properties, data_json, show_missing) if output_json: click.echo(json.dumps(check, indent=4, sort_keys=True)) else: From 67d5889e48ae152a90664d3242bd7af43966c3c5 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 27 Jun 2023 16:11:38 -0700 Subject: [PATCH 15/24] Flatten JSON output Signed-off-by: Simeon Widdis --- cli/osints/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/osints/diff.py b/cli/osints/diff.py index 56215af..754b7a7 100644 --- a/cli/osints/diff.py +++ b/cli/osints/diff.py @@ -133,6 +133,6 @@ def diff(mapping, data, output_json, show_missing): data_json = json.load(data_file) check = do_check(properties, data_json, show_missing) if output_json: - click.echo(json.dumps(check, indent=4, sort_keys=True)) + click.echo(json.dumps(check, sort_keys=True)) else: output_diff(check) From ca72763f0eb30a049cdb95f9fc14e48d48207953 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 27 Jun 2023 16:21:01 -0700 Subject: [PATCH 16/24] Fill out remainder of README Signed-off-by: Simeon Widdis --- cli/README.md | 32 ++++++++++++++++++++++++++++++-- cli/osints/diff.py | 2 +- cli/osints/main.py | 1 + cli/setup.py | 1 + 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/cli/README.md b/cli/README.md index d718619..6976629 100644 --- a/cli/README.md +++ b/cli/README.md @@ -1,4 +1,32 @@ # OsInts: the OpenSearch Integrations CLI -Under construction. -Installation: `pip install .` +The OsInts CLI is a utility CLI for developing integrations with OpenSearch Integrations. +It provides a few convenience methods: + +- `diff`: Type check your integration given a sample data record and the appropriate SS4O schema. + +## Installation + +Use the package manager [pip](https://pip.pypa.io/en/stable/) to install the CLI. + +```bash +$ cd cli +$ pip install . +... +Successfully installed osints-0.1.0 +``` + +## Usage + +```bash +$ osints --help +Usage: osints [OPTIONS] COMMAND [ARGS]... + + Various tools for working with OpenSearch Integrations. + +Options: + --help Show this message and exit. + +Commands: + diff Diff between a mapping and some data. +``` diff --git a/cli/osints/diff.py b/cli/osints/diff.py index 754b7a7..65806f2 100644 --- a/cli/osints/diff.py +++ b/cli/osints/diff.py @@ -127,7 +127,7 @@ def output_diff(difference: dict[str, object], prefix: str = '') -> None: help="Output fields that are expected in the mappings but missing in the data" ) def diff(mapping, data, output_json, show_missing): - """Diff between a mapping and some data. Experimental.""" + """Type check your integration given a sample data record and the appropriate SS4O schema.""" properties = load_mapping(mapping) with open(data, "r") as data_file: data_json = json.load(data_file) diff --git a/cli/osints/main.py b/cli/osints/main.py index 1874619..46defbc 100644 --- a/cli/osints/main.py +++ b/cli/osints/main.py @@ -4,6 +4,7 @@ @click.group() def cli(): + """Various tools for working with OpenSearch Integrations.""" pass diff --git a/cli/setup.py b/cli/setup.py index cca80cf..5337954 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -9,6 +9,7 @@ 'beartype', 'click', ], + python_requires='>3.10.0', entry_points={ 'console_scripts': [ 'osints = osints.main:cli' From 6b29c7dd6817f39eb8aff53735b2e2ce987d2a78 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 27 Jun 2023 16:42:02 -0700 Subject: [PATCH 17/24] Apply black formatting Signed-off-by: Simeon Widdis --- cli/osints/diff.py | 49 +++++++++++++++++++++++++++++----------------- cli/setup.py | 16 ++++++--------- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/cli/osints/diff.py b/cli/osints/diff.py index 65806f2..2e83905 100644 --- a/cli/osints/diff.py +++ b/cli/osints/diff.py @@ -17,15 +17,27 @@ def load_mapping(mapping: str) -> dict[str, dict]: for item in composed_of: item_glob = glob.glob(os.path.join(curr_dir, f"{item}*")) if len(item_glob) == 0: - click.secho(f"ERROR: mapping file {mapping} references component {item}, which does not exist.", err=True, fg='red') + click.secho( + f"ERROR: mapping file {mapping} references component {item}, which does not exist.", + err=True, + fg="red", + ) raise click.Abort() if properties.get(item) is not None: - click.secho(f"ERROR: mapping file {mapping} references component {item} and defines conflicting key '{item}'", err=True, fg='red') + click.secho( + f"ERROR: mapping file {mapping} references component {item} and defines conflicting key '{item}'", + err=True, + fg="red", + ) raise click.Abort() # Greedily take any mapping that matches the name for now. # Later, configuration will need to be implemented. if len(item_glob) > 1: - click.secho(f"WARNING: found more than one mapping for component {item}. Assuming {item_glob[0]}.", err=True, fg='yellow') + click.secho( + f"WARNING: found more than one mapping for component {item}. Assuming {item_glob[0]}.", + err=True, + fg="yellow", + ) properties.update(load_mapping(item_glob[0])) return properties @@ -35,18 +47,18 @@ def flat_type_check(expect: str, actual: object) -> dict[str, dict]: match expect: case "text" | "keyword": if not isinstance(actual, str): - return { "expected": expect, "actual": actual } + return {"expected": expect, "actual": actual} case "long" | "integer": if not isinstance(actual, int): - return { "expected": expect, "actual": actual } + return {"expected": expect, "actual": actual} case "alias": - # We assume aliases were already unwrapped by the caller and ignore them. + # We assume aliases were already unwrapped by the caller and ignore them. return {} case "date": if not isinstance(actual, str) and not isinstance(actual, int): - return { "expected": expect, "actual": actual } + return {"expected": expect, "actual": actual} case _: - click.secho(f"WARNING: unknown type '{expect}'", err=True, fg='yellow') + click.secho(f"WARNING: unknown type '{expect}'", err=True, fg="yellow") return {} @@ -54,19 +66,20 @@ def flat_type_check(expect: str, actual: object) -> dict[str, dict]: def get_type(mapping: dict) -> str | dict: if mapping.get("properties"): return { - key: get_type(value) - for key, value in mapping.get("properties").items() + key: get_type(value) for key, value in mapping.get("properties").items() } return mapping.get("type", "unknown") @beartype -def do_check(mapping: dict[str, dict], data: dict[str, object], show_missing: bool = False) -> dict[str, dict]: +def do_check( + mapping: dict[str, dict], data: dict[str, object], show_missing: bool = False +) -> dict[str, dict]: result = {} for key, value in mapping.items(): if key not in data: if show_missing and value.get("type") != "alias": - result[key] = { "expected": get_type(value), "actual": None } + result[key] = {"expected": get_type(value), "actual": None} continue elif "properties" in value and isinstance(data[key], dict): check = do_check(value["properties"], data[key]) @@ -74,7 +87,7 @@ def do_check(mapping: dict[str, dict], data: dict[str, object], show_missing: bo result[key] = check elif value.get("type") == "alias": # Unwrap aliases split by '.' - value_path = value["path"].split('.') + value_path = value["path"].split(".") curr_data = data for step in value_path[:-1]: if step not in curr_data: @@ -87,12 +100,12 @@ def do_check(mapping: dict[str, dict], data: dict[str, object], show_missing: bo result[key] = check for key, value in data.items(): if key not in mapping: - result[key] = { "expected": None, "actual": value } + result[key] = {"expected": None, "actual": value} return result @beartype -def output_diff(difference: dict[str, object], prefix: str = '') -> None: +def output_diff(difference: dict[str, object], prefix: str = "") -> None: for key, value in sorted(difference.items()): out_key = prefix + key if "expected" not in value and "actual" not in value: @@ -116,15 +129,15 @@ def output_diff(difference: dict[str, object], prefix: str = '') -> None: ) @click.option( "--json", - 'output_json', + "output_json", is_flag=True, - help="Output machine-readable JSON instead of the default diff format" + help="Output machine-readable JSON instead of the default diff format", ) @click.option( "--show-missing", "show_missing", is_flag=True, - help="Output fields that are expected in the mappings but missing in the data" + help="Output fields that are expected in the mappings but missing in the data", ) def diff(mapping, data, output_json, show_missing): """Type check your integration given a sample data record and the appropriate SS4O schema.""" diff --git a/cli/setup.py b/cli/setup.py index 5337954..4ce1b6f 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -1,18 +1,14 @@ from setuptools import setup, find_packages setup( - name='osints', - version='0.1.0', + name="osints", + version="0.1.0", packages=find_packages(), include_package_data=True, install_requires=[ - 'beartype', - 'click', + "beartype", + "click", ], - python_requires='>3.10.0', - entry_points={ - 'console_scripts': [ - 'osints = osints.main:cli' - ] - } + python_requires=">3.10.0", + entry_points={"console_scripts": ["osints = osints.main:cli"]}, ) From e146a78ce5a9466539f727c4944dd9ec172ace37 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 27 Jun 2023 16:50:38 -0700 Subject: [PATCH 18/24] Propagate show_missing through do_check Signed-off-by: Simeon Widdis --- cli/osints/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/osints/diff.py b/cli/osints/diff.py index 2e83905..cccca71 100644 --- a/cli/osints/diff.py +++ b/cli/osints/diff.py @@ -82,7 +82,7 @@ def do_check( result[key] = {"expected": get_type(value), "actual": None} continue elif "properties" in value and isinstance(data[key], dict): - check = do_check(value["properties"], data[key]) + check = do_check(value["properties"], data[key], show_missing) if check != {}: result[key] = check elif value.get("type") == "alias": From 181eb84147e6349eba26dd021790ef4320f13485 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 28 Jun 2023 11:50:17 -0700 Subject: [PATCH 19/24] Add functionality to unwrap data lists Signed-off-by: Simeon Widdis --- cli/osints/diff.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cli/osints/diff.py b/cli/osints/diff.py index cccca71..b4e295f 100644 --- a/cli/osints/diff.py +++ b/cli/osints/diff.py @@ -144,6 +144,9 @@ def diff(mapping, data, output_json, show_missing): properties = load_mapping(mapping) with open(data, "r") as data_file: data_json = json.load(data_file) + if isinstance(data_json, list): + # Unwrap list of data, assume first record is representative + data_json = data_json[0] check = do_check(properties, data_json, show_missing) if output_json: click.echo(json.dumps(check, sort_keys=True)) From b7b56ffa79b2e29589acf05229c374df7017a23e Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 28 Jun 2023 13:54:30 -0700 Subject: [PATCH 20/24] Rename files for more standard python package structure Signed-off-by: Simeon Widdis --- cli/.DS_Store | Bin 0 -> 8196 bytes cli/setup.py | 4 ++-- cli/{osints => src}/__init__.py | 0 cli/src/diff/__init__.py | 1 + cli/{osints => src/diff}/diff.py | 3 +++ cli/{osints => src}/main.py | 5 ++++- 6 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 cli/.DS_Store rename cli/{osints => src}/__init__.py (100%) create mode 100644 cli/src/diff/__init__.py rename cli/{osints => src/diff}/diff.py (99%) rename cli/{osints => src}/main.py (81%) diff --git a/cli/.DS_Store b/cli/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..a2787975394e1e364ae719a9c0c22a4ebf95fe58 GIT binary patch literal 8196 zcmeHMU2GIp6h3EK=$$D))6$mWWa&b|U}<4(EC0y$2M{T?_P^z4ncW%b$aJRc%{;+J;2Y0I8{!QA)OX9I69~aQUIbn z1qcfDDGvyA;vr3jbXw3rmDUv51BR*?rWh#HX+F-yi6%ojEvQfj6zYKClQGOtFnBuY z#RGA`q@Yn7c_8w@To3T@VPH|-?ty=6{+_noG|gR3PuP~{Clc3CDk)vGxQvyta(1+D z%q{f!x}Wpf_3UBJ^}DW}F81~Gpk)?PYE_5p_`2no*+9WEOcFUVWI2XgXm@jt;RU*V zVuHy`PN}OVCtJ5|Z*GilZE2lqj8ATFPBg|_TeeJ1$*j75^UmJXNqgLJ&kI{HJP%;$ zEZb)0GtBn#9m24ZnxbLXi-yTcxi&bYj0{^D*Xgs)nkjX;N0uERwkK=m$)cf*Yv;xt zFQrywbUTx?b>D2WZ7%E$`{rq%_eMQ8>vsf#DOJsRX0J(dR^IF+X;Q7^D|Ju!mg{(f zX4WHGpG=6?moF`;j;+0SW8#4wdpaNL>bbgf*>Y8>RH}yZmS?4HbD&@u{@94_WlYDg zoD-wuI>)yjv&^hqfunN2qmP?c)tZ&HtM1XX+0w;ThI47_RL=6xsP&AAXVhtHWc6Kk z#Gscv+QQ$y)?z+75wiR-bGTyAn86u9}4~er2tKwa^N?pa(|4fC9|G3vdBmgG=xpd;p)p zS8xTshHLOG`~W|}FYr4q!ZKtiVWG$1WVeK^(>rJccK5 z3?IjncnZ(rBu?Rzcn+VzXYo0F9$&@R@eRC)Z{lVA7(c;J@k_jpH}D($8Gpq;B#=;2 zq;hGgv`ktft(7)P35h=W@gfzYW>oOSR%)l-oa2C^Wpou`USggJ+X@k`GUQ#Ar@053OK=T+8k5V5(+0d&~0fT7DHz1HbqfODX_%c zuHD1}k9EPfwbM#GOxWKC>_vD7-iMFi3j*wS@ICwpzaijaTtWa{jd23$I$Td6O<*&& zU@Pv%J=lir1lcZp1bgsN?8U=)6jOK{bpou7+^;L~^kU&NR2 zWqc)o?_2n`fZk0&K3s&Np!nf@fXw%09ouzI&|DG7x2r|mop_)`2-WiZfA7NI|LW3mv9B*N#&?PL&rfH!WzOLJb$;<3T!3^yVLi)Q^XA$&gM9 cN;6dc@ecui{f>?9|KbnU=>89|csDhF0-h2fMgRZ+ literal 0 HcmV?d00001 diff --git a/cli/setup.py b/cli/setup.py index 4ce1b6f..504a8ea 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -2,7 +2,7 @@ setup( name="osints", - version="0.1.0", + version="0.1.1", packages=find_packages(), include_package_data=True, install_requires=[ @@ -10,5 +10,5 @@ "click", ], python_requires=">3.10.0", - entry_points={"console_scripts": ["osints = osints.main:cli"]}, + entry_points={"console_scripts": ["osints = src.main:cli"]}, ) diff --git a/cli/osints/__init__.py b/cli/src/__init__.py similarity index 100% rename from cli/osints/__init__.py rename to cli/src/__init__.py diff --git a/cli/src/diff/__init__.py b/cli/src/diff/__init__.py new file mode 100644 index 0000000..9c2ffef --- /dev/null +++ b/cli/src/diff/__init__.py @@ -0,0 +1 @@ +from .diff import diff diff --git a/cli/osints/diff.py b/cli/src/diff/diff.py similarity index 99% rename from cli/osints/diff.py rename to cli/src/diff/diff.py index b4e295f..3157627 100644 --- a/cli/osints/diff.py +++ b/cli/src/diff/diff.py @@ -152,3 +152,6 @@ def diff(mapping, data, output_json, show_missing): click.echo(json.dumps(check, sort_keys=True)) else: output_diff(check) + +if __name__ == "__main__": + diff() diff --git a/cli/osints/main.py b/cli/src/main.py similarity index 81% rename from cli/osints/main.py rename to cli/src/main.py index 46defbc..ebadf1a 100644 --- a/cli/osints/main.py +++ b/cli/src/main.py @@ -1,7 +1,6 @@ import click from .diff import diff - @click.group() def cli(): """Various tools for working with OpenSearch Integrations.""" @@ -9,3 +8,7 @@ def cli(): cli.add_command(diff) + + +if __name__ == "__main__": + cli() From 4d75e7e603c9daa2087dfc82019b4bbbf2a5234b Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 28 Jun 2023 14:06:20 -0700 Subject: [PATCH 21/24] Add more detail to documentation Signed-off-by: Simeon Widdis --- cli/README.md | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/cli/README.md b/cli/README.md index 6976629..1392f0e 100644 --- a/cli/README.md +++ b/cli/README.md @@ -14,19 +14,41 @@ $ cd cli $ pip install . ... Successfully installed osints-0.1.0 +$ osints ``` -## Usage +If you want the installation to be editable (for development), you can either specify the editable flag: + +```bash +$ pip install . --editable +$ osints +``` + +Or you can skip the install entirely and run it as a module: ```bash -$ osints --help -Usage: osints [OPTIONS] COMMAND [ARGS]... +$ python3 -m src.main +``` + +## Usage - Various tools for working with OpenSearch Integrations. +See `osints --help` for a summary of all commands. -Options: - --help Show this message and exit. +### Usage: `diff` -Commands: - diff Diff between a mapping and some data. +Here's an example usage of `diff` on the [current (buggy) version of the Nginx integration](https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx): +```bash +$ osints diff --mapping schemas/logs-1.0.0.mapping.json --data data/sample.json +- event.category: ["web"] ++ event.category: "keyword" +- event.type: ["access"] ++ event.type: "keyword" +- http.response.status_code: "200" ++ http.response.status_code: "integer" +- span_id: "abcdef1010" +- trace_id: "102981ABCD2901" ``` + +From this, we can gather: +- `event.category`, `event.type`, and `http.response.status_code` are all the wrong type. The first two should be a `keyword` instead of a list of strings, while the latter should be an integer `200` instead of a string `"200"`. +- `span_id` and `trace_id` are present in the data but not accounted for in the schema. This indicates that they are either redundant or incorrectly named. In this case, it turns out to be the latter, there are appropriate `spanId` and `traceId` fields. From bac5b6ba3e0a6ff009cca4c2d1f15bef324eacdd Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 28 Jun 2023 14:08:51 -0700 Subject: [PATCH 22/24] Correct doc link to permalink Signed-off-by: Simeon Widdis --- cli/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/README.md b/cli/README.md index 1392f0e..6b50306 100644 --- a/cli/README.md +++ b/cli/README.md @@ -36,7 +36,7 @@ See `osints --help` for a summary of all commands. ### Usage: `diff` -Here's an example usage of `diff` on the [current (buggy) version of the Nginx integration](https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx): +Here's an example usage of `diff` on the [current (buggy) version of the Nginx integration](https://github.com/opensearch-project/dashboards-observability/tree/6d5bd478704dc7342b1471767ced7036bb23f335/server/adaptors/integrations/__data__/repository/nginx): ```bash $ osints diff --mapping schemas/logs-1.0.0.mapping.json --data data/sample.json - event.category: ["web"] From 39e1ba11cfb343caa8696b591ed0118c8bb64209 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 28 Jun 2023 16:56:21 -0700 Subject: [PATCH 23/24] Remove .DS_Store Signed-off-by: Simeon Widdis --- cli/.DS_Store | Bin 8196 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 cli/.DS_Store diff --git a/cli/.DS_Store b/cli/.DS_Store deleted file mode 100644 index a2787975394e1e364ae719a9c0c22a4ebf95fe58..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHMU2GIp6h3EK=$$D))6$mWWa&b|U}<4(EC0y$2M{T?_P^z4ncW%b$aJRc%{;+J;2Y0I8{!QA)OX9I69~aQUIbn z1qcfDDGvyA;vr3jbXw3rmDUv51BR*?rWh#HX+F-yi6%ojEvQfj6zYKClQGOtFnBuY z#RGA`q@Yn7c_8w@To3T@VPH|-?ty=6{+_noG|gR3PuP~{Clc3CDk)vGxQvyta(1+D z%q{f!x}Wpf_3UBJ^}DW}F81~Gpk)?PYE_5p_`2no*+9WEOcFUVWI2XgXm@jt;RU*V zVuHy`PN}OVCtJ5|Z*GilZE2lqj8ATFPBg|_TeeJ1$*j75^UmJXNqgLJ&kI{HJP%;$ zEZb)0GtBn#9m24ZnxbLXi-yTcxi&bYj0{^D*Xgs)nkjX;N0uERwkK=m$)cf*Yv;xt zFQrywbUTx?b>D2WZ7%E$`{rq%_eMQ8>vsf#DOJsRX0J(dR^IF+X;Q7^D|Ju!mg{(f zX4WHGpG=6?moF`;j;+0SW8#4wdpaNL>bbgf*>Y8>RH}yZmS?4HbD&@u{@94_WlYDg zoD-wuI>)yjv&^hqfunN2qmP?c)tZ&HtM1XX+0w;ThI47_RL=6xsP&AAXVhtHWc6Kk z#Gscv+QQ$y)?z+75wiR-bGTyAn86u9}4~er2tKwa^N?pa(|4fC9|G3vdBmgG=xpd;p)p zS8xTshHLOG`~W|}FYr4q!ZKtiVWG$1WVeK^(>rJccK5 z3?IjncnZ(rBu?Rzcn+VzXYo0F9$&@R@eRC)Z{lVA7(c;J@k_jpH}D($8Gpq;B#=;2 zq;hGgv`ktft(7)P35h=W@gfzYW>oOSR%)l-oa2C^Wpou`USggJ+X@k`GUQ#Ar@053OK=T+8k5V5(+0d&~0fT7DHz1HbqfODX_%c zuHD1}k9EPfwbM#GOxWKC>_vD7-iMFi3j*wS@ICwpzaijaTtWa{jd23$I$Td6O<*&& zU@Pv%J=lir1lcZp1bgsN?8U=)6jOK{bpou7+^;L~^kU&NR2 zWqc)o?_2n`fZk0&K3s&Np!nf@fXw%09ouzI&|DG7x2r|mop_)`2-WiZfA7NI|LW3mv9B*N#&?PL&rfH!WzOLJb$;<3T!3^yVLi)Q^XA$&gM9 cN;6dc@ecui{f>?9|KbnU=>89|csDhF0-h2fMgRZ+ From 4f88a1b4d9cc65386d15fca80d6c9eb70e740548 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 6 Jul 2023 09:54:54 -0700 Subject: [PATCH 24/24] Add colored diff output to diff Signed-off-by: Simeon Widdis --- cli/src/diff/diff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/diff/diff.py b/cli/src/diff/diff.py index 3157627..65ef24e 100644 --- a/cli/src/diff/diff.py +++ b/cli/src/diff/diff.py @@ -111,9 +111,9 @@ def output_diff(difference: dict[str, object], prefix: str = "") -> None: if "expected" not in value and "actual" not in value: output_diff(value, f"{prefix}{key}.") if value.get("actual") is not None: - click.echo(f"- {out_key}: {json.dumps(value.get('actual'))}") + click.secho(f"- {out_key}: {json.dumps(value.get('actual'))}", fg="red") if value.get("expected") is not None: - click.echo(f"+ {out_key}: {json.dumps(value.get('expected'))}") + click.secho(f"+ {out_key}: {json.dumps(value.get('expected'))}", fg="green") @click.command()