diff --git a/HISTORY.md b/HISTORY.md index 8ee7c88..2c32bf1 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -2,6 +2,11 @@ # History +## Unreleased +* Permit invocation without configuration file for ad hoc operations. + In this mode, the Grafana URL can optionally be defined using the + environment variable `GRAFANA_URL`. + ## 0.2.0 (2022-02-05) * Migrated from grafana_api to grafana_client diff --git a/README.md b/README.md index 3090e52..1cbc051 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,63 @@ Currently, there is no up-to-date version on PyPI, so we recommend to install directly from the repository. -## Configuration +## Ad Hoc Usage -The configuration is stored in a YAML file. In order to connect to Grafana, you -will need an authentication token for Grafana. +You can use `grafana-import` in ad hoc mode without a configuration file. + +### Getting started + +In order to do some orientation flights, start a Grafana instance using Podman +or Docker. +```shell +docker run --rm -it --name=grafana --publish=3000:3000 \ + --env='GF_SECURITY_ADMIN_PASSWORD=admin' grafana/grafana:latest +``` + +Define Grafana endpoint. +```shell +export GRAFANA_URL=http://admin:admin@localhost:3000 +``` + +### Import +Import a dashboard from a JSON file into the `Applications` folder in Grafana. +```shell +grafana-import import -i grafana-dashboard.json -f Applications -o +``` +Please note the import action preserves the version history. + +### Export +Export the dashboard titled `my-first-dashboard` to the default export directory. +```bash +grafana-import export -d "my-first-dashboard" --pretty +``` + +### Delete +Delete the dashboard titled `my-first-dashboard` from folder `Applications`. +```bash +grafana-import remove -f Applications -d "my-first-dashboard" +``` + + +## Usage with Configuration File + +You can also use `grafana-import` with a configuration file. In this way, you +can manage and use different Grafana connection profiles, and also use presets +for application-wide configuration settings. + +The configuration is stored in a YAML file. In order to use it optimally, +build a directory structure like this: +``` +grafana-import/ +- conf/grafana-import.yml + Path to your main configuration file. +- exports/ + Path where exported dashboards will be stored. +- imports/ + Path where dashboards are imported from. +``` + +Then, enter into your directory, and type in your commands. The configuration file uses two sections, `general`, and `grafana`. @@ -71,19 +124,17 @@ server URL. -## Usage +## Authentication -build a directory structure: -- grafana-import/ - - conf/grafana-import.yml - where your main configuration file is - - exports/ - where your exported dashboards will be stored. - - imports/ - where your dashboards to import are stored. +In order to connect to Grafana, you can use either vanilla credentials +(username/password), or an authentication token. Because `grafana-import` +uses `grafana-client`, the same features for defining authentication +settings can be used. See also [grafana-client authentication variants]. -Then, enter into your directory, and type in your commands. -Please note the import action preserves the version history. +[grafana-client authentication variants]: https://github.com/panodata/grafana-client/#authentication + + +## Help `grafana-import --help` ```shell @@ -117,6 +168,8 @@ optional arguments: path to config files. -d DASHBOARD_NAME, --dashboard_name DASHBOARD_NAME name of dashboard to export. + -u GRAFANA_URL, --grafana_url GRAFANA_URL + Grafana URL to connect to. -g GRAFANA_LABEL, --grafana_label GRAFANA_LABEL label in the config file that represents the grafana to connect to. @@ -134,27 +187,6 @@ optional arguments: ``` -## Examples - -### Import -Import a dashboard from a JSON file to the folder `Applications` in Grafana. -```shell -grafana-import -i my-first-dashboard_202104011548.json -f Applications -o -``` - -### Export -Export the dashboard `my-first-dashboard` to the default export directory. -```bash -grafana-import -d "my-first-dashboard" -p export -``` - -### Delete -Delete the dashboard `my-first-dashboard` from folder `Applications`. -```bash -grafana-import -f Applications -d "my-first-dashboard" remove -``` - - ## Contributing Contributions are welcome, and they are greatly appreciated. You can contribute diff --git a/grafana_import/cli.py b/grafana_import/cli.py index c2e4d23..3cef4d3 100644 --- a/grafana_import/cli.py +++ b/grafana_import/cli.py @@ -104,6 +104,10 @@ def main(): parser.add_argument('-d', '--dashboard_name' , help='name of dashboard to export.') + parser.add_argument('-u', '--grafana_url' + , help='Grafana URL to connect to.' + , required=False) + parser.add_argument('-g', '--grafana_label' , help='label in the config file that represents the grafana to connect to.' , default='default') @@ -148,14 +152,15 @@ def main(): if args.base_path is not None: base_path = inArgs.base_path - config_file = os.path.join(base_path, CONFIG_NAME) - if args.config_file is not None: + if args.config_file is None: + config = {"general": {"debug": False}} + else: + config_file = os.path.join(base_path, CONFIG_NAME) if not re.search(r'^(\.|\/)?/', config_file): config_file = os.path.join(base_path,args.config_file) else: config_file = args.config_file - - config = load_yaml_config(config_file) + config = load_yaml_config(config_file) if args.verbose is None: if 'debug' in config['general']: @@ -192,7 +197,7 @@ def main(): if 'export_suffix' not in config['general'] or config['general']['export_suffix'] is None: config['general']['export_suffix'] = "_%Y%m%d%H%M%S" - params = grafana_settings(config=config, label=args.grafana_label) + params = grafana_settings(url=args.grafana_url, config=config, label=args.grafana_label) params.update({ 'overwrite': args.overwrite, 'allow_new': args.allow_new, diff --git a/grafana_import/grafana.py b/grafana_import/grafana.py index a8af5a8..edfa7b5 100644 --- a/grafana_import/grafana.py +++ b/grafana_import/grafana.py @@ -66,18 +66,29 @@ class Grafana(object): dashboards: t.List[t.Any] = [] #*********************************************** - def __init__( *args, **kwargs ): - self = args[0] + def __init__(self, **kwargs ): - config = { } - config['protocol'] = kwargs.get('protocol', 'http') - config['host'] = kwargs.get('host', 'localhost') - config['port'] = kwargs.get('port', 3000) - config['token'] = kwargs.get('token', None) - if config['token'] is None: - raise GrafanaClient.GrafanaBadInputError('grafana token is not defined') - - config['verify_ssl'] = kwargs.get('verify_ssl', True) + # Configure Grafana connectivity. + if "url" in kwargs: + self.grafana_api = GrafanaApi.GrafanaApi.from_url(kwargs["url"]) + else: + config = {} + config['protocol'] = kwargs.get('protocol', 'http') + config['host'] = kwargs.get('host', 'localhost') + config['port'] = kwargs.get('port', 3000) + config['token'] = kwargs.get('token', None) + if config['token'] is None: + raise GrafanaClient.GrafanaBadInputError('Grafana authentication token missing') + + config['verify_ssl'] = kwargs.get('verify_ssl', True) + + self.grafana_api = GrafanaApi.GrafanaApi( + auth=config['token'], + host=config['host'], + protocol=config['protocol'], + port=config['port'], + verify=config['verify_ssl'], + ) self.search_api_limit = kwargs.get('search_api_limit', 5000) #* set the default destination folder for dash @@ -90,14 +101,6 @@ def __init__( *args, **kwargs ): #* allow to create new dashboard with same name in specified folder. self.allow_new = kwargs.get('allow_new', False) - #* build an aapi object - self.grafana_api = GrafanaApi.GrafanaApi( - auth=config['token'], - host=config['host'], - protocol=config['protocol'], - port=config['port'], - verify=config['verify_ssl'], - ) #* try to connect to the API try: res = self.grafana_api.health.check() diff --git a/grafana_import/util.py b/grafana_import/util.py index 5204cb0..d65cc25 100644 --- a/grafana_import/util.py +++ b/grafana_import/util.py @@ -1,8 +1,10 @@ +import os import typing as t import yaml ConfigType = t.Dict[str, t.Any] +SettingsType = t.Dict[str, t.Union[str, int, bool]] def load_yaml_config(config_file: str) -> ConfigType: @@ -26,7 +28,31 @@ def load_yaml_config(config_file: str) -> ConfigType: raise ValueError(f"Reading configuration file failed: {ex}") from ex -def grafana_settings(config: ConfigType, label: str) -> t.Dict[str, t.Union[str, int, bool]]: +def grafana_settings( + url: t.Union[str, None], + config: t.Union[ConfigType, None], + label: t.Union[str, None]) -> SettingsType: + """ + Acquire Grafana connection profile settings, and application settings. + """ + + params: SettingsType + + # Grafana connectivity. + if url or "GRAFANA_URL" in os.environ: + params = {"url": url or os.environ["GRAFANA_URL"]} + elif config is not None: + params = grafana_settings_from_config_section(config=config, label=label) + + # Additional application parameters. + params.update({ + "search_api_limit": config.get("grafana", {}).get("search_api_limit", 5000), + "folder": config.get("general", {}).get("grafana_folder", "General"), + }) + return params + + +def grafana_settings_from_config_section(config: ConfigType, label: t.Union[str, None]) -> SettingsType: """ Extract Grafana connection profile from configuration dictionary, by label. @@ -34,15 +60,15 @@ def grafana_settings(config: ConfigType, label: str) -> t.Dict[str, t.Union[str, section. In order to address a specific profile, this function accepts a `label` string. """ - if not label or label not in config["grafana"]: + if not label or not config.get("grafana", {}).get(label): raise ValueError(f"Invalid Grafana configuration label: {label}") - # ** init default conf from grafana with set label. + # Initialize default configuration from Grafana by label. # FIXME: That is certainly a code smell. # Q: Has it been introduced later in order to support multiple connection profiles? + # Q: Is it needed to update the original `config` dict, or can it just be omitted? config["grafana"] = config["grafana"][label] - # ************ if "token" not in config["grafana"]: raise ValueError(f"Authentication token missing in Grafana configuration at: {label}") @@ -52,7 +78,6 @@ def grafana_settings(config: ConfigType, label: str) -> t.Dict[str, t.Union[str, "port": config["grafana"].get("port", "3000"), "token": config["grafana"].get("token"), "verify_ssl": config["grafana"].get("verify_ssl", True), - "search_api_limit": config["grafana"].get("search_api_limit", 5000), - "folder": config["general"].get("grafana_folder", "General"), } + return params diff --git a/tests/conftest.py b/tests/conftest.py index faca876..68ef924 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,5 @@ +import os +import typing as t from importlib.resources import files import pytest @@ -27,6 +29,15 @@ def niquests_patch_all(): modules["requests.packages.urllib3"] = urllib3 +@pytest.fixture(scope="session", autouse=True) +def reset_environment(): + """ + Make sure relevant environment variables do not leak into the test suite. + """ + if "GRAFANA_URL" in os.environ: + del os.environ["GRAFANA_URL"] + + @pytest.fixture def mocked_responses(): """ @@ -50,7 +61,7 @@ def config(): @pytest.fixture def settings(config): - return grafana_settings(config, label="default") + return grafana_settings(url=None, config=config, label="default") @pytest.fixture(autouse=True) @@ -60,5 +71,10 @@ def reset_grafana_importer(): @pytest.fixture -def gio(settings) -> Grafana: - return Grafana(**settings) +def gio_factory(settings) -> t.Callable: + def mkgrafana(use_settings: bool = True) -> Grafana: + if use_settings: + return Grafana(**settings) + else: + return Grafana(url="http://localhost:3000") + return mkgrafana diff --git a/tests/test_cli.py b/tests/test_cli.py index b7396b0..52f2246 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -13,7 +13,15 @@ CONFIG_FILE = "grafana_import/conf/grafana-import.yml" -def test_import_dashboard_success(mocked_grafana, mocked_responses, tmp_path, capsys): +def get_settings_arg(use_settings: bool = True): + if use_settings: + return f"--config_file {CONFIG_FILE}" + else: + return "--grafana_url http://localhost:3000" + + +@pytest.mark.parametrize("use_settings", [True, False], ids=["config-yes", "config-no"]) +def test_import_dashboard_success(mocked_grafana, mocked_responses, tmp_path, capsys, use_settings): """ Verify "import dashboard" works. """ @@ -28,7 +36,7 @@ def test_import_dashboard_success(mocked_grafana, mocked_responses, tmp_path, ca dashboard_file = Path(tmp_path / "dashboard.json") dashboard_file.write_text(json.dumps(dashboard, indent=2)) - sys.argv = shlex.split(f"grafana-import import --config_file {CONFIG_FILE} --dashboard_file {dashboard_file}") + sys.argv = shlex.split(f"grafana-import import {get_settings_arg(use_settings)} --dashboard_file {dashboard_file}") with pytest.raises(SystemExit) as ex: main() @@ -38,7 +46,8 @@ def test_import_dashboard_success(mocked_grafana, mocked_responses, tmp_path, ca assert "OK: Dashboard 'Dashboard One' imported into folder 'General'" in out -def test_export_dashboard_success(mocked_grafana, mocked_responses, capsys): +@pytest.mark.parametrize("use_settings", [True, False], ids=["config-yes", "config-no"]) +def test_export_dashboard_success(mocked_grafana, mocked_responses, capsys, use_settings): """ Verify "export dashboard" works. """ @@ -50,7 +59,7 @@ def test_export_dashboard_success(mocked_grafana, mocked_responses, capsys): content_type="application/json", ) - sys.argv = shlex.split(f"grafana-import export --config_file {CONFIG_FILE} --dashboard_name foobar") + sys.argv = shlex.split(f"grafana-import export {get_settings_arg(use_settings)} --dashboard_name foobar") with pytest.raises(SystemExit) as ex: m = mock.patch("builtins.open", open_write_noop) @@ -63,7 +72,8 @@ def test_export_dashboard_success(mocked_grafana, mocked_responses, capsys): assert re.match(r"OK: Dashboard 'foobar' exported to: ./foobar_\d+.json", out) -def test_export_dashboard_notfound(mocked_grafana, mocked_responses, capsys): +@pytest.mark.parametrize("use_settings", [True, False], ids=["config-yes", "config-no"]) +def test_export_dashboard_notfound(mocked_grafana, mocked_responses, capsys, use_settings): """ Verify "export dashboard" fails appropriately when addressed dashboard does not exist. """ @@ -75,7 +85,7 @@ def test_export_dashboard_notfound(mocked_grafana, mocked_responses, capsys): content_type="application/json", ) - sys.argv = shlex.split(f"grafana-import export --config_file {CONFIG_FILE} --dashboard_name foobar") + sys.argv = shlex.split(f"grafana-import export {get_settings_arg(use_settings)} --dashboard_name foobar") with pytest.raises(SystemExit) as ex: main() assert ex.match("1") @@ -84,7 +94,8 @@ def test_export_dashboard_notfound(mocked_grafana, mocked_responses, capsys): assert "Dashboard name not found: foobar" in out -def test_remove_dashboard_success(mocked_grafana, mocked_responses, settings, capsys): +@pytest.mark.parametrize("use_settings", [True, False], ids=["config-yes", "config-no"]) +def test_remove_dashboard_success(mocked_grafana, mocked_responses, capsys, use_settings): """ Verify "remove dashboard" works. """ @@ -95,11 +106,11 @@ def test_remove_dashboard_success(mocked_grafana, mocked_responses, settings, ca content_type="application/json", ) - sys.argv = shlex.split(f"grafana-import remove --config_file {CONFIG_FILE} --dashboard_name foobar") + sys.argv = shlex.split(f"grafana-import remove {get_settings_arg(use_settings)} --dashboard_name foobar") with pytest.raises(SystemExit) as ex: main() assert ex.match("0") out, err = capsys.readouterr() - assert "OK: Dashboard removed: foobar" in out + assert "OK: Dashboard removed: foobar" in out diff --git a/tests/test_core.py b/tests/test_core.py index c3aa586..6b1972d 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -4,19 +4,23 @@ from tests.util import mkdashboard, mock_grafana_health -def test_find_dashboard_success(mocked_grafana, gio): +@pytest.mark.parametrize("use_settings", [True, False], ids=["config-yes", "config-no"]) +def test_find_dashboard_success(mocked_grafana, gio_factory, use_settings): """ Verify "find dashboard" works. """ + gio = gio_factory(use_settings=use_settings) results = gio.find_dashboard("foobar") assert results == {"title": "foobar", "uid": "618f7589-7e3d-4399-a585-372df9fa5e85"} -def test_import_dashboard_success(mocked_grafana, mocked_responses, gio): +@pytest.mark.parametrize("use_settings", [True, False], ids=["config-yes", "config-no"]) +def test_import_dashboard_success(mocked_grafana, mocked_responses, gio_factory, use_settings): """ Verify "import dashboard" works. """ + mocked_responses.post( "http://localhost:3000/api/dashboards/db", json={"status": "ok"}, @@ -25,11 +29,15 @@ def test_import_dashboard_success(mocked_grafana, mocked_responses, gio): ) dashboard = mkdashboard() + + gio = gio_factory(use_settings=use_settings) outcome = gio.import_dashboard(dashboard) + assert outcome is True -def test_export_dashboard_success(mocked_grafana, mocked_responses, gio): +@pytest.mark.parametrize("use_settings", [True, False], ids=["config-yes", "config-no"]) +def test_export_dashboard_success(mocked_grafana, mocked_responses, gio_factory, use_settings): """ Verify "export dashboard" works. """ @@ -40,14 +48,17 @@ def test_export_dashboard_success(mocked_grafana, mocked_responses, gio): content_type="application/json", ) + gio = gio_factory(use_settings=use_settings) dashboard = gio.export_dashboard("foobar") + assert dashboard == {"dashboard": {}} -def test_export_dashboard_notfound(mocked_grafana, mocked_responses, gio): +def test_export_dashboard_notfound(mocked_grafana, mocked_responses, gio_factory): """ Verify "export dashboard" using an unknown dashboard croaks as expected. """ + gio = gio_factory() with pytest.raises(GrafanaDashboardNotFoundError) as ex: gio.export_dashboard("unknown") assert ex.match("Dashboard not found: unknown")