Optimize Grafana dashboards downloading (hitachienergy#3131)

ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py

@@ -37,11 +37,11 @@ def __init__(self, argv: List[str]):
 
         self.__add_args(argv)
 
+        self.__LINE_SIZE: int = 50  # used in printing
+
         if not self.rerun:
             self.__log_info_summary()
 
-        self.__LINE_SIZE: int = 50  # used in printing
-
     def __log_info_summary(self):
         """
         Helper function for printing all parsed arguments
@@ -61,7 +61,7 @@ def __log_info_summary(self):
         lines.append(f'Repos backup file: {str(self.repos_backup_file)}')
 
         if self.dest_manifest:
-            lines.append(f'Manifest used: {str(self.dest_manifest)}')
+            lines.append(f'Manifest used: {str(self.dest_manifest.absolute())}')
 
         if self.is_log_file_enabled:
             lines.append(f'Log file location: {str(self.log_file.absolute())}')
@@ -216,7 +216,7 @@ def __add_args(self, argv: List[str]):
         self.rerun = args['rerun']
         self.pyyaml_installed = args['pyyaml_installed']
 
-    def __print_parsed_manifest_data(self, output: Dict[str, Any]):
+    def __print_parsed_manifest_data(self, requirements: Dict[str, Any], output: Dict[str, Any]):
         lines: List[str] = ['Manifest summary:']
 
         lines.append('-' * self.__LINE_SIZE)
@@ -231,6 +231,13 @@ def __print_parsed_manifest_data(self, output: Dict[str, Any]):
         for feature in output['detected-features']:
             lines.append(f'- {feature}')
 
+        dashboards = requirements['grafana-dashboards']
+        if dashboards:
+            lines.append('')
+            lines.append('Dashboards to download:')
+            for dashboard in dashboards:
+                lines.append(f'- {dashboard}')
+
         lines.append('-' * self.__LINE_SIZE)
 
         logging.info('\n'.join(lines))
@@ -248,5 +255,8 @@ def read_manifest(self, requirements: Dict[str, Any]):
         mreader = ManifestReader(self.dest_manifest)
         output = mreader.parse_manifest()
 
+        if 'grafana' not in output['detected-features']:
+            requirements['grafana-dashboards'] = []
+
         if self.verbose_mode:
-            self.__print_parsed_manifest_data(output)
+            self.__print_parsed_manifest_data(requirements, output)

ansible/playbooks/roles/repository/files/download-requirements/src/mode/base_mode.py

@@ -229,10 +229,12 @@ def run(self):
         """
         # add required directories
         self._cfg.dest_files.mkdir(exist_ok=True, parents=True)
-        self._cfg.dest_grafana_dashboards.mkdir(exist_ok=True, parents=True)
         self._cfg.dest_images.mkdir(exist_ok=True, parents=True)
         self._cfg.dest_packages.mkdir(exist_ok=True, parents=True)
 
+        if self._requirements['grafana-dashboards']:
+            self._cfg.dest_grafana_dashboards.mkdir(exist_ok=True, parents=True)
+
         # provides tar which is required for backup
         logging.info('Installing base packages...')
         self._install_base_packages()
@@ -259,9 +261,10 @@ def run(self):
         self.__download_files(self._requirements['files'], self._cfg.dest_files)
         logging.info('Done downloading files.')
 
-        logging.info('Downloading grafana dashboards...')
-        self.__download_grafana_dashboards()
-        logging.info('Done downloading grafana dashboards.')
+        if self._requirements['grafana-dashboards']:
+            logging.info('Downloading grafana dashboards...')
+            self.__download_grafana_dashboards()
+            logging.info('Done downloading grafana dashboards.')
 
         logging.info('Downloading Crane...')
         self.__download_crane()

ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py

@@ -1,17 +1,21 @@
 import logging
 from pathlib import Path
 
+import pytest
 import yaml
 
 from src.config.config import Config
-from tests.data.config import EXPECTED_VERBOSE_OUTPUT
-from tests.data.manifest_reader import INPUT_MANIFEST_FEATURE_MAPPINGS
+from tests.data.config import EXPECTED_VERBOSE_OUTPUT, EXPECTED_VERBOSE_DASHBOARD_OUTPUT, DASHBOARD_REQUIREMENTS
+from tests.data.manifest_reader import INPUT_MANIFEST_FEATURE_MAPPINGS, INPUT_MANIFEST_WITH_DASHBOARDS
 
 
-def test_manifest_verbose_output(mocker, caplog):
+@pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC',
+                         [(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_VERBOSE_OUTPUT),
+                          (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_VERBOSE_DASHBOARD_OUTPUT)])
+def test_manifest_verbose_output(INPUT_DOC, EXPECTED_OUTPUT_DOC, mocker, caplog):
     ''' Check output produced when running download-requirements script with the `-v|--verbose` flag and with provided `-m|--manifest` '''
 
-    mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_MANIFEST_FEATURE_MAPPINGS))
+    mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_DOC))
     caplog.set_level(logging.INFO)
 
     # mock Config's init methods:
@@ -23,8 +27,8 @@ def test_manifest_verbose_output(mocker, caplog):
     # mock required config data:
     config.dest_manifest = Path('/some/path')
     config.verbose_mode = True
-    config.read_manifest({})
+    config.read_manifest(yaml.safe_load(DASHBOARD_REQUIREMENTS))
 
     log_output = f'\n{"".join(caplog.messages)}\n'
 
-    assert log_output == EXPECTED_VERBOSE_OUTPUT
+    assert log_output == EXPECTED_OUTPUT_DOC

ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py

@@ -1,13 +1,20 @@
 from pathlib import Path
 
+import pytest
 import yaml
 
 from src.config.manifest_reader import ManifestReader
-from tests.data.manifest_reader import EXPECTED_FEATURE_MAPPINGS, INPUT_MANIFEST_FEATURE_MAPPINGS
+from tests.data.manifest_reader import (EXPECTED_FEATURE_MAPPINGS,
+                                        EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS,
+                                        INPUT_MANIFEST_FEATURE_MAPPINGS,
+                                        INPUT_MANIFEST_WITH_DASHBOARDS)
 
-def test_parse_manifest(mocker):
+@pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC',
+                         [(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_FEATURE_MAPPINGS),
+                          (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS)])
+def test_parse_manifest(INPUT_DOC, EXPECTED_OUTPUT_DOC, mocker):
     ''' Check manifest file parsing '''
-    mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_MANIFEST_FEATURE_MAPPINGS))
+    mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_DOC))
 
     mreader = ManifestReader(Path('/some/path'))
-    assert mreader.parse_manifest() == EXPECTED_FEATURE_MAPPINGS
+    assert mreader.parse_manifest() == EXPECTED_OUTPUT_DOC

ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py

@@ -1,22 +1,97 @@
+DASHBOARD_REQUIREMENTS = """
+grafana-dashboards:
+  grafana_dashboard_7249:
+    url: 'https://grafana.com/api/dashboards/7249/revisions/1/download'
+    sha256: 41cc2794b1cc9fc537baf045fee12d086d23632b4c8b2e88985274bb9862e731
+  grafana_dashboard_315:
+    url: 'https://grafana.com/api/dashboards/315/revisions/3/download'
+    sha256: ee46dd6e68a9950aa78e8c88ae5e565c8ebde6cbdbe08972a70f06c5486618fb
+  grafana_dashboard_11074:
+    url: 'https://grafana.com/api/dashboards/11074/revisions/9/download'
+    sha256: 151b23305da46eab84930e99175e1c07e375af73dbbb4b8f501ca25f5ac62785
+  grafana_dashboard_405:
+    url: 'https://grafana.com/api/dashboards/405/revisions/8/download'
+    sha256: 97675027cbd5b7241e93a2b598654c4b466bc909eeb6358ba123d500094d913c
+  grafana_dashboard_455:
+    url: 'https://grafana.com/api/dashboards/455/revisions/2/download'
+    sha256: c66b91ab8d258b0dc005d3ee4dac3a5634a627c79cc8053875f76ab1e369a362
+  grafana_dashboard_9628:
+    url: 'https://grafana.com/api/dashboards/9628/revisions/7/download'
+    sha256: c64cc38ad9ebd7af09551ee83e669a38f62a76e7c80929af5668a5852732b376
+  grafana_dashboard_4279:
+    url: 'https://grafana.com/api/dashboards/4279/revisions/4/download'
+    sha256: 74d47be868da52c145240ab5586d91ace9e9218ca775af988f9d60e501907a25
+  grafana_dashboard_1860:
+    url: 'https://grafana.com/api/dashboards/1860/revisions/23/download'
+    sha256: 225faab8bf35c1723af14d4c069882ccb92b455d1941c6b1cf3d95a1576c13d7
+  grafana_dashboard_7589:
+    url: 'https://grafana.com/api/dashboards/7589/revisions/5/download'
+    sha256: cf020e14465626360418e8b5746818c80d77c0301422f3060879fddc099c2151
+  grafana_dashboard_789:
+    url: 'https://grafana.com/api/dashboards/789/revisions/1/download'
+    sha256: 6a9b4bdc386062287af4f7d56781103a2e45a51813596a65f03c1ae1d4d3e919
+  grafana_dashboard_179:
+    url: 'https://grafana.com/api/dashboards/179/revisions/7/download'
+    sha256: 8d67350ff74e715fb1463f2406f24a73377357d90344f8200dad9d1b2a8133c2
+  grafana_dashboard_6663:
+    url: 'https://grafana.com/api/dashboards/6663/revisions/1/download'
+    sha256: d544d88069e1b793ff3d8f6970df641ad9a66217e69b629621e1ecbb2f06aa05
+  grafana_dashboard_10991:
+    url: 'https://grafana.com/api/dashboards/10991/revisions/11/download'
+    sha256: 66340fa3256d432287cba75ab5177eb058c77afa7d521a75d58099f95b1bff50
+"""
+
+
 EXPECTED_VERBOSE_OUTPUT = """
 Manifest summary:
 --------------------------------------------------
 Components detected:
 - kafka
-- monitoring
 - repository
 
 Features detected:
 - filebeat
 - firewall
-- grafana
 - image-registry
 - jmx-exporter
 - kafka
 - kafka-exporter
 - node-exporter
-- prometheus
 - repository
 - zookeeper
 --------------------------------------------------
 """
+
+
+EXPECTED_VERBOSE_DASHBOARD_OUTPUT = """
+Manifest summary:
+--------------------------------------------------
+Components detected:
+- monitoring
+- repository
+
+Features detected:
+- filebeat
+- firewall
+- grafana
+- image-registry
+- node-exporter
+- prometheus
+- repository
+
+Dashboards to download:
+- grafana_dashboard_7249
+- grafana_dashboard_315
+- grafana_dashboard_11074
+- grafana_dashboard_405
+- grafana_dashboard_455
+- grafana_dashboard_9628
+- grafana_dashboard_4279
+- grafana_dashboard_1860
+- grafana_dashboard_7589
+- grafana_dashboard_789
+- grafana_dashboard_179
+- grafana_dashboard_6663
+- grafana_dashboard_10991
+--------------------------------------------------
+"""

ansible/playbooks/roles/repository/files/download-requirements/tests/data/manifest_reader.py

@@ -1,36 +1,4 @@
-INPUT_MANIFEST_FEATURE_MAPPINGS = """
----
-kind: epiphany-cluster
-title: Epiphany cluster Config
-provider: any
-name: default
-specification:
-  name: new_cluster
-  admin_user:
-    name: operations
-    key_path: /shared/.ssh/epiphany-operations/id_rsa
-  components:
-    repository:
-      count: 1
-    kubernetes_master:
-      count: 0
-    kubernetes_node:
-      count: 0
-    logging:
-      count: 0
-    monitoring:
-      count: 1
-    kafka:
-      count: 2
-    postgresql:
-      count: 0
-    load_balancer:
-      count: 0
-    rabbitmq:
-      count: 0
-    opendistro_for_elasticsearch:
-      count: 0
-version: 2.0.0dev
+FEATURE_MAPPINGS = """
 ---
 kind: configuration/feature-mappings
 title: Feature mapping to roles
@@ -126,17 +94,99 @@
 """
 
 
+INPUT_MANIFEST_FEATURE_MAPPINGS = f"""
+kind: epiphany-cluster
+title: Epiphany cluster Config
+provider: any
+name: default
+specification:
+  name: new_cluster
+  admin_user:
+    name: operations
+    key_path: /shared/.ssh/epiphany-operations/id_rsa
+  components:
+    repository:
+      count: 1
+    kubernetes_master:
+      count: 0
+    kubernetes_node:
+      count: 0
+    logging:
+      count: 0
+    monitoring:
+      count: 0
+    kafka:
+      count: 2
+    postgresql:
+      count: 0
+    load_balancer:
+      count: 0
+    rabbitmq:
+      count: 0
+    opendistro_for_elasticsearch:
+      count: 0
+version: 2.0.0dev
+{FEATURE_MAPPINGS}
+"""
+
+
+INPUT_MANIFEST_WITH_DASHBOARDS = f"""
+kind: epiphany-cluster
+title: Epiphany cluster Config
+provider: any
+name: default
+specification:
+  name: new_cluster
+  admin_user:
+    name: operations
+    key_path: /shared/.ssh/epiphany-operations/id_rsa
+  components:
+    repository:
+      count: 1
+    kubernetes_master:
+      count: 0
+    kubernetes_node:
+      count: 0
+    logging:
+      count: 0
+    monitoring:
+      count: 1
+    kafka:
+      count: 0
+    postgresql:
+      count: 0
+    load_balancer:
+      count: 0
+    rabbitmq:
+      count: 0
+    opendistro_for_elasticsearch:
+      count: 0
+version: 2.0.0dev
+{FEATURE_MAPPINGS}
+"""
+
+
 EXPECTED_FEATURE_MAPPINGS = {
-    'detected-components': ['kafka', 'monitoring', 'repository'],
+    'detected-components': ['kafka', 'repository'],
     'detected-features': ['filebeat',
                           'firewall',
-                          'grafana',
                           'image-registry',
                           'jmx-exporter',
                           'kafka',
                           'kafka-exporter',
                           'node-exporter',
-                          'prometheus',
                           'repository',
                           'zookeeper']
 }
+
+
+EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS = {
+    'detected-components': ['monitoring', 'repository'],
+    'detected-features': ['filebeat',
+                          'firewall',
+                          'grafana',
+                          'image-registry',
+                          'node-exporter',
+                          'prometheus',
+                          'repository']
+}