Ionosphere - Graphite now graphs - v1.1.0-alpha

IssueID #1842: Ionosphere - Graphite now graphs
IssueID #1658: Patterning Skyline Ionosphere
IssueID #1460: panorama check file fails
Panorama check file fails #24

- Added returns to skyline_functions.get_graphite_metric and specific webapp
  Ionosphere URL parameters for the Graphite NOW graphs
- Removed unused get_graphite_metric from skyline/webapp/backend.py
- Added get_graphite_metric, the graphite_now png context and retrieving the
  graphite_now_images at TARGET_HOURS, 24h, 7d and 30d
- Removed the reference to #14 from webapp Ionosphere templates
- Added order and graphite_now_images block to training_data.html
- Added sorted_images for order and graphite_now_images to webapp

Modified:
skyline/skyline_functions.py
skyline/webapp/backend.py
skyline/webapp/ionosphere_backend.py
skyline/webapp/templates/features_profiles.html
skyline/webapp/templates/training_data.html
skyline/webapp/webapp.py

skyline/skyline_functions.py

@@ -461,6 +461,11 @@ def get_graphite_metric(
         from sys import version_info
         python_version = int(version_info[0])
 
+    try:
+        quote(skyline_app, safe='')
+    except:
+        from requests.utils import quote
+
     current_skyline_app_logger = current_skyline_app + 'Log'
     current_logger = logging.getLogger(current_skyline_app_logger)
 
@@ -486,10 +491,12 @@ def get_graphite_metric(
 
     current_logger.info('read_timeout - %s' % str(read_timeout))
 
+    graphite_from = datetime.datetime.fromtimestamp(int(from_timestamp)).strftime('%H:%M_%Y%m%d')
+    current_logger.info('graphite_from - %s' % str(graphite_from))
+
     graphite_until = datetime.datetime.fromtimestamp(int(until_timestamp)).strftime('%H:%M_%Y%m%d')
     current_logger.info('graphite_until - %s' % str(graphite_until))
 
-    graphite_from = datetime.datetime.fromtimestamp(int(from_timestamp)).strftime('%H:%M_%Y%m%d')
     output_format = data_type
 
     # graphite URL
@@ -511,6 +518,7 @@ def get_graphite_metric(
         else:
             if settings.ENABLE_DEBUG:
                 current_logger.info('graph file exists - %s' % str(output_object))
+            return True
 
     if get_image:
         current_logger.info(
@@ -522,6 +530,26 @@ def get_graphite_metric(
             image_url += '&width=586'
         if 'height' not in image_url:
             image_url += '&height=308'
+        # @added 20170106 - Feature #1842: Ionosphere - Graphite now graphs
+        # settings, color and title
+        if current_skyline_app == 'webapp':
+            if 'graphite_now' in output_object:
+                no_extension = os.path.splitext(output_object)[0]
+                _hours = os.path.splitext(no_extension)[1]
+                hours = _hours.replace('.', '')
+                int_hours = hours.replace('h', '')
+                str_value = str(int_hours)
+                period = 'hours'
+                if int(int_hours) > 24:
+                    str_value = str(int(int_hours) / 24)
+                    period = 'days'
+                unencoded_graph_title = 'Graphite NOW at %s %s' % (str_value, period)
+                graph_title_string = quote(unencoded_graph_title, safe='')
+                graph_title = '&title=%s' % graph_title_string
+                add_parameters = '%s&colorList=blue%s' % (settings.GRAPHITE_GRAPH_SETTINGS, graph_title)
+                image_url += add_parameters
+                current_logger.info('Ionosphere graphite NOW url - %s' % (image_url))
+
         if settings.ENABLE_DEBUG:
             current_logger.info('graphite image url - %s' % (str(image_url)))
         image_url_timeout = int(connect_timeout)
@@ -550,6 +578,14 @@ def get_graphite_metric(
         if not os.path.isfile(graphite_image_file):
             msg = 'retrieve failed to surface %s graph from Graphite' % (metric)
             current_logger.error('error :: %s' % msg)
+        # @added 20170107 - Feature #1842: Ionosphere - Graphite now graphs
+        # In order to determine whether a Graphite image was retrieved or not
+        # this needs to return here.  This should not be backwards incompatible
+        # as it has never been used to determine the outcome before it appears,
+        # which as they say is my bad.
+            return False
+        else:
+            return True
 
     if data_type == 'json':
 
@@ -613,6 +649,8 @@ def get_graphite_metric(
         else:
             return True
 
+    return True
+
 # @added 20160922 - Branch #922: Ionosphere
 # Added the send_anomalous_metric_to function for Analyzer and Mirage
 # @modified 20161228 Feature #1828: ionosphere - mirage Redis data features

skyline/webapp/backend.py

@@ -11,7 +11,7 @@
 # from mysql.connector import errorcode
 
 import settings
-from skyline_functions import get_graphite_metric, mysql_select
+from skyline_functions import mysql_select
 
 import skyline_version
 skyline_version = skyline_version.__absolute_version__

skyline/webapp/ionosphere_backend.py

@@ -1,6 +1,6 @@
 from __future__ import division
 import logging
-from os import path, walk, listdir
+from os import path, walk, listdir, remove
 import string
 import operator
 import time
@@ -24,7 +24,8 @@
 
 import settings
 import skyline_version
-from skyline_functions import RepresentsInt, mkdir_p, write_data_to_file
+from skyline_functions import (
+    RepresentsInt, mkdir_p, write_data_to_file, get_graphite_metric)
 from tsfresh_feature_names import TSFRESH_FEATURES
 
 from database import get_engine, ionosphere_table_meta, metrics_table_meta
@@ -325,7 +326,10 @@ def new_load_metric_vars(metric_vars_file):
         metric_file = path.join(metric_data_dir, i_file)
         metric_paths.append([i_file, metric_file])
         if i_file.endswith('.png'):
-            images.append(str(metric_file))
+            # @modified 20170106 - Feature #1842: Ionosphere - Graphite now graphs
+            # Exclude any graphite_now png files from the images lists
+            if '.graphite_now.' not in i_file:
+                images.append(str(metric_file))
         if i_file == metric_var_filename:
             metric_vars_file = str(metric_file)
         if i_file == ts_json_filename:
@@ -396,7 +400,39 @@ def new_load_metric_vars(metric_vars_file):
             logger.error(traceback.format_exc())
             logger.error('error :: failed to get anomaly id from panorama response: %s' % str(r.text))
 
-    return (metric_paths, images, human_date, metric_vars, ts_json, data_to_process, panorama_anomaly_id)
+    # @added 20170106 - Feature #1842: Ionosphere - Graphite now graphs
+    # Graphite now graphs at TARGET_HOURS, 24h, 7d, 30d to fully inform the
+    # operator about the metric.
+    graphite_now_images = []
+    graphite_now = int(time.time())
+    graph_resolutions = [int(settings.TARGET_HOURS), 24, 168, 720]
+    for target_hours in graph_resolutions:
+        graph_image = False
+        try:
+            target_seconds = int((target_hours * 60) * 60)
+            from_timestamp = str(graphite_now - target_seconds)
+            until_timestamp = str(graphite_now)
+            graph_image_file = '%s/%s.graphite_now.%sh.png' % (metric_data_dir, base_name, str(target_hours))
+            # These are NOW graphs, so if the graph_image_file exists, remove it
+            if path.isfile(graph_image_file):
+                try:
+                    remove(str(graph_image_file))
+                    logger.info('graph_image_file removed - %s' % str(graph_image_file))
+                except OSError:
+                    pass
+            logger.info('getting Graphite graph for %s hours - from_timestamp - %s, until_timestamp - %s' % (str(target_hours), str(from_timestamp), str(until_timestamp)))
+            graph_image = get_graphite_metric(
+                skyline_app, base_name, from_timestamp, until_timestamp, 'image',
+                graph_image_file)
+            if graph_image:
+                graphite_now_images.append(graph_image_file)
+        except:
+            logger.error(traceback.format_exc())
+            logger.error('error :: failed to get Graphite graph at %s hours for %s' % (str(target_hours), base_name))
+
+    return (
+        metric_paths, images, human_date, metric_vars, ts_json, data_to_process,
+        panorama_anomaly_id, graphite_now_images)
 
 
 def get_an_engine():

skyline/webapp/templates/features_profiles.html

@@ -331,10 +331,17 @@ <h4><span class="logo"><span class="sky">Timeseries ::</span> <span class="re">n
     <table>
         <caption><b><span class="logo"><span class="sky">Known</span> <span class="re">bugs</span></span></b></caption>
         <tr>
-            <td>
-              <a target='_blank' href="https://github.com/earthgecko/skyline/issues/24">#24</a>
-            </td>
-            <td>An anomaly can sometimes not have a matching Panorama anomaly record due to Panorama check file fails #24</td>
+<!-- @modified 20170106 - Feature #1830: Ionosphere alerts
+#                         Bug #1460: panorama check file fails
+#                         Panorama check file fails #24
+# Fixed
+          <td>
+            <a target='_blank' href="https://github.com/earthgecko/skyline/issues/24">#24</a>
+          </td>
+          <td>An anomaly can sometimes not have a matching Panorama anomaly record due to Panorama check file fails #24</td>
+-->
+          <td>None</td>
+          <td>That are known of at the present time</td>
         </tr>
     </table>
   </div>

skyline/webapp/templates/training_data.html

@@ -378,10 +378,9 @@ <h4><span class="logo"><span class="sky">Panorama ::</span> <span class="re">ano
 the top right of this page.  Here is hoping that this day just gets better.
 	  </div>
   {% endif %}
+
   {% for image in metric_images %}
-    {% if '.redis.plot' in image %}
-      <h4><span class="logo"><span class="sky">Redis ::</span> <span class="re">anomaly ::</span></span></span> data plot at <strong>{{ redis_full_duration_in_hours }} hours</strong></h4>
-    {% else %}
+    {% if not '.redis.plot' in image %}
       {% if metric_second_order_resolution_hours != False %}
       <h4><span class="logo"><span class="sky">Graphite ::</span> <span class="re">anomaly ::</span></span></span> graph at <strong>{{ metric_second_order_resolution_hours }} hours</strong></h4>
         {% else %}
@@ -391,10 +390,31 @@ <h4><span class="logo"><span class="sky">Graphite ::</span> <span class="re">ano
 If you wish to see the <a target='_blank' href="{{ graphite_url }}">current
 Graphite graph</a> for {{ for_metric }} (opens in new tab)
       {% endif %}
+      <img src="ionosphere_images?image={{ image }}" alt="{{ for_metric }} timeseries graph" class="img-responsive center-block" /><br>
+Image file: {{ image }}<br><br>
     {% endif %}
+  {% endfor %}
+
+      <h4><span class="logo"><span class="sky">Graphite ::</span> <span class="re">graphs NOW ::</span></span></span> at <strong>7h, 24h, 7d and 30d</strong></h4>
+<!-- @added 20170106 - Feature #1842: Ionosphere - Graphite now graphs -->
+  {% for gimage in graphite_now_images %}
+<!--      <img src="ionosphere_images?image={{ gimage }}" alt="{{ for_metric }} timeseries graph" class="img-responsive center-block" />
+# Align 4 images - tried all of http://stackoverflow.com/questions/9382970/how-to-center-a-div-with-bootstrap2
+# end up using https://owlcation.com/stem/how-to-align-images-side-by-side thanks
+# Ellen Brundige
+-->
+        <img src="ionosphere_images?image={{ gimage }}" alt="{{ for_metric }} timeseries graph" style="float: left; width: 40%; margin-right: 1%; margin-bottom: 0.5em;">
+  {% endfor %}
+        <p style="clear: both;">
+
+  {% for image in metric_images %}
+    {% if '.redis.plot' in image %}
+      <h4><span class="logo"><span class="sky">Redis ::</span> <span class="re">anomaly ::</span></span></span> data plot at <strong>{{ redis_full_duration_in_hours }} hours</strong></h4>
       <img src="ionosphere_images?image={{ image }}" alt="{{ for_metric }} timeseries graph" class="img-responsive center-block" /><br>
 Image file: {{ image }}<br><br>
+    {% endif %}
   {% endfor %}
+
       <h4><span class="logo"><span class="sky">Training ::</span> <span class="re">data ::</span></span></span> files</h4>
       <table>
         <tr>
@@ -419,10 +439,17 @@ <h4><span class="logo"><span class="sky">Timeseries ::</span> <span class="re">a
     <table>
         <caption><b><span class="logo"><span class="sky">Known</span> <span class="re">bugs</span></span></b></caption>
         <tr>
+<!-- @modified 20170106 - Feature #1830: Ionosphere alerts
+#                         Bug #1460: panorama check file fails
+#                         Panorama check file fails #24
+# Fixed
             <td>
               <a target='_blank' href="https://github.com/earthgecko/skyline/issues/24">#24</a>
             </td>
             <td>An anomaly can sometimes not have a matching Panorama anomaly record due to Panorama check file fails #24</td>
+-->
+            <td>None</td>
+            <td>That are known of at the present time</td>
         </tr>
     </table>
   </div>

skyline/webapp/webapp.py

@@ -1105,7 +1105,9 @@ def ionosphere():
                 return internal_error(fail_msg, trace)
 
         try:
-            mpaths, images, hdate, m_vars, ts_json, data_to_process, p_id = ionosphere_metric_data(requested_timestamp, base_name, context)
+            # @added 20170106 - Feature #1842: Ionosphere - Graphite now graphs
+            # Added graphite_now_images gimages
+            mpaths, images, hdate, m_vars, ts_json, data_to_process, p_id, gimages = ionosphere_metric_data(requested_timestamp, base_name, context)
 
             # @added 20170104 - Feature #1842: Ionosphere - Graphite now graphs
             # Added the full_duration parameter so that the appropriate graphs can be
@@ -1127,10 +1129,17 @@ def ionosphere():
                 trace = traceback.format_exc()
                 return internal_error(message, trace)
 
+            # @added 20170105 - Feature #1842: Ionosphere - Graphite now graphs
+            # We want to sort the images so that the Graphite image is always
+            # displayed first in he training_data.html page AND we want Graphite
+            # now graphs at TARGET_HOURS, 24h, 7d, 30d to inform the operator
+            # about the metric
+            sorted_images = sorted(images)
+
             return render_template(
                 'ionosphere.html', timestamp=requested_timestamp,
                 for_metric=base_name, metric_vars=m_vars, metric_files=mpaths,
-                metric_images=images, human_date=hdate, timeseries=ts_json,
+                metric_images=sorted_images, human_date=hdate, timeseries=ts_json,
                 data_ok=data_to_process, td_files=mpaths,
                 panorama_anomaly_id=p_id, graphite_url=graph_url,
                 extracted_features=features, calc_time=f_calc,
@@ -1141,7 +1150,7 @@ def ionosphere():
                 metric_full_duration=m_full_duration,
                 metric_full_duration_in_hours=m_full_duration_in_hours,
                 metric_second_order_resolution_hours=second_order_resolution_hours,
-                tsfresh_version=TSFRESH_VERSION,
+                tsfresh_version=TSFRESH_VERSION, graphite_now_images=gimages,
                 version=skyline_version, duration=(time.time() - start),
                 print_debug=debug_on), 200
         except: