Report bytes scanned (re #69)

mozilla · Jan 8, 2018 · b193971 · b193971
1 parent cdf0053
commit b193971
Show file tree

Hide file tree

Showing 8 changed files with 18 additions and 8 deletions.
diff --git a/client/app/pages/queries/query.html b/client/app/pages/queries/query.html
@@ -208,7 +208,7 @@ <h3>
           <span class="zmdi zmdi-time"></span>
           <span class="text-muted">Data Scanned </span>
           <strong>
-            <span ng-if="queryResult.query_result.data_scanned != ''"> {{ queryResult.query_result.data_scanned }}</span>
+            <span ng-if="queryResult.query_result.data_scanned != '' || queryResult.query_result.data_scanned == 'N/A'"> {{ queryResult.query_result.data_scanned }}</span>
           </strong>
         </li>
       </ul>

diff --git a/redash/models.py b/redash/models.py
@@ -730,8 +730,8 @@ def get_latest(cls, data_source, query, max_age=0):
     def store_result(cls, org, data_source, query_hash, query, data, run_time, retrieved_at):
         try:
             data_scanned_information = json.loads(data)['data_scanned']
-        except (ValueError, TypeError) as e:
-            data_scanned_information = 'error'
+        except (ValueError, TypeError, KeyError) as e:
+            data_scanned_information = ''
 
         query_result = cls(org=org,
                            query_hash=query_hash,

diff --git a/redash/query_runner/activedata.py b/redash/query_runner/activedata.py
@@ -115,6 +115,7 @@ def run_query(self, annotated_query, user):
             return None, cause
 
         output = normalize(response)
+        output.update({'data_scanned':'N/A'})
         json_data = json.dumps(output, cls=JSONEncoder)
         return json_data, None
 

diff --git a/redash/query_runner/athena.py b/redash/query_runner/athena.py
@@ -177,18 +177,23 @@ def run_query(self, query, user):
             column_tuples = [(i[0], _TYPE_MAPPINGS.get(i[1], None)) for i in cursor.description]
             columns = self.fetch_columns(column_tuples)
             rows = [dict(zip(([c['name'] for c in columns]), r)) for i, r in enumerate(cursor.fetchall())]
-            data = {'columns': columns, 'rows': rows}
+            qbytes = 'upstream2'
+            try:
+                qbytes = cursor.data_scanned_in_bytes
+            except AttributeError as e:
+                logger.debug("Athena Upstream can't get data_scanned_in_bytes: %s", e)
+            data = {'columns': columns, 'rows': rows, 'data_scanned': qbytes}
             json_data = json.dumps(data, cls=JSONEncoder)
             error = None
         except KeyboardInterrupt:
             if cursor.query_id:
                 cursor.cancel()
             error = "Query cancelled by user."
             json_data = None
-        except Exception as ex:
+        except Exception as exc:
             if cursor.query_id:
                 cursor.cancel()
-            error = ex.message
+            error = exc.message
             json_data = None
 
         return json_data, error

diff --git a/redash/query_runner/big_query.py b/redash/query_runner/big_query.py
@@ -248,6 +248,7 @@ def run_query(self, query, user):
                     return None, "Larger than %d MBytes will be processed (%f MBytes)" % (limitMB, processedMB)
 
             data = self._get_query_result(jobs, query)
+            data.update({'data_scanned':'N/A'})
             error = None
 
             json_data = json.dumps(data, cls=JSONEncoder)

diff --git a/redash/query_runner/google_spreadsheets.py b/redash/query_runner/google_spreadsheets.py
@@ -205,6 +205,7 @@ def run_query(self, query, user):
             spreadsheet = spreadsheet_service.open_by_key(key)
 
             data = parse_spreadsheet(spreadsheet, worksheet_num)
+            data.update({'data_scanned': 'N/A'})
 
             json_data = json.dumps(data, cls=JSONEncoder)
             error = None

diff --git a/redash/query_runner/mysql.py b/redash/query_runner/mysql.py
@@ -161,7 +161,7 @@ def run_query(self, query, user):
                 columns = self.fetch_columns([(i[0], types_map.get(i[1], None)) for i in cursor.description])
                 rows = [dict(zip((c['name'] for c in columns), row)) for row in data]
 
-                data = {'columns': columns, 'rows': rows}
+                data = {'columns': columns, 'rows': rows, 'data_scanned': 'N/A'}
                 json_data = json.dumps(data, cls=JSONEncoder)
                 error = None
             else:

diff --git a/redash/query_runner/presto.py b/redash/query_runner/presto.py
@@ -34,6 +34,8 @@
 class Presto(BaseQueryRunner):
     noop_query = 'SHOW TABLES'
     default_doc_url = 'https://prestodb.io/docs/current/'
+    data_source_version_query = "SELECT node_version AS version FROM system.runtime.nodes WHERE coordinator = true AND state = 'active'"
+    data_source_version_post_process = "none"
 
     @classmethod
     def configuration_schema(cls):
@@ -116,7 +118,7 @@ def run_query(self, query, user):
             column_tuples = [(i[0], PRESTO_TYPES_MAPPING.get(i[1], None)) for i in cursor.description]
             columns = self.fetch_columns(column_tuples)
             rows = [dict(zip(([c['name'] for c in columns]), r)) for i, r in enumerate(cursor.fetchall())]
-            data = {'columns': columns, 'rows': rows}
+            data = {'columns': columns, 'rows': rows, 'data_scanned': 'N/A'}
             json_data = json.dumps(data, cls=JSONEncoder)
             error = None
         except DatabaseError as db: