Merge pull request getredash#139 from erans/master

Added support for running scripts as queries

data/manager.py

@@ -185,6 +185,12 @@ def start_workers(self, workers_count, connection_type, connection_string):
             from redash.data import query_runner_bigquery
             connection_params = json.loads(connection_string)
             runner = query_runner_bigquery.bigquery(connection_params)
+        elif connection_type == 'script':
+            from redash.data import query_runner_script
+            runner = query_runner_script.script(connection_string)
+        elif connection_type == 'url':
+            from redash.data import query_runner_url
+            runner = query_runner_url.url(connection_string)
         else:
             from redash.data import query_runner
             runner = query_runner.redshift(connection_string)

data/query_runner_script.py

@@ -0,0 +1,48 @@
+import json
+import logging
+import sys
+import os
+import subprocess
+
+# We use subprocess.check_output because we are lazy.
+# If someone will really want to run this on Python < 2.7 they can easily update the code to run
+# Popen, check the retcodes and other things and read the standard output to a variable.
+if not "check_output" in subprocess.__dict__:
+    print "ERROR: This runner uses subprocess.check_output function which exists in Python 2.7"
+
+def script(connection_string):
+
+    def query_runner(query):
+        try:
+            json_data = None
+            error = None
+
+            # Poor man's protection against running scripts from output the scripts directory
+            if connection_string.find("../") > -1:
+                return None, "Scripts can only be run from the configured scripts directory"
+
+            query = query.strip()
+
+            script = os.path.join(connection_string, query)
+            if not os.path.exists(script):
+                return None, "Script '%s' not found in script directory" % query
+
+            output = subprocess.check_output(script, shell=False)
+            if output != None:
+                output = output.strip()
+                if output != "":
+                    return output, None
+
+            error = "Error reading output"
+        except subprocess.CalledProcessError as e:
+            return None, str(e)
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+        return json_data, error
+
+    query_runner.annotate_query = False
+    return query_runner

data/query_runner_url.py

@@ -0,0 +1,45 @@
+import json
+import logging
+import sys
+import os
+import urllib2
+
+def url(connection_string):
+
+    def query_runner(query):
+        base_url = connection_string
+
+        try:
+            json_data = None
+            error = None
+
+            query = query.strip()
+
+            if base_url is not None and base_url != "":
+                if query.find("://") > -1:
+                    return None, "Accepting only relative URLs to '%s'" % base_url
+
+            if base_url is None:
+                base_url = ""
+
+            url = base_url + query
+
+            json_data = urllib2.urlopen(url).read().strip()
+
+            if not json_data:
+                error = "Error reading data from '%s'" % url
+
+            return json_data, error
+
+        except urllib2.URLError as e:
+            return None, str(e)
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+        return json_data, error
+
+    query_runner.annotate_query = False
+    return query_runner

settings.py

@@ -46,14 +46,19 @@ def parse_boolean(str):
 
 NAME = os.environ.get('REDASH_NAME', 're:dash')
 
-# "pg", "graphite" or "mysql"
+# "pg", "graphite", "mysql", "bigquery" or "script"
 CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg")
 # Connection string for the database that is used to run queries against. Examples:
 # -- mysql:    CONNECTION_STRING = "Server=;User=;Pwd=;Database="
 # -- pg:       CONNECTION_STRING = "user= password= host= port=5439 dbname="
 # -- graphite: CONNECTION_STRING = {"url": "https://graphite.yourcompany.com", "auth": ["user", "password"], "verify": true}
 # -- bigquery: CONNECTION_STRING = {"serviceAccount" : "43242343247-fjdfakljr3r2@developer.gserviceaccount.com", "privateKey" : "/somewhere/23fjkfjdsfj21312-privatekey.p12", "projectId" : "myproject-123" }
 #    to obtain bigquery credentials follow the guidelines at https://developers.google.com/bigquery/authorization#service-accounts
+# -- script: CONNECTION_STRING = "PATH TO ALL SCRIPTS"  (.i.e /home/user/redash_scripts/)
+#    all scripts must be have the executable flag set and reside in the path configured in CONNECTION_STRING.
+#    The output of the scripts must be in the output format defined here: <TODO: Add link to documentation of output format>
+# -- url: CONNECTION_STRING = "base URL" (i.e. http://myserver/somewhere)
+#    If CONNECTION_STRING is set, the query should be a relative URL. If it is not set a full URL can be used
 CONNECTION_STRING = os.environ.get("REDASH_CONNECTION_STRING", "user= password= host= port=5439 dbname=")
 
 # Connection settings for re:dash's own database (where we store the queries, results, etc)