From d435d122eb26142220d1a42637a0e86e9e9eacd3 Mon Sep 17 00:00:00 2001 From: Eran Sandler Date: Mon, 17 Mar 2014 16:36:51 +0200 Subject: [PATCH 1/3] Added support for running scripts as queries --- redash/data/manager.py | 3 ++ redash/data/query_runner_script.py | 51 ++++++++++++++++++++++++++++++ redash/settings.py | 5 ++- 3 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 redash/data/query_runner_script.py diff --git a/redash/data/manager.py b/redash/data/manager.py index 55a10066d1..93499eed40 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -185,6 +185,9 @@ def start_workers(self, workers_count, connection_type, connection_string): from redash.data import query_runner_bigquery connection_params = json.loads(connection_string) runner = query_runner_bigquery.bigquery(connection_params) + elif connection_type == 'script': + from redash.data import query_runner_script + runner = query_runner_script.script(connection_string) else: from redash.data import query_runner runner = query_runner.redshift(connection_string) diff --git a/redash/data/query_runner_script.py b/redash/data/query_runner_script.py new file mode 100644 index 0000000000..902bd6f0e5 --- /dev/null +++ b/redash/data/query_runner_script.py @@ -0,0 +1,51 @@ +import json +import logging +import sys +import os +import subprocess + +# We use subprocess.check_output because we are lazy. +# If someone will really want to run this on Python < 2.7 they can easily update the code to run +# Popen, check the retcodes and other things and read the standard output to a variable. +if not "check_output" in subprocess.__dict__: + print "ERROR: This runner uses subprocess.check_output function which exists in Python 2.7" + +def script(connection_string): + + def query_runner(query): + try: + json_data = None + error = None + + # Remove the SQL comment that Redash adds + if query.find("/*") > -1 and query.find("*/") > -1: + query = query[query.find("*/")+3:] + + # Poor man's protection against running scripts from output the scripts directory + if connection_string.find("../") > -1: + return None, "Scripts can only be run from the configured scripts directory" + + query = query.strip() + + script = os.path.join(connection_string, query) + if not os.path.exists(script): + return None, "Script '%s' not found in script directory" % query + + output = subprocess.check_output(script, shell=False) + if output != None: + output = output.strip() + if output != "": + return output, None + + error = "Error reading output" + except subprocess.CalledProcessError as e: + return None, str(e) + except KeyboardInterrupt: + error = "Query cancelled by user." + json_data = None + except Exception as e: + raise sys.exc_info()[1], None, sys.exc_info()[2] + + return json_data, error + + return query_runner diff --git a/redash/settings.py b/redash/settings.py index b454afa736..d28575effb 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -46,7 +46,7 @@ def parse_boolean(str): NAME = os.environ.get('REDASH_NAME', 're:dash') -# "pg", "graphite" or "mysql" +# "pg", "graphite", "mysql", "bigquery" or "script" CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg") # Connection string for the database that is used to run queries against. Examples: # -- mysql: CONNECTION_STRING = "Server=;User=;Pwd=;Database=" @@ -54,6 +54,9 @@ def parse_boolean(str): # -- graphite: CONNECTION_STRING = {"url": "https://graphite.yourcompany.com", "auth": ["user", "password"], "verify": true} # -- bigquery: CONNECTION_STRING = {"serviceAccount" : "43242343247-fjdfakljr3r2@developer.gserviceaccount.com", "privateKey" : "/somewhere/23fjkfjdsfj21312-privatekey.p12", "projectId" : "myproject-123" } # to obtain bigquery credentials follow the guidelines at https://developers.google.com/bigquery/authorization#service-accounts +# -- script: CONNECTION_STRING = "PATH TO ALL SCRIPTS" (.i.e /home/user/redash_scripts/) +# all scripts must be have the executable flag set and reside in the path configured in CONNECTION_STRING. +# The output of the scripts must be in the output format defined here: CONNECTION_STRING = os.environ.get("REDASH_CONNECTION_STRING", "user= password= host= port=5439 dbname=") # Connection settings for re:dash's own database (where we store the queries, results, etc) From d3e87a3d28bd010d0e45a8733d0b189caebe8c8c Mon Sep 17 00:00:00 2001 From: Eran Sandler Date: Mon, 17 Mar 2014 18:44:31 +0200 Subject: [PATCH 2/3] added support for a 'url' source where you can supply a URL to retrieve the same JSON result used in other query runners --- redash/data/manager.py | 3 +++ redash/data/query_runner_url.py | 48 +++++++++++++++++++++++++++++++++ redash/settings.py | 2 ++ 3 files changed, 53 insertions(+) create mode 100644 redash/data/query_runner_url.py diff --git a/redash/data/manager.py b/redash/data/manager.py index 93499eed40..5530c76a1c 100644 --- a/redash/data/manager.py +++ b/redash/data/manager.py @@ -188,6 +188,9 @@ def start_workers(self, workers_count, connection_type, connection_string): elif connection_type == 'script': from redash.data import query_runner_script runner = query_runner_script.script(connection_string) + elif connection_type == 'url': + from redash.data import query_runner_url + runner = query_runner_url.url(connection_string) else: from redash.data import query_runner runner = query_runner.redshift(connection_string) diff --git a/redash/data/query_runner_url.py b/redash/data/query_runner_url.py new file mode 100644 index 0000000000..1cb57f17b7 --- /dev/null +++ b/redash/data/query_runner_url.py @@ -0,0 +1,48 @@ +import json +import logging +import sys +import os +import urllib2 + +def url(connection_string): + + def query_runner(query): + base_url = connection_string + + try: + json_data = None + error = None + + query = query.strip() + + # Remove the SQL comment that Redash adds + if query.find("/*") > -1 and query.find("*/") > -1: + query = query[query.find("*/")+3:] + + if base_url is not None and base_url != "": + if query.find("://") > -1: + return None, "Accepting only relative URLs to '%s'" % base_url + + if base_url is None: + base_url = "" + + url = base_url + query + + json_data = urllib2.urlopen(url).read().strip() + + if not json_data: + error = "Error reading data from '%s'" % url + + return json_data, error + + except urllib2.URLError as e: + return None, str(e) + except KeyboardInterrupt: + error = "Query cancelled by user." + json_data = None + except Exception as e: + raise sys.exc_info()[1], None, sys.exc_info()[2] + + return json_data, error + + return query_runner diff --git a/redash/settings.py b/redash/settings.py index d28575effb..a931085bf8 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -57,6 +57,8 @@ def parse_boolean(str): # -- script: CONNECTION_STRING = "PATH TO ALL SCRIPTS" (.i.e /home/user/redash_scripts/) # all scripts must be have the executable flag set and reside in the path configured in CONNECTION_STRING. # The output of the scripts must be in the output format defined here: +# -- url: CONNECTION_STRING = "base URL" (i.e. http://myserver/somewhere) +# If CONNECTION_STRING is set, the query should be a relative URL. If it is not set a full URL can be used CONNECTION_STRING = os.environ.get("REDASH_CONNECTION_STRING", "user= password= host= port=5439 dbname=") # Connection settings for re:dash's own database (where we store the queries, results, etc) From a2257999a7bf2fde4ec084ac974619160c0c5e4d Mon Sep 17 00:00:00 2001 From: Eran Sandler Date: Mon, 17 Mar 2014 18:56:50 +0200 Subject: [PATCH 3/3] moved to use the query_runner.annotate_query flag so we won't get the SQL comment --- redash/data/query_runner_script.py | 5 +---- redash/data/query_runner_url.py | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/redash/data/query_runner_script.py b/redash/data/query_runner_script.py index 902bd6f0e5..dbee7c780a 100644 --- a/redash/data/query_runner_script.py +++ b/redash/data/query_runner_script.py @@ -17,10 +17,6 @@ def query_runner(query): json_data = None error = None - # Remove the SQL comment that Redash adds - if query.find("/*") > -1 and query.find("*/") > -1: - query = query[query.find("*/")+3:] - # Poor man's protection against running scripts from output the scripts directory if connection_string.find("../") > -1: return None, "Scripts can only be run from the configured scripts directory" @@ -48,4 +44,5 @@ def query_runner(query): return json_data, error + query_runner.annotate_query = False return query_runner diff --git a/redash/data/query_runner_url.py b/redash/data/query_runner_url.py index 1cb57f17b7..64f146bd54 100644 --- a/redash/data/query_runner_url.py +++ b/redash/data/query_runner_url.py @@ -15,10 +15,6 @@ def query_runner(query): query = query.strip() - # Remove the SQL comment that Redash adds - if query.find("/*") > -1 and query.find("*/") > -1: - query = query[query.find("*/")+3:] - if base_url is not None and base_url != "": if query.find("://") > -1: return None, "Accepting only relative URLs to '%s'" % base_url @@ -45,4 +41,5 @@ def query_runner(query): return json_data, error + query_runner.annotate_query = False return query_runner