Merge pull request #380 from EverythingMe/feature/datasources_v2

Refactor datasources (query runners)

migrations/0003_update_data_source_config.py

@@ -0,0 +1,63 @@
+import json
+
+from redash import query_runner
+from redash.models import DataSource
+
+
+def update(data_source):
+    print "[%s] Old options: %s" % (data_source.name, data_source.options)
+
+    if query_runner.validate_configuration(data_source.type, data_source.options):
+        print "[%s] configuration already valid. skipping." % data_source.name
+        return
+
+    if data_source.type == 'pg':
+        values = data_source.options.split(" ")
+        configuration = {}
+        for value in values:
+            k, v = value.split("=", 1)
+            configuration[k] = v
+        data_source.options = json.dumps(configuration)
+
+    elif data_source.type == 'mysql':
+        values = data_source.options.split(";")
+        configuration = {}
+        for value in values:
+            k, v = value.split("=", 1)
+            configuration[k] = v
+        data_source.options = json.dumps(configuration)
+
+    elif data_source.type == 'graphite':
+        old_config = json.loads(data_source.options)
+
+        configuration = {
+            "url": old_config["url"]
+        }
+
+        if "verify" in old_config:
+            configuration['verify'] = old_config['verify']
+
+        if "auth" in old_config:
+            configuration['username'], configuration['password'] = old_config["auth"]
+
+        data_source.options = json.dumps(configuration)
+
+    elif data_source.type == 'url':
+        data_source.options = json.dumps({"url": data_source.options})
+
+    elif data_source.type == 'script':
+        data_source.options = json.dumps({"path": data_source.options})
+
+    elif data_source.type == 'mongo':
+        data_source.type = 'mongodb'
+
+    else:
+        print "[%s] No need to convert type of: %s" % (data_source.name, data_source.type)
+
+    print "[%s] New options: %s" % (data_source.name, data_source.options)
+    data_source.save()
+
+
+if __name__ == '__main__':
+    for data_source in DataSource.all():
+        update(data_source)

redash/__init__.py

@@ -4,8 +4,9 @@
 from statsd import StatsClient
 
 from redash import settings
+from redash.query_runner import import_query_runners
 
-__version__ = '0.5.0'
+__version__ = '0.6.0'
 
 
 def setup_logging():
@@ -31,4 +32,6 @@ def create_redis_connection():
 
 setup_logging()
 redis_connection = create_redis_connection()
-statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX)
+statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX)
+
+import_query_runners(settings.QUERY_RUNNERS)

redash/cli/data_sources.py

@@ -1,5 +1,8 @@
+import json
+import click
 from flask.ext.script import Manager
 from redash import models
+from redash.query_runner import query_runners, validate_configuration
 
 manager = Manager(help="Data sources management commands.")
 
@@ -13,11 +16,70 @@ def list():
         print "Id: {}\nName: {}\nType: {}\nOptions: {}".format(ds.id, ds.name, ds.type, ds.options)
 
 
+def validate_data_source_type(type):
+    if type not in query_runners.keys():
+        print "Error: the type \"{}\" is not supported (supported types: {}).".format(type, ", ".join(query_runners.keys()))
+        exit()
+
+
+def validate_data_source_options(type, options):
+    if not validate_configuration(type, options):
+        print "Error: invalid configuration."
+        exit()
+
 @manager.command
-def new(name, type, options):
+def new(name=None, type=None, options=None):
     """Create new data source"""
-    # TODO: validate it's a valid type and in the future, validate the options.
+    if name is None:
+        name = click.prompt("Name")
+
+    if type is None:
+        print "Select type:"
+        for i, query_runner_name in enumerate(query_runners.keys()):
+            print "{}. {}".format(i+1, query_runner_name)
+
+        idx = 0
+        while idx < 1 or idx > len(query_runners.keys()):
+            idx = click.prompt("[{}-{}]".format(1, len(query_runners.keys())), type=int)
+
+        type = query_runners.keys()[idx-1]
+    else:
+        validate_data_source_type(type)
+
+    if options is None:
+        query_runner = query_runners[type]
+        schema = query_runner.configuration_schema()
+
+        types = {
+            'string': unicode,
+            'number': int,
+            'boolean': bool
+        }
+
+        options_obj = {}
+
+        for k, prop in schema['properties'].iteritems():
+            required = k in schema.get('required', [])
+            default_value = "<<DEFAULT_VALUE>>"
+            if required:
+                default_value = None
+
+            prompt = prop.get('title', k.capitalize())
+            if required:
+                prompt = "{} (required)".format(prompt)
+            else:
+                prompt = "{} (optional)".format(prompt)
+
+            value = click.prompt(prompt, default=default_value, type=types[prop['type']], show_default=False)
+            if value != default_value:
+                options_obj[k] = value
+
+        options = json.dumps(options_obj)
+
+    validate_data_source_options(type, options)
+
     print "Creating {} data source ({}) with options:\n{}".format(type, name, options)
+
     data_source = models.DataSource.create(name=name,
                                            type=type,
                                            options=options)
@@ -49,7 +111,14 @@ def update_attr(obj, attr, new_value):
 def edit(name, new_name=None, options=None, type=None):
     """Edit data source settings (name, options, type)"""
     try:
+        if type is not None:
+            validate_data_source_type(type)
+
         data_source = models.DataSource.get(models.DataSource.name==name)
+
+        if options is not None:
+            validate_data_source_options(data_source.type, options)
+
         update_attr(data_source, "name", new_name)
         update_attr(data_source, "type", type)
         update_attr(data_source, "options", options)

redash/controllers.py

@@ -23,6 +23,7 @@
 from redash.tasks import QueryTask, record_event
 from redash.cache import headers as cache_headers
 from redash.permissions import require_permission
+from redash.query_runner import query_runners, validate_configuration
 
 
 @app.route('/ping', methods=['GET'])
@@ -174,11 +175,35 @@ def post(self):
 api.add_resource(MetricsAPI, '/api/metrics/v1/send', endpoint='metrics')
 
 
+class DataSourceTypeListAPI(BaseResource):
+    @require_permission("admin")
+    def get(self):
+        return [q.to_dict() for q in query_runners.values()]
+
+api.add_resource(DataSourceTypeListAPI, '/api/data_sources/types', endpoint='data_source_types')
+
+
 class DataSourceListAPI(BaseResource):
     def get(self):
         data_sources = [ds.to_dict() for ds in models.DataSource.all()]
         return data_sources
 
+    @require_permission("admin")
+    def post(self):
+        req = request.get_json(True)
+        required_fields = ('options', 'name', 'type')
+        for f in required_fields:
+            if f not in req:
+                abort(400)
+
+        if not validate_configuration(req['type'], req['options']):
+            abort(400)
+
+        datasource = models.DataSource.create(name=req['name'], type=req['type'], options=req['options'])
+
+        return datasource.to_dict()
+
+
 api.add_resource(DataSourceListAPI, '/api/data_sources', endpoint='data_sources')