Docker based developer workflow

.dockerignore

@@ -1,4 +1,4 @@
 client/.tmp/
-client/node_modules/
+node_modules/
+.tmp/
 .git/
-.vagrant/

.gitignore

@@ -20,9 +20,6 @@ venv
 
 dump.rdb
 
-# Docker related
-docker-compose.yml
-
 node_modules
 .tmp
 .sass-cache

Dockerfile

@@ -1,51 +1,10 @@
-FROM ubuntu:trusty
+FROM redash/base:latest
 
-# Ubuntu packages
-RUN apt-get update && \
-  apt-get install -y python-pip python-dev curl build-essential pwgen libffi-dev sudo git-core wget \
-  # Postgres client
-  libpq-dev \
-  # Additional packages required for data sources:
-  libssl-dev libmysqlclient-dev freetds-dev libsasl2-dev && \
-  apt-get clean && \
-  rm -rf /var/lib/apt/lists/*
+# We first copy only the requirements file, to avoid rebuilding on every file
+# change.
+COPY requirements.txt requirements_dev.txt requirements_all_ds.txt ./
+RUN pip install -r requirements.txt -r requirements_dev.txt -r requirements_all_ds.txt
 
-# Users creation
-RUN useradd --system --comment " " --create-home redash
+COPY . ./
 
-# Pip requirements for all data source types
-RUN pip install -U setuptools==23.1.0 && \
-  pip install supervisor==3.1.2
-
-COPY . /opt/redash/current
-RUN chown -R redash /opt/redash/current
-
-# Setting working directory
-WORKDIR /opt/redash/current
-
-# Install project specific dependencies
-RUN pip install -r requirements_all_ds.txt && \
-  pip install -r requirements.txt
-
-RUN curl https://deb.nodesource.com/setup_4.x | bash - && \
-  apt-get install -y nodejs && \
-  sudo -u redash -H make deps && \
-  rm -rf node_modules client/node_modules /home/redash/.npm /home/redash/.cache && \
-  apt-get purge -y nodejs && \
-  apt-get clean && \
-  rm -rf /var/lib/apt/lists/*
-
-# Setup supervisord
-RUN mkdir -p /opt/redash/supervisord && \
-    mkdir -p /opt/redash/logs && \
-    cp /opt/redash/current/setup/docker/supervisord/supervisord.conf /opt/redash/supervisord/supervisord.conf
-
-# Fix permissions
-RUN chown -R redash /opt/redash
-
-# Expose ports
-EXPOSE 5000
-EXPOSE 9001
-
-# Startup script
-CMD ["supervisord", "-c", "/opt/redash/supervisord/supervisord.conf"]
+ENTRYPOINT ["/app/bin/docker-entrypoint"]

Makefile

@@ -11,7 +11,7 @@ deps:
 
 pack:
 	sed -ri "s/^__version__ = '([0-9.]*)'/__version__ = '$(FULL_VERSION)'/" redash/__init__.py
-	tar -zcv -f $(FILENAME) --exclude="optipng*" --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="node_modules" --exclude="client/app" *
+	tar -zcv -f $(FILENAME) --exclude="optipng*" --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="node_modules" *
 
 upload:
 	python bin/release_manager.py $(CIRCLE_SHA1) $(BASE_VERSION) $(FILENAME)

bin/docker-entrypoint

@@ -0,0 +1,73 @@
+#!/bin/bash
+set -e
+
+worker() {
+  WORKERS_COUNT=${WORKERS_COUNT:-2}
+  QUEUES=${QUEUES:-queries,scheduled_queries,celery}
+
+  echo "Starting $WORKERS_COUNT workers for queues: $QUEUES..."
+  exec sudo -E -u redash /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair
+}
+
+scheduler() {
+  WORKERS_COUNT=${WORKERS_COUNT:-1}
+  QUEUES=${QUEUES:-celery}
+
+  echo "Starting scheduler and $WORKERS_COUNT workers for queues: $QUEUES..."
+
+  exec sudo -E -u redash /usr/local/bin/celery worker --app=redash.worker --beat -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair
+}
+
+server() {
+  exec sudo -E -u redash /usr/local/bin/gunicorn -b 0.0.0.0:5000 --name redash -w4 redash:app
+}
+
+help() {
+  echo "Redash Docker."
+  echo ""
+  echo "Usage:"
+  echo ""
+
+  echo "server -- start Redash server (with gunicorn)"
+  echo "worker -- start Celery worker"
+  echo "scheduler -- start Celery worker with a beat (scheduler) process"
+  echo ""
+  echo "shell -- open shell"
+  echo "dev_server -- start Flask development server with debugger and auto reload"
+  echo "create_db -- create database tables"
+}
+
+tests() {
+  export REDASH_DATABASE_URL="postgresql://postgres@postgres/tests"
+  exec sudo -E -u redash make test
+}
+
+case "$1" in
+  worker)
+    shift
+    worker
+    ;;
+  server)
+    shift
+    server
+    ;;
+  scheduler)
+    shift
+    scheduler
+    ;;
+  dev_server)
+    exec sudo -E -u redash /app/manage.py runserver --debugger --reload -h 0.0.0.0
+    ;;
+  shell)
+    exec sudo -E -u redash /app/manage.py shell
+    ;;
+  create_db)
+    exec sudo -E -u redash /app/manage.py database create_tables
+    ;;
+  tests)
+    tests
+    ;;
+  *)
+    help
+    ;;
+esac

client/app/components/app-header/app-header.css

@@ -0,0 +1,7 @@
+.menu-search {
+    margin-top: 3px;
+}
+
+.menu-search input[type="text"] {
+    height: 30px;
+}

client/app/components/app-header/app-header.html

@@ -37,10 +37,12 @@
         </li>
       </ul>
       <form class="navbar-form navbar-left" role="search" ng-submit="$ctrl.searchQueries()">
-        <div class="form-group">
+        <div class="input-group menu-search">
           <input type="text" ng-model="$ctrl.term" class="form-control" placeholder="Search queries...">
+          <span class="input-group-btn">
+            <button type="submit" class="btn btn-default"><span class="zmdi zmdi-search"></span></button>
+          </span>
         </div>
-        <button type="submit" class="btn btn-default"><span class="zmdi zmdi-search"></span></button>
       </form>
       <ul class="nav navbar-nav navbar-right">
         <li ng-show="$ctrl.currentUser.isAdmin">

client/app/components/app-header/index.js

@@ -2,6 +2,7 @@ import debug from 'debug';
 
 import template from './app-header.html';
 import logoUrl from '../../assets/images/redash_icon_small.png';
+import './app-header.css';
 
 const logger = debug('redash:appHeader');
 

client/app/index.js

@@ -107,7 +107,7 @@ ngModule.config(($routeProvider, $locationProvider, $compileProvider,
 });
 
 // Update ui-select's template to use Font-Awesome instead of glyphicon.
-ngModule.run(($templateCache) => {
+ngModule.run(($templateCache, OfflineListener) => { // eslint-disable-line no-unused-vars
   const templateName = 'bootstrap/match.tpl.html';
   let template = $templateCache.get(templateName);
   template = template.replace('glyphicon glyphicon-remove', 'fa fa-remove');

client/app/services/index.js

@@ -12,5 +12,6 @@ export { default as DataSource } from './data-source';
 export { default as QuerySnippet } from './query-snippet';
 export { default as Notifications } from './notifications';
 export { default as KeyboardShortcuts } from './keyboard-shortcuts';
+export { default as OfflineListener } from './offline-listener';
 export { default as AlertDialog } from './alert-dialog';
 export { default as Auth } from './auth';

client/app/services/offline-listener.js

@@ -0,0 +1,23 @@
+function OfflineListener(toastr) {
+  function addOnlineListener(toast) {
+    function onlineStateHandler() {
+      toastr.remove(toast.toastId);
+      window.removeEventListener('online', onlineStateHandler);
+    }
+    window.addEventListener('online', onlineStateHandler);
+  }
+
+  window.addEventListener('offline', () => {
+    const toast = toastr.warning('<div>Please check your Internet connection.</div>', '', {
+      allowHtml: true,
+      autoDismiss: false,
+      timeOut: false,
+      tapToDismiss: true,
+    });
+    addOnlineListener(toast);
+  });
+}
+
+export default function (ngModule) {
+  ngModule.service('OfflineListener', OfflineListener);
+}

docker-compose.production.yml

@@ -0,0 +1,44 @@
+# This is an example configuration for Docker Compose. Make sure to atleast update
+# the cookie secret & postgres database password.
+#
+# Some other recommendations:
+# 1. To persist Postgres data, assign it a volume host location.
+# 2. Split the worker service to adhoc workers and scheduled queries workers.
+version: '2'
+services:
+  server:
+    build: .
+    command: server
+    depends_on:
+      - postgres
+      - redis
+    ports:
+      - "5000:5000"
+    environment:
+      PYTHONUNBUFFERED: 0
+      REDASH_LOG_LEVEL: "INFO"
+      REDASH_REDIS_URL: "redis://redis:6379/0"
+      REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres"
+      REDASH_COOKIE_SECRET: veryverysecret
+  worker:
+    build: .
+    command: scheduler
+    environment:
+      PYTHONUNBUFFERED: 0
+      REDASH_LOG_LEVEL: "INFO"
+      REDASH_REDIS_URL: "redis://redis:6379/0"
+      REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres"
+      QUEUES: "queries,scheduled_queries,celery"
+      WORKERS_COUNT: 2
+  redis:
+    image: redis:2.8
+  postgres:
+    image: postgres:9.3
+    # volumes:
+    #   - /opt/postgres-data:/var/lib/postgresql/data
+  nginx:
+    image: redash/nginx:latest
+    ports:
+      - "80:80"
+    depends_on:
+      - server

docker-compose.yml

@@ -0,0 +1,41 @@
+# This configuration file is for **development** setup. For production, refer to
+# docker-compose.production.yml.
+version: '2'
+services:
+  server:
+    build: .
+    command: dev_server
+    depends_on:
+      - postgres
+      - redis
+    ports:
+      - "5000:5000"
+    volumes:
+      - ".:/app"
+    environment:
+      PYTHONUNBUFFERED: 0
+      REDASH_LOG_LEVEL: "INFO"
+      REDASH_REDIS_URL: "redis://redis:6379/0"
+      REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres"
+  worker:
+    build: .
+    command: scheduler
+    volumes_from:
+      - server
+    depends_on:
+      - server
+    environment:
+      PYTHONUNBUFFERED: 0
+      REDASH_LOG_LEVEL: "INFO"
+      REDASH_REDIS_URL: "redis://redis:6379/0"
+      REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres"
+      QUEUES: "queries,scheduled_queries,celery"
+      WORKERS_COUNT: 2
+  redis:
+    image: redis:2.8
+  postgres:
+    image: postgres:9.3
+    # The following turns the DB into less durable, but gains significant performance improvements for the tests run (x3
+    # improvement on my personal machine). We should consider moving this into a dedicated Docker Compose configuration for
+    # tests.
+    command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF"

redash/__init__.py

@@ -25,7 +25,12 @@ def setup_logging():
     handler.setFormatter(formatter)
     logging.getLogger().addHandler(handler)
     logging.getLogger().setLevel(settings.LOG_LEVEL)
-    logging.getLogger("passlib").setLevel("ERROR")
+
+    # Make noisy libraries less noisy
+    if settings.LOG_LEVEL != "DEBUG":
+        logging.getLogger("passlib").setLevel("ERROR")
+        logging.getLogger("requests.packages.urllib3").setLevel("ERROR")
+        logging.getLogger("snowflake.connector").setLevel("ERROR")
 
 
 def create_redis_connection():

redash/cli/database.py

@@ -6,14 +6,12 @@
 @manager.command()
 def create_tables():
     """Create the database tables."""
-    from redash.models import db, init_db
+    from redash.models import db
     db.create_all()
 
     # Need to mark current DB as up to date
     stamp()
 
-    init_db()
-
 
 @manager.command()
 def drop_tables():

redash/handlers/__init__.py

@@ -21,6 +21,6 @@ def status_api():
 
 
 def init_app(app):
-    from redash.handlers import embed, queries, static, authentication, admin
+    from redash.handlers import embed, queries, static, authentication, admin, setup
     app.register_blueprint(routes)
     api.init_app(app)

redash/handlers/authentication.py

@@ -92,6 +92,13 @@ def forgot_password(org_slug=None):
 @routes.route(org_scoped_rule('/login'), methods=['GET', 'POST'])
 @limiter.limit(settings.THROTTLE_LOGIN_PATTERN)
 def login(org_slug=None):
+    # We intentionally use == as otherwise it won't actually use the proxy. So weird :O
+    # noinspection PyComparisonWithNone
+    if current_org == None and not settings.MULTI_ORG:
+        return redirect('/setup')
+    elif current_org == None:
+        return redirect('/')
+
     index_url = url_for("redash.index", org_slug=org_slug)
     next_path = request.args.get('next', index_url)
     if current_user.is_authenticated:

redash/handlers/setup.py

@@ -0,0 +1,56 @@
+from flask import redirect, request, render_template, url_for, g
+from flask_login import login_user
+from wtforms import Form, PasswordField, StringField, BooleanField, validators
+from wtforms.fields.html5 import EmailField
+
+from redash import settings
+from redash.tasks.general import subscribe
+from redash.handlers.base import routes
+from redash.models import Organization, Group, User, db
+from redash.authentication.org_resolving import current_org
+
+
+class SetupForm(Form):
+    name = StringField('Name', validators=[validators.InputRequired()])
+    email = EmailField('Email Address', validators=[validators.Email()])
+    password = PasswordField('Password', validators=[validators.Length(6)])
+    org_name = StringField("Organization Name", validators=[validators.InputRequired()])
+    security_notifications = BooleanField()
+    newsletter = BooleanField()
+
+
+@routes.route('/setup', methods=['GET', 'POST'])
+def setup():
+    if current_org != None or settings.MULTI_ORG:
+        return redirect('/')
+
+    form = SetupForm(request.form)
+    form.newsletter.data = True
+    form.security_notifications.data = True
+
+    if request.method == 'POST' and form.validate():
+        default_org = Organization(name=form.org_name.data, slug='default', settings={})
+        admin_group = Group(name='admin', permissions=['admin', 'super_admin'], org=default_org, type=Group.BUILTIN_GROUP)
+        default_group = Group(name='default', permissions=Group.DEFAULT_PERMISSIONS, org=default_org, type=Group.BUILTIN_GROUP)
+
+        db.session.add_all([default_org, admin_group, default_group])
+        db.session.commit()
+
+        user = User(org=default_org, name=form.name.data, email=form.email.data, group_ids=[admin_group.id, default_group.id])
+        user.hash_password(form.password.data)
+
+        db.session.add(user)
+        db.session.commit()
+
+        g.org = default_org
+        login_user(user)
+
+        # signup to newsletter if needed
+        if form.newsletter.data or form.security_notifications:
+            subscribe.delay(form.data)
+
+        return redirect(url_for('redash.index', org_slug=None))
+
+    return render_template('setup.html', form=form)
+
+

redash/tasks/general.py

@@ -29,6 +29,19 @@ def version_check():
     run_version_check()
 
 
+@celery.task(name="redash.tasks.subscribe")
+def subscribe(form):
+    logger.info("Subscribing to: [security notifications=%s], [newsletter=%s]", form['security_notifications'], form['newsletter'])
+    data = {
+        'admin_name': form['name'],
+        'admin_email': form['email'],
+        'org_name': form['org_name'],
+        'security_notifications': form['security_notifications'],
+        'newsletter': form['newsletter']
+    }
+    requests.post('https://beacon.redash.io/subscribe', json=data)
+
+
 @celery.task(name="redash.tasks.send_mail", base=BaseTask)
 def send_mail(to, subject, html, text):
     from redash.wsgi import app

redash/templates/setup.html

@@ -0,0 +1,64 @@
+{% extends "layouts/signed_out.html" %}
+
+{% block title %}Redash Initial Setup{% endblock %}
+
+{% macro render_field_errors(field) -%}
+    {% if field.errors %}
+        {% for e in field.errors %}
+            <p class="help-block">{{ e }}</p>
+        {% endfor %}
+    {% endif %}
+{%- endmacro %}
+
+{% macro render_field(field, help_block=None) -%}
+    <div class="form-group {% if field.errors %}has-error{% endif %}">
+        {{ field.label() }}
+        {{ field(class='form-control') }}
+        {% if help_block %}
+            <p class="help-block">{{ help_block }}</p>
+        {% endif %}
+        {{ render_field_errors(field) }}
+    </div>
+{%- endmacro %}
+
+{% block content %}
+    <h2>
+        Welcome to Redash!<br/><small>Before you can use your instance, you need to do a quick setup.</small>
+    </h2>
+
+    {% with messages = get_flashed_messages() %}
+        {% if messages %}
+            {% for message in messages %}
+                <div class="alert alert-warning" role="alert">{{ message }}</div>
+            {% endfor %}
+        {% endif %}
+    {% endwith %}
+
+    <form role="form" method="post" name="create_account">
+        <h3>Admin User</h3>
+        {{ render_field(form.name) }}
+        {{ render_field(form.email) }}
+        {{ render_field(form.password) }}
+
+        <div class="checkbox">
+            <label>
+                {{ form.security_notifications() }}
+                Subscribe to Security Notifications
+            </label>
+        </div>
+
+        <div class="checkbox">
+            <label>
+                {{ form.newsletter() }}
+                Subscribe to newsletter (version updates, no more than once a month)
+            </label>
+        </div>
+
+        <h3>General</h3>
+
+        {{ render_field(form.org_name, help_block="Used in email notifications and the UI.") }}
+
+        <button type="submit" class="btn btn btn-primary">Setup</button>
+    </form>
+{% endblock %}
+

tests/__init__.py

@@ -5,9 +5,9 @@
 from unittest import TestCase
 from contextlib import contextmanager
 
-os.environ['REDASH_REDIS_URL'] = "redis://localhost:6379/5"
+os.environ['REDASH_REDIS_URL'] = os.environ.get('REDASH_REDIS_URL', "redis://localhost:6379/0").replace("/0", "/5")
 # Use different url for Celery to avoid DB being cleaned up:
-os.environ['REDASH_CELERY_BROKER'] = "redis://localhost:6379/6"
+os.environ['REDASH_CELERY_BROKER'] = os.environ.get('REDASH_REDIS_URL', "redis://localhost:6379/0").replace("/5", "/6")
 
 # Dummy values for oauth login
 os.environ['REDASH_GOOGLE_CLIENT_ID'] = "dummy"

webpack.config.js

@@ -86,6 +86,10 @@ var config = {
         target: redashBackend + '/',
         secure: false
       },
+      '/setup': {
+        target: redashBackend + '/',
+        secure: false
+      },
       '/images': {
         target: redashBackend + '/',
         secure: false