Merge pull request #1530 from getredash/docker_workflow

Docker based developer workflow

Author: arikfr

.dockerignore

@@ -1,4 +1,4 @@
 client/.tmp/
-client/node_modules/
+node_modules/
+.tmp/
 .git/
-.vagrant/

.gitignore

@@ -20,9 +20,6 @@ venv
 
 dump.rdb
 
-# Docker related
-docker-compose.yml
-
 node_modules
 .tmp
 .sass-cache

Dockerfile

@@ -1,51 +1,10 @@
-FROM ubuntu:trusty
+FROM redash/base:latest
 
-# Ubuntu packages
-RUN apt-get update && \
-  apt-get install -y python-pip python-dev curl build-essential pwgen libffi-dev sudo git-core wget \
-  # Postgres client
-  libpq-dev \
-  # Additional packages required for data sources:
-  libssl-dev libmysqlclient-dev freetds-dev libsasl2-dev && \
-  apt-get clean && \
-  rm -rf /var/lib/apt/lists/*
+# We first copy only the requirements file, to avoid rebuilding on every file
+# change.
+COPY requirements.txt requirements_dev.txt requirements_all_ds.txt ./
+RUN pip install -r requirements.txt -r requirements_dev.txt -r requirements_all_ds.txt
 
-# Users creation
-RUN useradd --system --comment " " --create-home redash
+COPY . ./
 
-# Pip requirements for all data source types
-RUN pip install -U setuptools==23.1.0 && \
-  pip install supervisor==3.1.2
-
-COPY . /opt/redash/current
-RUN chown -R redash /opt/redash/current
-
-# Setting working directory
-WORKDIR /opt/redash/current
-
-# Install project specific dependencies
-RUN pip install -r requirements_all_ds.txt && \
-  pip install -r requirements.txt
-
-RUN curl https://deb.nodesource.com/setup_4.x | bash - && \
-  apt-get install -y nodejs && \
-  sudo -u redash -H make deps && \
-  rm -rf node_modules client/node_modules /home/redash/.npm /home/redash/.cache && \
-  apt-get purge -y nodejs && \
-  apt-get clean && \
-  rm -rf /var/lib/apt/lists/*
-
-# Setup supervisord
-RUN mkdir -p /opt/redash/supervisord && \
-    mkdir -p /opt/redash/logs && \
-    cp /opt/redash/current/setup/docker/supervisord/supervisord.conf /opt/redash/supervisord/supervisord.conf
-
-# Fix permissions
-RUN chown -R redash /opt/redash
-
-# Expose ports
-EXPOSE 5000
-EXPOSE 9001
-
-# Startup script
-CMD ["supervisord", "-c", "/opt/redash/supervisord/supervisord.conf"]
+ENTRYPOINT ["/app/bin/docker-entrypoint"]

Makefile

@@ -11,7 +11,7 @@ deps:
 
 pack:
 	sed -ri "s/^__version__ = '([0-9.]*)'/__version__ = '$(FULL_VERSION)'/" redash/__init__.py
-	tar -zcv -f $(FILENAME) --exclude="optipng*" --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="node_modules" --exclude="client/app" *
+	tar -zcv -f $(FILENAME) --exclude="optipng*" --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="node_modules" *
 
 upload:
 	python bin/release_manager.py $(CIRCLE_SHA1) $(BASE_VERSION) $(FILENAME)

Procfile.dev

@@ -0,0 +1,73 @@
+#!/bin/bash
+set -e
+
+worker() {
+  WORKERS_COUNT=${WORKERS_COUNT:-2}
+  QUEUES=${QUEUES:-queries,scheduled_queries,celery}
+
+  echo "Starting $WORKERS_COUNT workers for queues: $QUEUES..."
+  exec sudo -E -u redash /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair
+}
+
+scheduler() {
+  WORKERS_COUNT=${WORKERS_COUNT:-1}
+  QUEUES=${QUEUES:-celery}
+
+  echo "Starting scheduler and $WORKERS_COUNT workers for queues: $QUEUES..."
+
+  exec sudo -E -u redash /usr/local/bin/celery worker --app=redash.worker --beat -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair
+}
+
+server() {
+  exec sudo -E -u redash /usr/local/bin/gunicorn -b 0.0.0.0:5000 --name redash -w4 redash:app
+}
+
+help() {
+  echo "Redash Docker."
+  echo ""
+  echo "Usage:"
+  echo ""
+
+  echo "server -- start Redash server (with gunicorn)"
+  echo "worker -- start Celery worker"
+  echo "scheduler -- start Celery worker with a beat (scheduler) process"
+  echo ""
+  echo "shell -- open shell"
+  echo "dev_server -- start Flask development server with debugger and auto reload"
+  echo "create_db -- create database tables"
+}
+
+tests() {
+  export REDASH_DATABASE_URL="postgresql://postgres@postgres/tests"
+  exec sudo -E -u redash make test
+}
+
+case "$1" in
+  worker)
+    shift
+    worker
+    ;;
+  server)
+    shift
+    server
+    ;;
+  scheduler)
+    shift
+    scheduler
+    ;;
+  dev_server)
+    exec sudo -E -u redash /app/manage.py runserver --debugger --reload -h 0.0.0.0
+    ;;
+  shell)
+    exec sudo -E -u redash /app/manage.py shell
+    ;;
+  create_db)
+    exec sudo -E -u redash /app/manage.py database create_tables
+    ;;
+  tests)
+    tests
+    ;;
+  *)
+    help
+    ;;
+esac

Vagrantfile

@@ -0,0 +1,7 @@
+.menu-search {
+    margin-top: 3px;
+}
+
+.menu-search input[type="text"] {
+    height: 30px;
+}

bin/docker-entrypoint

@@ -37,10 +37,12 @@
         </li>
       </ul>
       <form class="navbar-form navbar-left" role="search" ng-submit="$ctrl.searchQueries()">
-        <div class="form-group">
+        <div class="input-group menu-search">
           <input type="text" ng-model="$ctrl.term" class="form-control" placeholder="Search queries...">
+          <span class="input-group-btn">
+            <button type="submit" class="btn btn-default"><span class="zmdi zmdi-search"></span></button>
+          </span>
         </div>
-        <button type="submit" class="btn btn-default"><span class="zmdi zmdi-search"></span></button>
       </form>
       <ul class="nav navbar-nav navbar-right">
         <li ng-show="$ctrl.currentUser.isAdmin">

bin/vagrant_ctl.sh

@@ -2,6 +2,7 @@ import debug from 'debug';
 
 import template from './app-header.html';
 import logoUrl from '../../assets/images/redash_icon_small.png';
+import './app-header.css';
 
 const logger = debug('redash:appHeader');
 

client/app/components/app-header/app-header.css

@@ -107,7 +107,7 @@ ngModule.config(($routeProvider, $locationProvider, $compileProvider,
 });
 
 // Update ui-select's template to use Font-Awesome instead of glyphicon.
-ngModule.run(($templateCache) => {
+ngModule.run(($templateCache, OfflineListener) => { // eslint-disable-line no-unused-vars
   const templateName = 'bootstrap/match.tpl.html';
   let template = $templateCache.get(templateName);
   template = template.replace('glyphicon glyphicon-remove', 'fa fa-remove');

client/app/components/app-header/app-header.html

@@ -12,5 +12,6 @@ export { default as DataSource } from './data-source';
 export { default as QuerySnippet } from './query-snippet';
 export { default as Notifications } from './notifications';
 export { default as KeyboardShortcuts } from './keyboard-shortcuts';
+export { default as OfflineListener } from './offline-listener';
 export { default as AlertDialog } from './alert-dialog';
 export { default as Auth } from './auth';

client/app/components/app-header/index.js

@@ -0,0 +1,23 @@
+function OfflineListener(toastr) {
+  function addOnlineListener(toast) {
+    function onlineStateHandler() {
+      toastr.remove(toast.toastId);
+      window.removeEventListener('online', onlineStateHandler);
+    }
+    window.addEventListener('online', onlineStateHandler);
+  }
+
+  window.addEventListener('offline', () => {
+    const toast = toastr.warning('<div>Please check your Internet connection.</div>', '', {
+      allowHtml: true,
+      autoDismiss: false,
+      timeOut: false,
+      tapToDismiss: true,
+    });
+    addOnlineListener(toast);
+  });
+}
+
+export default function (ngModule) {
+  ngModule.service('OfflineListener', OfflineListener);
+}

client/app/index.js

@@ -0,0 +1,44 @@
+# This is an example configuration for Docker Compose. Make sure to atleast update
+# the cookie secret & postgres database password.
+#
+# Some other recommendations:
+# 1. To persist Postgres data, assign it a volume host location.
+# 2. Split the worker service to adhoc workers and scheduled queries workers.
+version: '2'
+services:
+  server:
+    build: .
+    command: server
+    depends_on:
+      - postgres
+      - redis
+    ports:
+      - "5000:5000"
+    environment:
+      PYTHONUNBUFFERED: 0
+      REDASH_LOG_LEVEL: "INFO"
+      REDASH_REDIS_URL: "redis://redis:6379/0"
+      REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres"
+      REDASH_COOKIE_SECRET: veryverysecret
+  worker:
+    build: .
+    command: scheduler
+    environment:
+      PYTHONUNBUFFERED: 0
+      REDASH_LOG_LEVEL: "INFO"
+      REDASH_REDIS_URL: "redis://redis:6379/0"
+      REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres"
+      QUEUES: "queries,scheduled_queries,celery"
+      WORKERS_COUNT: 2
+  redis:
+    image: redis:2.8
+  postgres:
+    image: postgres:9.3
+    # volumes:
+    #   - /opt/postgres-data:/var/lib/postgresql/data
+  nginx:
+    image: redash/nginx:latest
+    ports:
+      - "80:80"
+    depends_on:
+      - server

client/app/services/index.js

@@ -0,0 +1,41 @@
+# This configuration file is for **development** setup. For production, refer to
+# docker-compose.production.yml.
+version: '2'
+services:
+  server:
+    build: .
+    command: dev_server
+    depends_on:
+      - postgres
+      - redis
+    ports:
+      - "5000:5000"
+    volumes:
+      - ".:/app"
+    environment:
+      PYTHONUNBUFFERED: 0
+      REDASH_LOG_LEVEL: "INFO"
+      REDASH_REDIS_URL: "redis://redis:6379/0"
+      REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres"
+  worker:
+    build: .
+    command: scheduler
+    volumes_from:
+      - server
+    depends_on:
+      - server
+    environment:
+      PYTHONUNBUFFERED: 0
+      REDASH_LOG_LEVEL: "INFO"
+      REDASH_REDIS_URL: "redis://redis:6379/0"
+      REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres"
+      QUEUES: "queries,scheduled_queries,celery"
+      WORKERS_COUNT: 2
+  redis:
+    image: redis:2.8
+  postgres:
+    image: postgres:9.3
+    # The following turns the DB into less durable, but gains significant performance improvements for the tests run (x3
+    # improvement on my personal machine). We should consider moving this into a dedicated Docker Compose configuration for
+    # tests.
+    command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF"

client/app/services/offline-listener.js

@@ -25,7 +25,12 @@ def setup_logging():
     handler.setFormatter(formatter)
     logging.getLogger().addHandler(handler)
     logging.getLogger().setLevel(settings.LOG_LEVEL)
-    logging.getLogger("passlib").setLevel("ERROR")
+
+    # Make noisy libraries less noisy
+    if settings.LOG_LEVEL != "DEBUG":
+        logging.getLogger("passlib").setLevel("ERROR")
+        logging.getLogger("requests.packages.urllib3").setLevel("ERROR")
+        logging.getLogger("snowflake.connector").setLevel("ERROR")
 
 
 def create_redis_connection():