From 08730ad11343766f9882c2172b0a023cfa7358e8 Mon Sep 17 00:00:00 2001 From: masaru Date: Fri, 31 Jul 2015 14:05:48 +0900 Subject: [PATCH 1/3] created bootstrap script for amazon linux --- setup/bootstrap_amazon_linux.sh | 199 ++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 setup/bootstrap_amazon_linux.sh diff --git a/setup/bootstrap_amazon_linux.sh b/setup/bootstrap_amazon_linux.sh new file mode 100644 index 0000000000..6c47f84f1e --- /dev/null +++ b/setup/bootstrap_amazon_linux.sh @@ -0,0 +1,199 @@ +#!/bin/bash +set -eu + +REDASH_BASE_PATH=/opt/redash +FILES_BASE_URL=https://raw.githubusercontent.com/EverythingMe/redash/docs_setup/setup/files/ + +# Verify running as root: +if [ "$(id -u)" != "0" ]; then + if [ $# -ne 0 ]; then + echo "Failed running with sudo. Exiting." 1>&2 + exit 1 + fi + echo "This script must be run as root. Trying to run with sudo." + sudo bash $0 --with-sudo + exit 0 +fi + +# Base packages +yum update +yum install -y python-pip python-devel nginx curl +yes | yum groupinstall -y "Development Tools" +yum install -y libffi-devel openssl-devel + +# redash user +# TODO: check user doesn't exist yet? +if [-x $(adduser --system --no-create-home --comment "" redash)]; then + echo "redash user have already registered." +fi +add_service() { + service_name=$1 + service_command="/etc/init.d/$service_name" + + echo "Adding service: $service_name (/etc/init.d/$service_name)." + chmod +x $service_command + + if command -v chkconfig >/dev/null 2>&1; then + # we're chkconfig, so lets add to chkconfig and put in runlevel 345 + chkconfig --add $service_name && echo "Successfully added to chkconfig!" + chkconfig --level 345 $service_name on && echo "Successfully added to runlevels 345!" + elif command -v update-rc.d >/dev/null 2>&1; then + #if we're not a chkconfig box assume we're able to use update-rc.d + update-rc.d $service_name defaults && echo "Success!" + else + echo "No supported init tool found." + fi + + $service_command start +} + +# PostgreSQL +pg_available=0 +psql --version || pg_available=$? +if [ $pg_available -ne 0 ]; then + # wget $FILES_BASE_URL"postgres_apt.sh" -O /tmp/postgres_apt.sh + # bash /tmp/postgres_apt.sh + yum update + yum -y install postgresql93-server postgresql93-devel + service postgresql93 initdb + add_service "postgresql93" +fi + + + +# Redis +redis_available=0 +redis-cli --version || redis_available=$? +if [ $redis_available -ne 0 ]; then + wget http://download.redis.io/releases/redis-2.8.17.tar.gz + tar xzf redis-2.8.17.tar.gz + rm redis-2.8.17.tar.gz + cd redis-2.8.17 + make + make install + + # Setup process init & configuration + + REDIS_PORT=6379 + REDIS_CONFIG_FILE="/etc/redis/$REDIS_PORT.conf" + REDIS_LOG_FILE="/var/log/redis_$REDIS_PORT.log" + REDIS_DATA_DIR="/var/lib/redis/$REDIS_PORT" + + mkdir -p `dirname "$REDIS_CONFIG_FILE"` || die "Could not create redis config directory" + mkdir -p `dirname "$REDIS_LOG_FILE"` || die "Could not create redis log dir" + mkdir -p "$REDIS_DATA_DIR" || die "Could not create redis data directory" + + wget -O /etc/init.d/redis_6379 $FILES_BASE_URL"redis_init" + wget -O $REDIS_CONFIG_FILE $FILES_BASE_URL"redis.conf" + + add_service "redis_$REDIS_PORT" + + cd .. + rm -rf redis-2.8.17 +fi + + +if [ ! -d "$REDASH_BASE_PATH" ]; then + sudo mkdir /opt/redash + sudo chown redash /opt/redash + sudo -u redash mkdir /opt/redash/logs +fi + +# Default config file +if [ ! -f "/opt/redash/.env" ]; then + sudo -u redash wget $FILES_BASE_URL"env" -O /opt/redash/.env +fi + +# Install latest version +REDASH_VERSION=${REDASH_VERSION-0.6.3.b906} +LATEST_URL="https://github.com/EverythingMe/redash/releases/download/v${REDASH_VERSION}/redash.$REDASH_VERSION.tar.gz" +VERSION_DIR="/opt/redash/redash.$REDASH_VERSION" +REDASH_TARBALL=/tmp/redash.tar.gz +REDASH_TARBALL=/tmp/redash.tar.gz + +if [ ! -d "$VERSION_DIR" ]; then + sudo -u redash wget $LATEST_URL -O $REDASH_TARBALL + sudo -u redash mkdir $VERSION_DIR + sudo -u redash tar -C $VERSION_DIR -xvf $REDASH_TARBALL + ln -nfs $VERSION_DIR /opt/redash/current + ln -nfs /opt/redash/.env /opt/redash/current/.env + + cd /opt/redash/current + + # TODO: venv? + pip install -r requirements.txt +fi + +# InfluxDB dependencies: +pip install influxdb==2.6.0 + +# BigQuery dependencies: +pip install google-api-python-client==1.2 pyOpenSSL==0.14 oauth2client==1.2 + +# MySQL dependencies: +yum install -y mysql-devel +pip install MySQL-python==1.2.5 + +# Mongo dependencies: +pip install pymongo==2.7.2 + +# Setup supervisord + sysv init startup script +sudo -u redash mkdir -p /opt/redash/supervisord +pip install supervisor==3.1.2 # TODO: move to requirements.txt + +# Create database / tables +pg_user_exists=0 +sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='redash'" | grep -q 1 || pg_user_exists=$? +if [ $pg_user_exists -ne 0 ]; then + echo "Creating redash postgres user & database." + sudo -u postgres createuser redash --no-superuser --no-createdb --no-createrole + sudo -u postgres createdb redash --owner=redash + + cd /opt/redash/current + sudo -u redash bin/run ./manage.py database create_tables +fi + +# Create default admin user +cd /opt/redash/current +# TODO: make sure user created only once +# TODO: generate temp password and print to screen +sudo -u redash bin/run ./manage.py users create --admin --password admin "Admin" "admin" + +# Create re:dash read only pg user & setup data source +pg_user_exists=0 +sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='redash_reader'" | grep -q 1 || pg_user_exists=$? +if [ $pg_user_exists -ne 0 ]; then + echo "Creating redash reader postgres user." + + sudo yum install expect + + REDASH_READER_PASSWORD=$(mkpasswd) + sudo -u postgres psql -c "CREATE ROLE redash_reader WITH PASSWORD '$REDASH_READER_PASSWORD' NOCREATEROLE NOCREATEDB NOSUPERUSER LOGIN" + sudo -u redash psql -c "grant select(id,name,type) ON data_sources to redash_reader;" redash + sudo -u redash psql -c "grant select on activity_log, events, queries, dashboards, widgets, visualizations, query_results to redash_reader;" redash + + cd /opt/redash/current + sudo -u redash bin/run ./manage.py ds new -n "re:dash metadata" -t "pg" -o "{\"user\": \"redash_reader\", \"password\": \"$REDASH_READER_PASSWORD\", \"host\": \"localhost\", \"dbname\": \"redash\"}" +fi + + +# Get supervisord startup script +sudo -u redash wget -O /opt/redash/supervisord/supervisord.conf $FILES_BASE_URL"supervisord.conf" + +# install start-stop-daemon +wget http://developer.axis.com/download/distribution/apps-sys-utils-start-stop-daemon-IR1_9_18-2.tar.gz +tar xvzf apps-sys-utils-start-stop-daemon-IR1_9_18-2.tar.gz +cd apps/sys-utils/start-stop-daemon-IR1_9_18-2/ +gcc start-stop-daemon.c -o start-stop-daemon +cp start-stop-daemon /sbin/ + +# wget -O /etc/init.d/redash_supervisord $FILES_BASE_URL"redash_supervisord_init" +add_service "redash_supervisord" + +# Nginx setup +if [-x $(mkdir /etc/nginx/sites-available)]; then + echo "/etc/nginx/sites-available exists." +fi +wget -O /etc/nginx/sites-available/redash $FILES_BASE_URL"nginx_redash_site" +ln -nfs /etc/nginx/sites-available/redash /etc/nginx/conf.d/redash.conf +service nginx restart From a3cb698be0f61943911763536b6729b4a9032a8d Mon Sep 17 00:00:00 2001 From: masaru Date: Fri, 31 Jul 2015 14:34:25 +0900 Subject: [PATCH 2/3] fixed file path the fixed pathes will work if my original files are uploaded --- setup/bootstrap_amazon_linux.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/bootstrap_amazon_linux.sh b/setup/bootstrap_amazon_linux.sh index 6c47f84f1e..016f88c8a2 100644 --- a/setup/bootstrap_amazon_linux.sh +++ b/setup/bootstrap_amazon_linux.sh @@ -178,7 +178,7 @@ fi # Get supervisord startup script -sudo -u redash wget -O /opt/redash/supervisord/supervisord.conf $FILES_BASE_URL"supervisord.conf" +sudo -u redash wget -O /opt/redash/supervisord/supervisord.conf $FILES_BASE_URL"supervisord_for_amazon_linux.conf" # install start-stop-daemon wget http://developer.axis.com/download/distribution/apps-sys-utils-start-stop-daemon-IR1_9_18-2.tar.gz @@ -187,7 +187,7 @@ cd apps/sys-utils/start-stop-daemon-IR1_9_18-2/ gcc start-stop-daemon.c -o start-stop-daemon cp start-stop-daemon /sbin/ -# wget -O /etc/init.d/redash_supervisord $FILES_BASE_URL"redash_supervisord_init" +wget -O /etc/init.d/redash_supervisord $FILES_BASE_URL"redash_supervisord_init_for_amazon_linux" add_service "redash_supervisord" # Nginx setup From 0e46a24112592c647ed806add5544f0bb4fa01d2 Mon Sep 17 00:00:00 2001 From: masaru Date: Fri, 31 Jul 2015 19:21:36 +0900 Subject: [PATCH 3/3] fixed config file pathes --- setup/bootstrap_amazon_linux.sh | 6 +- .../redash_supervisord_init_for_amazon_linux | 125 ++++++++++++++++++ setup/supervisord_for_amazon_linux.conf | 31 +++++ 3 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 setup/redash_supervisord_init_for_amazon_linux create mode 100644 setup/supervisord_for_amazon_linux.conf diff --git a/setup/bootstrap_amazon_linux.sh b/setup/bootstrap_amazon_linux.sh index 016f88c8a2..9d6aa837b9 100644 --- a/setup/bootstrap_amazon_linux.sh +++ b/setup/bootstrap_amazon_linux.sh @@ -3,7 +3,7 @@ set -eu REDASH_BASE_PATH=/opt/redash FILES_BASE_URL=https://raw.githubusercontent.com/EverythingMe/redash/docs_setup/setup/files/ - +FILE_BASE_URL_FOR_AMAZON_LINUX=https://raw.githubusercontent.com/EverythingMe/redash/master/setup/files/ # Verify running as root: if [ "$(id -u)" != "0" ]; then if [ $# -ne 0 ]; then @@ -178,7 +178,7 @@ fi # Get supervisord startup script -sudo -u redash wget -O /opt/redash/supervisord/supervisord.conf $FILES_BASE_URL"supervisord_for_amazon_linux.conf" +sudo -u redash wget -O /opt/redash/supervisord/supervisord.conf $FILE_BASE_URL_FOR_AMAZON_LINUX"supervisord_for_amazon_linux.conf" # install start-stop-daemon wget http://developer.axis.com/download/distribution/apps-sys-utils-start-stop-daemon-IR1_9_18-2.tar.gz @@ -187,7 +187,7 @@ cd apps/sys-utils/start-stop-daemon-IR1_9_18-2/ gcc start-stop-daemon.c -o start-stop-daemon cp start-stop-daemon /sbin/ -wget -O /etc/init.d/redash_supervisord $FILES_BASE_URL"redash_supervisord_init_for_amazon_linux" +wget -O /etc/init.d/redash_supervisord $FILE_BASE_URL_FOR_AMAZON_LINUX"redash_supervisord_init_for_amazon_linux" add_service "redash_supervisord" # Nginx setup diff --git a/setup/redash_supervisord_init_for_amazon_linux b/setup/redash_supervisord_init_for_amazon_linux new file mode 100644 index 0000000000..e058fe2186 --- /dev/null +++ b/setup/redash_supervisord_init_for_amazon_linux @@ -0,0 +1,125 @@ +#!/bin/sh +# /etc/init.d/redash_supervisord +### BEGIN INIT INFO +# Provides: supervisord +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: process supervisor +### END INIT INFO + +# Author: Ron DuPlain + +# Do NOT "set -e" + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin +NAME=supervisord +DESC="process supervisor" +DAEMON=/usr/local/bin/$NAME +DAEMON_ARGS="--configuration /opt/redash/supervisord/supervisord.conf " +PIDFILE=/opt/redash/supervisord/supervisord.pid +SCRIPTNAME=/etc/init.d/redash_supervisord +USER=redash + +# Exit if the package is not installed +[ -x "$DAEMON" ] || exit 0 + +# Read configuration variable file if it is present +[ -r /etc/default/$NAME ] && . /etc/default/$NAME + +. /etc/rc.d/init.d/functions + +# +# Function that starts the daemon/service +# + +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + start-stop-daemon --start --quiet --pidfile $PIDFILE --user $USER --chuid $USER --exec $DAEMON --test > /dev/null \ + || return 1 + start-stop-daemon --start --quiet --pidfile $PIDFILE --user $USER --chuid $USER --exec $DAEMON -- \ + $DAEMON_ARGS \ + || return 2 + # Add code here, if necessary, that waits for the process to be ready + # to handle requests from services started subsequently which depend + # on this one. As a last resort, sleep for some time. +} + +# +# Function that stops the daemon/service +# + +do_stop() +{ + # Return + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --user $USER --chuid $USER --name $NAME + RETVAL="$?" + [ "$RETVAL" = 2 ] && return 2 + # Wait for children to finish too if this is a daemon that forks + # and if the daemon is only ever run from this initscript. + # If the above conditions are not satisfied then add some other code + # that waits for the process to drop all resources that could be + # needed by services started subsequently. A last resort is to + # sleep for some time. + start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --user $USER --chuid $USER --exec $DAEMON + [ "$?" = 2 ] && return 2 + # Many daemons don't delete their pidfiles when they exit. + rm -f $PIDFILE + return "$RETVAL" +} + +case "$1" in + start) + [ "$VERBOSE" != no ] && echo "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) [ "$VERBOSE" != no ] && echo 0 ;; + 2) [ "$VERBOSE" != no ] && echo 1 ;; + esac + ;; + stop) + [ "$VERBOSE" != no ] && echo "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) [ "$VERBOSE" != no ] && echo 0 ;; + 2) [ "$VERBOSE" != no ] && echo 1 ;; + esac + ;; + status) + status -p "$STASH_PID" stash + # status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? + ;; + restart) + echo "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) echo 0 ;; + 1) echo 1 ;; # Old process is still running + *) echo 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + echo 1 + ;; + esac + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart}" >&2 + exit 3 + ;; +esac + diff --git a/setup/supervisord_for_amazon_linux.conf b/setup/supervisord_for_amazon_linux.conf new file mode 100644 index 0000000000..3adabacdc2 --- /dev/null +++ b/setup/supervisord_for_amazon_linux.conf @@ -0,0 +1,31 @@ + +[supervisord] +nodaemon=false +logfile=/opt/redash/logs/supervisord.log +pidfile=/opt/redash/supervisord/supervisord.pid +directory=/opt/redash/current + +[inet_http_server] +port = 127.0.0.1:9001 + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[program:redash_server] +command=/opt/redash/current/bin/run /usr/local/bin/gunicorn -b 127.0.0.1:5000 --name redash -w 4 redash.wsgi:app +process_name=redash_server +numprocs=1 +priority=999 +autostart=true +autorestart=true +stdout_logfile=/opt/redash/logs/api.log +stderr_logfile=/opt/redash/logs/api_error.log + +[program:redash_celery] +command=/opt/redash/current/bin/run /usr/local/bin/celery worker --app=redash.worker --beat -Qqueries,celery,scheduled_queries +process_name=redash_celery +numprocs=1 +priority=999 +autostart=true +autorestart=true +stdout_logfile=/opt/redash/logs/celery.log