adding new scripts

max-allan-surevine · Nov 8, 2013 · 416f308 · 416f308
1 parent d193084
commit 416f308
Show file tree

Hide file tree

Showing 3 changed files with 396 additions and 0 deletions.
diff --git a/cascading.compatiblity/install-cascading-compatiblity-bootstrap.sh b/cascading.compatiblity/install-cascading-compatiblity-bootstrap.sh
@@ -0,0 +1,145 @@
+#!/bin/bash
+set -e -x
+
+# Usage:
+#  --user-home - an alternative user to install into, default /home/hadoop
+#  --tmpdir - an alternative temporary directory, default TMPDIR or /tmp if not set
+#  --no-screen - do not install screen, screen is installed by default on the master as a convenience
+#  --latest - url to text file referencing the latest version
+#  --no-bash - do not update .bashrc
+#  --gradle-ver - the version of name of gradel. defaults to "gradle-1.7"
+#  --gradle-file - the packaged file extension of the file. defaults to "<-gradel-ver>all-zip" 
+#  --gradle-bucket - the root site url or S3 bucket of the install file defaults to "http://services.gradle.org/distributions/"
+#  --gradle-home - the directory that gradle will be install into. defaults to "/usr/share/"
+
+CASCADING_COMP_REPO=https://github.com/AmazonEMR/cascading.compatibility.git
+HADOOP_TEST_JAR=http://preprod.us-east-1.elasticmapreduce.testing.s3.amazonaws.com/libs/hadoop/hadoop-test-1.0.3.jar
+GRADLE_VER=gradle-1.7
+GRADLE_FILE=$GRADLE_VER-all.zip
+GRADLE_BUCKET=http://services.gradle.org/distributions/
+GRADLE_HOME=/usr/share/
+
+LATEST=$GRADLE_BUCKET$GRADLE_FILE
+
+
+case "`uname`" in
+  Darwin)
+    USER_HOME=/Users/$USER;;
+  *)
+    USER_HOME=/home/$USER;;
+esac
+
+INSTALL_ON_SLAVES=false
+BASH_PROFILE=.bashrc
+INSTALL_SCREEN=y
+INSTALL_GIT=y
+UPDATE_BASH=y
+
+IS_MASTER=true
+if [ -f /mnt/var/lib/info/instance.json ]
+then
+  IS_MASTER=`cat /mnt/var/lib/info/instance.json | tr -d '\n ' | sed -n 's|.*\"isMaster\":\([^,]*\).*|\1|p'`
+  USER_HOME=/home/hadoop
+fi
+
+[ -z "$TMPDIR" ] && TMPDIR=/tmp/
+
+error_msg () # msg
+{
+  echo 1>&2 "Error: $1"
+}
+
+error_exit () # <msg> <cod>
+{
+  error_msg "$1"
+  exit ${2:-1}
+}
+
+while [ $# -gt 0 ]
+do
+  case "$1" in
+    --latest)
+      shift
+      LATEST=$1
+      ;;
+    --user-home)
+      shift
+      USER_HOME=$1
+      ;;
+    --tmpdir)
+      shift
+      TMPDIR=$1
+      ;;
+    --no-screen)
+      INSTALL_SCREEN=
+      ;;
+    --no-bash)
+      UPDATE_BASH=
+      ;;
+    --slaves)
+      INSTALL_ON_SLAVES=true
+      ;;
+    --gradle-ver)
+      GRADLE_VER=
+      ;;
+    --gradle-file)
+      GRADLE_FILE=
+      ;;
+    --gradle-bucket)
+      GRADLE_BUCKET=
+      ;;
+    --gradle-home)
+      GRADLE_HOME=
+      ;;
+    -*)
+      # do not exit out, just note failure
+      error_msg "unrecognized option: $1"
+      ;;
+    *)
+      break;
+      ;;
+  esac
+  shift
+done
+
+if [ "$IS_MASTER" = "false" && "$INSTALL_ON_SLAVES" = "false" ]; then
+  exit 0
+fi
+
+# fetch the install zip file
+wget -S -T 10 -t 5 $LATEST -P $TMPDIR
+
+sudo unzip -o /$TMPDIR$GRADLE_FILE -d $GRADLE_HOME
+
+if [ -n "$UPDATE_BASH" ]; then
+cat >> $USER_HOME/$BASH_PROFILE <<- EOF
+
+# Amazon Elastic Map Reduce Gradle Bootstrap
+export PATH=$GRADLE_HOME/$GRADLE_VER/bin:\$PATH
+
+EOF
+fi
+
+if [ "$IS_MASTER" = "true" ]; then
+  [ -n "$INSTALL_SCREEN" ] && sudo apt-get --force-yes install screen -y
+fi
+
+if [ "$IS_MASTER" = "true" ]; then
+  [ -n "$INSTALL_GIT" ] && sudo apt-get --force-yes install git -y
+fi
+
+# clean up
+sudo rm $TMPDIR$GRADLE_FILE
+
+#pulldown hadoop-test.jar file
+wget -S -T 10 -t 5 $HADOOP_TEST_JAR -P $USER_HOME/
+
+if [ "$IS_MASTER" = "true" ]; then
+  #pull down the cascading compatiblity repo
+  CASCADING_COMP_DIR=$USER_HOME/cascading.compatibility
+  mkdir $CASCADING_COMP_DIR
+  git clone $CASCADING_COMP_REPO $CASCADING_COMP_DIR
+  cd $CASCADING_COMP_DIR
+  git checkout 2.2
+  cd
+fi
diff --git a/gradle/install-gradle-bootstrap.sh b/gradle/install-gradle-bootstrap.sh
@@ -0,0 +1,137 @@
+#!/bin/bash
+set -e -x
+
+# Usage:
+#  --user-home - an alternative user to install into, default /home/hadoop
+#  --tmpdir - an alternative temporary directory, default TMPDIR or /tmp if not set
+#  --no-screen - do not install screen, screen is installed by default on the master as a convenience
+#  --latest - url to text file referencing the latest version
+#  --no-bash - do not update .bashrc
+#  --gradle-ver - the version of name of gradel. defaults to "gradle-1.7"
+#  --gradle-file - the packaged file extension of the file. defaults to "<-gradel-ver>all-zip" 
+#  --gradle-bucket - the root site url or S3 bucket of the install file defaults to "http://services.gradle.org/distributions/"
+#  --gradle-home - the directory that gradle will be install into. defaults to "/usr/share/"
+
+CASCADING_COMP_REPO=https://github.com/AmazonEMR/cascading.compatibility.git
+GRADLE_VER=gradle-1.7
+GRADLE_FILE=$GRADLE_VER-all.zip
+GRADLE_BUCKET=http://services.gradle.org/distributions/
+GRADLE_HOME=/usr/share/
+
+LATEST=$GRADLE_BUCKET$GRADLE_FILE
+
+
+case "`uname`" in
+  Darwin)
+    USER_HOME=/Users/$USER;;
+  *)
+    USER_HOME=/home/$USER;;
+esac
+
+INSTALL_ON_SLAVES=false
+BASH_PROFILE=.bashrc
+INSTALL_SCREEN=y
+INSTALL_GIT=y
+UPDATE_BASH=y
+
+IS_MASTER=true
+if [ -f /mnt/var/lib/info/instance.json ]
+then
+  IS_MASTER=`cat /mnt/var/lib/info/instance.json | tr -d '\n ' | sed -n 's|.*\"isMaster\":\([^,]*\).*|\1|p'`
+  USER_HOME=/home/hadoop
+fi
+
+[ -z "$TMPDIR" ] && TMPDIR=/tmp/
+
+error_msg () # msg
+{
+  echo 1>&2 "Error: $1"
+}
+
+error_exit () # <msg> <cod>
+{
+  error_msg "$1"
+  exit ${2:-1}
+}
+
+while [ $# -gt 0 ]
+do
+  case "$1" in
+    --latest)
+      shift
+      LATEST=$1
+      ;;
+    --user-home)
+      shift
+      USER_HOME=$1
+      ;;
+    --tmpdir)
+      shift
+      TMPDIR=$1
+      ;;
+    --no-screen)
+      INSTALL_SCREEN=
+      ;;
+    --no-bash)
+      UPDATE_BASH=
+      ;;
+    --slaves)
+      INSTALL_ON_SLAVES=true
+      ;;
+    --gradle-ver)
+      GRADLE_VER=
+      ;;
+    --gradle-file)
+      GRADLE_FILE=
+      ;;
+    --gradle-bucket)
+      GRADLE_BUCKET=
+      ;;
+    --gradle-home)
+      GRADLE_HOME=
+      ;;
+    -*)
+      # do not exit out, just note failure
+      error_msg "unrecognized option: $1"
+      ;;
+    *)
+      break;
+      ;;
+  esac
+  shift
+done
+
+if [ "$IS_MASTER" = "false" && "$INSTALL_ON_SLAVES" = "false" ]; then
+  exit 0
+fi
+
+# fetch the install zip file
+wget -S -T 10 -t 5 $LATEST -P $TMPDIR
+
+sudo unzip -o /$TMPDIR$GRADLE_FILE -d $GRADLE_HOME
+
+if [ -n "$UPDATE_BASH" ]; then
+cat >> $USER_HOME/$BASH_PROFILE <<- EOF
+
+# Amazon Elastic Map Reduce Gradle Bootstrap
+export PATH=$GRADLE_HOME/$GRADLE_VER/bin:\$PATH
+
+EOF
+fi
+
+if [ "$IS_MASTER" = "true" ]; then
+  [ -n "$INSTALL_SCREEN" ] && sudo apt-get --force-yes install screen -y
+fi
+
+if [ "$IS_MASTER" = "true" ]; then
+  [ -n "$INSTALL_GIT" ] && sudo apt-get --force-yes install git -y
+fi
+
+# clean up
+sudo rm $TMPDIR$GRADLE_FILE
+
+mkdir $USER_HOME/cascading
+git clone $CASCADING_COMP_REPO $USER_HOME/cascading
+cd $USER_HOME/cascading
+git checkout 2.2
+cd 
diff --git a/phoenix/install-phoenix-bootstrap.sh b/phoenix/install-phoenix-bootstrap.sh
@@ -0,0 +1,114 @@
+#!/bin/bash
+set -e -x
+
+# Usage:
+#  --download-location - Base URL to find the Phoenix files, default http://s3.amazonaws.com/beta.elasticmapreduce/bootstrap-actions/phoenix/2.0.2/
+#  --phoenix-server - Phoenix Server file that will be copied to the Hbase Lib folder on all servers, default is phoenix-2.0.2.jar
+#  --phoenix-client - Phoenix Client file that will be copied to the Hbase Lib folder on the master node, default is phoenix-2.0.2-client.jar
+#  --hbase-version - an alternative HBase Verison path to use, default /home/hadoop/.versions/<$HBASE_VERSION>/
+#  --slaves - indicates that the Phoenix service jar will be deployed on all slave nodes, defaults to true
+
+PRODUCTNAME=phoenix
+VERSION=2.1.0
+DOWNLOAD_LOCATION=http://s3.amazonaws.com/beta.elasticmapreduce/bootstrap-actions/$PRODUCTNAME/$VERSION/
+HBASE_VERSION=hbase-0.94.7
+
+
+INSTALL_DIR=/home/hadoop/.versions/$HBASE_VERSION/lib/
+PHOENIX_SERVER=$PRODUCTNAME-$VERSION.jar
+PHOENIX_PACKAGE_NAME=$PRODUCTNAME-$VERSION-install
+PHOENIX_PACKAGE=$PHOENIX_PACKAGE_NAME.tar
+
+INSTALL_ON_SLAVES=true
+
+
+IS_MASTER=true
+if [ -f /mnt/var/lib/info/instance.json ]
+then
+  IS_MASTER=`cat /mnt/var/lib/info/instance.json | tr -d '\n ' | sed -n 's|.*\"isMaster\":\([^,]*\).*|\1|p'`
+  USER_HOME=/home/hadoop
+fi
+
+
+error_msg () # msg
+{
+  echo 1>&2 "Error: $1"
+}
+
+error_exit () # <msg> <cod>
+{
+  error_msg "$1"
+  exit ${2:-1}
+}
+
+while [ $# -gt 0 ]
+do
+  case "$1" in
+    --hbase-version)
+      shift
+      HBASE_VERSION=$1
+      ;;
+    --phoenix-server)
+      shift
+      PHOENIX_SERVER=$1
+      ;;
+    --phoenix-version)
+      shift
+      VERSION=$1
+      ;;
+    --slaves)
+      INSTALL_ON_SLAVES=true
+      ;;
+    -*)
+      # do not exit out, just note failure
+      error_msg "unrecognized option: $1"
+      ;;
+    *)
+      break;
+      ;;
+  esac
+  shift
+done
+
+
+if [ "$IS_MASTER" = "false" && "$INSTALL_ON_SLAVES" = "false" ]; then
+  exit 0
+fi
+
+# show error and exit if HBase version is not present
+if ! [ -d $INSTALL_DIR ]; then
+    error_exit "The specifie HBase path is not present: $1"
+fi
+
+# install the phoenix service jar on the all nodes
+if [ -f $INSTALL_DIR$PHOENIX_SERVER ]; then
+    rm $INSTALL_DIR$PHOENIX_SERVER
+fi
+
+wget -S -T 10 -t 5 $DOWNLOAD_LOCATION$PHOENIX_SERVER -P $INSTALL_DIR
+chmod 755 $INSTALL_DIR$PHOENIX_SERVER
+
+
+# also, if we're on the master node, get the full install package
+if [ "$IS_MASTER" = "true" ]; then
+
+    PHOENIX_DIR=$INSTALL_DIR$PRODUCTNAME/
+
+
+    if [ -d $PHOENIX_DIR ]; then
+        sudo rm -rf $PHOENIX_DIR
+    fi
+    wget -S -T 10 -t 5 $DOWNLOAD_LOCATION$PHOENIX_PACKAGE -P $INSTALL_DIR
+    mkdir $PHOENIX_DIR
+    tar xvf $INSTALL_DIR$PHOENIX_PACKAGE -C $PHOENIX_DIR
+
+    chmod -R 755 $PHOENIX_DIR
+
+    #remove downloaded jar
+    rm $INSTALL_DIR$PHOENIX_PACKAGE
+    #remove duplicate service jar to avoid confusion of which jar is installed 
+    rm $PHOENIX_DIR$PRODUCTNAME-$VERSION.jar
+
+fi
+
+