From dc3aca926577f02d4f7ea7143dbdc27347882349 Mon Sep 17 00:00:00 2001 From: Arun Das Date: Thu, 8 Dec 2016 12:22:30 -0600 Subject: [PATCH 01/17] Create Dockerfile Please note: Some of the Hadoop part extracted from "https://hub.docker.com/r/sequenceiq/hadoop-docker/~/dockerfile/" CaffeOnSpark is in directory /opt/CaffeOnSpark. Hadoop and Spark is set up correctly. COS working properly. Did MNIST example successfully. --- docker/standalone/cpu/Dockerfile | 155 +++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 docker/standalone/cpu/Dockerfile diff --git a/docker/standalone/cpu/Dockerfile b/docker/standalone/cpu/Dockerfile new file mode 100644 index 0000000..78e9798 --- /dev/null +++ b/docker/standalone/cpu/Dockerfile @@ -0,0 +1,155 @@ +FROM ubuntu:14.04 +MAINTAINER arun.das@my.utsa.edu + +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository ppa:openjdk-r/ppa +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + vim \ + cmake \ + git \ + wget \ + libatlas-base-dev \ + libboost-all-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libhdf5-serial-dev \ + libleveldb-dev \ + liblmdb-dev \ + libopencv-dev \ + libprotobuf-dev \ + libsnappy-dev \ + protobuf-compiler \ + python-dev \ + python-numpy \ + python-pip \ + python-scipy \ + maven \ + unzip \ + zip \ + unzip \ + libopenblas-dev \ + openssh-server \ + openssh-client \ + libopenblas-dev \ + libboost-all-dev +RUN apt-get install -y software-properties-common +RUN add-apt-repository ppa:openjdk-r/ppa +RUN apt-get update && apt-get install -y openjdk-8-jdk +RUN rm -rf /var/lib/apt/lists/* + + +# Passwordless SSH +RUN ssh-keygen -y -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key +RUN ssh-keygen -y -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key +RUN ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa +RUN cp /root/.ssh/id_rsa.pub ~/.ssh/authorized_keys + + +# Apache Hadoop and Spark section +RUN wget http://apache.mirrors.tds.net/hadoop/common/hadoop-2.6.4/hadoop-2.6.4.tar.gz +RUN wget http://archive.apache.org/dist/spark/spark-1.6.0/spark-1.6.0-bin-hadoop2.6.tgz + +RUN gunzip hadoop-2.6.4.tar.gz +RUN gunzip spark-1.6.0-bin-hadoop2.6.tgz +RUN tar -xf hadoop-2.6.4.tar +RUN tar -xf spark-1.6.0-bin-hadoop2.6.tar + +RUN sudo cp -r hadoop-2.6.4 /usr/local/hadoop +RUN sudo cp -r spark-1.6.0-bin-hadoop2.6 /usr/local/spark + +RUN rm hadoop-2.6.4.tar spark-1.6.0-bin-hadoop2.6.tar +RUN rm -rf hadoop-2.6.4/ spark-1.6.0-bin-hadoop2.6/ + +RUN sudo mkdir -p /usr/local/hadoop/hadoop_data/hdfs/namenode +RUN sudo mkdir -p /usr/local/hadoop/hadoop_data/hdfs/datanode + +# Environment variables +ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64 +ENV HADOOP_HOME=/usr/local/hadoop +ENV SPARK_HOME=/usr/local/spark +ENV PATH $PATH:$JAVA_HOME/bin +ENV PATH $PATH:$HADOOP_HOME/bin +ENV PATH $PATH:$HADOOP_HOME/sbin +ENV PATH $PATH:$SPARK_HOME/bin +ENV PATH $PATH:$SPARK_HOME/sbin +ENV HADOOP_MAPRED_HOME /usr/local/hadoop +ENV HADOOP_COMMON_HOME /usr/local/hadoop +ENV HADOOP_HDFS_HOME /usr/local/hadoop +ENV HADOOP_CONF_DIR /usr/local/hadoop/etc/hadoop +ENV YARN_HOME /usr/local/hadoop +ENV HADOOP_COMMON_LIB_NATIVE_DIR /usr/local/hadoop/lib/native +ENV HADOOP_OPTS "-Djava.library.path=$HADOOP_HOME/lib" + +# Clone CaffeOnSpark +ENV CAFFE_ON_SPARK=/opt/CaffeOnSpark +WORKDIR $CAFFE_ON_SPARK +RUN git clone https://github.com/yahoo/CaffeOnSpark.git . --recursive + +# Some of the Hadoop part extracted from "https://hub.docker.com/r/sequenceiq/hadoop-docker/~/dockerfile/" +RUN mkdir $HADOOP_HOME/input +RUN cp $HADOOP_HOME/etc/hadoop/*.xml $HADOOP_HOME/input +RUN cd /usr/local/hadoop/input + +# Copy .xml files. +RUN cp ${CAFFE_ON_SPARK}/scripts/*.xml ${HADOOP_HOME}/etc/hadoop + +# Format namenode and finish hadoop, spark installations. +RUN $HADOOP_HOME/bin/hdfs namenode -format + +RUN ls /root/.ssh/ +ADD config/ssh_config /root/.ssh/config +RUN chmod 600 /root/.ssh/config +RUN chown root:root /root/.ssh/config + +ADD config/bootstrap.sh /etc/bootstrap.sh +RUN chown root:root /etc/bootstrap.sh +RUN chmod 700 /etc/bootstrap.sh + +ENV BOOTSTRAP /etc/bootstrap.sh + +RUN sed -i '/^export JAVA_HOME/ s:.*:export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64\nexport HADOOP_HOME=/usr/local/hadoop\n:' $HADOOP_HOME/etc/hadoop/hadoop-env.sh +RUN sed -i '/^export HADOOP_CONF_DIR/ s:.*:export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/:' $HADOOP_HOME/etc/hadoop/hadoop-env.sh + +# workingaround docker.io build error +RUN ls -la /usr/local/hadoop/etc/hadoop/*-env.sh +RUN chmod +x /usr/local/hadoop/etc/hadoop/*-env.sh +RUN ls -la /usr/local/hadoop/etc/hadoop/*-env.sh + +# fix the 254 error code +RUN sed -i "/^[^#]*UsePAM/ s/.*/#&/" /etc/ssh/sshd_config +RUN echo "UsePAM no" >> /etc/ssh/sshd_config +RUN echo "Port 2122" >> /etc/ssh/sshd_config + +RUN service ssh start && $HADOOP_HOME/etc/hadoop/hadoop-env.sh && $HADOOP_HOME/sbin/start-dfs.sh && $HADOOP_HOME/bin/hdfs dfs -mkdir -p /user/root +RUN service ssh start && $HADOOP_HOME/etc/hadoop/hadoop-env.sh && $HADOOP_HOME/sbin/start-dfs.sh && $HADOOP_HOME/bin/hdfs dfs -put $HADOOP_HOME/etc/hadoop/ input + +CMD ["/etc/bootstrap.sh", "-bash"] + +# Hdfs ports +EXPOSE 50010 50020 50070 50075 50090 8020 9000 +# Mapred ports +EXPOSE 10020 19888 +#Yarn ports +EXPOSE 8030 8031 8032 8033 8040 8042 8088 +#Other ports +EXPOSE 49707 2122 + + +# Continue with CaffeOnSpark build. +# ENV CAFFE_ON_SPARK=/opt/CaffeOnSpark +WORKDIR $CAFFE_ON_SPARK +# RUN git clone https://github.com/yahoo/CaffeOnSpark.git . --recursive +RUN cp caffe-public/Makefile.config.example caffe-public/Makefile.config +RUN echo "INCLUDE_DIRS += ${JAVA_HOME}/include" >> caffe-public/Makefile.config +RUN sed -i "s/# CPU_ONLY := 1/CPU_ONLY := 1/g" caffe-public/Makefile.config +RUN sed -i "s|CUDA_DIR := /usr/local/cuda|# CUDA_DIR := /usr/local/cuda|g" caffe-public/Makefile.config +RUN sed -i "s|CUDA_ARCH :=|# CUDA_ARCH :=|g" caffe-public/Makefile.config +RUN sed -i "s|BLAS := atlas|BLAS := open|g" caffe-public/Makefile.config +RUN sed -i "s|TEST_GPUID := 0|# TEST_GPUID := 0|g" caffe-public/Makefile.config + +RUN make build + +ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:$CAFFE_ON_SPARK/caffe-public/distribute/lib:$CAFFE_ON_SPARK/caffe-distri/distribute/lib + +WORKDIR /root From 363b09ea2e261daa0e45b89d6ad31d12e9f3253a Mon Sep 17 00:00:00 2001 From: Arun Das Date: Thu, 8 Dec 2016 12:29:43 -0600 Subject: [PATCH 02/17] Create ssh_config Required: Proper SSH without password for Hadoop. --- docker/standalone/cpu/ssh_config | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docker/standalone/cpu/ssh_config diff --git a/docker/standalone/cpu/ssh_config b/docker/standalone/cpu/ssh_config new file mode 100644 index 0000000..535f9d3 --- /dev/null +++ b/docker/standalone/cpu/ssh_config @@ -0,0 +1,5 @@ +Host * + UserKnownHostsFile /dev/null + StrictHostKeyChecking no + LogLevel quiet + Port 2122 From 2fde6008a20fc92d9826923d275406805040331a Mon Sep 17 00:00:00 2001 From: Arun Das Date: Thu, 8 Dec 2016 12:30:14 -0600 Subject: [PATCH 03/17] Rename docker/standalone/cpu/ssh_config to docker/standalone/cpu/config/ssh_config --- docker/standalone/cpu/{ => config}/ssh_config | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker/standalone/cpu/{ => config}/ssh_config (100%) diff --git a/docker/standalone/cpu/ssh_config b/docker/standalone/cpu/config/ssh_config similarity index 100% rename from docker/standalone/cpu/ssh_config rename to docker/standalone/cpu/config/ssh_config From d431ac7616584cbd3228b0f2b553f8a592e61696 Mon Sep 17 00:00:00 2001 From: Arun Das Date: Thu, 8 Dec 2016 12:32:16 -0600 Subject: [PATCH 04/17] Create bootstrap.sh Required: To start hadoop dfs and yarn when starting container and to pass required environment variables. --- docker/standalone/cpu/config/bootstrap.sh | 27 +++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 docker/standalone/cpu/config/bootstrap.sh diff --git a/docker/standalone/cpu/config/bootstrap.sh b/docker/standalone/cpu/config/bootstrap.sh new file mode 100644 index 0000000..5797e47 --- /dev/null +++ b/docker/standalone/cpu/config/bootstrap.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +: ${HADOOP_PREFIX:=/usr/local/hadoop} + +$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh + +rm /tmp/*.pid + +# installing libraries if any - (resource urls added comma separated to the ACP system variable) +cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; curl -LO $cp ; done; cd - + +# adding necessary paths to environment variables (FIXME: These are already in Dockerfile, but does not work. So giving them explicitly.) +export PATH=$PATH:$SPARK_HOME/bin +export PATH=$PATH:$HADOOP_HOME/bin +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CAFFE_ON_SPARK/caffe-public/distribute/lib:$CAFFE_ON_SPARK/caffe-distri/distribute/lib + +service ssh start +$HADOOP_PREFIX/sbin/start-dfs.sh +$HADOOP_PREFIX/sbin/start-yarn.sh + +if [[ $1 == "-d" ]]; then + while true; do sleep 1000; done +fi + +if [[ $1 == "-bash" ]]; then + /bin/bash +fi From a6407a1559306718c0cdde40524f934097fa7b02 Mon Sep 17 00:00:00 2001 From: Arun Das Date: Thu, 8 Dec 2016 12:49:49 -0600 Subject: [PATCH 05/17] Create README.md --- docker/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 docker/README.md diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..aaa8b4d --- /dev/null +++ b/docker/README.md @@ -0,0 +1,19 @@ +# CaffeOnSpark Standalone Docker + +Dockerfiles for CPU build is provided in `standalone/cpu` folder. The image can be built by running: +``` +docker build -t caffeonspark:cpu standalone/cpu +``` +After the image is built, use `docker images` to validate. + +## Launching CaffeOnSpark container +Hadoop and Spark are essential requirements for CaffeOnSpark. To ensure that both process runs flawless, we have included `standalone/cpu/config/bootstrap.sh` script which must be run everytime the container is started. + +To launch a container running CaffeOnSpark please use: +``` +docker run -it caffeonspark:cpu /etc/bootstrap.sh -bash +``` + +Now you have a working environment with CaffeOnSpark. + +To verify installation, please follow [GetStarted_yarn](https://github.com/yahoo/CaffeOnSpark/wiki/GetStarted_yarn) guide from `Step 7`. From 361051feb8dc009eff9c8bd746a3b1b818b91797 Mon Sep 17 00:00:00 2001 From: Arun Das Date: Tue, 10 Jan 2017 23:28:54 -0600 Subject: [PATCH 06/17] Create bootstrap.sh --- docker/standalone/gpu/config/bootstrap.sh | 27 +++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 docker/standalone/gpu/config/bootstrap.sh diff --git a/docker/standalone/gpu/config/bootstrap.sh b/docker/standalone/gpu/config/bootstrap.sh new file mode 100644 index 0000000..5797e47 --- /dev/null +++ b/docker/standalone/gpu/config/bootstrap.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +: ${HADOOP_PREFIX:=/usr/local/hadoop} + +$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh + +rm /tmp/*.pid + +# installing libraries if any - (resource urls added comma separated to the ACP system variable) +cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; curl -LO $cp ; done; cd - + +# adding necessary paths to environment variables (FIXME: These are already in Dockerfile, but does not work. So giving them explicitly.) +export PATH=$PATH:$SPARK_HOME/bin +export PATH=$PATH:$HADOOP_HOME/bin +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CAFFE_ON_SPARK/caffe-public/distribute/lib:$CAFFE_ON_SPARK/caffe-distri/distribute/lib + +service ssh start +$HADOOP_PREFIX/sbin/start-dfs.sh +$HADOOP_PREFIX/sbin/start-yarn.sh + +if [[ $1 == "-d" ]]; then + while true; do sleep 1000; done +fi + +if [[ $1 == "-bash" ]]; then + /bin/bash +fi From 21ae6e6bcf08bea223d59bdedf319222238d5526 Mon Sep 17 00:00:00 2001 From: Arun Das Date: Tue, 10 Jan 2017 23:29:28 -0600 Subject: [PATCH 07/17] Create ssh_config --- docker/standalone/gpu/config/ssh_config | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docker/standalone/gpu/config/ssh_config diff --git a/docker/standalone/gpu/config/ssh_config b/docker/standalone/gpu/config/ssh_config new file mode 100644 index 0000000..7795f49 --- /dev/null +++ b/docker/standalone/gpu/config/ssh_config @@ -0,0 +1,5 @@ +Host * +UserKnownHostsFile /dev/null +StrictHostKeyChecking no +LogLevel quiet +Port 2122 From 6b5f4d41094026464f845bd20275cb494e888ca2 Mon Sep 17 00:00:00 2001 From: Arun Das Date: Tue, 10 Jan 2017 23:36:41 -0600 Subject: [PATCH 08/17] Update Dockerfile removed redundant lines. --- docker/standalone/cpu/Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docker/standalone/cpu/Dockerfile b/docker/standalone/cpu/Dockerfile index 78e9798..76bbd01 100644 --- a/docker/standalone/cpu/Dockerfile +++ b/docker/standalone/cpu/Dockerfile @@ -32,10 +32,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ openssh-server \ openssh-client \ libopenblas-dev \ - libboost-all-dev -RUN apt-get install -y software-properties-common -RUN add-apt-repository ppa:openjdk-r/ppa -RUN apt-get update && apt-get install -y openjdk-8-jdk + libboost-all-dev \ + openjdk-8-jdk + RUN rm -rf /var/lib/apt/lists/* From 7237da244ce97a0923138fcbec17f03377af94bd Mon Sep 17 00:00:00 2001 From: Arun Das Date: Tue, 10 Jan 2017 23:37:54 -0600 Subject: [PATCH 09/17] Dockerfile-GPU Adding GPU support for docker version of CaffeOnSpark --- docker/standalone/gpu/config/Dockerfile | 151 ++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 docker/standalone/gpu/config/Dockerfile diff --git a/docker/standalone/gpu/config/Dockerfile b/docker/standalone/gpu/config/Dockerfile new file mode 100644 index 0000000..cc50144 --- /dev/null +++ b/docker/standalone/gpu/config/Dockerfile @@ -0,0 +1,151 @@ +FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 +MAINTAINER arun.das@my.utsa.edu + +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository ppa:openjdk-r/ppa +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + vim \ + cmake \ + git \ + wget \ + libatlas-base-dev \ + libboost-all-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libhdf5-serial-dev \ + libleveldb-dev \ + liblmdb-dev \ + libopencv-dev \ + libprotobuf-dev \ + libsnappy-dev \ + protobuf-compiler \ + python-dev \ + python-numpy \ + python-pip \ + python-scipy \ + maven \ + unzip \ + zip \ + unzip \ + libopenblas-dev \ + openssh-server \ + openssh-client \ + libopenblas-dev \ + libboost-all-dev \ + openjdk-8-jdk + +RUN rm -rf /var/lib/apt/lists/* + + +# Passwordless SSH +RUN ssh-keygen -y -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key +RUN ssh-keygen -y -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key +RUN ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa +RUN cp /root/.ssh/id_rsa.pub ~/.ssh/authorized_keys + + +# Apache Hadoop and Spark section +RUN wget http://apache.mirrors.tds.net/hadoop/common/hadoop-2.6.4/hadoop-2.6.4.tar.gz +RUN wget http://archive.apache.org/dist/spark/spark-1.6.0/spark-1.6.0-bin-hadoop2.6.tgz + +RUN gunzip hadoop-2.6.4.tar.gz +RUN gunzip spark-1.6.0-bin-hadoop2.6.tgz +RUN tar -xf hadoop-2.6.4.tar +RUN tar -xf spark-1.6.0-bin-hadoop2.6.tar + +RUN sudo cp -r hadoop-2.6.4 /usr/local/hadoop +RUN sudo cp -r spark-1.6.0-bin-hadoop2.6 /usr/local/spark + +RUN rm hadoop-2.6.4.tar spark-1.6.0-bin-hadoop2.6.tar +RUN rm -rf hadoop-2.6.4/ spark-1.6.0-bin-hadoop2.6/ + +RUN sudo mkdir -p /usr/local/hadoop/hadoop_data/hdfs/namenode +RUN sudo mkdir -p /usr/local/hadoop/hadoop_data/hdfs/datanode + +# Environment variables +ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64 +ENV HADOOP_HOME=/usr/local/hadoop +ENV SPARK_HOME=/usr/local/spark +ENV PATH $PATH:$JAVA_HOME/bin +ENV PATH $PATH:$HADOOP_HOME/bin +ENV PATH $PATH:$HADOOP_HOME/sbin +ENV PATH $PATH:$SPARK_HOME/bin +ENV PATH $PATH:$SPARK_HOME/sbin +ENV HADOOP_MAPRED_HOME /usr/local/hadoop +ENV HADOOP_COMMON_HOME /usr/local/hadoop +ENV HADOOP_HDFS_HOME /usr/local/hadoop +ENV HADOOP_CONF_DIR /usr/local/hadoop/etc/hadoop +ENV YARN_CONF_DIR /usr/local/hadoop/etc/hadoop +ENV YARN_HOME /usr/local/hadoop +ENV HADOOP_COMMON_LIB_NATIVE_DIR /usr/local/hadoop/lib/native +ENV HADOOP_OPTS "-Djava.library.path=$HADOOP_HOME/lib" + +# Clone CaffeOnSpark +ENV CAFFE_ON_SPARK=/opt/CaffeOnSpark +WORKDIR $CAFFE_ON_SPARK +RUN git clone https://github.com/yahoo/CaffeOnSpark.git . --recursive + +# Some of the Hadoop part extracted from "https://hub.docker.com/r/sequenceiq/hadoop-docker/~/dockerfile/" +RUN mkdir $HADOOP_HOME/input +RUN cp $HADOOP_HOME/etc/hadoop/*.xml $HADOOP_HOME/input +RUN cd /usr/local/hadoop/input + +# Copy .xml files. +RUN cp ${CAFFE_ON_SPARK}/scripts/*.xml ${HADOOP_HOME}/etc/hadoop + +# Format namenode and finish hadoop, spark installations. +RUN $HADOOP_HOME/bin/hdfs namenode -format + +RUN ls /root/.ssh/ +ADD config/ssh_config /root/.ssh/config +RUN chmod 600 /root/.ssh/config +RUN chown root:root /root/.ssh/config + +ADD config/bootstrap.sh /etc/bootstrap.sh +RUN chown root:root /etc/bootstrap.sh +RUN chmod 700 /etc/bootstrap.sh + +ENV BOOTSTRAP /etc/bootstrap.sh + +RUN sed -i '/^export JAVA_HOME/ s:.*:export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64\nexport HADOOP_HOME=/usr/local/hadoop\n:' $HADOOP_HOME/etc/hadoop/hadoop-env.sh +RUN sed -i '/^export HADOOP_CONF_DIR/ s:.*:export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/:' $HADOOP_HOME/etc/hadoop/hadoop-env.sh + +# workingaround docker.io build error +RUN ls -la /usr/local/hadoop/etc/hadoop/*-env.sh +RUN chmod +x /usr/local/hadoop/etc/hadoop/*-env.sh +RUN ls -la /usr/local/hadoop/etc/hadoop/*-env.sh + +# fix the 254 error code +RUN sed -i "/^[^#]*UsePAM/ s/.*/#&/" /etc/ssh/sshd_config +RUN echo "UsePAM no" >> /etc/ssh/sshd_config +RUN echo "Port 2122" >> /etc/ssh/sshd_config + +RUN service ssh start && $HADOOP_HOME/etc/hadoop/hadoop-env.sh && $HADOOP_HOME/sbin/start-dfs.sh && $HADOOP_HOME/bin/hdfs dfs -mkdir -p /user/root +RUN service ssh start && $HADOOP_HOME/etc/hadoop/hadoop-env.sh && $HADOOP_HOME/sbin/start-dfs.sh && $HADOOP_HOME/bin/hdfs dfs -put $HADOOP_HOME/etc/hadoop/ input + +CMD ["/etc/bootstrap.sh", "-bash"] + +# Hdfs ports +EXPOSE 50010 50020 50070 50075 50090 8020 9000 +# Mapred ports +EXPOSE 10020 19888 +#Yarn ports +EXPOSE 8030 8031 8032 8033 8040 8042 8088 +#Other ports +EXPOSE 49707 2122 + +# Continue with CaffeOnSpark build. +# ENV CAFFE_ON_SPARK=/opt/CaffeOnSpark +WORKDIR $CAFFE_ON_SPARK +# RUN git clone https://github.com/yahoo/CaffeOnSpark.git . --recursive +RUN cp caffe-public/Makefile.config.example caffe-public/Makefile.config +RUN echo "INCLUDE_DIRS += ${JAVA_HOME}/include" >> caffe-public/Makefile.config +#RUN sed -i "s/# USE_CUDNN := 1/USE_CUDNN := 1/g" caffe-public/Makefile.config +RUN sed -i "s|BLAS := atlas|BLAS := open|g" caffe-public/Makefile.config + +RUN make build + +ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:$CAFFE_ON_SPARK/caffe-public/distribute/lib:$CAFFE_ON_SPARK/caffe-distri/distribute/lib + +WORKDIR /root From 33a651a5cb528331cf5b1cdac1a00392521a6d74 Mon Sep 17 00:00:00 2001 From: Arun Das Date: Tue, 10 Jan 2017 23:38:22 -0600 Subject: [PATCH 10/17] Rename docker/standalone/gpu/config/Dockerfile to docker/standalone/gpu/Dockerfile --- docker/standalone/gpu/{config => }/Dockerfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker/standalone/gpu/{config => }/Dockerfile (100%) diff --git a/docker/standalone/gpu/config/Dockerfile b/docker/standalone/gpu/Dockerfile similarity index 100% rename from docker/standalone/gpu/config/Dockerfile rename to docker/standalone/gpu/Dockerfile From 329f1d1ac0e180bb10b5a7f454d5d309fcc54433 Mon Sep 17 00:00:00 2001 From: Arun Das Date: Tue, 10 Jan 2017 23:46:42 -0600 Subject: [PATCH 11/17] Update README.md --- docker/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/README.md b/docker/README.md index aaa8b4d..6810c62 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,5 +1,7 @@ # CaffeOnSpark Standalone Docker +Dockerfiles for both CPU and GPU builds are available in `standalone` folder. To use the CPU only version use the commands given. A GPU version of docker can be run using the command [`nvidia-docker`](https://github.com/NVIDIA/nvidia-docker) instead of `docker` using the `standalone/gpu` folder. + Dockerfiles for CPU build is provided in `standalone/cpu` folder. The image can be built by running: ``` docker build -t caffeonspark:cpu standalone/cpu From e8d8399c82486320bc46f0e79a5b17ade5080fe7 Mon Sep 17 00:00:00 2001 From: Arun Das Date: Fri, 20 Jan 2017 18:18:00 -0600 Subject: [PATCH 12/17] Update Dockerfile added copyright --- docker/standalone/cpu/Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/standalone/cpu/Dockerfile b/docker/standalone/cpu/Dockerfile index 76bbd01..8bbb3b6 100644 --- a/docker/standalone/cpu/Dockerfile +++ b/docker/standalone/cpu/Dockerfile @@ -1,5 +1,10 @@ +# Copyright 2016 Yahoo Inc. +# Licensed under the terms of the Apache 2.0 license. +# Please see LICENSE file in the project root for terms. +# +# This file is the dockerfile to setup caffeonspark cpu standalone version. + FROM ubuntu:14.04 -MAINTAINER arun.das@my.utsa.edu RUN apt-get update && apt-get install -y software-properties-common RUN add-apt-repository ppa:openjdk-r/ppa From d3c12ce7dbac4e91a358c619254290d94a902eaa Mon Sep 17 00:00:00 2001 From: Arun Das Date: Fri, 20 Jan 2017 18:20:08 -0600 Subject: [PATCH 13/17] Update bootstrap.sh --- docker/standalone/cpu/config/bootstrap.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/standalone/cpu/config/bootstrap.sh b/docker/standalone/cpu/config/bootstrap.sh index 5797e47..a510078 100644 --- a/docker/standalone/cpu/config/bootstrap.sh +++ b/docker/standalone/cpu/config/bootstrap.sh @@ -1,4 +1,9 @@ #!/bin/bash +# Copyright 2016 Yahoo Inc. +# Licensed under the terms of the Apache 2.0 license. +# Please see LICENSE file in the project root for terms. +# +# This script starts hadoop dfs and yarn while the docker container is started. : ${HADOOP_PREFIX:=/usr/local/hadoop} From 039378fe3faab75b8a58cf265a3ad431ea283026 Mon Sep 17 00:00:00 2001 From: Arun Das Date: Fri, 20 Jan 2017 18:24:58 -0600 Subject: [PATCH 14/17] Update Dockerfile Added copyright --- docker/standalone/gpu/Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/standalone/gpu/Dockerfile b/docker/standalone/gpu/Dockerfile index cc50144..43c154f 100644 --- a/docker/standalone/gpu/Dockerfile +++ b/docker/standalone/gpu/Dockerfile @@ -1,5 +1,10 @@ +# Copyright 2016 Yahoo Inc. +# Licensed under the terms of the Apache 2.0 license. +# Please see LICENSE file in the project root for terms. +# +# This file is the dockerfile to setup caffeonspark cpu standalone version. + FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 -MAINTAINER arun.das@my.utsa.edu RUN apt-get update && apt-get install -y software-properties-common RUN add-apt-repository ppa:openjdk-r/ppa From 36de124c952c71d0788c18a957081d71606d29dd Mon Sep 17 00:00:00 2001 From: Arun Das Date: Fri, 20 Jan 2017 18:25:42 -0600 Subject: [PATCH 15/17] Update bootstrap.sh Added copyright --- docker/standalone/gpu/config/bootstrap.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/standalone/gpu/config/bootstrap.sh b/docker/standalone/gpu/config/bootstrap.sh index 5797e47..a510078 100644 --- a/docker/standalone/gpu/config/bootstrap.sh +++ b/docker/standalone/gpu/config/bootstrap.sh @@ -1,4 +1,9 @@ #!/bin/bash +# Copyright 2016 Yahoo Inc. +# Licensed under the terms of the Apache 2.0 license. +# Please see LICENSE file in the project root for terms. +# +# This script starts hadoop dfs and yarn while the docker container is started. : ${HADOOP_PREFIX:=/usr/local/hadoop} From 473e1e005a77f7869e8835af1ad3e5a8a1124121 Mon Sep 17 00:00:00 2001 From: Arun Das Date: Fri, 20 Jan 2017 19:45:17 -0600 Subject: [PATCH 16/17] Update ssh_config Added copyright --- docker/standalone/cpu/config/ssh_config | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/standalone/cpu/config/ssh_config b/docker/standalone/cpu/config/ssh_config index 535f9d3..f926b36 100644 --- a/docker/standalone/cpu/config/ssh_config +++ b/docker/standalone/cpu/config/ssh_config @@ -1,3 +1,9 @@ +# Copyright 2016 Yahoo Inc. +# Licensed under the terms of the Apache 2.0 license. +# Please see LICENSE file in the project root for terms. +# +# This file creates user specific ssh configuration +# Host * UserKnownHostsFile /dev/null StrictHostKeyChecking no From 7ff5306144dc4ce4ddedc8c235bcacf657583abb Mon Sep 17 00:00:00 2001 From: Arun Das Date: Fri, 20 Jan 2017 19:45:36 -0600 Subject: [PATCH 17/17] Update ssh_config --- docker/standalone/gpu/config/ssh_config | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/standalone/gpu/config/ssh_config b/docker/standalone/gpu/config/ssh_config index 7795f49..b2f8659 100644 --- a/docker/standalone/gpu/config/ssh_config +++ b/docker/standalone/gpu/config/ssh_config @@ -1,3 +1,9 @@ +# Copyright 2016 Yahoo Inc. +# Licensed under the terms of the Apache 2.0 license. +# Please see LICENSE file in the project root for terms. +# +# This file creates user specific ssh configuration +# Host * UserKnownHostsFile /dev/null StrictHostKeyChecking no