[TST] Migrate CircleCI tests to docker

.noserc

@@ -1,11 +1,10 @@
 [nosetests]
 verbosity=3
 
+logging-level=DEBUG
+with-doctest=1
+with-xunit=1
 with-coverage=1
 cover-branches=1
 cover-xml=1
-cover-xml-file=./coverage.xml
 cover-min-percentage=50
-
-
-with-xunit=1

docker/test-image/Dockerfile → Dockerfile

@@ -26,26 +26,49 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-FROM nipype/testnipype
+FROM nipype/testnipypedata:latest
 MAINTAINER Stanford Center for Reproducible Neuroscience <crn.poldracklab@gmail.com>
 
 # Preparations
 RUN ln -snf /bin/bash /bin/sh
-WORKDIR /root
-
-RUN mkdir -p ~/examples/ && ln -sf /usr/share/fsl-feeds/ ~/examples/feeds
-RUN curl -sSL "https://dl.dropbox.com/s/jzgq2nupxyz36bp/nipype-tutorial.tar.bz2" && \
-    tar jxvf nipype-tutorial.tar.bz2 && \
-    mv nipype-tutorial/* ~/examples/ && \
-    curl -sSL "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt1.tar.gz" && \
-    curl -sSL "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt2.tar.gz" && \
-    curl -sSL "http://fsl.fmrib.ox.ac.uk/fslcourse/tbss.tar.gz" && \
-    mkdir ~/examples/fsl_course_data && \
-    tar zxvf fdt1.tar.gz -C ~/examples/fsl_course_data && \
-    tar zxvf fdt2.tar.gz -C ~/examples/fsl_course_data && \
-    tar zxvf tbss.tar.gz -C ~/examples/fsl_course_data && \
-    echo 'export FSL_COURSE_DATA=/root/examples/fsl_course_data' >> /etc/profile.d/crn_neuro.sh && \
-    echo 'export FSL_COURSE_DATA=/root/examples/fsl_course_data' >> /etc/bash.bashrc
-ENV FSL_COURSE_DATA /root/examples/fsl_course_data
-
-CMD ["/bin/bash"]
+
+# Install this branch's code
+WORKDIR /root/src
+
+# Install matplotlib, sphinx and coverage to build documentation
+# and run tests with coverage
+RUN source activate nipypetests-2.7 && \
+    pip install matplotlib sphinx coverage && \
+    source activate nipypetests-3.4 && \
+    pip install matplotlib sphinx coverage && \
+    source activate nipypetests-3.5 && \
+    pip install matplotlib sphinx coverage
+
+ADD . nipype/
+
+# Install the checked out version of nipype, check that requirements are
+# installed and install it for each of the three environments.
+RUN cd nipype/ && \
+    source activate nipypetests-2.7 && \
+    pip install -r requirements.txt && \
+    pip install -e .
+
+RUN cd nipype/ && \
+    source activate nipypetests-3.4 && \
+    pip install -r requirements.txt && \
+    pip install -e .
+
+RUN cd nipype/ && \
+    source activate nipypetests-3.5 && \
+    pip install -r requirements.txt && \
+    pip install -e .
+
+WORKDIR /scratch
+
+# Install entrypoints
+ADD docker/circleci/run_* /usr/bin/
+RUN chmod +x /usr/bin/run_*
+
+# RUN echo 'source /etc/profile.d/nipype_tests.sh' >> /etc/bash.bashrc
+ENTRYPOINT ["/usr/bin/run_examples.sh"]
+

circle.yml

@@ -1,78 +1,46 @@
+machine:
+  services:
+    - docker
+
 dependencies:
   cache_directories:
-    - "~/.apt-cache"
-    - "~/examples/data"
-    - "~/examples/fsdata"
-    - "~/mcr"
-    - "~/spm12"
-    - "~/examples/fsl_course_data"
+    - "~/docker"
+
   pre:
-    # Let CircleCI cache the apt archive
-    - sudo rm -rf /var/cache/apt/archives && sudo ln -s ~/.apt-cache /var/cache/apt/archives && mkdir -p ~/.apt-cache/partial
-    - bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh)
+    - mkdir -p "~/scratch/nose"
+
   override:
-    # Install apt packages
-    - sudo apt-get install -y fsl-core fsl-atlases fsl-mni152-templates fsl-feeds afni swig python-vtk xvfb fusefat
-    - echo 'source /etc/fsl/fsl.sh' >> $HOME/.profile
-    - echo 'source /etc/afni/afni.sh' >> $HOME/.profile
-    - mkdir -p ~/examples/ && ln -sf /usr/share/fsl-feeds/ ~/examples/feeds
-    # Enable system-wide vtk
-    - ln -sf /usr/lib/pymodules/python2.7/vtk ~/virtualenvs/venv-system/lib/python2.7/site-packages/
-    # Set up python environment
-    - pip install --upgrade pip
-    - pip install -e .
-    - pip install matplotlib sphinx ipython boto coverage dipy
-    # Add tvtk
-    - pip install http://effbot.org/downloads/Imaging-1.1.7.tar.gz
-    - pip install -e git+https://github.com/enthought/etsdevtools.git#egg=etsdevtools
-    - pip install -e git+https://github.com/enthought/blockcanvas.git#egg=blockcanvas
-    - pip install -e git+https://github.com/enthought/etsproxy.git#egg=etsproxy
-    - pip install -e git+https://github.com/enthought/ets.git#egg=ets
-    - pip install https://github.com/rtfd/readthedocs-sphinx-ext/archive/master.zip
-    - gem install fakes3
-    - if [[ ! -d ~/examples/data ]]; then wget "https://dl.dropbox.com/s/jzgq2nupxyz36bp/nipype-tutorial.tar.bz2" && tar jxvf nipype-tutorial.tar.bz2 && mv nipype-tutorial/* ~/examples/; fi
-    - if [[ ! -d ~/examples/fsl_course_data ]]; then wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt1.tar.gz" && wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt2.tar.gz" && wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/tbss.tar.gz" && mkdir ~/examples/fsl_course_data && tar zxvf fdt1.tar.gz -C ~/examples/fsl_course_data && tar zxvf fdt2.tar.gz -C ~/examples/fsl_course_data && tar zxvf tbss.tar.gz -C ~/examples/fsl_course_data; fi
-    - bash ~/nipype/tools/install_spm_mcr.sh
-    - mkdir -p ~/.nipype && echo '[logging]' > ~/.nipype/nipype.cfg && echo 'workflow_level = DEBUG' >> ~/.nipype/nipype.cfg && echo 'interface_level = DEBUG' >> ~/.nipype/nipype.cfg && echo 'filemanip_level = DEBUG' >> ~/.nipype/nipype.cfg
-machine:
-  environment:
-    FSLOUTPUTTYPE: NIFTI_GZ
+    - if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi
+    - docker build -t nipype/testbench:latest . :
+        timeout: 1600
+    - mkdir -p ~/docker; docker save nipype/testbench:latest > ~/docker/image.tar :
+        timeout: 1600
+
 test:
   override:
-    - mkdir -p ${CIRCLE_TEST_REPORTS}/nose
-    - source $HOME/.profile; nosetests --with-doctest --xunit-file="${CIRCLE_TEST_REPORTS}/nose/${CIRCLE_PROJECT_REPONAME}.xml" -c ./.noserc --logging-level=DEBUG --verbosity=3:
-        environment:
-          SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
-          FORCE_SPMMCR: 1
-          FSL_COURSE_DATA: "$HOME/examples/fsl_course_data"
-        timeout: 2600
-    - set -o pipefail && cd doc && make html 2>&1 | tee ~/log.txt
-    - cat ~/log.txt && if grep -q "ERROR" ~/log.txt; then false; else true; fi
-    - source $HOME/.profile; python ~/nipype/tools/run_examples.py test_spm Linear workflow3d workflow4d:
-        pwd: ../examples
-        environment:
-          SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
-          FORCE_SPMMCR: 1
+    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch --entrypoint="/usr/bin/run_builddocs.sh" nipype/testbench
+    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench test_spm Linear /root/examples/ workflow3d :
         timeout: 1600
-    - source $HOME/.profile; python ~/nipype/tools/run_examples.py fmri_fsl_feeds Linear l1pipeline:
-        pwd: ../examples
-    - source $HOME/.profile; python ~/nipype/tools/run_examples.py fmri_spm_dartel Linear level1 l2pipeline:
-        pwd: ../examples
-        environment:
-          SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
-          FORCE_SPMMCR: 1
+    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench test_spm Linear /root/examples/ workflow4d :
         timeout: 1600
-    - source $HOME/.profile; python ~/nipype/tools/run_examples.py fmri_fsl_reuse Linear level1_workflow:
-        pwd: ../examples
-    - source $HOME/.profile; python ~/nipype/tools/run_examples.py fmri_spm_nested Linear level1 l2pipeline:
-        pwd: ../examples
-        environment:
-          SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
-          FORCE_SPMMCR: 1
+    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_fsl_feeds Linear /root/examples/ l1pipeline
+    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_spm_dartel Linear /root/examples/ level1 :
+        timeout: 1600
+    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_spm_dartel Linear /root/examples/ l2pipeline :
+        timeout: 1600
+    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_fsl_reuse Linear /root/examples/ level1_workflow
+    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_spm_nested Linear /root/examples/ level1
+    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_spm_nested Linear /root/examples/ l2pipeline
+    - docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch --entrypoint="/usr/bin/run_nosetests.sh" nipype/testbench :
+        timeout: 2600
+  post:
+    - bash docker/circleci/teardown.sh
 
 general:
   artifacts:
-    - "doc/_build/html"
-    - "~/log.txt"
-    - "nosetests.xml"
-    - "coverage.xml"
+    - "~/docs"
+    - "~/logs"
+    - "~/coverage.xml"
+    - "~/nosetests.xml"
+    - "~/builddocs.log"
+    - "~/scratch"

docker/circleci/run_builddocs.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+for i in /etc/profile.d/*.sh; do
+    source $i
+done
+source activate nipypetests-2.7
+
+mkdir -p /scratch/docs
+set -o pipefail && cd /root/src/nipype/doc && make html 2>&1 | tee /scratch/builddocs.log
+cp -r /root/src/nipype/doc/_build/html/* /scratch/docs/
+chmod 777 -R /scratch/docs
+chmod 777 /scratch/builddocs.log
+cat /scratch/builddocs.log && if grep -q "ERROR" /scratch/builddocs.log; then false; else true; fi

docker/circleci/run_examples.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+for i in /etc/profile.d/*.sh; do
+    source $i
+done
+
+mkdir -p /root/.nipype
+echo '[logging]' > /root/.nipype/nipype.cfg
+echo 'workflow_level = DEBUG' >> /root/.nipype/nipype.cfg
+echo 'interface_level = DEBUG' >> /root/.nipype/nipype.cfg
+echo 'filemanip_level = DEBUG' >> /root/.nipype/nipype.cfg
+
+source activate nipypetests-2.7
+python /root/src/nipype/tools/run_examples.py $@

docker/circleci/run_nosetests.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+for i in /etc/profile.d/*.sh; do
+    source $i
+done
+source activate nipypetests-2.7
+cd /root/src/nipype
+mkdir -p /scratch/nose
+nosetests -c /root/src/nipype/.noserc --xunit-file="/scratch/nosetests.xml" --cover-xml-file="/scratch/coverage.xml"
+chmod 777 /scratch/nosetests.xml
+chmod 777 /scratch/coverage.xml
+chmod 777 -R /scratch/nose

docker/circleci/teardown.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# This script puts all artifacts in place after the smoke tests
+#
+#
+set -u
+set -e
+
+mkdir -p ${CIRCLE_TEST_REPORTS}/nose
+sudo mv ~/scratch/builddocs.log ~/builddocs.log
+sudo cp ~/scratch/nosetests.xml ${CIRCLE_TEST_REPORTS}/nose/${CIRCLE_PROJECT_REPONAME}.xml
+sudo mv ~/scratch/coverage.xml ~/coverage.xml
+mkdir -p ~/docs
+sudo mv ~/scratch/docs/* ~/docs/
+mkdir -p ~/logs
+sudo mv ~/scratch/logs/* ~/logs/

docker/test-image-base/Dockerfile

@@ -57,7 +57,8 @@ RUN apt-get update && \
                        libqhull-dev \
                        libxft-dev \
                        libjpeg-dev \
-                       libyaml-dev
+                       libyaml-dev \
+                       graphviz
 
 
 # Install ANTs
@@ -96,11 +97,10 @@ RUN echo "destinationFolder=/root/mcr" > mcr_options.txt && \
     unzip matlab_installer/installer.zip -d matlab_installer/ && \
     matlab_installer/install -inputFile mcr_options.txt && \
     rm -rf matlab_installer mcr_options.txt && \
-    echo 'export LD_LIBRARY_PATH=/root/mcr/v85/runtime/glnxa64:/root/mcr/v85/bin/glnxa64:/root/mcr/v85/sys/os/glnxa64:$LD_LIBRARY_PATH' >> /etc/profile.d/nipype_deps.sh && \
+#    echo 'export LD_LIBRARY_PATH=/root/mcr/v85/runtime/glnxa64:/root/mcr/v85/bin/glnxa64:/root/mcr/v85/sys/os/glnxa64:$LD_LIBRARY_PATH' >> /etc/profile.d/nipype_deps.sh && \
     echo 'export SPMMCRCMD="/root/spm12/run_spm12.sh /root/mcr/v85/ script"' >> /etc/profile.d/nipype_deps.sh && \ 
     echo 'export FORCE_SPMMCR=1' >> /etc/profile.d/nipype_deps.sh
 
-ENV LD_LIBRARY_PATH "/root/mcr/v85/runtime/glnxa64:/root/mcr/v85/bin/glnxa64:/root/mcr/v85/sys/os/glnxa64:$LD_LIBRARY_PATH"
 ENV SPMMCRCMD "/root/spm12/run_spm12.sh /root/mcr/v85/ script"
 ENV FORCE_SPMMCR 1
 

docker/test-image-data/Dockerfile

@@ -0,0 +1,58 @@
+# Copyright (c) 2016, The developers of the Stanford CRN
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of crn_base nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+FROM nipype/testnipype:latest
+MAINTAINER Stanford Center for Reproducible Neuroscience <crn.poldracklab@gmail.com>
+
+# Preparations
+RUN ln -snf /bin/bash /bin/sh
+WORKDIR /root
+
+# Install fsl-feeds and the nipype-tutorial data
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install -y fsl-feeds && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
+    mkdir -p ~/examples/ && \
+    ln -sf /usr/share/fsl-feeds/ ~/examples/feeds && \
+    curl -sSL "https://dl.dropbox.com/s/jzgq2nupxyz36bp/nipype-tutorial.tar.bz2" -o nipype-tutorial.tar.bz2 && \
+    tar jxf nipype-tutorial.tar.bz2 -C /root/examples
+
+# RUN curl -sSL "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt1.tar.gz" -o fdt1.tar.gz && \
+#     curl -sSL "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt2.tar.gz" -o fdt2.tar.gz && \
+#     curl -sSL "http://fsl.fmrib.ox.ac.uk/fslcourse/tbss.tar.gz" -o tbss.tar.gz && \
+#     mkdir ~/examples/fsl_course_data && \
+#     echo 'Untarring fsl course data...' && \
+#     tar zxf fdt1.tar.gz -C ~/examples/fsl_course_data && \
+#     tar zxf fdt2.tar.gz -C ~/examples/fsl_course_data && \
+#     tar zxf tbss.tar.gz -C ~/examples/fsl_course_data && \
+#     echo 'export FSL_COURSE_DATA=/root/examples/fsl_course_data' >> /etc/profile.d/nipype_data.sh
+# ENV FSL_COURSE_DATA /root/examples/fsl_course_data
+# RUN echo 'source /etc/profile.d/nipype_data.sh' >> /etc/bash.bashrc
+
+CMD ["/bin/bash"]

docker/test-image-nipype/Dockerfile

@@ -26,7 +26,7 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-FROM nipype/testbase
+FROM nipype/testbase:latest
 MAINTAINER Stanford Center for Reproducible Neuroscience <crn.poldracklab@gmail.com>
 
 # Preparations
@@ -53,18 +53,21 @@ RUN conda config --add channels conda-forge
 RUN conda create -y -n nipypetests-2.7 lockfile nipype && \
     echo '#!/bin/bash' >> /etc/profile.d/nipype.sh && \
     echo '#!/bin/bash' >> /etc/bashrc && \
-    echo 'source activate nipypetests-2.7' >> /etc/profile.d/nipype.sh && \
-    echo 'source activate nipypetests-2.7' >> /etc/bashrc
+    echo 'source activate nipypetests-2.7' >> /etc/profile.d/nipype.sh
 
 # Create conda environment
 RUN conda create -y -n nipypetests-3.4 lockfile nipype python=3.4 
 
 # Create conda environment
 RUN conda create -y -n nipypetests-3.5 lockfile nipype python=3.5
 
-# Install pip
-# RUN source activate nipypetests-2.7 && \
-#     pip install --upgrade pip && \
+# Install dipy
+RUN source activate nipypetests-2.7 && \
+    pip install dipy && \
+    source activate nipypetests-3.4 && \
+    pip install dipy && \
+    source activate nipypetests-3.5 && \
+    pip install dipy
 
 RUN echo "source /etc/profile.d/nipype.sh" >> /etc/bash.bashrc
 CMD ["/bin/bash"]