diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..c23fa4a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,18 @@ +language: python + +python: + - 3.6 +services: + - docker + +os: + - linux + + +notifications: + email: true + +before_install: + - docker build -t pns . +script: + - docker run pns python -c "print('built')" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..98d25f5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +# Distributed under the terms of the Modified BSD License. +ARG BASE_CONTAINER=jupyter/scipy-notebook +FROM $BASE_CONTAINER +LABEL maintainer="Russell " +USER root +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + fonts-dejavu \ + gfortran \ + gcc && \ + rm -rf /var/lib/apt/lists/* +RUN apt-get upgrade +RUN apt-get install -y ncurses-base ncurses-bin +RUN apt-get update && apt-get install -y bzip2 ca-certificates automake libtool \ + libncurses5-dev libreadline-dev libgsl0-dev cmake ssh +USER jovyan + +WORKDIR $HOME +RUN \ + wget http://www.neuron.yale.edu/ftp/neuron/versions/v7.7/nrn-7.7.tar.gz && \ + tar -xzf nrn-7.7.tar.gz && \ + rm nrn-7.7.tar.gz +WORKDIR $HOME/nrn-7.7 +ENV PATH /usr/bin/python3/python:/opt/conda/bin:/opt/conda/bin/conda:/opt/conda/bin/python:$PATH +RUN ./configure --prefix=`pwd` --without-iv --with-nrnpython=/opt/conda/bin/python3 +USER root +RUN sudo make all && \ + make install +USER jovyan +WORKDIR src/nrnpython +RUN python setup.py install +RUN python -c "import neuron" +ENV NEURON_HOME $HOME/nrn-7.7/x86_64 +ENV PATH $NEURON_HOME/bin:$PATH diff --git a/README.md b/README.md index 5038e3f..655db92 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ +[![Build Status](https://travis-ci.com/russelljjarvis/docker-stacks-returned.svg?branch=scidash)](https://travis-ci.com/russelljjarvis/docker-stacks-returned) + +[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/russelljjarvis/docker-stacks-returned/master) + # A hierarchy of Docker images for using SciDash tools -Purpose: The building of development environments raises an unnecessary technical problem. In many cases even if someone has the technical capacity to build, it does not mean they will have time to do so. In order to remove this barrier to participation we have created some Docker images to allow people to use our tools. Each of these images serves Jupyter notebooks and including a Python scientific computing stack (extended from jupyter/docker-stacks/scipy-notebook), and some combination of NEURON-7.4, scoop, DEAP, sciunit, and neuronunit. +Purpose: The building of development environments raises an unnecessary technical problem. In many cases even if someone has the technical capacity to build, it does not mean they will have time to do so. In order to remove this barrier to participation we have created some Docker images to allow people to use our tools. Each of these images serves Jupyter notebooks and including a Python scientific computing stack (extended from jupyter/docker-stacks/scipy-notebook), and some combination of NEURON-7.7, DASK, DEAP, sciunit, and neuronunit. ![Docker container hierarchy](https://github.com/scidash/assets/blob/master/mockups/Containerization%20and%20Continuous%20Integration%20Workflow.png) @@ -77,3 +81,19 @@ c.DockerSpawner.extra_create_kwargs.update({ }) ``` Then run `jupyterhub`, visit the indicated URL (e.g. `10.200.11.1:8000`) in your browser, and log in as a system user. + +Ipython Parallel seems to benefit from tini + +https://github.com/krallin/tini + +NOTE: If you are using Docker 1.13 or greater, Tini is included in Docker itself. This includes all versions of Docker CE. To enable Tini, just pass the --init flag to docker run. + +Why tini? (init backwards) + +Using Tini has several benefits: + +It protects you from software that accidentally creates zombie processes, which can (over time!) starve your entire system for PIDs (and make it unusable). +It ensures that the default signal handlers work for the software you run in your Docker image. For example, with Tini, SIGTERM properly terminates your process even if you didn't explicitly install a signal handler for it. +It does so completely transparently! Docker images that work without Tini will work with Tini without any changes. +If you'd like more detail on why this is useful, review this issue discussion: What is advantage of Tini?. + diff --git a/build b/build old mode 100755 new mode 100644 diff --git a/build-all b/build-all index 28a8b66..6d66c68 100755 --- a/build-all +++ b/build-all @@ -1,6 +1,6 @@ -for NAME in scipy-notebook-plus sciunit neuron-mpi-neuroml neuronunit neuronunit-openworm neuronunit-neuroconstruct neuronunit-optimization; do +for name in scipy-notebook-plus sciunit neuron-mpi-neuroml neuronunit neuronunit-optimization; do echo "********************************" - echo "Building scidash/$NAME" + echo "Building scidash/$name" echo "********************************" - ./build $NAME + ./build $name done \ No newline at end of file diff --git a/dev/Dockerfile b/dev/Dockerfile new file mode 100644 index 0000000..fa1b40c --- /dev/null +++ b/dev/Dockerfile @@ -0,0 +1,15 @@ +FROM scidash/neuronunit-optimization +COPY stdout_worker.py $HOME/stdout_worker.py +COPY func2rc.sh $HOME/func2rc.sh +RUN cat $HOME/func2rc.sh >> ~/.bashrc +RUN sudo chown -R jovyan /home/jovyan/work +ENV QT_QPA_PLATFORM offscreen +ENV PYTHONPATH $PYTHONPATH/home/jovyan/neuronunit/neuronunit/optimization +RUN /bin/bash -c "source $HOME/func2rc.sh" +WORKDIR $HOME/neuronunit/neuronunit +RUN sudo /opt/conda/bin/pip install dask + + + +RUN sudo rm -r $HOME/work +RUN sudo rm -r /opt/conda/lib/python3.5/site-packages/neuronunit* \ No newline at end of file diff --git a/efel_dm/Dockerfile b/efel_dm/Dockerfile new file mode 100644 index 0000000..5c79284 --- /dev/null +++ b/efel_dm/Dockerfile @@ -0,0 +1,56 @@ +# author Russell Jarvis rjjarvis@asu.edu +# author Rick Gerkin rgerkin@asu.edu +# neuronunit-showcase + +FROM scidash/neuronunit-optimization +RUN sudo rm -rf /opt/conda/lib/python3.5/site-packages/setuptools-27.2.0-py3.5.egg +RUN sudo /opt/conda/bin/pip install --upgrade pip +RUN sudo /opt/conda/bin/pip install --upgrade setuptools +RUN sudo rm -rf /opt/conda/lib/python3.5/site-packages/allensdk-0.12.4.1-py3.5.egg +RUN pip install git+https://github.com/fun-zoological-computing/AllenSDK.git +# RUN conda update -n base conda +USER $NB_USER + +# RUN pip uninstall tornado +# move to a docker install script. +RUN sudo /opt/conda/bin/pip install --upgrade pip +# RUN sudo /opt/conda/bin/pip install git+https://github.com/scidash/sciunit.git@dev # --upgrade# --process-dependency-links --upgrade +RUN sudo /opt/conda/bin/conda install -y cython + +RUN sudo /opt/conda/bin/pip install git+https://github.com/brian-team/brian2.git +# faster glif +# RUN sudo /opt/conda/bin/pip install allensdk --upgrade +# RUN sudo /opt/conda/bin/conda install -y numpy deap dask numba +# RUN sudo /opt/conda/bin/pip install allensdk --upgrade + +RUN sudo /opt/conda/bin/pip install git+git://github.com/BlueBrain/eFEL + +WORKDIR $HOME +# RUN sudo /opt/conda/bin/pip install git+ +# RUN git clone https://github.com/AllenInstitute/sonata +# WORKDIR /sonata/src/pysonata +# RUN echo $pwd +# RUN ls *.py +# RUN python setup.py install + +RUN git clone https://github.com/AllenInstitute/bmtk.git +WORKDIR bmtk +RUN python setup.py install + +WORKDIR $HOME +RUN sudo apt-get update -y +RUN sudo apt-get upgrade -y +RUN sudo apt-get -y install libhdf5-dev vim +# RUN sudo /opt/bin/conda/pip install setuptools +# RUN sudo /opt/conda/bin/pip install --upgrade protobuf + +# RUN git clone +# ing/AllenSDK +# WORKDIR AllenSDK +# RUN python setup.py install +RUN sudo /opt/conda/bin/pip3 install pyscaffold +RUN sudo /opt/conda/bin/pip install pip --upgrade +RUN sudo /opt/conda/bin/pip3 install SALib +# git+https://github.com/SALib/SALib.git + +ENTRYPOINT /bin/bash \ No newline at end of file diff --git a/julia_nb/Dockerfile b/julia_nb/Dockerfile new file mode 100644 index 0000000..445d27b --- /dev/null +++ b/julia_nb/Dockerfile @@ -0,0 +1,41 @@ +FROM with_r +USER root +#RUN apt-get install -y julia +# install Julia packages in /opt/julia instead of $HOME +ENV JULIA_DEPOT_PATH=/opt/julia +ENV JULIA_PKGDIR=/opt/julia +ENV JULIA_VERSION=1.0.0 + +RUN mkdir /opt/julia-${JULIA_VERSION} && \ + cd /tmp && \ + wget -q https://julialang-s3.julialang.org/bin/linux/x64/`echo ${JULIA_VERSION} | cut -d. -f 1,2`/julia-${JULIA_VERSION}-linux-x86_64.tar.gz && \ + echo "bea4570d7358016d8ed29d2c15787dbefaea3e746c570763e7ad6040f17831f3 *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \ + tar xzf julia-${JULIA_VERSION}-linux-x86_64.tar.gz -C /opt/julia-${JULIA_VERSION} --strip-components=1 && \ + rm /tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz +RUN ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia + +# Show Julia where conda libraries are \ +RUN mkdir /etc/julia && \ + echo "push!(Libdl.DL_LOAD_PATH, \"$CONDA_DIR/lib\")" >> /etc/julia/juliarc.jl && \ + # Create JULIA_PKGDIR \ + mkdir $JULIA_PKGDIR && \ + chown $NB_USER $JULIA_PKGDIR + +RUN julia -e 'import Pkg; Pkg.update()' && \ + (test $TEST_ONLY_BUILD || julia -e 'import Pkg; Pkg.add("HDF5")') && \ + julia -e 'import Pkg; Pkg.add("Gadfly")' && \ + julia -e 'import Pkg; Pkg.add("RDatasets")' && \ + julia -e 'import Pkg; Pkg.add("IJulia"); Pkg.add("StatsPlots");Pkg.add("Plots"); Pkg.add("StatsPlots");Pkg.add("DataFrames")' + +RUN julia -e 'import Pkg; Pkg.add("Seaborn"); Pkg.add("PyPlot")' +RUN julia -e 'import Pkg;Pkg.clone("https://github.com/gsoleilhac/NSGAII.jl")' +USER jovyan +# Precompile Julia packages \ +RUN sudo julia -e 'using Pkg; Pkg.add("IJulia"); using IJulia' +ADD stochastic_gd.jl . +RUN sudo julia stochastic_gd.jl + +# move kernelspec out of home \ +# mv $HOME/.local/share/jupyter/kernels/julia* $CONDA_DIR/share/jupyter/kernels/ && \ +# chmod -R go+rx $CONDA_DIR/share/jupyter && \ +# rm -rf $HOME/.local && $JULIA_PKGDIR $CONDA_DIR/share/jupyter diff --git a/julia_nb/stochastic_gd.jl b/julia_nb/stochastic_gd.jl new file mode 100644 index 0000000..d40b79f --- /dev/null +++ b/julia_nb/stochastic_gd.jl @@ -0,0 +1,53 @@ +println("reminder to Russell") +println("to install packages use") +println("launch julia using sudo") +println("try to import module first using, 'using'") +println("include('ml.jl')") +#println("Pkg.add('PyCall')") +println("\n\n\n\n\n\n") +using Pkg + +function trypy() + try + ENV["PYTHON"] = "/opt/conda/bin/python" + Pkg.rm("PyCall") + Pkg.build("PyCall") + Pkg.test("PyCall") + plt = pyimport("matplotlib.pyplot") + x = range(0;stop=2*pi,length=1000); y = sin.(3*x + 4*cos.(2*x)); + plt.plot(x, y, color="red", linewidth=2.0, linestyle="--") + #plt.show() + + #return unreliable_connect() + catch ex + ENV["PYTHON"] = "/anaconda3/bin/python" + Pkg.rm("PyCall") + Pkg.build("PyCall") + Pkg.test("PyCall") + + math = pyimport("math") + Pkg.test("PyCall") + plt = pyimport("matplotlib.pyplot") + x = range(0;stop=2*pi,length=1000); y = sin.(3*x + 4*cos.(2*x)); + plt.plot(x, y, color="red", linewidth=2.0, linestyle="--") + + math.sin(math.pi / 4) # returns ≈ 1/√2 = 0.70710678... + end +end +Pkg.add("StatsPlots") +Pkg.add("PyCall") +Pkg.add("GR") +Pkg.add("IJulia"); +Pkg.add("Plots"); +Pkg.add("StatsPlots"); #to install the StatPlots package. +Pkg.add("DataFrames"); +Pkg.add("Seaborn") +Pkg.add("PyPlot") +Pkg.add("Knet") +using DataFrames +# using IJulia +using Knet +using Plots; +using StatsPlots; +using PyCall; +println("done") diff --git a/network/#Dockerfile# b/network/#Dockerfile# new file mode 100644 index 0000000..bd43f56 --- /dev/null +++ b/network/#Dockerfile# @@ -0,0 +1,23 @@ +FROM scidash/neuronunit-optimization + # move to a docker install script. + +RUN sudo /opt/conda/bin/pip install git+https://github.com/scidash/sciunit.git@dev --process-dependency-links --upgrade +RUN sudo /opt/conda/bin/conda install -y cython +RUN sudo /opt/conda/bin/pip install pynn lazyarray + +RUN sudo /opt/conda/bin/pip install git+https://github.com/brian-team/brian2.git + # faster glif +#RUN sudo /opt/conda/bin/pip install git+https://github.com/russelljjarvis/AllenSDK.git +RUN sudo /opt/conda/bin/conda install -y numpy deap dask numba +RUN sudo /opt/conda/bin/pip install neurodynex + +RUN sudo /opt/conda/bin/pip install coverage lmfit +RUN sudo /opt/conda/bin/pip install allensdk #--upgrade +RUN sudo /opt/conda/bin/conda install pandas +RUN sudo /opt/conda/bin/pip install toolz cloudpickle +RUN sudo /opt/conda/bin/pip install pyscaffold + +RUN git clone https://github.com/SALib/SALib.git +WORKDIR SALib +RUN python setup.py develop +RUN sudo /opt/conda/bin/pip install SALib \ No newline at end of file diff --git a/network/Dockerfile b/network/Dockerfile new file mode 100644 index 0000000..e78d382 --- /dev/null +++ b/network/Dockerfile @@ -0,0 +1,36 @@ +FROM scidash/neuronunit-optimization + # move to a docker install script. + +RUN sudo /opt/conda/bin/pip install git+https://github.com/scidash/sciunit.git@dev --process-dependency-links --upgrade +RUN sudo /opt/conda/bin/conda install -y cython +RUN sudo /opt/conda/bin/pip install pynn lazyarray + +RUN sudo /opt/conda/bin/pip install git+https://github.com/brian-team/brian2.git + # faster glif +#RUN sudo /opt/conda/bin/pip install git+https://github.com/russelljjarvis/AllenSDK.git +RUN sudo /opt/conda/bin/conda install -y numpy deap dask numba +RUN sudo /opt/conda/bin/pip install neurodynex + +RUN sudo /opt/conda/bin/pip install coverage lmfit +RUN sudo /opt/conda/bin/pip install allensdk #--upgrade +RUN sudo /opt/conda/bin/conda install pandas +RUN sudo /opt/conda/bin/pip install toolz cloudpickle +RUN sudo /opt/conda/bin/pip install setuptools +RUN sudo /opt/conda/bin/conda install -c conda-forge pyscaffold +RUN sudo /opt/conda/bin/pip install --upgrade pip +RUN sudo /opt/conda/bin/conda install python=3.5.0 +# RUN sudo /opt/conda/bin/pip install pyscaffold +# RUN sudo /opt/conda/bin/pip install SALib + +# RUN git clone https://github.com/SALib/SALib.git +# WORKDIR SALib +# RUN python3 setup.py develop + +# uncomment to give PyNN mpi. +# note this is a bad idea. For this project. +# MPI is used for non-embarrasingly parallel computations, +# we use dask here, for embarrassingly parallel. +# RUN sudo apt-get update -y +# RUN sudo apt-get install -y mpich +# RUN pip install mpi4py +RUN pip install git+https://github.com/russelljjarvis/modality \ No newline at end of file diff --git a/network/Dockerfile (rjjarvis's conflicted copy 2019-04-02) b/network/Dockerfile (rjjarvis's conflicted copy 2019-04-02) new file mode 100644 index 0000000..dea3caf --- /dev/null +++ b/network/Dockerfile (rjjarvis's conflicted copy 2019-04-02) @@ -0,0 +1,23 @@ +FROM scidash/neuronunit-optimization + # move to a docker install script. + +RUN sudo /opt/conda/bin/pip install git+https://github.com/scidash/sciunit.git@dev --process-dependency-links --upgrade +RUN sudo /opt/conda/bin/conda install -y cython +RUN sudo /opt/conda/bin/pip install pynn lazyarray + +RUN sudo /opt/conda/bin/pip install git+https://github.com/brian-team/brian2.git + # faster glif +#RUN sudo /opt/conda/bin/pip install git+https://github.com/russelljjarvis/AllenSDK.git +RUN sudo /opt/conda/bin/conda install -y numpy deap dask numba +RUN sudo /opt/conda/bin/pip install neurodynex + +RUN sudo /opt/conda/bin/pip install coverage lmfit +RUN sudo /opt/conda/bin/pip install allensdk #--upgrade +RUN sudo /opt/conda/bin/conda install pandas +RUN sudo /opt/conda/bin/pip install toolz cloudpickle +RUN sudo /opt/conda/bin/pip install pyscaffold + +# RUN git clone https://github.com/SALib/SALib.git +# WORKDIR SALib +# RUN python setup.py develop +RUN sudo /opt/conda/bin/pip install SALib \ No newline at end of file diff --git a/neuron-mpi-neuroml-extra/Dockerfile b/neuron-mpi-neuroml-extra/Dockerfile deleted file mode 100644 index bc51d3e..0000000 --- a/neuron-mpi-neuroml-extra/Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ -FROM scidash/neuron-mpi-neuroml - -RUN /opt/conda/bin/ipython3 setup.py install -RUN /opt/conda/bin/ipython -c "from neuron import h" -RUN /opt/conda/bin/ipython3 -c "from neuron import h" - -ENV HOME /home/jovyan -ENV PATH $HOME/nrn-7.4/x86_64/bin:$PATH -ENV PATH /opt/conda/bin:/opt/conda/bin/conda:/opt/conda/bin/python:$PATH - -RUN sudo chown -R jovyan /home/jovyan -WORKDIR /home/jovyan/nrn-7.4/src/parallel -RUN nrniv test0.hoc -RUN mpiexec -np 4 nrniv -mpi test0.hoc -RUN mpiexec -np 4 nrniv -mpi test1.hoc -RUN mpiexec -np 4 nrniv -mpi test2.hoc -RUN mpiexec -np 4 nrniv -mpi test3.hoc -RUN mpiexec -np 4 nrniv -mpi test4.hoc -RUN mpiexec -np 4 nrniv -mpi test5.hoc -RUN mpiexec -np 4 nrniv -mpi test6.hoc -RUN mpiexec -np 4 nrniv -mpi test7.hoc -ADD ./test_fixed.py /home/jovyan/nrn-7.4/src/parallel/ -ADD ./test_fixed2.py /home/jovyan/nrn-7.4/src/parallel/ - -RUN mpiexec -np 4 nrniv -mpi -python test_fixed.py - -RUN sudo /opt/conda/bin/pip install zmq -RUN sudo /opt/conda/bin/pip install packaging -#RUN sudo /opt/conda/bin/pip install --upgrade pip#notebook -RUN sudo /opt/conda/bin/pip install notebook -RUN sudo /opt/conda/bin/pip install ipyparallel -RUN which /opt/conda/bin/ipcluster -RUN sudo /opt/conda/bin/ipcluster nbextension enable -RUN sudo /opt/conda/bin/ipython profile create chase -ENV PYTHONPATH "/opt/conda/bin/python3.5/dist-packages" -RUN echo $PYTHONPATH -RUN which python -RUN /opt/conda/bin/ipython3 -c "import sys; print(sys.path)" -RUN /opt/conda/bin/ipython3 -c "from ipyparallel import Client" -#RUN /opt/conda/bin/ipython -c "import os;os.system('ipcluster start -n 4 --engines=MPIEngineSetLauncher')" - -#RUN sudo ipcluster start --profile=chase --debug & -#RUN mpiexec -n 4 ipengine --mpi=mpi4py - -COPY ./test_fixed.py /home/jovyan/nrn-7.4/src/parallel -COPY ./test_fixed2.py /home/jovyan/nrn-7.4/src/parallel -RUN ls /home/jovyan/nrn-7.4/src/parallel/*.py -RUN ls *.py -RUN pwd - -RUN ls /home/jovyan/nrn-7.4/src/parallel/test_fixed2.py -RUN echo "import os">> run_ipp1.py -#RUN echo "os.system('ipcluster start --profile=chase --debug &')" >> run_ipp1.py -RUN echo "os.system('ipcluster start -n 4 --engines=MPIEngineSetLauncher --profile=chase --debug &')" >> run_ipp1.py -#RUN conda install mpi4py -#RUN echo "import ipyparallel as ipp ; rc = ipp.Client(profile='chase'); print('hello from before cpu '); print(rc.ids);" >> run_ipp2.py -#RUN ipython run_ipp1.py && sleep 15 && ipython run_ipp2.py -RUN /opt/conda/bin/ipython3 -c "from neuron import h" - -RUN /opt/conda/bin/ipython3 run_ipp1.py && sleep 15 && mpiexec -n 4 nrniv -mpi -python /home/jovyan/nrn-7.4/src/parallel/test_fixed2.py -#RUN /opt/conda/bin/ipython3 run_ipp1.py && sleep 15 && mpiexec -n 4 /opt/conda/bin/ipengine --mpi=mpi4py /home/jovyan/nrn-7.4/src/parallel/test_fixed2.py diff --git a/neuron-mpi-neuroml/Dockerfile b/neuron-mpi-neuroml/Dockerfile index 4c5c0ae..2391063 100644 --- a/neuron-mpi-neuroml/Dockerfile +++ b/neuron-mpi-neuroml/Dockerfile @@ -34,11 +34,11 @@ ENV PATH /opt/conda/bin:/opt/conda/bin/conda:/opt/conda/bin/python:$PATH WORKDIR $HOME RUN \ - wget http://www.neuron.yale.edu/ftp/neuron/versions/v7.4/nrn-7.4.tar.gz && \ - tar -xzf nrn-7.4.tar.gz && \ - rm nrn-7.4.tar.gz + wget http://www.neuron.yale.edu/ftp/neuron/versions/v7.5/nrn-7.5.tar.gz && \ + tar -xzf nrn-7.5.tar.gz && \ + rm nrn-7.5.tar.gz -WORKDIR $HOME/nrn-7.4 +WORKDIR $HOME/nrn-7.5 ENV PATH /usr/bin/python3/python:/opt/conda/bin:/opt/conda/bin/conda:/opt/conda/bin/python:$PATH RUN ./configure --prefix=`pwd` --with-paranrn --without-iv --with-nrnpython=/opt/conda/bin/python3 RUN sudo make all && \ @@ -49,14 +49,16 @@ RUN make install WORKDIR src/nrnpython RUN python setup.py install -ENV NEURON_HOME $HOME/nrn-7.4/x86_64 +ENV NEURON_HOME $HOME/nrn-7.5/x86_64 ENV PATH $NEURON_HOME/bin:$PATH + # Get JNeuroML #Change to this when PR is accepted -#RUN git clone https://github.com/NeuroML/jNeuroML +RUN git clone https://github.com/NeuroML/jNeuroML WORKDIR $HOME -RUN git clone https://github.com/russelljjarvis/jNeuroML.git +#RUN git clone https://github.com/russelljjarvis/jNeuroML.git WORKDIR jNeuroML +RUN ls -ltr *.py RUN python getNeuroML.py -WORKDIR $WORK_HOME \ No newline at end of file +WORKDIR $WORK_HOME diff --git a/neuron-serial/Dockerfile b/neuron-serial/Dockerfile new file mode 100644 index 0000000..0968bf4 --- /dev/null +++ b/neuron-serial/Dockerfile @@ -0,0 +1,56 @@ +#author russell jarvis rjjarvis@asu.edu +#NEURON Dockerfile +#Set the base image to Ubuntu +FROM scidash/scipy-notebook-plus + +#Get a whole lot of GNU core development tools +#version control java development, maven +#Libraries required for building MPI from source +#Libraries required for building NEURON from source +#Also DO this part as root. +USER root +RUN /opt/conda/bin/conda update conda +RUN apt-get update -y && apt-get upgrade -y +RUN apt-get update && apt-get install -y wget bzip2 ca-certificates default-jre default-jdk maven automake libtool \ + wget python3 libpython3-dev libncurses5-dev libreadline-dev libgsl0-dev cython3 \ + python3-pip python3-numpy python3-scipy python3-matplotlib python3-mock \ + ipython3 python3-docutils \ + python3-mpi4py cmake ssh + + +#Do the rest of the build as user: +#This will create a more familiar environment to continue developing in. +#with less of a need to chown and chmod everything done as root at dockerbuild completion + +RUN wget https://neuron.yale.edu/ftp/neuron/versions/v7.6/nrn-7.6.x86_64-linux.deb +RUN dpkg -i nrn-7.6.x86_64-linux.deb +ENV PATH /usr/bin/python/python:/opt/conda/bin:/opt/conda/bin/conda:/opt/conda/bin/python:$PATH + +USER jovyan +ENV PATH /usr/bin/python/python:/opt/conda/bin:/opt/conda/bin/conda:/opt/conda/bin/python:$PATH +# Use numpy 1.12.1 until quantities is compatible with 1.13. +RUN sudo /opt/conda/bin/conda install -y scipy numpy==1.12.1 matplotlib +RUN sudo chown -R jovyan /home/jovyan +ENV HOME /home/jovyan +ENV PATH /opt/conda/bin:/opt/conda/bin/conda:/opt/conda/bin/python:$PATH + +RUN wget https://neuron.yale.edu/ftp/neuron/versions/v7.6/nrn-7.6.x86_64-linux.deb +RUN sudo dpkg -i nrn-7.6.x86_64-linux.deb +RUN sudo git clone https://github.com/neuronsimulator/nrn +WORKDIR nrn +RUN sudo apt-get install flex -y +RUN sudo ./build.sh +RUN sudo ./configure --prefix=`pwd` --without-iv --with-nrnpython=/opt/conda/bin/python +#RUN sudo apt-get install bison -y +RUN sudo apt-get install byacc -y +RUN cat ./build.sh +RUN sudo make +RUN sudo make install +WORKDIR nrn/src/nrnpython +RUN python setup.py install + +ENV NEURON_HOME $HOME/nrn-7.6/x86_64 +ENV PATH $NEURON_HOME/bin:$PATH + + + diff --git a/neuron-serial/Dockerfile~ b/neuron-serial/Dockerfile~ new file mode 100644 index 0000000..7b91dac --- /dev/null +++ b/neuron-serial/Dockerfile~ @@ -0,0 +1,68 @@ +#author russell jarvis rjjarvis@asu.edu +#NEURON Dockerfile +#Set the base image to Ubuntu +FROM scidash/scipy-notebook-plus + +#Get a whole lot of GNU core development tools +#version control java development, maven +#Libraries required for building MPI from source +#Libraries required for building NEURON from source +#Also DO this part as root. +USER root + +RUN apt-get update && apt-get install -y wget bzip2 ca-certificates default-jre default-jdk maven automake libtool \ + wget python3 libpython3-dev libncurses5-dev libreadline-dev libgsl0-dev cython3 \ + python3-pip python3-numpy python3-scipy python3-matplotlib python3-mock \ + ipython3 python3-docutils \ + python3-mpi4py cmake ssh + + +#Do the rest of the build as user: +#This will create a more familiar environment to continue developing in. +#with less of a need to chown and chmod everything done as root at dockerbuild completion + +RUN git clone https://github.com/neuronsimulator/nrn +WORKDIR nrn +RUN wget https://neuron.yale.edu/ftp/neuron/versions/v7.6/nrn-7.6.x86_64-linux.deb +ENV PATH /usr/bin/python3/python:/opt/conda/bin:/opt/conda/bin/conda:/opt/conda/bin/python:$PATH + +# RUN ./build.sh + +# RUN ./configure --prefix=`pwd` --without-iv --with-nrnpython=/opt/conda/bin/python3 +# RUN make # all +# RUN make install + + +USER jovyan +ENV PATH /usr/bin/python3/python:/opt/conda/bin:/opt/conda/bin/conda:/opt/conda/bin/python:$PATH + +# Use numpy 1.12.1 until quantities is compatible with 1.13. +RUN conda install -y scipy numpy==1.12.1 matplotlib +RUN sudo chown -R jovyan /home/jovyan +ENV HOME /home/jovyan +ENV PATH /opt/conda/bin:/opt/conda/bin/conda:/opt/conda/bin/python:$PATH + + + +WORKDIR nrn/src/nrnpython +RUN sudo ./build.sh + +RUN sudo ./configure --prefix=`pwd` --without-iv --with-nrnpython=/opt/conda/bin/python3 +RUN sudo make # all +RUN sudo make install + + +RUN python setup.py install +ENV NEURON_HOME $HOME/nrn-7.5/x86_64 +ENV PATH $NEURON_HOME/bin:$PATH + + +# Get JNeuroML +# Change to this when PR is accepted +# RUN git clone https://github.com/NeuroML/jNeuroML +# WORKDIR $HOME +# RUN git clone https://github.com/russelljjarvis/jNeuroML.git +# WORKDIR jNeuroML +# RUN ls -ltr *.py +# RUN python getNeuroML.py +# WORKDIR $WORK_HOME diff --git a/neuronunit-neuroconstruct/Dockerfile b/neuronunit-neuroconstruct/Dockerfile deleted file mode 100644 index 8fe1483..0000000 --- a/neuronunit-neuroconstruct/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# author Russell Jarvis rjjarvis@asu.edu -# author Rick Gerkin rgerkin@asu.edu -# neuronunit-showcase - -FROM scidash/neuronunit - -################# -# Showcase for NeuroConstruct (http://www.neuroconstruct.org/) by Padraig Gleeson, -# a subproject of Neural Ensemble (http://neuralensemble.org/) -################# -USER root -RUN apt-get update -RUN apt-get install -y subversion -USER $NB_USER -RUN pip install execnet -WORKDIR $HOME -RUN git clone https://github.com/NeuralEnsemble/neuroConstruct.git -WORKDIR neuroConstruct -RUN bash updatenC.sh -RUN bash nC.sh -make -RUN ant run -WORKDIR osb -RUN ./checkOsb.sh -USER root -RUN ln -s $NEURON_HOME/bin/nrnivmodl /usr/local/bin/ -USER $NB_USER -WORKDIR $HOME/neuroConstruct -RUN ant test -Dsimulators='NEURON' -ENV NC_HOME $HOME/neuroConstruct -ENV PYTHONPATH $NC_HOME -USER $NB_USER -RUN python -c "import pythonnC" -WORKDIR $WORK_HOME diff --git a/neuronunit-optimization/Dockerfile b/neuronunit-optimization/Dockerfile index 3e4896d..baa942c 100644 --- a/neuronunit-optimization/Dockerfile +++ b/neuronunit-optimization/Dockerfile @@ -9,70 +9,63 @@ FROM scidash/neuronunit ################# USER root + + + RUN apt-get update RUN apt-get install -y gvfs-bin libxss1 python3-setuptools RUN apt-get install -y python-tk curl apt-utils -#RUN apt-get install -y povray eog -RUN chown -R jovyan $HOME -RUN chown -R jovyan /opt/conda/lib/python3.5/site-packages/ -USER $NB_USER +#RUN sudo chown -R jovyan $HOME +#RUN sudo chown -R jovyan /opt/conda/lib/python3.5/site-packages/ + RUN pip install --upgrade pip +RUN pip install matplotlib RUN pip install IPython \ jupyterhub \ notebook \ ipykernel \ - ipyparallel \ - enum34 -RUN conda install mpi4py -RUN easy_install gevent -RUN easy_install greenlet + enum34 \ + numba +RUN /opt/conda/bin/conda info --envs +RUN /opt/conda/bin/conda update -n base conda -##### Scoop ##### -WORKDIR $HOME -RUN git clone https://github.com/soravux/scoop -WORKDIR scoop -RUN python setup.py install +# RUN conda update -n base conda +#numba is for optimized code, bqplot is for interactive plotting. +RUN conda install -c dask distributed +RUN conda install -c anaconda pyzmq +RUN conda install -c conda-forge libsodium -##### BluePyOpt ###### -WORKDIR $HOME -RUN git clone https://github.com/russelljjarvis/BluePyOpt.git -WORKDIR BluePyOpt -RUN python setup.py install -RUN python -c "import bluepyopt" -##### IPython Cluster ##### -USER $NB_USER -WORKDIR $HOME -RUN ipython profile create default -RUN pip install git+https://github.com/roryk/ipython-cluster-helper -# Must be done as root to create directories in /usr/local -RUN sudo /opt/conda/bin/ipcluster nbextension enable +RUN easy_install gevent +RUN easy_install greenlet ENV PATH $PATH:/opt/conda/bin -ENV PATH $PATH:/opt/conda/bin/ipcluster ENV PATH $PATH:/opt/conda/bin/ipython ENV PATH $PATH:/opt/conda/bin/pip ENV PATH $PATH:/opt/conda/bin/python ENV PATH $PATH:/opt/conda/lib/python3.5/site-packages ENV PATH $PATH:$PYTHONPATH +RUN pip install dask +RUN pip install tornado zmq +RUN pip install --upgrade zmq tornado pip +USER $NB_USER + RUN printenv PATH RUN python -c "import pyneuroml" RUN python -c "import neuronunit" -RUN python -c "from neuronunit.models.reduced import ReducedModel" +# RUN python -c "from neuronunit.models.reduced import ReducedModel" RUN python -c "import quantities" RUN python -c "import neuron" RUN python -c "import pyneuroml" RUN nrnivmodl -RUN python -c "import scoop" -RUN python -c "import deap" RUN nrniv +WORKDIR $HOME +RUN sudo chown -R $NB_USER $HOME +ENV PYTHONPATH $HOME/neuronunit:$PYTHONPATH +RUN pip install -U jupyter +# ENTRYPOINT /bin/bash -#RUN export PATH=$PATH - -#un comment the following line to test ipcluster during build. -#RUN ipcluster start --profile=chase --debug & -#https://github.com/russelljjarvis/neuronunit.git -WORKDIR $WORK_HOME \ No newline at end of file +# RUN pip uninstall tornado diff --git a/neuronunit-optimization/func2rc.sh b/neuronunit-optimization/func2rc.sh new file mode 100644 index 0000000..e6e19e6 --- /dev/null +++ b/neuronunit-optimization/func2rc.sh @@ -0,0 +1,18 @@ + + +function sc(){ + export PYTHONPATH=$1 + ipcluster start -n 8 --profile=default & + sleep 5 + python stdout_worker.py & + ipython -i nsga_parallel.py + # explanation + # there is some ugly inconsistencies between the + # controller and the engines on ipyparallel + # basically the controllers fixed location of neuronunit is fixed to /home/jovyan/work/neuronunit/neuronunit + # this probably + # because PYTHONPATH was set to this early with ENV + # this is very static and it can't be updated + #ln -s "`$1`../../neuronunit /home/jovyan/work/neuronunit/neuronunit" +} + diff --git a/neuronunit-optimization/stdout_worker.py b/neuronunit-optimization/stdout_worker.py new file mode 100644 index 0000000..595043c --- /dev/null +++ b/neuronunit-optimization/stdout_worker.py @@ -0,0 +1,86 @@ +"""A script for watching all traffic on the IOPub channel (stdout/stderr/pyerr) of engines. + +This connects to the default cluster, or you can pass the path to your ipcontroller-client.json + +Try running this script, and then running a few jobs that print (and call sys.stdout.flush), +and you will see the print statements as they arrive, notably not waiting for the results +to finish. + +You can use the zeromq SUBSCRIBE mechanism to only receive information from specific engines, +and easily filter by message type. + +Authors +------- +* MinRK +""" + +import sys +import json +import zmq + +from jupyter_client.session import Session +from ipykernel.connect import find_connection_file + +def main(connection_file): + """watch iopub channel, and print messages""" + + ctx = zmq.Context.instance() + + with open(connection_file) as f: + cfg = json.loads(f.read()) + + reg_url = cfg['interface'] + iopub_port = cfg['iopub'] + iopub_url = "%s:%s"%(reg_url, iopub_port) + + session = Session(key=cfg['key'].encode('ascii')) + sub = ctx.socket(zmq.SUB) + + # This will subscribe to all messages: + sub.SUBSCRIBE = b'' + # replace with b'' with b'engine.1.stdout' to subscribe only to engine 1's stdout + # 0MQ subscriptions are simple 'foo*' matches, so 'engine.1.' subscribes + # to everything from engine 1, but there is no way to subscribe to + # just stdout from everyone. + # multiple calls to subscribe will add subscriptions, e.g. to subscribe to + # engine 1's stderr and engine 2's stdout: + # sub.SUBSCRIBE = b'engine.1.stderr' + # sub.SUBSCRIBE = b'engine.2.stdout' + sub.connect(iopub_url) + while True: + try: + idents,msg = session.recv(sub, mode=0) + except KeyboardInterrupt: + return + # ident always length 1 here + topic = idents[0].decode('utf8', 'replace') + if msg['msg_type'] == 'stream': + # stdout/stderr + # stream names are in msg['content']['name'], if you want to handle + # them differently + print("%s: %s" % (topic, msg['content']['text'])) + elif msg['msg_type'] == 'error': + # Python traceback + c = msg['content'] + print(topic + ':') + for line in c['traceback']: + # indent lines + print(' ' + line) + elif msg['msg_type'] == 'error': + # Python traceback + c = msg['content'] + print(topic + ':') + for line in c['traceback']: + # indent lines + print(' ' + line) + +if __name__ == '__main__': + if len(sys.argv) > 1: + pattern = sys.argv[1] + else: + # This gets the security file for the default profile: + pattern = 'ipcontroller-client.json' + cf = find_connection_file(pattern) + print("Using connection file %s" % cf) + main(cf) + #import nsga_parallel diff --git a/neuronunit/Dockerfile b/neuronunit/Dockerfile index 6289d05..f18dc88 100644 --- a/neuronunit/Dockerfile +++ b/neuronunit/Dockerfile @@ -8,22 +8,13 @@ FROM scidash/neuron-mpi-neuroml # then a new tar of the repo will be downloaded. ARG HEAD=1 WORKDIR $HOME - -RUN wget http://github.com/scidash/sciunit/tarball/dev -O sciunit.tar.gz -RUN mkdir -p sciunit -RUN tar -xvzf sciunit.tar.gz -C sciunit --strip-components=1 -RUN rm sciunit.tar.gz - -RUN wget http://github.com/scidash/neuronunit/tarball/dev -O neuronunit.tar.gz -RUN mkdir -p neuronunit -RUN tar -xvzf neuronunit.tar.gz -C neuronunit --strip-components=1 -RUN rm neuronunit.tar.gz +RUN git clone -b dev https://github.com/scidash/neuronunit.git WORKDIR neuronunit RUN pip install --process-dependency-links -e . -# RUN python setup.py install - -WORKDIR $WORK_HOME +#RUN sudo /opt/conda/bin/python setup.py install +WORKDIR $HOME # Put the optional local working copy (not the one we just cloned) # on the working path. ENV PYTHONPATH $WORK_HOME/sciunit:$PYTHONPATH -ENV PYTHONPATH $WORK_HOME/neuronunit:$PYTHONPATH \ No newline at end of file +ENV PYTHONPATH $WORK_HOME/neuronunit:$PYTHONPATH + diff --git a/run b/run index d2f3a71..e604e85 100755 --- a/run +++ b/run @@ -1,28 +1,13 @@ #!/bin/bash -NAME=$1 -CONTAINER_STATUS=$(docker ps -a --filter="name=$NAME" -q | xargs) # Is the container running? -if [ $CONTAINER_STATUS ]; then # If so... - docker rm -f $CONTAINER_STATUS # Stop it and remove it. -fi -if ! [ $NOTEBOOK_HOME ]; then - NOTEBOOK_HOME=$HOME -fi + WORK_HOME=/home/jovyan/work -CMD="start-notebook.sh" -CMD_ARGS="--ip=0.0.0.0 --NotebookApp.token=\"\" --NotebookApp.disable_check_xsrf=True" -HOST_PORT=8888 # Open port on the host machine. +ENTRYPOINT=/bin/bash if [ "$2" = "-dev" ]; then - docker run -it --name="$NAME" \ + docker run -it --entrypoint=$ENTRYPOINT \ -v $SCIDASH_HOME/sciunit:$WORK_HOME/sciunit \ -v $SCIDASH_HOME/neuronunit:$WORK_HOME/neuronunit \ - -p $HOST_PORT:8888 -v $NOTEBOOK_HOME:$WORK_HOME/host \ - scidash/$NAME $CMD $CMD_ARGS + scidash/$1 else - docker run -d --name="$NAME" \ - -p $HOST_PORT:8888 -v $NOTEBOOK_HOME:$WORK_HOME/host \ - scidash/$NAME $CMD $CMD_ARGS - docker exec "$NAME" bash -c "ln -s \$HOME/sciunit/docs $WORK_HOME/sciunit-docs" - docker exec "$NAME" bash -c "ln -s \$HOME/neuronunit/docs $WORK_HOME/neuronunit-docs" + docker run -it --entrypoint=$ENTRYPOINT scidash/$1 fi -# Now access the notebook server at http://localhost:$HOST_PORT \ No newline at end of file diff --git a/scipy-notebook-plus/Dockerfile b/scipy-notebook-plus/Dockerfile index c65f814..4b2bc32 100644 --- a/scipy-notebook-plus/Dockerfile +++ b/scipy-notebook-plus/Dockerfile @@ -13,6 +13,8 @@ RUN chown -R $NB_USER $HOME #Libraries required for building NEURON from source RUN apt-get update +RUN apt-get -y install apt-transport-https ca-certificates +RUN apt-get -y install apt-transport-https curl RUN apt-get install -y wget bzip2 ca-certificates \ libglib2.0-0 libxext6 libsm6 libxrender1 \ git gcc g++ build-essential \ @@ -24,7 +26,9 @@ RUN pip install --upgrade pip # Upgrade to version 2.0 RUN conda install -y matplotlib - +RUN conda update -n base conda +# RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh +# RUN bash Miniconda3-latest-Linux-x86_64.sh # Make sure every Python file belongs to jovyan RUN find /opt/conda ! -user $NB_USER -print0 | xargs -0 -I {} chown -h $NB_USER {} # Remove dangling symlinks diff --git a/sciunit/Dockerfile b/sciunit/Dockerfile index c768b9d..3cdb1bb 100644 --- a/sciunit/Dockerfile +++ b/sciunit/Dockerfile @@ -7,6 +7,9 @@ FROM scidash/scipy-notebook-plus # HEAD, which will ensure that if the HEAD has changed (i.e. new commits), # then a new tar of the repo will be downloaded. ARG HEAD=1 +RUN sudo apt-get update +RUN sudo apt-get install -y apt-transport-https + WORKDIR $HOME RUN wget http://github.com/scidash/sciunit/tarball/dev -O sciunit.tar.gz RUN mkdir -p sciunit diff --git a/test-all b/test-all old mode 100755 new mode 100644 diff --git a/test_fixed.py b/test_fixed.py new file mode 100644 index 0000000..5dd5e90 --- /dev/null +++ b/test_fixed.py @@ -0,0 +1,11 @@ +from neuron import h +pc = h.ParallelContext() + +id = int(pc.id()) +nhost = int(pc.nhost()) + +print('I am %d of %d'%(id, nhost)) + +pc.runworker() +pc.done() +h.quit() diff --git a/with-r/Dockerfile b/with-r/Dockerfile new file mode 100644 index 0000000..24e44f5 --- /dev/null +++ b/with-r/Dockerfile @@ -0,0 +1,2 @@ +FROM network +RUN sudo apt-get install r-base diff --git a/with_julia/Dockerfile b/with_julia/Dockerfile new file mode 100644 index 0000000..89400ec --- /dev/null +++ b/with_julia/Dockerfile @@ -0,0 +1,49 @@ +FROM with_r +USER root +#RUN apt-get install -y julia +# install Julia packages in /opt/julia instead of $HOME +ENV JULIA_DEPOT_PATH=/opt/julia +ENV JULIA_PKGDIR=/opt/julia +ENV JULIA_VERSION=1.0.0 + +RUN mkdir /opt/julia-${JULIA_VERSION} && \ + cd /tmp && \ + wget -q https://julialang-s3.julialang.org/bin/linux/x64/`echo ${JULIA_VERSION} | cut -d. -f 1,2`/julia-${JULIA_VERSION}-linux-x86_64.tar.gz && \ + echo "bea4570d7358016d8ed29d2c15787dbefaea3e746c570763e7ad6040f17831f3 *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \ + tar xzf julia-${JULIA_VERSION}-linux-x86_64.tar.gz -C /opt/julia-${JULIA_VERSION} --strip-components=1 && \ + rm /tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz +RUN ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia + +# Show Julia where conda libraries are \ +RUN mkdir /etc/julia && \ + echo "push!(Libdl.DL_LOAD_PATH, \"$CONDA_DIR/lib\")" >> /etc/julia/juliarc.jl && \ + # Create JULIA_PKGDIR \ + mkdir $JULIA_PKGDIR + +RUN julia -e 'import Pkg; Pkg.update()' && \ + (test $TEST_ONLY_BUILD || julia -e 'import Pkg; Pkg.add("HDF5")') && \ + julia -e 'import Pkg; Pkg.add("Gadfly")' && \ + julia -e 'import Pkg; Pkg.add("RDatasets")' && \ + julia -e 'import Pkg; Pkg.add("IJulia")' && \ + julia -e 'import Pkg; Pkg.add("StatsPlots");Pkg.add("Plots");' + +RUN julia -e 'import Pkg; Pkg.add("DataFrames")' + +RUN julia -e 'import Pkg; Pkg.add("Seaborn"); Pkg.add("PyPlot")' +RUN julia -e 'import Pkg;Pkg.clone("https://github.com/gsoleilhac/NSGAII.jl")' +RUN julia -e 'import Pkg;Pkg.clone("https://github.com/fun-zoological-computing/SNN.jl")' +USER jovyan +# Precompile Julia packages \ +RUN sudo julia -e 'using Pkg; Pkg.add("IJulia"); using IJulia' +ADD packages.jl . +RUN sudo chown -R jovyan $HOME +RUN sudo chown -R jovyan $JULIA_PKGDIR +#RUN sudo /opt/conda/bin/pip install --upgrade numpy +RUN julia packages.jl +RUN sudo /opt/conda/bin/conda install -f numpy +RUN sudo /opt/conda/bin/conda install nomkl numpy scipy scikit-learn numexpr +RUN sudo /opt/conda/bin/conda remove mkl mkl-service +RUN sudo /opt/conda/bin/conda update numpy +RUN sudo /opt/conda/bin/pip install git+git://github.com/BlueBrain/eFEL +RUN sudo /opt/conda/bin/pip install --upgrade allensdk +RUN sudo /opt/conda/bin/pip install -e git+https://github.com/scidash/sciunit@dev