From 7e118062c4e62f54d2d2010c07cdf967b90aaceb Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Fri, 15 Jan 2021 00:29:22 -0500 Subject: [PATCH 01/26] Add new dockerfiles for raspberry pi --- docker/dbmongo/Dockerfile_raspberrypi | 34 ++++ docker/lab/Dockerfile_raspberrypi | 240 ++++++++++++++++++++++++++ docker/machine/Dockerfile_raspberrypi | 97 +++++++++++ 3 files changed, 371 insertions(+) create mode 100644 docker/dbmongo/Dockerfile_raspberrypi create mode 100644 docker/lab/Dockerfile_raspberrypi create mode 100644 docker/machine/Dockerfile_raspberrypi diff --git a/docker/dbmongo/Dockerfile_raspberrypi b/docker/dbmongo/Dockerfile_raspberrypi new file mode 100644 index 000000000..89f3330b0 --- /dev/null +++ b/docker/dbmongo/Dockerfile_raspberrypi @@ -0,0 +1,34 @@ +FROM arm64v8/mongo:bionic +WORKDIR /opt/ + +ARG docker_filepath=docker/dbmongo/files + +#add repo for mongodb +#RUN echo "deb http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.2 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.2.list +#RUN apt-get update --fix-missing && \ +# apt-get install -y wget gnupg +#RUN wget -qO - https://www.mongodb.org/static/pgp/server-4.2.asc | apt-key add - + +#RUN echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.2 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.2.list + +RUN apt-get update --fix-missing && \ + apt-get install -y --allow-unauthenticated \ + vim npm openssh-client htop dos2unix \ + net-tools iputils-ping \ + --no-install-recommends && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +#check if we need to rebuild +COPY ${docker_filepath}/mongod.conf /etc/ +COPY ${docker_filepath}/users.json /root/ +COPY ${docker_filepath}/projects.json /root/ +COPY ${docker_filepath}/entrypoint.sh /root/ +COPY ${docker_filepath}/sync.sh /root/ +RUN dos2unix /root/entrypoint.sh /root/sync.sh +RUN dos2unix /root/users.json /root/projects.json +RUN dos2unix /etc/mongod.conf + +EXPOSE 27017 + +CMD ["/bin/bash", "/root/entrypoint.sh"] diff --git a/docker/lab/Dockerfile_raspberrypi b/docker/lab/Dockerfile_raspberrypi new file mode 100644 index 000000000..70d83b22b --- /dev/null +++ b/docker/lab/Dockerfile_raspberrypi @@ -0,0 +1,240 @@ +FROM ubuntu:bionic + +#RUN apt-get update --fix-missing && apt-get install -y wget + +#nodejs +#RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ +# tar -xvf ~/node.tar.xz -C /opt/ && \ +# rm ~/node.tar.xz +#ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH + +RUN groupadd --gid 1000 node \ + && useradd --uid 1000 --gid node --shell /bin/bash --create-home node + +ENV NODE_VERSION 12.20.0 + +RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) ARCH='x64';; \ + ppc64el) ARCH='ppc64le';; \ + s390x) ARCH='s390x';; \ + arm64) ARCH='arm64';; \ + armhf) ARCH='armv7l';; \ + i386) ARCH='x86';; \ + *) echo "unsupported architecture"; exit 1 ;; \ + esac \ + && set -ex \ + # libatomic1 for arm + && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + && for key in \ + 4ED778F539E3634C779C87C6D7062848A1AB005C \ + 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ + 1C050899334244A8AF75E53792EF661D867B9DFA \ + 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ + 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ + C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ + C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \ + DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ + A48C2BEE680E841632CD4E44F07496B3EB3C1762 \ + 108F52B48DB57BB0CC439B2997B01419BD92F80A \ + B9E2F5981AA6E0CD28160D9FF13993A75599653C \ + ; do \ + gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ + gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ + done \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ + && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ + && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ + && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ + && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ + && apt-mark auto '.*' > /dev/null \ + && find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + && ln -s /usr/local/bin/node /usr/local/bin/nodejs \ + # smoke tests + && node --version \ + && npm --version + +ENV YARN_VERSION 1.22.5 + +RUN set -ex \ + && savedAptMark="$(apt-mark showmanual)" \ + && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + && for key in \ + 6A010C5166006599AA17F08146C2130DFD2497F5 \ + ; do \ + gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ + gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ + done \ + && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" \ + && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc" \ + && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ + && mkdir -p /opt \ + && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ \ + && ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn \ + && ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg \ + && rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ + && apt-mark auto '.*' > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; } \ + && find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + # smoke test + && yarn --version + +ARG docker_filepath=docker/lab/files + +RUN apt-get update --fix-missing && apt-get install -y \ + vim openssh-client openssh-server telnet apache2 \ + net-tools iputils-ping xz-utils \ + screen ngrep ca-cacert \ + mercurial subversion \ + build-essential cmake lsb-core cpio mesa-common-dev \ + dos2unix \ + curl \ + git \ + libatlas-base-dev \ + gfortran \ + --no-install-recommends && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Ensure specific python version +# See: https://stackoverflow.com/a/58562728/1730417 +# and: https://askubuntu.com/a/1176271/260220 +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update +RUN apt-get install -y python3.7 python3.7-dev python3.7-distutils +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 +RUN update-alternatives --set python /usr/bin/python3.7 +RUN curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python get-pip.py --force-reinstall && \ + rm get-pip.py + +# ensure local python is preferred over distribution python +ENV PATH /usr/local/bin:$PATH + +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +ENV TZ=America/New_York +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + libbluetooth-dev \ + tk-dev \ + uuid-dev \ + && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget python3.7-venv python3-numpy \ + && apt-get clean \ + && rm -rf /var/lib/apt/list/* + +RUN python3 --version +RUN python --version +RUN python -m venv /venv +ENV PATH=/venv/bin:$PATH +RUN python --version +RUN pip --version + +# setup python environment +COPY ${docker_filepath}/requirements.txt /root/ +RUN mkdir /root/wheel +COPY ${docker_filepath}/wheel /root/wheel +RUN pip3 install --no-cache-dir cython +#RUN pip3 install --no-cache-dir -r /root/requirements.txt +RUN pip3 install --no-index --find-links=/root/wheel -r /root/requirements.txt +### bill's surprise fork +# NOTE: For some reason, this install doesn't seem to recognize the installed numpy +# when run on Docker. As a workaround, the `wheel` directory includes a wheel +# for this install. +#RUN pip3 install --no-cache-dir git+https://github.com/lacava/surprise.git@1.0.8.3 +RUN pip3 install --no-index --find-links=/root/wheel scikit_surprise + +## Webserver + +RUN rm /etc/apache2/sites-enabled/* +COPY ${docker_filepath}/ports.conf /etc/apache2/ +RUN cp /etc/apache2/mods-available/rewrite* /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/ssl* /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/socache* /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/proxy.* /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/proxy_wstunnel.load /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/proxy_http.load /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/headers.load /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/expires.load /etc/apache2/mods-enabled + +# setup node environment +RUN npm install -g pm2 --silent --progress=false + +# install lab/webapp/node_modules to an anon volume +WORKDIR /appsrc/lab/webapp +COPY lab/webapp/package.json /appsrc/lab/webapp/ +COPY lab/webapp/package-lock.json /appsrc/lab/webapp/ +RUN dos2unix /appsrc/lab/webapp/*.json +RUN npm install --silent --progress=false + +# install lab/node_modules to an anon volume +WORKDIR /appsrc/lab +COPY lab/package.json /appsrc/lab/ +COPY lab/package-lock.json /appsrc/lab/ +RUN dos2unix /appsrc/lab/*.json +RUN npm install --silent --progress=false + +RUN apt-get install -y libgfortran5 liblapack-dev + +COPY ${docker_filepath}/001-pennai.conf /etc/apache2/sites-enabled/ +COPY ${docker_filepath}/htpasswd /etc/apache2/htpasswd +COPY ${docker_filepath}/certs/* /usr/lib/ssl/private/ + +WORKDIR /root/ + +# Webserver - paiwww +COPY ${docker_filepath}/start.sh /root/ + +## Utility script, used when starting ai +COPY ${docker_filepath}/wait-for-it.sh /root/ +RUN ["chmod", "+x", "/root/wait-for-it.sh"] + + +## PennAI Lab server +COPY ${docker_filepath}/entrypoint.sh /root/ +RUN ["chmod", "+x", "/root/entrypoint.sh"] + +RUN dos2unix /root/start.sh \ + && dos2unix /root/wait-for-it.sh \ + && dos2unix /root/entrypoint.sh + +# set version and build environment; tag.sh is sourced in entrypoint.sh +ENV BUILD_ENV='dev' +COPY .env /etc/profile.d/ +RUN cp '/etc/profile.d/.env' '/etc/profile.d/tag.sh' +RUN dos2unix /etc/profile.d/tag.sh +RUN sed -i "s/TAG=/export TAG=/g" /etc/profile.d/tag.sh + + +# Start the webserver +CMD ["/bin/bash", "/root/start.sh"] + +# EXPOSE 443 +EXPOSE 5080 +WORKDIR /appsrc/lab/ +ENTRYPOINT ["/root/entrypoint.sh"] diff --git a/docker/machine/Dockerfile_raspberrypi b/docker/machine/Dockerfile_raspberrypi new file mode 100644 index 000000000..b1941fde6 --- /dev/null +++ b/docker/machine/Dockerfile_raspberrypi @@ -0,0 +1,97 @@ +FROM python:3.7-stretch + +#nodejs +#RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ +# tar -xvf ~/node.tar.xz -C /opt/ && \ +# rm ~/node.tar.xz +#ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH + +ENV NODE_VERSION 12.20.0 + +RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) ARCH='x64';; \ + ppc64el) ARCH='ppc64le';; \ + s390x) ARCH='s390x';; \ + arm64) ARCH='arm64';; \ + armhf) ARCH='armv7l';; \ + i386) ARCH='x86';; \ + *) echo "unsupported architecture"; exit 1 ;; \ + esac \ + && set -ex \ + # libatomic1 for arm + && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + && for key in \ + 4ED778F539E3634C779C87C6D7062848A1AB005C \ + 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ + 1C050899334244A8AF75E53792EF661D867B9DFA \ + 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ + 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ + C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ + C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \ + DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ + A48C2BEE680E841632CD4E44F07496B3EB3C1762 \ + 108F52B48DB57BB0CC439B2997B01419BD92F80A \ + B9E2F5981AA6E0CD28160D9FF13993A75599653C \ + ; do \ + gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ + gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ + done \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ + && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ + && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ + && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ + && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ + && apt-mark auto '.*' > /dev/null \ + && find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + && ln -s /usr/local/bin/node /usr/local/bin/nodejs \ + # smoke tests + && node --version \ + && npm --version + +ARG docker_filepath=docker/machine/files + +RUN apt-get update --fix-missing && apt-get install -y \ + vim openssh-client openssh-server graphviz \ + net-tools iputils-ping xz-utils \ + screen ngrep ca-cacert \ + mercurial subversion \ + build-essential cmake lsb-core cpio mesa-common-dev \ + libglib2.0-0 libxext6 libsm6 libxrender1 dos2unix \ + --no-install-recommends && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# setup python environment +COPY ${docker_filepath}/requirements.txt /root/ +RUN mkdir /root/wheel +COPY ${docker_filepath}/wheel /root/wheel +RUN python -m pip install --no-index --find-links=/root/wheel -r /root/requirements.txt + +# setup node environment +RUN npm install -g pm2 --silent --progress=false + +# install node_modules to an anon volume +WORKDIR /appsrc/machine + +COPY machine/package.json /appsrc/machine/ +COPY machine/package-lock.json /appsrc/machine/ +RUN dos2unix /appsrc/machine/*.json +RUN npm install --silent --progress=false + +COPY ${docker_filepath}/entrypoint.sh /root/ +COPY ${docker_filepath}/wait-for-it.sh /root/ +RUN dos2unix /root/wait-for-it.sh && dos2unix /root/entrypoint.sh +RUN ["chmod", "+x", "/root/wait-for-it.sh"] + +CMD ["/bin/bash", "/root/entrypoint.sh"] From 4c56e102666f1c25b23016be535431e6a6cd3438 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Fri, 15 Jan 2021 15:33:32 -0500 Subject: [PATCH 02/26] Create new docker compose configuration for rpi --- docker-compose-raspberrypi.yml | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 docker-compose-raspberrypi.yml diff --git a/docker-compose-raspberrypi.yml b/docker-compose-raspberrypi.yml new file mode 100644 index 000000000..7b40dd500 --- /dev/null +++ b/docker-compose-raspberrypi.yml @@ -0,0 +1,48 @@ +version: '3' + +services: + lab: + build: + context: . + dockerfile: docker/lab/Dockerfile_raspberrypi + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/lab/webapp/node_modules" + - "/appsrc/lab/node_modules" + ports: + - "5080:5080" + env_file: + - ./config/common.env + - ./config/ai.env + depends_on: + - dbmongo + + machine: + build: + context: . + dockerfile: docker/machine/Dockerfile_raspberrypi + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/machine/node_modules" + ports: + - "5081:5081" + env_file: ./config/common.env + environment: + - MACHINE_PORT=5081 + depends_on: + - lab + - dbmongo + + dbmongo: + build: + context: . + dockerfile: docker/dbmongo/Dockerfile_raspberrypi + tty: true + stdin_open: true + ports: + - "27017:27017" + env_file: ./config/common.env From bde6cb511336286e6b95b2c8eab23637ecce0f25 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Fri, 15 Jan 2021 16:06:02 -0500 Subject: [PATCH 03/26] Add rpi dependency wheels as a Git submodule --- .gitmodules | 3 +++ docker/pennai-arm64-deps | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 docker/pennai-arm64-deps diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..363522775 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "docker/pennai-arm64-deps"] + path = docker/pennai-arm64-deps + url = https://github.com/EpistasisLab/pennai-arm64-deps diff --git a/docker/pennai-arm64-deps b/docker/pennai-arm64-deps new file mode 160000 index 000000000..83b7045aa --- /dev/null +++ b/docker/pennai-arm64-deps @@ -0,0 +1 @@ +Subproject commit 83b7045aa973cda00a830af46ef0d6dc1f52f04b From e9f7da92b1ca2f8edbe9e29c6ea7a7a8ed429700 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Fri, 15 Jan 2021 16:33:16 -0500 Subject: [PATCH 04/26] Make Dockerfiles for rpi see wheel files --- docker/dbmongo/Dockerfile_raspberrypi | 2 +- docker/lab/Dockerfile_raspberrypi | 17 ++++++----------- docker/machine/Dockerfile_production | 1 + docker/machine/Dockerfile_raspberrypi | 13 ++++--------- 4 files changed, 12 insertions(+), 21 deletions(-) diff --git a/docker/dbmongo/Dockerfile_raspberrypi b/docker/dbmongo/Dockerfile_raspberrypi index 89f3330b0..8bf6e0133 100644 --- a/docker/dbmongo/Dockerfile_raspberrypi +++ b/docker/dbmongo/Dockerfile_raspberrypi @@ -13,7 +13,7 @@ ARG docker_filepath=docker/dbmongo/files RUN apt-get update --fix-missing && \ apt-get install -y --allow-unauthenticated \ - vim npm openssh-client htop dos2unix \ + npm openssh-client htop dos2unix \ net-tools iputils-ping \ --no-install-recommends && \ apt-get clean && \ diff --git a/docker/lab/Dockerfile_raspberrypi b/docker/lab/Dockerfile_raspberrypi index 70d83b22b..a0d864aa8 100644 --- a/docker/lab/Dockerfile_raspberrypi +++ b/docker/lab/Dockerfile_raspberrypi @@ -99,18 +99,14 @@ RUN set -ex \ && yarn --version ARG docker_filepath=docker/lab/files +ARG python_wheel_directory=docker/pennai-arm64-deps/lab RUN apt-get update --fix-missing && apt-get install -y \ - vim openssh-client openssh-server telnet apache2 \ + openssh-client openssh-server telnet apache2 \ net-tools iputils-ping xz-utils \ - screen ngrep ca-cacert \ - mercurial subversion \ + ngrep ca-cacert \ build-essential cmake lsb-core cpio mesa-common-dev \ - dos2unix \ - curl \ - git \ - libatlas-base-dev \ - gfortran \ + dos2unix curl git libatlas-base-dev gfortran \ --no-install-recommends && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -158,9 +154,8 @@ RUN pip --version # setup python environment COPY ${docker_filepath}/requirements.txt /root/ RUN mkdir /root/wheel -COPY ${docker_filepath}/wheel /root/wheel +COPY ${python_wheel_directory}/lab /root/wheel RUN pip3 install --no-cache-dir cython -#RUN pip3 install --no-cache-dir -r /root/requirements.txt RUN pip3 install --no-index --find-links=/root/wheel -r /root/requirements.txt ### bill's surprise fork # NOTE: For some reason, this install doesn't seem to recognize the installed numpy @@ -183,7 +178,7 @@ RUN cp /etc/apache2/mods-available/rewrite* /etc/apache2/mods-enabled \ && cp /etc/apache2/mods-available/expires.load /etc/apache2/mods-enabled # setup node environment -RUN npm install -g pm2 --silent --progress=false +RUN npm install -g pm2 webpack --silent --progress=false # install lab/webapp/node_modules to an anon volume WORKDIR /appsrc/lab/webapp diff --git a/docker/machine/Dockerfile_production b/docker/machine/Dockerfile_production index 570a6c481..c0cbb2f46 100644 --- a/docker/machine/Dockerfile_production +++ b/docker/machine/Dockerfile_production @@ -11,6 +11,7 @@ RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH ARG docker_filepath=docker/machine/files +ARG python_wheels=docker/pennai-arm64-deps/lab/ RUN apt-get update --fix-missing && apt-get install -y \ vim openssh-client openssh-server graphviz \ diff --git a/docker/machine/Dockerfile_raspberrypi b/docker/machine/Dockerfile_raspberrypi index b1941fde6..b044f5e07 100644 --- a/docker/machine/Dockerfile_raspberrypi +++ b/docker/machine/Dockerfile_raspberrypi @@ -1,11 +1,5 @@ FROM python:3.7-stretch -#nodejs -#RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ -# tar -xvf ~/node.tar.xz -C /opt/ && \ -# rm ~/node.tar.xz -#ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH - ENV NODE_VERSION 12.20.0 RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ @@ -60,12 +54,12 @@ RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ && npm --version ARG docker_filepath=docker/machine/files +ARG python_wheel_directory=docker/pennai-arm64-deps RUN apt-get update --fix-missing && apt-get install -y \ - vim openssh-client openssh-server graphviz \ + openssh-client openssh-server graphviz \ net-tools iputils-ping xz-utils \ - screen ngrep ca-cacert \ - mercurial subversion \ + ngrep ca-cacert \ build-essential cmake lsb-core cpio mesa-common-dev \ libglib2.0-0 libxext6 libsm6 libxrender1 dos2unix \ --no-install-recommends && \ @@ -76,6 +70,7 @@ RUN apt-get update --fix-missing && apt-get install -y \ COPY ${docker_filepath}/requirements.txt /root/ RUN mkdir /root/wheel COPY ${docker_filepath}/wheel /root/wheel +COPY ${python_wheel_directory}/machine /root/wheel RUN python -m pip install --no-index --find-links=/root/wheel -r /root/requirements.txt # setup node environment From add2dd78ed9367cc921f8eb119d41833ab08aeb7 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Fri, 15 Jan 2021 16:46:47 -0500 Subject: [PATCH 05/26] Revert machine Dockerfile_production after accidental change --- docker/machine/Dockerfile_production | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/machine/Dockerfile_production b/docker/machine/Dockerfile_production index c0cbb2f46..570a6c481 100644 --- a/docker/machine/Dockerfile_production +++ b/docker/machine/Dockerfile_production @@ -11,7 +11,6 @@ RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH ARG docker_filepath=docker/machine/files -ARG python_wheels=docker/pennai-arm64-deps/lab/ RUN apt-get update --fix-missing && apt-get install -y \ vim openssh-client openssh-server graphviz \ From e5cadd404bc64a58ea026a93ee936b1e38a49093 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Sat, 16 Jan 2021 13:28:11 -0500 Subject: [PATCH 06/26] Builds all docker containers --- docker/lab/Dockerfile_raspberrypi | 8 ++++---- docker/lab/files/requirements-raspberrypi.txt | 9 +++++++++ docker/machine/Dockerfile_raspberrypi | 9 ++++----- docker/machine/files/requirements-raspberrypi.txt | 11 +++++++++++ 4 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 docker/lab/files/requirements-raspberrypi.txt create mode 100644 docker/machine/files/requirements-raspberrypi.txt diff --git a/docker/lab/Dockerfile_raspberrypi b/docker/lab/Dockerfile_raspberrypi index a0d864aa8..f3236b30c 100644 --- a/docker/lab/Dockerfile_raspberrypi +++ b/docker/lab/Dockerfile_raspberrypi @@ -99,7 +99,7 @@ RUN set -ex \ && yarn --version ARG docker_filepath=docker/lab/files -ARG python_wheel_directory=docker/pennai-arm64-deps/lab +ARG python_wheel_directory=docker/pennai-arm64-deps/lab/wheel RUN apt-get update --fix-missing && apt-get install -y \ openssh-client openssh-server telnet apache2 \ @@ -152,11 +152,11 @@ RUN python --version RUN pip --version # setup python environment -COPY ${docker_filepath}/requirements.txt /root/ +COPY ${docker_filepath}/requirements-raspberrypi.txt /root/ RUN mkdir /root/wheel -COPY ${python_wheel_directory}/lab /root/wheel +COPY ${python_wheel_directory}/*.whl /root/wheel/ RUN pip3 install --no-cache-dir cython -RUN pip3 install --no-index --find-links=/root/wheel -r /root/requirements.txt +RUN pip3 install --no-index --find-links=/root/wheel -r /root/requirements-raspberrypi.txt ### bill's surprise fork # NOTE: For some reason, this install doesn't seem to recognize the installed numpy # when run on Docker. As a workaround, the `wheel` directory includes a wheel diff --git a/docker/lab/files/requirements-raspberrypi.txt b/docker/lab/files/requirements-raspberrypi.txt new file mode 100644 index 000000000..b33b0a57b --- /dev/null +++ b/docker/lab/files/requirements-raspberrypi.txt @@ -0,0 +1,9 @@ +scikit-learn==0.22.1 +tqdm==4.32.2 +pymongo==3.8.0 +pandas==1.0.3 +cython==0.29.2 +xgboost==0.90 +simplejson==3.16.0 +requests==2.22.0 +joblib==0.16.0 diff --git a/docker/machine/Dockerfile_raspberrypi b/docker/machine/Dockerfile_raspberrypi index b044f5e07..cf083bd7c 100644 --- a/docker/machine/Dockerfile_raspberrypi +++ b/docker/machine/Dockerfile_raspberrypi @@ -54,7 +54,7 @@ RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ && npm --version ARG docker_filepath=docker/machine/files -ARG python_wheel_directory=docker/pennai-arm64-deps +ARG python_wheel_directory=docker/pennai-arm64-deps/machine/wheel RUN apt-get update --fix-missing && apt-get install -y \ openssh-client openssh-server graphviz \ @@ -67,11 +67,10 @@ RUN apt-get update --fix-missing && apt-get install -y \ rm -rf /var/lib/apt/lists/* # setup python environment -COPY ${docker_filepath}/requirements.txt /root/ +COPY ${docker_filepath}/requirements-raspberrypi.txt /root/ RUN mkdir /root/wheel -COPY ${docker_filepath}/wheel /root/wheel -COPY ${python_wheel_directory}/machine /root/wheel -RUN python -m pip install --no-index --find-links=/root/wheel -r /root/requirements.txt +COPY ${python_wheel_directory}/*.whl /root/wheel/ +RUN python -m pip install --no-index --find-links=/root/wheel -r /root/requirements-raspberrypi.txt # setup node environment RUN npm install -g pm2 --silent --progress=false diff --git a/docker/machine/files/requirements-raspberrypi.txt b/docker/machine/files/requirements-raspberrypi.txt new file mode 100644 index 000000000..1bcbe8cfd --- /dev/null +++ b/docker/machine/files/requirements-raspberrypi.txt @@ -0,0 +1,11 @@ +scikit-learn==0.22.1 +tqdm==4.32.2 +pymongo==3.8.0 +pandas==1.0.3 +matplotlib==3.1.0 +mlxtend==0.16.0 +pydot==1.4.1 +requests==2.22.0 +xgboost==0.90 +shap==0.34.0 +joblib==0.16.0 From 611f55ee4706ce21739e6b4acc98eefc2aa0f0ce Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Sat, 16 Jan 2021 15:38:35 -0500 Subject: [PATCH 07/26] Create mongodb data directory in dbmongo container if it doesn't exist --- docker/dbmongo/files/entrypoint.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/dbmongo/files/entrypoint.sh b/docker/dbmongo/files/entrypoint.sh index 0c9a68dc3..5f24460e3 100644 --- a/docker/dbmongo/files/entrypoint.sh +++ b/docker/dbmongo/files/entrypoint.sh @@ -3,6 +3,9 @@ wget localhost:51678/v1/metadata -t 1 -qO- &> /dev/null if [ ! -v PROJECTS_FILE ]; then PROJECTS_FILE='projects.json'; fi +if [ ! -d /var/lib/mongodb ]; then + mkdir -p /var/lib/mongodb; +fi mongod -f /etc/mongod.conf --fork #check to see if db was loaded if [ ! -f '/root/forum' ]; then From f4315239a3cf74d95dd55f0c714d6a3af2977ba8 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Sun, 24 Jan 2021 14:21:27 -0500 Subject: [PATCH 08/26] Bump machine (rpi) to buster to fix arm64 glibc distribution --- docker/machine/Dockerfile_raspberrypi | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/machine/Dockerfile_raspberrypi b/docker/machine/Dockerfile_raspberrypi index cf083bd7c..c38a3f358 100644 --- a/docker/machine/Dockerfile_raspberrypi +++ b/docker/machine/Dockerfile_raspberrypi @@ -1,4 +1,4 @@ -FROM python:3.7-stretch +FROM python:3.7-buster ENV NODE_VERSION 12.20.0 @@ -59,12 +59,14 @@ ARG python_wheel_directory=docker/pennai-arm64-deps/machine/wheel RUN apt-get update --fix-missing && apt-get install -y \ openssh-client openssh-server graphviz \ net-tools iputils-ping xz-utils \ - ngrep ca-cacert \ - build-essential cmake lsb-core cpio mesa-common-dev \ + ngrep \ + libatlas-base-dev \ + build-essential cmake cpio mesa-common-dev \ libglib2.0-0 libxext6 libsm6 libxrender1 dos2unix \ --no-install-recommends && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +# Removed: lsb-core ca-cacert glibc # setup python environment COPY ${docker_filepath}/requirements-raspberrypi.txt /root/ From ed25cd6bac319740c9b7fb2257a27c21a2816a3a Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Sun, 24 Jan 2021 14:37:20 -0500 Subject: [PATCH 09/26] Improve naming convention for RPi dockerfiles and create RPi test dockerfiles --- docker-compose-int-test-rpi.yml | 70 +++++++++++++++++++ docker-compose-raspberrypi.yml | 6 +- docker-compose-unit-test-rpi.yml | 15 ++++ ...{Dockerfile_raspberrypi => rpi.Dockerfile} | 0 ...{Dockerfile_raspberrypi => rpi.Dockerfile} | 0 ...{Dockerfile_raspberrypi => rpi.Dockerfile} | 0 tests/integration/rpi.Dockerfile | 19 +++++ tests/unit/rpi.Dockerfile | 44 ++++++++++++ 8 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 docker-compose-int-test-rpi.yml create mode 100644 docker-compose-unit-test-rpi.yml rename docker/dbmongo/{Dockerfile_raspberrypi => rpi.Dockerfile} (100%) rename docker/lab/{Dockerfile_raspberrypi => rpi.Dockerfile} (100%) rename docker/machine/{Dockerfile_raspberrypi => rpi.Dockerfile} (100%) create mode 100644 tests/integration/rpi.Dockerfile create mode 100644 tests/unit/rpi.Dockerfile diff --git a/docker-compose-int-test-rpi.yml b/docker-compose-int-test-rpi.yml new file mode 100644 index 000000000..380a0e168 --- /dev/null +++ b/docker-compose-int-test-rpi.yml @@ -0,0 +1,70 @@ +version: '3' + +services: + + tester: + build: ./tests/integration + depends_on: + - lab + - machine + - dbmongo + tty: true + stdin_open: true + command: bash -c "sh /root/wait_pennai.sh && sh /root/int_test_runner.sh" + volumes: + - "./tests/integration/jest:/src/jest" + - "./data/datasets/test/integration:/appsrc/data/datasets/test/integration" + - "./target:/target" + - "./ai/__pycache__:/appsrc/ai/__pycache__" + - "./ai/metalearning/__pycache__:/appsrc/ai/metalearning/__pycache__" + + lab: + build: + context: . + dockerfile: docker/lab/Dockerfile + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/lab/webapp/node_modules" + - "/appsrc/lab/node_modules" + ports: + #- "443:443" + - "5080:5080" + environment: + - AI_RECOMMENDER=random + - AI_NUMRECOMMEND=1 + - AI_AUTOSTART=1 + - STARTUP_DATASET_PATH=/appsrc/data/datasets/test/test_mixed + env_file: ./config/common.env + depends_on: + - dbmongo + + machine: + build: + context: . + dockerfile: docker/machine/Dockerfile + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/machine/node_modules" + ports: + - "5081:5081" + environment: + - STARTUP_DATASET_PATH=/appsrc/data/datasets/test/test_mixed + - MACHINE_PORT=5081 + env_file: ./config/common.env + depends_on: + - lab + - dbmongo + + dbmongo: + build: + context: . + dockerfile: docker/dbmongo/Dockerfile + tty: true + stdin_open: true + ports: + - "27017:27017" + env_file: ./config/common.env diff --git a/docker-compose-raspberrypi.yml b/docker-compose-raspberrypi.yml index 7b40dd500..397a5ac43 100644 --- a/docker-compose-raspberrypi.yml +++ b/docker-compose-raspberrypi.yml @@ -4,7 +4,7 @@ services: lab: build: context: . - dockerfile: docker/lab/Dockerfile_raspberrypi + dockerfile: docker/lab/rpi.Dockerfile tty: true stdin_open: true volumes: @@ -22,7 +22,7 @@ services: machine: build: context: . - dockerfile: docker/machine/Dockerfile_raspberrypi + dockerfile: docker/machine/rpi.Dockerfile tty: true stdin_open: true volumes: @@ -40,7 +40,7 @@ services: dbmongo: build: context: . - dockerfile: docker/dbmongo/Dockerfile_raspberrypi + dockerfile: docker/dbmongo/rpi.Dockerfile tty: true stdin_open: true ports: diff --git a/docker-compose-unit-test-rpi.yml b/docker-compose-unit-test-rpi.yml new file mode 100644 index 000000000..1993eac34 --- /dev/null +++ b/docker-compose-unit-test-rpi.yml @@ -0,0 +1,15 @@ +version: '3' + +services: + unit_tester: + build: + context: . + dockerfile: tests/unit/Dockerfile + working_dir: /appsrc + command: bash -c "dos2unix tests/unit/unit_test_runner.sh && sh tests/unit/unit_test_runner.sh" + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/lab/webapp/node_modules" + - "/appsrc/lab/node_modules" diff --git a/docker/dbmongo/Dockerfile_raspberrypi b/docker/dbmongo/rpi.Dockerfile similarity index 100% rename from docker/dbmongo/Dockerfile_raspberrypi rename to docker/dbmongo/rpi.Dockerfile diff --git a/docker/lab/Dockerfile_raspberrypi b/docker/lab/rpi.Dockerfile similarity index 100% rename from docker/lab/Dockerfile_raspberrypi rename to docker/lab/rpi.Dockerfile diff --git a/docker/machine/Dockerfile_raspberrypi b/docker/machine/rpi.Dockerfile similarity index 100% rename from docker/machine/Dockerfile_raspberrypi rename to docker/machine/rpi.Dockerfile diff --git a/tests/integration/rpi.Dockerfile b/tests/integration/rpi.Dockerfile new file mode 100644 index 000000000..39bbd5e26 --- /dev/null +++ b/tests/integration/rpi.Dockerfile @@ -0,0 +1,19 @@ +FROM node:8.15.1-slim +WORKDIR /src + +RUN apt-get update && apt-get install -y vim dos2unix + +COPY package.json . +COPY tsconfig.json . +# RUN npm install --save es6-promise isomorphic-fetch +# RUN npm install --save-dev typescript jest ts-jest @types/jest +# RUN npm install --save-dev babel-jest babel-preset-react react-test-renderer babel-preset-es2015 +# RUN npm install --save-dev babel-jest babel-preset-react react-test-renderer babel-preset-env +RUN npm install --no-optional --progress=false + +COPY wait-for-it.sh /opt +RUN dos2unix /opt/wait-for-it.sh +RUN ["chmod", "+x", "/opt/wait-for-it.sh"] +COPY wait_pennai.sh /root +COPY int_test_runner.sh /root +RUN dos2unix /root/*.sh diff --git a/tests/unit/rpi.Dockerfile b/tests/unit/rpi.Dockerfile new file mode 100644 index 000000000..f04fee9e6 --- /dev/null +++ b/tests/unit/rpi.Dockerfile @@ -0,0 +1,44 @@ +FROM python:3.7.4-stretch + +#nodejs +RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ + tar -xvf ~/node.tar.xz -C /opt/ && \ + rm ~/node.tar.xz +ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH + +RUN apt-get update --fix-missing \ + && apt-get install -y --no-install-recommends \ + graphviz dos2unix && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN npm install -g mocha + +RUN pip install --upgrade pip + +## tests +COPY /tests/unit/requirements.txt /root/test_requirements.txt +RUN pip install --no-cache-dir -r /root/test_requirements.txt + +## machine and lab container +COPY /docker/lab/files/requirements.txt /root/lab_requirements.txt +RUN pip install --no-cache-dir -r /root/lab_requirements.txt + +COPY /docker/machine/files/requirements.txt /root/mach_requirements.txt +RUN pip install --no-cache-dir -r /root/mach_requirements.txt + +### Bill's surprise fork +RUN pip install --no-cache-dir git+https://github.com/lacava/surprise.git@1.0.8.3 + + +# install lab/node_modules to an anon volume +WORKDIR /appsrc/lab +COPY lab/package.json /appsrc/lab/ +RUN dos2unix /appsrc/lab/package.json +RUN npm install --silent --progress=false + +# install lab/webapp/node_modules to an anon volume +WORKDIR /appsrc/lab/webapp +COPY lab/webapp/package.json /appsrc/lab/webapp/ +RUN dos2unix /appsrc/lab/webapp/package.json +RUN npm install --silent --progress=false From b63c2f584654924dd59eced24f89a88e8193c7ea Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Sun, 24 Jan 2021 14:40:44 -0500 Subject: [PATCH 10/26] Make new test docker-compose files point to the right Dockerfiles --- docker-compose-int-test-rpi.yml | 6 +++--- docker-compose-unit-test-rpi.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose-int-test-rpi.yml b/docker-compose-int-test-rpi.yml index 380a0e168..666a42daf 100644 --- a/docker-compose-int-test-rpi.yml +++ b/docker-compose-int-test-rpi.yml @@ -21,7 +21,7 @@ services: lab: build: context: . - dockerfile: docker/lab/Dockerfile + dockerfile: docker/lab/rpi.Dockerfile tty: true stdin_open: true volumes: @@ -43,7 +43,7 @@ services: machine: build: context: . - dockerfile: docker/machine/Dockerfile + dockerfile: docker/machine/rpi.Dockerfile tty: true stdin_open: true volumes: @@ -62,7 +62,7 @@ services: dbmongo: build: context: . - dockerfile: docker/dbmongo/Dockerfile + dockerfile: docker/dbmongo/rpi.Dockerfile tty: true stdin_open: true ports: diff --git a/docker-compose-unit-test-rpi.yml b/docker-compose-unit-test-rpi.yml index 1993eac34..b2cf26966 100644 --- a/docker-compose-unit-test-rpi.yml +++ b/docker-compose-unit-test-rpi.yml @@ -4,7 +4,7 @@ services: unit_tester: build: context: . - dockerfile: tests/unit/Dockerfile + dockerfile: tests/unit/rpi.Dockerfile working_dir: /appsrc command: bash -c "dos2unix tests/unit/unit_test_runner.sh && sh tests/unit/unit_test_runner.sh" tty: true From 6157f68666d57632f83a685ab5521145d6c923fc Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Mon, 25 Jan 2021 14:24:22 -0500 Subject: [PATCH 11/26] Pass all unit tests on Raspberry Pi --- tests/unit/requirements-rpi.txt | 4 ++++ tests/unit/rpi.Dockerfile | 39 ++++++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 tests/unit/requirements-rpi.txt diff --git a/tests/unit/requirements-rpi.txt b/tests/unit/requirements-rpi.txt new file mode 100644 index 000000000..a7fa3f931 --- /dev/null +++ b/tests/unit/requirements-rpi.txt @@ -0,0 +1,4 @@ +coverage==5.3 +nose==1.3.7 +nose-htmloutput==0.6.0 +parameterized==0.7.0 \ No newline at end of file diff --git a/tests/unit/rpi.Dockerfile b/tests/unit/rpi.Dockerfile index f04fee9e6..0b951e24b 100644 --- a/tests/unit/rpi.Dockerfile +++ b/tests/unit/rpi.Dockerfile @@ -1,34 +1,53 @@ -FROM python:3.7.4-stretch +FROM ubuntu:bionic + +RUN apt-get update && apt-get install -y wget xz-utils curl #nodejs -RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ +RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-arm64.tar.xz -O ~/node.tar.xz && \ tar -xvf ~/node.tar.xz -C /opt/ && \ rm ~/node.tar.xz -ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH +ENV PATH /opt/node-v11.14.0-linux-arm64/bin:$PATH RUN apt-get update --fix-missing \ && apt-get install -y --no-install-recommends \ - graphviz dos2unix && \ + graphviz dos2unix python3-numpy libatlas-base-dev gfortran libgfortran5 && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +# Ensure specific python version +# See: https://stackoverflow.com/a/58562728/1730417 +# and: https://askubuntu.com/a/1176271/260220 +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update +RUN apt-get install -y python3.7 python3.7-dev python3.7-distutils +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 +RUN update-alternatives --set python /usr/bin/python3.7 +RUN curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python get-pip.py --force-reinstall && \ + rm get-pip.py + RUN npm install -g mocha RUN pip install --upgrade pip ## tests -COPY /tests/unit/requirements.txt /root/test_requirements.txt +COPY /tests/unit/requirements-rpi.txt /root/test_requirements.txt RUN pip install --no-cache-dir -r /root/test_requirements.txt ## machine and lab container -COPY /docker/lab/files/requirements.txt /root/lab_requirements.txt -RUN pip install --no-cache-dir -r /root/lab_requirements.txt +RUN mkdir /root/wheel +COPY /docker/pennai-arm64-deps/lab/wheel/*.whl /root/wheel/ +COPY /docker/pennai-arm64-deps/machine/wheel/*.whl /root/wheel/ +COPY /docker/lab/files/requirements-raspberrypi.txt /root/lab_requirements.txt +RUN pip3 install --no-index --find-links=/root/wheel -r /root/lab_requirements.txt -COPY /docker/machine/files/requirements.txt /root/mach_requirements.txt -RUN pip install --no-cache-dir -r /root/mach_requirements.txt +COPY /docker/machine/files/requirements-raspberrypi.txt /root/mach_requirements.txt +RUN pip install --no-index --find-links=/root/wheel -r /root/mach_requirements.txt ### Bill's surprise fork -RUN pip install --no-cache-dir git+https://github.com/lacava/surprise.git@1.0.8.3 +#RUN pip install --verbose --verbose --verbose --no-cache-dir git+https://github.com/lacava/surprise.git@1.0.8.3 +RUN pip install --no-index --find-links=/root/wheel scikit_surprise # install lab/node_modules to an anon volume From 98a9d7e8ed0448309d5164c830f303a98395e726 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 22 Aug 2022 16:17:57 -0700 Subject: [PATCH 12/26] Fixed dev guide production release list, new theme --- .env | 2 +- .gitignore | 1 + docs/guides/developerGuide.md | 26 +++++++++++++------------- docs/requirements.txt | 2 +- docs/source/conf.py | 2 +- 5 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.env b/.env index 39219e769..e394a55ae 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ #!/bin/sh # Docker production build environment -TAG=0.19a0 +TAG=0.18a0 diff --git a/.gitignore b/.gitignore index 9e5da28fb..561b872a2 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,4 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +.vscode \ No newline at end of file diff --git a/docs/guides/developerGuide.md b/docs/guides/developerGuide.md index 4b42b5997..7d88a8d09 100644 --- a/docs/guides/developerGuide.md +++ b/docs/guides/developerGuide.md @@ -207,26 +207,26 @@ Release procedure: 2. **Push changes to github.** Merge the master branch into the `production` branch and push the changes to github. 3. **Build production docker images with `bash release/generate_production_release.sh`.** While in the prodution branch, build the production images and generate the user production .zip by running `bash release/generate_production_release.sh`. This will: -* Create local lab, machine, and dbmongo production docker images with the tag defined in the .env file -* Create the production .zip named `target/production/Aliro-${VERSION}.zip` -``` -git checkout production -bash release/generate_production_release.sh -``` + * Create local lab, machine, and dbmongo production docker images with the tag defined in the .env file + * Create the production .zip named `target/production/Aliro-${VERSION}.zip` + ``` + git checkout production + bash release/generate_production_release.sh + ``` 4. **Push docker images to DockerHub and tag the production git branch by running `deploy_production_release.sh`.** While in the produciton branch, run `bash release/deploy_production_release.sh`. This will: -* Push the production lab, machine and dbmongo production docker images to dockerHub -* Tag the production git branch with the version defined in `.env` -``` -git checkout production -bash release/deploy_production_release.sh -``` + * Push the production lab, machine and dbmongo production docker images to dockerHub + * Tag the production git branch with the version defined in `.env` + ``` + git checkout production + bash release/deploy_production_release.sh + ``` 5. **Test DockerHub images and production code.** Test that the production release works with the newly uploaded DockerHub images by navigating to the directory `target/production/Aliro-${VERSION}` and running `docker-compose up`. This should start an instance of Aliro that loads the newest images from DockerHub. Test that this works as expected. Check that in the enviromental variables section of the admin page, 'TAG' matches the current version. 6. **Create Github Release.** If the test is successful, create a github release using the github web interface. Base the release on the tagged production commit. Attach the file `target/production/Aliro-${VERSION}.zip` as an archive asset. -7. **Update the .env file in the master branch with the new dev version.** Update the `.env` file in the master branch with the next version number and the `a0` suffix (see [Pre-release versioning conventions](https://www.python.org/dev/peps/pep-0440/#pre-releases) and push the changes to git. For example, `0.14` was just released, the new dev tag should be `0.15a0`. +7. **Update the .env file in the master branch with the new dev version.** Update the `.env` file in the master branch with the next version number and the `a0` suffix (see [Pre-release versioning conventions](https://www.python.org/dev/peps/pep-0440/#pre-releases) and push the changes to git. For example, `0.18` was just released, the new dev tag should be `0.19a0`. ### Installing a production build 1. Download a production build from github diff --git a/docs/requirements.txt b/docs/requirements.txt index 152830250..ed26a41e3 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ sphinx==4.5.0 -sphinx-rtd-theme==1.0.0 +piccolo_theme environs==9.5.0 m2r2==0.3.2 \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index a8950a19b..6a0704efe 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -116,7 +116,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = 'piccolo_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the From 34c5ddcf359c8386b132db1878400477e2e19cdd Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Tue, 23 Aug 2022 11:24:51 -0700 Subject: [PATCH 13/26] Release version 0.18 --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index e394a55ae..03eaa284b 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ #!/bin/sh # Docker production build environment -TAG=0.18a0 +TAG=0.18 From 509de36a145fd8c8a4231b4563b188e033c0b8a9 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Fri, 15 Jan 2021 00:29:22 -0500 Subject: [PATCH 14/26] Add new dockerfiles for raspberry pi --- docker/dbmongo/Dockerfile_raspberrypi | 34 ++++ docker/lab/Dockerfile_raspberrypi | 240 ++++++++++++++++++++++++++ docker/machine/Dockerfile_raspberrypi | 97 +++++++++++ 3 files changed, 371 insertions(+) create mode 100644 docker/dbmongo/Dockerfile_raspberrypi create mode 100644 docker/lab/Dockerfile_raspberrypi create mode 100644 docker/machine/Dockerfile_raspberrypi diff --git a/docker/dbmongo/Dockerfile_raspberrypi b/docker/dbmongo/Dockerfile_raspberrypi new file mode 100644 index 000000000..89f3330b0 --- /dev/null +++ b/docker/dbmongo/Dockerfile_raspberrypi @@ -0,0 +1,34 @@ +FROM arm64v8/mongo:bionic +WORKDIR /opt/ + +ARG docker_filepath=docker/dbmongo/files + +#add repo for mongodb +#RUN echo "deb http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.2 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.2.list +#RUN apt-get update --fix-missing && \ +# apt-get install -y wget gnupg +#RUN wget -qO - https://www.mongodb.org/static/pgp/server-4.2.asc | apt-key add - + +#RUN echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.2 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.2.list + +RUN apt-get update --fix-missing && \ + apt-get install -y --allow-unauthenticated \ + vim npm openssh-client htop dos2unix \ + net-tools iputils-ping \ + --no-install-recommends && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +#check if we need to rebuild +COPY ${docker_filepath}/mongod.conf /etc/ +COPY ${docker_filepath}/users.json /root/ +COPY ${docker_filepath}/projects.json /root/ +COPY ${docker_filepath}/entrypoint.sh /root/ +COPY ${docker_filepath}/sync.sh /root/ +RUN dos2unix /root/entrypoint.sh /root/sync.sh +RUN dos2unix /root/users.json /root/projects.json +RUN dos2unix /etc/mongod.conf + +EXPOSE 27017 + +CMD ["/bin/bash", "/root/entrypoint.sh"] diff --git a/docker/lab/Dockerfile_raspberrypi b/docker/lab/Dockerfile_raspberrypi new file mode 100644 index 000000000..70d83b22b --- /dev/null +++ b/docker/lab/Dockerfile_raspberrypi @@ -0,0 +1,240 @@ +FROM ubuntu:bionic + +#RUN apt-get update --fix-missing && apt-get install -y wget + +#nodejs +#RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ +# tar -xvf ~/node.tar.xz -C /opt/ && \ +# rm ~/node.tar.xz +#ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH + +RUN groupadd --gid 1000 node \ + && useradd --uid 1000 --gid node --shell /bin/bash --create-home node + +ENV NODE_VERSION 12.20.0 + +RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) ARCH='x64';; \ + ppc64el) ARCH='ppc64le';; \ + s390x) ARCH='s390x';; \ + arm64) ARCH='arm64';; \ + armhf) ARCH='armv7l';; \ + i386) ARCH='x86';; \ + *) echo "unsupported architecture"; exit 1 ;; \ + esac \ + && set -ex \ + # libatomic1 for arm + && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + && for key in \ + 4ED778F539E3634C779C87C6D7062848A1AB005C \ + 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ + 1C050899334244A8AF75E53792EF661D867B9DFA \ + 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ + 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ + C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ + C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \ + DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ + A48C2BEE680E841632CD4E44F07496B3EB3C1762 \ + 108F52B48DB57BB0CC439B2997B01419BD92F80A \ + B9E2F5981AA6E0CD28160D9FF13993A75599653C \ + ; do \ + gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ + gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ + done \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ + && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ + && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ + && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ + && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ + && apt-mark auto '.*' > /dev/null \ + && find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + && ln -s /usr/local/bin/node /usr/local/bin/nodejs \ + # smoke tests + && node --version \ + && npm --version + +ENV YARN_VERSION 1.22.5 + +RUN set -ex \ + && savedAptMark="$(apt-mark showmanual)" \ + && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + && for key in \ + 6A010C5166006599AA17F08146C2130DFD2497F5 \ + ; do \ + gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ + gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ + done \ + && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" \ + && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc" \ + && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ + && mkdir -p /opt \ + && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ \ + && ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn \ + && ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg \ + && rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ + && apt-mark auto '.*' > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; } \ + && find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + # smoke test + && yarn --version + +ARG docker_filepath=docker/lab/files + +RUN apt-get update --fix-missing && apt-get install -y \ + vim openssh-client openssh-server telnet apache2 \ + net-tools iputils-ping xz-utils \ + screen ngrep ca-cacert \ + mercurial subversion \ + build-essential cmake lsb-core cpio mesa-common-dev \ + dos2unix \ + curl \ + git \ + libatlas-base-dev \ + gfortran \ + --no-install-recommends && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Ensure specific python version +# See: https://stackoverflow.com/a/58562728/1730417 +# and: https://askubuntu.com/a/1176271/260220 +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update +RUN apt-get install -y python3.7 python3.7-dev python3.7-distutils +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 +RUN update-alternatives --set python /usr/bin/python3.7 +RUN curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python get-pip.py --force-reinstall && \ + rm get-pip.py + +# ensure local python is preferred over distribution python +ENV PATH /usr/local/bin:$PATH + +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +ENV TZ=America/New_York +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + libbluetooth-dev \ + tk-dev \ + uuid-dev \ + && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget python3.7-venv python3-numpy \ + && apt-get clean \ + && rm -rf /var/lib/apt/list/* + +RUN python3 --version +RUN python --version +RUN python -m venv /venv +ENV PATH=/venv/bin:$PATH +RUN python --version +RUN pip --version + +# setup python environment +COPY ${docker_filepath}/requirements.txt /root/ +RUN mkdir /root/wheel +COPY ${docker_filepath}/wheel /root/wheel +RUN pip3 install --no-cache-dir cython +#RUN pip3 install --no-cache-dir -r /root/requirements.txt +RUN pip3 install --no-index --find-links=/root/wheel -r /root/requirements.txt +### bill's surprise fork +# NOTE: For some reason, this install doesn't seem to recognize the installed numpy +# when run on Docker. As a workaround, the `wheel` directory includes a wheel +# for this install. +#RUN pip3 install --no-cache-dir git+https://github.com/lacava/surprise.git@1.0.8.3 +RUN pip3 install --no-index --find-links=/root/wheel scikit_surprise + +## Webserver + +RUN rm /etc/apache2/sites-enabled/* +COPY ${docker_filepath}/ports.conf /etc/apache2/ +RUN cp /etc/apache2/mods-available/rewrite* /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/ssl* /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/socache* /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/proxy.* /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/proxy_wstunnel.load /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/proxy_http.load /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/headers.load /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/expires.load /etc/apache2/mods-enabled + +# setup node environment +RUN npm install -g pm2 --silent --progress=false + +# install lab/webapp/node_modules to an anon volume +WORKDIR /appsrc/lab/webapp +COPY lab/webapp/package.json /appsrc/lab/webapp/ +COPY lab/webapp/package-lock.json /appsrc/lab/webapp/ +RUN dos2unix /appsrc/lab/webapp/*.json +RUN npm install --silent --progress=false + +# install lab/node_modules to an anon volume +WORKDIR /appsrc/lab +COPY lab/package.json /appsrc/lab/ +COPY lab/package-lock.json /appsrc/lab/ +RUN dos2unix /appsrc/lab/*.json +RUN npm install --silent --progress=false + +RUN apt-get install -y libgfortran5 liblapack-dev + +COPY ${docker_filepath}/001-pennai.conf /etc/apache2/sites-enabled/ +COPY ${docker_filepath}/htpasswd /etc/apache2/htpasswd +COPY ${docker_filepath}/certs/* /usr/lib/ssl/private/ + +WORKDIR /root/ + +# Webserver - paiwww +COPY ${docker_filepath}/start.sh /root/ + +## Utility script, used when starting ai +COPY ${docker_filepath}/wait-for-it.sh /root/ +RUN ["chmod", "+x", "/root/wait-for-it.sh"] + + +## PennAI Lab server +COPY ${docker_filepath}/entrypoint.sh /root/ +RUN ["chmod", "+x", "/root/entrypoint.sh"] + +RUN dos2unix /root/start.sh \ + && dos2unix /root/wait-for-it.sh \ + && dos2unix /root/entrypoint.sh + +# set version and build environment; tag.sh is sourced in entrypoint.sh +ENV BUILD_ENV='dev' +COPY .env /etc/profile.d/ +RUN cp '/etc/profile.d/.env' '/etc/profile.d/tag.sh' +RUN dos2unix /etc/profile.d/tag.sh +RUN sed -i "s/TAG=/export TAG=/g" /etc/profile.d/tag.sh + + +# Start the webserver +CMD ["/bin/bash", "/root/start.sh"] + +# EXPOSE 443 +EXPOSE 5080 +WORKDIR /appsrc/lab/ +ENTRYPOINT ["/root/entrypoint.sh"] diff --git a/docker/machine/Dockerfile_raspberrypi b/docker/machine/Dockerfile_raspberrypi new file mode 100644 index 000000000..b1941fde6 --- /dev/null +++ b/docker/machine/Dockerfile_raspberrypi @@ -0,0 +1,97 @@ +FROM python:3.7-stretch + +#nodejs +#RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ +# tar -xvf ~/node.tar.xz -C /opt/ && \ +# rm ~/node.tar.xz +#ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH + +ENV NODE_VERSION 12.20.0 + +RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) ARCH='x64';; \ + ppc64el) ARCH='ppc64le';; \ + s390x) ARCH='s390x';; \ + arm64) ARCH='arm64';; \ + armhf) ARCH='armv7l';; \ + i386) ARCH='x86';; \ + *) echo "unsupported architecture"; exit 1 ;; \ + esac \ + && set -ex \ + # libatomic1 for arm + && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + && for key in \ + 4ED778F539E3634C779C87C6D7062848A1AB005C \ + 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ + 1C050899334244A8AF75E53792EF661D867B9DFA \ + 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ + 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ + C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ + C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \ + DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ + A48C2BEE680E841632CD4E44F07496B3EB3C1762 \ + 108F52B48DB57BB0CC439B2997B01419BD92F80A \ + B9E2F5981AA6E0CD28160D9FF13993A75599653C \ + ; do \ + gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ + gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ + done \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ + && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ + && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ + && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ + && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ + && apt-mark auto '.*' > /dev/null \ + && find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + && ln -s /usr/local/bin/node /usr/local/bin/nodejs \ + # smoke tests + && node --version \ + && npm --version + +ARG docker_filepath=docker/machine/files + +RUN apt-get update --fix-missing && apt-get install -y \ + vim openssh-client openssh-server graphviz \ + net-tools iputils-ping xz-utils \ + screen ngrep ca-cacert \ + mercurial subversion \ + build-essential cmake lsb-core cpio mesa-common-dev \ + libglib2.0-0 libxext6 libsm6 libxrender1 dos2unix \ + --no-install-recommends && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# setup python environment +COPY ${docker_filepath}/requirements.txt /root/ +RUN mkdir /root/wheel +COPY ${docker_filepath}/wheel /root/wheel +RUN python -m pip install --no-index --find-links=/root/wheel -r /root/requirements.txt + +# setup node environment +RUN npm install -g pm2 --silent --progress=false + +# install node_modules to an anon volume +WORKDIR /appsrc/machine + +COPY machine/package.json /appsrc/machine/ +COPY machine/package-lock.json /appsrc/machine/ +RUN dos2unix /appsrc/machine/*.json +RUN npm install --silent --progress=false + +COPY ${docker_filepath}/entrypoint.sh /root/ +COPY ${docker_filepath}/wait-for-it.sh /root/ +RUN dos2unix /root/wait-for-it.sh && dos2unix /root/entrypoint.sh +RUN ["chmod", "+x", "/root/wait-for-it.sh"] + +CMD ["/bin/bash", "/root/entrypoint.sh"] From b478c9885b4aee8267fa3e8a22bbf7b1a3cddbe3 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Fri, 15 Jan 2021 15:33:32 -0500 Subject: [PATCH 15/26] Create new docker compose configuration for rpi --- docker-compose-raspberrypi.yml | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 docker-compose-raspberrypi.yml diff --git a/docker-compose-raspberrypi.yml b/docker-compose-raspberrypi.yml new file mode 100644 index 000000000..7b40dd500 --- /dev/null +++ b/docker-compose-raspberrypi.yml @@ -0,0 +1,48 @@ +version: '3' + +services: + lab: + build: + context: . + dockerfile: docker/lab/Dockerfile_raspberrypi + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/lab/webapp/node_modules" + - "/appsrc/lab/node_modules" + ports: + - "5080:5080" + env_file: + - ./config/common.env + - ./config/ai.env + depends_on: + - dbmongo + + machine: + build: + context: . + dockerfile: docker/machine/Dockerfile_raspberrypi + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/machine/node_modules" + ports: + - "5081:5081" + env_file: ./config/common.env + environment: + - MACHINE_PORT=5081 + depends_on: + - lab + - dbmongo + + dbmongo: + build: + context: . + dockerfile: docker/dbmongo/Dockerfile_raspberrypi + tty: true + stdin_open: true + ports: + - "27017:27017" + env_file: ./config/common.env From 6e1e97a56f1df247f463296ba6bea7c5462c32ce Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Fri, 15 Jan 2021 16:06:02 -0500 Subject: [PATCH 16/26] Add rpi dependency wheels as a Git submodule --- .gitmodules | 3 +++ docker/pennai-arm64-deps | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 docker/pennai-arm64-deps diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..363522775 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "docker/pennai-arm64-deps"] + path = docker/pennai-arm64-deps + url = https://github.com/EpistasisLab/pennai-arm64-deps diff --git a/docker/pennai-arm64-deps b/docker/pennai-arm64-deps new file mode 160000 index 000000000..83b7045aa --- /dev/null +++ b/docker/pennai-arm64-deps @@ -0,0 +1 @@ +Subproject commit 83b7045aa973cda00a830af46ef0d6dc1f52f04b From c4abad146ec1fa3f18a560d08ab47c0039ebcac2 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Fri, 15 Jan 2021 16:33:16 -0500 Subject: [PATCH 17/26] Make Dockerfiles for rpi see wheel files --- docker/dbmongo/Dockerfile_raspberrypi | 2 +- docker/lab/Dockerfile_raspberrypi | 17 ++++++----------- docker/machine/Dockerfile_production | 1 + docker/machine/Dockerfile_raspberrypi | 13 ++++--------- 4 files changed, 12 insertions(+), 21 deletions(-) diff --git a/docker/dbmongo/Dockerfile_raspberrypi b/docker/dbmongo/Dockerfile_raspberrypi index 89f3330b0..8bf6e0133 100644 --- a/docker/dbmongo/Dockerfile_raspberrypi +++ b/docker/dbmongo/Dockerfile_raspberrypi @@ -13,7 +13,7 @@ ARG docker_filepath=docker/dbmongo/files RUN apt-get update --fix-missing && \ apt-get install -y --allow-unauthenticated \ - vim npm openssh-client htop dos2unix \ + npm openssh-client htop dos2unix \ net-tools iputils-ping \ --no-install-recommends && \ apt-get clean && \ diff --git a/docker/lab/Dockerfile_raspberrypi b/docker/lab/Dockerfile_raspberrypi index 70d83b22b..a0d864aa8 100644 --- a/docker/lab/Dockerfile_raspberrypi +++ b/docker/lab/Dockerfile_raspberrypi @@ -99,18 +99,14 @@ RUN set -ex \ && yarn --version ARG docker_filepath=docker/lab/files +ARG python_wheel_directory=docker/pennai-arm64-deps/lab RUN apt-get update --fix-missing && apt-get install -y \ - vim openssh-client openssh-server telnet apache2 \ + openssh-client openssh-server telnet apache2 \ net-tools iputils-ping xz-utils \ - screen ngrep ca-cacert \ - mercurial subversion \ + ngrep ca-cacert \ build-essential cmake lsb-core cpio mesa-common-dev \ - dos2unix \ - curl \ - git \ - libatlas-base-dev \ - gfortran \ + dos2unix curl git libatlas-base-dev gfortran \ --no-install-recommends && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -158,9 +154,8 @@ RUN pip --version # setup python environment COPY ${docker_filepath}/requirements.txt /root/ RUN mkdir /root/wheel -COPY ${docker_filepath}/wheel /root/wheel +COPY ${python_wheel_directory}/lab /root/wheel RUN pip3 install --no-cache-dir cython -#RUN pip3 install --no-cache-dir -r /root/requirements.txt RUN pip3 install --no-index --find-links=/root/wheel -r /root/requirements.txt ### bill's surprise fork # NOTE: For some reason, this install doesn't seem to recognize the installed numpy @@ -183,7 +178,7 @@ RUN cp /etc/apache2/mods-available/rewrite* /etc/apache2/mods-enabled \ && cp /etc/apache2/mods-available/expires.load /etc/apache2/mods-enabled # setup node environment -RUN npm install -g pm2 --silent --progress=false +RUN npm install -g pm2 webpack --silent --progress=false # install lab/webapp/node_modules to an anon volume WORKDIR /appsrc/lab/webapp diff --git a/docker/machine/Dockerfile_production b/docker/machine/Dockerfile_production index e0f7f1439..63d57ccfa 100644 --- a/docker/machine/Dockerfile_production +++ b/docker/machine/Dockerfile_production @@ -11,6 +11,7 @@ RUN wget --quiet https://nodejs.org/dist/v12.22.9/node-v12.22.9-linux-x64.tar.xz ENV PATH /opt/node-v12.22.9-linux-x64/bin:$PATH ARG docker_filepath=docker/machine/files +ARG python_wheels=docker/pennai-arm64-deps/lab/ RUN apt-get update --fix-missing && apt-get install -y \ vim openssh-client openssh-server graphviz \ diff --git a/docker/machine/Dockerfile_raspberrypi b/docker/machine/Dockerfile_raspberrypi index b1941fde6..b044f5e07 100644 --- a/docker/machine/Dockerfile_raspberrypi +++ b/docker/machine/Dockerfile_raspberrypi @@ -1,11 +1,5 @@ FROM python:3.7-stretch -#nodejs -#RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ -# tar -xvf ~/node.tar.xz -C /opt/ && \ -# rm ~/node.tar.xz -#ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH - ENV NODE_VERSION 12.20.0 RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ @@ -60,12 +54,12 @@ RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ && npm --version ARG docker_filepath=docker/machine/files +ARG python_wheel_directory=docker/pennai-arm64-deps RUN apt-get update --fix-missing && apt-get install -y \ - vim openssh-client openssh-server graphviz \ + openssh-client openssh-server graphviz \ net-tools iputils-ping xz-utils \ - screen ngrep ca-cacert \ - mercurial subversion \ + ngrep ca-cacert \ build-essential cmake lsb-core cpio mesa-common-dev \ libglib2.0-0 libxext6 libsm6 libxrender1 dos2unix \ --no-install-recommends && \ @@ -76,6 +70,7 @@ RUN apt-get update --fix-missing && apt-get install -y \ COPY ${docker_filepath}/requirements.txt /root/ RUN mkdir /root/wheel COPY ${docker_filepath}/wheel /root/wheel +COPY ${python_wheel_directory}/machine /root/wheel RUN python -m pip install --no-index --find-links=/root/wheel -r /root/requirements.txt # setup node environment From 1e761e599f7b4b9164982e394aef5ab1fb5acb3f Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Fri, 15 Jan 2021 16:46:47 -0500 Subject: [PATCH 18/26] Revert machine Dockerfile_production after accidental change --- docker/machine/Dockerfile_production | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/machine/Dockerfile_production b/docker/machine/Dockerfile_production index 63d57ccfa..e0f7f1439 100644 --- a/docker/machine/Dockerfile_production +++ b/docker/machine/Dockerfile_production @@ -11,7 +11,6 @@ RUN wget --quiet https://nodejs.org/dist/v12.22.9/node-v12.22.9-linux-x64.tar.xz ENV PATH /opt/node-v12.22.9-linux-x64/bin:$PATH ARG docker_filepath=docker/machine/files -ARG python_wheels=docker/pennai-arm64-deps/lab/ RUN apt-get update --fix-missing && apt-get install -y \ vim openssh-client openssh-server graphviz \ From 3bb4b001815e4dd3e06d87656251a98f76e2c1aa Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Sat, 16 Jan 2021 13:28:11 -0500 Subject: [PATCH 19/26] Builds all docker containers --- docker/lab/Dockerfile_raspberrypi | 8 ++++---- docker/lab/files/requirements-raspberrypi.txt | 9 +++++++++ docker/machine/Dockerfile_raspberrypi | 9 ++++----- docker/machine/files/requirements-raspberrypi.txt | 11 +++++++++++ 4 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 docker/lab/files/requirements-raspberrypi.txt create mode 100644 docker/machine/files/requirements-raspberrypi.txt diff --git a/docker/lab/Dockerfile_raspberrypi b/docker/lab/Dockerfile_raspberrypi index a0d864aa8..f3236b30c 100644 --- a/docker/lab/Dockerfile_raspberrypi +++ b/docker/lab/Dockerfile_raspberrypi @@ -99,7 +99,7 @@ RUN set -ex \ && yarn --version ARG docker_filepath=docker/lab/files -ARG python_wheel_directory=docker/pennai-arm64-deps/lab +ARG python_wheel_directory=docker/pennai-arm64-deps/lab/wheel RUN apt-get update --fix-missing && apt-get install -y \ openssh-client openssh-server telnet apache2 \ @@ -152,11 +152,11 @@ RUN python --version RUN pip --version # setup python environment -COPY ${docker_filepath}/requirements.txt /root/ +COPY ${docker_filepath}/requirements-raspberrypi.txt /root/ RUN mkdir /root/wheel -COPY ${python_wheel_directory}/lab /root/wheel +COPY ${python_wheel_directory}/*.whl /root/wheel/ RUN pip3 install --no-cache-dir cython -RUN pip3 install --no-index --find-links=/root/wheel -r /root/requirements.txt +RUN pip3 install --no-index --find-links=/root/wheel -r /root/requirements-raspberrypi.txt ### bill's surprise fork # NOTE: For some reason, this install doesn't seem to recognize the installed numpy # when run on Docker. As a workaround, the `wheel` directory includes a wheel diff --git a/docker/lab/files/requirements-raspberrypi.txt b/docker/lab/files/requirements-raspberrypi.txt new file mode 100644 index 000000000..b33b0a57b --- /dev/null +++ b/docker/lab/files/requirements-raspberrypi.txt @@ -0,0 +1,9 @@ +scikit-learn==0.22.1 +tqdm==4.32.2 +pymongo==3.8.0 +pandas==1.0.3 +cython==0.29.2 +xgboost==0.90 +simplejson==3.16.0 +requests==2.22.0 +joblib==0.16.0 diff --git a/docker/machine/Dockerfile_raspberrypi b/docker/machine/Dockerfile_raspberrypi index b044f5e07..cf083bd7c 100644 --- a/docker/machine/Dockerfile_raspberrypi +++ b/docker/machine/Dockerfile_raspberrypi @@ -54,7 +54,7 @@ RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ && npm --version ARG docker_filepath=docker/machine/files -ARG python_wheel_directory=docker/pennai-arm64-deps +ARG python_wheel_directory=docker/pennai-arm64-deps/machine/wheel RUN apt-get update --fix-missing && apt-get install -y \ openssh-client openssh-server graphviz \ @@ -67,11 +67,10 @@ RUN apt-get update --fix-missing && apt-get install -y \ rm -rf /var/lib/apt/lists/* # setup python environment -COPY ${docker_filepath}/requirements.txt /root/ +COPY ${docker_filepath}/requirements-raspberrypi.txt /root/ RUN mkdir /root/wheel -COPY ${docker_filepath}/wheel /root/wheel -COPY ${python_wheel_directory}/machine /root/wheel -RUN python -m pip install --no-index --find-links=/root/wheel -r /root/requirements.txt +COPY ${python_wheel_directory}/*.whl /root/wheel/ +RUN python -m pip install --no-index --find-links=/root/wheel -r /root/requirements-raspberrypi.txt # setup node environment RUN npm install -g pm2 --silent --progress=false diff --git a/docker/machine/files/requirements-raspberrypi.txt b/docker/machine/files/requirements-raspberrypi.txt new file mode 100644 index 000000000..1bcbe8cfd --- /dev/null +++ b/docker/machine/files/requirements-raspberrypi.txt @@ -0,0 +1,11 @@ +scikit-learn==0.22.1 +tqdm==4.32.2 +pymongo==3.8.0 +pandas==1.0.3 +matplotlib==3.1.0 +mlxtend==0.16.0 +pydot==1.4.1 +requests==2.22.0 +xgboost==0.90 +shap==0.34.0 +joblib==0.16.0 From 20f7e5898d9fedabda1e904615a62cf4fbc3235b Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Sat, 16 Jan 2021 15:38:35 -0500 Subject: [PATCH 20/26] Create mongodb data directory in dbmongo container if it doesn't exist --- docker/dbmongo/files/entrypoint.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/dbmongo/files/entrypoint.sh b/docker/dbmongo/files/entrypoint.sh index 0c9a68dc3..5f24460e3 100644 --- a/docker/dbmongo/files/entrypoint.sh +++ b/docker/dbmongo/files/entrypoint.sh @@ -3,6 +3,9 @@ wget localhost:51678/v1/metadata -t 1 -qO- &> /dev/null if [ ! -v PROJECTS_FILE ]; then PROJECTS_FILE='projects.json'; fi +if [ ! -d /var/lib/mongodb ]; then + mkdir -p /var/lib/mongodb; +fi mongod -f /etc/mongod.conf --fork #check to see if db was loaded if [ ! -f '/root/forum' ]; then From 89004faf384df468253856b668777d611b5efddd Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Sun, 24 Jan 2021 14:21:27 -0500 Subject: [PATCH 21/26] Bump machine (rpi) to buster to fix arm64 glibc distribution --- docker/machine/Dockerfile_raspberrypi | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/machine/Dockerfile_raspberrypi b/docker/machine/Dockerfile_raspberrypi index cf083bd7c..c38a3f358 100644 --- a/docker/machine/Dockerfile_raspberrypi +++ b/docker/machine/Dockerfile_raspberrypi @@ -1,4 +1,4 @@ -FROM python:3.7-stretch +FROM python:3.7-buster ENV NODE_VERSION 12.20.0 @@ -59,12 +59,14 @@ ARG python_wheel_directory=docker/pennai-arm64-deps/machine/wheel RUN apt-get update --fix-missing && apt-get install -y \ openssh-client openssh-server graphviz \ net-tools iputils-ping xz-utils \ - ngrep ca-cacert \ - build-essential cmake lsb-core cpio mesa-common-dev \ + ngrep \ + libatlas-base-dev \ + build-essential cmake cpio mesa-common-dev \ libglib2.0-0 libxext6 libsm6 libxrender1 dos2unix \ --no-install-recommends && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +# Removed: lsb-core ca-cacert glibc # setup python environment COPY ${docker_filepath}/requirements-raspberrypi.txt /root/ From 7a36cd74fc744949466e23744585eb65b5874f07 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Sun, 24 Jan 2021 14:37:20 -0500 Subject: [PATCH 22/26] Improve naming convention for RPi dockerfiles and create RPi test dockerfiles --- docker-compose-int-test-rpi.yml | 70 +++++++++++++++++++ docker-compose-raspberrypi.yml | 6 +- docker-compose-unit-test-rpi.yml | 15 ++++ ...{Dockerfile_raspberrypi => rpi.Dockerfile} | 0 ...{Dockerfile_raspberrypi => rpi.Dockerfile} | 0 ...{Dockerfile_raspberrypi => rpi.Dockerfile} | 0 tests/integration/rpi.Dockerfile | 19 +++++ tests/unit/rpi.Dockerfile | 44 ++++++++++++ 8 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 docker-compose-int-test-rpi.yml create mode 100644 docker-compose-unit-test-rpi.yml rename docker/dbmongo/{Dockerfile_raspberrypi => rpi.Dockerfile} (100%) rename docker/lab/{Dockerfile_raspberrypi => rpi.Dockerfile} (100%) rename docker/machine/{Dockerfile_raspberrypi => rpi.Dockerfile} (100%) create mode 100644 tests/integration/rpi.Dockerfile create mode 100644 tests/unit/rpi.Dockerfile diff --git a/docker-compose-int-test-rpi.yml b/docker-compose-int-test-rpi.yml new file mode 100644 index 000000000..380a0e168 --- /dev/null +++ b/docker-compose-int-test-rpi.yml @@ -0,0 +1,70 @@ +version: '3' + +services: + + tester: + build: ./tests/integration + depends_on: + - lab + - machine + - dbmongo + tty: true + stdin_open: true + command: bash -c "sh /root/wait_pennai.sh && sh /root/int_test_runner.sh" + volumes: + - "./tests/integration/jest:/src/jest" + - "./data/datasets/test/integration:/appsrc/data/datasets/test/integration" + - "./target:/target" + - "./ai/__pycache__:/appsrc/ai/__pycache__" + - "./ai/metalearning/__pycache__:/appsrc/ai/metalearning/__pycache__" + + lab: + build: + context: . + dockerfile: docker/lab/Dockerfile + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/lab/webapp/node_modules" + - "/appsrc/lab/node_modules" + ports: + #- "443:443" + - "5080:5080" + environment: + - AI_RECOMMENDER=random + - AI_NUMRECOMMEND=1 + - AI_AUTOSTART=1 + - STARTUP_DATASET_PATH=/appsrc/data/datasets/test/test_mixed + env_file: ./config/common.env + depends_on: + - dbmongo + + machine: + build: + context: . + dockerfile: docker/machine/Dockerfile + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/machine/node_modules" + ports: + - "5081:5081" + environment: + - STARTUP_DATASET_PATH=/appsrc/data/datasets/test/test_mixed + - MACHINE_PORT=5081 + env_file: ./config/common.env + depends_on: + - lab + - dbmongo + + dbmongo: + build: + context: . + dockerfile: docker/dbmongo/Dockerfile + tty: true + stdin_open: true + ports: + - "27017:27017" + env_file: ./config/common.env diff --git a/docker-compose-raspberrypi.yml b/docker-compose-raspberrypi.yml index 7b40dd500..397a5ac43 100644 --- a/docker-compose-raspberrypi.yml +++ b/docker-compose-raspberrypi.yml @@ -4,7 +4,7 @@ services: lab: build: context: . - dockerfile: docker/lab/Dockerfile_raspberrypi + dockerfile: docker/lab/rpi.Dockerfile tty: true stdin_open: true volumes: @@ -22,7 +22,7 @@ services: machine: build: context: . - dockerfile: docker/machine/Dockerfile_raspberrypi + dockerfile: docker/machine/rpi.Dockerfile tty: true stdin_open: true volumes: @@ -40,7 +40,7 @@ services: dbmongo: build: context: . - dockerfile: docker/dbmongo/Dockerfile_raspberrypi + dockerfile: docker/dbmongo/rpi.Dockerfile tty: true stdin_open: true ports: diff --git a/docker-compose-unit-test-rpi.yml b/docker-compose-unit-test-rpi.yml new file mode 100644 index 000000000..1993eac34 --- /dev/null +++ b/docker-compose-unit-test-rpi.yml @@ -0,0 +1,15 @@ +version: '3' + +services: + unit_tester: + build: + context: . + dockerfile: tests/unit/Dockerfile + working_dir: /appsrc + command: bash -c "dos2unix tests/unit/unit_test_runner.sh && sh tests/unit/unit_test_runner.sh" + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/lab/webapp/node_modules" + - "/appsrc/lab/node_modules" diff --git a/docker/dbmongo/Dockerfile_raspberrypi b/docker/dbmongo/rpi.Dockerfile similarity index 100% rename from docker/dbmongo/Dockerfile_raspberrypi rename to docker/dbmongo/rpi.Dockerfile diff --git a/docker/lab/Dockerfile_raspberrypi b/docker/lab/rpi.Dockerfile similarity index 100% rename from docker/lab/Dockerfile_raspberrypi rename to docker/lab/rpi.Dockerfile diff --git a/docker/machine/Dockerfile_raspberrypi b/docker/machine/rpi.Dockerfile similarity index 100% rename from docker/machine/Dockerfile_raspberrypi rename to docker/machine/rpi.Dockerfile diff --git a/tests/integration/rpi.Dockerfile b/tests/integration/rpi.Dockerfile new file mode 100644 index 000000000..39bbd5e26 --- /dev/null +++ b/tests/integration/rpi.Dockerfile @@ -0,0 +1,19 @@ +FROM node:8.15.1-slim +WORKDIR /src + +RUN apt-get update && apt-get install -y vim dos2unix + +COPY package.json . +COPY tsconfig.json . +# RUN npm install --save es6-promise isomorphic-fetch +# RUN npm install --save-dev typescript jest ts-jest @types/jest +# RUN npm install --save-dev babel-jest babel-preset-react react-test-renderer babel-preset-es2015 +# RUN npm install --save-dev babel-jest babel-preset-react react-test-renderer babel-preset-env +RUN npm install --no-optional --progress=false + +COPY wait-for-it.sh /opt +RUN dos2unix /opt/wait-for-it.sh +RUN ["chmod", "+x", "/opt/wait-for-it.sh"] +COPY wait_pennai.sh /root +COPY int_test_runner.sh /root +RUN dos2unix /root/*.sh diff --git a/tests/unit/rpi.Dockerfile b/tests/unit/rpi.Dockerfile new file mode 100644 index 000000000..f04fee9e6 --- /dev/null +++ b/tests/unit/rpi.Dockerfile @@ -0,0 +1,44 @@ +FROM python:3.7.4-stretch + +#nodejs +RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ + tar -xvf ~/node.tar.xz -C /opt/ && \ + rm ~/node.tar.xz +ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH + +RUN apt-get update --fix-missing \ + && apt-get install -y --no-install-recommends \ + graphviz dos2unix && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN npm install -g mocha + +RUN pip install --upgrade pip + +## tests +COPY /tests/unit/requirements.txt /root/test_requirements.txt +RUN pip install --no-cache-dir -r /root/test_requirements.txt + +## machine and lab container +COPY /docker/lab/files/requirements.txt /root/lab_requirements.txt +RUN pip install --no-cache-dir -r /root/lab_requirements.txt + +COPY /docker/machine/files/requirements.txt /root/mach_requirements.txt +RUN pip install --no-cache-dir -r /root/mach_requirements.txt + +### Bill's surprise fork +RUN pip install --no-cache-dir git+https://github.com/lacava/surprise.git@1.0.8.3 + + +# install lab/node_modules to an anon volume +WORKDIR /appsrc/lab +COPY lab/package.json /appsrc/lab/ +RUN dos2unix /appsrc/lab/package.json +RUN npm install --silent --progress=false + +# install lab/webapp/node_modules to an anon volume +WORKDIR /appsrc/lab/webapp +COPY lab/webapp/package.json /appsrc/lab/webapp/ +RUN dos2unix /appsrc/lab/webapp/package.json +RUN npm install --silent --progress=false From 4c91f7444ce8d9425bdab932cb66fc55a9b31247 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Sun, 24 Jan 2021 14:40:44 -0500 Subject: [PATCH 23/26] Make new test docker-compose files point to the right Dockerfiles --- docker-compose-int-test-rpi.yml | 6 +++--- docker-compose-unit-test-rpi.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose-int-test-rpi.yml b/docker-compose-int-test-rpi.yml index 380a0e168..666a42daf 100644 --- a/docker-compose-int-test-rpi.yml +++ b/docker-compose-int-test-rpi.yml @@ -21,7 +21,7 @@ services: lab: build: context: . - dockerfile: docker/lab/Dockerfile + dockerfile: docker/lab/rpi.Dockerfile tty: true stdin_open: true volumes: @@ -43,7 +43,7 @@ services: machine: build: context: . - dockerfile: docker/machine/Dockerfile + dockerfile: docker/machine/rpi.Dockerfile tty: true stdin_open: true volumes: @@ -62,7 +62,7 @@ services: dbmongo: build: context: . - dockerfile: docker/dbmongo/Dockerfile + dockerfile: docker/dbmongo/rpi.Dockerfile tty: true stdin_open: true ports: diff --git a/docker-compose-unit-test-rpi.yml b/docker-compose-unit-test-rpi.yml index 1993eac34..b2cf26966 100644 --- a/docker-compose-unit-test-rpi.yml +++ b/docker-compose-unit-test-rpi.yml @@ -4,7 +4,7 @@ services: unit_tester: build: context: . - dockerfile: tests/unit/Dockerfile + dockerfile: tests/unit/rpi.Dockerfile working_dir: /appsrc command: bash -c "dos2unix tests/unit/unit_test_runner.sh && sh tests/unit/unit_test_runner.sh" tty: true From 61b3ed1ac735f39d2de8afef04e566473d388a76 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Mon, 25 Jan 2021 14:24:22 -0500 Subject: [PATCH 24/26] Pass all unit tests on Raspberry Pi --- tests/unit/requirements-rpi.txt | 4 ++++ tests/unit/rpi.Dockerfile | 39 ++++++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 tests/unit/requirements-rpi.txt diff --git a/tests/unit/requirements-rpi.txt b/tests/unit/requirements-rpi.txt new file mode 100644 index 000000000..a7fa3f931 --- /dev/null +++ b/tests/unit/requirements-rpi.txt @@ -0,0 +1,4 @@ +coverage==5.3 +nose==1.3.7 +nose-htmloutput==0.6.0 +parameterized==0.7.0 \ No newline at end of file diff --git a/tests/unit/rpi.Dockerfile b/tests/unit/rpi.Dockerfile index f04fee9e6..0b951e24b 100644 --- a/tests/unit/rpi.Dockerfile +++ b/tests/unit/rpi.Dockerfile @@ -1,34 +1,53 @@ -FROM python:3.7.4-stretch +FROM ubuntu:bionic + +RUN apt-get update && apt-get install -y wget xz-utils curl #nodejs -RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ +RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-arm64.tar.xz -O ~/node.tar.xz && \ tar -xvf ~/node.tar.xz -C /opt/ && \ rm ~/node.tar.xz -ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH +ENV PATH /opt/node-v11.14.0-linux-arm64/bin:$PATH RUN apt-get update --fix-missing \ && apt-get install -y --no-install-recommends \ - graphviz dos2unix && \ + graphviz dos2unix python3-numpy libatlas-base-dev gfortran libgfortran5 && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +# Ensure specific python version +# See: https://stackoverflow.com/a/58562728/1730417 +# and: https://askubuntu.com/a/1176271/260220 +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update +RUN apt-get install -y python3.7 python3.7-dev python3.7-distutils +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 +RUN update-alternatives --set python /usr/bin/python3.7 +RUN curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python get-pip.py --force-reinstall && \ + rm get-pip.py + RUN npm install -g mocha RUN pip install --upgrade pip ## tests -COPY /tests/unit/requirements.txt /root/test_requirements.txt +COPY /tests/unit/requirements-rpi.txt /root/test_requirements.txt RUN pip install --no-cache-dir -r /root/test_requirements.txt ## machine and lab container -COPY /docker/lab/files/requirements.txt /root/lab_requirements.txt -RUN pip install --no-cache-dir -r /root/lab_requirements.txt +RUN mkdir /root/wheel +COPY /docker/pennai-arm64-deps/lab/wheel/*.whl /root/wheel/ +COPY /docker/pennai-arm64-deps/machine/wheel/*.whl /root/wheel/ +COPY /docker/lab/files/requirements-raspberrypi.txt /root/lab_requirements.txt +RUN pip3 install --no-index --find-links=/root/wheel -r /root/lab_requirements.txt -COPY /docker/machine/files/requirements.txt /root/mach_requirements.txt -RUN pip install --no-cache-dir -r /root/mach_requirements.txt +COPY /docker/machine/files/requirements-raspberrypi.txt /root/mach_requirements.txt +RUN pip install --no-index --find-links=/root/wheel -r /root/mach_requirements.txt ### Bill's surprise fork -RUN pip install --no-cache-dir git+https://github.com/lacava/surprise.git@1.0.8.3 +#RUN pip install --verbose --verbose --verbose --no-cache-dir git+https://github.com/lacava/surprise.git@1.0.8.3 +RUN pip install --no-index --find-links=/root/wheel scikit_surprise # install lab/node_modules to an anon volume From b2869e564caa19597370bd7ee876f6d83002d262 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Tue, 23 Aug 2022 14:20:04 -0700 Subject: [PATCH 25/26] Added rpi production yml and Dockerfiles --- docker-compose-raspberrypi-production.yml | 53 +++++ docker-compose-raspberrypi.yml | 6 +- docker/lab/rpi.Dockerfile | 6 +- docker/lab/rpi_production.Dockerfile | 249 ++++++++++++++++++++++ docker/machine/rpi.Dockerfile | 4 +- docker/machine/rpi_production.Dockerfile | 99 +++++++++ 6 files changed, 412 insertions(+), 5 deletions(-) create mode 100644 docker-compose-raspberrypi-production.yml create mode 100644 docker/lab/rpi_production.Dockerfile create mode 100644 docker/machine/rpi_production.Dockerfile diff --git a/docker-compose-raspberrypi-production.yml b/docker-compose-raspberrypi-production.yml new file mode 100644 index 000000000..c06e0879e --- /dev/null +++ b/docker-compose-raspberrypi-production.yml @@ -0,0 +1,53 @@ +# Build production images +version: '3' + +services: + lab_rpi: + build: + context: . + dockerfile: docker/lab/rpi_production.Dockerfile + image: "aliro_lab_rpi:${TAG}" + tty: true + stdin_open: true + volumes: + - "/appsrc" + - "./data/datasets/user:/appsrc/data/datasets/user" + - "./data/recommenders/pennaiweb:/appsrc/data/recommenders/pennaiweb" + - "./target:/appsrc/target" + ports: + - "5080:5080" + env_file: + - ./config/common.env + - ./config/ai.env + depends_on: + - dbmongo + + machine_rpi: + build: + context: . + dockerfile: docker/machine/rpi_production.Dockerfile + image: "aliro_machine_rpi:${TAG}" + tty: true + stdin_open: true + volumes: + - "/appsrc" + - "./target:/appsrc/target" + ports: + - "5081:5081" + env_file: ./config/common.env + environment: + - MACHINE_PORT=5081 + depends_on: + - lab + - dbmongo + + dbmongo_rpi: + build: + context: . + dockerfile: docker/dbmongo/rpi.Dockerfile + image: "aliro_dbmongo_rpi:${TAG}" + tty: true + stdin_open: true + ports: + - "27017:27017" + env_file: ./config/common.env diff --git a/docker-compose-raspberrypi.yml b/docker-compose-raspberrypi.yml index 397a5ac43..afd383063 100644 --- a/docker-compose-raspberrypi.yml +++ b/docker-compose-raspberrypi.yml @@ -1,7 +1,7 @@ version: '3' services: - lab: + lab_rpi: build: context: . dockerfile: docker/lab/rpi.Dockerfile @@ -19,7 +19,7 @@ services: depends_on: - dbmongo - machine: + machine_rpi: build: context: . dockerfile: docker/machine/rpi.Dockerfile @@ -37,7 +37,7 @@ services: - lab - dbmongo - dbmongo: + dbmongo_rpi: build: context: . dockerfile: docker/dbmongo/rpi.Dockerfile diff --git a/docker/lab/rpi.Dockerfile b/docker/lab/rpi.Dockerfile index f3236b30c..39d4f4896 100644 --- a/docker/lab/rpi.Dockerfile +++ b/docker/lab/rpi.Dockerfile @@ -11,7 +11,7 @@ FROM ubuntu:bionic RUN groupadd --gid 1000 node \ && useradd --uid 1000 --gid node --shell /bin/bash --create-home node -ENV NODE_VERSION 12.20.0 +ENV NODE_VERSION 12.22.9 RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ && case "${dpkgArch##*-}" in \ @@ -40,6 +40,8 @@ RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ 108F52B48DB57BB0CC439B2997B01419BD92F80A \ B9E2F5981AA6E0CD28160D9FF13993A75599653C \ ; do \ + gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \ gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ @@ -73,6 +75,8 @@ RUN set -ex \ && for key in \ 6A010C5166006599AA17F08146C2130DFD2497F5 \ ; do \ + gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \ gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ diff --git a/docker/lab/rpi_production.Dockerfile b/docker/lab/rpi_production.Dockerfile new file mode 100644 index 000000000..2b75b0cf6 --- /dev/null +++ b/docker/lab/rpi_production.Dockerfile @@ -0,0 +1,249 @@ +FROM ubuntu:bionic + +# copy src for lab +COPY lab /appsrc/lab +COPY ai /appsrc/ai + +#RUN apt-get update --fix-missing && apt-get install -y wget + +#nodejs +#RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ +# tar -xvf ~/node.tar.xz -C /opt/ && \ +# rm ~/node.tar.xz +#ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH + +RUN groupadd --gid 1000 node \ + && useradd --uid 1000 --gid node --shell /bin/bash --create-home node + +ENV NODE_VERSION 12.22.9 + +RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) ARCH='x64';; \ + ppc64el) ARCH='ppc64le';; \ + s390x) ARCH='s390x';; \ + arm64) ARCH='arm64';; \ + armhf) ARCH='armv7l';; \ + i386) ARCH='x86';; \ + *) echo "unsupported architecture"; exit 1 ;; \ + esac \ + && set -ex \ + # libatomic1 for arm + && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + && for key in \ + 4ED778F539E3634C779C87C6D7062848A1AB005C \ + 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ + 1C050899334244A8AF75E53792EF661D867B9DFA \ + 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ + 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ + C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ + C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \ + DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ + A48C2BEE680E841632CD4E44F07496B3EB3C1762 \ + 108F52B48DB57BB0CC439B2997B01419BD92F80A \ + B9E2F5981AA6E0CD28160D9FF13993A75599653C \ + ; do \ + gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ + gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ + done \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ + && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ + && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ + && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ + && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ + && apt-mark auto '.*' > /dev/null \ + && find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + && ln -s /usr/local/bin/node /usr/local/bin/nodejs \ + # smoke tests + && node --version \ + && npm --version + +ENV YARN_VERSION 1.22.5 + +RUN set -ex \ + && savedAptMark="$(apt-mark showmanual)" \ + && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + && for key in \ + 6A010C5166006599AA17F08146C2130DFD2497F5 \ + ; do \ + gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ + gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ + done \ + && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" \ + && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc" \ + && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ + && mkdir -p /opt \ + && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ \ + && ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn \ + && ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg \ + && rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ + && apt-mark auto '.*' > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; } \ + && find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + # smoke test + && yarn --version + +ARG docker_filepath=docker/lab/files +ARG python_wheel_directory=docker/pennai-arm64-deps/lab/wheel + +RUN apt-get update --fix-missing && apt-get install -y \ + openssh-client openssh-server telnet apache2 \ + net-tools iputils-ping xz-utils \ + ngrep ca-cacert \ + build-essential cmake lsb-core cpio mesa-common-dev \ + dos2unix curl git libatlas-base-dev gfortran \ + --no-install-recommends && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Ensure specific python version +# See: https://stackoverflow.com/a/58562728/1730417 +# and: https://askubuntu.com/a/1176271/260220 +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update +RUN apt-get install -y python3.7 python3.7-dev python3.7-distutils +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 +RUN update-alternatives --set python /usr/bin/python3.7 +RUN curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python get-pip.py --force-reinstall && \ + rm get-pip.py + +# ensure local python is preferred over distribution python +ENV PATH /usr/local/bin:$PATH + +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +ENV TZ=America/New_York +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + libbluetooth-dev \ + tk-dev \ + uuid-dev \ + && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget python3.7-venv python3-numpy \ + && apt-get clean \ + && rm -rf /var/lib/apt/list/* + +RUN python3 --version +RUN python --version +RUN python -m venv /venv +ENV PATH=/venv/bin:$PATH +RUN python --version +RUN pip --version + +# setup python environment +COPY ${docker_filepath}/requirements-raspberrypi.txt /root/ +RUN mkdir /root/wheel +COPY ${python_wheel_directory}/*.whl /root/wheel/ +RUN pip3 install --no-cache-dir cython +RUN pip3 install --no-index --find-links=/root/wheel -r /root/requirements-raspberrypi.txt +### bill's surprise fork +# NOTE: For some reason, this install doesn't seem to recognize the installed numpy +# when run on Docker. As a workaround, the `wheel` directory includes a wheel +# for this install. +#RUN pip3 install --no-cache-dir git+https://github.com/lacava/surprise.git@1.0.8.3 +RUN pip3 install --no-index --find-links=/root/wheel scikit_surprise + +## Webserver + +RUN rm /etc/apache2/sites-enabled/* +COPY ${docker_filepath}/ports.conf /etc/apache2/ +RUN cp /etc/apache2/mods-available/rewrite* /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/ssl* /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/socache* /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/proxy.* /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/proxy_wstunnel.load /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/proxy_http.load /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/headers.load /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/expires.load /etc/apache2/mods-enabled + +# setup node environment +RUN npm install -g pm2 webpack --silent --progress=false + +# install lab/webapp/node_modules to an anon volume +WORKDIR /appsrc/lab/webapp +COPY lab/webapp/package.json /appsrc/lab/webapp/ +COPY lab/webapp/package-lock.json /appsrc/lab/webapp/ +RUN dos2unix /appsrc/lab/webapp/*.json +RUN npm install --silent --progress=false + +# install lab/node_modules to an anon volume +WORKDIR /appsrc/lab +COPY lab/package.json /appsrc/lab/ +COPY lab/package-lock.json /appsrc/lab/ +RUN dos2unix /appsrc/lab/*.json +RUN npm install --silent --progress=false + +RUN apt-get install -y libgfortran5 liblapack-dev + +COPY ${docker_filepath}/001-pennai.conf /etc/apache2/sites-enabled/ +COPY ${docker_filepath}/htpasswd /etc/apache2/htpasswd +COPY ${docker_filepath}/certs/* /usr/lib/ssl/private/ + +WORKDIR /root/ + +# Webserver - paiwww +COPY ${docker_filepath}/start.sh /root/ + +## Utility script, used when starting ai +COPY ${docker_filepath}/wait-for-it.sh /root/ +RUN ["chmod", "+x", "/root/wait-for-it.sh"] + + +## PennAI Lab server +COPY ${docker_filepath}/entrypoint.sh /root/ +RUN ["chmod", "+x", "/root/entrypoint.sh"] + +RUN dos2unix /root/start.sh \ + && dos2unix /root/wait-for-it.sh \ + && dos2unix /root/entrypoint.sh + +# copy knowledgebase +COPY 'data/knowledgebases/sklearn-benchmark-data-knowledgebase-r6.tsv.gz' /appsrc/data/knowledgebases/ +COPY 'data/knowledgebases/pmlb_classification_metafeatures.csv.gz' /appsrc/data/knowledgebases/ +COPY 'data/knowledgebases/pmlb_regression_results.pkl.gz' /appsrc/data/knowledgebases/ +COPY 'data/knowledgebases/pmlb_regression_metafeatures.csv.gz' /appsrc/data/knowledgebases/ + +# set version and build environment; tag.sh is sourced in entrypoint.sh +ENV BUILD_ENV='prod' +COPY .env /etc/profile.d/ +RUN cp '/etc/profile.d/.env' '/etc/profile.d/tag.sh' +RUN dos2unix /etc/profile.d/tag.sh +RUN sed -i "s/TAG=/export TAG=/g" /etc/profile.d/tag.sh + + +# Start the webserver +CMD ["/bin/bash", "/root/start.sh"] + +# EXPOSE 443 +EXPOSE 5080 +WORKDIR /appsrc/lab/ +ENTRYPOINT ["/root/entrypoint.sh"] diff --git a/docker/machine/rpi.Dockerfile b/docker/machine/rpi.Dockerfile index c38a3f358..e3d5dfcb8 100644 --- a/docker/machine/rpi.Dockerfile +++ b/docker/machine/rpi.Dockerfile @@ -1,6 +1,6 @@ FROM python:3.7-buster -ENV NODE_VERSION 12.20.0 +ENV NODE_VERSION 12.22.9 RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ && case "${dpkgArch##*-}" in \ @@ -29,6 +29,8 @@ RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ 108F52B48DB57BB0CC439B2997B01419BD92F80A \ B9E2F5981AA6E0CD28160D9FF13993A75599653C \ ; do \ + gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \ gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ diff --git a/docker/machine/rpi_production.Dockerfile b/docker/machine/rpi_production.Dockerfile new file mode 100644 index 000000000..39a61fb65 --- /dev/null +++ b/docker/machine/rpi_production.Dockerfile @@ -0,0 +1,99 @@ +FROM python:3.7-buster + +# copy src for machine +COPY machine /appsrc/machine/ +COPY config/machine_config.json /appsrc/config/machine_config.json + +ENV NODE_VERSION 12.22.9 + +RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) ARCH='x64';; \ + ppc64el) ARCH='ppc64le';; \ + s390x) ARCH='s390x';; \ + arm64) ARCH='arm64';; \ + armhf) ARCH='armv7l';; \ + i386) ARCH='x86';; \ + *) echo "unsupported architecture"; exit 1 ;; \ + esac \ + && set -ex \ + # libatomic1 for arm + && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + && for key in \ + 4ED778F539E3634C779C87C6D7062848A1AB005C \ + 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ + 1C050899334244A8AF75E53792EF661D867B9DFA \ + 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ + 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ + C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ + C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \ + DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ + A48C2BEE680E841632CD4E44F07496B3EB3C1762 \ + 108F52B48DB57BB0CC439B2997B01419BD92F80A \ + B9E2F5981AA6E0CD28160D9FF13993A75599653C \ + ; do \ + gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ + gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ + done \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ + && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ + && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ + && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ + && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ + && apt-mark auto '.*' > /dev/null \ + && find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + && ln -s /usr/local/bin/node /usr/local/bin/nodejs \ + # smoke tests + && node --version \ + && npm --version + +ARG docker_filepath=docker/machine/files +ARG python_wheel_directory=docker/pennai-arm64-deps/machine/wheel + +RUN apt-get update --fix-missing && apt-get install -y \ + openssh-client openssh-server graphviz \ + net-tools iputils-ping xz-utils \ + ngrep \ + libatlas-base-dev \ + build-essential cmake cpio mesa-common-dev \ + libglib2.0-0 libxext6 libsm6 libxrender1 dos2unix \ + --no-install-recommends && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +# Removed: lsb-core ca-cacert glibc + +# setup python environment +COPY ${docker_filepath}/requirements-raspberrypi.txt /root/ +RUN mkdir /root/wheel +COPY ${python_wheel_directory}/*.whl /root/wheel/ +RUN python -m pip install --no-index --find-links=/root/wheel -r /root/requirements-raspberrypi.txt + +# setup node environment +RUN npm install -g pm2 --silent --progress=false + +# install node_modules to an anon volume +WORKDIR /appsrc/machine + +COPY machine/package.json /appsrc/machine/ +COPY machine/package-lock.json /appsrc/machine/ +RUN dos2unix /appsrc/machine/*.json +RUN npm install --silent --progress=false + +COPY ${docker_filepath}/entrypoint.sh /root/ +COPY ${docker_filepath}/wait-for-it.sh /root/ +RUN dos2unix /root/wait-for-it.sh && dos2unix /root/entrypoint.sh +RUN ["chmod", "+x", "/root/wait-for-it.sh"] + +CMD ["/bin/bash", "/root/entrypoint.sh"] From 5df840f600087b99fc8779eee01619de385978c0 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Tue, 23 Aug 2022 14:26:15 -0700 Subject: [PATCH 26/26] Fixed node version and keyserver conflicts --- docker/lab/rpi.Dockerfile | 10 ---------- docker/machine/rpi.Dockerfile | 7 ------- 2 files changed, 17 deletions(-) diff --git a/docker/lab/rpi.Dockerfile b/docker/lab/rpi.Dockerfile index d02933e53..39d4f4896 100644 --- a/docker/lab/rpi.Dockerfile +++ b/docker/lab/rpi.Dockerfile @@ -11,11 +11,7 @@ FROM ubuntu:bionic RUN groupadd --gid 1000 node \ && useradd --uid 1000 --gid node --shell /bin/bash --create-home node -<<<<<<< HEAD ENV NODE_VERSION 12.22.9 -======= -ENV NODE_VERSION 12.20.0 ->>>>>>> 6157f68666d57632f83a685ab5521145d6c923fc RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ && case "${dpkgArch##*-}" in \ @@ -44,11 +40,8 @@ RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ 108F52B48DB57BB0CC439B2997B01419BD92F80A \ B9E2F5981AA6E0CD28160D9FF13993A75599653C \ ; do \ -<<<<<<< HEAD gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \ gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \ -======= ->>>>>>> 6157f68666d57632f83a685ab5521145d6c923fc gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ @@ -82,11 +75,8 @@ RUN set -ex \ && for key in \ 6A010C5166006599AA17F08146C2130DFD2497F5 \ ; do \ -<<<<<<< HEAD gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \ gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \ -======= ->>>>>>> 6157f68666d57632f83a685ab5521145d6c923fc gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ diff --git a/docker/machine/rpi.Dockerfile b/docker/machine/rpi.Dockerfile index 3a81e9b0d..e3d5dfcb8 100644 --- a/docker/machine/rpi.Dockerfile +++ b/docker/machine/rpi.Dockerfile @@ -1,10 +1,6 @@ FROM python:3.7-buster -<<<<<<< HEAD ENV NODE_VERSION 12.22.9 -======= -ENV NODE_VERSION 12.20.0 ->>>>>>> 6157f68666d57632f83a685ab5521145d6c923fc RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ && case "${dpkgArch##*-}" in \ @@ -33,11 +29,8 @@ RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ 108F52B48DB57BB0CC439B2997B01419BD92F80A \ B9E2F5981AA6E0CD28160D9FF13993A75599653C \ ; do \ -<<<<<<< HEAD gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \ gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \ -======= ->>>>>>> 6157f68666d57632f83a685ab5521145d6c923fc gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \