diff --git a/.env b/.env index 58fba5a77..87df3c4cc 100644 --- a/.env +++ b/.env @@ -1,3 +1,6 @@ #!/bin/sh # Docker production build environment +# ARCH refers to the architecture of the development machine. +# If you are on arm64, set this to arm64, otherwise use x64 TAG=0.20.a0 +ARCH=x64 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 950c5f852..6307f1aa5 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,4 @@ MANIFEST .xz .exe +run*Test.sh \ No newline at end of file diff --git a/docker-compose-int-test.yml b/docker-compose-int-test.yml index 380a0e168..2057b2b7d 100644 --- a/docker-compose-int-test.yml +++ b/docker-compose-int-test.yml @@ -22,6 +22,8 @@ services: build: context: . dockerfile: docker/lab/Dockerfile + args: + - ARCH=${ARCH} tty: true stdin_open: true volumes: diff --git a/docker-compose-multiarch.yml b/docker-compose-multiarch.yml new file mode 100644 index 000000000..6f20a77ff --- /dev/null +++ b/docker-compose-multiarch.yml @@ -0,0 +1,57 @@ +version: '3' + +services: + lab: + build: + context: . + dockerfile: docker/lab/Dockerfile + x-bake: + platforms: + - linux/arm64 + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/lab/webapp/node_modules" + - "/appsrc/lab/node_modules" + ports: + - "5080:5080" + env_file: + - ./config/common.env + - ./config/ai.env + depends_on: + - dbmongo + + machine: + build: + context: . + dockerfile: docker/machine/Dockerfile + x-bake: + platforms: + - linux/arm64 + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/machine/node_modules" + ports: + - "5081:5081" + env_file: ./config/common.env + environment: + - MACHINE_PORT=5081 + depends_on: + - lab + - dbmongo + + dbmongo: + build: + context: . + dockerfile: docker/dbmongo/Dockerfile + x-bake: + platforms: + - linux/arm64 + tty: true + stdin_open: true + ports: + - "27017:27017" + env_file: ./config/common.env diff --git a/docker-compose-unit-test.yml b/docker-compose-unit-test.yml index e4f85cae6..4eea12503 100644 --- a/docker-compose-unit-test.yml +++ b/docker-compose-unit-test.yml @@ -5,6 +5,8 @@ services: build: context: . dockerfile: tests/unit/Dockerfile + args: + - ARCH=${ARCH} working_dir: /appsrc command: bash -c "dos2unix tests/unit/unit_test_runner.sh && export PENNAI_UNIT_TESTS_JS_ONLY=$PENNAI_UNIT_TESTS_JS_ONLY && export PENNAI_UNIT_TESTS_PY_ONLY=$PENNAI_UNIT_TESTS_PY_ONLY && sh tests/unit/unit_test_runner.sh" tty: true diff --git a/docker-compose.yml b/docker-compose.yml index 6f20a77ff..fbace78e4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,9 +5,8 @@ services: build: context: . dockerfile: docker/lab/Dockerfile - x-bake: - platforms: - - linux/arm64 + args: + - ARCH=${ARCH} tty: true stdin_open: true volumes: @@ -26,9 +25,6 @@ services: build: context: . dockerfile: docker/machine/Dockerfile - x-bake: - platforms: - - linux/arm64 tty: true stdin_open: true volumes: @@ -47,9 +43,6 @@ services: build: context: . dockerfile: docker/dbmongo/Dockerfile - x-bake: - platforms: - - linux/arm64 tty: true stdin_open: true ports: diff --git a/docker/lab/Dockerfile b/docker/lab/Dockerfile index 30e6d0b07..bc6063528 100644 --- a/docker/lab/Dockerfile +++ b/docker/lab/Dockerfile @@ -1,11 +1,7 @@ -# FROM python:3.7.11-stretch FROM python:3.7.16-slim-buster -#nodejs -# RUN wget --quiet https://nodejs.org/dist/v12.22.9/node-v12.22.9-linux-x64.tar.xz -O ~/node.tar.xz && \ -# tar -xvf ~/node.tar.xz -C /opt/ && \ -# rm ~/node.tar.xz -# ENV PATH /opt/node-v12.22.9-linux-x64/bin:$PATH +ARG ARCH +ARG wheel_directory=docker/pywheels RUN apt-get update --fix-missing && \ apt-get install -y wget xz-utils git \ @@ -48,8 +44,13 @@ ARG docker_filepath=docker/lab/files # rm -rf /var/lib/apt/lists/* # setup python environment +RUN mkdir /root/wheel +COPY ${wheel_directory}/*.whl /root/wheel/ COPY ${docker_filepath}/requirements.txt /root/ -RUN pip install --no-cache-dir -r /root/requirements.txt +COPY ${docker_filepath}/requirements-surprise-${ARCH}.txt /root/ +RUN pip install --no-cache-dir --find-links=/root/wheel -r /root/requirements.txt +RUN pip install --no-cache-dir --find-links=/root/wheel -r /root/requirements-surprise-${ARCH}.txt +RUN rm -rf /root/wheel ## Webserver diff --git a/docker/lab/files/requirements-surprise-arm64.txt b/docker/lab/files/requirements-surprise-arm64.txt new file mode 100644 index 000000000..429a4ae52 --- /dev/null +++ b/docker/lab/files/requirements-surprise-arm64.txt @@ -0,0 +1 @@ +scikit-surprise==1.0.8.2 \ No newline at end of file diff --git a/docker/lab/files/requirements-surprise-x64.txt b/docker/lab/files/requirements-surprise-x64.txt new file mode 100644 index 000000000..87ecfc9ab --- /dev/null +++ b/docker/lab/files/requirements-surprise-x64.txt @@ -0,0 +1 @@ +-e git+https://github.com/lacava/surprise.git@1.1.1.1#egg=scikit-surprise \ No newline at end of file diff --git a/docker/lab/files/requirements.txt b/docker/lab/files/requirements.txt index a918995a9..ef2e673bc 100644 --- a/docker/lab/files/requirements.txt +++ b/docker/lab/files/requirements.txt @@ -7,5 +7,4 @@ cython==0.29.32 xgboost==1.6.2 simplejson==3.17.2 requests==2.25.1 -joblib==1.2.0 --e git+https://github.com/lacava/surprise.git@1.1.1.1#egg=scikit-surprise \ No newline at end of file +joblib==1.2.0 \ No newline at end of file diff --git a/docker/machine/Dockerfile b/docker/machine/Dockerfile index f9fe32753..b73d3cf28 100644 --- a/docker/machine/Dockerfile +++ b/docker/machine/Dockerfile @@ -1,12 +1,6 @@ -# FROM python:3.7.11-stretch FROM python:3.7.16-slim-buster -#nodejs -# RUN wget --quiet https://nodejs.org/dist/v12.22.9/node-v12.22.9-linux-x64.tar.xz -O ~/node.tar.xz && \ -# tar -xvf ~/node.tar.xz -C /opt/ && \ -# rm ~/node.tar.xz -# ENV PATH /opt/node-v12.22.9-linux-x64/bin:$PATH - +ARG wheel_directory=docker/pywheels ARG docker_filepath=docker/machine/files RUN apt-get update --fix-missing && apt-get install -y \ @@ -36,11 +30,11 @@ RUN apt-get update --fix-missing && apt-get install -y \ apt-get clean # setup python environment +RUN mkdir /root/wheel +COPY ${wheel_directory}/*.whl /root/wheel/ COPY ${docker_filepath}/requirements.txt /root/ -RUN pip install --no-cache-dir -r /root/requirements.txt -COPY ${docker_filepath}/requirements_shap.txt /root/ -RUN pip install --no-cache-dir -r /root/requirements_shap.txt - +RUN pip install --no-cache-dir --find-links=/root/wheel -r /root/requirements.txt +RUN rm -rf /root/wheel # setup node environment RUN npm install -g pm2 --silent --progress=false diff --git a/docker/machine/files/requirements.txt b/docker/machine/files/requirements.txt index 90f23bd0a..1b97e440b 100644 --- a/docker/machine/files/requirements.txt +++ b/docker/machine/files/requirements.txt @@ -8,4 +8,5 @@ mlxtend==0.16.0 pydot==1.4.2 requests==2.25.1 xgboost==1.6.2 -joblib==1.2.0 \ No newline at end of file +joblib==1.2.0 +shap==0.37.0 \ No newline at end of file diff --git a/docker/pywheels/numpy-1.19.5-cp39-cp39-macosx_12_0_arm64.whl b/docker/pywheels/numpy-1.19.5-cp39-cp39-macosx_12_0_arm64.whl new file mode 100644 index 000000000..836e7a4d4 Binary files /dev/null and b/docker/pywheels/numpy-1.19.5-cp39-cp39-macosx_12_0_arm64.whl differ diff --git a/docker/pywheels/pandas-1.0.5-cp39-cp39-macosx_11_0_arm64.whl b/docker/pywheels/pandas-1.0.5-cp39-cp39-macosx_11_0_arm64.whl new file mode 100644 index 000000000..752494e40 Binary files /dev/null and b/docker/pywheels/pandas-1.0.5-cp39-cp39-macosx_11_0_arm64.whl differ diff --git a/docker/pywheels/scikit_learn-0.23.2-cp39-cp39-macosx_12_0_arm64.whl b/docker/pywheels/scikit_learn-0.23.2-cp39-cp39-macosx_12_0_arm64.whl new file mode 100644 index 000000000..b65b55f7b Binary files /dev/null and b/docker/pywheels/scikit_learn-0.23.2-cp39-cp39-macosx_12_0_arm64.whl differ diff --git a/docker/pywheels/scikit_surprise-1.0.8.2-cp37-cp37m-linux_aarch64.whl b/docker/pywheels/scikit_surprise-1.0.8.2-cp37-cp37m-linux_aarch64.whl new file mode 100644 index 000000000..a914e338a Binary files /dev/null and b/docker/pywheels/scikit_surprise-1.0.8.2-cp37-cp37m-linux_aarch64.whl differ diff --git a/docs/guides/userGuide.md b/docs/guides/userGuide.md index d2fc6f409..4ea364bee 100644 --- a/docs/guides/userGuide.md +++ b/docs/guides/userGuide.md @@ -34,12 +34,7 @@ We have built a custom Raspberry Pi OS Image containing Aliro (configured to be 7. Insert the MicroSD Card into your Raspberry Pi 400 and start it up. 8. When the Operating System has finished starting up, double-click the Aliro-Ed Icon on the Destop or launch the Web Browser. -Now that Aliro is up and running, you are ready to run experiments, Aliro Ed comes preloaded with the following datasets: - - [BNGBreastTumor.tsv](https://github.com/EpistasisLab/pmlb/tree/master/datasets/1201_BNG_breastTumor) - - [ESL.tsv](https://github.com/EpistasisLab/pmlb/tree/master/datasets/1027_ESL) - - [Iris.tsv](https://github.com/EpistasisLab/pmlb/tree/master/datasets/iris) - - [Titanic.tsv](https://github.com/EpistasisLab/pmlb/tree/master/datasets/titanic) - - Other datasets can be downloaded from the [Penn Machine Learning Benchmarks](https://github.com/EpistasisLab/pmlb) +Now that Aliro is up and running, you are ready to run experiments, Aliro Ed comes preloaded with some data sets for you to experiment with. Other datasets can be downloaded from the [Penn Machine Learning Benchmarks](https://github.com/EpistasisLab/pmlb) You may also load your own datasets, please see the following sections below for further instructions: - [Adding Data Sets](#adding-datasets) diff --git a/runIntTest.sh b/runIntTest.sh index cf3154f3d..e0a29ea46 100755 --- a/runIntTest.sh +++ b/runIntTest.sh @@ -1,4 +1,4 @@ #!/usr/bin/bash docker system prune -a -f --volumes -docker-compose -f docker-compose-int-test.yml up --abort-on-container-exit -V +docker compose -f docker-compose-int-test.yml up --abort-on-container-exit -V diff --git a/runUnitTest.sh b/runUnitTest.sh index 1a5365329..918847987 100755 --- a/runUnitTest.sh +++ b/runUnitTest.sh @@ -1,4 +1,4 @@ #!/usr/bin/bash docker system prune -a -f --volumes -docker-compose -f docker-compose-unit-test.yml up --abort-on-container-exit -V +docker compose -f docker-compose-unit-test.yml up --abort-on-container-exit -V diff --git a/tests/unit/Dockerfile b/tests/unit/Dockerfile index a71f2427f..902f9fbb6 100644 --- a/tests/unit/Dockerfile +++ b/tests/unit/Dockerfile @@ -1,5 +1,8 @@ FROM python:3.7.16-buster +ARG ARCH +ARG wheel_directory=docker/pywheels + #nodejs RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ && case "${dpkgArch##*-}" in \ @@ -36,13 +39,17 @@ COPY /tests/unit/requirements.txt /root/test_requirements.txt RUN pip install --no-cache-dir -r /root/test_requirements.txt ## machine and lab container +COPY ${wheel_directory}/*.whl /root/wheel/ COPY /docker/lab/files/requirements.txt /root/lab_requirements.txt -RUN pip install --no-cache-dir -r /root/lab_requirements.txt +RUN pip install --no-cache-dir --find-links=/root/wheel -r /root/lab_requirements.txt +COPY /docker/lab/files/requirements-surprise-${ARCH}.txt /root/surprise_requirements.txt +RUN pip install --no-cache-dir --find-links=/root/wheel -r /root/surprise_requirements.txt COPY /docker/machine/files/requirements.txt /root/mach_requirements.txt -RUN pip install --no-cache-dir -r /root/mach_requirements.txt -COPY /docker/machine/files/requirements_shap.txt /root/mach_requirements_shap.txt -RUN pip install --no-cache-dir -r /root/mach_requirements_shap.txt +RUN pip install --no-cache-dir --find-links=/root/wheel -r /root/mach_requirements.txt +# COPY /docker/machine/files/requirements_shap.txt /root/mach_requirements_shap.txt +# RUN pip install --no-cache-dir -r /root/mach_requirements_shap.txt +RUN rm -rf /root/wheel # install lab/node_modules to an anon volume WORKDIR /appsrc/lab