diff --git a/.gitignore b/.gitignore index f3c28571bdf..d4b817a1ae5 100644 --- a/.gitignore +++ b/.gitignore @@ -150,3 +150,7 @@ venv/* # resource optimization scripts/resource/output *.pem + +# docker tests +docker/mountFolder/*.bin +docker/mountFolder/*.bin.mtd diff --git a/docker/build.sh b/docker/build.sh index 2898effdc22..eddc29d7108 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -26,7 +26,7 @@ # docker image build -f docker/sysds.Dockerfile -t apache/systemds:latest . # The second build is for testing systemds. This image installs the R dependencies needed to run the tests. -docker image build -f docker/testsysds.Dockerfile -t apache/systemds:testing-latest . +# docker image build -f docker/testsysds.Dockerfile -t apache/systemds:testing-latest . # The third build is python docker for systemds. # docker image build -f docker/pythonsysds.Dockerfile -t apache/systemds:python-nightly . diff --git a/docker/mountFolder/main.dml b/docker/mountFolder/main.dml index 07c4862e622..0efbb6e948a 100644 --- a/docker/mountFolder/main.dml +++ b/docker/mountFolder/main.dml @@ -19,4 +19,15 @@ # #------------------------------------------------------------- -print("Hello, World!"); \ No newline at end of file +# kmeans_test.dml +# Simple KMeans clustering test + +# Arguments +X = matrix("1.0 2.0 1.5 1.8 5.0 8.0 8.0 8.0 1.0 0.6 9.0 11.0", rows = 6, cols = 2) + +# Run KMeans +[centroids, assignments] = kmeans(X=X, k=2, max_iter=100, eps=0.01) + +print(centroids) +print(assignments) +write(centroids, "centroids.bin") diff --git a/docker/sysds.Dockerfile b/docker/sysds.Dockerfile index cc6ef605d53..2238aa878a8 100644 --- a/docker/sysds.Dockerfile +++ b/docker/sysds.Dockerfile @@ -19,18 +19,16 @@ # #------------------------------------------------------------- -FROM ubuntu:24.04@sha256:6015f66923d7afbc53558d7ccffd325d43b4e249f41a6e93eef074c9505d2233 +FROM alpine:3.20@sha256:de4fe7064d8f98419ea6b49190df1abbf43450c1702eeb864fe9ced453c1cc5f AS compile-image WORKDIR /usr/src/ # Do basic updates on the image -RUN apt-get update -qq \ - && apt-get upgrade -y \ - && apt-get install -y --no-install-recommends \ +RUN apk add --no-cache \ wget \ git \ ca-certificates \ - && apt-get clean + bash # Set environment variables # Maven @@ -43,11 +41,11 @@ ENV SYSTEMDS_ROOT=/usr/src/systemds ENV PATH=$SYSTEMDS_ROOT/bin:$PATH ENV SYSDS_QUIET=1 -# Download Java and Mvn +# Download Mvn and JDK RUN mkdir -p /usr/lib/jvm \ && wget -qO- \ - https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.15%2B6/OpenJDK17U-jdk_x64_linux_hotspot_17.0.15_6.tar.gz | tar xzf - \ - && mv jdk-17.0.15+6 /usr/lib/jvm/jdk-17.0.15+6 \ + https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.15%2B6/OpenJDK17U-jdk_x64_alpine-linux_hotspot_17.0.15_6.tar.gz | tar xzf - \ + && mv jdk-17.0.15+6 $JAVA_HOME \ && wget -qO- \ http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz | tar xzf - \ && mv apache-maven-$MAVEN_VERSION /usr/lib/mvn @@ -57,8 +55,11 @@ RUN git clone --depth 1 https://github.com/apache/systemds.git systemds && \ cd /usr/src/systemds/ && \ mvn --no-transfer-progress clean package -P distribution +COPY docker/mountFolder/main.dml /input/main.dml + # Remove all unnecessary files from the Image -RUN rm -rf .git && \ +RUN cd /usr/src/systemds/ && \ + rm -rf .git && \ rm -rf .github && \ rm -rf target/javadoc** && \ rm -rf target/apidocs** && \ @@ -71,9 +72,55 @@ RUN rm -rf .git && \ rm -rf /usr/lib/mvn && \ rm -rf CONTRIBUTING.md && \ rm -rf pom.xml && \ - rm -rf ~/.m2 + rm -rf ~/.m2 && \ + rm -rf docker && \ + rm -rf .mvn +FROM alpine:3.20@sha256:de4fe7064d8f98419ea6b49190df1abbf43450c1702eeb864fe9ced453c1cc5f -COPY docker/mountFolder/main.dml /input/main.dml +RUN apk add --no-cache bash \ + snappy \ + lz4 \ + zlib + +ENV JAVA_HOME=/usr/lib/jvm/jdk-17.0.15+6 +ENV PATH=$JAVA_HOME/bin:$PATH +ENV SYSTEMDS_ROOT=/systemds +ENV PATH=$SYSTEMDS_ROOT/bin:$PATH +ENV SYSDS_QUIET=1 + +ENV HADOOP_VERSION=3.3.6 +ENV HADOOP_HOME=/opt/hadoop +ENV LD_LIBRARY_PATH=/opt/hadoop/lib/native +ENV HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native" +ENV GLIBC_VERSION=2.35-r1 + +RUN wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub \ + && wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk \ + && apk add glibc-${GLIBC_VERSION}.apk \ + && rm glibc-${GLIBC_VERSION}.apk + +RUN mkdir -p /usr/lib/jvm \ + && wget -qO- \ + https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.15%2B6/OpenJDK17U-jre_x64_alpine-linux_hotspot_17.0.15_6.tar.gz | tar xzf - \ + && mv jdk-17.0.15+6-jre $JAVA_HOME + +RUN mkdir -p $HADOOP_HOME/lib/native \ + && wget -q https://downloads.apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz && \ + tar --strip-components=2 -xzf hadoop-${HADOOP_VERSION}.tar.gz \ + hadoop-${HADOOP_VERSION}/lib/native && \ + mv native/libhadoop.so.1.0.0 /opt/hadoop/lib/native && \ + mv native/libhadoop.so /opt/hadoop/lib/native && \ + rm hadoop-${HADOOP_VERSION}.tar.gz && \ + rm -rf native + +COPY --from=compile-image /usr/src/systemds /systemds +COPY --from=compile-image /input/main.dml /input/main.dml + +WORKDIR /input + +RUN addgroup -S default && adduser -S systemds -G default +USER systemds -CMD ["systemds", "/input/main.dml"] +ENTRYPOINT ["systemds"] +CMD ["main.dml"] diff --git a/docker/testsysds.Dockerfile b/docker/testsysds.Dockerfile index 2f63dace7fa..fc6eb1491e1 100644 --- a/docker/testsysds.Dockerfile +++ b/docker/testsysds.Dockerfile @@ -18,8 +18,29 @@ # under the License. # #------------------------------------------------------------- +# Stage 1: Build SEAL +FROM debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973 AS seal-build -FROM ubuntu:24.04@sha256:6015f66923d7afbc53558d7ccffd325d43b4e249f41a6e93eef074c9505d2233 +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + wget \ + tar \ + git \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /seal + +# Install SEAL +RUN wget -qO- https://github.com/microsoft/SEAL/archive/refs/tags/v3.7.0.tar.gz | tar xzf - \ + && cd SEAL-3.7.0 \ + && cmake -S . -B build -DBUILD_SHARED_LIBS=ON \ + && cmake --build build \ + && cmake --install build --prefix /seal-install + +# Stage 2: Final image with R, JDK, Maven, SEAL +FROM debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973 WORKDIR /usr/src/ ENV MAVEN_VERSION=3.9.9 @@ -28,61 +49,37 @@ ENV MAVEN_HOME=/usr/lib/mvn ENV JAVA_HOME=/usr/lib/jvm/jdk-17.0.15+6 ENV PATH=$JAVA_HOME/bin:$MAVEN_HOME/bin:$PATH -ENV LANGUAGE=en_US:en -ENV LC_ALL=en_US.UTF-8 -ENV LANG=en_US.UTF-8 -ENV LD_LIBRARY_PATH=/usr/local/lib/ - -RUN apt-get update -qq \ - && apt-get upgrade -y \ - && apt-get install -y --no-install-recommends \ - libcurl4-openssl-dev \ - libxml2-dev \ - locales \ - software-properties-common \ - dirmngr \ - gnupg \ - apt-transport-https \ - wget \ - ca-certificates \ - git \ - cmake \ - patchelf \ - && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 \ - && add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" \ - && apt-get update -qq \ - && apt-get upgrade -y \ - && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \ - && locale-gen en_US.utf8 \ - && /usr/sbin/update-locale LANG=en_US.UTF-8 \ +RUN apt-get update && apt-get install -y --no-install-recommends \ + r-base \ + wget \ + cmake \ + r-base-dev \ + libcurl4-openssl-dev \ + libssl-dev \ + libxml2-dev \ + ca-certificates \ + patchelf \ + git \ + libssl-dev \ + r-base-dev \ + r-base-core \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ && mkdir -p /usr/lib/jvm \ && wget -qO- \ https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.15%2B6/OpenJDK17U-jdk_x64_linux_hotspot_17.0.15_6.tar.gz | tar xzf - \ - && mv jdk-17.0.15+6 /usr/lib/jvm/jdk-17.0.15+6 \ + && mv jdk-17.0.15+6 $JAVA_HOME \ && wget -qO- \ - http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz | tar xzf - \ + http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz | tar xzf - \ && mv apache-maven-$MAVEN_VERSION /usr/lib/mvn -# Install R Base -RUN apt-get install -y --no-install-recommends \ - libssl-dev \ - r-base \ - r-base-dev \ - r-base-core - # Install R packages -COPY ./src/test/scripts/installDependencies.R installDependencies.R +COPY ./src/test/scripts/installDependencies.R installDependencies.R RUN Rscript installDependencies.R \ - && rm -rf installDependencies.R \ - && rm -rf /var/lib/apt/lists/* + && rm -f installDependencies.R -# Install SEAL -RUN wget -qO- https://github.com/microsoft/SEAL/archive/refs/tags/v3.7.0.tar.gz | tar xzf - \ - && cd SEAL-3.7.0 \ - && cmake -S . -B build -DBUILD_SHARED_LIBS=ON \ - && cmake --build build \ - && cmake --install build +# Copy SEAL +COPY --from=seal-build /seal-install /usr/local # Finally copy the entrypoint script # This is last to enable quick updates to the script after initial local build.