#1026 create test cuda ml flavor (#483)

related to exasol/script-languages-release#1026

exaudfclient/base/exaudfclient.template.sh

@@ -4,4 +4,5 @@ SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
 echo "Changing to script directory  $SCRIPT_DIR"
 cd "$SCRIPT_DIR" || return 1
 export LIBEXAUDFLIB_PATH="$SCRIPT_DIR/base/libexaudflib_complete.so"
-export LIBPYEXADATAFRAME_DIR="$SCRIPT_DIR/python/python3"
+export LIBPYEXADATAFRAME_DIR="$SCRIPT_DIR/base/python/python3"
+export LD_LIBRARY_PATH="/opt/conda/cuda-compat/:$LD_LIBRARY_PATH" #Temporary hack for the Cuda ML flavor(s)

flavors/template-Exasol-8-python-3.10-cuda-conda/FLAVOR_DESCRIPTION.md

@@ -4,6 +4,8 @@
 
 **Remarks:** The packages are grouped by there main usage and their type.
 
+- CondaDeps dependencies
+  - [Ubuntu packages](flavor_base/conda_deps/packages/apt_get_packages)
 - UDFclient dependencies
   - [Conda Packages](flavor_base/udfclient_deps/packages/conda_packages)
   - [Conda Channels](flavor_base/udfclient_deps/packages/conda_channels)

flavors/template-Exasol-8-python-3.10-cuda-conda/flavor_base/base_test_build_run/Dockerfile

@@ -11,7 +11,6 @@ WORKDIR /exaudfclient/
 ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
 ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
 ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
-# We need to build with optimize here, because the debug build fails to run with a runtime linking error regarding protobuf
 RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries
 RUN cp -r -L bazel-bin/* /exaudf
 

flavors/test-Exasol-8-cuda-ml/FLAVOR_DESCRIPTION.md

@@ -0,0 +1,18 @@
+# Details for template-Exasol-8-python-3.10-cuda-conda flavor
+
+## Packages
+
+**Remarks:** The packages are grouped by there main usage and their type.
+
+- CondaDeps dependencies
+  - [Ubuntu packages](flavor_base/conda_deps/packages/apt_get_packages)
+- UDFclient dependencies
+  - [Conda Packages](flavor_base/udfclient_deps/packages/conda_packages)
+  - [Conda Channels](flavor_base/udfclient_deps/packages/conda_channels)
+- Language dependencies
+  - [Conda Packages](flavor_base/language_deps/packages/conda_packages)
+  - [Conda Channels](flavor_base/language_deps/packages/conda_channels)
+- Flavor packages
+  - [Conda Packages](flavor_base/flavor_base_deps/packages/conda_packages)
+  - [Conda Channels](flavor_base/flavor_base_deps/packages/conda_channels)
+  - [Python3 pip packages](flavor_base/flavor_base_deps/packages/python3_pip_packages)

flavors/test-Exasol-8-cuda-ml/flavor_base/base_test_build_run/Dockerfile

@@ -0,0 +1,25 @@
+FROM {{base_test_deps}}
+
+RUN mkdir /conf /buckets
+
+RUN ldconfig
+
+RUN mkdir /exaudfclient /exaudf
+COPY /exaudfclient/ /exaudfclient/
+
+WORKDIR /exaudfclient/
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
+ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
+ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
+RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries
+RUN cp -r -L bazel-bin/* /exaudf
+
+WORKDIR /exaudfclient/base
+RUN ./test_udfclient.sh /exaudf/exaudfclient
+RUN ./test_udfclient.sh /exaudf/exaudfclient_static
+
+WORKDIR /
+RUN mkdir /exasol_emulator
+COPY emulator/ /exasol_emulator
+COPY /exaudfclient/base/exaudflib/zmqcontainer.proto /exasol_emulator
+RUN cd /exasol_emulator && protoc zmqcontainer.proto --python_out=.

flavors/test-Exasol-8-cuda-ml/flavor_base/base_test_deps/Dockerfile

@@ -0,0 +1,6 @@
+FROM {{build_deps}}
+
+RUN mkdir -p /build_info/packages
+COPY base_test_deps/packages /build_info/packages/base_test_deps
+
+RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/base_test_deps/apt_get_packages --with-versions

flavors/test-Exasol-8-cuda-ml/flavor_base/base_test_deps/packages/apt_get_packages

@@ -0,0 +1,6 @@
+gdb|12.1-0ubuntu1~22.04.2
+valgrind|1:3.18.1-1ubuntu2
+gdbserver|12.1-0ubuntu1~22.04.2
+binutils|2.38-4ubuntu2.6
+patchelf|0.14.3-1
+strace|5.16-0ubuntu3

flavors/test-Exasol-8-cuda-ml/flavor_base/build_deps/Dockerfile

@@ -0,0 +1,13 @@
+FROM {{language_deps}}
+
+RUN mkdir -p /build_info/packages
+COPY build_deps/packages /build_info/packages/build_deps
+
+RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/build_deps/conda_packages --channel-file /build_info/packages/build_deps/conda_channels --with-versions --conda-binary "$MAMBA_ROOT_PREFIX/bin/mamba"
+
+RUN curl -L -o swig-2.0.4.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/swig-2.0.4.tar.gz && \
+    tar zxf swig-2.0.4.tar.gz && \
+    (cd swig-2.0.4 && ./configure --prefix=/usr && make && make install) && \
+    rm -rf swig-2.0.4 swig-2.0.4.tar.gz
+
+ENV PROTOBUF_BIN=$MAMBA_ROOT_PREFIX/bin/protoc

flavors/test-Exasol-8-cuda-ml/flavor_base/build_deps/packages/conda_channels

@@ -0,0 +1 @@
+conda-forge

flavors/test-Exasol-8-cuda-ml/flavor_base/build_deps/packages/conda_packages

@@ -0,0 +1,7 @@
+protobuf|5.28.2
+pcre|8.45
+bazel|7.3.2
+make|4.3
+chrpath|0.16
+gxx_linux-64|12.4.0
+gcc_linux-64|12.4.0

flavors/test-Exasol-8-cuda-ml/flavor_base/build_run/Dockerfile

@@ -0,0 +1,24 @@
+FROM {{build_deps}}
+
+RUN mkdir /exaudfclient /exaudf
+COPY exaudfclient/ /exaudfclient/
+
+WORKDIR /exaudfclient/
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
+ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
+ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
+RUN ./build.sh --config no-tty --config optimize --config python --config slow-wrapper
+RUN cp -r -L bazel-bin/* /exaudf
+
+WORKDIR /exaudfclient/base
+RUN ./test_udfclient.sh /exaudf/exaudfclient
+
+WORKDIR /exaudf
+
+RUN rm -r /exaudfclient
+
+COPY --from={{build_deps}} /build_info /build_info
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+
+RUN mkdir /conf /buckets

flavors/test-Exasol-8-cuda-ml/flavor_base/build_steps.py

@@ -0,0 +1,153 @@
+from typing import Dict
+
+from exasol.slc.internal.tasks.build.docker_flavor_image_task import (DockerFlavorAnalyzeImageTask)
+
+
+class AnalyzeCondaDeps(DockerFlavorAnalyzeImageTask):
+
+    def get_build_step(self) -> str:
+        return "conda_deps"
+
+    def get_additional_build_directories_mapping(self) -> Dict[str, str]:
+        return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+class AnalyzeUDFClientDeps(DockerFlavorAnalyzeImageTask):
+
+    def get_build_step(self) -> str:
+        return "udfclient_deps"
+
+    def get_additional_build_directories_mapping(self) -> Dict[str, str]:
+        return {"scripts": "ext/scripts"}
+
+    def requires_tasks(self):
+        return {"conda_deps": AnalyzeCondaDeps}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeLanguageDeps(DockerFlavorAnalyzeImageTask):
+
+    def get_build_step(self) -> str:
+        return "language_deps"
+
+    def get_additional_build_directories_mapping(self) -> Dict[str, str]:
+        return {"scripts": "ext/scripts"}
+
+    def requires_tasks(self):
+        return {"udfclient_deps": AnalyzeUDFClientDeps}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeBuildDeps(DockerFlavorAnalyzeImageTask):
+
+    def get_build_step(self) -> str:
+        return "build_deps"
+
+    def get_additional_build_directories_mapping(self) -> Dict[str, str]:
+        return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"}
+
+    def requires_tasks(self):
+        return {"language_deps": AnalyzeLanguageDeps}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeBuildRun(DockerFlavorAnalyzeImageTask):
+
+    def get_build_step(self) -> str:
+        return "build_run"
+
+    def requires_tasks(self):
+        return {"build_deps": AnalyzeBuildDeps}
+
+    def get_additional_build_directories_mapping(self) -> Dict[str, str]:
+        return {"exaudfclient": "exaudfclient"}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeBaseTestDeps(DockerFlavorAnalyzeImageTask):
+
+    def get_build_step(self) -> str:
+        return "base_test_deps"
+
+    def requires_tasks(self):
+        return {"build_deps": AnalyzeBuildDeps}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeBaseTestBuildRun(DockerFlavorAnalyzeImageTask):
+
+    def get_build_step(self) -> str:
+        return "base_test_build_run"
+
+    def requires_tasks(self):
+        return {"base_test_deps": AnalyzeBaseTestDeps}
+
+    def get_additional_build_directories_mapping(self) -> Dict[str, str]:
+        return {"exaudfclient": "exaudfclient", "emulator": "emulator"}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeFlavorBaseDeps(DockerFlavorAnalyzeImageTask):
+
+    def get_build_step(self) -> str:
+        return "flavor_base_deps"
+
+    def requires_tasks(self):
+        return {"language_deps": AnalyzeLanguageDeps}
+
+    def get_additional_build_directories_mapping(self):
+        return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeFlavorTestBuildRun(DockerFlavorAnalyzeImageTask):
+
+    def get_build_step(self) -> str:
+        return "flavor_test_build_run"
+
+    def requires_tasks(self):
+        return {"base_test_build_run": AnalyzeBaseTestBuildRun,
+                "flavor_base_deps": AnalyzeFlavorBaseDeps}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeRelease(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "release"
+
+    def requires_tasks(self):
+        return {"build_run": AnalyzeBuildRun,
+                "language_deps": AnalyzeLanguageDeps,
+                "flavor_base_deps": AnalyzeFlavorBaseDeps}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class SecurityScan(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "security_scan"
+
+    def requires_tasks(self):
+        return {"release": AnalyzeRelease}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"

flavors/test-Exasol-8-cuda-ml/flavor_base/conda_deps/Dockerfile

@@ -0,0 +1,33 @@
+FROM ubuntu:22.04
+ENV DEBIAN_FRONTEND=noninteractive
+
+ENV ARCHIVE_UBUNTU_PREFIX=""
+RUN sed --in-place --regexp-extended "s/(\/\/)(archive\.ubuntu)/\1$ARCHIVE_UBUNTU_PREFIX\2/" /etc/apt/sources.list
+
+COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc
+
+COPY scripts /scripts
+
+RUN mkdir -p /build_info/packages
+COPY conda_deps/packages /build_info/packages/conda_deps
+RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/conda_deps/apt_get_packages --with-versions
+
+ENV ENV_NAME="base"
+ENV MAMBA_ROOT_PREFIX="/opt/conda"
+ENV MAMBA_EXE="/bin/micromamba"
+ENV MAMBA_DOCKERFILE_ACTIVATE=1
+
+COPY conda_deps/_activate_current_env.sh /usr/local/bin/_activate_current_env.sh
+
+RUN addgroup --gid 1000 exasolution
+RUN adduser --disabled-login --uid 1000 --gid 1000 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone"
+RUN addgroup --gid 500 exausers
+RUN adduser --disabled-login --uid 500 --gid 500 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone"
+
+SHELL ["/bin/bash", "-l", "-c"]
+ENV MICROMAMBA_VERSION=0.24.0
+RUN /scripts/install_scripts/install_micromamba.sh "$MICROMAMBA_VERSION"
+
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8

flavors/test-Exasol-8-cuda-ml/flavor_base/conda_deps/_activate_current_env.sh

@@ -0,0 +1,15 @@
+# This script should never be called directly, only sourced:
+
+#     source _activate_current_env.sh
+
+if [[ "${MAMBA_SKIP_ACTIVATE}" == "1" ]]; then
+  return
+fi
+
+# Initialize the current shell
+eval "$("${MAMBA_EXE}" shell hook --shell=bash)"
+
+# For robustness, try all possible activate commands.
+conda activate "${ENV_NAME}" 2>/dev/null \
+  || mamba activate "${ENV_NAME}" 2>/dev/null \
+  || micromamba activate "${ENV_NAME}"

flavors/test-Exasol-8-cuda-ml/flavor_base/conda_deps/packages/apt_get_packages

@@ -0,0 +1,5 @@
+coreutils|8.32-4.1ubuntu1.2
+locales|2.35-0ubuntu3.8
+curl|7.81.0-1ubuntu1.19
+ca-certificates|20240203~22.04.1
+bzip2|1.0.8-5build1

flavors/test-Exasol-8-cuda-ml/flavor_base/derived_from

@@ -0,0 +1 @@
+flavors/template-Exasol-8-python-3.10-cuda-conda

flavors/test-Exasol-8-cuda-ml/flavor_base/flavor_base_deps/Dockerfile

@@ -0,0 +1,8 @@
+FROM {{ language_deps }}
+
+RUN mkdir -p /build_info/packages
+COPY flavor_base_deps/packages /build_info/packages/flavor_base_deps
+
+RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/flavor_base_deps/conda_packages --channel-file /build_info/packages/flavor_base_deps/conda_channels --with-versions --conda-binary "$MAMBA_ROOT_PREFIX/bin/mamba"
+
+RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary "$MAMBA_ROOT_PREFIX/bin/python3.10" --with-versions --allow-no-version

flavors/test-Exasol-8-cuda-ml/flavor_base/flavor_base_deps/packages/conda_channels

@@ -0,0 +1,2 @@
+conda-forge
+nvidia

flavors/test-Exasol-8-cuda-ml/flavor_base/flavor_base_deps/packages/conda_packages

@@ -0,0 +1,7 @@
+cudnn|9.3.0.75
+cuda-toolkit|12.6.2
+cuda-compat|12.6.2
+cuda-nvcc|12.6.77
+cuda-nvrtc|12.6.77
+numba|0.60.0
+pytorch|2.4.1