Skip to content

Commit 0b6f994

Browse files
ma-pinedaTyler Titsworthpre-commit-ci[bot]jafraustro
authored
Adding classical-ml and data-analytics presets (#255)
Signed-off-by: Tyler Titsworth <tyler.titsworth@intel.com> Signed-off-by: tylertitsworth <tyler.titsworth@intel.com> Co-authored-by: Tyler Titsworth <tyler.titsworth@intel.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: jafraustro <jaime.fraustro.valdez@intel.com>
1 parent f9cad1c commit 0b6f994

File tree

18 files changed

+709
-781
lines changed

18 files changed

+709
-781
lines changed

‎.github/workflows/container-ci.yaml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ jobs:
6363
setup-build:
6464
outputs:
6565
matrix: ${{ steps.build-matrix.outputs.matrix }}
66-
runs-on: ubuntu-latest # ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
66+
runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
6767
steps:
6868
- name: Harden Runner
6969
uses: step-security/harden-runner@0d381219ddf674d61a7572ddd19d7941e271515c # v2.9.0
@@ -79,13 +79,14 @@ jobs:
7979
build-containers:
8080
needs: [setup-build]
8181
env: ${{ matrix }}
82-
runs-on: ubuntu-latest # ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
82+
runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
8383
strategy:
8484
matrix: ${{ fromJson(needs.setup-build.outputs.matrix) }}
8585
fail-fast: false
8686
outputs:
8787
group: ${{ steps.build-group.outputs.container-group }}
8888
steps:
89+
- uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
8990
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
9091
if: ${{ !inputs.no_build }}
9192
- uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
@@ -111,7 +112,7 @@ jobs:
111112
setup-scan:
112113
needs: [build-containers]
113114
if: ${{ github.event_name == 'pull_request' }}
114-
runs-on: ubuntu-latest # ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
115+
runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
115116
outputs:
116117
matrix: ${{ steps.scan-matrix.outputs.matrix }}
117118
steps:
@@ -164,7 +165,7 @@ jobs:
164165
####################################################################################################
165166
setup-test:
166167
needs: [build-containers]
167-
runs-on: ubuntu-latest # ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
168+
runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
168169
outputs:
169170
matrix: ${{ steps.test-matrix.outputs.matrix }}
170171
steps:

‎.github/workflows/integration-test.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ jobs:
113113
path: output.txt
114114
recreate: true
115115
status-check:
116-
needs: [group-diff, pipeline-ci]
116+
needs: [group-diff, pipeline-ci, merge-logs]
117117
runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
118118
if: always()
119119
steps:

‎preset/classical-ml/Dockerfile‎

Lines changed: 61 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -12,40 +12,35 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15+
1516
ARG BASE_IMAGE="ubuntu"
1617
ARG BASE_TAG="22.04"
1718

18-
FROM ${BASE_IMAGE}:${BASE_TAG} as classical-ml-base
19+
FROM ${BASE_IMAGE}:${BASE_TAG} as classical-ml
1920

2021
ENV DEBIAN_FRONTEND=noninteractive
2122

22-
# See http://bugs.python.org/issue19846
23-
2423
ENV LANG=C.UTF-8
2524

2625
SHELL ["/bin/bash", "-c"]
2726

2827
RUN apt-get update -y && \
2928
apt-get install -y --no-install-recommends --fix-missing \
30-
bzip2 \
31-
ca-certificates \
32-
diffutils \
33-
gcc \
34-
git \
35-
gzip \
36-
make \
37-
patch \
38-
rsync \
39-
unzip \
40-
wget \
41-
xz-utils && \
29+
bzip2 \
30+
ca-certificates \
31+
diffutils \
32+
gcc \
33+
git \
34+
gzip \
35+
make \
36+
patch \
37+
rsync \
38+
unzip \
39+
wget \
40+
xz-utils && \
4241
rm -rf /var/lib/apt/lists/*
4342

44-
FROM classical-ml-base as classical-ml-python
45-
46-
# Setting up non-root directories
4743
RUN useradd --uid 1000 -d /home/dev -s /bin/bash -m dev
48-
# Set a password for the user (Optional)
4944
RUN echo 'dev:password' | chpasswd
5045
USER dev
5146
WORKDIR /home/dev
@@ -56,77 +51,76 @@ ARG PYTHON_VERSION
5651
ARG IDP_VERSION
5752
ARG INTEL_CHANNEL
5853

59-
RUN wget --progress=dot:giga --no-check-certificate https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-${MINIFORGE_VERSION}.sh -O miniforge.sh && \
54+
RUN wget --progress=dot:giga --no-check-certificate https://github.com/conda-forge/miniforge/releases/latest/download/${MINIFORGE_VERSION}.sh -O miniforge.sh && \
6055
chmod +x miniforge.sh && \
6156
./miniforge.sh -b -p "${CONDA_ROOT}" && \
6257
rm ./miniforge.sh && \
63-
ln -s "${CONDA_ROOT}" "${CONDA_ROOT}/../miniforge3" && \
58+
ln -s "${CONDA_ROOT}" "${CONDA_ROOT}/../miniforge" && \
6459
export PATH="${CONDA_ROOT}/bin/:${PATH}" && \
65-
conda update -y conda && \
66-
conda config --add channels conda-forge && \
67-
conda config --add channels https://software.repos.intel.com/python/conda/ && \
6860
conda init --all && \
6961
conda install -y \
70-
'jupyterlab>=4.1.8' \
71-
'notebook>=7.1.3' \
72-
'jupyterhub>=4.1.5' \
73-
'jupyter-server-proxy>=4.1.2' \
74-
'mako>=1.2.2' \
75-
'pyjwt>=2.4.0' \
76-
'cryptography>=42.0.5' \
77-
'nodejs>=20.12.2' \
78-
'aiohttp>=3.9.4' \
62+
'colorama==0.4.6' \
63+
'conda==24.5.0' \
64+
'jupyterhub==5.1.0' \
65+
'jupyter-server-proxy==4.3.0' \
66+
'mamba==1.5.8' \
67+
'networkx==3.3' \
68+
'notebook==7.2.1' \
69+
'pip==24.0' \
70+
'python==3.10.14' \
7971
'idna>=3.7' \
80-
'oauthlib>=3.2.2' \
81-
&& \
82-
jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \
83-
conda clean -y --all
72+
'requests>=2.32.0' \
73+
'setuptools>=70.0.0' \
74+
'tqdm>=4.66.3' \
75+
'urllib3>=2.2.2' \
76+
'nodejs==22.5.1' \
77+
&& \
78+
jupyter labextension disable "@jupyterlab/apputils-extension:announcements" \
79+
&& \
80+
conda clean -y --all \
81+
&& \
82+
conda config --add channels ${INTEL_CHANNEL}
8483

8584
ENV PATH ${CONDA_ROOT}/condabin:${CONDA_ROOT}/bin/:${PATH}
85+
RUN conda config --set pip_interop_enabled True
8686

8787
ARG IDP_VERSION
88+
ARG DAAL4PY_VERSION
8889
ARG DPNP_VERSION
8990
ARG XGBOOST_VERSION
9091
ARG MODIN_VERSION
9192
ARG NUMPY_VERSION
9293
ARG SKLEARNEX_VERSION
9394

94-
# Conda packages
95-
RUN conda create -yn classical-ml -c ${INTEL_CHANNEL} -c conda-forge \
96-
dpnp=${DPNP_VERSION} \
97-
numpy=${NUMPY_VERSION} \
98-
python=${PYTHON_VERSION} \
99-
scikit-learn-intelex==${SKLEARNEX_VERSION} \
100-
xgboost=${XGBOOST_VERSION} \
101-
modin-ray=${MODIN_VERSION} \
102-
'python-dotenv>=1.0.1' \
103-
'tqdm>=4.66.2' \
104-
'matplotlib-base>=3.4.3' \
105-
'threadpoolctl>=3.3.0' \
106-
'ipython>=8.18.1' \
107-
'ipykernel>=6.29.3' \
108-
'kernda>=0.3.0' \
109-
'protobuf>=4.24' \
110-
'pillow>=10.2.0' \
111-
'tornado>=6.3.3' && \
95+
RUN conda create -yn classical-ml \
96+
"python=${PYTHON_VERSION}" \
97+
"daal4py=${DAAL4PY_VERSION}" \
98+
"dpnp=${DPNP_VERSION}" \
99+
'ipykernel==6.29.5' \
100+
'kernda==0.3.0' \
101+
'matplotlib-base==3.8.4' \
102+
"modin-ray=${MODIN_VERSION}" \
103+
'python-dotenv==1.0.1' \
104+
"scikit-learn-intelex=${SKLEARNEX_VERSION}" \
105+
'tqdm==4.66.4' \
106+
"xgboost=${XGBOOST_VERSION}" \
107+
'idna>=3.7' \
108+
'requests>=2.32.0' \
109+
'setuptools>=70.0.0' \
110+
'tqdm>=4.66.3' \
111+
'urllib3>=2.2.2' \
112+
&& \
112113
conda clean -y --all
113114

114-
115-
116-
# PyPI packages
117115
RUN conda run -n classical-ml python -m pip install --no-deps --no-cache-dir \
118-
'dataset-librarian==1.0.4' \
119-
'cloud-data-connector==1.0.3'
120-
116+
'dataset-librarian==1.0.4'
121117

122118
ENV PYTHONSTARTUP=~/.patch_sklearn.py
123119
COPY base/.patch_sklearn.py ~/.patch_sklearn.py
124120

125121
ENV PYTHONSTARTUP=/home/dev/.patch_sklearn.py
126122
COPY base/.patch_sklearn.py /home/dev/.patch_sklearn.py
127123

128-
FROM classical-ml-python as classical-ml-jupyter
129-
130124
EXPOSE 8888
131125

132126
RUN mkdir -p ~/jupyter/ && chmod -R a+rwx ~/jupyter/ && \
@@ -136,10 +130,10 @@ WORKDIR /home/dev
136130
COPY --chown=dev notebooks /home/dev/jupyter
137131
COPY --chown=dev tests /home/dev/sample-tests
138132

139-
RUN "${CONDA_ROOT}/envs/classical-ml/bin/python" -m ipykernel install --user --name classical-ml --display-name "Classical ML" && \
140-
"${CONDA_ROOT}/envs/classical-ml/bin/kernda" -o -y "$HOME/.local/share/jupyter/kernels/$(echo classical-ml | sed -e 's/\(.*\)/\L\1/')/kernel.json" && \
141-
"${CONDA_ROOT}/envs/classical-ml/bin/python" -m ipykernel.kernelspec --user && \
142-
conda clean -y --all
133+
RUN KERNEL_DIR="${CONDA_ROOT}/share/jupyter/kernels/classical-ml" && \
134+
conda run -n classical-ml python -m ipykernel install --prefix "$CONDA_ROOT" --name classical-ml --display-name "Classical ML" && \
135+
conda run -n classical-ml kernda -o -y "$KERNEL_DIR/kernel.json" && \
136+
conda run -n base jupyter kernelspec list
143137

144138
CMD ["bash", "-c", "source activate classical-ml && jupyter lab --notebook-dir=~/jupyter --port 8888 --ip 0.0.0.0 --no-browser --allow-root"]
145139

‎preset/classical-ml/docker-compose.yaml‎

Lines changed: 39 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -15,35 +15,38 @@
1515
# -*- coding: utf-8 -*-
1616
#
1717

18+
1819
version: '3'
1920
services:
2021
classical-ml:
2122
build:
2223
args:
2324
BASE_IMAGE: ${BASE_IMAGE:-ubuntu}
2425
BASE_TAG: ${BASE_TAG:-22.04}
25-
DPNP_VERSION: ${NUMBA_DPEX_VERSION:-0.14.0}
26-
IDP_VERSION: ${IDP_VERSION:-2024.1.0}
26+
DAAL4PY_VERSION: ${DAAL4PY_VERSION:-2024.5.0}
27+
DPNP_VERSION: ${DPNP_VERSION:-0.15.0}
28+
IDP_VERSION: ${IDP_VERSION:-2024.2}
2729
INTEL_CHANNEL: ${INTEL_CHANNEL:-https://software.repos.intel.com/python/conda/}
28-
MINIFORGE_VERSION: ${MINIFORGE_VERSION:-Linux-x86_64}
29-
MODIN_VERSION: ${MODIN_VERSION:-0.26.1}
30-
MPI_VERSION: ${MPI_VERSION:-2021.12.0}
31-
NUMBA_DPEX_VERSION: ${NUMBA_DPEX_VERSION:-0.22.1}
30+
MINIFORGE_VERSION: ${MINIFORGE_VERSION:-Miniforge3-Linux-x86_64}
31+
MODIN_VERSION: ${MODIN_VERSION:-0.30.0}
32+
MPI_VERSION: ${MPI_VERSION:-2021.13}
33+
NUMBA_DPEX_VERSION: ${NUMBA_DPEX_VERSION:-0.23.0}
3234
NUMPY_VERSION: ${NUMPY_VERSION:-1.26.4}
33-
PYTHON_VERSION: ${PYTHON_VERSION:-3.10}
34-
SKLEARNEX_VERSION: ${SKLEARNEX_VERSION:-2024.2.0}
35+
PYTHON_VERSION: ${PYTHON_VERSION:-3.9}
36+
SKLEARNEX_VERSION: ${SKLEARNEX_VERSION:-2024.5.0}
3537
XGBOOST_VERSION: ${XGBOOST_VERSION:-2.0.3}
3638
http_proxy: ${http_proxy}
3739
https_proxy: ${https_proxy}
3840
no_proxy: ''
3941
context: .
42+
target: classical-ml
4043
labels:
4144
docs: classical_ml
4245
org.opencontainers.image.title: "Intel® AI Tools Selector Preset Containers - Classical ML"
4346
org.opencontainers.base.name: "ubuntu:22.04"
4447
org.opencontainers.image.name: "intel/classical-ml"
45-
org.opencontainers.image.version: 2024.1.0-py${PYTHON_VERSION:-3.10}
46-
dependency.python: ${PYTHON_VERSION:-3.10}
48+
org.opencontainers.image.version: 2024.2.0-py${PYTHON_VERSION:-3.9}
49+
dependency.python: ${PYTHON_VERSION:-3.9}
4750
dependency.python.pip: requirements.txt
4851
dependency.apt.bzip2: true
4952
dependency.apt.ca-certificates: true
@@ -57,43 +60,36 @@ services:
5760
dependency.apt.unzip: true
5861
dependency.apt.wget: true
5962
dependency.apt.xz-utils: true
60-
dependency.conda.jupyterlab: '>=4.1.8'
61-
dependency.conda.notebook: '>=7.1.3'
62-
dependency.conda.jupyterhub: '>=4.1.5'
63-
dependency.conda.jupyter-server-proxy: '>=4.1.2'
64-
dependency.conda.mako: '>=1.2.2'
65-
dependency.conda.pyjwt: '>=2.4.0'
66-
dependency.conda.cryptography: '>=42.0.5'
67-
dependency.conda.nodejs: '>=20.12.2'
68-
dependency.conda.aiohttp: '>=3.9.4'
69-
dependency.conda.idna: '>=3.7'
70-
dependency.conda.oauthlib: '>=3.2.2'
71-
dependency.conda.dpnp: '>=0.14.0'
72-
dependency.conda.numpy: '>=1.26.4'
73-
dependency.conda.python: "=${PYTHON_VERSION:-3.10}"
74-
dependency.conda.scikit-learn-intelex: '>=2024.2.0'
75-
dependency.conda.xgboost: '>=2.0.3'
76-
dependency.conda.modin-ray: '>=0.26.1'
77-
dependency.conda.python-dotenv: '>=1.0.1'
78-
dependency.conda.tqdm: '>=4.66.2'
79-
dependency.conda.matplotlib-base: '>=3.4.3'
80-
dependency.conda.dataset_librarian: '>=1.0.4'
81-
dependency.conda.threadpoolctl: '>=3.3.0'
82-
dependency.conda.ipython: '>=8.18.1'
83-
dependency.conda.ipykernel: '>=6.29.3'
84-
dependency.conda.kernda: '>=0.3.0'
85-
dependency.conda.protobuf: '>=4.24'
86-
dependency.conda.pillow: '>=10.2.0'
87-
dependency.conda.tornado: '>=6.3.3'
88-
target: classical-ml-jupyter
89-
command: |
90-
bash -c "conda run -n classical-ml python -c 'import sklearn; import xgboost; print(\"SciKit:\", sklearn.__version__, \" XGBoost:\",xgboost.__version__)' && \
91-
conda run -n classical-ml python -c 'import modin.pandas as pd, modin.config as cfg; cfg.Engine.put(\"Ray\"); df = pd.DataFrame([1]);print(df+1)'"
92-
image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-classical-ml-2024.1.0-py${PYTHON_VERSION:-3.10}
63+
dependency.conda.colorama: '==0.4.6'
64+
dependency.conda.conda: '==24.5.0'
65+
dependency.conda.daal4py: '=2024.5.0'
66+
dependency.conda.dpnp: '=0.15.0'
67+
dependency.conda.ipykernel: '==6.29.5'
68+
dependency.conda.jupyterhub: '==5.1.0'
69+
dependency.conda.jupyter-server-proxy: '==4.3.0'
70+
dependency.conda.kernda: '==0.3.0'
71+
dependency.conda.mamba: '==1.5.8'
72+
dependency.conda.matplotlib-base: '==3.8.4'
73+
dependency.conda.modin-ray: '=0.30.0'
74+
dependency.conda.networkx: '==3.3'
75+
dependency.conda.notebook: '==7.2.1'
76+
dependency.conda.pip: '==24.0'
77+
dependency.conda.python: '==3.10.14'
78+
dependency.conda.python-dotenv: '==1.0.1'
79+
dependency.conda.scikit-learn-intelex: '=2024.5.0'
80+
dependency.conda.tqdm: '==4.66.4'
81+
dependency.conda.xgboost: '=2.0.3'
82+
image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-classical-ml-${RELEASE:-2024.2.0}-py${PYTHON_VERSION:-3.9}
9383
environment:
9484
http_proxy: ${http_proxy}
9585
https_proxy: ${https_proxy}
9686
network_mode: host
9787
shm_size: 12GB
9888
volumes:
9989
- /dev/dri/by-path:/dev/dri/by-path
90+
command: >
91+
bash -c " conda run -n classical-ml python -c 'import sklearn;import xgboost;print(\"SciKit:\",
92+
sklearn.__version__, \" XGBoost:\", xgboost.__version__)' &&
93+
94+
conda run -n classical-ml python -c 'import modin.pandas as pd;import modin.config
95+
as cfg;cfg.Engine.put(\"Ray\");df = pd.DataFrame([1]);print(df+1)' "

‎preset/classical-ml/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
cloud-data-connector==1.0.3
1+
dataset-librarian==1.0.4

‎preset/classical-ml/tests/scikit/kmeans.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262

6363
data, labels = load_digits(return_X_y=True)
6464
(n_samples, n_features), n_digits = data.shape, np.unique(labels).size
65+
data = np.array(data, dtype=np.float64)
6566

6667
print(f"# digits: {n_digits}; # samples: {n_samples}; # features {n_features}")
6768

‎preset/classical-ml/tests/scikit/test_scikit.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@
1414
# limitations under the License.
1515

1616
set -xe
17+
1718
SCRIPT_DIR=$(dirname "$0")
1819

1920
python "${SCRIPT_DIR}/kmeans.py"
20-
21-
python "${SCRIPT_DIR}/kmeans.py" true
21+
python "${SCRIPT_DIR}/kmeans.py" true # Enable intel opt

0 commit comments

Comments
 (0)