feat(deps): add kubeflow-training to workbench images

This commit adds `kubeflow-training[huggingface]` to the following workbench images:
- `./jupyter/datascience/ubi9-python-3.11`
- `./jupyter/pytorch/ubi9-python-3.11`
- `./jupyter/rocm/pytorch/ubi9-python-3.11`
- `./jupyter/trustyai/ubi9-python-3.11`
- `./codeserver/ubi9-python-3.11`

This outcome comes with a slew of caveats and disclaimers:
- Due to a dependency conflict, `codeflare-sdk==0.24.3` was **also** added to the following workbench images.
    - `./jupyter/datascience/ubi9-python-3.11`
    - `./jupyter/pytorch/ubi9-python-3.11`
    - `./jupyter/rocm/pytorch/ubi9-python-3.11`
    - `./jupyter/trustyai/ubi9-python-3.11`
- ⚠️ In what may be a "controversial" decision, `codeflare-sdk` was **NOT** updated on other workbench images.  Since `0.24.3` was a "one-off" release to unblock the `kubeflow-training` inclusion - the thought process here is that normal "sync" procedures on the next official release will standardize the `codeflare-sdk` dependency across all workbench images.  This allows us to restrict the testing effort of this commit.
- `jupyter/minmal/ubi9-python-3.11` was deliberately excluded from receiving `kubeflow-training` per discussions with team.
- Due to dependency conflicts discovered `tensorflow`-based workbench images,`kubeflow-training` has not been added to those workbench images at this time.  This decision was agreed to by affect stakeholders.  Core blocking issue can be seen here:
    - onnx/tensorflow-onnx#2328
- Due to a dependency conflict, `transformers = "==4.38.0"` was **also** added to the`./jupyter/trustyai/ubi9-python-3.11` workbench image after discussion with the developer that last worked on the `trustyai` image.  While it certainly must be tested, there was no strict requirement that necessitated pinning the `transformers` dependency to `4.36.2` - and the `huggingface` `extras` now introduces a `4.38.0` constraint for `transformers`.

Related-to: https://issues.redhat.com/browse/RHOAIENG-12822

codeserver/ubi9-python-3.11/Pipfile

@@ -11,11 +11,8 @@ verify_ssl = true
 [dev-packages]
 
 [packages]
-# Base packages
 wheel = "~=0.43.0"
 setuptools = "~=70.0.0"
-
-# Datascience packages
 boto3 = "~=1.34.60"
 kafka-python = "~=2.0.2"
 matplotlib = "~=3.8.3"
@@ -26,13 +23,13 @@ scikit-learn = "~=1.4.0"
 scipy = "~=1.12.0"
 skl2onnx = "~=1.16.0"
 ipykernel = "~=6.29.3"
-
-# Some extra usefull packages
 opencensus = "~=0.11.4"
 smart-open = "~=7.0.1"
 virtualenv = "~=20.25.1"
 py-spy = "~=0.3.14"
 prometheus-client = "~=0.20.0"
+codeflare-sdk = "==0.24.3"
+kubeflow-training = {extras = ["huggingface"], version = "==1.8.1"}
 
 [requires]
 python_version = "3.11"

jupyter/datascience/ubi9-python-3.11/Pipfile

@@ -6,7 +6,6 @@ name = "pypi"
 [dev-packages]
 
 [packages]
-# Datascience and useful extensions
 boto3 = "~=1.35.14"
 kafka-python-ng = "~=2.2.2"
 kfp = "~=2.9.0"
@@ -18,18 +17,11 @@ scikit-learn = "~=1.5.1"
 scipy = "~=1.14.1"
 skl2onnx = "~=1.17.0"
 onnxconverter-common = "~=1.13.0" # Required for skl2onnx, as upgraded version is not compatible with protobuf
-codeflare-sdk = "~=0.24.0"
-
-# DB connectors
 pymongo = "~=4.8.0"
 psycopg = "~=3.2.1"
 pyodbc = "~=5.1.0"
 mysql-connector-python = "~=9.0.0"
-
-# JupyterLab packages
-
 odh-elyra = "==4.2.0"
-
 jupyterlab = "~=4.2.4"
 jupyter-bokeh = "~=4.0.5"
 jupyter-server = "~=2.14.2"
@@ -41,10 +33,10 @@ jupyterlab-widgets = "~=3.0.13"
 jupyter-resource-usage = "~=1.1.0"
 nbdime = "~=4.0.1"
 nbgitpuller = "~=1.2.1"
-
-# Base packages
 wheel = "~=0.44.0"
 setuptools = "~=74.1.2"
+codeflare-sdk = "==0.24.3"
+kubeflow-training = {extras = ["huggingface"], version = "==1.8.1"}
 
 [requires]
 python_version = "3.11"

jupyter/pytorch/ubi9-python-3.11/Pipfile

@@ -11,12 +11,9 @@ name = "pytorch"
 [dev-packages]
 
 [packages]
-# PyTorch packages
 tensorboard = "~=2.17.1"
 torch = {version = "~=2.4.0", index = "pytorch"}
 torchvision = {version = "~=0.19.0", index = "pytorch"}
-
-# Datascience and useful extensions
 boto3 = "~=1.35.14"
 kafka-python-ng = "~=2.2.2"
 kfp = "~=2.9.0"
@@ -28,18 +25,11 @@ scikit-learn = "~=1.5.1"
 scipy = "~=1.14.1"
 skl2onnx = "~=1.17.0"
 onnxconverter-common = "~=1.13.0" # Required for skl2onnx, as upgraded version is not compatible with protobuf
-codeflare-sdk = "~=0.24.0"
-
-# DB connectors
 pymongo = "~=4.8.0"
 psycopg = "~=3.2.1"
 pyodbc = "~=5.1.0"
 mysql-connector-python = "~=9.0.0"
-
-# JupyterLab packages
-
 odh-elyra = "==4.2.0"
-
 jupyterlab = "~=4.2.4"
 jupyter-bokeh = "~=4.0.5"
 jupyter-server = "~=2.14.2"
@@ -51,10 +41,10 @@ jupyterlab-widgets = "~=3.0.13"
 jupyter-resource-usage = "~=1.1.0"
 nbdime = "~=4.0.1"
 nbgitpuller = "~=1.2.1"
-
-# Base packages
 wheel = "~=0.44.0"
 setuptools = "~=74.1.2"
+codeflare-sdk = "==0.24.3"
+kubeflow-training = {extras = ["huggingface"], version = "==1.8.1"}
 
 [requires]
 python_version = "3.11"

jupyter/rocm/pytorch/ubi9-python-3.11/Pipfile

@@ -12,13 +12,10 @@ name = "pytorch"
 [dev-packages]
 
 [packages]
-# ROCm PyTorch packages
 tensorboard = "~=2.16.2"
 torch = {version = "~=2.4.0", index = "pytorch"}
 torchvision = {version = "~=0.19.0", index = "pytorch"} # https://pytorch.org/get-started/previous-versions/#linux-and-windows-1
 pytorch-triton-rocm = {version = "~=3.0.0", index = "pytorch"}
-
-# Datascience and useful extensions
 boto3 = "~=1.35.14"
 kafka-python-ng = "~=2.2.2"
 kfp = "~=2.9.0"
@@ -30,18 +27,11 @@ scikit-learn = "~=1.5.1"
 scipy = "~=1.14.1"
 skl2onnx = "~=1.17.0"
 onnxconverter-common = "~=1.13.0" # Required for skl2onnx, as upgraded version is not compatible with protobuf
-codeflare-sdk = "~=0.24.0"
-
-# DB connectors
 pymongo = "~=4.8.0"
 psycopg = "~=3.2.1"
 pyodbc = "~=5.1.0"
 mysql-connector-python = "~=9.0.0"
-
-# JupyterLab packages
-
 odh-elyra = "==4.2.0"
-
 jupyterlab = "~=4.2.4"
 jupyter-bokeh = "~=4.0.5"
 jupyter-server = "~=2.14.2"
@@ -53,10 +43,10 @@ jupyterlab-widgets = "~=3.0.13"
 jupyter-resource-usage = "~=1.1.0"
 nbdime = "~=4.0.1"
 nbgitpuller = "~=1.2.1"
-
-# Base packages
 wheel = "~=0.44.0"
 setuptools = "~=74.1.2"
+codeflare-sdk = "==0.24.3"
+kubeflow-training = {extras = ["huggingface"], version = "==1.8.1"}
 
 [requires]
 python_version = "3.11"

jupyter/trustyai/ubi9-python-3.11/Pipfile

@@ -11,14 +11,11 @@ name = "pytorch"
 [dev-packages]
 
 [packages]
-# PyTorch packages
 torch = {version = "~=2.2.1", index = "pytorch"}
-# TrustyAI packages
-transformers = "~=4.36.2"
+transformers = "~=4.38.0"
 datasets = "~=2.21.0"
 accelerate = "~=0.34.2"
 trustyai = "~=0.6.1"
-# Datascience and useful extensions
 boto3 = "~=1.35.14"
 kafka-python-ng = "~=2.2.2"
 kfp = "~=2.9.0"
@@ -30,17 +27,11 @@ scikit-learn = "~=1.2.1" # Should be pinned down to this version in order to be
 scipy = "~=1.14.1"
 skl2onnx = "~=1.17.0"
 onnxconverter-common = "~=1.13.0" # Required for skl2onnx, as upgraded version is not compatible with protobuf
-codeflare-sdk = "~=0.24.0"
-
-# DB connectors
 pymongo = "~=4.8.0"
 psycopg = "~=3.2.1"
 pyodbc = "~=5.1.0"
 mysql-connector-python = "~=9.0.0"
-
-# JupyterLab packages
 odh-elyra = "==4.2.0"
-
 jupyterlab = "~=4.2.4"
 jupyter-bokeh = "~=3.0.5" # Should be pinned down to this version in order to be compatible with trustyai
 jupyter-server = "~=2.14.2"
@@ -52,10 +43,10 @@ jupyterlab-widgets = "~=3.0.13"
 jupyter-resource-usage = "~=1.1.0"
 nbdime = "~=4.0.1"
 nbgitpuller = "~=1.2.1"
-
-# Base packages
 wheel = "~=0.44.0"
 setuptools = "~=74.1.2"
+codeflare-sdk = "==0.24.3"
+kubeflow-training = {extras = ["huggingface"], version = "==1.8.1"}
 
 [requires]
 python_version = "3.11"