move horovod to separte tox testenv

ci/test_unit.sh

@@ -18,3 +18,8 @@
 set -e
 
 tox -e test-gpu $@
+
+# skip multi-gpu testing if any arguments are supplied.
+if [[ $# -eq 0 ]] ; then
+  tox -e test-multi-gpu
+fi

tox.ini

@@ -10,7 +10,28 @@ commands =
     pip install -e .[all]
 
 [testenv:test-gpu]
-passenv = *
+passenv =
+    WANDB_API_KEY
+sitepackages = true
+; Runs in: Internal Jenkins
+; Runs GPU-based tests.
+; The jenkins jobs run on an image based on merlin-hugectr. This will include all cudf configuration
+; and other gpu-specific libraries that we can enxpect will always exist. Thus, we don't need
+; to install requirements.txt yet. As we get better at python environment isolation, we will
+; need to add some back.
+deps =
+    -rrequirements/dev.txt
+    tensorflow<2.10
+setenv =
+    TF_GPU_ALLOCATOR=cuda_malloc_async
+commands =
+    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git
+    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/nvtabular.git
+    python -m pytest --cov-report term --cov merlin -rxs {posargs:tests/unit}
+
+[testenv:test-multi-gpu]
+passenv =
+    OPAL_PREFIX
 sitepackages = true
 ; Runs in: Internal Jenkins
 ; Runs GPU-based tests.
@@ -23,11 +44,9 @@ deps =
     tensorflow<2.10
 setenv =
     TF_GPU_ALLOCATOR=cuda_malloc_async
-    HOROVOD_GPU_OPERATIONS=NCCL
 allowlist_externals =
     horovodrun
 commands =
     python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git
     python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/nvtabular.git
-    python -m pytest --cov-report term --cov merlin -rxs {posargs:tests/unit}
     horovodrun -np 2 sh examples/usecases/multi-gpu/hvd_wrapper.sh python -m pytest -m horovod -rxs tests/unit