Use conda to build python packages during GPU tests

Signed-off-by: Jordan Jacobelli <jjacobelli@nvidia.com>
rapidsai · May 6, 2022 · a87fe2e · a87fe2e
1 parent 081df0a
commit a87fe2e
Show file tree

Hide file tree

Showing 72 changed files with 127 additions and 137 deletions.
diff --git a/benchmarks/python_pytest_based/bench_algos.py b/benchmarks/python_pytest_based/bench_algos.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2021, NVIDIA CORPORATION.
+# Copyright (c) 2020-2022, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -31,7 +31,7 @@ def setFixtureParamNames(*args, **kwargs):
 
 import cugraph
 from cugraph.structure.number_map import NumberMap
-from cugraph.tests import utils
+from cugraph.testing import utils
 from cugraph.utilities.utils import is_device_version_less_than
 import rmm
 

diff --git a/benchmarks/python_pytest_based/params.py b/benchmarks/python_pytest_based/params.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2021, NVIDIA CORPORATION.
+# Copyright (c) 2020-2022, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -13,8 +13,7 @@
 
 import pytest
 
-from cugraph.tests.utils import genFixtureParamsProduct
-from cugraph.tests import utils
+from cugraph.testing import utils
 from pathlib import PurePath
 
 # FIXME: write and use mechanism described here for specifying datasets:
@@ -52,7 +51,7 @@
  marks=[pytest.mark.poolallocator_off]),
 ]
 
-FIXTURE_PARAMS = genFixtureParamsProduct(
+FIXTURE_PARAMS = utils.genFixtureParamsProduct(
  (DIRECTED_DATASETS + UNDIRECTED_DATASETS, "ds"),
  (MANAGED_MEMORY, "mm"),
  (POOL_ALLOCATOR, "pa"))
diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh
@@ -38,6 +38,7 @@ export HOME=$WORKSPACE
 cd $WORKSPACE
 export GIT_DESCRIBE_TAG=`git describe --tags`
 export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
+unset GIT_DESCRIBE_TAG
 
 # ucx-py version
 export UCX_PY_VERSION='0.26.*'
@@ -102,15 +103,15 @@ if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
  gpuci_logger "Build from source"
  $WORKSPACE/build.sh -v clean libcugraph pylibcugraph cugraph
 else
- echo "Installing libcugraph-tests"
+ gpuci_logger "Installing libcugraph-tests"
  gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} libcugraph libcugraph_etl libcugraph-tests
 
- gpuci_logger "Install the master version of dask and distributed"
- pip install "git+https://github.com/dask/distributed.git" --upgrade --no-deps
- pip install "git+https://github.com/dask/dask.git" --upgrade --no-deps
-
- echo "Build pylibcugraph and cugraph..."
- $WORKSPACE/build.sh pylibcugraph cugraph
+ gpuci_logger "Building and installing pylibcugraph and cugraph..."
+ export CONDA_BLD_DIR="${WORKSPACE}/.conda-bld"
+ export VERSION_SUFFIX=""
+ gpuci_conda_retry build conda/recipes/pylibcugraph --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_ARTIFACT_PATH} --python=${PYTHON}
+ gpuci_conda_retry build conda/recipes/cugraph --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_ARTIFACT_PATH} --python=${PYTHON}
+ gpuci_mamba_retry install cugraph pylibcugraph -c ${CONDA_BLD_DIR}
 fi
 
 ################################################################################

diff --git a/...libcugraph/pylibcugraph/tests/__init__.py → python/cugraph/cugraph/testing/__init__.py b/...libcugraph/pylibcugraph/tests/__init__.py → python/cugraph/cugraph/testing/__init__.py
diff --git a/python/cugraph/cugraph/tests/utils.py → python/cugraph/cugraph/testing/utils.py b/python/cugraph/cugraph/tests/utils.py → python/cugraph/cugraph/testing/utils.py
@@ -374,7 +374,7 @@ def random_edgelist(
 
  Examples
  --------
- >>> from cugraph.tests import utils
+ >>> from cugraph.testing import utils
  >>> # genrates 20 df with 100M edges each and write to disk
  >>> for x in range(20):
  >>> df = utils.random_edgelist(e=100000000, ef=64,

diff --git a/python/cugraph/cugraph/tests/__init__.py b/python/cugraph/cugraph/tests/__init__.py
diff --git a/python/cugraph/cugraph/tests/mg/test_mg_batch_betweenness_centrality.py b/python/cugraph/cugraph/tests/mg/test_mg_batch_betweenness_centrality.py
@@ -12,15 +12,18 @@
 # limitations under the License.
 
 import gc
+import sys
 
 import pytest
 import numpy as np
 
 from cugraph.dask.common.mg_utils import is_single_gpu
-from cugraph.tests.utils import RAPIDS_DATASET_ROOT_DIR_PATH
+from cugraph.testing.utils import RAPIDS_DATASET_ROOT_DIR_PATH
 
 # Get parameters from standard betwenness_centrality_test
-from cugraph.tests.test_betweenness_centrality import (
+# As tests directory is not a module, we need to add it to the path
+sys.path.insert(0, '../')
+from test_betweenness_centrality import ( # noqa: E402
  DIRECTED_GRAPH_OPTIONS,
  ENDPOINTS_OPTIONS,
  NORMALIZED_OPTIONS,
@@ -29,7 +32,7 @@
  SUBSET_SEED_OPTIONS,
 )
 
-from cugraph.tests.test_betweenness_centrality import (
+from test_betweenness_centrality import ( # noqa: E402
  calc_betweenness_centrality,
  compare_scores,
 )

diff --git a/python/cugraph/cugraph/tests/mg/test_mg_batch_edge_betweenness_centrality.py b/python/cugraph/cugraph/tests/mg/test_mg_batch_edge_betweenness_centrality.py
@@ -12,24 +12,27 @@
 # limitations under the License.
 
 import gc
+import sys
 
 import pytest
 import numpy as np
 
 from cugraph.dask.common.mg_utils import is_single_gpu
 
-from cugraph.tests.utils import RAPIDS_DATASET_ROOT_DIR_PATH
+from cugraph.testing.utils import RAPIDS_DATASET_ROOT_DIR_PATH
 
 # Get parameters from standard betwenness_centrality_test
-from cugraph.tests.test_edge_betweenness_centrality import (
+# As tests directory is not a module, we need to add it to the path
+sys.path.insert(0, '.')
+from test_edge_betweenness_centrality import ( # noqa: E402
  DIRECTED_GRAPH_OPTIONS,
  NORMALIZED_OPTIONS,
  DEFAULT_EPSILON,
  SUBSET_SIZE_OPTIONS,
  SUBSET_SEED_OPTIONS,
 )
 
-from cugraph.tests.test_edge_betweenness_centrality import (
+from test_edge_betweenness_centrality import ( # noqa: E402
  calc_edge_betweenness_centrality,
  compare_scores,
 )

diff --git a/python/cugraph/cugraph/tests/mg/test_mg_bfs.py b/python/cugraph/cugraph/tests/mg/test_mg_bfs.py
@@ -19,7 +19,7 @@
 import dask_cudf
 import cudf
 # from cugraph.dask.common.mg_utils import is_single_gpu
-from cugraph.tests.utils import RAPIDS_DATASET_ROOT_DIR_PATH
+from cugraph.testing.utils import RAPIDS_DATASET_ROOT_DIR_PATH
 
 # =============================================================================
 # Pytest Setup / Teardown - called for each test function

diff --git a/python/cugraph/cugraph/tests/mg/test_mg_comms.py b/python/cugraph/cugraph/tests/mg/test_mg_comms.py
@@ -19,7 +19,7 @@
 import dask_cudf
 import cudf
 # from cugraph.dask.common.mg_utils import is_single_gpu
-from cugraph.tests.utils import RAPIDS_DATASET_ROOT_DIR_PATH
+from cugraph.testing.utils import RAPIDS_DATASET_ROOT_DIR_PATH
 
 # =============================================================================
 # Pytest Setup / Teardown - called for each test function

diff --git a/python/cugraph/cugraph/tests/mg/test_mg_connectivity.py b/python/cugraph/cugraph/tests/mg/test_mg_connectivity.py
@@ -19,7 +19,7 @@
 import dask_cudf
 import cudf
 # from cugraph.dask.common.mg_utils import is_single_gpu
-from cugraph.tests.utils import RAPIDS_DATASET_ROOT_DIR_PATH
+from cugraph.testing.utils import RAPIDS_DATASET_ROOT_DIR_PATH
 
 # =============================================================================
 # Pytest Setup / Teardown - called for each test function

diff --git a/python/cugraph/cugraph/tests/mg/test_mg_degree.py b/python/cugraph/cugraph/tests/mg/test_mg_degree.py
@@ -19,7 +19,7 @@
 
 import cugraph
 from cugraph.dask.common.mg_utils import is_single_gpu
-from cugraph.tests.utils import RAPIDS_DATASET_ROOT_DIR_PATH
+from cugraph.testing.utils import RAPIDS_DATASET_ROOT_DIR_PATH
 
 # =============================================================================
 # Pytest Setup / Teardown - called for each test function

diff --git a/python/cugraph/cugraph/tests/mg/test_mg_doctests.py b/python/cugraph/cugraph/tests/mg/test_mg_doctests.py
@@ -24,7 +24,7 @@
 
 import cugraph
 import cudf
-from cugraph.tests import utils
+from cugraph.testing import utils
 
 from dask.distributed import Client
 from dask_cuda import LocalCUDACluster

diff --git a/python/cugraph/cugraph/tests/mg/test_mg_hits.py b/python/cugraph/cugraph/tests/mg/test_mg_hits.py
@@ -17,7 +17,7 @@
 import cugraph
 import dask_cudf
 # from cugraph.dask.common.mg_utils import is_single_gpu
-from cugraph.tests import utils
+from cugraph.testing import utils
 
 
 # =============================================================================

diff --git a/python/cugraph/cugraph/tests/mg/test_mg_katz_centrality.py b/python/cugraph/cugraph/tests/mg/test_mg_katz_centrality.py
@@ -19,7 +19,7 @@
 import dask_cudf
 import cudf
 from cugraph.dask.common.mg_utils import is_single_gpu
-from cugraph.tests.utils import RAPIDS_DATASET_ROOT_DIR_PATH
+from cugraph.testing.utils import RAPIDS_DATASET_ROOT_DIR_PATH
 
 
 # =============================================================================
@@ -65,7 +65,7 @@ def test_dask_katz_centrality(dask_client, directed):
  mg_res = mg_res.compute()
 
  import networkx as nx
- from cugraph.tests import utils
+ from cugraph.testing import utils
  NM = utils.read_csv_for_nx(input_data_path)
  if directed:
  Gnx = nx.from_pandas_edgelist(

diff --git a/python/cugraph/cugraph/tests/mg/test_mg_louvain.py b/python/cugraph/cugraph/tests/mg/test_mg_louvain.py
@@ -16,7 +16,7 @@
 import cugraph.dask as dcg
 import cugraph
 import dask_cudf
-from cugraph.tests import utils
+from cugraph.testing import utils
 # from cugraph.dask.common.mg_utils import is_single_gpu
 
 try:

diff --git a/python/cugraph/cugraph/tests/mg/test_mg_neighborhood_sampling.py b/python/cugraph/cugraph/tests/mg/test_mg_neighborhood_sampling.py
@@ -18,7 +18,7 @@
 import dask_cudf
 import cudf
 from cugraph.dask.common.mg_utils import is_single_gpu
-from cugraph.tests import utils
+from cugraph.testing import utils
 
 
 # =============================================================================

diff --git a/python/cugraph/cugraph/tests/mg/test_mg_pagerank.py b/python/cugraph/cugraph/tests/mg/test_mg_pagerank.py
@@ -18,7 +18,7 @@
 import dask_cudf
 import cudf
 # from cugraph.dask.common.mg_utils import is_single_gpu
-from cugraph.tests.utils import RAPIDS_DATASET_ROOT_DIR_PATH
+from cugraph.testing.utils import RAPIDS_DATASET_ROOT_DIR_PATH
 
 
 # The function selects personalization_perc% of accessible vertices in graph M

diff --git a/python/cugraph/cugraph/tests/mg/test_mg_renumber.py b/python/cugraph/cugraph/tests/mg/test_mg_renumber.py
@@ -25,10 +25,10 @@
 
 import cugraph.dask as dcg
 import cugraph
-from cugraph.tests import utils
+from cugraph.testing import utils
 from cugraph.structure.number_map import NumberMap
 from cugraph.dask.common.mg_utils import is_single_gpu
-from cugraph.tests.utils import RAPIDS_DATASET_ROOT_DIR_PATH
+from cugraph.testing.utils import RAPIDS_DATASET_ROOT_DIR_PATH
 
 
 # =============================================================================

diff --git a/python/cugraph/cugraph/tests/mg/test_mg_replication.py b/python/cugraph/cugraph/tests/mg/test_mg_replication.py
@@ -20,7 +20,7 @@
 import cugraph
 import cugraph.dask.structure.replication as replication
 from cugraph.dask.common.mg_utils import is_single_gpu
-import cugraph.tests.utils as utils
+import cugraph.testing.utils as utils
 
 DATASETS_OPTIONS = utils.DATASETS_SMALL
 DIRECTED_GRAPH_OPTIONS = [False, True]

diff --git a/python/cugraph/cugraph/tests/mg/test_mg_sssp.py b/python/cugraph/cugraph/tests/mg/test_mg_sssp.py
@@ -19,7 +19,7 @@
 import dask_cudf
 import cudf
 # from cugraph.dask.common.mg_utils import is_single_gpu
-from cugraph.tests.utils import RAPIDS_DATASET_ROOT_DIR_PATH
+from cugraph.testing.utils import RAPIDS_DATASET_ROOT_DIR_PATH
 
 # =============================================================================
 # Pytest Setup / Teardown - called for each test function

diff --git a/python/cugraph/cugraph/tests/mg/test_mg_symmetrize.py b/python/cugraph/cugraph/tests/mg/test_mg_symmetrize.py
@@ -18,7 +18,7 @@
 import pandas as pd
 import cudf
 import cugraph
-from cugraph.tests import utils
+from cugraph.testing import utils
 from cugraph.dask.common.mg_utils import (is_single_gpu,
  setup_local_dask_cluster,
  teardown_local_dask_cluster)

diff --git a/python/cugraph/cugraph/tests/mg/test_mg_utility.py b/python/cugraph/cugraph/tests/mg/test_mg_utility.py
@@ -20,12 +20,12 @@
 from cugraph.dask.common.part_utils import concat_within_workers
 from cugraph.dask.common.read_utils import get_n_workers
 from cugraph.dask.common.mg_utils import is_single_gpu
-from cugraph.tests.utils import RAPIDS_DATASET_ROOT_DIR_PATH
+from cugraph.testing.utils import RAPIDS_DATASET_ROOT_DIR_PATH
 
 import os
 import time
 import numpy as np
-from cugraph.tests import utils
+from cugraph.testing import utils
 
 
 # =============================================================================

diff --git a/python/cugraph/cugraph/tests/test_balanced_cut.py b/python/cugraph/cugraph/tests/test_balanced_cut.py
@@ -19,7 +19,7 @@
 import pandas as pd
 import cudf
 import cugraph
-from cugraph.tests import utils
+from cugraph.testing import utils
 
 
 def cugraph_call(G, partitions):

diff --git a/python/cugraph/cugraph/tests/test_betweenness_centrality.py b/python/cugraph/cugraph/tests/test_betweenness_centrality.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2021, NVIDIA CORPORATION.:
+# Copyright (c) 2020-2022, NVIDIA CORPORATION.:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -16,7 +16,7 @@
 import pytest
 
 import cugraph
-from cugraph.tests import utils
+from cugraph.testing import utils
 import random
 import numpy as np
 import cudf

diff --git a/python/cugraph/cugraph/tests/test_bfs.py b/python/cugraph/cugraph/tests/test_bfs.py
@@ -18,7 +18,7 @@
 import cudf
 import pytest
 import cugraph
-from cugraph.tests import utils
+from cugraph.testing import utils
 import random
 
 import pandas as pd

diff --git a/python/cugraph/cugraph/tests/test_compat_pr.py b/python/cugraph/cugraph/tests/test_compat_pr.py
@@ -18,7 +18,7 @@
 # python 3.7. Also, this import networkx needs to be relocated in the
 # third-party group once this gets fixed.
 import pytest
-from cugraph.tests import utils
+from cugraph.testing import utils
 import numpy as np
 import gc
 import importlib

diff --git a/python/cugraph/cugraph/tests/test_connectivity.py b/python/cugraph/cugraph/tests/test_connectivity.py
@@ -27,7 +27,7 @@
 
 import cudf
 import cugraph
-from cugraph.tests import utils
+from cugraph.testing import utils
 
 # Temporarily suppress warnings till networkX fixes deprecation warnings
 # (Using or importing the ABCs from 'collections' instead of from

diff --git a/python/cugraph/cugraph/tests/test_convert_matrix.py b/python/cugraph/cugraph/tests/test_convert_matrix.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.
+# Copyright (c) 2019-2022, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -14,7 +14,7 @@
 import gc
 import pytest
 import cugraph
-from cugraph.tests import utils
+from cugraph.testing import utils
 import numpy as np
 
 # Temporarily suppress warnings till networkX fixes deprecation warnings

diff --git a/python/cugraph/cugraph/tests/test_core_number.py b/python/cugraph/cugraph/tests/test_core_number.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.
+# Copyright (c) 2019-2022, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -14,7 +14,7 @@
 import gc
 import pytest
 import cugraph
-from cugraph.tests import utils
+from cugraph.testing import utils
 from cugraph.utilities import df_score_to_dictionary
 
 # Temporarily suppress warnings till networkX fixes deprecation warnings

diff --git a/python/cugraph/cugraph/tests/test_doctests.py b/python/cugraph/cugraph/tests/test_doctests.py
@@ -26,7 +26,7 @@
 import pylibcugraph
 import cudf
 from numba import cuda
-from cugraph.tests import utils
+from cugraph.testing import utils
 
 
 modules_to_skip = ["dask", "proto", "raft"]