From 86abfd61587ded7661032b732f5bf662a296524e Mon Sep 17 00:00:00 2001 From: Ralph Liu <137829296+nv-rliu@users.noreply.github.com> Date: Tue, 16 Jul 2024 22:16:14 -0400 Subject: [PATCH 01/11] Fix Warning from `simpleDistributedGraph.py` (#4540) This PR fixes a warning seen when accessing a `dask_cudf.Series` object directly Authors: - Ralph Liu (https://github.com/nv-rliu) Approvers: - Rick Ratzel (https://github.com/rlratzel) URL: https://github.com/rapidsai/cugraph/pull/4540 --- .../structure/graph_implementation/simpleDistributedGraph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cugraph/cugraph/structure/graph_implementation/simpleDistributedGraph.py b/python/cugraph/cugraph/structure/graph_implementation/simpleDistributedGraph.py index e50462c001e..7f3f7e83e59 100644 --- a/python/cugraph/cugraph/structure/graph_implementation/simpleDistributedGraph.py +++ b/python/cugraph/cugraph/structure/graph_implementation/simpleDistributedGraph.py @@ -891,9 +891,9 @@ def _call_plc_two_hop_neighbors(sID, mg_graph_x, start_vertices): if start_vertices is not None: if self.renumbered: start_vertices = self.renumber_map.to_internal_vertex_id(start_vertices) - start_vertices_type = self.edgelist.edgelist_df.dtypes[0] + start_vertices_type = self.edgelist.edgelist_df.dtypes.iloc[0] else: - start_vertices_type = self.input_df.dtypes[0] + start_vertices_type = self.input_df.dtypes.iloc[0] start_vertices = start_vertices.astype(start_vertices_type) From 710e31ad0766a5b6d8f713e2a9c70268f75dc8a3 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 17 Jul 2024 12:43:36 -0400 Subject: [PATCH 02/11] Build and test with CUDA 12.5.1 (#4534) This PR updates the latest CUDA build/test version 12.2.2 to 12.5.1. Contributes to https://github.com/rapidsai/build-planning/issues/73 Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) - https://github.com/jakirkham Approvers: - James Lamb (https://github.com/jameslamb) - https://github.com/jakirkham - Chuck Hastings (https://github.com/ChuckHastings) URL: https://github.com/rapidsai/cugraph/pull/4534 --- .../devcontainer.json | 8 ++-- .../devcontainer.json | 12 ++--- .github/workflows/build.yaml | 32 ++++++------- .github/workflows/pr.yaml | 46 +++++++++---------- .github/workflows/test.yaml | 18 ++++---- ..._64.yaml => all_cuda-125_arch-x86_64.yaml} | 4 +- dependencies.yaml | 6 ++- .../source/graph_support/DGL_support.md | 2 +- .../source/installation/source_build.md | 4 +- 9 files changed, 68 insertions(+), 64 deletions(-) rename .devcontainer/{cuda12.2-conda => cuda12.5-conda}/devcontainer.json (91%) rename .devcontainer/{cuda12.2-pip => cuda12.5-pip}/devcontainer.json (88%) rename conda/environments/{all_cuda-122_arch-x86_64.yaml => all_cuda-125_arch-x86_64.yaml} (96%) diff --git a/.devcontainer/cuda12.2-conda/devcontainer.json b/.devcontainer/cuda12.5-conda/devcontainer.json similarity index 91% rename from .devcontainer/cuda12.2-conda/devcontainer.json rename to .devcontainer/cuda12.5-conda/devcontainer.json index 434fc2c5885..c4149f04bed 100644 --- a/.devcontainer/cuda12.2-conda/devcontainer.json +++ b/.devcontainer/cuda12.5-conda/devcontainer.json @@ -3,7 +3,7 @@ "context": "${localWorkspaceFolder}/.devcontainer", "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", "args": { - "CUDA": "12.2", + "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "conda", "BASE": "rapidsai/devcontainers:24.08-cpp-mambaforge-ubuntu22.04" } @@ -11,7 +11,7 @@ "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.2-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.5-conda" ], "hostRequirements": {"gpu": "optional"}, "features": { @@ -20,7 +20,7 @@ "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.2-envs}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.5-envs}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cugraph,type=bind,consistency=consistent", @@ -29,7 +29,7 @@ "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.2-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.5-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.devcontainer/cuda12.2-pip/devcontainer.json b/.devcontainer/cuda12.5-pip/devcontainer.json similarity index 88% rename from .devcontainer/cuda12.2-pip/devcontainer.json rename to .devcontainer/cuda12.5-pip/devcontainer.json index 4a4bea7bbb0..5e42537b46d 100644 --- a/.devcontainer/cuda12.2-pip/devcontainer.json +++ b/.devcontainer/cuda12.5-pip/devcontainer.json @@ -3,20 +3,20 @@ "context": "${localWorkspaceFolder}/.devcontainer", "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", "args": { - "CUDA": "12.2", + "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.08-cpp-cuda12.2-ucx1.15.0-openmpi-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.08-cpp-cuda12.5-ucx1.15.0-openmpi-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.2-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.5-pip" ], "hostRequirements": {"gpu": "optional"}, "features": { "ghcr.io/rapidsai/devcontainers/features/cuda:24.8": { - "version": "12.2", + "version": "12.5", "installcuBLAS": true, "installcuSOLVER": true, "installcuRAND": true, @@ -28,7 +28,7 @@ "ghcr.io/rapidsai/devcontainers/features/cuda", "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.2-venvs}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.5-venvs}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cugraph,type=bind,consistency=consistent", @@ -36,7 +36,7 @@ "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.2-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.5-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 03d3162bdbf..704d632efe6 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -38,7 +38,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -47,7 +47,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibcugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -83,7 +83,7 @@ jobs: wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -93,7 +93,7 @@ jobs: wheel-build-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -106,7 +106,7 @@ jobs: wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -116,7 +116,7 @@ jobs: wheel-build-nx-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -126,7 +126,7 @@ jobs: wheel-publish-nx-cugraph: needs: wheel-build-nx-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -136,7 +136,7 @@ jobs: wheel-build-cugraph-dgl: needs: wheel-publish-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -146,7 +146,7 @@ jobs: wheel-publish-cugraph-dgl: needs: wheel-build-cugraph-dgl secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -156,7 +156,7 @@ jobs: wheel-build-cugraph-pyg: needs: wheel-publish-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -166,7 +166,7 @@ jobs: wheel-publish-cugraph-pyg: needs: wheel-build-cugraph-pyg secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -175,7 +175,7 @@ jobs: package-name: cugraph-pyg wheel-build-cugraph-equivariant: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -185,7 +185,7 @@ jobs: wheel-publish-cugraph-equivariant: needs: wheel-build-cugraph-equivariant secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4b5d0c26d0b..e41839bbf4f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -34,29 +34,29 @@ jobs: - wheel-tests-cugraph-equivariant - devcontainer secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@cuda-12.5.1 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@cuda-12.5.1 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@cuda-12.5.1 with: build_type: pull-request node_type: cpu32 conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-12.5.1 with: build_type: pull-request conda-cpp-checks: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@cuda-12.5.1 with: build_type: pull-request enable_check_symbols: true @@ -64,19 +64,19 @@ jobs: conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@cuda-12.5.1 with: build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda-12.5.1 with: build_type: pull-request conda-notebook-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -86,7 +86,7 @@ jobs: docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -96,7 +96,7 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: pull-request script: ci/build_wheel_pylibcugraph.sh @@ -107,14 +107,14 @@ jobs: wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: pull-request script: ci/test_wheel_pylibcugraph.sh wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: pull-request script: ci/build_wheel_cugraph.sh @@ -124,35 +124,35 @@ jobs: wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: pull-request script: ci/test_wheel_cugraph.sh wheel-build-nx-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: pull-request script: ci/build_wheel_nx-cugraph.sh wheel-tests-nx-cugraph: needs: wheel-build-nx-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: pull-request script: ci/test_wheel_nx-cugraph.sh wheel-build-cugraph-dgl: needs: wheel-tests-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: pull-request script: ci/build_wheel_cugraph-dgl.sh wheel-tests-cugraph-dgl: needs: wheel-build-cugraph-dgl secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: pull-request script: ci/test_wheel_cugraph-dgl.sh @@ -160,38 +160,38 @@ jobs: wheel-build-cugraph-pyg: needs: wheel-tests-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: pull-request script: ci/build_wheel_cugraph-pyg.sh wheel-tests-cugraph-pyg: needs: wheel-build-cugraph-pyg secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: pull-request script: ci/test_wheel_cugraph-pyg.sh matrix_filter: map(select(.ARCH == "amd64")) wheel-build-cugraph-equivariant: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: pull-request script: ci/build_wheel_cugraph-equivariant.sh wheel-tests-cugraph-equivariant: needs: wheel-build-cugraph-equivariant secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: pull-request script: ci/test_wheel_cugraph-equivariant.sh matrix_filter: map(select(.ARCH == "amd64")) devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@cuda-12.5.1 with: arch: '["amd64"]' - cuda: '["12.2"]' + cuda: '["12.5"]' node_type: cpu32 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY build_command: | diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 03d8b73a412..cab2063115a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -26,7 +26,7 @@ jobs: symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel) conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -42,7 +42,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibcugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: script: ci/test_wheel_pylibcugraph.sh wheel-tests-cugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -60,7 +60,7 @@ jobs: script: ci/test_wheel_cugraph.sh wheel-tests-nx-cugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: script: ci/test_wheel_nx-cugraph.sh wheel-tests-cugraph-dgl: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -79,7 +79,7 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) wheel-tests-cugraph-pyg: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -89,7 +89,7 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) wheel-tests-cugraph-equivariant: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml similarity index 96% rename from conda/environments/all_cuda-122_arch-x86_64.yaml rename to conda/environments/all_cuda-125_arch-x86_64.yaml index c48f3b386d5..8e7c2156b93 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -17,7 +17,7 @@ dependencies: - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api -- cuda-version=12.2 +- cuda-version=12.5 - cudf==24.8.*,>=0.0.0a0 - cupy>=12.0.0 - cxx-compiler @@ -81,4 +81,4 @@ dependencies: - ucx-py==0.39.*,>=0.0.0a0 - wget - wheel -name: all_cuda-122_arch-x86_64 +name: all_cuda-125_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index 03f4760b6d1..1ef92274bde 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -3,7 +3,7 @@ files: all: output: [conda] matrix: - cuda: ["11.8", "12.2"] + cuda: ["11.8", "12.5"] arch: [x86_64] includes: - checks @@ -353,6 +353,10 @@ dependencies: cuda: "12.2" packages: - cuda-version=12.2 + - matrix: + cuda: "12.5" + packages: + - cuda-version=12.5 cuda: specific: - output_types: [conda] diff --git a/docs/cugraph/source/graph_support/DGL_support.md b/docs/cugraph/source/graph_support/DGL_support.md index 9df462155fd..ba9a28e3170 100644 --- a/docs/cugraph/source/graph_support/DGL_support.md +++ b/docs/cugraph/source/graph_support/DGL_support.md @@ -17,7 +17,7 @@ mamba install cugraph-dgl -c rapidsai-nightly -c rapidsai -c pytorch -c conda-fo ### Create the conda development environment ``` -conda env create -n cugraph_dgl_dev --file conda/environments/all_cuda-122_arch-x86_64.yaml +conda env create -n cugraph_dgl_dev --file conda/environments/all_cuda-125_arch-x86_64.yaml ``` ### Install in editable mode diff --git a/docs/cugraph/source/installation/source_build.md b/docs/cugraph/source/installation/source_build.md index 49af1375781..89e63badef8 100644 --- a/docs/cugraph/source/installation/source_build.md +++ b/docs/cugraph/source/installation/source_build.md @@ -43,7 +43,7 @@ files](https://github.com/rapidsai/cugraph/blob/main/conda/environments). conda env create --name cugraph_dev --file $CUGRAPH_HOME/conda/environments/all_cuda-118_arch-x86_64.yaml # for CUDA 12.x -conda env create --name cugraph_dev --file $CUGRAPH_HOME/conda/environments/all_cuda-12.2_arch-x86_64.yaml +conda env create --name cugraph_dev --file $CUGRAPH_HOME/conda/environments/all_cuda-125_arch-x86_64.yaml # activate the environment @@ -61,7 +61,7 @@ conda env update --name cugraph_dev --file $CUGRAPH_HOME/conda/environments/all_ conda activate cugraph_dev # for CUDA 12.x -conda env update --name cugraph_dev --file $CUGRAPH_HOME/conda/environments/all_cuda-122_arch-x86_64.yaml +conda env update --name cugraph_dev --file $CUGRAPH_HOME/conda/environments/all_cuda-125_arch-x86_64.yaml conda activate cugraph_dev From 8829aa9ece9b57fd6bca795f0633b066f50f8afb Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Wed, 17 Jul 2024 12:24:47 -0700 Subject: [PATCH 03/11] Ensure `get_test_data.sh` doesn't re-download datasets (#4536) * support calling `get_test_data.sh` from any directory * only download datasets if newer than local copies * extract datasets in parallel After this PR, datasets will be stored in `datasets/tmp` and not deleted after extraction, because `wget -N` (aka `--timestamping`) relies on comparing the local and remote mtimes. Authors: - Paul Taylor (https://github.com/trxcllnt) Approvers: - Chuck Hastings (https://github.com/ChuckHastings) - Rick Ratzel (https://github.com/rlratzel) URL: https://github.com/rapidsai/cugraph/pull/4536 --- datasets/get_test_data.sh | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/datasets/get_test_data.sh b/datasets/get_test_data.sh index be2126be771..eea789ef3e3 100755 --- a/datasets/get_test_data.sh +++ b/datasets/get_test_data.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -15,6 +15,9 @@ set -e set -o pipefail +# Ensure we're in the cugraph/datasets dir +cd "$( cd "$( dirname "$(realpath -m "${BASH_SOURCE[0]}")" )" && pwd )"; + # Update this to add/remove/change a dataset, using the following format: # # comment about the dataset @@ -99,28 +102,19 @@ DESTDIRS=($(echo "$DATASET_DATA"|awk '{if (NR%4 == 0) print $0}')) # extract 4t echo Downloading ... # Download all tarfiles to a tmp dir -rm -rf tmp -mkdir tmp +mkdir -p tmp cd tmp for url in ${URLS[*]}; do - time wget --progress=dot:giga ${url} + time wget -N --progress=dot:giga ${url} done cd .. -# Setup the destination dirs, removing any existing ones first! -for index in ${!DESTDIRS[*]}; do - rm -rf ${DESTDIRS[$index]} -done -for index in ${!DESTDIRS[*]}; do - mkdir -p ${DESTDIRS[$index]} -done +# create the destination dirs +mkdir -p "${DESTDIRS[@]}" # Iterate over the arrays and untar the nth tarfile to the nth dest directory. # The tarfile name is derived from the download url. echo Decompressing ... for index in ${!DESTDIRS[*]}; do - tfname=$(basename ${URLS[$index]}) - tar xvzf tmp/${tfname} -C ${DESTDIRS[$index]} -done - -rm -rf tmp + echo "tmp/$(basename "${URLS[$index]}") -C ${DESTDIRS[$index]}" | tr '\n' '\0' +done | xargs -0 -t -r -n1 -P$(nproc --all) sh -c 'tar -xzvf $0 --overwrite' From 55cb99232ebda72a29688a689cd704dedd66c99f Mon Sep 17 00:00:00 2001 From: Seunghwa Kang <45857425+seunghwak@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:44:42 -0700 Subject: [PATCH 04/11] Create a graph from the edge list in multiple chunks (#4539) Added a graph creation function that takes edge list in multiple chunks. This helps to cut peak memory footprint. Authors: - Seunghwa Kang (https://github.com/seunghwak) Approvers: - Chuck Hastings (https://github.com/ChuckHastings) URL: https://github.com/rapidsai/cugraph/pull/4539 --- cpp/include/cugraph/graph_functions.hpp | 66 + .../create_graph_from_edgelist_impl.cuh | 1196 +++++++++++++---- .../create_graph_from_edgelist_mg_v32_e32.cu | 92 +- .../create_graph_from_edgelist_mg_v32_e64.cu | 91 +- .../create_graph_from_edgelist_mg_v64_e64.cu | 92 +- .../create_graph_from_edgelist_sg_v32_e32.cu | 92 +- .../create_graph_from_edgelist_sg_v32_e64.cu | 92 +- .../create_graph_from_edgelist_sg_v64_e64.cu | 92 +- cpp/tests/mtmg/multi_node_threaded_test.cu | 19 +- cpp/tests/mtmg/threaded_test.cu | 19 +- cpp/tests/mtmg/threaded_test_jaccard.cu | 19 +- cpp/tests/mtmg/threaded_test_louvain.cu | 19 +- cpp/tests/structure/renumbering_test.cpp | 14 +- cpp/tests/utilities/test_graphs.hpp | 368 +++-- 14 files changed, 1843 insertions(+), 428 deletions(-) diff --git a/cpp/include/cugraph/graph_functions.hpp b/cpp/include/cugraph/graph_functions.hpp index 79ff576571e..e1364f69991 100644 --- a/cpp/include/cugraph/graph_functions.hpp +++ b/cpp/include/cugraph/graph_functions.hpp @@ -730,6 +730,72 @@ create_graph_from_edgelist(raft::handle_t const& handle, bool renumber, bool do_expensive_check = false); +/** + * @brief create a graph from (the optional vertex list and) the given edge list (with optional edge + * IDs and types). + * + * This version takes edge list in multiple chunks (e.g. edge data from multiple files). + * + * @tparam vertex_t Type of vertex identifiers. Needs to be an integral type. + * @tparam edge_t Type of edge identifiers. Needs to be an integral type. + * @tparam weight_t Type of edge weight. Needs to be floating point type + * @tparam edge_id_t Type of edge id. Needs to be an integral type + * @tparam edge_type_t Type of edge type. Needs to be an integral type, currently only int32_t is + * supported + * @tparam store_transposed Flag indicating whether to use sources (if false) or destinations (if + * true) as major indices in storing edges using a 2D sparse matrix. transposed. + * @tparam multi_gpu Flag indicating whether template instantiation should target single-GPU (false) + * or multi-GPU (true). + * @param handle RAFT handle object to encapsulate resources (e.g. CUDA stream, communicator, and + * handles to various CUDA libraries) to run graph algorithms. + * @param vertices If valid, part of the entire set of vertices in the graph to be renumbered. + * This parameter can be used to include isolated vertices. If @p renumber is false and @p vertices + * is valid, @p vertices elements should be consecutive integers starting from 0. If multi-GPU, + * applying the compute_gpu_id_from_vertex_t to every vertex should return the local GPU ID for this + * function to work (vertices should be pre-shuffled). + * @param edgelist_srcs Vectors of edge source vertex IDs. If multi-GPU, applying the + * compute_gpu_id_from_ext_edge_endpoints_t to every edge should return the local GPU ID for this + * function to work (edges should be pre-shuffled). + * @param edgelist_dsts Vectors of edge destination vertex IDs. + * @param edgelist_weights Vectors of weight values for edges + * @param edgelist_edge_ids Vectors of edge_id values for edges + * @param edgelist_edge_types Vectors of edge_type values for edges + * @param graph_properties Properties of the graph represented by the input (optional vertex list + * and) edge list. + * @param renumber Flag indicating whether to renumber vertices or not (must be true if @p multi_gpu + * is true). + * @param do_expensive_check A flag to run expensive checks for input arguments (if set to `true`). + * @return Tuple of the generated graph and optional edge_property_t objects storing the provided + * edge properties and a renumber map (if @p renumber is true). + */ +template +std::tuple< + graph_t, + std::optional< + edge_property_t, weight_t>>, + std::optional< + edge_property_t, edge_id_t>>, + std::optional< + edge_property_t, edge_type_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check = false); + /** * @brief Find all 2-hop neighbors in the graph * diff --git a/cpp/src/structure/create_graph_from_edgelist_impl.cuh b/cpp/src/structure/create_graph_from_edgelist_impl.cuh index 1c15842982b..9796ddd60a1 100644 --- a/cpp/src/structure/create_graph_from_edgelist_impl.cuh +++ b/cpp/src/structure/create_graph_from_edgelist_impl.cuh @@ -317,206 +317,57 @@ std::enable_if_t< std::optional< edge_property_t, edge_type_t>>, std::optional>>> -create_graph_from_edgelist_impl( +create_graph_from_partitioned_edgelist( raft::handle_t const& handle, std::optional>&& local_vertices, - rmm::device_uvector&& edgelist_srcs, - rmm::device_uvector&& edgelist_dsts, - std::optional>&& edgelist_weights, - std::optional>&& edgelist_edge_ids, - std::optional>&& edgelist_edge_types, + std::vector>&& edge_partition_edgelist_srcs, + std::vector>&& edge_partition_edgelist_dsts, + std::optional>>&& edge_partition_edgelist_weights, + std::optional>>&& edge_partition_edgelist_edge_ids, + std::optional>>&& edge_partition_edgelist_edge_types, + std::vector> const& edgelist_intra_partition_segment_offsets, graph_properties_t graph_properties, - bool renumber, - bool do_expensive_check) + bool renumber) { auto& major_comm = handle.get_subcomm(cugraph::partition_manager::major_comm_name()); auto const major_comm_size = major_comm.get_size(); auto& minor_comm = handle.get_subcomm(cugraph::partition_manager::minor_comm_name()); auto const minor_comm_size = minor_comm.get_size(); - CUGRAPH_EXPECTS(edgelist_srcs.size() == edgelist_dsts.size(), - "Invalid input arguments: edgelist_srcs.size() != edgelist_dsts.size()."); - CUGRAPH_EXPECTS(!edgelist_weights || (edgelist_srcs.size() == (*edgelist_weights).size()), - "Invalid input arguments: edgelist_srcs.size() != edgelist_weights.size()."); - CUGRAPH_EXPECTS(!edgelist_edge_ids || (edgelist_srcs.size() == (*edgelist_edge_ids).size()), - "Invalid input arguments: edgelist_srcs.size() != " - "std::get<0>((*edgelist_edge_ids)).size()."); - CUGRAPH_EXPECTS(!edgelist_edge_types || (edgelist_srcs.size() == (*edgelist_edge_types).size()), - "Invalid input arguments: edgelist_srcs.size() != " - "std::get<1>((*edgelist_edge_types)).size()."); - CUGRAPH_EXPECTS(renumber, - "Invalid input arguments: renumber should be true if multi_gpu is true."); - - if (do_expensive_check) { - expensive_check_edgelist(handle, - local_vertices, - store_transposed ? edgelist_dsts : edgelist_srcs, - store_transposed ? edgelist_srcs : edgelist_dsts, - renumber); - - if (graph_properties.is_symmetric) { - CUGRAPH_EXPECTS( - (check_symmetric( - handle, - raft::device_span(edgelist_srcs.data(), edgelist_srcs.size()), - raft::device_span(edgelist_dsts.data(), edgelist_dsts.size()))), - "Invalid input arguments: graph_properties.is_symmetric is true but the input edge list is " - "not symmetric."); - } - - if (!graph_properties.is_multigraph) { - CUGRAPH_EXPECTS( - check_no_parallel_edge( - handle, - raft::device_span(edgelist_srcs.data(), edgelist_srcs.size()), - raft::device_span(edgelist_dsts.data(), edgelist_dsts.size())), - "Invalid input arguments: graph_properties.is_multigraph is false but the input edge list " - "has parallel edges."); - } - } - - // 1. groupby edges to their target local adjacency matrix partition (and further groupby within - // the local partition by applying the compute_gpu_id_from_vertex_t to minor vertex IDs). - - auto d_edge_counts = cugraph::detail::groupby_and_count_edgelist_by_local_partition_id( - handle, - store_transposed ? edgelist_dsts : edgelist_srcs, - store_transposed ? edgelist_srcs : edgelist_dsts, - edgelist_weights, - edgelist_edge_ids, - edgelist_edge_types, - true); - - std::vector h_edge_counts(d_edge_counts.size()); - raft::update_host( - h_edge_counts.data(), d_edge_counts.data(), d_edge_counts.size(), handle.get_stream()); - handle.sync_stream(); + // 1. renumber std::vector edgelist_edge_counts(minor_comm_size, edge_t{0}); - auto edgelist_intra_partition_segment_offsets = - std::make_optional>>( - minor_comm_size, std::vector(major_comm_size + 1, edge_t{0})); - for (int i = 0; i < minor_comm_size; ++i) { - edgelist_edge_counts[i] = std::accumulate(h_edge_counts.begin() + major_comm_size * i, - h_edge_counts.begin() + major_comm_size * (i + 1), - edge_t{0}); - std::partial_sum(h_edge_counts.begin() + major_comm_size * i, - h_edge_counts.begin() + major_comm_size * (i + 1), - (*edgelist_intra_partition_segment_offsets)[i].begin() + 1); - } - std::vector edgelist_displacements(minor_comm_size, edge_t{0}); - std::partial_sum(edgelist_edge_counts.begin(), - edgelist_edge_counts.end() - 1, - edgelist_displacements.begin() + 1); - - // 2. split the input edges to local partitions - - std::vector> edge_partition_edgelist_srcs{}; - edge_partition_edgelist_srcs.reserve(minor_comm_size); - for (int i = 0; i < minor_comm_size; ++i) { - rmm::device_uvector tmp_srcs(edgelist_edge_counts[i], handle.get_stream()); - thrust::copy(handle.get_thrust_policy(), - edgelist_srcs.begin() + edgelist_displacements[i], - edgelist_srcs.begin() + edgelist_displacements[i] + edgelist_edge_counts[i], - tmp_srcs.begin()); - edge_partition_edgelist_srcs.push_back(std::move(tmp_srcs)); - } - edgelist_srcs.resize(0, handle.get_stream()); - edgelist_srcs.shrink_to_fit(handle.get_stream()); - - std::vector> edge_partition_edgelist_dsts{}; - edge_partition_edgelist_dsts.reserve(minor_comm_size); - for (int i = 0; i < minor_comm_size; ++i) { - rmm::device_uvector tmp_dsts(edgelist_edge_counts[i], handle.get_stream()); - thrust::copy(handle.get_thrust_policy(), - edgelist_dsts.begin() + edgelist_displacements[i], - edgelist_dsts.begin() + edgelist_displacements[i] + edgelist_edge_counts[i], - tmp_dsts.begin()); - edge_partition_edgelist_dsts.push_back(std::move(tmp_dsts)); - } - edgelist_dsts.resize(0, handle.get_stream()); - edgelist_dsts.shrink_to_fit(handle.get_stream()); - - std::optional>> edge_partition_edgelist_weights{}; - if (edgelist_weights) { - edge_partition_edgelist_weights = std::vector>{}; - (*edge_partition_edgelist_weights).reserve(minor_comm_size); - for (int i = 0; i < minor_comm_size; ++i) { - rmm::device_uvector tmp_weights(edgelist_edge_counts[i], handle.get_stream()); - thrust::copy( - handle.get_thrust_policy(), - (*edgelist_weights).begin() + edgelist_displacements[i], - (*edgelist_weights).begin() + edgelist_displacements[i] + edgelist_edge_counts[i], - tmp_weights.begin()); - (*edge_partition_edgelist_weights).push_back(std::move(tmp_weights)); - } - (*edgelist_weights).resize(0, handle.get_stream()); - (*edgelist_weights).shrink_to_fit(handle.get_stream()); - } - - std::optional>> edge_partition_edgelist_edge_ids{}; - if (edgelist_edge_ids) { - edge_partition_edgelist_edge_ids = std::vector>{}; - (*edge_partition_edgelist_edge_ids).reserve(minor_comm_size); - for (int i = 0; i < minor_comm_size; ++i) { - rmm::device_uvector tmp_edge_ids(edgelist_edge_counts[i], handle.get_stream()); - thrust::copy( - handle.get_thrust_policy(), - (*edgelist_edge_ids).begin() + edgelist_displacements[i], - (*edgelist_edge_ids).begin() + edgelist_displacements[i] + edgelist_edge_counts[i], - tmp_edge_ids.begin()); - (*edge_partition_edgelist_edge_ids).push_back(std::move(tmp_edge_ids)); - } - (*edgelist_edge_ids).resize(0, handle.get_stream()); - (*edgelist_edge_ids).shrink_to_fit(handle.get_stream()); - } - - std::optional>> edge_partition_edgelist_edge_types{}; - if (edgelist_edge_types) { - edge_partition_edgelist_edge_types = std::vector>{}; - (*edge_partition_edgelist_edge_types).reserve(minor_comm_size); - for (int i = 0; i < minor_comm_size; ++i) { - rmm::device_uvector tmp_edge_types(edgelist_edge_counts[i], handle.get_stream()); - thrust::copy( - handle.get_thrust_policy(), - (*edgelist_edge_types).begin() + edgelist_displacements[i], - (*edgelist_edge_types).begin() + edgelist_displacements[i] + edgelist_edge_counts[i], - tmp_edge_types.begin()); - (*edge_partition_edgelist_edge_types).push_back(std::move(tmp_edge_types)); - } - (*edgelist_edge_types).resize(0, handle.get_stream()); - (*edgelist_edge_types).shrink_to_fit(handle.get_stream()); + for (size_t i = 0; i < edgelist_edge_counts.size(); ++i) { + edgelist_edge_counts[i] = static_cast(edge_partition_edgelist_srcs[i].size()); } - // 3. renumber - std::vector src_ptrs(minor_comm_size); std::vector dst_ptrs(src_ptrs.size()); for (int i = 0; i < minor_comm_size; ++i) { src_ptrs[i] = edge_partition_edgelist_srcs[i].begin(); dst_ptrs[i] = edge_partition_edgelist_dsts[i].begin(); } - auto [renumber_map_labels, meta] = cugraph::renumber_edgelist( - handle, - std::move(local_vertices), - src_ptrs, - dst_ptrs, - edgelist_edge_counts, - edgelist_intra_partition_segment_offsets, - store_transposed); + auto [renumber_map_labels, meta] = + cugraph::renumber_edgelist(handle, + std::move(local_vertices), + src_ptrs, + dst_ptrs, + edgelist_edge_counts, + edgelist_intra_partition_segment_offsets, + store_transposed); auto num_segments_per_vertex_partition = static_cast(meta.edge_partition_segment_offsets.size() / minor_comm_size); auto use_dcs = num_segments_per_vertex_partition > (detail::num_sparse_segments_per_vertex_partition + 2); - // 4. sort and compress edge list (COO) to CSR (or CSC) or CSR + DCSR (CSC + DCSC) hybrid + // 2. sort and compress edge list (COO) to CSR (or CSC) or CSR + DCSR (CSC + DCSC) hybrid auto total_global_mem = handle.get_device_properties().totalGlobalMem; size_t element_size = sizeof(vertex_t) * 2; - if (edgelist_weights) { element_size += sizeof(weight_t); } - if (edgelist_edge_ids) { element_size += sizeof(edge_id_t); } - if (edgelist_edge_types) { element_size += sizeof(edge_type_t); } + if (edge_partition_edgelist_weights) { element_size += sizeof(weight_t); } + if (edge_partition_edgelist_edge_ids) { element_size += sizeof(edge_id_t); } + if (edge_partition_edgelist_edge_types) { element_size += sizeof(edge_type_t); } auto constexpr mem_frugal_ratio = 0.25; // if the expected temporary buffer size exceeds the mem_frugal_ratio of the // total_global_mem, switch to the memory frugal approach @@ -567,9 +418,9 @@ create_graph_from_edgelist_impl( meta.edge_partition_segment_offsets[num_segments_per_vertex_partition * i + detail::num_sparse_segments_per_vertex_partition]) : std::nullopt; - if (edgelist_weights) { - if (edgelist_edge_ids) { - if (edgelist_edge_types) { + if (edge_partition_edgelist_weights) { + if (edge_partition_edgelist_edge_ids) { + if (edge_partition_edgelist_edge_types) { std::forward_as_tuple( offsets, indices, std::tie(weights, edge_ids, edge_types), dcs_nzd_vertices) = detail::sort_and_compress_edgelist( std::move(edge_partition_edgelist_srcs[i]), @@ -715,7 +566,7 @@ create_graph_from_edgelist_impl( } } - // 5. segmented sort neighbors + // 3. segmented sort neighbors for (size_t i = 0; i < edge_partition_offsets.size(); ++i) { if (edge_partition_weights) { @@ -801,46 +652,43 @@ create_graph_from_edgelist_impl( } } - // 6. create a graph and an edge_property_t object. + // 4. create a graph and an edge_property_t object. - std::optional< - edge_property_t, weight_t>> + std::optional, weight_t>> edge_weights{std::nullopt}; if (edge_partition_weights) { edge_weights = - edge_property_t, weight_t>( + edge_property_t, weight_t>( std::move(*edge_partition_weights)); } - std::optional< - edge_property_t, edge_id_t>> + std::optional, edge_id_t>> edge_ids{std::nullopt}; if (edge_partition_edge_ids) { - edge_ids = - edge_property_t, edge_id_t>( - std::move(*edge_partition_edge_ids)); + edge_ids = edge_property_t, edge_id_t>( + std::move(*edge_partition_edge_ids)); } std::optional< - edge_property_t, edge_type_t>> + edge_property_t, edge_type_t>> edge_types{std::nullopt}; if (edge_partition_edge_types) { edge_types = - edge_property_t, edge_type_t>( + edge_property_t, edge_type_t>( std::move(*edge_partition_edge_types)); } return std::make_tuple( - cugraph::graph_t( + cugraph::graph_t( handle, std::move(edge_partition_offsets), std::move(edge_partition_indices), std::move(edge_partition_dcs_nzd_vertices), - cugraph::graph_meta_t{meta.number_of_vertices, - meta.number_of_edges, - graph_properties, - meta.partition, - meta.edge_partition_segment_offsets}), + cugraph::graph_meta_t{meta.number_of_vertices, + meta.number_of_edges, + graph_properties, + meta.partition, + meta.edge_partition_segment_offsets}), std::move(edge_weights), std::move(edge_ids), std::move(edge_types), @@ -855,7 +703,7 @@ template std::enable_if_t< - !multi_gpu, + multi_gpu, std::tuple< cugraph::graph_t, std::optional< @@ -867,7 +715,7 @@ std::enable_if_t< std::optional>>> create_graph_from_edgelist_impl( raft::handle_t const& handle, - std::optional>&& vertices, + std::optional>&& local_vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -877,24 +725,28 @@ create_graph_from_edgelist_impl( bool renumber, bool do_expensive_check) { - CUGRAPH_EXPECTS( - !vertices || ((*vertices).size() < static_cast(std::numeric_limits::max())), - "Invalid input arguments: # unique vertex IDs should be smaller than " - "std::numeric_limits::Max()."); + auto& major_comm = handle.get_subcomm(cugraph::partition_manager::major_comm_name()); + auto const major_comm_size = major_comm.get_size(); + auto& minor_comm = handle.get_subcomm(cugraph::partition_manager::minor_comm_name()); + auto const minor_comm_size = minor_comm.get_size(); + CUGRAPH_EXPECTS(edgelist_srcs.size() == edgelist_dsts.size(), "Invalid input arguments: edgelist_srcs.size() != edgelist_dsts.size()."); CUGRAPH_EXPECTS(!edgelist_weights || (edgelist_srcs.size() == (*edgelist_weights).size()), - "Invalid input arguments: edgelist_srcs.size() != edgelist_weights.size()."); + "Invalid input arguments: edgelist_weights.has_value() is true and " + "edgelist_srcs.size() != (*edgelist_weights).size()."); CUGRAPH_EXPECTS(!edgelist_edge_ids || (edgelist_srcs.size() == (*edgelist_edge_ids).size()), - "Invalid input arguments: edgelist_srcs.size() != " - "(*edgelist_edge_ids).size()."); + "Invalid input arguments: edgelist_edge_ids.has_value() is true and " + "edgelist_srcs.size() != (*edgelist_edge_ids).size()."); CUGRAPH_EXPECTS(!edgelist_edge_types || (edgelist_srcs.size() == (*edgelist_edge_types).size()), - "Invalid input arguments: edgelist_srcs.size() != " - "(*edgelist_edge_types).size()."); + "Invalid input arguments: edgelist_edge_types.has_value() is true, " + "edgelist_srcs.size() != (*edgelist_edge_types).size()."); + CUGRAPH_EXPECTS(renumber, + "Invalid input arguments: renumber should be true if multi_gpu is true."); if (do_expensive_check) { expensive_check_edgelist(handle, - vertices, + local_vertices, store_transposed ? edgelist_dsts : edgelist_srcs, store_transposed ? edgelist_srcs : edgelist_dsts, renumber); @@ -920,50 +772,660 @@ create_graph_from_edgelist_impl( } } - // renumber + // 1. groupby edges to their target local adjacency matrix partition (and further groupby within + // the local partition by applying the compute_gpu_id_from_vertex_t to minor vertex IDs). - auto renumber_map_labels = - renumber ? std::make_optional>(0, handle.get_stream()) - : std::nullopt; - renumber_meta_t meta{}; - if (renumber) { - std::tie(*renumber_map_labels, meta) = cugraph::renumber_edgelist( - handle, - std::move(vertices), - edgelist_srcs.data(), - edgelist_dsts.data(), - static_cast(edgelist_srcs.size()), - store_transposed); - } + auto d_edge_counts = cugraph::detail::groupby_and_count_edgelist_by_local_partition_id( + handle, + store_transposed ? edgelist_dsts : edgelist_srcs, + store_transposed ? edgelist_srcs : edgelist_dsts, + edgelist_weights, + edgelist_edge_ids, + edgelist_edge_types, + true); - vertex_t num_vertices{}; - if (renumber) { - num_vertices = static_cast((*renumber_map_labels).size()); - } else { - if (vertices) { - num_vertices = (*vertices).size(); - } else { - num_vertices = 1 + cugraph::detail::compute_maximum_vertex_id( - handle.get_stream(), edgelist_srcs, edgelist_dsts); - } - } + std::vector h_edge_counts(d_edge_counts.size()); + raft::update_host( + h_edge_counts.data(), d_edge_counts.data(), d_edge_counts.size(), handle.get_stream()); + handle.sync_stream(); - // convert edge list (COO) to compressed sparse format (CSR or CSC) + std::vector edgelist_edge_counts(minor_comm_size, edge_t{0}); + auto edgelist_intra_partition_segment_offsets = std::vector>( + minor_comm_size, std::vector(major_comm_size + 1, edge_t{0})); + for (int i = 0; i < minor_comm_size; ++i) { + edgelist_edge_counts[i] = std::accumulate(h_edge_counts.begin() + major_comm_size * i, + h_edge_counts.begin() + major_comm_size * (i + 1), + edge_t{0}); + std::partial_sum(h_edge_counts.begin() + major_comm_size * i, + h_edge_counts.begin() + major_comm_size * (i + 1), + edgelist_intra_partition_segment_offsets[i].begin() + 1); + } + std::vector edgelist_displacements(minor_comm_size, edge_t{0}); + std::partial_sum(edgelist_edge_counts.begin(), + edgelist_edge_counts.end() - 1, + edgelist_displacements.begin() + 1); - auto total_global_mem = handle.get_device_properties().totalGlobalMem; - size_t element_size = sizeof(vertex_t) * 2; - if (edgelist_weights) { element_size += sizeof(weight_t); } - if (edgelist_edge_ids) { element_size += sizeof(edge_id_t); } - if (edgelist_edge_types) { element_size += sizeof(edge_type_t); } - auto constexpr mem_frugal_ratio = - 0.25; // if the expected temporary buffer size exceeds the mem_frugal_ratio of the - // total_global_mem, switch to the memory frugal approach - auto mem_frugal_threshold = - static_cast(static_cast(total_global_mem / element_size) * mem_frugal_ratio); + // 2. split the input edges to local partitions - rmm::device_uvector offsets(size_t{0}, handle.get_stream()); - rmm::device_uvector indices(size_t{0}, handle.get_stream()); - std::optional> weights{std::nullopt}; + std::vector> edge_partition_edgelist_srcs{}; + edge_partition_edgelist_srcs.reserve(minor_comm_size); + for (int i = 0; i < minor_comm_size; ++i) { + rmm::device_uvector tmp_srcs(edgelist_edge_counts[i], handle.get_stream()); + thrust::copy(handle.get_thrust_policy(), + edgelist_srcs.begin() + edgelist_displacements[i], + edgelist_srcs.begin() + edgelist_displacements[i] + edgelist_edge_counts[i], + tmp_srcs.begin()); + edge_partition_edgelist_srcs.push_back(std::move(tmp_srcs)); + } + edgelist_srcs.resize(0, handle.get_stream()); + edgelist_srcs.shrink_to_fit(handle.get_stream()); + + std::vector> edge_partition_edgelist_dsts{}; + edge_partition_edgelist_dsts.reserve(minor_comm_size); + for (int i = 0; i < minor_comm_size; ++i) { + rmm::device_uvector tmp_dsts(edgelist_edge_counts[i], handle.get_stream()); + thrust::copy(handle.get_thrust_policy(), + edgelist_dsts.begin() + edgelist_displacements[i], + edgelist_dsts.begin() + edgelist_displacements[i] + edgelist_edge_counts[i], + tmp_dsts.begin()); + edge_partition_edgelist_dsts.push_back(std::move(tmp_dsts)); + } + edgelist_dsts.resize(0, handle.get_stream()); + edgelist_dsts.shrink_to_fit(handle.get_stream()); + + std::optional>> edge_partition_edgelist_weights{}; + if (edgelist_weights) { + edge_partition_edgelist_weights = std::vector>{}; + (*edge_partition_edgelist_weights).reserve(minor_comm_size); + for (int i = 0; i < minor_comm_size; ++i) { + rmm::device_uvector tmp_weights(edgelist_edge_counts[i], handle.get_stream()); + thrust::copy( + handle.get_thrust_policy(), + (*edgelist_weights).begin() + edgelist_displacements[i], + (*edgelist_weights).begin() + edgelist_displacements[i] + edgelist_edge_counts[i], + tmp_weights.begin()); + (*edge_partition_edgelist_weights).push_back(std::move(tmp_weights)); + } + (*edgelist_weights).resize(0, handle.get_stream()); + (*edgelist_weights).shrink_to_fit(handle.get_stream()); + } + + std::optional>> edge_partition_edgelist_edge_ids{}; + if (edgelist_edge_ids) { + edge_partition_edgelist_edge_ids = std::vector>{}; + (*edge_partition_edgelist_edge_ids).reserve(minor_comm_size); + for (int i = 0; i < minor_comm_size; ++i) { + rmm::device_uvector tmp_edge_ids(edgelist_edge_counts[i], handle.get_stream()); + thrust::copy( + handle.get_thrust_policy(), + (*edgelist_edge_ids).begin() + edgelist_displacements[i], + (*edgelist_edge_ids).begin() + edgelist_displacements[i] + edgelist_edge_counts[i], + tmp_edge_ids.begin()); + (*edge_partition_edgelist_edge_ids).push_back(std::move(tmp_edge_ids)); + } + (*edgelist_edge_ids).resize(0, handle.get_stream()); + (*edgelist_edge_ids).shrink_to_fit(handle.get_stream()); + } + + std::optional>> edge_partition_edgelist_edge_types{}; + if (edgelist_edge_types) { + edge_partition_edgelist_edge_types = std::vector>{}; + (*edge_partition_edgelist_edge_types).reserve(minor_comm_size); + for (int i = 0; i < minor_comm_size; ++i) { + rmm::device_uvector tmp_edge_types(edgelist_edge_counts[i], handle.get_stream()); + thrust::copy( + handle.get_thrust_policy(), + (*edgelist_edge_types).begin() + edgelist_displacements[i], + (*edgelist_edge_types).begin() + edgelist_displacements[i] + edgelist_edge_counts[i], + tmp_edge_types.begin()); + (*edge_partition_edgelist_edge_types).push_back(std::move(tmp_edge_types)); + } + (*edgelist_edge_types).resize(0, handle.get_stream()); + (*edgelist_edge_types).shrink_to_fit(handle.get_stream()); + } + + return create_graph_from_partitioned_edgelist( + handle, + std::move(local_vertices), + std::move(edge_partition_edgelist_srcs), + std::move(edge_partition_edgelist_dsts), + std::move(edge_partition_edgelist_weights), + std::move(edge_partition_edgelist_edge_ids), + std::move(edge_partition_edgelist_edge_types), + edgelist_intra_partition_segment_offsets, + graph_properties, + renumber); +} + +template +std::enable_if_t< + multi_gpu, + std::tuple< + cugraph::graph_t, + std::optional< + edge_property_t, weight_t>>, + std::optional< + edge_property_t, edge_id_t>>, + std::optional< + edge_property_t, edge_type_t>>, + std::optional>>> +create_graph_from_edgelist_impl( + raft::handle_t const& handle, + std::optional>&& local_vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check) +{ + auto& major_comm = handle.get_subcomm(cugraph::partition_manager::major_comm_name()); + auto const major_comm_size = major_comm.get_size(); + auto& minor_comm = handle.get_subcomm(cugraph::partition_manager::minor_comm_name()); + auto const minor_comm_size = minor_comm.get_size(); + + CUGRAPH_EXPECTS(edgelist_srcs.size() == edgelist_dsts.size(), + "Invalid input arguments: edgelist_srcs.size() != edgelist_dsts.size()."); + CUGRAPH_EXPECTS(!edgelist_weights || (edgelist_srcs.size() == (*edgelist_weights).size()), + "Invalid input arguments: edgelist_weights.has_value() is true and " + "edgelist_srcs.size() != (*edgelist_weights).size()."); + CUGRAPH_EXPECTS(!edgelist_edge_ids || (edgelist_srcs.size() == (*edgelist_edge_ids).size()), + "Invalid input arguments: edgelist_edge_ids.has_value() is true and " + "edgelist_srcs.size() != (*edgelist_edge_ids).size()."); + CUGRAPH_EXPECTS(!edgelist_edge_types || (edgelist_srcs.size() == (*edgelist_edge_types).size()), + "Invalid input arguments: edgelist_edge_types.has_value() is true, " + "edgelist_srcs.size() != (*edgelist_edge_types).size()."); + for (size_t i = 0; i < edgelist_srcs.size(); ++i) { + CUGRAPH_EXPECTS(edgelist_srcs[i].size() == edgelist_dsts[i].size(), + "Invalid input arguments: edgelist_srcs[i].size() != edgelist_dsts[i].size()."); + CUGRAPH_EXPECTS(!edgelist_weights || (edgelist_srcs[i].size() == (*edgelist_weights)[i].size()), + "Invalid input arguments: edgelist_weights.has_value() is true and " + "edgelist_srcs[i].size() != (*edgelist_weights)[i].size()."); + CUGRAPH_EXPECTS( + !edgelist_edge_ids || (edgelist_srcs[i].size() == (*edgelist_edge_ids)[i].size()), + "Invalid input arguments: edgelist_edge_ids.has_value() is true and " + "edgelist_srcs[i].size() != (*edgelist_edge_ids)[i].size()."); + CUGRAPH_EXPECTS( + !edgelist_edge_types || (edgelist_srcs[i].size() == (*edgelist_edge_types)[i].size()), + "Invalid input arguments: edgelist_edge_types.has_value() is true, " + "edgelist_srcs[i].size() != (*edgelist_edge_types)[i].size()."); + } + CUGRAPH_EXPECTS(renumber, + "Invalid input arguments: renumber should be true if multi_gpu is true."); + + if (do_expensive_check) { + edge_t aggregate_edge_count{0}; + for (size_t i = 0; i < edgelist_srcs.size(); ++i) { + aggregate_edge_count += edgelist_srcs[i].size(); + } + + rmm::device_uvector aggregate_edgelist_srcs(aggregate_edge_count, + handle.get_stream()); + rmm::device_uvector aggregate_edgelist_dsts(aggregate_edge_count, + handle.get_stream()); + edge_t output_offset{0}; + for (size_t i = 0; i < edgelist_srcs.size(); ++i) { + thrust::copy(handle.get_thrust_policy(), + edgelist_srcs[i].begin(), + edgelist_srcs[i].end(), + aggregate_edgelist_srcs.begin() + output_offset); + thrust::copy(handle.get_thrust_policy(), + edgelist_dsts[i].begin(), + edgelist_dsts[i].end(), + aggregate_edgelist_dsts.begin() + output_offset); + output_offset += edgelist_srcs[i].size(); + } + + expensive_check_edgelist( + handle, + local_vertices, + store_transposed ? aggregate_edgelist_dsts : aggregate_edgelist_srcs, + store_transposed ? aggregate_edgelist_srcs : aggregate_edgelist_dsts, + renumber); + + if (graph_properties.is_symmetric) { + CUGRAPH_EXPECTS( + (check_symmetric( + handle, + raft::device_span(aggregate_edgelist_srcs.data(), + aggregate_edgelist_srcs.size()), + raft::device_span(aggregate_edgelist_dsts.data(), + aggregate_edgelist_dsts.size()))), + "Invalid input arguments: graph_properties.is_symmetric is true but the input edge list is " + "not symmetric."); + } + + if (!graph_properties.is_multigraph) { + CUGRAPH_EXPECTS( + check_no_parallel_edge(handle, + raft::device_span(aggregate_edgelist_srcs.data(), + aggregate_edgelist_srcs.size()), + raft::device_span(aggregate_edgelist_dsts.data(), + aggregate_edgelist_dsts.size())), + "Invalid input arguments: graph_properties.is_multigraph is false but the input edge list " + "has parallel edges."); + } + } + + // 1. groupby each edge chunks to their target local adjacency matrix partition (and further + // groupby within the local partition by applying the compute_gpu_id_from_vertex_t to minor vertex + // IDs). + + std::vector>> edgelist_partitioned_srcs( + edgelist_srcs.size()); + std::vector>> edgelist_partitioned_dsts( + edgelist_srcs.size()); + auto edgelist_partitioned_weights = + edgelist_weights ? std::make_optional>>>( + edgelist_srcs.size()) + : std::nullopt; + auto edgelist_partitioned_edge_ids = + edgelist_edge_ids + ? std::make_optional>>>( + edgelist_srcs.size()) + : std::nullopt; + auto edgelist_partitioned_edge_types = + edgelist_edge_types + ? std::make_optional>>>( + edgelist_srcs.size()) + : std::nullopt; + + for (size_t i = 0; i < edgelist_srcs.size(); ++i) { // iterate over input edge chunks + std::optional> this_chunk_weights{std::nullopt}; + if (edgelist_weights) { this_chunk_weights = std::move((*edgelist_weights)[i]); } + std::optional> this_chunk_edge_ids{std::nullopt}; + if (edgelist_edge_ids) { this_chunk_edge_ids = std::move((*edgelist_edge_ids)[i]); } + std::optional> this_chunk_edge_types{std::nullopt}; + if (edgelist_edge_types) { this_chunk_edge_types = std::move((*edgelist_edge_types)[i]); } + auto d_this_chunk_edge_counts = + cugraph::detail::groupby_and_count_edgelist_by_local_partition_id( + handle, + store_transposed ? edgelist_dsts[i] : edgelist_srcs[i], + store_transposed ? edgelist_srcs[i] : edgelist_dsts[i], + this_chunk_weights, + this_chunk_edge_ids, + this_chunk_edge_types, + true); + + std::vector h_this_chunk_edge_counts(d_this_chunk_edge_counts.size()); + raft::update_host(h_this_chunk_edge_counts.data(), + d_this_chunk_edge_counts.data(), + d_this_chunk_edge_counts.size(), + handle.get_stream()); + handle.sync_stream(); + std::vector h_this_chunk_edge_displacements(h_this_chunk_edge_counts.size()); + std::exclusive_scan(h_this_chunk_edge_counts.begin(), + h_this_chunk_edge_counts.end(), + h_this_chunk_edge_displacements.begin(), + size_t{0}); + + for (int j = 0; j < minor_comm_size /* # local edge partitions */ * + major_comm_size /* # segments in the local minor range */; + ++j) { + rmm::device_uvector tmp_srcs(h_this_chunk_edge_counts[j], handle.get_stream()); + auto input_first = edgelist_srcs[i].begin() + h_this_chunk_edge_displacements[j]; + thrust::copy( + handle.get_thrust_policy(), input_first, input_first + tmp_srcs.size(), tmp_srcs.begin()); + edgelist_partitioned_srcs[i].push_back(std::move(tmp_srcs)); + } + edgelist_srcs[i].resize(0, handle.get_stream()); + edgelist_srcs[i].shrink_to_fit(handle.get_stream()); + + for (int j = 0; j < minor_comm_size /* # local edge partitions */ * + major_comm_size /* # segments in the local minor range */; + ++j) { + rmm::device_uvector tmp_dsts(h_this_chunk_edge_counts[j], handle.get_stream()); + auto input_first = edgelist_dsts[i].begin() + h_this_chunk_edge_displacements[j]; + thrust::copy( + handle.get_thrust_policy(), input_first, input_first + tmp_dsts.size(), tmp_dsts.begin()); + edgelist_partitioned_dsts[i].push_back(std::move(tmp_dsts)); + } + edgelist_dsts[i].resize(0, handle.get_stream()); + edgelist_dsts[i].shrink_to_fit(handle.get_stream()); + + if (this_chunk_weights) { + for (int j = 0; j < minor_comm_size /* # local edge partitions */ * + major_comm_size /* # segments in the local minor range */; + ++j) { + rmm::device_uvector tmp_weights(h_this_chunk_edge_counts[j], handle.get_stream()); + auto input_first = (*this_chunk_weights).begin() + h_this_chunk_edge_displacements[j]; + thrust::copy(handle.get_thrust_policy(), + input_first, + input_first + tmp_weights.size(), + tmp_weights.begin()); + (*edgelist_partitioned_weights)[i].push_back(std::move(tmp_weights)); + } + (*this_chunk_weights).resize(0, handle.get_stream()); + (*this_chunk_weights).shrink_to_fit(handle.get_stream()); + } + + if (this_chunk_edge_ids) { + for (int j = 0; j < minor_comm_size /* # local edge partitions */ * + major_comm_size /* # segments in the local minor range */; + ++j) { + rmm::device_uvector tmp_edge_ids(h_this_chunk_edge_counts[j], + handle.get_stream()); + auto input_first = (*this_chunk_edge_ids).begin() + h_this_chunk_edge_displacements[j]; + thrust::copy(handle.get_thrust_policy(), + input_first, + input_first + tmp_edge_ids.size(), + tmp_edge_ids.begin()); + (*edgelist_partitioned_edge_ids)[i].push_back(std::move(tmp_edge_ids)); + } + (*this_chunk_edge_ids).resize(0, handle.get_stream()); + (*this_chunk_edge_ids).shrink_to_fit(handle.get_stream()); + } + + if (this_chunk_edge_types) { + for (int j = 0; j < minor_comm_size /* # local edge partitions */ * + major_comm_size /* # segments in the local minor range */; + ++j) { + rmm::device_uvector tmp_edge_types(h_this_chunk_edge_counts[j], + handle.get_stream()); + auto input_first = (*this_chunk_edge_types).begin() + h_this_chunk_edge_displacements[j]; + thrust::copy(handle.get_thrust_policy(), + input_first, + input_first + tmp_edge_types.size(), + tmp_edge_types.begin()); + (*edgelist_partitioned_edge_types)[i].push_back(std::move(tmp_edge_types)); + } + (*this_chunk_edge_types).resize(0, handle.get_stream()); + (*this_chunk_edge_types).shrink_to_fit(handle.get_stream()); + } + } + edgelist_srcs.clear(); + edgelist_dsts.clear(); + if (edgelist_weights) { (*edgelist_weights).clear(); } + if (edgelist_edge_ids) { (*edgelist_edge_ids).clear(); } + if (edgelist_edge_types) { (*edgelist_edge_types).clear(); } + + // 2. split the grouped edge chunks to local partitions + + auto edgelist_intra_partition_segment_offsets = std::vector>(minor_comm_size); + + std::vector> edge_partition_edgelist_srcs{}; + edge_partition_edgelist_srcs.reserve(minor_comm_size); + std::vector> edge_partition_edgelist_dsts{}; + edge_partition_edgelist_dsts.reserve(minor_comm_size); + auto edge_partition_edgelist_weights = + edgelist_partitioned_weights ? std::make_optional>>() + : std::nullopt; + if (edgelist_partitioned_weights) { (*edge_partition_edgelist_weights).reserve(minor_comm_size); } + auto edge_partition_edgelist_edge_ids = + edgelist_partitioned_edge_ids + ? std::make_optional>>() + : std::nullopt; + if (edgelist_partitioned_edge_ids) { + (*edge_partition_edgelist_edge_ids).reserve(minor_comm_size); + } + auto edge_partition_edgelist_edge_types = + edgelist_partitioned_edge_types + ? std::make_optional>>() + : std::nullopt; + if (edgelist_partitioned_edge_types) { + (*edge_partition_edgelist_edge_types).reserve(minor_comm_size); + } + + for (int i = 0; i < minor_comm_size; ++i) { // iterate over local edge partitions + edge_t edge_count{0}; + std::vector intra_partition_segment_sizes(major_comm_size, 0); + std::vector intra_segment_copy_output_displacements(major_comm_size * + edgelist_partitioned_srcs.size()); + for (int j = 0; j < major_comm_size /* # segments in the local minor range */; ++j) { + edge_t displacement{0}; + for (size_t k = 0; k < edgelist_partitioned_srcs.size() /* # input edge chunks */; ++k) { + auto segment_size = edgelist_partitioned_srcs[k][i * major_comm_size + j].size(); + edge_count += segment_size; + intra_partition_segment_sizes[j] += segment_size; + intra_segment_copy_output_displacements[j * edgelist_partitioned_srcs.size() + k] = + displacement; + displacement += segment_size; + } + } + std::vector intra_partition_segment_offsets(major_comm_size + 1, 0); + std::inclusive_scan(intra_partition_segment_sizes.begin(), + intra_partition_segment_sizes.end(), + intra_partition_segment_offsets.begin() + 1); + + rmm::device_uvector tmp_srcs(edge_count, handle.get_stream()); + for (int j = 0; j < major_comm_size; ++j) { + for (size_t k = 0; k < edgelist_partitioned_srcs.size(); ++k) { + auto& input_buffer = edgelist_partitioned_srcs[k][i * major_comm_size + j]; + thrust::copy( + handle.get_thrust_policy(), + input_buffer.begin(), + input_buffer.end(), + tmp_srcs.begin() + intra_partition_segment_offsets[j] + + intra_segment_copy_output_displacements[j * edgelist_partitioned_srcs.size() + k]); + input_buffer.resize(0, handle.get_stream()); + input_buffer.shrink_to_fit(handle.get_stream()); + } + } + edge_partition_edgelist_srcs.push_back(std::move(tmp_srcs)); + + rmm::device_uvector tmp_dsts(edge_count, handle.get_stream()); + for (int j = 0; j < major_comm_size; ++j) { + for (size_t k = 0; k < edgelist_partitioned_dsts.size(); ++k) { + auto& input_buffer = edgelist_partitioned_dsts[k][i * major_comm_size + j]; + thrust::copy( + handle.get_thrust_policy(), + input_buffer.begin(), + input_buffer.end(), + tmp_dsts.begin() + intra_partition_segment_offsets[j] + + intra_segment_copy_output_displacements[j * edgelist_partitioned_dsts.size() + k]); + input_buffer.resize(0, handle.get_stream()); + input_buffer.shrink_to_fit(handle.get_stream()); + } + } + edge_partition_edgelist_dsts.push_back(std::move(tmp_dsts)); + + if (edge_partition_edgelist_weights) { + rmm::device_uvector tmp_weights(edge_count, handle.get_stream()); + for (int j = 0; j < major_comm_size; ++j) { + for (size_t k = 0; k < edgelist_partitioned_dsts.size(); ++k) { + auto& input_buffer = (*edgelist_partitioned_weights)[k][i * major_comm_size + j]; + thrust::copy( + handle.get_thrust_policy(), + input_buffer.begin(), + input_buffer.end(), + tmp_weights.begin() + intra_partition_segment_offsets[j] + + intra_segment_copy_output_displacements[j * edgelist_partitioned_dsts.size() + k]); + input_buffer.resize(0, handle.get_stream()); + input_buffer.shrink_to_fit(handle.get_stream()); + } + } + (*edge_partition_edgelist_weights).push_back(std::move(tmp_weights)); + } + + if (edge_partition_edgelist_edge_ids) { + rmm::device_uvector tmp_edge_ids(edge_count, handle.get_stream()); + for (int j = 0; j < major_comm_size; ++j) { + for (size_t k = 0; k < edgelist_partitioned_dsts.size(); ++k) { + auto& input_buffer = (*edgelist_partitioned_edge_ids)[k][i * major_comm_size + j]; + thrust::copy( + handle.get_thrust_policy(), + input_buffer.begin(), + input_buffer.end(), + tmp_edge_ids.begin() + intra_partition_segment_offsets[j] + + intra_segment_copy_output_displacements[j * edgelist_partitioned_dsts.size() + k]); + input_buffer.resize(0, handle.get_stream()); + input_buffer.shrink_to_fit(handle.get_stream()); + } + } + (*edge_partition_edgelist_edge_ids).push_back(std::move(tmp_edge_ids)); + } + + if (edge_partition_edgelist_edge_types) { + rmm::device_uvector tmp_edge_types(edge_count, handle.get_stream()); + for (int j = 0; j < major_comm_size; ++j) { + for (size_t k = 0; k < edgelist_partitioned_dsts.size(); ++k) { + auto& input_buffer = (*edgelist_partitioned_edge_types)[k][i * major_comm_size + j]; + thrust::copy( + handle.get_thrust_policy(), + input_buffer.begin(), + input_buffer.end(), + tmp_edge_types.begin() + intra_partition_segment_offsets[j] + + intra_segment_copy_output_displacements[j * edgelist_partitioned_dsts.size() + k]); + input_buffer.resize(0, handle.get_stream()); + input_buffer.shrink_to_fit(handle.get_stream()); + } + } + (*edge_partition_edgelist_edge_types).push_back(std::move(tmp_edge_types)); + } + + edgelist_intra_partition_segment_offsets[i] = std::move(intra_partition_segment_offsets); + } + + return create_graph_from_partitioned_edgelist( + handle, + std::move(local_vertices), + std::move(edge_partition_edgelist_srcs), + std::move(edge_partition_edgelist_dsts), + std::move(edge_partition_edgelist_weights), + std::move(edge_partition_edgelist_edge_ids), + std::move(edge_partition_edgelist_edge_types), + edgelist_intra_partition_segment_offsets, + graph_properties, + renumber); +} + +template +std::enable_if_t< + !multi_gpu, + std::tuple< + cugraph::graph_t, + std::optional< + edge_property_t, weight_t>>, + std::optional< + edge_property_t, edge_id_t>>, + std::optional< + edge_property_t, edge_type_t>>, + std::optional>>> +create_graph_from_edgelist_impl( + raft::handle_t const& handle, + std::optional>&& vertices, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check) +{ + CUGRAPH_EXPECTS( + !vertices || ((*vertices).size() < static_cast(std::numeric_limits::max())), + "Invalid input arguments: # unique vertex IDs should be smaller than " + "std::numeric_limits::Max()."); + CUGRAPH_EXPECTS(edgelist_srcs.size() == edgelist_dsts.size(), + "Invalid input arguments: edgelist_srcs.size() != edgelist_dsts.size()."); + CUGRAPH_EXPECTS(!edgelist_weights || (edgelist_srcs.size() == (*edgelist_weights).size()), + "Invalid input arguments: edgelist_srcs.size() != edgelist_weights.size()."); + CUGRAPH_EXPECTS(!edgelist_edge_ids || (edgelist_srcs.size() == (*edgelist_edge_ids).size()), + "Invalid input arguments: edgelist_srcs.size() != " + "(*edgelist_edge_ids).size()."); + CUGRAPH_EXPECTS(!edgelist_edge_types || (edgelist_srcs.size() == (*edgelist_edge_types).size()), + "Invalid input arguments: edgelist_srcs.size() != " + "(*edgelist_edge_types).size()."); + + if (do_expensive_check) { + expensive_check_edgelist(handle, + vertices, + store_transposed ? edgelist_dsts : edgelist_srcs, + store_transposed ? edgelist_srcs : edgelist_dsts, + renumber); + + if (graph_properties.is_symmetric) { + CUGRAPH_EXPECTS( + (check_symmetric( + handle, + raft::device_span(edgelist_srcs.data(), edgelist_srcs.size()), + raft::device_span(edgelist_dsts.data(), edgelist_dsts.size()))), + "Invalid input arguments: graph_properties.is_symmetric is true but the input edge list is " + "not symmetric."); + } + + if (!graph_properties.is_multigraph) { + CUGRAPH_EXPECTS( + check_no_parallel_edge( + handle, + raft::device_span(edgelist_srcs.data(), edgelist_srcs.size()), + raft::device_span(edgelist_dsts.data(), edgelist_dsts.size())), + "Invalid input arguments: graph_properties.is_multigraph is false but the input edge list " + "has parallel edges."); + } + } + + // 1. renumber + + auto renumber_map_labels = + renumber ? std::make_optional>(0, handle.get_stream()) + : std::nullopt; + renumber_meta_t meta{}; + if (renumber) { + std::tie(*renumber_map_labels, meta) = cugraph::renumber_edgelist( + handle, + std::move(vertices), + edgelist_srcs.data(), + edgelist_dsts.data(), + static_cast(edgelist_srcs.size()), + store_transposed); + } + + vertex_t num_vertices{}; + if (renumber) { + num_vertices = static_cast((*renumber_map_labels).size()); + } else { + if (vertices) { + num_vertices = (*vertices).size(); + } else { + num_vertices = 1 + cugraph::detail::compute_maximum_vertex_id( + handle.get_stream(), edgelist_srcs, edgelist_dsts); + } + } + + // 2. convert edge list (COO) to compressed sparse format (CSR or CSC) + + auto total_global_mem = handle.get_device_properties().totalGlobalMem; + size_t element_size = sizeof(vertex_t) * 2; + if (edgelist_weights) { element_size += sizeof(weight_t); } + if (edgelist_edge_ids) { element_size += sizeof(edge_id_t); } + if (edgelist_edge_types) { element_size += sizeof(edge_type_t); } + auto constexpr mem_frugal_ratio = + 0.25; // if the expected temporary buffer size exceeds the mem_frugal_ratio of the + // total_global_mem, switch to the memory frugal approach + auto mem_frugal_threshold = + static_cast(static_cast(total_global_mem / element_size) * mem_frugal_ratio); + + rmm::device_uvector offsets(size_t{0}, handle.get_stream()); + rmm::device_uvector indices(size_t{0}, handle.get_stream()); + std::optional> weights{std::nullopt}; std::optional> ids{std::nullopt}; std::optional> types{std::nullopt}; @@ -1098,7 +1560,7 @@ create_graph_from_edgelist_impl( } } - // create a graph and an edge_property_t object. + // 3. create a graph and an edge_property_t object. std::optional< edge_property_t, weight_t>> @@ -1133,7 +1595,7 @@ create_graph_from_edgelist_impl( std::move(buffers)); } - // graph_t constructor + // 4. graph_t constructor return std::make_tuple( cugraph::graph_t( @@ -1150,6 +1612,194 @@ create_graph_from_edgelist_impl( std::move(renumber_map_labels)); } +template +std::enable_if_t< + !multi_gpu, + std::tuple< + cugraph::graph_t, + std::optional< + edge_property_t, weight_t>>, + std::optional< + edge_property_t, edge_id_t>>, + std::optional< + edge_property_t, edge_type_t>>, + std::optional>>> +create_graph_from_edgelist_impl( + raft::handle_t const& handle, + std::optional>&& local_vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check) +{ + CUGRAPH_EXPECTS(edgelist_srcs.size() == edgelist_dsts.size(), + "Invalid input arguments: edgelist_srcs.size() != edgelist_dsts.size()."); + CUGRAPH_EXPECTS(!edgelist_weights || (edgelist_srcs.size() == (*edgelist_weights).size()), + "Invalid input arguments: edgelist_weights.has_value() is true and " + "edgelist_srcs.size() != (*edgelist_weights).size()."); + CUGRAPH_EXPECTS(!edgelist_edge_ids || (edgelist_srcs.size() == (*edgelist_edge_ids).size()), + "Invalid input arguments: edgelist_edge_ids.has_value() is true and " + "edgelist_srcs.size() != (*edgelist_edge_ids).size()."); + CUGRAPH_EXPECTS(!edgelist_edge_types || (edgelist_srcs.size() == (*edgelist_edge_types).size()), + "Invalid input arguments: edgelist_edge_types.has_value() is true, " + "edgelist_srcs.size() != (*edgelist_edge_types).size()."); + for (size_t i = 0; i < edgelist_srcs.size(); ++i) { + CUGRAPH_EXPECTS(edgelist_srcs[i].size() == edgelist_dsts[i].size(), + "Invalid input arguments: edgelist_srcs[i].size() != edgelist_dsts[i].size()."); + CUGRAPH_EXPECTS(!edgelist_weights || (edgelist_srcs[i].size() == (*edgelist_weights)[i].size()), + "Invalid input arguments: edgelist_weights.has_value() is true and " + "edgelist_srcs[i].size() != (*edgelist_weights)[i].size()."); + CUGRAPH_EXPECTS( + !edgelist_edge_ids || (edgelist_srcs[i].size() == (*edgelist_edge_ids)[i].size()), + "Invalid input arguments: edgelist_edge_ids.has_value() is true and " + "edgelist_srcs[i].size() != (*edgelist_edge_ids)[i].size()."); + CUGRAPH_EXPECTS( + !edgelist_edge_types || (edgelist_srcs[i].size() == (*edgelist_edge_types)[i].size()), + "Invalid input arguments: edgelist_edge_types.has_value() is true, " + "edgelist_srcs[i].size() != (*edgelist_edge_types)[i].size()."); + } + + std::vector chunk_edge_counts(edgelist_srcs.size()); + for (size_t i = 0; i < edgelist_srcs.size(); ++i) { + chunk_edge_counts[i] = edgelist_srcs[i].size(); + } + std::vector chunk_edge_displacements(chunk_edge_counts.size()); + std::exclusive_scan(chunk_edge_counts.begin(), + chunk_edge_counts.end(), + chunk_edge_displacements.begin(), + edge_t{0}); + auto aggregate_edge_count = chunk_edge_displacements.back() + chunk_edge_counts.back(); + + rmm::device_uvector aggregate_edgelist_srcs(aggregate_edge_count, handle.get_stream()); + for (size_t i = 0; i < edgelist_srcs.size(); ++i) { + thrust::copy(handle.get_thrust_policy(), + edgelist_srcs[i].begin(), + edgelist_srcs[i].end(), + aggregate_edgelist_srcs.begin() + chunk_edge_displacements[i]); + edgelist_srcs[i].resize(0, handle.get_stream()); + edgelist_srcs[i].shrink_to_fit(handle.get_stream()); + } + edgelist_srcs.clear(); + + rmm::device_uvector aggregate_edgelist_dsts(aggregate_edge_count, handle.get_stream()); + for (size_t i = 0; i < edgelist_dsts.size(); ++i) { + thrust::copy(handle.get_thrust_policy(), + edgelist_dsts[i].begin(), + edgelist_dsts[i].end(), + aggregate_edgelist_dsts.begin() + chunk_edge_displacements[i]); + edgelist_dsts[i].resize(0, handle.get_stream()); + edgelist_dsts[i].shrink_to_fit(handle.get_stream()); + } + edgelist_dsts.clear(); + + auto aggregate_edgelist_weights = + edgelist_weights + ? std::make_optional>(aggregate_edge_count, handle.get_stream()) + : std::nullopt; + if (aggregate_edgelist_weights) { + for (size_t i = 0; i < (*edgelist_weights).size(); ++i) { + thrust::copy(handle.get_thrust_policy(), + (*edgelist_weights)[i].begin(), + (*edgelist_weights)[i].end(), + (*aggregate_edgelist_weights).begin() + chunk_edge_displacements[i]); + (*edgelist_weights)[i].resize(0, handle.get_stream()); + (*edgelist_weights)[i].shrink_to_fit(handle.get_stream()); + } + (*edgelist_weights).clear(); + } + + auto aggregate_edgelist_edge_ids = edgelist_edge_ids + ? std::make_optional>( + aggregate_edge_count, handle.get_stream()) + : std::nullopt; + if (aggregate_edgelist_edge_ids) { + for (size_t i = 0; i < (*edgelist_edge_ids).size(); ++i) { + thrust::copy(handle.get_thrust_policy(), + (*edgelist_edge_ids)[i].begin(), + (*edgelist_edge_ids)[i].end(), + (*aggregate_edgelist_edge_ids).begin() + chunk_edge_displacements[i]); + (*edgelist_edge_ids)[i].resize(0, handle.get_stream()); + (*edgelist_edge_ids)[i].shrink_to_fit(handle.get_stream()); + } + (*edgelist_edge_ids).clear(); + } + + auto aggregate_edgelist_edge_types = edgelist_edge_types + ? std::make_optional>( + aggregate_edge_count, handle.get_stream()) + : std::nullopt; + if (aggregate_edgelist_edge_types) { + for (size_t i = 0; i < (*edgelist_edge_types).size(); ++i) { + thrust::copy(handle.get_thrust_policy(), + (*edgelist_edge_types)[i].begin(), + (*edgelist_edge_types)[i].end(), + (*aggregate_edgelist_edge_types).begin() + chunk_edge_displacements[i]); + (*edgelist_edge_types)[i].resize(0, handle.get_stream()); + (*edgelist_edge_types)[i].shrink_to_fit(handle.get_stream()); + } + (*edgelist_edge_types).clear(); + } + + if (do_expensive_check) { + expensive_check_edgelist( + handle, + local_vertices, + store_transposed ? aggregate_edgelist_dsts : aggregate_edgelist_srcs, + store_transposed ? aggregate_edgelist_srcs : aggregate_edgelist_dsts, + renumber); + + if (graph_properties.is_symmetric) { + CUGRAPH_EXPECTS( + (check_symmetric( + handle, + raft::device_span(aggregate_edgelist_srcs.data(), + aggregate_edgelist_srcs.size()), + raft::device_span(aggregate_edgelist_dsts.data(), + aggregate_edgelist_dsts.size()))), + "Invalid input arguments: graph_properties.is_symmetric is true but the input edge list is " + "not symmetric."); + } + + if (!graph_properties.is_multigraph) { + CUGRAPH_EXPECTS( + check_no_parallel_edge(handle, + raft::device_span(aggregate_edgelist_srcs.data(), + aggregate_edgelist_srcs.size()), + raft::device_span(aggregate_edgelist_dsts.data(), + aggregate_edgelist_dsts.size())), + "Invalid input arguments: graph_properties.is_multigraph is false but the input edge list " + "has parallel edges."); + } + } + + return create_graph_from_edgelist_impl(handle, + std::move(local_vertices), + std::move(aggregate_edgelist_srcs), + std::move(aggregate_edgelist_dsts), + std::move(aggregate_edgelist_weights), + std::move(aggregate_edgelist_edge_ids), + std::move(aggregate_edgelist_edge_types), + graph_properties, + renumber, + do_expensive_check); +} + } // namespace template +std::tuple< + cugraph::graph_t, + std::optional< + edge_property_t, weight_t>>, + std::optional< + edge_property_t, edge_id_t>>, + std::optional< + edge_property_t, edge_type_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check) +{ + return create_graph_from_edgelist_impl(handle, + std::move(vertices), + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::move(edgelist_weights), + std::move(edgelist_edge_ids), + std::move(edgelist_edge_types), + graph_properties, + renumber, + do_expensive_check); +} + } // namespace cugraph diff --git a/cpp/src/structure/create_graph_from_edgelist_mg_v32_e32.cu b/cpp/src/structure/create_graph_from_edgelist_mg_v32_e32.cu index 62eb4ccbd96..d0e41734365 100644 --- a/cpp/src/structure/create_graph_from_edgelist_mg_v32_e32.cu +++ b/cpp/src/structure/create_graph_from_edgelist_mg_v32_e32.cu @@ -30,7 +30,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -51,7 +51,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -72,7 +72,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -93,7 +93,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -103,4 +103,88 @@ create_graph_from_edgelist, + std::optional< + cugraph::edge_property_t, float>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, float>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, double>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, double>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + } // namespace cugraph diff --git a/cpp/src/structure/create_graph_from_edgelist_mg_v32_e64.cu b/cpp/src/structure/create_graph_from_edgelist_mg_v32_e64.cu index efbe8058c41..380d3474292 100644 --- a/cpp/src/structure/create_graph_from_edgelist_mg_v32_e64.cu +++ b/cpp/src/structure/create_graph_from_edgelist_mg_v32_e64.cu @@ -30,7 +30,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -51,7 +51,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -72,7 +72,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -93,7 +93,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -103,4 +103,87 @@ create_graph_from_edgelist, + std::optional< + cugraph::edge_property_t, float>>, + std::optional< + cugraph::edge_property_t, int64_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, float>>, + std::optional< + cugraph::edge_property_t, int64_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, double>>, + std::optional< + cugraph::edge_property_t, int64_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, double>>, + std::optional< + cugraph::edge_property_t, int64_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); } // namespace cugraph diff --git a/cpp/src/structure/create_graph_from_edgelist_mg_v64_e64.cu b/cpp/src/structure/create_graph_from_edgelist_mg_v64_e64.cu index cc62166a7af..cbbaf025856 100644 --- a/cpp/src/structure/create_graph_from_edgelist_mg_v64_e64.cu +++ b/cpp/src/structure/create_graph_from_edgelist_mg_v64_e64.cu @@ -30,7 +30,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -51,7 +51,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -72,7 +72,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -93,7 +93,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -103,4 +103,88 @@ create_graph_from_edgelist, + std::optional< + cugraph::edge_property_t, float>>, + std::optional< + cugraph::edge_property_t, int64_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, float>>, + std::optional< + cugraph::edge_property_t, int64_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, double>>, + std::optional< + cugraph::edge_property_t, int64_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, double>>, + std::optional< + cugraph::edge_property_t, int64_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + } // namespace cugraph diff --git a/cpp/src/structure/create_graph_from_edgelist_sg_v32_e32.cu b/cpp/src/structure/create_graph_from_edgelist_sg_v32_e32.cu index 34c91494c8b..28dc3befd8d 100644 --- a/cpp/src/structure/create_graph_from_edgelist_sg_v32_e32.cu +++ b/cpp/src/structure/create_graph_from_edgelist_sg_v32_e32.cu @@ -30,7 +30,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -51,7 +51,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -72,7 +72,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -93,7 +93,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -103,4 +103,88 @@ create_graph_from_edgelist, + std::optional< + cugraph::edge_property_t, float>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, float>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, double>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, double>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + } // namespace cugraph diff --git a/cpp/src/structure/create_graph_from_edgelist_sg_v32_e64.cu b/cpp/src/structure/create_graph_from_edgelist_sg_v32_e64.cu index d09e50e0f81..71bd74c1a44 100644 --- a/cpp/src/structure/create_graph_from_edgelist_sg_v32_e64.cu +++ b/cpp/src/structure/create_graph_from_edgelist_sg_v32_e64.cu @@ -30,7 +30,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -51,7 +51,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -72,7 +72,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -93,7 +93,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -103,4 +103,88 @@ create_graph_from_edgelist, + std::optional< + cugraph::edge_property_t, float>>, + std::optional< + cugraph::edge_property_t, int64_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, float>>, + std::optional< + cugraph::edge_property_t, int64_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, double>>, + std::optional< + cugraph::edge_property_t, int64_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, double>>, + std::optional< + cugraph::edge_property_t, int64_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + } // namespace cugraph diff --git a/cpp/src/structure/create_graph_from_edgelist_sg_v64_e64.cu b/cpp/src/structure/create_graph_from_edgelist_sg_v64_e64.cu index 099c6801f0d..7db38452c72 100644 --- a/cpp/src/structure/create_graph_from_edgelist_sg_v64_e64.cu +++ b/cpp/src/structure/create_graph_from_edgelist_sg_v64_e64.cu @@ -30,7 +30,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -51,7 +51,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -72,7 +72,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -93,7 +93,7 @@ template std::tuple< std::optional>> create_graph_from_edgelist( raft::handle_t const& handle, - std::optional>&& vertex_span, + std::optional>&& vertices, rmm::device_uvector&& edgelist_srcs, rmm::device_uvector&& edgelist_dsts, std::optional>&& edgelist_weights, @@ -103,4 +103,88 @@ create_graph_from_edgelist, + std::optional< + cugraph::edge_property_t, float>>, + std::optional< + cugraph::edge_property_t, int64_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, float>>, + std::optional< + cugraph::edge_property_t, int64_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, double>>, + std::optional< + cugraph::edge_property_t, int64_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + +template std::tuple< + cugraph::graph_t, + std::optional< + cugraph::edge_property_t, double>>, + std::optional< + cugraph::edge_property_t, int64_t>>, + std::optional< + cugraph::edge_property_t, int32_t>>, + std::optional>> +create_graph_from_edgelist( + raft::handle_t const& handle, + std::optional>&& vertices, + std::vector>&& edgelist_srcs, + std::vector>&& edgelist_dsts, + std::optional>>&& edgelist_weights, + std::optional>>&& edgelist_edge_ids, + std::optional>>&& edgelist_edge_types, + graph_properties_t graph_properties, + bool renumber, + bool do_expensive_check); + } // namespace cugraph diff --git a/cpp/tests/mtmg/multi_node_threaded_test.cu b/cpp/tests/mtmg/multi_node_threaded_test.cu index 24852562b86..06ccd4a7fa1 100644 --- a/cpp/tests/mtmg/multi_node_threaded_test.cu +++ b/cpp/tests/mtmg/multi_node_threaded_test.cu @@ -150,9 +150,22 @@ class Tests_Multithreaded instance_manager->reset_threads(); // Load SG edge list - auto [d_src_v, d_dst_v, d_weights_v, d_vertices_v, is_symmetric] = - input_usecase.template construct_edgelist( - handle, multithreaded_usecase.test_weighted, false, false); + rmm::device_uvector d_src_v(0, handle.get_stream()); + rmm::device_uvector d_dst_v(0, handle.get_stream()); + std::optional> d_weights_v{std::nullopt}; + std::optional> d_vertices_v{std::nullopt}; + bool is_symmetric{}; + { + std::vector> src_chunks{}; + std::vector> dst_chunks{}; + std::optional>> weight_chunks{std::nullopt}; + std::tie(src_chunks, dst_chunks, weight_chunks, d_vertices_v, is_symmetric) = + input_usecase.template construct_edgelist( + handle, multithreaded_usecase.test_weighted, false, false); + + std::tie(d_src_v, d_dst_v, d_weights_v) = cugraph::test::detail::concatenate_edge_chunks( + handle, std::move(src_chunks), std::move(dst_chunks), std::move(weight_chunks)); + } auto h_src_v = cugraph::test::to_host(handle, d_src_v); auto h_dst_v = cugraph::test::to_host(handle, d_dst_v); diff --git a/cpp/tests/mtmg/threaded_test.cu b/cpp/tests/mtmg/threaded_test.cu index df5a9e079df..a288ef63da9 100644 --- a/cpp/tests/mtmg/threaded_test.cu +++ b/cpp/tests/mtmg/threaded_test.cu @@ -151,9 +151,22 @@ class Tests_Multithreaded instance_manager->reset_threads(); // Load SG edge list - auto [d_src_v, d_dst_v, d_weights_v, d_vertices_v, is_symmetric] = - input_usecase.template construct_edgelist( - handle, multithreaded_usecase.test_weighted, false, false); + rmm::device_uvector d_src_v(0, handle.get_stream()); + rmm::device_uvector d_dst_v(0, handle.get_stream()); + std::optional> d_weights_v{std::nullopt}; + std::optional> d_vertices_v{std::nullopt}; + bool is_symmetric{}; + { + std::vector> src_chunks{}; + std::vector> dst_chunks{}; + std::optional>> weight_chunks{std::nullopt}; + std::tie(src_chunks, dst_chunks, weight_chunks, d_vertices_v, is_symmetric) = + input_usecase.template construct_edgelist( + handle, multithreaded_usecase.test_weighted, false, false); + + std::tie(d_src_v, d_dst_v, d_weights_v) = cugraph::test::detail::concatenate_edge_chunks( + handle, std::move(src_chunks), std::move(dst_chunks), std::move(weight_chunks)); + } rmm::device_uvector d_unique_vertices(2 * d_src_v.size(), handle.get_stream()); thrust::copy( diff --git a/cpp/tests/mtmg/threaded_test_jaccard.cu b/cpp/tests/mtmg/threaded_test_jaccard.cu index 0f531796cff..bed2f193130 100644 --- a/cpp/tests/mtmg/threaded_test_jaccard.cu +++ b/cpp/tests/mtmg/threaded_test_jaccard.cu @@ -144,9 +144,22 @@ class Tests_Multithreaded instance_manager->reset_threads(); // Load SG edge list - auto [d_src_v, d_dst_v, d_weights_v, d_vertices_v, is_symmetric] = - input_usecase.template construct_edgelist( - handle, test_weighted, store_transposed, false); + rmm::device_uvector d_src_v(0, handle.get_stream()); + rmm::device_uvector d_dst_v(0, handle.get_stream()); + std::optional> d_weights_v{std::nullopt}; + std::optional> d_vertices_v{std::nullopt}; + bool is_symmetric{}; + { + std::vector> src_chunks{}; + std::vector> dst_chunks{}; + std::optional>> weight_chunks{std::nullopt}; + std::tie(src_chunks, dst_chunks, weight_chunks, d_vertices_v, is_symmetric) = + input_usecase.template construct_edgelist( + handle, test_weighted, store_transposed, false); + + std::tie(d_src_v, d_dst_v, d_weights_v) = cugraph::test::detail::concatenate_edge_chunks( + handle, std::move(src_chunks), std::move(dst_chunks), std::move(weight_chunks)); + } rmm::device_uvector d_unique_vertices(2 * d_src_v.size(), handle.get_stream()); thrust::copy( diff --git a/cpp/tests/mtmg/threaded_test_louvain.cu b/cpp/tests/mtmg/threaded_test_louvain.cu index b9c8f621ab8..a2ec9244b38 100644 --- a/cpp/tests/mtmg/threaded_test_louvain.cu +++ b/cpp/tests/mtmg/threaded_test_louvain.cu @@ -151,9 +151,22 @@ class Tests_Multithreaded instance_manager->reset_threads(); // Load SG edge list - auto [d_src_v, d_dst_v, d_weights_v, d_vertices_v, is_symmetric] = - input_usecase.template construct_edgelist( - handle, multithreaded_usecase.test_weighted, false, false); + rmm::device_uvector d_src_v(0, handle.get_stream()); + rmm::device_uvector d_dst_v(0, handle.get_stream()); + std::optional> d_weights_v{std::nullopt}; + std::optional> d_vertices_v{std::nullopt}; + bool is_symmetric{}; + { + std::vector> src_chunks{}; + std::vector> dst_chunks{}; + std::optional>> weight_chunks{std::nullopt}; + std::tie(src_chunks, dst_chunks, weight_chunks, d_vertices_v, is_symmetric) = + input_usecase.template construct_edgelist( + handle, multithreaded_usecase.test_weighted, false, false); + + std::tie(d_src_v, d_dst_v, d_weights_v) = cugraph::test::detail::concatenate_edge_chunks( + handle, std::move(src_chunks), std::move(dst_chunks), std::move(weight_chunks)); + } rmm::device_uvector d_unique_vertices(2 * d_src_v.size(), handle.get_stream()); thrust::copy( diff --git a/cpp/tests/structure/renumbering_test.cpp b/cpp/tests/structure/renumbering_test.cpp index ffb51943660..d5829be6fe3 100644 --- a/cpp/tests/structure/renumbering_test.cpp +++ b/cpp/tests/structure/renumbering_test.cpp @@ -69,10 +69,17 @@ class Tests_Renumbering rmm::device_uvector src_v(0, handle.get_stream()); rmm::device_uvector dst_v(0, handle.get_stream()); - rmm::device_uvector renumber_map_labels_v(0, handle.get_stream()); - std::tie(src_v, dst_v, std::ignore, std::ignore, std::ignore) = - input_usecase.template construct_edgelist(handle, false, false, false); + { + std::vector> src_chunks{}; + std::vector> dst_chunks{}; + std::tie(src_chunks, dst_chunks, std::ignore, std::ignore, std::ignore) = + input_usecase.template construct_edgelist(handle, false, false, false); + + std::tie(src_v, dst_v, std::ignore) = + cugraph::test::detail::concatenate_edge_chunks( + handle, std::move(src_chunks), std::move(dst_chunks), std::nullopt); + } if (renumbering_usecase.check_correctness) { h_original_src_v = cugraph::test::to_host(handle, src_v); @@ -84,6 +91,7 @@ class Tests_Renumbering hr_timer.start("Renumbering"); } + rmm::device_uvector renumber_map_labels_v(0, handle.get_stream()); std::tie(renumber_map_labels_v, std::ignore) = cugraph::renumber_edgelist( handle, std::nullopt, src_v.begin(), dst_v.begin(), src_v.size(), false); diff --git a/cpp/tests/utilities/test_graphs.hpp b/cpp/tests/utilities/test_graphs.hpp index 0f3224bfc52..b6898fbaf78 100644 --- a/cpp/tests/utilities/test_graphs.hpp +++ b/cpp/tests/utilities/test_graphs.hpp @@ -35,51 +35,72 @@ namespace test { namespace detail { -template -std::optional> try_allocate(raft::handle_t const& handle, size_t size) +template +std::tuple, + rmm::device_uvector, + std::optional>> +concatenate_edge_chunks(raft::handle_t const& handle, + std::vector>&& src_chunks, + std::vector>&& dst_chunks, + std::optional>> weight_chunks) { - try { - return std::make_optional>(size, handle.get_stream()); - } catch (std::exception const& e) { - return std::nullopt; - } -} + if (src_chunks.size() == 1) { + return std::make_tuple(std::move(src_chunks[0]), + std::move(dst_chunks[0]), + weight_chunks ? std::make_optional>( + std::move((*weight_chunks)[0])) + : std::nullopt); + } else { + size_t edge_count{0}; + for (size_t i = 0; i < src_chunks.size(); ++i) { + edge_count += src_chunks[i].size(); + } -// use host memory as temporary buffer if memroy allocation on device fails -template -rmm::device_uvector concatenate(raft::handle_t const& handle, - std::vector>&& inputs) -{ - size_t tot_count{0}; - for (size_t i = 0; i < inputs.size(); ++i) { - tot_count += inputs[i].size(); - } + rmm::device_uvector srcs(edge_count, handle.get_stream()); + { + size_t offset{0}; + for (size_t i = 0; i < src_chunks.size(); ++i) { + raft::copy( + srcs.data() + offset, src_chunks[i].data(), src_chunks[i].size(), handle.get_stream()); + offset += src_chunks[i].size(); + src_chunks[i].resize(0, handle.get_stream()); + src_chunks[i].shrink_to_fit(handle.get_stream()); + } + src_chunks.clear(); + } - auto output = try_allocate(handle, tot_count); - if (output) { - size_t offset{0}; - for (size_t i = 0; i < inputs.size(); ++i) { - raft::copy( - (*output).data() + offset, inputs[i].data(), inputs[i].size(), handle.get_stream()); - offset += inputs[i].size(); + rmm::device_uvector dsts(edge_count, handle.get_stream()); + { + size_t offset{0}; + for (size_t i = 0; i < dst_chunks.size(); ++i) { + raft::copy( + dsts.data() + offset, dst_chunks[i].data(), dst_chunks[i].size(), handle.get_stream()); + offset += dst_chunks[i].size(); + dst_chunks[i].resize(0, handle.get_stream()); + dst_chunks[i].shrink_to_fit(handle.get_stream()); + } + dst_chunks.clear(); } - inputs.clear(); - inputs.shrink_to_fit(); - } else { - std::vector h_buffer(tot_count); - size_t offset{0}; - for (size_t i = 0; i < inputs.size(); ++i) { - raft::update_host( - h_buffer.data() + offset, inputs[i].data(), inputs[i].size(), handle.get_stream()); - offset += inputs[i].size(); + + auto weights = weight_chunks ? std::make_optional>( + edge_count, handle.get_stream()) + : std::nullopt; + if (weights) { + size_t offset{0}; + for (size_t i = 0; i < (*weight_chunks).size(); ++i) { + raft::copy((*weights).data() + offset, + (*weight_chunks)[i].data(), + (*weight_chunks)[i].size(), + handle.get_stream()); + offset += (*weight_chunks)[i].size(); + (*weight_chunks)[i].resize(0, handle.get_stream()); + (*weight_chunks)[i].shrink_to_fit(handle.get_stream()); + } + (*weight_chunks).clear(); } - inputs.clear(); - inputs.shrink_to_fit(); - output = rmm::device_uvector(tot_count, handle.get_stream()); - raft::update_device((*output).data(), h_buffer.data(), h_buffer.size(), handle.get_stream()); - } - return std::move(*output); + return std::make_tuple(std::move(srcs), std::move(dsts), std::move(weights)); + } } class TranslateGraph_Usecase { @@ -131,9 +152,9 @@ class File_Usecase : public detail::TranslateGraph_Usecase { } template - std::tuple, - rmm::device_uvector, - std::optional>, + std::tuple>, + std::vector>, + std::optional>>, std::optional>, bool> construct_edgelist(raft::handle_t const& handle, @@ -159,8 +180,20 @@ class File_Usecase : public detail::TranslateGraph_Usecase { translate(handle, srcs, dsts); if (vertices) { translate(handle, *vertices); } - return std::make_tuple( - std::move(srcs), std::move(dsts), std::move(weights), std::move(vertices), is_symmetric); + std::vector> edge_src_chunks{}; + edge_src_chunks.push_back(std::move(srcs)); + std::vector> edge_dst_chunks{}; + edge_dst_chunks.push_back(std::move(dsts)); + std::optional>> edge_weight_chunks{std::nullopt}; + if (weights) { + edge_weight_chunks = std::vector>{}; + (*edge_weight_chunks).push_back(std::move(*weights)); + } + return std::make_tuple(std::move(edge_src_chunks), + std::move(edge_dst_chunks), + std::move(edge_weight_chunks), + std::move(vertices), + is_symmetric); } private: @@ -193,9 +226,9 @@ class Rmat_Usecase : public detail::TranslateGraph_Usecase { } template - std::tuple, - rmm::device_uvector, - std::optional>, + std::tuple>, + std::vector>, + std::optional>>, std::optional>, bool> construct_edgelist(raft::handle_t const& handle, @@ -213,7 +246,7 @@ class Rmat_Usecase : public detail::TranslateGraph_Usecase { // cuMemAddressReserve // (https://developer.nvidia.com/blog/introducing-low-level-gpu-virtual-memory-management), we // can reduce the temporary memory requirement to (1 / num_partitions) * (original data size) - size_t constexpr num_partitions_per_gpu = 2; + size_t constexpr num_partitions_per_gpu = 4; size_t num_partitions = num_partitions_per_gpu * static_cast(multi_gpu ? handle.get_comms().get_size() : 1); @@ -253,14 +286,14 @@ class Rmat_Usecase : public detail::TranslateGraph_Usecase { raft::random::RngState rng_state{ base_seed_ + static_cast(multi_gpu ? handle.get_comms().get_rank() : 0)}; - std::vector> src_partitions{}; - std::vector> dst_partitions{}; - auto weight_partitions = test_weighted - ? std::make_optional>>() - : std::nullopt; - src_partitions.reserve(num_partitions_per_gpu); - dst_partitions.reserve(num_partitions_per_gpu); - if (weight_partitions) { (*weight_partitions).reserve(num_partitions_per_gpu); } + std::vector> edge_src_chunks{}; + std::vector> edge_dst_chunks{}; + auto edge_weight_chunks = test_weighted + ? std::make_optional>>() + : std::nullopt; + edge_src_chunks.reserve(num_partitions_per_gpu); + edge_dst_chunks.reserve(num_partitions_per_gpu); + if (edge_weight_chunks) { (*edge_weight_chunks).reserve(num_partitions_per_gpu); } for (size_t i = 0; i < num_partitions_per_gpu; ++i) { auto [tmp_src_v, tmp_dst_v] = cugraph::generate_rmat_edgelist(handle, @@ -277,7 +310,7 @@ class Rmat_Usecase : public detail::TranslateGraph_Usecase { } std::optional> tmp_weights_v{std::nullopt}; - if (weight_partitions) { + if (edge_weight_chunks) { tmp_weights_v = std::make_optional>(tmp_src_v.size(), handle.get_stream()); @@ -315,27 +348,9 @@ class Rmat_Usecase : public detail::TranslateGraph_Usecase { std::nullopt); } - src_partitions.push_back(std::move(tmp_src_v)); - dst_partitions.push_back(std::move(tmp_dst_v)); - if (weight_partitions) { (*weight_partitions).push_back(std::move(*tmp_weights_v)); } - } - - size_t tot_edge_counts{0}; - for (size_t i = 0; i < src_partitions.size(); ++i) { - tot_edge_counts += src_partitions[i].size(); - } - - // detail::concatenate uses a host buffer to store input vectors if initial device memory - // allocation for the return vector fails. This does not improve peak memory usage and is not - // helpful with the rmm_mode = cuda. However, if rmm_mode = pool, memory allocation can fail - // even when the aggregate free memory size far exceeds the requested size. This heuristic is - // helpful in this case. - - auto src_v = detail::concatenate(handle, std::move(src_partitions)); - auto dst_v = detail::concatenate(handle, std::move(dst_partitions)); - std::optional> weight_v{std::nullopt}; - if (weight_partitions) { - weight_v = detail::concatenate(handle, std::move(*weight_partitions)); + edge_src_chunks.push_back(std::move(tmp_src_v)); + edge_dst_chunks.push_back(std::move(tmp_dst_v)); + if (edge_weight_chunks) { (*edge_weight_chunks).push_back(std::move(*tmp_weights_v)); } } // 3. generate vertices @@ -364,8 +379,11 @@ class Rmat_Usecase : public detail::TranslateGraph_Usecase { handle, std::move(vertex_v)); } - return std::make_tuple( - std::move(src_v), std::move(dst_v), std::move(weight_v), std::move(vertex_v), undirected_); + return std::make_tuple(std::move(edge_src_chunks), + std::move(edge_dst_chunks), + std::move(edge_weight_chunks), + std::move(vertex_v), + undirected_); } void set_scale(size_t scale) { scale_ = scale; } @@ -388,17 +406,15 @@ class PathGraph_Usecase { public: PathGraph_Usecase() = delete; - PathGraph_Usecase(std::vector> parms, - bool weighted = false, - bool scramble = false) - : parms_(parms), weighted_(weighted) + PathGraph_Usecase(std::vector> parms, bool scramble = false) + : parms_(parms) { } template - std::tuple, - rmm::device_uvector, - std::optional>, + std::tuple>, + std::vector>, + std::optional>>, std::optional>, bool> construct_edgelist(raft::handle_t const& handle, @@ -415,21 +431,44 @@ class PathGraph_Usecase { static_cast(std::get<1>(p))); }); - auto [src_v, dst_v] = cugraph::generate_path_graph_edgelist(handle, converted_parms); - std::tie(src_v, dst_v, std::ignore) = + auto [srcs, dsts] = cugraph::generate_path_graph_edgelist(handle, converted_parms); + + raft::random::RngState rng_state{ + base_seed_ + static_cast(multi_gpu ? handle.get_comms().get_rank() : 0)}; + + std::optional> weights{std::nullopt}; + if (test_weighted) { + weights = std::make_optional>(srcs.size(), handle.get_stream()); + + cugraph::detail::uniform_random_fill(handle.get_stream(), + weights->data(), + weights->size(), + weight_t{0.0}, + weight_t{1.0}, + rng_state); + } + + std::tie(srcs, dsts, weights) = cugraph::symmetrize_edgelist_from_triangular( - handle, std::move(src_v), std::move(dst_v), std::nullopt); + handle, std::move(srcs), std::move(dsts), std::move(weights)); rmm::device_uvector d_vertices(num_vertices_, handle.get_stream()); cugraph::detail::sequence_fill( handle.get_stream(), d_vertices.data(), num_vertices_, vertex_t{0}); handle.sync_stream(); - return std::make_tuple(std::move(src_v), - std::move(dst_v), - test_weighted ? std::make_optional>( - src_v.size(), handle.get_stream()) - : std::nullopt, + std::vector> edge_src_chunks{}; + edge_src_chunks.push_back(std::move(srcs)); + std::vector> edge_dst_chunks{}; + edge_dst_chunks.push_back(std::move(dsts)); + std::optional>> edge_weight_chunks{std::nullopt}; + if (weights) { + edge_weight_chunks = std::vector>{}; + (*edge_weight_chunks).push_back(std::move(*weights)); + } + return std::make_tuple(std::move(edge_src_chunks), + std::move(edge_dst_chunks), + std::move(edge_weight_chunks), std::move(d_vertices), symmetric); } @@ -437,7 +476,7 @@ class PathGraph_Usecase { private: std::vector> parms_{}; size_t num_vertices_{0}; - bool weighted_{false}; + uint64_t base_seed_{}; }; class Mesh2DGraph_Usecase { @@ -450,9 +489,9 @@ class Mesh2DGraph_Usecase { } template - std::tuple, - rmm::device_uvector, - std::optional>, + std::tuple>, + std::vector>, + std::optional>>, std::optional>, bool> construct_edgelist(raft::handle_t const& handle, @@ -476,9 +515,9 @@ class Mesh3DGraph_Usecase { } template - std::tuple, - rmm::device_uvector, - std::optional>, + std::tuple>, + std::vector>, + std::optional>>, std::optional>, bool> construct_edgelist(raft::handle_t const& handle, @@ -501,9 +540,9 @@ class CompleteGraph_Usecase { } template - std::tuple, - rmm::device_uvector, - std::optional>, + std::tuple>, + std::vector>, + std::optional>>, std::optional>, bool> construct_edgelist(raft::handle_t const& handle, @@ -573,10 +612,10 @@ class CombinedGenerator_Usecase { CombinedGenerator_Usecase(generator_tuple_t const& tuple) : generator_tuple_(tuple) {} template - std::tuple, - rmm::device_uvector, - std::optional>, - rmm::device_uvector, + std::tuple>, + std::vector>, + std::optional>>, + std::optional>, vertex_t, bool> construct_edgelist(raft::handle_t const& handle, @@ -593,6 +632,13 @@ class CombinedGenerator_Usecase { // Need to combine elements. We have a vector of tuples, we want to combine // the elements of each component of the tuple CUGRAPH_FAIL("not implemented"); + + std::vector> edge_src_chunks{}; + edge_src_chunks.push_back(rmm::device_uvector(0, handle.get_stream())); + std::vector> edge_dst_chunks{}; + edge_dst_chunks.push_back(rmm::device_uvector(0, handle.get_stream())); + return std::make_tuple( + std::move(edge_src_chunks), std::move(edge_dst_chunks), std::nullopt, std::nullopt, false); } private: @@ -617,32 +663,56 @@ construct_graph(raft::handle_t const& handle, bool drop_self_loops = false, bool drop_multi_edges = false) { - auto [d_src_v, d_dst_v, d_weights_v, d_vertices_v, is_symmetric] = + auto [edge_src_chunks, edge_dst_chunks, edge_weight_chunks, d_vertices_v, is_symmetric] = input_usecase.template construct_edgelist( handle, test_weighted, store_transposed, multi_gpu); - CUGRAPH_EXPECTS(d_src_v.size() <= static_cast(std::numeric_limits::max()), + size_t num_edges{0}; + for (size_t i = 0; i < edge_src_chunks.size(); ++i) { + num_edges += edge_src_chunks[i].size(); + } + CUGRAPH_EXPECTS(num_edges <= static_cast(std::numeric_limits::max()), "Invalid template parameter: edge_t overflow."); if (drop_self_loops) { - std::tie(d_src_v, d_dst_v, d_weights_v, std::ignore, std::ignore) = - cugraph::remove_self_loops(handle, - std::move(d_src_v), - std::move(d_dst_v), - std::move(d_weights_v), - std::nullopt, - std::nullopt); + for (size_t i = 0; i < edge_src_chunks.size(); ++i) { + std::optional> tmp_weights{std::nullopt}; + std::tie(edge_src_chunks[i], edge_dst_chunks[i], tmp_weights, std::ignore, std::ignore) = + cugraph::remove_self_loops( + handle, + std::move(edge_src_chunks[i]), + std::move(edge_dst_chunks[i]), + edge_weight_chunks + ? std::make_optional>(std::move((*edge_weight_chunks)[i])) + : std::nullopt, + std::nullopt, + std::nullopt); + if (tmp_weights) { (*edge_weight_chunks)[i] = std::move(*tmp_weights); } + } } if (drop_multi_edges) { - std::tie(d_src_v, d_dst_v, d_weights_v, std::ignore, std::ignore) = + auto [srcs, dsts, weights] = detail::concatenate_edge_chunks(handle, + std::move(edge_src_chunks), + std::move(edge_dst_chunks), + std::move(edge_weight_chunks)); + std::tie(srcs, dsts, weights, std::ignore, std::ignore) = cugraph::remove_multi_edges( handle, - std::move(d_src_v), - std::move(d_dst_v), - std::move(d_weights_v), + std::move(srcs), + std::move(dsts), + std::move(weights), std::nullopt, std::nullopt, is_symmetric ? true /* keep minimum weight edges to maintain symmetry */ : false); + edge_src_chunks = std::vector>{}; + edge_src_chunks.push_back(std::move(srcs)); + edge_dst_chunks = std::vector>{}; + edge_dst_chunks.push_back(std::move(dsts)); + edge_weight_chunks = std::nullopt; + if (weights) { + edge_weight_chunks = std::vector>{}; + (*edge_weight_chunks).push_back(std::move(*weights)); + } } graph_t graph(handle); @@ -650,23 +720,43 @@ construct_graph(raft::handle_t const& handle, edge_property_t, weight_t>> edge_weights{std::nullopt}; std::optional> renumber_map{std::nullopt}; - std::tie(graph, edge_weights, std::ignore, std::ignore, renumber_map) = - cugraph::create_graph_from_edgelist( - handle, - std::move(d_vertices_v), - std::move(d_src_v), - std::move(d_dst_v), - std::move(d_weights_v), - std::nullopt, - std::nullopt, - cugraph::graph_properties_t{is_symmetric, drop_multi_edges ? false : true}, - renumber); + if (edge_src_chunks.size() == 1) { + std::tie(graph, edge_weights, std::ignore, std::ignore, renumber_map) = + cugraph::create_graph_from_edgelist( + handle, + std::move(d_vertices_v), + std::move(edge_src_chunks[0]), + std::move(edge_dst_chunks[0]), + edge_weight_chunks ? std::make_optional(std::move((*edge_weight_chunks)[0])) : std::nullopt, + std::nullopt, + std::nullopt, + cugraph::graph_properties_t{is_symmetric, drop_multi_edges ? false : true}, + renumber); + } else { + std::tie(graph, edge_weights, std::ignore, std::ignore, renumber_map) = + cugraph::create_graph_from_edgelist( + handle, + std::move(d_vertices_v), + std::move(edge_src_chunks), + std::move(edge_dst_chunks), + std::move(edge_weight_chunks), + std::nullopt, + std::nullopt, + cugraph::graph_properties_t{is_symmetric, drop_multi_edges ? false : true}, + renumber); + } return std::make_tuple(std::move(graph), std::move(edge_weights), std::move(renumber_map)); } From 599a3bf5717532867f8909d4bfb967d9d7240c1a Mon Sep 17 00:00:00 2001 From: Chuck Hastings <45364586+ChuckHastings@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:52:25 -0400 Subject: [PATCH 05/11] Refactor C++ unit tests to allow finer grained filtering (#4533) Running all of the C++ unit tests can take a long time. This PR splits the file-based tests into small versus large tests. The large tests (usually named file_large_test) can be skipped by setting the following environment variable: `export GTEST_FILTER="-*large*"` You can skip both the large tests and the benchmark tests with the following: `export GTEST_FILTER="-*benchmark*:*large*"` By default all tests will be run. Authors: - Chuck Hastings (https://github.com/ChuckHastings) Approvers: - Joseph Nke (https://github.com/jnke2016) - Seunghwa Kang (https://github.com/seunghwak) URL: https://github.com/rapidsai/cugraph/pull/4533 --- .../edge_betweenness_centrality_test.cpp | 20 +++++++++-- .../eigenvector_centrality_test.cpp | 16 +++++++-- cpp/tests/centrality/katz_centrality_test.cpp | 14 ++++++-- .../mg_betweenness_centrality_test.cpp | 20 +++++++++-- .../mg_edge_betweenness_centrality_test.cpp | 16 +++++++-- .../mg_eigenvector_centrality_test.cpp | 14 ++++++-- .../centrality/mg_katz_centrality_test.cpp | 14 ++++++-- .../community/mg_weighted_matching_test.cpp | 2 +- cpp/tests/link_analysis/hits_test.cpp | 16 +++++++-- cpp/tests/link_analysis/mg_hits_test.cpp | 14 ++++++-- cpp/tests/link_analysis/mg_pagerank_test.cpp | 19 ++++++++-- cpp/tests/prims/mg_count_if_e.cu | 12 +++++-- cpp/tests/prims/mg_count_if_v.cu | 9 +++-- cpp/tests/prims/mg_extract_transform_e.cu | 9 +++-- ...extract_transform_v_frontier_outgoing_e.cu | 9 +++-- ...er_v_random_select_transform_outgoing_e.cu | 25 +++++++++++-- ...rm_reduce_dst_key_aggregated_outgoing_e.cu | 12 +++++-- ..._v_transform_reduce_incoming_outgoing_e.cu | 12 +++++-- cpp/tests/prims/mg_reduce_v.cu | 10 ++++-- cpp/tests/prims/mg_transform_e.cu | 12 +++++-- cpp/tests/prims/mg_transform_reduce_e.cu | 12 +++++-- .../mg_transform_reduce_e_by_src_dst_key.cu | 12 +++++-- cpp/tests/prims/mg_transform_reduce_v.cu | 9 +++-- ...reduce_v_frontier_outgoing_e_by_src_dst.cu | 9 +++-- .../sampling/biased_neighbor_sampling.cpp | 12 +++++-- .../sampling/mg_biased_neighbor_sampling.cpp | 12 +++++-- cpp/tests/sampling/mg_random_walks_test.cpp | 36 ++++++++++++++----- .../sampling/mg_uniform_neighbor_sampling.cpp | 13 +++++-- cpp/tests/sampling/sg_random_walks_test.cpp | 24 +++++++++---- .../sampling/uniform_neighbor_sampling.cpp | 13 +++++-- cpp/tests/structure/degree_test.cpp | 9 +++-- cpp/tests/structure/induced_subgraph_test.cpp | 8 ++--- .../structure/mg_induced_subgraph_test.cu | 6 ++-- cpp/tests/structure/mg_symmetrize_test.cpp | 14 ++++++-- .../structure/mg_transpose_storage_test.cpp | 11 ++++-- cpp/tests/structure/mg_transpose_test.cpp | 11 ++++-- cpp/tests/structure/renumbering_test.cpp | 14 +++++--- cpp/tests/structure/symmetrize_test.cpp | 14 ++++++-- .../structure/transpose_storage_test.cpp | 11 ++++-- cpp/tests/structure/transpose_test.cpp | 11 ++++-- cpp/tests/structure/weight_sum_test.cpp | 11 ++++-- cpp/tests/traversal/mg_bfs_test.cpp | 11 ++++-- cpp/tests/traversal/mg_k_hop_nbrs_test.cpp | 14 ++++++-- 43 files changed, 464 insertions(+), 108 deletions(-) diff --git a/cpp/tests/centrality/edge_betweenness_centrality_test.cpp b/cpp/tests/centrality/edge_betweenness_centrality_test.cpp index b8661e76a11..4f9cdbe6458 100644 --- a/cpp/tests/centrality/edge_betweenness_centrality_test.cpp +++ b/cpp/tests/centrality/edge_betweenness_centrality_test.cpp @@ -218,7 +218,7 @@ TEST_P(Tests_EdgeBetweennessCentrality_Rmat, CheckInt64Int64FloatFloat) } INSTANTIATE_TEST_SUITE_P( - file_test_pass, + file_test, Tests_EdgeBetweennessCentrality_File, ::testing::Combine( // enable correctness checks @@ -230,8 +230,22 @@ INSTANTIATE_TEST_SUITE_P( EdgeBetweennessCentrality_Usecase{20, true, false, true}, EdgeBetweennessCentrality_Usecase{20, true, true, false}, EdgeBetweennessCentrality_Usecase{20, true, true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_EdgeBetweennessCentrality_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(EdgeBetweennessCentrality_Usecase{20, false, false, false}, + EdgeBetweennessCentrality_Usecase{20, false, false, true}, + EdgeBetweennessCentrality_Usecase{20, false, true, false}, + EdgeBetweennessCentrality_Usecase{20, false, true, true}, + EdgeBetweennessCentrality_Usecase{20, true, false, false}, + EdgeBetweennessCentrality_Usecase{20, true, false, true}, + EdgeBetweennessCentrality_Usecase{20, true, true, false}, + EdgeBetweennessCentrality_Usecase{20, true, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); INSTANTIATE_TEST_SUITE_P( diff --git a/cpp/tests/centrality/eigenvector_centrality_test.cpp b/cpp/tests/centrality/eigenvector_centrality_test.cpp index ab87cccd0e3..9e232888e77 100644 --- a/cpp/tests/centrality/eigenvector_centrality_test.cpp +++ b/cpp/tests/centrality/eigenvector_centrality_test.cpp @@ -269,7 +269,7 @@ TEST_P(Tests_EigenvectorCentrality_Rmat, CheckInt64Int64FloatFloat) } INSTANTIATE_TEST_SUITE_P( - file_test_pass, + file_test_test, Tests_EigenvectorCentrality_File, ::testing::Combine( // enable correctness checks @@ -277,8 +277,18 @@ INSTANTIATE_TEST_SUITE_P( EigenvectorCentrality_Usecase{500, false, true}, EigenvectorCentrality_Usecase{500, true, false}, EigenvectorCentrality_Usecase{500, true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_EigenvectorCentrality_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(EigenvectorCentrality_Usecase{500, false, false}, + EigenvectorCentrality_Usecase{500, false, true}, + EigenvectorCentrality_Usecase{500, true, false}, + EigenvectorCentrality_Usecase{500, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/centrality/katz_centrality_test.cpp b/cpp/tests/centrality/katz_centrality_test.cpp index 190007c38e5..fc5aead5d90 100644 --- a/cpp/tests/centrality/katz_centrality_test.cpp +++ b/cpp/tests/centrality/katz_centrality_test.cpp @@ -281,8 +281,18 @@ INSTANTIATE_TEST_SUITE_P( KatzCentrality_Usecase{false, true}, KatzCentrality_Usecase{true, false}, KatzCentrality_Usecase{true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_KatzCentrality_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(KatzCentrality_Usecase{false, false}, + KatzCentrality_Usecase{false, true}, + KatzCentrality_Usecase{true, false}, + KatzCentrality_Usecase{true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/centrality/mg_betweenness_centrality_test.cpp b/cpp/tests/centrality/mg_betweenness_centrality_test.cpp index 798e767085e..65a1098033b 100644 --- a/cpp/tests/centrality/mg_betweenness_centrality_test.cpp +++ b/cpp/tests/centrality/mg_betweenness_centrality_test.cpp @@ -218,7 +218,7 @@ TEST_P(Tests_MGBetweennessCentrality_Rmat, CheckInt64Int64FloatFloat) } INSTANTIATE_TEST_SUITE_P( - file_test_pass, + file_test, Tests_MGBetweennessCentrality_File, ::testing::Combine( // enable correctness checks @@ -230,8 +230,22 @@ INSTANTIATE_TEST_SUITE_P( BetweennessCentrality_Usecase{20, false, true, false, true}, BetweennessCentrality_Usecase{20, false, true, true, false}, BetweennessCentrality_Usecase{20, false, true, true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGBetweennessCentrality_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(BetweennessCentrality_Usecase{20, false, false, false, false}, + BetweennessCentrality_Usecase{20, false, false, false, true}, + BetweennessCentrality_Usecase{20, false, false, true, false}, + BetweennessCentrality_Usecase{20, false, false, true, true}, + BetweennessCentrality_Usecase{20, false, true, false, false}, + BetweennessCentrality_Usecase{20, false, true, false, true}, + BetweennessCentrality_Usecase{20, false, true, true, false}, + BetweennessCentrality_Usecase{20, false, true, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/centrality/mg_edge_betweenness_centrality_test.cpp b/cpp/tests/centrality/mg_edge_betweenness_centrality_test.cpp index 1703f198a4c..8dbeb076c00 100644 --- a/cpp/tests/centrality/mg_edge_betweenness_centrality_test.cpp +++ b/cpp/tests/centrality/mg_edge_betweenness_centrality_test.cpp @@ -222,7 +222,7 @@ TEST_P(Tests_MGEdgeBetweennessCentrality_Rmat, CheckInt64Int64FloatFloat) } INSTANTIATE_TEST_SUITE_P( - file_test_pass, + file_test, Tests_MGEdgeBetweennessCentrality_File, ::testing::Combine( // enable correctness checks @@ -230,8 +230,18 @@ INSTANTIATE_TEST_SUITE_P( EdgeBetweennessCentrality_Usecase{20, false, false, true}, EdgeBetweennessCentrality_Usecase{20, false, true, false}, EdgeBetweennessCentrality_Usecase{20, false, true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGEdgeBetweennessCentrality_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(EdgeBetweennessCentrality_Usecase{20, false, false, false}, + EdgeBetweennessCentrality_Usecase{20, false, false, true}, + EdgeBetweennessCentrality_Usecase{20, false, true, false}, + EdgeBetweennessCentrality_Usecase{20, false, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/centrality/mg_eigenvector_centrality_test.cpp b/cpp/tests/centrality/mg_eigenvector_centrality_test.cpp index 76c52d52bfd..39406e05d6c 100644 --- a/cpp/tests/centrality/mg_eigenvector_centrality_test.cpp +++ b/cpp/tests/centrality/mg_eigenvector_centrality_test.cpp @@ -261,8 +261,18 @@ INSTANTIATE_TEST_SUITE_P( EigenvectorCentrality_Usecase{500, false, true}, EigenvectorCentrality_Usecase{500, true, false}, EigenvectorCentrality_Usecase{500, true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGEigenvectorCentrality_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(EigenvectorCentrality_Usecase{500, false, false}, + EigenvectorCentrality_Usecase{500, false, true}, + EigenvectorCentrality_Usecase{500, true, false}, + EigenvectorCentrality_Usecase{500, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/centrality/mg_katz_centrality_test.cpp b/cpp/tests/centrality/mg_katz_centrality_test.cpp index e38f87749b8..15a4089fa1b 100644 --- a/cpp/tests/centrality/mg_katz_centrality_test.cpp +++ b/cpp/tests/centrality/mg_katz_centrality_test.cpp @@ -252,8 +252,18 @@ INSTANTIATE_TEST_SUITE_P( KatzCentrality_Usecase{false, true}, KatzCentrality_Usecase{true, false}, KatzCentrality_Usecase{true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGKatzCentrality_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(KatzCentrality_Usecase{false, false}, + KatzCentrality_Usecase{false, true}, + KatzCentrality_Usecase{true, false}, + KatzCentrality_Usecase{true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/community/mg_weighted_matching_test.cpp b/cpp/tests/community/mg_weighted_matching_test.cpp index 8abd7646065..5a150bbc1f7 100644 --- a/cpp/tests/community/mg_weighted_matching_test.cpp +++ b/cpp/tests/community/mg_weighted_matching_test.cpp @@ -69,7 +69,7 @@ class Tests_MGWeightedMatching constexpr bool multi_gpu = true; - bool test_weighted = false; + bool test_weighted = true; bool renumber = true; bool drop_self_loops = false; bool drop_multi_edges = false; diff --git a/cpp/tests/link_analysis/hits_test.cpp b/cpp/tests/link_analysis/hits_test.cpp index 31ed5537a6b..89bc0ca34c5 100644 --- a/cpp/tests/link_analysis/hits_test.cpp +++ b/cpp/tests/link_analysis/hits_test.cpp @@ -338,11 +338,21 @@ INSTANTIATE_TEST_SUITE_P( Hits_Usecase{true, false, true}, Hits_Usecase{true, true, true}), ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), - cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), - cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx"), cugraph::test::File_Usecase("test/datasets/dolphins.mtx")))); +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_Hits_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(Hits_Usecase{false, false, true}, + Hits_Usecase{false, true, true}, + Hits_Usecase{true, false, true}, + Hits_Usecase{true, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), + cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); + INSTANTIATE_TEST_SUITE_P(rmat_small_test, Tests_Hits_Rmat, // enable correctness checks diff --git a/cpp/tests/link_analysis/mg_hits_test.cpp b/cpp/tests/link_analysis/mg_hits_test.cpp index 40a439ffc4c..8a13204a5aa 100644 --- a/cpp/tests/link_analysis/mg_hits_test.cpp +++ b/cpp/tests/link_analysis/mg_hits_test.cpp @@ -289,8 +289,18 @@ INSTANTIATE_TEST_SUITE_P( Hits_Usecase{false, true}, Hits_Usecase{true, false}, Hits_Usecase{true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGHits_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(Hits_Usecase{false, false}, + Hits_Usecase{false, true}, + Hits_Usecase{true, false}, + Hits_Usecase{true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/link_analysis/mg_pagerank_test.cpp b/cpp/tests/link_analysis/mg_pagerank_test.cpp index 26136c8c9d2..a9e3a10b2ae 100644 --- a/cpp/tests/link_analysis/mg_pagerank_test.cpp +++ b/cpp/tests/link_analysis/mg_pagerank_test.cpp @@ -297,8 +297,22 @@ TEST_P(Tests_MGPageRank_Rmat, CheckInt64Int64FloatFloat) std::get<0>(param), override_Rmat_Usecase_with_cmd_line_arguments(std::get<1>(param))); } +INSTANTIATE_TEST_SUITE_P(file_tests, + Tests_MGPageRank_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(PageRank_Usecase{0.0, false, false}, + PageRank_Usecase{0.0, false, true}, + PageRank_Usecase{0.0, true, false}, + PageRank_Usecase{0.0, true, true}, + PageRank_Usecase{0.5, false, false}, + PageRank_Usecase{0.5, false, true}, + PageRank_Usecase{0.5, true, false}, + PageRank_Usecase{0.5, true, true}), + ::testing::Values(cugraph::test::File_Usecase("karate.csv")))); + INSTANTIATE_TEST_SUITE_P( - file_tests, + file_large_tests, Tests_MGPageRank_File, ::testing::Combine( // enable correctness checks @@ -310,8 +324,7 @@ INSTANTIATE_TEST_SUITE_P( PageRank_Usecase{0.5, false, true}, PageRank_Usecase{0.5, true, false}, PageRank_Usecase{0.5, true, true}), - ::testing::Values(cugraph::test::File_Usecase("karate.csv"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/prims/mg_count_if_e.cu b/cpp/tests/prims/mg_count_if_e.cu index 8ad34a5a724..88494132ac7 100644 --- a/cpp/tests/prims/mg_count_if_e.cu +++ b/cpp/tests/prims/mg_count_if_e.cu @@ -322,13 +322,21 @@ TEST_P(Tests_MGCountIfE_Rmat, CheckInt64Int64FloatTransposeTrue) INSTANTIATE_TEST_SUITE_P( file_test, Tests_MGCountIfE_File, + ::testing::Combine(::testing::Values(Prims_Usecase{false, false, true}, + Prims_Usecase{false, true, true}, + Prims_Usecase{true, false, true}, + Prims_Usecase{true, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGCountIfE_File, ::testing::Combine( ::testing::Values(Prims_Usecase{false, false, true}, Prims_Usecase{false, true, true}, Prims_Usecase{true, false, true}, Prims_Usecase{true, true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/prims/mg_count_if_v.cu b/cpp/tests/prims/mg_count_if_v.cu index e3f30e37729..56550027936 100644 --- a/cpp/tests/prims/mg_count_if_v.cu +++ b/cpp/tests/prims/mg_count_if_v.cu @@ -212,10 +212,15 @@ TEST_P(Tests_MGCountIfV_Rmat, CheckInt64Int64FloatTransposeTrue) INSTANTIATE_TEST_SUITE_P( file_test, Tests_MGCountIfV_File, + ::testing::Combine(::testing::Values(Prims_Usecase{true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGCountIfV_File, ::testing::Combine( ::testing::Values(Prims_Usecase{true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/prims/mg_extract_transform_e.cu b/cpp/tests/prims/mg_extract_transform_e.cu index d7aa953ef7c..e45f10c3175 100644 --- a/cpp/tests/prims/mg_extract_transform_e.cu +++ b/cpp/tests/prims/mg_extract_transform_e.cu @@ -365,10 +365,15 @@ TEST_P(Tests_MGExtractTransformE_Rmat, CheckInt64Int64FloatInt32Int32) INSTANTIATE_TEST_SUITE_P( file_test, Tests_MGExtractTransformE_File, + ::testing::Combine(::testing::Values(Prims_Usecase{false, true}, Prims_Usecase{true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGExtractTransformE_File, ::testing::Combine( ::testing::Values(Prims_Usecase{false, true}, Prims_Usecase{true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/prims/mg_extract_transform_v_frontier_outgoing_e.cu b/cpp/tests/prims/mg_extract_transform_v_frontier_outgoing_e.cu index 9e7611190ae..03f64e422c9 100644 --- a/cpp/tests/prims/mg_extract_transform_v_frontier_outgoing_e.cu +++ b/cpp/tests/prims/mg_extract_transform_v_frontier_outgoing_e.cu @@ -468,10 +468,15 @@ TEST_P(Tests_MGExtractTransformVFrontierOutgoingE_Rmat, CheckInt64Int64FloatInt3 INSTANTIATE_TEST_SUITE_P( file_test, Tests_MGExtractTransformVFrontierOutgoingE_File, + ::testing::Combine(::testing::Values(Prims_Usecase{false, true}, Prims_Usecase{true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGExtractTransformVFrontierOutgoingE_File, ::testing::Combine( ::testing::Values(Prims_Usecase{false, true}, Prims_Usecase{true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/prims/mg_per_v_random_select_transform_outgoing_e.cu b/cpp/tests/prims/mg_per_v_random_select_transform_outgoing_e.cu index 49c14631839..d77d8a7659e 100644 --- a/cpp/tests/prims/mg_per_v_random_select_transform_outgoing_e.cu +++ b/cpp/tests/prims/mg_per_v_random_select_transform_outgoing_e.cu @@ -563,8 +563,29 @@ INSTANTIATE_TEST_SUITE_P( Prims_Usecase{size_t{1000}, size_t{4}, true, true, false, true}, Prims_Usecase{size_t{1000}, size_t{4}, true, true, true, false}, Prims_Usecase{size_t{1000}, size_t{4}, true, true, true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGPerVRandomSelectTransformOutgoingE_File, + ::testing::Combine( + ::testing::Values(Prims_Usecase{size_t{1000}, size_t{4}, false, false, false, false}, + Prims_Usecase{size_t{1000}, size_t{4}, false, false, false, true}, + Prims_Usecase{size_t{1000}, size_t{4}, false, false, true, false}, + Prims_Usecase{size_t{1000}, size_t{4}, false, false, true, true}, + Prims_Usecase{size_t{1000}, size_t{4}, false, true, false, false}, + Prims_Usecase{size_t{1000}, size_t{4}, false, true, false, true}, + Prims_Usecase{size_t{1000}, size_t{4}, false, true, true, false}, + Prims_Usecase{size_t{1000}, size_t{4}, false, true, true, true}, + Prims_Usecase{size_t{1000}, size_t{4}, true, false, false, false}, + Prims_Usecase{size_t{1000}, size_t{4}, true, false, false, true}, + Prims_Usecase{size_t{1000}, size_t{4}, true, false, true, false}, + Prims_Usecase{size_t{1000}, size_t{4}, true, false, true, true}, + Prims_Usecase{size_t{1000}, size_t{4}, true, true, false, false}, + Prims_Usecase{size_t{1000}, size_t{4}, true, true, false, true}, + Prims_Usecase{size_t{1000}, size_t{4}, true, true, true, false}, + Prims_Usecase{size_t{1000}, size_t{4}, true, true, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/prims/mg_per_v_transform_reduce_dst_key_aggregated_outgoing_e.cu b/cpp/tests/prims/mg_per_v_transform_reduce_dst_key_aggregated_outgoing_e.cu index bef6395a780..e08baa610e5 100644 --- a/cpp/tests/prims/mg_per_v_transform_reduce_dst_key_aggregated_outgoing_e.cu +++ b/cpp/tests/prims/mg_per_v_transform_reduce_dst_key_aggregated_outgoing_e.cu @@ -568,13 +568,21 @@ TEST_P(Tests_MGPerVTransformReduceDstKeyAggregatedOutgoingE_Rmat, INSTANTIATE_TEST_SUITE_P( file_test, Tests_MGPerVTransformReduceDstKeyAggregatedOutgoingE_File, + ::testing::Combine(::testing::Values(Prims_Usecase{false, false, true}, + Prims_Usecase{false, true, true}, + Prims_Usecase{true, false, true}, + Prims_Usecase{true, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGPerVTransformReduceDstKeyAggregatedOutgoingE_File, ::testing::Combine( ::testing::Values(Prims_Usecase{false, false, true}, Prims_Usecase{false, true, true}, Prims_Usecase{true, false, true}, Prims_Usecase{true, true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/prims/mg_per_v_transform_reduce_incoming_outgoing_e.cu b/cpp/tests/prims/mg_per_v_transform_reduce_incoming_outgoing_e.cu index d4f102127c5..eb350ddb435 100644 --- a/cpp/tests/prims/mg_per_v_transform_reduce_incoming_outgoing_e.cu +++ b/cpp/tests/prims/mg_per_v_transform_reduce_incoming_outgoing_e.cu @@ -610,13 +610,21 @@ TEST_P(Tests_MGPerVTransformReduceIncomingOutgoingE_Rmat, CheckInt64Int64FloatTr INSTANTIATE_TEST_SUITE_P( file_test, Tests_MGPerVTransformReduceIncomingOutgoingE_File, + ::testing::Combine(::testing::Values(Prims_Usecase{false, false, true}, + Prims_Usecase{false, true, true}, + Prims_Usecase{true, false, true}, + Prims_Usecase{true, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGPerVTransformReduceIncomingOutgoingE_File, ::testing::Combine( ::testing::Values(Prims_Usecase{false, false, true}, Prims_Usecase{false, true, true}, Prims_Usecase{true, false, true}, Prims_Usecase{true, true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/prims/mg_reduce_v.cu b/cpp/tests/prims/mg_reduce_v.cu index e91db5fa6ad..0f53adcc71c 100644 --- a/cpp/tests/prims/mg_reduce_v.cu +++ b/cpp/tests/prims/mg_reduce_v.cu @@ -314,12 +314,18 @@ TEST_P(Tests_MGReduceV_Rmat, CheckInt64Int64FloatTransposeTrue) INSTANTIATE_TEST_SUITE_P( file_test, Tests_MGReduceV_File, + ::testing::Combine(::testing::Values(Prims_Usecase{true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGReduceV_File, ::testing::Combine( ::testing::Values(Prims_Usecase{true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); + INSTANTIATE_TEST_SUITE_P(rmat_small_test, Tests_MGReduceV_Rmat, ::testing::Combine(::testing::Values(Prims_Usecase{true}), diff --git a/cpp/tests/prims/mg_transform_e.cu b/cpp/tests/prims/mg_transform_e.cu index 2e8b2feaef3..2f131b4d54f 100644 --- a/cpp/tests/prims/mg_transform_e.cu +++ b/cpp/tests/prims/mg_transform_e.cu @@ -448,13 +448,21 @@ TEST_P(Tests_MGTransformE_Rmat, CheckInt64Int64FloatBoolTransposeTrue) INSTANTIATE_TEST_SUITE_P( file_test, Tests_MGTransformE_File, + ::testing::Combine(::testing::Values(Prims_Usecase{false, false, true}, + Prims_Usecase{false, true, true}, + Prims_Usecase{true, false, true}, + Prims_Usecase{true, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGTransformE_File, ::testing::Combine( ::testing::Values(Prims_Usecase{false, false, true}, Prims_Usecase{false, true, true}, Prims_Usecase{true, false, true}, Prims_Usecase{true, true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/prims/mg_transform_reduce_e.cu b/cpp/tests/prims/mg_transform_reduce_e.cu index 9f9e71f6e55..ff8bd42f1ad 100644 --- a/cpp/tests/prims/mg_transform_reduce_e.cu +++ b/cpp/tests/prims/mg_transform_reduce_e.cu @@ -338,13 +338,21 @@ TEST_P(Tests_MGTransformReduceE_Rmat, CheckInt64Int64FloatTransposeTrue) INSTANTIATE_TEST_SUITE_P( file_test, Tests_MGTransformReduceE_File, + ::testing::Combine(::testing::Values(Prims_Usecase{false, false, true}, + Prims_Usecase{false, true, true}, + Prims_Usecase{true, false, true}, + Prims_Usecase{true, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGTransformReduceE_File, ::testing::Combine( ::testing::Values(Prims_Usecase{false, false, true}, Prims_Usecase{false, true, true}, Prims_Usecase{true, false, true}, Prims_Usecase{true, true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/prims/mg_transform_reduce_e_by_src_dst_key.cu b/cpp/tests/prims/mg_transform_reduce_e_by_src_dst_key.cu index 67cacb27e0c..c7bb51e3635 100644 --- a/cpp/tests/prims/mg_transform_reduce_e_by_src_dst_key.cu +++ b/cpp/tests/prims/mg_transform_reduce_e_by_src_dst_key.cu @@ -464,13 +464,21 @@ TEST_P(Tests_MGTransformReduceEBySrcDstKey_Rmat, CheckInt64Int64FloatTransposeTr INSTANTIATE_TEST_SUITE_P( file_test, Tests_MGTransformReduceEBySrcDstKey_File, + ::testing::Combine(::testing::Values(Prims_Usecase{false, false, true}, + Prims_Usecase{false, true, true}, + Prims_Usecase{true, false, true}, + Prims_Usecase{true, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGTransformReduceEBySrcDstKey_File, ::testing::Combine( ::testing::Values(Prims_Usecase{false, false, true}, Prims_Usecase{false, true, true}, Prims_Usecase{true, false, true}, Prims_Usecase{true, true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/prims/mg_transform_reduce_v.cu b/cpp/tests/prims/mg_transform_reduce_v.cu index f6f07bc03ab..0e6d71094bd 100644 --- a/cpp/tests/prims/mg_transform_reduce_v.cu +++ b/cpp/tests/prims/mg_transform_reduce_v.cu @@ -356,10 +356,15 @@ TEST_P(Tests_MGTransformReduceV_Rmat, CheckInt64Int64FloatTransposeTrue) INSTANTIATE_TEST_SUITE_P( file_test, Tests_MGTransformReduceV_File, + ::testing::Combine(::testing::Values(Prims_Usecase{true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGTransformReduceV_File, ::testing::Combine( ::testing::Values(Prims_Usecase{true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/prims/mg_transform_reduce_v_frontier_outgoing_e_by_src_dst.cu b/cpp/tests/prims/mg_transform_reduce_v_frontier_outgoing_e_by_src_dst.cu index 596bb3688fb..5947dd9a560 100644 --- a/cpp/tests/prims/mg_transform_reduce_v_frontier_outgoing_e_by_src_dst.cu +++ b/cpp/tests/prims/mg_transform_reduce_v_frontier_outgoing_e_by_src_dst.cu @@ -721,10 +721,15 @@ TEST_P(Tests_MGTransformReduceVFrontierOutgoingEBySrcDst_Rmat, CheckInt64Int64Fl INSTANTIATE_TEST_SUITE_P( file_test, Tests_MGTransformReduceVFrontierOutgoingEBySrcDst_File, + ::testing::Combine(::testing::Values(Prims_Usecase{false, true}, Prims_Usecase{true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGTransformReduceVFrontierOutgoingEBySrcDst_File, ::testing::Combine( ::testing::Values(Prims_Usecase{false, true}, Prims_Usecase{true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/sampling/biased_neighbor_sampling.cpp b/cpp/tests/sampling/biased_neighbor_sampling.cpp index 8ee3ab27833..0b7307bef7d 100644 --- a/cpp/tests/sampling/biased_neighbor_sampling.cpp +++ b/cpp/tests/sampling/biased_neighbor_sampling.cpp @@ -258,13 +258,21 @@ TEST_P(Tests_Biased_Neighbor_Sampling_Rmat, CheckInt64Int64Float) INSTANTIATE_TEST_SUITE_P( file_test, Tests_Biased_Neighbor_Sampling_File, + ::testing::Combine(::testing::Values(Biased_Neighbor_Sampling_Usecase{{4, 10}, 128, false, false}, + Biased_Neighbor_Sampling_Usecase{{4, 10}, 128, false, true}, + Biased_Neighbor_Sampling_Usecase{{4, 10}, 128, true, false}, + Biased_Neighbor_Sampling_Usecase{{4, 10}, 128, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_Biased_Neighbor_Sampling_File, ::testing::Combine( ::testing::Values(Biased_Neighbor_Sampling_Usecase{{4, 10}, 128, false, false}, Biased_Neighbor_Sampling_Usecase{{4, 10}, 128, false, true}, Biased_Neighbor_Sampling_Usecase{{4, 10}, 128, true, false}, Biased_Neighbor_Sampling_Usecase{{4, 10}, 128, true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/sampling/mg_biased_neighbor_sampling.cpp b/cpp/tests/sampling/mg_biased_neighbor_sampling.cpp index 000b0a31e4b..8f08320aac8 100644 --- a/cpp/tests/sampling/mg_biased_neighbor_sampling.cpp +++ b/cpp/tests/sampling/mg_biased_neighbor_sampling.cpp @@ -313,13 +313,21 @@ TEST_P(Tests_MGBiased_Neighbor_Sampling_Rmat, CheckInt64Int64Float) INSTANTIATE_TEST_SUITE_P( file_test, Tests_MGBiased_Neighbor_Sampling_File, + ::testing::Combine(::testing::Values(Biased_Neighbor_Sampling_Usecase{{4, 10}, 128, false, false}, + Biased_Neighbor_Sampling_Usecase{{4, 10}, 128, false, true}, + Biased_Neighbor_Sampling_Usecase{{4, 10}, 128, true, false}, + Biased_Neighbor_Sampling_Usecase{{4, 10}, 128, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGBiased_Neighbor_Sampling_File, ::testing::Combine( ::testing::Values(Biased_Neighbor_Sampling_Usecase{{4, 10}, 128, false, false}, Biased_Neighbor_Sampling_Usecase{{4, 10}, 128, false, true}, Biased_Neighbor_Sampling_Usecase{{4, 10}, 128, true, false}, Biased_Neighbor_Sampling_Usecase{{4, 10}, 128, true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/sampling/mg_random_walks_test.cpp b/cpp/tests/sampling/mg_random_walks_test.cpp index dbafa9ac1f2..c2ad5c37e9e 100644 --- a/cpp/tests/sampling/mg_random_walks_test.cpp +++ b/cpp/tests/sampling/mg_random_walks_test.cpp @@ -286,35 +286,53 @@ TEST_P(Tests_Node2VecRandomWalks_Rmat, Initialize_i32_i32_f) } INSTANTIATE_TEST_SUITE_P( - simple_test, + file_test, + Tests_UniformRandomWalks_File, + ::testing::Combine(::testing::Values(UniformRandomWalks_Usecase{false, 0, true}, + UniformRandomWalks_Usecase{true, 0, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_test, + Tests_BiasedRandomWalks_File, + ::testing::Combine(::testing::Values(BiasedRandomWalks_Usecase{false, 0, true}, + BiasedRandomWalks_Usecase{true, 0, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_test, + Tests_Node2VecRandomWalks_File, + ::testing::Combine(::testing::Values(Node2VecRandomWalks_Usecase{4, 8, false, 0, true}, + Node2VecRandomWalks_Usecase{4, 8, true, 0, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, Tests_UniformRandomWalks_File, ::testing::Combine( ::testing::Values(UniformRandomWalks_Usecase{false, 0, true}, UniformRandomWalks_Usecase{true, 0, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); INSTANTIATE_TEST_SUITE_P( - simple_test, + file_large_test, Tests_BiasedRandomWalks_File, ::testing::Combine( ::testing::Values(BiasedRandomWalks_Usecase{false, 0, true}, BiasedRandomWalks_Usecase{true, 0, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); INSTANTIATE_TEST_SUITE_P( - simple_test, + file_large_test, Tests_Node2VecRandomWalks_File, ::testing::Combine( ::testing::Values(Node2VecRandomWalks_Usecase{4, 8, false, 0, true}, Node2VecRandomWalks_Usecase{4, 8, true, 0, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/sampling/mg_uniform_neighbor_sampling.cpp b/cpp/tests/sampling/mg_uniform_neighbor_sampling.cpp index ee9651d5e5d..23f1da40cf4 100644 --- a/cpp/tests/sampling/mg_uniform_neighbor_sampling.cpp +++ b/cpp/tests/sampling/mg_uniform_neighbor_sampling.cpp @@ -340,8 +340,17 @@ INSTANTIATE_TEST_SUITE_P( Uniform_Neighbor_Sampling_Usecase{{4, 10}, 128, false, true}, Uniform_Neighbor_Sampling_Usecase{{4, 10}, 128, true, false}, Uniform_Neighbor_Sampling_Usecase{{4, 10}, 128, true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGUniform_Neighbor_Sampling_File, + ::testing::Combine( + ::testing::Values(Uniform_Neighbor_Sampling_Usecase{{4, 10}, 128, false, false}, + Uniform_Neighbor_Sampling_Usecase{{4, 10}, 128, false, true}, + Uniform_Neighbor_Sampling_Usecase{{4, 10}, 128, true, false}, + Uniform_Neighbor_Sampling_Usecase{{4, 10}, 128, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/sampling/sg_random_walks_test.cpp b/cpp/tests/sampling/sg_random_walks_test.cpp index 6b0ce32fdfe..7409c2ab758 100644 --- a/cpp/tests/sampling/sg_random_walks_test.cpp +++ b/cpp/tests/sampling/sg_random_walks_test.cpp @@ -252,24 +252,36 @@ INSTANTIATE_TEST_SUITE_P( #endif INSTANTIATE_TEST_SUITE_P( - simple_test, + file_test, + Tests_BiasedRandomWalks_File, + ::testing::Combine(::testing::Values(BiasedRandomWalks_Usecase{false, 0, true}, + BiasedRandomWalks_Usecase{true, 0, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_test, + Tests_Node2VecRandomWalks_File, + ::testing::Combine(::testing::Values(Node2VecRandomWalks_Usecase{4, 8, false, 0, true}, + Node2VecRandomWalks_Usecase{4, 8, true, 0, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, Tests_BiasedRandomWalks_File, ::testing::Combine( ::testing::Values(BiasedRandomWalks_Usecase{false, 0, true}, BiasedRandomWalks_Usecase{true, 0, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); INSTANTIATE_TEST_SUITE_P( - simple_test, + file_large_test, Tests_Node2VecRandomWalks_File, ::testing::Combine( ::testing::Values(Node2VecRandomWalks_Usecase{4, 8, false, 0, true}, Node2VecRandomWalks_Usecase{4, 8, true, 0, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/sampling/uniform_neighbor_sampling.cpp b/cpp/tests/sampling/uniform_neighbor_sampling.cpp index 6ab11cc6c29..e348780e14b 100644 --- a/cpp/tests/sampling/uniform_neighbor_sampling.cpp +++ b/cpp/tests/sampling/uniform_neighbor_sampling.cpp @@ -278,8 +278,17 @@ INSTANTIATE_TEST_SUITE_P( Uniform_Neighbor_Sampling_Usecase{{4, 10}, 128, false, true}, Uniform_Neighbor_Sampling_Usecase{{4, 10}, 128, true, false}, Uniform_Neighbor_Sampling_Usecase{{4, 10}, 128, true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_Uniform_Neighbor_Sampling_File, + ::testing::Combine( + ::testing::Values(Uniform_Neighbor_Sampling_Usecase{{4, 10}, 128, false, false}, + Uniform_Neighbor_Sampling_Usecase{{4, 10}, 128, false, true}, + Uniform_Neighbor_Sampling_Usecase{{4, 10}, 128, true, false}, + Uniform_Neighbor_Sampling_Usecase{{4, 10}, 128, true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/structure/degree_test.cpp b/cpp/tests/structure/degree_test.cpp index 2a899c49f5e..ba387945424 100644 --- a/cpp/tests/structure/degree_test.cpp +++ b/cpp/tests/structure/degree_test.cpp @@ -145,10 +145,13 @@ TEST_P(Tests_Degree, CheckInt32Int32FloatTransposeTrue) run_current_test(GetParam()); } -INSTANTIATE_TEST_SUITE_P(simple_test, +INSTANTIATE_TEST_SUITE_P(file_test, Tests_Degree, - ::testing::Values(Degree_Usecase("test/datasets/karate.mtx"), - Degree_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(Degree_Usecase("test/datasets/karate.mtx"))); + +INSTANTIATE_TEST_SUITE_P(file_large_test, + Tests_Degree, + ::testing::Values(Degree_Usecase("test/datasets/web-Google.mtx"), Degree_Usecase("test/datasets/ljournal-2008.mtx"), Degree_Usecase("test/datasets/webbase-1M.mtx"))); diff --git a/cpp/tests/structure/induced_subgraph_test.cpp b/cpp/tests/structure/induced_subgraph_test.cpp index 73168819600..2df8df0a4e5 100644 --- a/cpp/tests/structure/induced_subgraph_test.cpp +++ b/cpp/tests/structure/induced_subgraph_test.cpp @@ -274,7 +274,7 @@ TEST_P(Tests_InducedSubgraph_Rmat, CheckInt32Int32FloatTransposeTrue) #endif INSTANTIATE_TEST_SUITE_P( - karate_test, + file_test, Tests_InducedSubgraph_File, ::testing::Combine( ::testing::Values(InducedSubgraph_Usecase{std::vector{0}, false, false}, @@ -294,7 +294,7 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); INSTANTIATE_TEST_SUITE_P( - web_google_test, + web_google_large_test, Tests_InducedSubgraph_File, ::testing::Combine( ::testing::Values(InducedSubgraph_Usecase{std::vector{250, 130, 15}, false, false}, @@ -304,7 +304,7 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx")))); INSTANTIATE_TEST_SUITE_P( - ljournal_2008_test, + ljournal_2008_large_test, Tests_InducedSubgraph_File, ::testing::Combine( ::testing::Values(InducedSubgraph_Usecase{std::vector{9130, 1200, 300}, false, false}, @@ -314,7 +314,7 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values(cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx")))); INSTANTIATE_TEST_SUITE_P( - webbase_1M_test, + webbase_1M_large_test, Tests_InducedSubgraph_File, ::testing::Combine( ::testing::Values(InducedSubgraph_Usecase{std::vector{700}, false, false}, diff --git a/cpp/tests/structure/mg_induced_subgraph_test.cu b/cpp/tests/structure/mg_induced_subgraph_test.cu index 9f8ef1debcf..fd636e8c4c9 100644 --- a/cpp/tests/structure/mg_induced_subgraph_test.cu +++ b/cpp/tests/structure/mg_induced_subgraph_test.cu @@ -322,7 +322,7 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); INSTANTIATE_TEST_SUITE_P( - web_google_test, + web_google_large_test, Tests_MGInducedSubgraph_File, ::testing::Combine( ::testing::Values(InducedSubgraph_Usecase{std::vector{250, 130, 15}, false, false}, @@ -332,7 +332,7 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx")))); INSTANTIATE_TEST_SUITE_P( - ljournal_2008_test, + ljournal_2008_large_test, Tests_MGInducedSubgraph_File, ::testing::Combine( ::testing::Values(InducedSubgraph_Usecase{std::vector{300, 20, 400}, false, false}, @@ -342,7 +342,7 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values(cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx")))); INSTANTIATE_TEST_SUITE_P( - webbase_1M_test, + webbase_1M_large_test, Tests_MGInducedSubgraph_File, ::testing::Combine( ::testing::Values(InducedSubgraph_Usecase{std::vector{700}, false, false}, diff --git a/cpp/tests/structure/mg_symmetrize_test.cpp b/cpp/tests/structure/mg_symmetrize_test.cpp index 7f1e4f04dc7..7d83b482ccf 100644 --- a/cpp/tests/structure/mg_symmetrize_test.cpp +++ b/cpp/tests/structure/mg_symmetrize_test.cpp @@ -299,8 +299,18 @@ INSTANTIATE_TEST_SUITE_P( Symmetrize_Usecase{true, false}, Symmetrize_Usecase{false, true}, Symmetrize_Usecase{true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGSymmetrize_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(Symmetrize_Usecase{false, false}, + Symmetrize_Usecase{true, false}, + Symmetrize_Usecase{false, true}, + Symmetrize_Usecase{true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); INSTANTIATE_TEST_SUITE_P( diff --git a/cpp/tests/structure/mg_transpose_storage_test.cpp b/cpp/tests/structure/mg_transpose_storage_test.cpp index e870f648039..c43561a4251 100644 --- a/cpp/tests/structure/mg_transpose_storage_test.cpp +++ b/cpp/tests/structure/mg_transpose_storage_test.cpp @@ -295,8 +295,15 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Combine( // enable correctness checks ::testing::Values(TransposeStorage_Usecase{false}, TransposeStorage_Usecase{true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGTransposeStorage_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(TransposeStorage_Usecase{false}, TransposeStorage_Usecase{true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); INSTANTIATE_TEST_SUITE_P( diff --git a/cpp/tests/structure/mg_transpose_test.cpp b/cpp/tests/structure/mg_transpose_test.cpp index 921cef42595..a1a816e6f4b 100644 --- a/cpp/tests/structure/mg_transpose_test.cpp +++ b/cpp/tests/structure/mg_transpose_test.cpp @@ -291,8 +291,15 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Combine( // enable correctness checks ::testing::Values(Transpose_Usecase{false}, Transpose_Usecase{true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGTranspose_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(Transpose_Usecase{false}, Transpose_Usecase{true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); INSTANTIATE_TEST_SUITE_P( diff --git a/cpp/tests/structure/renumbering_test.cpp b/cpp/tests/structure/renumbering_test.cpp index d5829be6fe3..dcf278ab05a 100644 --- a/cpp/tests/structure/renumbering_test.cpp +++ b/cpp/tests/structure/renumbering_test.cpp @@ -142,15 +142,21 @@ TEST_P(Tests_Renumbering_Rmat, CheckInt32Int32FloatFloat) run_current_test(std::get<0>(param), std::get<1>(param)); } +INSTANTIATE_TEST_SUITE_P(file_test, + Tests_Renumbering_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(Renumbering_Usecase{}), + ::testing::Values(cugraph::test::File_Usecase("negative-vertex-id.csv"), + cugraph::test::File_Usecase("karate.csv")))); + INSTANTIATE_TEST_SUITE_P( - file_test, + file_large_test, Tests_Renumbering_File, ::testing::Combine( // enable correctness checks ::testing::Values(Renumbering_Usecase{}), - ::testing::Values(cugraph::test::File_Usecase("negative-vertex-id.csv"), - cugraph::test::File_Usecase("karate.csv"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/structure/symmetrize_test.cpp b/cpp/tests/structure/symmetrize_test.cpp index a0d7118b098..bf5e28e472f 100644 --- a/cpp/tests/structure/symmetrize_test.cpp +++ b/cpp/tests/structure/symmetrize_test.cpp @@ -471,8 +471,18 @@ INSTANTIATE_TEST_SUITE_P( Symmetrize_Usecase{true, false}, Symmetrize_Usecase{false, true}, Symmetrize_Usecase{true, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_Symmetrize_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(Symmetrize_Usecase{false, false}, + Symmetrize_Usecase{true, false}, + Symmetrize_Usecase{false, true}, + Symmetrize_Usecase{true, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); INSTANTIATE_TEST_SUITE_P( diff --git a/cpp/tests/structure/transpose_storage_test.cpp b/cpp/tests/structure/transpose_storage_test.cpp index 144cebd98d6..d14f975b382 100644 --- a/cpp/tests/structure/transpose_storage_test.cpp +++ b/cpp/tests/structure/transpose_storage_test.cpp @@ -248,8 +248,15 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Combine( // enable correctness checks ::testing::Values(TransposeStorage_Usecase{false}, TransposeStorage_Usecase{true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_TransposeStorage_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(TransposeStorage_Usecase{false}, TransposeStorage_Usecase{true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); INSTANTIATE_TEST_SUITE_P( diff --git a/cpp/tests/structure/transpose_test.cpp b/cpp/tests/structure/transpose_test.cpp index eba99b52730..4b666d72814 100644 --- a/cpp/tests/structure/transpose_test.cpp +++ b/cpp/tests/structure/transpose_test.cpp @@ -232,8 +232,15 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Combine( // enable correctness checks ::testing::Values(Transpose_Usecase{false}, Transpose_Usecase{true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_Transpose_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(Transpose_Usecase{false}, Transpose_Usecase{true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); INSTANTIATE_TEST_SUITE_P( diff --git a/cpp/tests/structure/weight_sum_test.cpp b/cpp/tests/structure/weight_sum_test.cpp index fc22232f21c..6f600e69431 100644 --- a/cpp/tests/structure/weight_sum_test.cpp +++ b/cpp/tests/structure/weight_sum_test.cpp @@ -247,8 +247,15 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Combine( // enable correctness checks ::testing::Values(WeightSum_Usecase{false}, WeightSum_Usecase{true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_WeightSum_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(WeightSum_Usecase{false}, WeightSum_Usecase{true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/traversal/mg_bfs_test.cpp b/cpp/tests/traversal/mg_bfs_test.cpp index aa59719f814..c294c6d0091 100644 --- a/cpp/tests/traversal/mg_bfs_test.cpp +++ b/cpp/tests/traversal/mg_bfs_test.cpp @@ -314,8 +314,15 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Combine( // enable correctness checks ::testing::Values(BFS_Usecase{0, false}, BFS_Usecase{0, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGBFS_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(BFS_Usecase{0, false}, BFS_Usecase{0, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); diff --git a/cpp/tests/traversal/mg_k_hop_nbrs_test.cpp b/cpp/tests/traversal/mg_k_hop_nbrs_test.cpp index 64674fb3799..311d1f55b9c 100644 --- a/cpp/tests/traversal/mg_k_hop_nbrs_test.cpp +++ b/cpp/tests/traversal/mg_k_hop_nbrs_test.cpp @@ -277,8 +277,18 @@ INSTANTIATE_TEST_SUITE_P( KHopNbrs_Usecase{1024, 2, true}, KHopNbrs_Usecase{1024, 1, false}, KHopNbrs_Usecase{1024, 1, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); + +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_MGKHopNbrs_File, + ::testing::Combine( + // enable correctness checks + ::testing::Values(KHopNbrs_Usecase{1024, 2, false}, + KHopNbrs_Usecase{1024, 2, true}, + KHopNbrs_Usecase{1024, 1, false}, + KHopNbrs_Usecase{1024, 1, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); From 79a2b2a8b72a0d7125e54a5d25a66fb643715d1f Mon Sep 17 00:00:00 2001 From: Chuck Hastings <45364586+ChuckHastings@users.noreply.github.com> Date: Thu, 18 Jul 2024 18:42:45 -0400 Subject: [PATCH 06/11] Define and Implement C API for biased sampling (#4535) Defined and implements C API for biased sampling. API includes a parameter for handling edge bias input that is not the weight of the edge. However, there is currently no mechanism to create the `cugraph_edge_property_view_t *` value that would need to be passed in. There are some proposed functions to support that included in `property.h`, but they are inside an `#ifdef 0` block so that they won't be compiled. Authors: - Chuck Hastings (https://github.com/ChuckHastings) Approvers: - Alex Barghi (https://github.com/alexbarghi-nv) - Seunghwa Kang (https://github.com/seunghwak) URL: https://github.com/rapidsai/cugraph/pull/4535 --- cpp/CMakeLists.txt | 2 +- cpp/include/cugraph_c/properties.h | 193 +++ cpp/include/cugraph_c/sampling_algorithms.h | 60 + ...bor_sampling.cpp => neighbor_sampling.cpp} | 429 +++++ cpp/src/c_api/properties.hpp | 44 + cpp/tests/CMakeLists.txt | 2 + cpp/tests/c_api/biased_neighbor_sample_test.c | 973 ++++++++++++ .../c_api/mg_biased_neighbor_sample_test.c | 1398 +++++++++++++++++ .../pylibcugraph/_cugraph_c/properties.pxd | 29 + .../_cugraph_c/sampling_algorithms.pxd | 21 + 10 files changed, 3150 insertions(+), 1 deletion(-) create mode 100644 cpp/include/cugraph_c/properties.h rename cpp/src/c_api/{uniform_neighbor_sampling.cpp => neighbor_sampling.cpp} (67%) create mode 100644 cpp/src/c_api/properties.hpp create mode 100644 cpp/tests/c_api/biased_neighbor_sample_test.c create mode 100644 cpp/tests/c_api/mg_biased_neighbor_sample_test.c create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/properties.pxd diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index cf511b1e08a..61ec34c3319 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -652,7 +652,7 @@ add_library(cugraph_c src/c_api/lookup_src_dst.cpp src/c_api/louvain.cpp src/c_api/triangle_count.cpp - src/c_api/uniform_neighbor_sampling.cpp + src/c_api/neighbor_sampling.cpp src/c_api/labeling_result.cpp src/c_api/weakly_connected_components.cpp src/c_api/strongly_connected_components.cpp diff --git a/cpp/include/cugraph_c/properties.h b/cpp/include/cugraph_c/properties.h new file mode 100644 index 00000000000..d7775bbf783 --- /dev/null +++ b/cpp/include/cugraph_c/properties.h @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// +// Speculative description of handling generic vertex and edge properties. +// +// If we have vertex properties and edge properties that we want to apply to an existing graph +// (after it was created) we could use these methods to construct C++ objects to represent these +// properties. +// +// These assume the use of external vertex ids and external edge ids as the mechanism for +// correlating a property to a particular vertex or edge. +// + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + int32_t align_; +} cugraph_vertex_property_t; + +typedef struct { + int32_t align_; +} cugraph_edge_property_t; + +typedef struct { + int32_t align_; +} cugraph_vertex_property_view_t; + +typedef struct { + int32_t align_; +} cugraph_edge_property_view_t; + +#if 0 +// Blocking out definition of these since this is speculative work. + +/** + * @brief Create a vertex property + * + * @param [in] handle Handle for accessing resources + * @param [in] graph Pointer to graph. + * @param [in] vertex_ids Device array of vertex ids + * @param [in] property Device array of vertex property + * @param [in] default_property Device array of vertex property + * @param [out] result Pointer to the location to store the pointer to the vertex property object + * @param [out] error Pointer to an error object storing details of any error. Will + * be populated if error code is not CUGRAPH_SUCCESS + * @return error code + */ +cugraph_error_code_t cugraph_vertex_property_create( + const cugraph_resource_handle_t* handle, + const cugraph_graph_t * graph, + const cugraph_type_erased_device_array_t* vertex_ids, + const cugraph_type_erased_device_array_t* properties, + const cugraph_type_erased_scalar_t* default_property, + cugraph_vertex_property_t** result, + cugraph_error_t** error); + +/** + * @brief Create an edge property + * + * @param [in] handle Handle for accessing resources + * @param [in] graph Pointer to graph. + * @param [in] lookup_container Lookup map + * @param [in] edge_ids Device array of edge ids + * @param [in] property Device array of edge property + * @param [in] default_property Device array of vertex property + * @param [out] result Pointer to the location to store the pointer to the edge property object + * @param [out] error Pointer to an error object storing details of any error. Will + * be populated if error code is not CUGRAPH_SUCCESS + * @return error code + */ +cugraph_error_code_t cugraph_edge_property_create( + const cugraph_resource_handle_t* handle, + const cugraph_graph_t * graph, + const cugraph_lookup_container_t* lookup_container, + const cugraph_type_erased_device_array_t* edge_ids, + const cugraph_type_erased_device_array_t* properties, + const cugraph_type_erased_scalar_t* default_property, + cugraph_edge_property_t** result, + cugraph_error_t** error); + +/** + * @brief Update an existing vertex property + * + * @param [in] handle Handle for accessing resources + * @param [in] graph Pointer to graph. + * @param [in] vertex_ids Device array of vertex ids to update + * @param [in] property Device array of vertex properties to update + * @param [in/out] result Pointer to the vertex property object to update + * @param [out] error Pointer to an error object storing details of any error. Will + * be populated if error code is not CUGRAPH_SUCCESS + * @return error code + */ +cugraph_error_code_t cugraph_vertex_property_update( + const cugraph_resource_handle_t* handle, + const cugraph_graph_t * graph, + const cugraph_type_erased_device_array_t* vertex_ids, + const cugraph_type_erased_device_array_t* properties, + const cugraph_type_erased_scalar_t* default_property, + cugraph_vertex_property_view_t* result, + cugraph_error_t** error); + +/** + * @brief Update an existing edge property + * + * @param [in] handle Handle for accessing resources + * @param [in] graph Pointer to graph. + * @param [in] lookup_container Lookup map + * @param [in] edge_ids Device array of edge ids to update + * @param [in] property Device array of edge properties to update + * @param [in/out] result Pointer to the edge property object to update + * @param [out] error Pointer to an error object storing details of any error. Will + * be populated if error code is not CUGRAPH_SUCCESS + * @return error code + */ +cugraph_error_code_t cugraph_edge_property_create( + const cugraph_resource_handle_t* handle, + const cugraph_graph_t * graph, + const cugraph_lookup_container_t* lookup_container, + const cugraph_type_erased_device_array_t* edge_ids, + const cugraph_type_erased_device_array_t* properties, + cugraph_edge_property_view_t* result, + cugraph_error_t** error); + +/** + * @brief Create a vertex_property_view from a vertex property + * + * @param [in] vertex_property Pointer to the vertex property object + * @return Pointer to the view of the host array + */ +cugraph_vertex_property_view_t* cugraph_vertex_property_view( + cugraph_vertex_property_view* vertex_property); + +/** + * @brief Create a edge_property_view from a edge property + * + * @param [in] edge_property Pointer to the edge property object + * @return Pointer to the view of the host array + */ +cugraph_edge_property_view_t* cugraph_edge_property_view( + cugraph_edge_property_view* edge_property); + +/** + * @brief Destroy a vertex_property object + * + * @param [in] p Pointer to the vertex_property object + */ +void cugraph_vertex_property_free(cugraph_vertex_property_t* p); + +/** + * @brief Destroy a edge_property object + * + * @param [in] p Pointer to the edge_property object + */ +void cugraph_edge_property_free(cugraph_edge_property_t* p); + +/** + * @brief Destroy a vertex_property_view object + * + * @param [in] p Pointer to the vertex_property_view object + */ +void cugraph_vertex_property_view_free(cugraph_vertex_property__viewt* p); + +/** + * @brief Destroy a edge_property_view object + * + * @param [in] p Pointer to the edge_property_view object + */ +void cugraph_edge_property_view_free(cugraph_edge_property_view_t* p); +#endif + +#ifdef __cplusplus +} +#endif diff --git a/cpp/include/cugraph_c/sampling_algorithms.h b/cpp/include/cugraph_c/sampling_algorithms.h index a7490ad2c63..1a3d20b9339 100644 --- a/cpp/include/cugraph_c/sampling_algorithms.h +++ b/cpp/include/cugraph_c/sampling_algorithms.h @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -373,6 +374,65 @@ cugraph_error_code_t cugraph_uniform_neighbor_sample( cugraph_sample_result_t** result, cugraph_error_t** error); +/** + * @brief Biased Neighborhood Sampling + * + * Returns a sample of the neighborhood around specified start vertices. Optionally, each + * start vertex can be associated with a label, allowing the caller to specify multiple batches + * of sampling requests in the same function call - which should improve GPU utilization. + * + * If label is NULL then all start vertices will be considered part of the same batch and the + * return value will not have a label column. + * + * @param [in] handle Handle for accessing resources + * @param [in] graph Pointer to graph. NOTE: Graph might be modified if the storage + * needs to be transposed + * @param [in] edge_biases Device array of edge biases to use for sampling. If NULL + * use the edge weight as the bias. NOTE: This is a placeholder for future capability, the + * value for edge_biases should always be set to NULL at the moment. + * @param [in] start_vertices Device array of start vertices for the sampling + * @param [in] start_vertex_labels Device array of start vertex labels for the sampling. The + * labels associated with each start vertex will be included in the output associated with results + * that were derived from that start vertex. We only support label of type INT32. If label is + * NULL, the return data will not be labeled. + * @param [in] label_list Device array of the labels included in @p start_vertex_labels. If + * @p label_to_comm_rank is not specified this parameter is ignored. If specified, label_list + * must be sorted in ascending order. + * @param [in] label_to_comm_rank Device array identifying which comm rank the output for a + * particular label should be shuffled in the output. If not specifed the data is not organized in + * output. If specified then the all data from @p label_list[i] will be shuffled to rank @p. This + * cannot be specified unless @p start_vertex_labels is also specified + * label_to_comm_rank[i]. If not specified then the output data will not be shuffled between ranks. + * @param [in] label_offsets Device array of the offsets for each label in the seed list. This + * parameter is only used with the retain_seeds option. + * @param [in] fanout Host array defining the fan out at each step in the sampling algorithm. + * We only support fanout values of type INT32 + * @param [in,out] rng_state State of the random number generator, updated with each call + * @param [in] sampling_options + * Opaque pointer defining the sampling options. + * @param [in] do_expensive_check + * A flag to run expensive checks for input arguments (if set to true) + * @param [out] result Output from the uniform_neighbor_sample call + * @param [out] error Pointer to an error object storing details of any error. Will + * be populated if error code is not CUGRAPH_SUCCESS + * @return error code + */ +cugraph_error_code_t cugraph_biased_neighbor_sample( + const cugraph_resource_handle_t* handle, + cugraph_graph_t* graph, + const cugraph_edge_property_view_t* edge_biases, + const cugraph_type_erased_device_array_view_t* start_vertices, + const cugraph_type_erased_device_array_view_t* start_vertex_labels, + const cugraph_type_erased_device_array_view_t* label_list, + const cugraph_type_erased_device_array_view_t* label_to_comm_rank, + const cugraph_type_erased_device_array_view_t* label_offsets, + const cugraph_type_erased_host_array_view_t* fan_out, + cugraph_rng_state_t* rng_state, + const cugraph_sampling_options_t* options, + bool_t do_expensive_check, + cugraph_sample_result_t** result, + cugraph_error_t** error); + /** * @deprecated This call should be replaced with cugraph_sample_result_get_majors * @brief Get the source vertices from the sampling algorithm result diff --git a/cpp/src/c_api/uniform_neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp similarity index 67% rename from cpp/src/c_api/uniform_neighbor_sampling.cpp rename to cpp/src/c_api/neighbor_sampling.cpp index 45609fc0e01..69306806030 100644 --- a/cpp/src/c_api/uniform_neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -16,6 +16,7 @@ #include "c_api/abstract_functor.hpp" #include "c_api/graph.hpp" +#include "c_api/properties.hpp" #include "c_api/random.hpp" #include "c_api/resource_handle.hpp" #include "c_api/utils.hpp" @@ -402,6 +403,356 @@ struct uniform_neighbor_sampling_functor : public cugraph::c_api::abstract_funct } }; +struct biased_neighbor_sampling_functor : public cugraph::c_api::abstract_functor { + raft::handle_t const& handle_; + cugraph::c_api::cugraph_graph_t* graph_{nullptr}; + cugraph::c_api::cugraph_edge_property_view_t const* edge_biases_{nullptr}; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* start_vertices_{nullptr}; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* start_vertex_labels_{nullptr}; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* label_list_{nullptr}; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* label_to_comm_rank_{nullptr}; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* label_offsets_{nullptr}; + cugraph::c_api::cugraph_type_erased_host_array_view_t const* fan_out_{nullptr}; + cugraph::c_api::cugraph_rng_state_t* rng_state_{nullptr}; + cugraph::c_api::cugraph_sampling_options_t options_{}; + bool do_expensive_check_{false}; + cugraph::c_api::cugraph_sample_result_t* result_{nullptr}; + + biased_neighbor_sampling_functor( + cugraph_resource_handle_t const* handle, + cugraph_graph_t* graph, + cugraph_edge_property_view_t const* edge_biases, + cugraph_type_erased_device_array_view_t const* start_vertices, + cugraph_type_erased_device_array_view_t const* start_vertex_labels, + cugraph_type_erased_device_array_view_t const* label_list, + cugraph_type_erased_device_array_view_t const* label_to_comm_rank, + cugraph_type_erased_device_array_view_t const* label_offsets, + cugraph_type_erased_host_array_view_t const* fan_out, + cugraph_rng_state_t* rng_state, + cugraph::c_api::cugraph_sampling_options_t options, + bool do_expensive_check) + : abstract_functor(), + handle_(*reinterpret_cast(handle)->handle_), + graph_(reinterpret_cast(graph)), + edge_biases_( + reinterpret_cast(edge_biases)), + start_vertices_( + reinterpret_cast( + start_vertices)), + start_vertex_labels_( + reinterpret_cast( + start_vertex_labels)), + label_list_(reinterpret_cast( + label_list)), + label_to_comm_rank_( + reinterpret_cast( + label_to_comm_rank)), + label_offsets_( + reinterpret_cast( + label_offsets)), + fan_out_( + reinterpret_cast(fan_out)), + rng_state_(reinterpret_cast(rng_state)), + options_(options), + do_expensive_check_(do_expensive_check) + { + } + + template + void operator()() + { + using label_t = int32_t; + + // FIXME: Think about how to handle SG vice MG + if constexpr (!cugraph::is_candidate::value) { + unsupported(); + } else { + // uniform_nbr_sample expects store_transposed == false + if constexpr (store_transposed) { + error_code_ = cugraph::c_api:: + transpose_storage( + handle_, graph_, error_.get()); + if (error_code_ != CUGRAPH_SUCCESS) return; + } + + auto graph = + reinterpret_cast*>(graph_->graph_); + + auto graph_view = graph->view(); + + auto edge_weights = reinterpret_cast< + cugraph::edge_property_t, + weight_t>*>(graph_->edge_weights_); + + auto edge_ids = reinterpret_cast< + cugraph::edge_property_t, + edge_t>*>(graph_->edge_ids_); + + auto edge_types = reinterpret_cast< + cugraph::edge_property_t, + edge_type_t>*>(graph_->edge_types_); + + auto number_map = reinterpret_cast*>(graph_->number_map_); + + auto edge_biases = + edge_biases_ ? reinterpret_cast*>( + edge_biases_->edge_property_) + : nullptr; + + rmm::device_uvector start_vertices(start_vertices_->size_, handle_.get_stream()); + raft::copy(start_vertices.data(), + start_vertices_->as_type(), + start_vertices.size(), + handle_.get_stream()); + + std::optional> start_vertex_labels{std::nullopt}; + + if (start_vertex_labels_ != nullptr) { + start_vertex_labels = + rmm::device_uvector{start_vertex_labels_->size_, handle_.get_stream()}; + raft::copy(start_vertex_labels->data(), + start_vertex_labels_->as_type(), + start_vertex_labels_->size_, + handle_.get_stream()); + } + + if constexpr (multi_gpu) { + if (start_vertex_labels) { + std::tie(start_vertices, *start_vertex_labels) = + cugraph::detail::shuffle_ext_vertex_value_pairs_to_local_gpu_by_vertex_partitioning( + handle_, std::move(start_vertices), std::move(*start_vertex_labels)); + } else { + start_vertices = + cugraph::detail::shuffle_ext_vertices_to_local_gpu_by_vertex_partitioning( + handle_, std::move(start_vertices)); + } + } + + // + // Need to renumber start_vertices + // + cugraph::renumber_local_ext_vertices( + handle_, + start_vertices.data(), + start_vertices.size(), + number_map->data(), + graph_view.local_vertex_partition_range_first(), + graph_view.local_vertex_partition_range_last(), + do_expensive_check_); + + auto&& [src, dst, wgt, edge_id, edge_type, hop, edge_label, offsets] = + cugraph::biased_neighbor_sample( + handle_, + graph_view, + (edge_weights != nullptr) ? std::make_optional(edge_weights->view()) : std::nullopt, + (edge_ids != nullptr) ? std::make_optional(edge_ids->view()) : std::nullopt, + (edge_types != nullptr) ? std::make_optional(edge_types->view()) : std::nullopt, + (edge_biases != nullptr) ? *edge_biases : edge_weights->view(), + raft::device_span{start_vertices.data(), start_vertices.size()}, + (start_vertex_labels_ != nullptr) + ? std::make_optional>(start_vertex_labels->data(), + start_vertex_labels->size()) + : std::nullopt, + (label_list_ != nullptr) + ? std::make_optional(std::make_tuple( + raft::device_span{label_list_->as_type(), + label_list_->size_}, + raft::device_span{label_to_comm_rank_->as_type(), + label_to_comm_rank_->size_})) + : std::nullopt, + raft::host_span(fan_out_->as_type(), fan_out_->size_), + rng_state_->rng_state_, + options_.return_hops_, + options_.with_replacement_, + options_.prior_sources_behavior_, + options_.dedupe_sources_, + do_expensive_check_); + + std::vector vertex_partition_lasts = graph_view.vertex_partition_range_lasts(); + + cugraph::unrenumber_int_vertices(handle_, + src.data(), + src.size(), + number_map->data(), + vertex_partition_lasts, + do_expensive_check_); + + cugraph::unrenumber_int_vertices(handle_, + dst.data(), + dst.size(), + number_map->data(), + vertex_partition_lasts, + do_expensive_check_); + + std::optional> majors{std::nullopt}; + rmm::device_uvector minors(0, handle_.get_stream()); + std::optional> major_offsets{std::nullopt}; + + std::optional> label_hop_offsets{std::nullopt}; + + std::optional> renumber_map{std::nullopt}; + std::optional> renumber_map_offsets{std::nullopt}; + + bool src_is_major = (options_.compression_type_ == cugraph_compression_type_t::CSR) || + (options_.compression_type_ == cugraph_compression_type_t::DCSR) || + (options_.compression_type_ == cugraph_compression_type_t::COO); + + if (options_.renumber_results_) { + if (options_.compression_type_ == cugraph_compression_type_t::COO) { + // COO + + rmm::device_uvector output_majors(0, handle_.get_stream()); + rmm::device_uvector output_renumber_map(0, handle_.get_stream()); + std::tie(output_majors, + minors, + wgt, + edge_id, + edge_type, + label_hop_offsets, + output_renumber_map, + renumber_map_offsets) = + cugraph::renumber_and_sort_sampled_edgelist( + handle_, + std::move(src), + std::move(dst), + std::move(wgt), + std::move(edge_id), + std::move(edge_type), + std::move(hop), + options_.retain_seeds_ + ? std::make_optional(raft::device_span{ + start_vertices_->as_type(), start_vertices_->size_}) + : std::nullopt, + options_.retain_seeds_ ? std::make_optional(raft::device_span{ + label_offsets_->as_type(), label_offsets_->size_}) + : std::nullopt, + offsets ? std::make_optional( + raft::device_span{offsets->data(), offsets->size()}) + : std::nullopt, + edge_label ? edge_label->size() : size_t{1}, + hop ? fan_out_->size_ : size_t{1}, + src_is_major, + do_expensive_check_); + + majors.emplace(std::move(output_majors)); + renumber_map.emplace(std::move(output_renumber_map)); + } else { + // (D)CSC, (D)CSR + + bool doubly_compress = (options_.compression_type_ == cugraph_compression_type_t::DCSR) || + (options_.compression_type_ == cugraph_compression_type_t::DCSC); + + rmm::device_uvector output_major_offsets(0, handle_.get_stream()); + rmm::device_uvector output_renumber_map(0, handle_.get_stream()); + std::tie(majors, + output_major_offsets, + minors, + wgt, + edge_id, + edge_type, + label_hop_offsets, + output_renumber_map, + renumber_map_offsets) = + cugraph::renumber_and_compress_sampled_edgelist( + handle_, + std::move(src), + std::move(dst), + std::move(wgt), + std::move(edge_id), + std::move(edge_type), + std::move(hop), + options_.retain_seeds_ + ? std::make_optional(raft::device_span{ + start_vertices_->as_type(), start_vertices_->size_}) + : std::nullopt, + options_.retain_seeds_ ? std::make_optional(raft::device_span{ + label_offsets_->as_type(), label_offsets_->size_}) + : std::nullopt, + offsets ? std::make_optional( + raft::device_span{offsets->data(), offsets->size()}) + : std::nullopt, + edge_label ? edge_label->size() : size_t{1}, + hop ? fan_out_->size_ : size_t{1}, + src_is_major, + options_.compress_per_hop_, + doubly_compress, + do_expensive_check_); + + renumber_map.emplace(std::move(output_renumber_map)); + major_offsets.emplace(std::move(output_major_offsets)); + } + + // These are now represented by label_hop_offsets + hop.reset(); + offsets.reset(); + } else { + if (options_.compression_type_ != cugraph_compression_type_t::COO) { + CUGRAPH_FAIL("Can only use COO format if not renumbering"); + } + + std::tie(src, dst, wgt, edge_id, edge_type, label_hop_offsets) = + cugraph::sort_sampled_edgelist(handle_, + std::move(src), + std::move(dst), + std::move(wgt), + std::move(edge_id), + std::move(edge_type), + std::move(hop), + offsets + ? std::make_optional(raft::device_span{ + offsets->data(), offsets->size()}) + : std::nullopt, + edge_label ? edge_label->size() : size_t{1}, + hop ? fan_out_->size_ : size_t{1}, + src_is_major, + do_expensive_check_); + + majors.emplace(std::move(src)); + minors = std::move(dst); + + hop.reset(); + offsets.reset(); + } + + result_ = new cugraph::c_api::cugraph_sample_result_t{ + (major_offsets) + ? new cugraph::c_api::cugraph_type_erased_device_array_t(*major_offsets, SIZE_T) + : nullptr, + (majors) + ? new cugraph::c_api::cugraph_type_erased_device_array_t(*majors, graph_->vertex_type_) + : nullptr, + new cugraph::c_api::cugraph_type_erased_device_array_t(minors, graph_->vertex_type_), + (edge_id) + ? new cugraph::c_api::cugraph_type_erased_device_array_t(*edge_id, graph_->edge_type_) + : nullptr, + (edge_type) ? new cugraph::c_api::cugraph_type_erased_device_array_t( + *edge_type, graph_->edge_type_id_type_) + : nullptr, + (wgt) ? new cugraph::c_api::cugraph_type_erased_device_array_t(*wgt, graph_->weight_type_) + : nullptr, + (hop) ? new cugraph::c_api::cugraph_type_erased_device_array_t(*hop, INT32) + : nullptr, // FIXME get rid of this + (label_hop_offsets) + ? new cugraph::c_api::cugraph_type_erased_device_array_t(*label_hop_offsets, SIZE_T) + : nullptr, + (edge_label) + ? new cugraph::c_api::cugraph_type_erased_device_array_t(edge_label.value(), INT32) + : nullptr, + (renumber_map) ? new cugraph::c_api::cugraph_type_erased_device_array_t( + renumber_map.value(), graph_->vertex_type_) + : nullptr, + (renumber_map_offsets) ? new cugraph::c_api::cugraph_type_erased_device_array_t( + renumber_map_offsets.value(), SIZE_T) + : nullptr}; + } + } +}; + } // namespace extern "C" cugraph_error_code_t cugraph_sampling_options_create( @@ -954,3 +1305,81 @@ cugraph_error_code_t cugraph_uniform_neighbor_sample( do_expensive_check}; return cugraph::c_api::run_algorithm(graph, functor, result, error); } + +cugraph_error_code_t cugraph_biased_neighbor_sample( + const cugraph_resource_handle_t* handle, + cugraph_graph_t* graph, + const cugraph_edge_property_view_t* edge_biases, + const cugraph_type_erased_device_array_view_t* start_vertices, + const cugraph_type_erased_device_array_view_t* start_vertex_labels, + const cugraph_type_erased_device_array_view_t* label_list, + const cugraph_type_erased_device_array_view_t* label_to_comm_rank, + const cugraph_type_erased_device_array_view_t* label_offsets, + const cugraph_type_erased_host_array_view_t* fan_out, + cugraph_rng_state_t* rng_state, + const cugraph_sampling_options_t* options, + bool_t do_expensive_check, + cugraph_sample_result_t** result, + cugraph_error_t** error) +{ + auto options_cpp = *reinterpret_cast(options); + + CAPI_EXPECTS( + (edge_biases != nullptr) || + (reinterpret_cast(graph)->edge_weights_ != nullptr), + CUGRAPH_INVALID_INPUT, + "edge_biases is required if the graph is not weighted", + *error); + + CAPI_EXPECTS((!options_cpp.retain_seeds_) || (label_offsets != nullptr), + CUGRAPH_INVALID_INPUT, + "must specify label_offsets if retain_seeds is true", + *error); + + CAPI_EXPECTS((start_vertex_labels == nullptr) || + (reinterpret_cast( + start_vertex_labels) + ->type_ == INT32), + CUGRAPH_INVALID_INPUT, + "start_vertex_labels should be of type int", + *error); + + CAPI_EXPECTS((label_to_comm_rank == nullptr) || (start_vertex_labels != nullptr), + CUGRAPH_INVALID_INPUT, + "cannot specify label_to_comm_rank unless start_vertex_labels is also specified", + *error); + + CAPI_EXPECTS((label_to_comm_rank == nullptr) || (label_list != nullptr), + CUGRAPH_INVALID_INPUT, + "cannot specify label_to_comm_rank unless label_list is also specified", + *error); + + CAPI_EXPECTS(reinterpret_cast(graph)->vertex_type_ == + reinterpret_cast( + start_vertices) + ->type_, + CUGRAPH_INVALID_INPUT, + "vertex type of graph and start_vertices must match", + *error); + + CAPI_EXPECTS( + reinterpret_cast(fan_out) + ->type_ == INT32, + CUGRAPH_INVALID_INPUT, + "fan_out should be of type int", + *error); + + biased_neighbor_sampling_functor functor{handle, + graph, + edge_biases, + start_vertices, + start_vertex_labels, + label_list, + label_to_comm_rank, + label_offsets, + fan_out, + rng_state, + std::move(options_cpp), + do_expensive_check}; + return cugraph::c_api::run_algorithm(graph, functor, result, error); +} diff --git a/cpp/src/c_api/properties.hpp b/cpp/src/c_api/properties.hpp new file mode 100644 index 00000000000..971607586bb --- /dev/null +++ b/cpp/src/c_api/properties.hpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +namespace cugraph { +namespace c_api { + +typedef struct { + cugraph_data_type_id_t property_type_; + void* vertex_property_; +} cugraph_vertex_property_t; + +typedef struct { + cugraph_data_type_id_t property_type_; + void* edge_property_; +} cugraph_edge_property_t; + +typedef struct { + cugraph_data_type_id_t property_type_; + void* vertex_property_; +} cugraph_vertex_property_view_t; + +typedef struct { + cugraph_data_type_id_t property_type_; + void* edge_property_; +} cugraph_edge_property_view_t; + +} // namespace c_api +} // namespace cugraph diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 73a3104f27b..892ba91af86 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -768,6 +768,7 @@ if(BUILD_CUGRAPH_MG_TESTS) ConfigureCTestMG(MG_CAPI_EDGE_BETWEENNESS_CENTRALITY_TEST c_api/mg_edge_betweenness_centrality_test.c) ConfigureCTestMG(MG_CAPI_HITS_TEST c_api/mg_hits_test.c) ConfigureCTestMG(MG_CAPI_UNIFORM_NEIGHBOR_SAMPLE_TEST c_api/mg_uniform_neighbor_sample_test.c) + ConfigureCTestMG(MG_CAPI_BIASED_NEIGHBOR_SAMPLE_TEST c_api/mg_biased_neighbor_sample_test.c) ConfigureCTestMG(MG_CAPI_LOOKUP_SRC_DST_TEST c_api/mg_lookup_src_dst_test.c) ConfigureCTestMG(MG_CAPI_RANDOM_WALKS_TEST c_api/mg_random_walks_test.c) ConfigureCTestMG(MG_CAPI_TRIANGLE_COUNT_TEST c_api/mg_triangle_count_test.c) @@ -805,6 +806,7 @@ ConfigureCTest(CAPI_NODE2VEC_TEST c_api/node2vec_test.c) ConfigureCTest(CAPI_WEAKLY_CONNECTED_COMPONENTS_TEST c_api/weakly_connected_components_test.c) ConfigureCTest(CAPI_STRONGLY_CONNECTED_COMPONENTS_TEST c_api/strongly_connected_components_test.c) ConfigureCTest(CAPI_UNIFORM_NEIGHBOR_SAMPLE_TEST c_api/uniform_neighbor_sample_test.c) +ConfigureCTest(CAPI_BIASED_NEIGHBOR_SAMPLE_TEST c_api/biased_neighbor_sample_test.c) ConfigureCTest(CAPI_RANDOM_WALKS_TEST c_api/sg_random_walks_test.c) ConfigureCTest(CAPI_TRIANGLE_COUNT_TEST c_api/triangle_count_test.c) ConfigureCTest(CAPI_LOUVAIN_TEST c_api/louvain_test.c) diff --git a/cpp/tests/c_api/biased_neighbor_sample_test.c b/cpp/tests/c_api/biased_neighbor_sample_test.c new file mode 100644 index 00000000000..fe80514c825 --- /dev/null +++ b/cpp/tests/c_api/biased_neighbor_sample_test.c @@ -0,0 +1,973 @@ +/* + * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "c_test_utils.h" /* RUN_TEST */ + +#include +#include + +#include +#include +#include + +typedef int32_t vertex_t; +typedef int32_t edge_t; +typedef float weight_t; + +data_type_id_t vertex_tid = INT32; +data_type_id_t edge_tid = INT32; +data_type_id_t weight_tid = FLOAT32; +data_type_id_t edge_id_tid = INT32; +data_type_id_t edge_type_tid = INT32; + +int vertex_id_compare_function(const void* a, const void* b) +{ + if (*((vertex_t*)a) < *((vertex_t*)b)) + return -1; + else if (*((vertex_t*)a) > *((vertex_t*)b)) + return 1; + else + return 0; +} + +int generic_biased_neighbor_sample_test(const cugraph_resource_handle_t* handle, + vertex_t* h_src, + vertex_t* h_dst, + weight_t* h_wgt, + edge_t* h_edge_ids, + int32_t* h_edge_types, + size_t num_vertices, + size_t num_edges, + vertex_t* h_start, + int* h_start_labels, + size_t num_start_vertices, + size_t num_start_labels, + int* fan_out, + size_t fan_out_size, + bool_t with_replacement, + bool_t return_hops, + cugraph_prior_sources_behavior_t prior_sources_behavior, + bool_t dedupe_sources, + bool_t renumber_results) +{ + // Create graph + int test_ret_value = 0; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error = NULL; + cugraph_graph_t* graph = NULL; + cugraph_sample_result_t* result = NULL; + + ret_code = create_sg_test_graph(handle, + vertex_tid, + edge_tid, + h_src, + h_dst, + weight_tid, + h_wgt, + edge_type_tid, + h_edge_types, + edge_id_tid, + h_edge_ids, + num_edges, + FALSE, + TRUE, + FALSE, + FALSE, + &graph, + &ret_error); + + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + + cugraph_type_erased_device_array_t* d_start = NULL; + cugraph_type_erased_device_array_view_t* d_start_view = NULL; + cugraph_type_erased_device_array_t* d_start_labels = NULL; + cugraph_type_erased_device_array_view_t* d_start_labels_view = NULL; + cugraph_type_erased_host_array_view_t* h_fan_out_view = NULL; + + ret_code = cugraph_type_erased_device_array_create( + handle, num_start_vertices, INT32, &d_start, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "d_start create failed."); + + d_start_view = cugraph_type_erased_device_array_view(d_start); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, d_start_view, (byte_t*)h_start, &ret_error); + + ret_code = cugraph_type_erased_device_array_create( + handle, num_start_vertices, INT32, &d_start_labels, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "d_start_labels create failed."); + + d_start_labels_view = cugraph_type_erased_device_array_view(d_start_labels); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, d_start_labels_view, (byte_t*)h_start_labels, &ret_error); + + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "start_labels copy_from_host failed."); + + h_fan_out_view = cugraph_type_erased_host_array_view_create(fan_out, fan_out_size, INT32); + + cugraph_rng_state_t* rng_state; + ret_code = cugraph_rng_state_create(handle, 0, &rng_state, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "rng_state create failed."); + + cugraph_sampling_options_t* sampling_options; + + ret_code = cugraph_sampling_options_create(&sampling_options, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "sampling_options create failed."); + + cugraph_sampling_set_with_replacement(sampling_options, with_replacement); + cugraph_sampling_set_return_hops(sampling_options, return_hops); + cugraph_sampling_set_prior_sources_behavior(sampling_options, prior_sources_behavior); + cugraph_sampling_set_dedupe_sources(sampling_options, dedupe_sources); + cugraph_sampling_set_renumber_results(sampling_options, renumber_results); + + ret_code = cugraph_biased_neighbor_sample(handle, + graph, + NULL, + d_start_view, + d_start_labels_view, + NULL, + NULL, + NULL, + h_fan_out_view, + rng_state, + sampling_options, + FALSE, + &result, + &ret_error); + +#ifdef NO_CUGRAPH_OPS + TEST_ASSERT( + test_ret_value, ret_code != CUGRAPH_SUCCESS, "biased_neighbor_sample should have failed") +#else + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "biased_neighbor_sample failed."); + + cugraph_sampling_options_free(sampling_options); + + cugraph_type_erased_device_array_view_t* result_srcs; + cugraph_type_erased_device_array_view_t* result_dsts; + cugraph_type_erased_device_array_view_t* result_edge_id; + cugraph_type_erased_device_array_view_t* result_weights; + cugraph_type_erased_device_array_view_t* result_edge_types; + cugraph_type_erased_device_array_view_t* result_hops; + cugraph_type_erased_device_array_view_t* result_offsets; + cugraph_type_erased_device_array_view_t* result_labels; + cugraph_type_erased_device_array_view_t* result_renumber_map; + cugraph_type_erased_device_array_view_t* result_renumber_map_offsets; + + result_srcs = cugraph_sample_result_get_sources(result); + result_dsts = cugraph_sample_result_get_destinations(result); + result_edge_id = cugraph_sample_result_get_edge_id(result); + result_weights = cugraph_sample_result_get_edge_weight(result); + result_edge_types = cugraph_sample_result_get_edge_type(result); + result_hops = cugraph_sample_result_get_hop(result); + result_hops = cugraph_sample_result_get_hop(result); + result_offsets = cugraph_sample_result_get_offsets(result); + result_labels = cugraph_sample_result_get_start_labels(result); + result_renumber_map = cugraph_sample_result_get_renumber_map(result); + result_renumber_map_offsets = cugraph_sample_result_get_renumber_map_offsets(result); + + size_t result_size = cugraph_type_erased_device_array_view_size(result_srcs); + size_t result_offsets_size = cugraph_type_erased_device_array_view_size(result_offsets); + size_t renumber_map_size = 0; + + if (renumber_results) { + renumber_map_size = cugraph_type_erased_device_array_view_size(result_renumber_map); + } + + vertex_t h_result_srcs[result_size]; + vertex_t h_result_dsts[result_size]; + edge_t h_result_edge_id[result_size]; + weight_t h_result_weight[result_size]; + int32_t h_result_edge_types[result_size]; + int32_t h_result_hops[result_size]; + size_t h_result_offsets[result_offsets_size]; + int h_result_labels[num_start_labels]; + vertex_t h_renumber_map[renumber_map_size]; + size_t h_renumber_map_offsets[result_offsets_size]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_srcs, result_srcs, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_dsts, result_dsts, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_edge_id, result_edge_id, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_weight, result_weights, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_edge_types, result_edge_types, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + TEST_ASSERT(test_ret_value, result_hops == NULL, "hops was not empty"); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_offsets, result_offsets, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_labels, result_labels, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + for (int k = 0; k < result_offsets_size - 1; k += fan_out_size) { + for (int h = 0; h < fan_out_size; ++h) { + int hop_start = h_result_offsets[k + h]; + int hop_end = h_result_offsets[k + h + 1]; + for (int i = hop_start; i < hop_end; ++i) { + h_result_hops[i] = h; + } + } + } + + for (int k = 0; k < num_start_labels + 1; ++k) { + h_result_offsets[k] = h_result_offsets[k * fan_out_size]; + } + result_offsets_size = num_start_labels + 1; + + if (renumber_results) { + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_renumber_map, result_renumber_map, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_renumber_map_offsets, result_renumber_map_offsets, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + } + + // First, check that all edges are actually part of the graph + weight_t M_w[num_vertices][num_vertices]; + edge_t M_edge_id[num_vertices][num_vertices]; + int32_t M_edge_type[num_vertices][num_vertices]; + + for (int i = 0; i < num_vertices; ++i) + for (int j = 0; j < num_vertices; ++j) { + M_w[i][j] = 0.0; + M_edge_id[i][j] = -1; + M_edge_type[i][j] = -1; + } + + for (int i = 0; i < num_edges; ++i) { + M_w[h_src[i]][h_dst[i]] = h_wgt[i]; + M_edge_id[h_src[i]][h_dst[i]] = h_edge_ids[i]; + M_edge_type[h_src[i]][h_dst[i]] = h_edge_types[i]; + } + + if (renumber_results) { + for (int label_id = 0; label_id < (result_offsets_size - 1); ++label_id) { + for (size_t i = h_result_offsets[label_id]; + (i < h_result_offsets[label_id + 1]) && (test_ret_value == 0); + ++i) { + vertex_t src = h_renumber_map[h_renumber_map_offsets[label_id] + h_result_srcs[i]]; + vertex_t dst = h_renumber_map[h_renumber_map_offsets[label_id] + h_result_dsts[i]]; + + TEST_ASSERT(test_ret_value, + M_w[src][dst] == h_result_weight[i], + "biased_neighbor_sample got edge that doesn't exist"); + TEST_ASSERT(test_ret_value, + M_edge_id[src][dst] == h_result_edge_id[i], + "biased_neighbor_sample got edge that doesn't exist"); + TEST_ASSERT(test_ret_value, + M_edge_type[src][dst] == h_result_edge_types[i], + "biased_neighbor_sample got edge that doesn't exist"); + } + } + } else { + for (int i = 0; (i < result_size) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + M_w[h_result_srcs[i]][h_result_dsts[i]] == h_result_weight[i], + "biased_neighbor_sample got edge that doesn't exist"); + TEST_ASSERT(test_ret_value, + M_edge_id[h_result_srcs[i]][h_result_dsts[i]] == h_result_edge_id[i], + "biased_neighbor_sample got edge that doesn't exist"); + TEST_ASSERT(test_ret_value, + M_edge_type[h_result_srcs[i]][h_result_dsts[i]] == h_result_edge_types[i], + "biased_neighbor_sample got edge that doesn't exist"); + } + } + + // + // For the sampling result to make sense, all sources in hop 0 must be in the seeds, + // all sources in hop 1 must be a result from hop 0, etc. + // + vertex_t check_v1[result_size]; + vertex_t check_v2[result_size]; + vertex_t* check_sources = check_v1; + vertex_t* check_destinations = check_v2; + + size_t degree[num_vertices]; + for (size_t i = 0; i < num_vertices; ++i) + degree[i] = 0; + + for (size_t i = 0; i < num_edges; ++i) { + degree[h_src[i]]++; + } + + for (int label_id = 0; label_id < (result_offsets_size - 1); ++label_id) { + size_t sources_size = 0; + size_t destinations_size = 0; + + // Fill sources with the input sources + for (size_t i = 0; i < num_start_vertices; ++i) { + if (h_start_labels[i] == h_result_labels[label_id]) { + check_sources[sources_size] = h_start[i]; + ++sources_size; + } + } + + if (renumber_results) { + size_t num_vertex_ids = 2 * (h_result_offsets[label_id + 1] - h_result_offsets[label_id]); + vertex_t vertex_ids[num_vertex_ids]; + + for (size_t i = 0; (i < (h_result_offsets[label_id + 1] - h_result_offsets[label_id])) && + (test_ret_value == 0); + ++i) { + vertex_ids[2 * i] = h_result_srcs[h_result_offsets[label_id] + i]; + vertex_ids[2 * i + 1] = h_result_dsts[h_result_offsets[label_id] + i]; + } + + qsort(vertex_ids, num_vertex_ids, sizeof(vertex_t), vertex_id_compare_function); + + vertex_t current_v = 0; + for (size_t i = 0; (i < num_vertex_ids) && (test_ret_value == 0); ++i) { + if (vertex_ids[i] == current_v) + ++current_v; + else + TEST_ASSERT(test_ret_value, + vertex_ids[i] == (current_v - 1), + "vertices are not properly renumbered"); + } + } + + for (int hop = 0; hop < fan_out_size; ++hop) { + if (prior_sources_behavior == CARRY_OVER) { + destinations_size = sources_size; + for (size_t i = 0; i < sources_size; ++i) { + check_destinations[i] = check_sources[i]; + } + } + + for (size_t i = h_result_offsets[label_id]; + (i < h_result_offsets[label_id + 1]) && (test_ret_value == 0); + ++i) { + if (h_result_hops[i] == hop) { + bool found = false; + for (size_t j = 0; (!found) && (j < sources_size); ++j) { + found = renumber_results + ? (h_renumber_map[h_renumber_map_offsets[label_id] + h_result_srcs[i]] == + check_sources[j]) + : (h_result_srcs[i] == check_sources[j]); + } + + TEST_ASSERT(test_ret_value, + found, + "encountered source vertex that was not part of previous frontier"); + } + + if (prior_sources_behavior == CARRY_OVER) { + // Make sure destination isn't already in the source list + bool found = false; + for (size_t j = 0; (!found) && (j < destinations_size); ++j) { + found = renumber_results + ? (h_renumber_map[h_renumber_map_offsets[label_id] + h_result_dsts[i]] == + check_destinations[j]) + : (h_result_dsts[i] == check_destinations[j]); + } + + if (!found) { + check_destinations[destinations_size] = + renumber_results ? h_renumber_map[h_renumber_map_offsets[label_id] + h_result_dsts[i]] + : h_result_dsts[i]; + ++destinations_size; + } + } else { + check_destinations[destinations_size] = + renumber_results ? h_renumber_map[h_renumber_map_offsets[label_id] + h_result_dsts[i]] + : h_result_dsts[i]; + ++destinations_size; + } + } + + vertex_t* tmp = check_sources; + check_sources = check_destinations; + check_destinations = tmp; + sources_size = destinations_size; + destinations_size = 0; + } + + if (prior_sources_behavior == EXCLUDE) { + // Make sure vertex v only appears as source in the first hop after it is encountered + for (size_t i = h_result_offsets[label_id]; + (i < h_result_offsets[label_id + 1]) && (test_ret_value == 0); + ++i) { + for (size_t j = i + 1; (j < h_result_offsets[label_id + 1]) && (test_ret_value == 0); ++j) { + if (h_result_srcs[i] == h_result_srcs[j]) { + TEST_ASSERT(test_ret_value, + h_result_hops[i] == h_result_hops[j], + "source vertex should not have been used in diferent hops"); + } + } + } + } + + if (dedupe_sources) { + // Make sure vertex v only appears as source once for each edge after it appears as + // destination Externally test this by verifying that vertex v only appears in <= hop + // size/degree + for (size_t i = h_result_offsets[label_id]; + (i < h_result_offsets[label_id + 1]) && (test_ret_value == 0); + ++i) { + if (h_result_hops[i] > 0) { + size_t num_occurrences = 1; + for (size_t j = i + 1; j < h_result_offsets[label_id + 1]; ++j) { + if ((h_result_srcs[j] == h_result_srcs[i]) && (h_result_hops[j] == h_result_hops[i])) + num_occurrences++; + } + + if (fan_out[h_result_hops[i]] < 0) { + TEST_ASSERT(test_ret_value, + num_occurrences <= degree[h_result_srcs[i]], + "source vertex used in too many return edges"); + } else { + TEST_ASSERT(test_ret_value, + num_occurrences < fan_out[h_result_hops[i]], + "source vertex used in too many return edges"); + } + } + } + } + } + + cugraph_sample_result_free(result); +#endif + + cugraph_sg_graph_free(graph); + cugraph_error_free(ret_error); + return test_ret_value; +} + +int test_biased_neighbor_sample_with_labels(const cugraph_resource_handle_t* handle) +{ + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + data_type_id_t edge_id_tid = INT32; + data_type_id_t edge_type_tid = INT32; + + size_t num_edges = 8; + size_t num_vertices = 6; + size_t fan_out_size = 1; + size_t num_starts = 2; + + vertex_t src[] = {0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; + edge_t edge_ids[] = {0, 1, 2, 3, 4, 5, 6, 7}; + weight_t weight[] = {0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8}; + int32_t edge_types[] = {7, 6, 5, 4, 3, 2, 1, 0}; + vertex_t start[] = {2, 3}; + size_t start_labels[] = {6, 12}; + int fan_out[] = {-1}; + + // Create graph + int test_ret_value = 0; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error = NULL; + cugraph_graph_t* graph = NULL; + cugraph_sample_result_t* result = NULL; + + bool_t with_replacement = TRUE; + bool_t return_hops = TRUE; + cugraph_prior_sources_behavior_t prior_sources_behavior = DEFAULT; + bool_t dedupe_sources = FALSE; + bool_t renumber_results = FALSE; + cugraph_compression_type_t compression = COO; + bool_t compress_per_hop = FALSE; + + ret_code = create_sg_test_graph(handle, + vertex_tid, + edge_tid, + src, + dst, + weight_tid, + weight, + edge_type_tid, + edge_types, + edge_id_tid, + edge_ids, + num_edges, + FALSE, + TRUE, + FALSE, + FALSE, + &graph, + &ret_error); + + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + + cugraph_type_erased_device_array_t* d_start = NULL; + cugraph_type_erased_device_array_view_t* d_start_view = NULL; + cugraph_type_erased_device_array_t* d_start_labels = NULL; + cugraph_type_erased_device_array_view_t* d_start_labels_view = NULL; + cugraph_type_erased_host_array_view_t* h_fan_out_view = NULL; + + ret_code = + cugraph_type_erased_device_array_create(handle, num_starts, INT32, &d_start, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "d_start create failed."); + + d_start_view = cugraph_type_erased_device_array_view(d_start); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, d_start_view, (byte_t*)start, &ret_error); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_starts, INT32, &d_start_labels, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "d_start_labels create failed."); + + d_start_labels_view = cugraph_type_erased_device_array_view(d_start_labels); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, d_start_labels_view, (byte_t*)start_labels, &ret_error); + + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "start_labels copy_from_host failed."); + + h_fan_out_view = cugraph_type_erased_host_array_view_create(fan_out, 1, INT32); + + cugraph_rng_state_t* rng_state; + ret_code = cugraph_rng_state_create(handle, 0, &rng_state, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "rng_state create failed."); + + cugraph_sampling_options_t* sampling_options; + + ret_code = cugraph_sampling_options_create(&sampling_options, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "sampling_options create failed."); + + cugraph_sampling_set_with_replacement(sampling_options, with_replacement); + cugraph_sampling_set_return_hops(sampling_options, return_hops); + cugraph_sampling_set_prior_sources_behavior(sampling_options, prior_sources_behavior); + cugraph_sampling_set_dedupe_sources(sampling_options, dedupe_sources); + cugraph_sampling_set_renumber_results(sampling_options, renumber_results); + cugraph_sampling_set_compression_type(sampling_options, compression); + cugraph_sampling_set_compress_per_hop(sampling_options, compress_per_hop); + + ret_code = cugraph_biased_neighbor_sample(handle, + graph, + NULL, + d_start_view, + d_start_labels_view, + NULL, + NULL, + NULL, + h_fan_out_view, + rng_state, + sampling_options, + FALSE, + &result, + &ret_error); + +#ifdef NO_CUGRAPH_OPS + TEST_ASSERT( + test_ret_value, ret_code != CUGRAPH_SUCCESS, "biased_neighbor_sample should have failed") +#else + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "biased_neighbor_sample failed."); + + cugraph_type_erased_device_array_view_t* result_srcs; + cugraph_type_erased_device_array_view_t* result_dsts; + cugraph_type_erased_device_array_view_t* result_edge_id; + cugraph_type_erased_device_array_view_t* result_weights; + cugraph_type_erased_device_array_view_t* result_edge_types; + cugraph_type_erased_device_array_view_t* result_hops; + cugraph_type_erased_device_array_view_t* result_offsets; + + result_srcs = cugraph_sample_result_get_sources(result); + result_dsts = cugraph_sample_result_get_destinations(result); + result_edge_id = cugraph_sample_result_get_edge_id(result); + result_weights = cugraph_sample_result_get_edge_weight(result); + result_edge_types = cugraph_sample_result_get_edge_type(result); + result_hops = cugraph_sample_result_get_hop(result); + result_offsets = cugraph_sample_result_get_offsets(result); + + size_t result_size = cugraph_type_erased_device_array_view_size(result_srcs); + size_t result_offsets_size = cugraph_type_erased_device_array_view_size(result_offsets); + + vertex_t h_srcs[result_size]; + vertex_t h_dsts[result_size]; + edge_t h_edge_id[result_size]; + weight_t h_weight[result_size]; + int32_t h_edge_types[result_size]; + int32_t h_hops[result_size]; + size_t h_result_offsets[result_offsets_size]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_srcs, result_srcs, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_dsts, result_dsts, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_edge_id, result_edge_id, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_weight, result_weights, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_edge_types, result_edge_types, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + TEST_ASSERT(test_ret_value, result_hops == NULL, "hops was not empty"); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_offsets, result_offsets, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + // NOTE: The C++ tester does a more thorough validation. For our purposes + // here we will do a simpler validation, merely checking that all edges + // are actually part of the graph + weight_t M_w[num_vertices][num_vertices]; + edge_t M_edge_id[num_vertices][num_vertices]; + int32_t M_edge_type[num_vertices][num_vertices]; + + for (int i = 0; i < num_vertices; ++i) + for (int j = 0; j < num_vertices; ++j) { + M_w[i][j] = 0.0; + M_edge_id[i][j] = -1; + M_edge_type[i][j] = -1; + } + + for (int i = 0; i < num_edges; ++i) { + M_w[src[i]][dst[i]] = weight[i]; + M_edge_id[src[i]][dst[i]] = edge_ids[i]; + M_edge_type[src[i]][dst[i]] = edge_types[i]; + } + + for (int i = 0; (i < result_size) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + M_w[h_srcs[i]][h_dsts[i]] == h_weight[i], + "biased_neighbor_sample got edge that doesn't exist"); + TEST_ASSERT(test_ret_value, + M_edge_id[h_srcs[i]][h_dsts[i]] == h_edge_id[i], + "biased_neighbor_sample got edge that doesn't exist"); + TEST_ASSERT(test_ret_value, + M_edge_type[h_srcs[i]][h_dsts[i]] == h_edge_types[i], + "biased_neighbor_sample got edge that doesn't exist"); + } + + cugraph_sample_result_free(result); + cugraph_sampling_options_free(sampling_options); +#endif + + cugraph_sg_graph_free(graph); + cugraph_error_free(ret_error); +} + +int test_biased_neighbor_sample_clean(const cugraph_resource_handle_t* handle) +{ + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + data_type_id_t edge_id_tid = INT32; + data_type_id_t edge_type_tid = INT32; + + size_t num_edges = 9; + size_t num_vertices = 6; + size_t fan_out_size = 3; + size_t num_starts = 2; + size_t num_start_labels = 2; + + vertex_t src[] = {0, 0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t dst[] = {1, 3, 3, 4, 0, 1, 3, 5, 5}; + edge_t edge_ids[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + weight_t weight[] = {0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9}; + int32_t edge_types[] = {8, 7, 6, 5, 4, 3, 2, 1, 0}; + vertex_t start[] = {2, 3}; + int start_labels[] = {6, 12}; + int fan_out[] = {-1, -1, -1}; + + int test_ret_value = 0; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error = NULL; + + bool_t with_replacement = FALSE; + bool_t return_hops = TRUE; + cugraph_prior_sources_behavior_t prior_sources_behavior = DEFAULT; + bool_t dedupe_sources = FALSE; + bool_t renumber_results = FALSE; + + return generic_biased_neighbor_sample_test(handle, + src, + dst, + weight, + edge_ids, + edge_types, + num_vertices, + num_edges, + start, + start_labels, + num_starts, + num_start_labels, + fan_out, + fan_out_size, + with_replacement, + return_hops, + prior_sources_behavior, + dedupe_sources, + renumber_results); +} + +int test_biased_neighbor_sample_dedupe_sources(const cugraph_resource_handle_t* handle) +{ + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + data_type_id_t edge_id_tid = INT32; + data_type_id_t edge_type_tid = INT32; + + size_t num_edges = 9; + size_t num_vertices = 6; + size_t fan_out_size = 3; + size_t num_starts = 2; + size_t num_start_labels = 2; + + vertex_t src[] = {0, 0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t dst[] = {1, 3, 3, 4, 0, 1, 3, 5, 5}; + edge_t edge_ids[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + weight_t weight[] = {0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9}; + int32_t edge_types[] = {8, 7, 6, 5, 4, 3, 2, 1, 0}; + vertex_t start[] = {2, 3}; + int start_labels[] = {6, 12}; + int fan_out[] = {-1, -1, -1}; + + int test_ret_value = 0; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error = NULL; + + bool_t with_replacement = FALSE; + bool_t return_hops = TRUE; + cugraph_prior_sources_behavior_t prior_sources_behavior = DEFAULT; + bool_t dedupe_sources = TRUE; + bool_t renumber_results = FALSE; + + return generic_biased_neighbor_sample_test(handle, + src, + dst, + weight, + edge_ids, + edge_types, + num_vertices, + num_edges, + start, + start_labels, + num_starts, + num_start_labels, + fan_out, + fan_out_size, + with_replacement, + return_hops, + prior_sources_behavior, + dedupe_sources, + renumber_results); +} + +int test_biased_neighbor_sample_unique_sources(const cugraph_resource_handle_t* handle) +{ + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + data_type_id_t edge_id_tid = INT32; + data_type_id_t edge_type_tid = INT32; + + size_t num_edges = 9; + size_t num_vertices = 6; + size_t fan_out_size = 3; + size_t num_starts = 2; + size_t num_start_labels = 2; + + vertex_t src[] = {0, 0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t dst[] = {1, 2, 3, 4, 0, 1, 3, 5, 5}; + edge_t edge_ids[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + weight_t weight[] = {0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9}; + int32_t edge_types[] = {8, 7, 6, 5, 4, 3, 2, 1, 0}; + vertex_t start[] = {2, 3}; + int start_labels[] = {6, 12}; + int fan_out[] = {-1, -1, -1}; + + int test_ret_value = 0; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error = NULL; + + bool_t with_replacement = FALSE; + bool_t return_hops = TRUE; + cugraph_prior_sources_behavior_t prior_sources_behavior = EXCLUDE; + bool_t dedupe_sources = FALSE; + bool_t renumber_results = FALSE; + + return generic_biased_neighbor_sample_test(handle, + src, + dst, + weight, + edge_ids, + edge_types, + num_vertices, + num_edges, + start, + start_labels, + num_starts, + num_start_labels, + fan_out, + fan_out_size, + with_replacement, + return_hops, + prior_sources_behavior, + dedupe_sources, + renumber_results); +} + +int test_biased_neighbor_sample_carry_over_sources(const cugraph_resource_handle_t* handle) +{ + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + data_type_id_t edge_id_tid = INT32; + data_type_id_t edge_type_tid = INT32; + + size_t num_edges = 9; + size_t num_vertices = 6; + size_t fan_out_size = 3; + size_t num_starts = 2; + size_t num_start_labels = 2; + + vertex_t src[] = {0, 0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t dst[] = {1, 2, 3, 4, 0, 1, 3, 5, 5}; + edge_t edge_ids[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + weight_t weight[] = {0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9}; + int32_t edge_types[] = {8, 7, 6, 5, 4, 3, 2, 1, 0}; + vertex_t start[] = {2, 3}; + int start_labels[] = {6, 12}; + int fan_out[] = {-1, -1, -1}; + + int test_ret_value = 0; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error = NULL; + + bool_t with_replacement = FALSE; + bool_t return_hops = TRUE; + cugraph_prior_sources_behavior_t prior_sources_behavior = CARRY_OVER; + bool_t dedupe_sources = FALSE; + bool_t renumber_results = FALSE; + + return generic_biased_neighbor_sample_test(handle, + src, + dst, + weight, + edge_ids, + edge_types, + num_vertices, + num_edges, + start, + start_labels, + num_starts, + num_start_labels, + fan_out, + fan_out_size, + with_replacement, + return_hops, + prior_sources_behavior, + dedupe_sources, + renumber_results); +} + +int test_biased_neighbor_sample_renumber_results(const cugraph_resource_handle_t* handle) +{ + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + data_type_id_t edge_id_tid = INT32; + data_type_id_t edge_type_tid = INT32; + + size_t num_edges = 9; + size_t num_vertices = 6; + size_t fan_out_size = 3; + size_t num_starts = 2; + size_t num_start_labels = 2; + + vertex_t src[] = {0, 0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t dst[] = {1, 2, 3, 4, 0, 1, 3, 5, 5}; + edge_t edge_ids[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + weight_t weight[] = {0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9}; + int32_t edge_types[] = {8, 7, 6, 5, 4, 3, 2, 1, 0}; + vertex_t start[] = {2, 3}; + int start_labels[] = {6, 12}; + int fan_out[] = {-1, -1, -1}; + + int test_ret_value = 0; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error = NULL; + + bool_t with_replacement = FALSE; + bool_t return_hops = TRUE; + cugraph_prior_sources_behavior_t prior_sources_behavior = DEFAULT; + bool_t dedupe_sources = FALSE; + bool_t renumber_results = TRUE; + + return generic_biased_neighbor_sample_test(handle, + src, + dst, + weight, + edge_ids, + edge_types, + num_vertices, + num_edges, + start, + start_labels, + num_starts, + num_start_labels, + fan_out, + fan_out_size, + with_replacement, + return_hops, + prior_sources_behavior, + dedupe_sources, + renumber_results); +} + +int main(int argc, char** argv) +{ + cugraph_resource_handle_t* handle = NULL; + + handle = cugraph_create_resource_handle(NULL); + + int result = 0; + result |= RUN_TEST_NEW(test_biased_neighbor_sample_clean, handle); + result |= RUN_TEST_NEW(test_biased_neighbor_sample_dedupe_sources, handle); + result |= RUN_TEST_NEW(test_biased_neighbor_sample_unique_sources, handle); + result |= RUN_TEST_NEW(test_biased_neighbor_sample_carry_over_sources, handle); + result |= RUN_TEST_NEW(test_biased_neighbor_sample_renumber_results, handle); + + cugraph_free_resource_handle(handle); + + return result; +} diff --git a/cpp/tests/c_api/mg_biased_neighbor_sample_test.c b/cpp/tests/c_api/mg_biased_neighbor_sample_test.c new file mode 100644 index 00000000000..ce96fc6f5f7 --- /dev/null +++ b/cpp/tests/c_api/mg_biased_neighbor_sample_test.c @@ -0,0 +1,1398 @@ +/* + * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mg_test_utils.h" /* RUN_MG_TEST */ + +#include +#include + +#include +#include +#include + +typedef int32_t vertex_t; +typedef int32_t edge_t; +typedef float weight_t; + +data_type_id_t vertex_tid = INT32; +data_type_id_t edge_tid = INT32; +data_type_id_t weight_tid = FLOAT32; +data_type_id_t edge_id_tid = INT32; +data_type_id_t edge_type_tid = INT32; + +int generic_biased_neighbor_sample_test(const cugraph_resource_handle_t* handle, + vertex_t* h_src, + vertex_t* h_dst, + weight_t* h_wgt, + edge_t* h_edge_ids, + int32_t* h_edge_types, + size_t num_vertices, + size_t num_edges, + vertex_t* h_start, + int* h_start_labels, + size_t num_start_vertices, + int* fan_out, + size_t fan_out_size, + bool_t with_replacement, + bool_t return_hops, + cugraph_prior_sources_behavior_t prior_sources_behavior, + bool_t dedupe_sources) +{ + // Create graph + int test_ret_value = 0; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error = NULL; + cugraph_graph_t* graph = NULL; + cugraph_sample_result_t* result = NULL; + + int rank = cugraph_resource_handle_get_rank(handle); + + ret_code = create_mg_test_graph_new(handle, + vertex_tid, + edge_tid, + h_src, + h_dst, + weight_tid, + h_wgt, + edge_type_tid, + h_edge_types, + edge_id_tid, + h_edge_ids, + num_edges, + FALSE, + TRUE, + FALSE, + FALSE, + &graph, + &ret_error); + + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + + cugraph_type_erased_device_array_t* d_start = NULL; + cugraph_type_erased_device_array_view_t* d_start_view = NULL; + cugraph_type_erased_device_array_t* d_start_labels = NULL; + cugraph_type_erased_device_array_view_t* d_start_labels_view = NULL; + cugraph_type_erased_host_array_view_t* h_fan_out_view = NULL; + + if (rank > 0) num_start_vertices = 0; + + ret_code = cugraph_type_erased_device_array_create( + handle, num_start_vertices, INT32, &d_start, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "d_start create failed."); + + d_start_view = cugraph_type_erased_device_array_view(d_start); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, d_start_view, (byte_t*)h_start, &ret_error); + + if (h_start_labels != NULL) { + ret_code = cugraph_type_erased_device_array_create( + handle, num_start_vertices, INT32, &d_start_labels, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "d_start_labels create failed."); + + d_start_labels_view = cugraph_type_erased_device_array_view(d_start_labels); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, d_start_labels_view, (byte_t*)h_start_labels, &ret_error); + + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "start_labels copy_from_host failed."); + } + + h_fan_out_view = cugraph_type_erased_host_array_view_create(fan_out, fan_out_size, INT32); + + cugraph_rng_state_t* rng_state; + ret_code = cugraph_rng_state_create(handle, rank, &rng_state, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "rng_state create failed."); + + cugraph_sampling_options_t* sampling_options; + + ret_code = cugraph_sampling_options_create(&sampling_options, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "sampling_options create failed."); + + cugraph_sampling_set_with_replacement(sampling_options, with_replacement); + cugraph_sampling_set_return_hops(sampling_options, return_hops); + cugraph_sampling_set_prior_sources_behavior(sampling_options, prior_sources_behavior); + cugraph_sampling_set_dedupe_sources(sampling_options, dedupe_sources); + + ret_code = cugraph_biased_neighbor_sample(handle, + graph, + NULL, + d_start_view, + d_start_labels_view, + NULL, + NULL, + NULL, + h_fan_out_view, + rng_state, + sampling_options, + FALSE, + &result, + &ret_error); + +#ifdef NO_CUGRAPH_OPS + TEST_ASSERT( + test_ret_value, ret_code != CUGRAPH_SUCCESS, "biased_neighbor_sample should have failed") +#else + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "biased_neighbor_sample failed."); + + cugraph_sampling_options_free(sampling_options); + + cugraph_type_erased_device_array_view_t* result_srcs; + cugraph_type_erased_device_array_view_t* result_dsts; + cugraph_type_erased_device_array_view_t* result_edge_id; + cugraph_type_erased_device_array_view_t* result_weights; + cugraph_type_erased_device_array_view_t* result_edge_types; + cugraph_type_erased_device_array_view_t* result_hops; + cugraph_type_erased_device_array_view_t* result_offsets = NULL; + cugraph_type_erased_device_array_view_t* result_labels = NULL; + + result_srcs = cugraph_sample_result_get_sources(result); + result_dsts = cugraph_sample_result_get_destinations(result); + result_edge_id = cugraph_sample_result_get_edge_id(result); + result_weights = cugraph_sample_result_get_edge_weight(result); + result_edge_types = cugraph_sample_result_get_edge_type(result); + result_hops = cugraph_sample_result_get_hop(result); + result_hops = cugraph_sample_result_get_hop(result); + + size_t result_offsets_size = 2; + + if (d_start_labels != NULL) { + result_offsets = cugraph_sample_result_get_offsets(result); + result_labels = cugraph_sample_result_get_start_labels(result); + result_offsets_size = + 1 + cugraph_test_scalar_reduce( + handle, cugraph_type_erased_device_array_view_size(result_offsets) - 1); + } + + size_t result_size = cugraph_test_device_gatherv_size(handle, result_srcs); + + vertex_t h_result_srcs[result_size]; + vertex_t h_result_dsts[result_size]; + edge_t h_result_edge_id[result_size]; + weight_t h_result_weight[result_size]; + int32_t h_result_edge_types[result_size]; + int32_t h_result_hops[result_size]; + size_t h_result_offsets[result_offsets_size]; + int h_result_labels[result_offsets_size - 1]; + + if (result_offsets_size == 2) { + h_result_offsets[0] = 0; + h_result_offsets[1] = result_size; + } + + ret_code = cugraph_test_device_gatherv_fill(handle, result_srcs, h_result_srcs); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "gatherv_fill failed."); + + ret_code = cugraph_test_device_gatherv_fill(handle, result_dsts, h_result_dsts); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "gatherv_fill failed."); + + if (h_edge_ids != NULL) { + ret_code = cugraph_test_device_gatherv_fill(handle, result_edge_id, h_result_edge_id); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "gatherv_fill failed."); + } + + if (h_wgt != NULL) { + ret_code = cugraph_test_device_gatherv_fill(handle, result_weights, h_result_weight); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "gatherv_fill failed."); + } + + if (h_edge_types != NULL) { + ret_code = cugraph_test_device_gatherv_fill(handle, result_edge_types, h_result_edge_types); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "gatherv_fill failed."); + } + + if (d_start_labels != NULL) { + size_t sz = cugraph_type_erased_device_array_view_size(result_offsets); + + ret_code = cugraph_test_device_gatherv_fill(handle, result_labels, h_result_labels); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "gatherv_fill failed."); + + size_t tmp_result_offsets[sz]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)tmp_result_offsets, result_offsets, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + // convert to size + for (size_t i = 1; i < sz; ++i) { + tmp_result_offsets[i - 1] = tmp_result_offsets[i] - tmp_result_offsets[i - 1]; + } + + cugraph_test_host_gatherv_fill( + handle, tmp_result_offsets, sz - 1, SIZE_T, h_result_offsets + 1); + + h_result_offsets[0] = 0; + for (size_t i = 1; i < result_offsets_size; ++i) { + h_result_offsets[i] += h_result_offsets[i - 1]; + } + } + + // First, check that all edges are actually part of the graph + weight_t M_w[num_vertices][num_vertices]; + edge_t M_edge_id[num_vertices][num_vertices]; + int32_t M_edge_type[num_vertices][num_vertices]; + + for (int i = 0; i < num_vertices; ++i) + for (int j = 0; j < num_vertices; ++j) { + M_w[i][j] = 0.0; + M_edge_id[i][j] = -1; + M_edge_type[i][j] = -1; + } + + for (int i = 0; i < num_edges; ++i) { + if (h_wgt != NULL) + M_w[h_src[i]][h_dst[i]] = h_wgt[i]; + else + M_w[h_src[i]][h_dst[i]] = 1.0; + + if (h_edge_ids != NULL) M_edge_id[h_src[i]][h_dst[i]] = h_edge_ids[i]; + if (h_edge_types != NULL) M_edge_type[h_src[i]][h_dst[i]] = h_edge_types[i]; + } + + for (int i = 0; (i < result_size) && (test_ret_value == 0); ++i) { + if (h_wgt != NULL) { + TEST_ASSERT(test_ret_value, + M_w[h_result_srcs[i]][h_result_dsts[i]] == h_result_weight[i], + "biased_neighbor_sample got edge that doesn't exist"); + } else { + TEST_ASSERT(test_ret_value, + M_w[h_result_srcs[i]][h_result_dsts[i]] == 1.0, + "biased_neighbor_sample got edge that doesn't exist"); + } + + if (h_edge_ids != NULL) + TEST_ASSERT(test_ret_value, + M_edge_id[h_result_srcs[i]][h_result_dsts[i]] == h_result_edge_id[i], + "biased_neighbor_sample got edge that doesn't exist"); + if (h_edge_types != NULL) + TEST_ASSERT(test_ret_value, + M_edge_type[h_result_srcs[i]][h_result_dsts[i]] == h_result_edge_types[i], + "biased_neighbor_sample got edge that doesn't exist"); + } + + if ((return_hops) && (d_start_labels != NULL) && (result_offsets_size > 0)) { + // + // For the sampling result to make sense, all sources in hop 0 must be in the seeds, + // all sources in hop 1 must be a result from hop 0, etc. + // + vertex_t check_v1[result_size]; + vertex_t check_v2[result_size]; + vertex_t* check_sources = check_v1; + vertex_t* check_destinations = check_v2; + + size_t degree[num_vertices]; + for (size_t i = 0; i < num_vertices; ++i) + degree[i] = 0; + + for (size_t i = 0; i < num_edges; ++i) { + degree[h_src[i]]++; + } + + for (size_t label_id = 0; label_id < (result_offsets_size - 1); ++label_id) { + // Skip any labels we already processed + bool already_processed = false; + for (size_t i = 0; (i < label_id) && !already_processed; ++i) + already_processed = (h_result_labels[label_id] == h_result_labels[i]); + + if (already_processed) continue; + + size_t sources_size = 0; + size_t destinations_size = 0; + + // Fill sources with the input sources + for (size_t i = 0; i < num_start_vertices; ++i) { + if (h_start_labels[i] == h_result_labels[label_id]) { + check_sources[sources_size] = h_start[i]; + ++sources_size; + } + } + + for (int hop = 0; hop < fan_out_size; ++hop) { + if (prior_sources_behavior == CARRY_OVER) { + destinations_size = sources_size; + for (size_t i = 0; i < sources_size; ++i) { + check_destinations[i] = check_sources[i]; + } + } + + for (size_t current_label_id = label_id; current_label_id < (result_offsets_size - 1); + ++current_label_id) { + if (h_result_labels[current_label_id] == h_result_labels[label_id]) { + for (size_t i = h_result_offsets[current_label_id]; + (i < h_result_offsets[current_label_id + 1]) && (test_ret_value == 0); + ++i) { + if (h_result_hops[i] == hop) { + bool found = false; + for (size_t j = 0; (!found) && (j < sources_size); ++j) { + found = (h_result_srcs[i] == check_sources[j]); + } + + TEST_ASSERT(test_ret_value, + found, + "encountered source vertex that was not part of previous frontier"); + } + + if (prior_sources_behavior == CARRY_OVER) { + // Make sure destination isn't already in the source list + bool found = false; + for (size_t j = 0; (!found) && (j < destinations_size); ++j) { + found = (h_result_dsts[i] == check_destinations[j]); + } + + if (!found) { + check_destinations[destinations_size] = h_result_dsts[i]; + ++destinations_size; + } + } else { + check_destinations[destinations_size] = h_result_dsts[i]; + ++destinations_size; + } + } + } + } + + vertex_t* tmp = check_sources; + check_sources = check_destinations; + check_destinations = tmp; + sources_size = destinations_size; + destinations_size = 0; + } + + if (prior_sources_behavior == EXCLUDE) { + // Make sure vertex v only appears as source in the first hop after it is encountered + for (size_t current_label_id = label_id; current_label_id < (result_offsets_size - 1); + ++current_label_id) { + if (h_result_labels[current_label_id] == h_result_labels[label_id]) { + for (size_t i = h_result_offsets[current_label_id]; + (i < h_result_offsets[current_label_id + 1]) && (test_ret_value == 0); + ++i) { + for (size_t j = i + 1; + (j < h_result_offsets[current_label_id + 1]) && (test_ret_value == 0); + ++j) { + if (h_result_srcs[i] == h_result_srcs[j]) { + TEST_ASSERT(test_ret_value, + h_result_hops[i] == h_result_hops[j], + "source vertex should not have been used in diferent hops"); + } + } + } + } + } + } + + if (dedupe_sources) { + // Make sure vertex v only appears as source once for each edge after it appears as + // destination Externally test this by verifying that vertex v only appears in <= hop + // size/degree + for (size_t current_label_id = label_id; current_label_id < (result_offsets_size - 1); + ++current_label_id) { + if (h_result_labels[current_label_id] == h_result_labels[label_id]) { + for (size_t i = h_result_offsets[current_label_id]; + (i < h_result_offsets[current_label_id + 1]) && (test_ret_value == 0); + ++i) { + if (h_result_hops[i] > 0) { + size_t num_occurrences = 1; + for (size_t j = i + 1; j < h_result_offsets[current_label_id + 1]; ++j) { + if ((h_result_srcs[j] == h_result_srcs[i]) && + (h_result_hops[j] == h_result_hops[i])) + num_occurrences++; + } + + if (fan_out[h_result_hops[i]] < 0) { + TEST_ASSERT(test_ret_value, + num_occurrences <= degree[h_result_srcs[i]], + "source vertex used in too many return edges"); + } else { + TEST_ASSERT(test_ret_value, + num_occurrences < fan_out[h_result_hops[i]], + "source vertex used in too many return edges"); + } + } + } + } + } + } + } + } + + cugraph_sample_result_free(result); +#endif + + cugraph_mg_graph_free(graph); + cugraph_error_free(ret_error); + return test_ret_value; +} + +int test_biased_neighbor_sample(const cugraph_resource_handle_t* handle) +{ + size_t num_edges = 8; + size_t num_vertices = 6; + size_t fan_out_size = 2; + size_t num_starts = 2; + + vertex_t src[] = {0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; + weight_t wgt[] = {1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7}; + vertex_t start[] = {2, 2}; + int fan_out[] = {1, 2}; + + bool_t with_replacement = FALSE; + bool_t return_hops = TRUE; + cugraph_prior_sources_behavior_t prior_sources_behavior = DEFAULT; + bool_t dedupe_sources = FALSE; + + return generic_biased_neighbor_sample_test(handle, + src, + dst, + wgt, + NULL, + NULL, + num_vertices, + num_edges, + start, + NULL, + num_starts, + fan_out, + fan_out_size, + with_replacement, + return_hops, + prior_sources_behavior, + dedupe_sources); +} + +int test_biased_neighbor_from_alex(const cugraph_resource_handle_t* handle) +{ + size_t num_edges = 12; + size_t num_vertices = 5; + size_t fan_out_size = 2; + size_t num_starts = 2; + size_t num_start_labels = 2; + + vertex_t src[] = {0, 1, 2, 3, 4, 3, 4, 2, 0, 1, 0, 2}; + vertex_t dst[] = {1, 2, 4, 2, 3, 4, 1, 1, 2, 3, 4, 4}; + edge_t idx[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + int32_t typ[] = {0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 0}; + weight_t wgt[] = {0.0, 0.1, 0.2, 3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.10, 0.11}; + vertex_t start[] = {0, 4}; + int32_t batch[] = {0, 1}; + int fan_out[] = {2, 2}; + + bool_t store_transposed = FALSE; + + int test_ret_value = 0; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error = NULL; + + cugraph_graph_t* graph = NULL; + cugraph_sample_result_t* result = NULL; + + bool_t with_replacement = FALSE; + bool_t return_hops = TRUE; + cugraph_prior_sources_behavior_t prior_sources_behavior = DEFAULT; + bool_t dedupe_sources = FALSE; + bool_t renumber_results = FALSE; + cugraph_compression_type_t compression = COO; + bool_t compress_per_hop = FALSE; + + cugraph_type_erased_device_array_t* d_start = NULL; + cugraph_type_erased_device_array_t* d_label = NULL; + cugraph_type_erased_device_array_view_t* d_start_view = NULL; + cugraph_type_erased_device_array_view_t* d_label_view = NULL; + cugraph_type_erased_host_array_view_t* h_fan_out_view = NULL; + + int rank = cugraph_resource_handle_get_rank(handle); + + cugraph_rng_state_t* rng_state; + ret_code = cugraph_rng_state_create(handle, rank, &rng_state, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "rng_state create failed."); + TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + ret_code = create_mg_test_graph_with_properties( + handle, src, dst, idx, typ, wgt, num_edges, store_transposed, FALSE, &graph, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_starts, INT32, &d_start, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "d_start create failed."); + + d_start_view = cugraph_type_erased_device_array_view(d_start); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, d_start_view, (byte_t*)start, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "start copy_from_host failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_starts, INT32, &d_label, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "d_label create failed."); + + d_label_view = cugraph_type_erased_device_array_view(d_label); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, d_label_view, (byte_t*)batch, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "start copy_from_host failed."); + + h_fan_out_view = cugraph_type_erased_host_array_view_create(fan_out, fan_out_size, INT32); + + cugraph_sampling_options_t* sampling_options; + + ret_code = cugraph_sampling_options_create(&sampling_options, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "sampling_options create failed."); + + cugraph_sampling_set_with_replacement(sampling_options, with_replacement); + cugraph_sampling_set_return_hops(sampling_options, return_hops); + cugraph_sampling_set_prior_sources_behavior(sampling_options, prior_sources_behavior); + cugraph_sampling_set_dedupe_sources(sampling_options, dedupe_sources); + cugraph_sampling_set_renumber_results(sampling_options, renumber_results); + cugraph_sampling_set_compression_type(sampling_options, compression); + cugraph_sampling_set_compress_per_hop(sampling_options, compress_per_hop); + + ret_code = cugraph_biased_neighbor_sample(handle, + graph, + NULL, + d_start_view, + d_label_view, + NULL, + NULL, + NULL, + h_fan_out_view, + rng_state, + sampling_options, + FALSE, + &result, + &ret_error); + +#ifdef NO_CUGRAPH_OPS + TEST_ASSERT( + test_ret_value, ret_code != CUGRAPH_SUCCESS, "biased_neighbor_sample should have failed"); +#else + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "biased_neighbor_sample failed."); + + cugraph_type_erased_device_array_view_t* result_src; + cugraph_type_erased_device_array_view_t* result_dst; + cugraph_type_erased_device_array_view_t* result_index; + cugraph_type_erased_device_array_view_t* result_type; + cugraph_type_erased_device_array_view_t* result_weight; + cugraph_type_erased_device_array_view_t* result_labels; + cugraph_type_erased_device_array_view_t* result_hops; + cugraph_type_erased_device_array_view_t* result_offsets; + + result_src = cugraph_sample_result_get_sources(result); + result_dst = cugraph_sample_result_get_destinations(result); + result_index = cugraph_sample_result_get_edge_id(result); + result_type = cugraph_sample_result_get_edge_type(result); + result_weight = cugraph_sample_result_get_edge_weight(result); + result_labels = cugraph_sample_result_get_start_labels(result); + result_hops = cugraph_sample_result_get_hop(result); + result_offsets = cugraph_sample_result_get_offsets(result); + + size_t result_size = cugraph_type_erased_device_array_view_size(result_src); + size_t offsets_size = cugraph_type_erased_device_array_view_size(result_offsets); + + vertex_t h_srcs[result_size]; + vertex_t h_dsts[result_size]; + edge_t h_index[result_size]; + int h_type[result_size]; + weight_t h_wgt[result_size]; + int h_labels[result_size]; + int h_hop[result_size]; + int h_offsets[offsets_size]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_srcs, result_src, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_dsts, result_dst, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_index, result_index, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_type, result_type, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_wgt, result_weight, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_labels, result_labels, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_offsets, result_offsets, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + for (int k = 0; k < offsets_size - 1; k += fan_out_size) { + for (int h = 0; h < fan_out_size; ++h) { + int hop_start = h_offsets[k + h]; + int hop_end = h_offsets[k + h + 1]; + for (int i = hop_start; i < hop_end; ++i) { + h_hop[i] = h; + } + } + } + + for (int k = 0; k < num_start_labels + 1; ++k) { + h_offsets[k] = h_offsets[k * fan_out_size]; + } + offsets_size = num_start_labels + 1; + + // NOTE: The C++ tester does a more thorough validation. For our purposes + // here we will do a simpler validation, merely checking that all edges + // are actually part of the graph + edge_t M[num_vertices][num_vertices]; + + for (int i = 0; i < num_vertices; ++i) + for (int j = 0; j < num_vertices; ++j) + M[i][j] = -1; + + for (int i = 0; i < num_edges; ++i) + M[src[i]][dst[i]] = idx[i]; + + for (int i = 0; (i < result_size) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + M[h_srcs[i]][h_dsts[i]] >= 0, + "biased_neighbor_sample got edge that doesn't exist"); + } +#endif + + cugraph_sample_result_free(result); + + cugraph_type_erased_host_array_view_free(h_fan_out_view); + cugraph_mg_graph_free(graph); + cugraph_error_free(ret_error); + cugraph_sampling_options_free(sampling_options); + + return test_ret_value; +} + +int test_biased_neighbor_sample_alex_bug(const cugraph_resource_handle_t* handle) +{ + size_t num_edges = 156; + size_t num_vertices = 34; + size_t fan_out_size = 2; + size_t num_starts = 4; + size_t num_labels = 3; + + vertex_t src[] = {1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 17, 19, 21, 31, 2, 3, 7, 13, + 17, 19, 21, 30, 3, 7, 8, 9, 13, 27, 28, 32, 7, 12, 13, 6, 10, 6, 10, 16, + 16, 30, 32, 33, 33, 33, 32, 33, 32, 33, 32, 33, 33, 32, 33, 32, 33, 25, 27, 29, + 32, 33, 25, 27, 31, 31, 29, 33, 33, 31, 33, 32, 33, 32, 33, 32, 33, 33, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, + 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 6, 8, + 8, 8, 9, 13, 14, 14, 15, 15, 18, 18, 19, 20, 20, 22, 22, 23, 23, 23, 23, 23, + 24, 24, 24, 25, 26, 26, 27, 28, 28, 29, 29, 30, 30, 31, 31, 32}; + vertex_t dst[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, + 6, 8, 8, 8, 9, 13, 14, 14, 15, 15, 18, 18, 19, 20, 20, 22, 22, 23, 23, 23, + 23, 23, 24, 24, 24, 25, 26, 26, 27, 28, 28, 29, 29, 30, 30, 31, 31, 32, 1, 2, + 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 17, 19, 21, 31, 2, 3, 7, 13, 17, 19, + 21, 30, 3, 7, 8, 9, 13, 27, 28, 32, 7, 12, 13, 6, 10, 6, 10, 16, 16, 30, + 32, 33, 33, 33, 32, 33, 32, 33, 32, 33, 33, 32, 33, 32, 33, 25, 27, 29, 32, 33, + 25, 27, 31, 31, 29, 33, 33, 31, 33, 32, 33, 32, 33, 32, 33, 33}; + weight_t wgt[] = { + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}; + + edge_t edge_ids[] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, + 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, + 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155}; + + vertex_t start[] = {0, 1, 2, 5}; + int32_t start_labels[] = {0, 0, 1, 2}; + int32_t label_list[] = {0, 1, 2}; + int32_t label_to_output_comm_rank[] = {0, 0, 1}; + int fan_out[] = {2, 3}; + + size_t expected_size[] = {3, 2, 1, 1, 1, 1, 1, 1}; + + bool_t with_replacement = FALSE; + bool_t return_hops = TRUE; + cugraph_prior_sources_behavior_t prior_sources_behavior = CARRY_OVER; + bool_t dedupe_sources = TRUE; + bool_t renumber_results = FALSE; + cugraph_compression_type_t compression = COO; + bool_t compress_per_hop = FALSE; + + // Create graph + int test_ret_value = 0; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error = NULL; + cugraph_graph_t* graph = NULL; + cugraph_sample_result_t* result = NULL; + + ret_code = create_mg_test_graph_with_properties( + handle, src, dst, edge_ids, NULL, wgt, num_edges, FALSE, TRUE, &graph, &ret_error); + + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + + cugraph_type_erased_device_array_t* d_start = NULL; + cugraph_type_erased_device_array_view_t* d_start_view = NULL; + cugraph_type_erased_device_array_t* d_start_labels = NULL; + cugraph_type_erased_device_array_view_t* d_start_labels_view = NULL; + cugraph_type_erased_device_array_t* d_label_list = NULL; + cugraph_type_erased_device_array_view_t* d_label_list_view = NULL; + cugraph_type_erased_device_array_t* d_label_to_output_comm_rank = NULL; + cugraph_type_erased_device_array_view_t* d_label_to_output_comm_rank_view = NULL; + cugraph_type_erased_host_array_view_t* h_fan_out_view = NULL; + + int rank = cugraph_resource_handle_get_rank(handle); + + if (rank > 0) { num_starts = 0; } + + cugraph_rng_state_t* rng_state; + ret_code = cugraph_rng_state_create(handle, rank, &rng_state, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "rng_state create failed."); + TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_starts, INT32, &d_start, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "d_start create failed."); + + d_start_view = cugraph_type_erased_device_array_view(d_start); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, d_start_view, (byte_t*)start, &ret_error); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_starts, INT32, &d_start_labels, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "d_start_labels create failed."); + + d_start_labels_view = cugraph_type_erased_device_array_view(d_start_labels); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, d_start_labels_view, (byte_t*)start_labels, &ret_error); + + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "start_labels copy_from_host failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_labels, INT32, &d_label_list, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "d_label_list create failed."); + + d_label_list_view = cugraph_type_erased_device_array_view(d_label_list); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, d_label_list_view, (byte_t*)label_list, &ret_error); + + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "label_list copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_create( + handle, num_labels, INT32, &d_label_to_output_comm_rank, &ret_error); + TEST_ASSERT( + test_ret_value, ret_code == CUGRAPH_SUCCESS, "d_label_to_output_comm_rank create failed."); + + d_label_to_output_comm_rank_view = + cugraph_type_erased_device_array_view(d_label_to_output_comm_rank); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, d_label_to_output_comm_rank_view, (byte_t*)label_to_output_comm_rank, &ret_error); + + TEST_ASSERT(test_ret_value, + ret_code == CUGRAPH_SUCCESS, + "label_to_output_comm_rank copy_from_host failed."); + + h_fan_out_view = cugraph_type_erased_host_array_view_create(fan_out, fan_out_size, INT32); + + cugraph_sampling_options_t* sampling_options; + ret_code = cugraph_sampling_options_create(&sampling_options, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "sampling_options create failed."); + + cugraph_sampling_set_with_replacement(sampling_options, with_replacement); + cugraph_sampling_set_return_hops(sampling_options, return_hops); + cugraph_sampling_set_prior_sources_behavior(sampling_options, prior_sources_behavior); + cugraph_sampling_set_dedupe_sources(sampling_options, dedupe_sources); + cugraph_sampling_set_renumber_results(sampling_options, renumber_results); + cugraph_sampling_set_compression_type(sampling_options, compression); + cugraph_sampling_set_compress_per_hop(sampling_options, compress_per_hop); + + ret_code = cugraph_biased_neighbor_sample(handle, + graph, + NULL, + d_start_view, + d_start_labels_view, + d_label_list_view, + d_label_to_output_comm_rank_view, + NULL, + h_fan_out_view, + rng_state, + sampling_options, + FALSE, + &result, + &ret_error); + +#ifdef NO_CUGRAPH_OPS + TEST_ASSERT( + test_ret_value, ret_code != CUGRAPH_SUCCESS, "biased_neighbor_sample should have failed") +#else + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "biased_neighbor_sample failed."); + + cugraph_type_erased_device_array_view_t* result_srcs = NULL; + cugraph_type_erased_device_array_view_t* result_dsts = NULL; + cugraph_type_erased_device_array_view_t* result_edge_id = NULL; + cugraph_type_erased_device_array_view_t* result_weights = NULL; + cugraph_type_erased_device_array_view_t* result_hops = NULL; + cugraph_type_erased_device_array_view_t* result_offsets = NULL; + + result_srcs = cugraph_sample_result_get_sources(result); + result_dsts = cugraph_sample_result_get_destinations(result); + result_edge_id = cugraph_sample_result_get_edge_id(result); + result_weights = cugraph_sample_result_get_edge_weight(result); + result_hops = cugraph_sample_result_get_hop(result); + result_offsets = cugraph_sample_result_get_offsets(result); + + size_t result_size = cugraph_type_erased_device_array_view_size(result_srcs); + size_t result_offsets_size = cugraph_type_erased_device_array_view_size(result_offsets); + + vertex_t h_srcs[result_size]; + vertex_t h_dsts[result_size]; + edge_t h_edge_id[result_size]; + weight_t h_weight[result_size]; + int32_t h_hops[result_size]; + size_t h_result_offsets[result_offsets_size]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_srcs, result_srcs, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_dsts, result_dsts, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_edge_id, result_edge_id, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_weight, result_weights, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_hops, result_hops, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_offsets, result_offsets, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + // NOTE: The C++ tester does a more thorough validation. For our purposes + // here we will do a simpler validation, merely checking that all edges + // are actually part of the graph + weight_t M_w[num_vertices][num_vertices]; + edge_t M_edge_id[num_vertices][num_vertices]; + + for (int i = 0; i < num_vertices; ++i) + for (int j = 0; j < num_vertices; ++j) { + M_w[i][j] = 0.0; + M_edge_id[i][j] = -1; + } + + for (int i = 0; i < num_edges; ++i) { + M_w[src[i]][dst[i]] = wgt[i]; + M_edge_id[src[i]][dst[i]] = edge_ids[i]; + } + + for (int i = 0; (i < result_size) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + M_w[h_srcs[i]][h_dsts[i]] == h_weight[i], + "biased_neighbor_sample got edge that doesn't exist"); + TEST_ASSERT(test_ret_value, + M_edge_id[h_srcs[i]][h_dsts[i]] == h_edge_id[i], + "biased_neighbor_sample got edge that doesn't exist"); + } + + TEST_ASSERT( + test_ret_value, result_offsets_size == expected_size[rank], "incorrect number of results"); + + cugraph_sample_result_free(result); +#endif + + cugraph_sg_graph_free(graph); + cugraph_error_free(ret_error); +} + +int test_biased_neighbor_sample_sort_by_hop(const cugraph_resource_handle_t* handle) +{ + size_t num_edges = 156; + size_t num_vertices = 34; + size_t fan_out_size = 2; + size_t num_starts = 4; + size_t num_labels = 3; + + vertex_t src[] = {1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 17, 19, 21, 31, 2, 3, 7, 13, + 17, 19, 21, 30, 3, 7, 8, 9, 13, 27, 28, 32, 7, 12, 13, 6, 10, 6, 10, 16, + 16, 30, 32, 33, 33, 33, 32, 33, 32, 33, 32, 33, 33, 32, 33, 32, 33, 25, 27, 29, + 32, 33, 25, 27, 31, 31, 29, 33, 33, 31, 33, 32, 33, 32, 33, 32, 33, 33, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, + 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 6, 8, + 8, 8, 9, 13, 14, 14, 15, 15, 18, 18, 19, 20, 20, 22, 22, 23, 23, 23, 23, 23, + 24, 24, 24, 25, 26, 26, 27, 28, 28, 29, 29, 30, 30, 31, 31, 32}; + vertex_t dst[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, + 6, 8, 8, 8, 9, 13, 14, 14, 15, 15, 18, 18, 19, 20, 20, 22, 22, 23, 23, 23, + 23, 23, 24, 24, 24, 25, 26, 26, 27, 28, 28, 29, 29, 30, 30, 31, 31, 32, 1, 2, + 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 17, 19, 21, 31, 2, 3, 7, 13, 17, 19, + 21, 30, 3, 7, 8, 9, 13, 27, 28, 32, 7, 12, 13, 6, 10, 6, 10, 16, 16, 30, + 32, 33, 33, 33, 32, 33, 32, 33, 32, 33, 33, 32, 33, 32, 33, 25, 27, 29, 32, 33, + 25, 27, 31, 31, 29, 33, 33, 31, 33, 32, 33, 32, 33, 32, 33, 33}; + weight_t wgt[] = { + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}; + + edge_t edge_ids[] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, + 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, + 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155}; + + vertex_t start[] = {0, 1, 2, 5}; + int32_t start_labels[] = {0, 0, 1, 2}; + int32_t label_list[] = {0, 1, 2}; + int32_t label_to_output_comm_rank[] = {0, 0, 1}; + int fan_out[] = {2, 3}; + + size_t expected_size[] = {3, 2, 1, 1, 1, 1, 1, 1}; + + bool_t with_replacement = FALSE; + bool_t return_hops = TRUE; + cugraph_prior_sources_behavior_t prior_sources_behavior = CARRY_OVER; + bool_t dedupe_sources = TRUE; + bool_t renumber_results = FALSE; + cugraph_compression_type_t compression = COO; + bool_t compress_per_hop = FALSE; + + // Create graph + int test_ret_value = 0; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error = NULL; + cugraph_graph_t* graph = NULL; + cugraph_sample_result_t* result = NULL; + + ret_code = create_mg_test_graph_with_properties( + handle, src, dst, edge_ids, NULL, wgt, num_edges, FALSE, TRUE, &graph, &ret_error); + + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + + cugraph_type_erased_device_array_t* d_start = NULL; + cugraph_type_erased_device_array_view_t* d_start_view = NULL; + cugraph_type_erased_device_array_t* d_start_labels = NULL; + cugraph_type_erased_device_array_view_t* d_start_labels_view = NULL; + cugraph_type_erased_device_array_t* d_label_list = NULL; + cugraph_type_erased_device_array_view_t* d_label_list_view = NULL; + cugraph_type_erased_device_array_t* d_label_to_output_comm_rank = NULL; + cugraph_type_erased_device_array_view_t* d_label_to_output_comm_rank_view = NULL; + cugraph_type_erased_host_array_view_t* h_fan_out_view = NULL; + + int rank = cugraph_resource_handle_get_rank(handle); + + if (rank > 0) { num_starts = 0; } + + cugraph_rng_state_t* rng_state; + ret_code = cugraph_rng_state_create(handle, rank, &rng_state, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "rng_state create failed."); + TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_starts, INT32, &d_start, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "d_start create failed."); + + d_start_view = cugraph_type_erased_device_array_view(d_start); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, d_start_view, (byte_t*)start, &ret_error); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_starts, INT32, &d_start_labels, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "d_start_labels create failed."); + + d_start_labels_view = cugraph_type_erased_device_array_view(d_start_labels); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, d_start_labels_view, (byte_t*)start_labels, &ret_error); + + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "start_labels copy_from_host failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_labels, INT32, &d_label_list, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "d_label_list create failed."); + + d_label_list_view = cugraph_type_erased_device_array_view(d_label_list); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, d_label_list_view, (byte_t*)label_list, &ret_error); + + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "label_list copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_create( + handle, num_labels, INT32, &d_label_to_output_comm_rank, &ret_error); + TEST_ASSERT( + test_ret_value, ret_code == CUGRAPH_SUCCESS, "d_label_to_output_comm_rank create failed."); + + d_label_to_output_comm_rank_view = + cugraph_type_erased_device_array_view(d_label_to_output_comm_rank); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, d_label_to_output_comm_rank_view, (byte_t*)label_to_output_comm_rank, &ret_error); + + TEST_ASSERT(test_ret_value, + ret_code == CUGRAPH_SUCCESS, + "label_to_output_comm_rank copy_from_host failed."); + + h_fan_out_view = cugraph_type_erased_host_array_view_create(fan_out, fan_out_size, INT32); + + cugraph_sampling_options_t* sampling_options; + ret_code = cugraph_sampling_options_create(&sampling_options, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "sampling_options create failed."); + + cugraph_sampling_set_with_replacement(sampling_options, with_replacement); + cugraph_sampling_set_return_hops(sampling_options, return_hops); + cugraph_sampling_set_prior_sources_behavior(sampling_options, prior_sources_behavior); + cugraph_sampling_set_dedupe_sources(sampling_options, dedupe_sources); + cugraph_sampling_set_renumber_results(sampling_options, renumber_results); + cugraph_sampling_set_compression_type(sampling_options, compression); + cugraph_sampling_set_compress_per_hop(sampling_options, compress_per_hop); + + ret_code = cugraph_biased_neighbor_sample(handle, + graph, + NULL, + d_start_view, + d_start_labels_view, + d_label_list_view, + d_label_to_output_comm_rank_view, + NULL, + h_fan_out_view, + rng_state, + sampling_options, + FALSE, + &result, + &ret_error); + +#ifdef NO_CUGRAPH_OPS + TEST_ASSERT( + test_ret_value, ret_code != CUGRAPH_SUCCESS, "biased_neighbor_sample should have failed") +#else + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "biased_neighbor_sample failed."); + + cugraph_type_erased_device_array_view_t* result_srcs = NULL; + cugraph_type_erased_device_array_view_t* result_dsts = NULL; + cugraph_type_erased_device_array_view_t* result_edge_id = NULL; + cugraph_type_erased_device_array_view_t* result_weights = NULL; + cugraph_type_erased_device_array_view_t* result_hops = NULL; + cugraph_type_erased_device_array_view_t* result_offsets = NULL; + + result_srcs = cugraph_sample_result_get_sources(result); + result_dsts = cugraph_sample_result_get_destinations(result); + result_edge_id = cugraph_sample_result_get_edge_id(result); + result_weights = cugraph_sample_result_get_edge_weight(result); + result_hops = cugraph_sample_result_get_hop(result); + result_offsets = cugraph_sample_result_get_offsets(result); + + size_t result_size = cugraph_type_erased_device_array_view_size(result_srcs); + size_t result_offsets_size = cugraph_type_erased_device_array_view_size(result_offsets); + + vertex_t h_srcs[result_size]; + vertex_t h_dsts[result_size]; + edge_t h_edge_id[result_size]; + weight_t h_weight[result_size]; + int32_t h_hops[result_size]; + size_t h_result_offsets[result_offsets_size]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_srcs, result_srcs, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_dsts, result_dsts, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_edge_id, result_edge_id, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_weight, result_weights, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_offsets, result_offsets, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + for (int k = 0; k < result_offsets_size - 1; k += fan_out_size) { + for (int h = 0; h < fan_out_size; ++h) { + int hop_start = h_result_offsets[k + h]; + int hop_end = h_result_offsets[k + h + 1]; + for (int i = hop_start; i < hop_end; ++i) { + h_hops[i] = h; + } + } + } + + size_t num_local_labels = (result_offsets_size - 1) / fan_out_size; + + for (int k = 0; k < num_local_labels + 1; ++k) { + h_result_offsets[k] = h_result_offsets[k * fan_out_size]; + } + result_offsets_size = num_local_labels + 1; + + // NOTE: The C++ tester does a more thorough validation. For our purposes + // here we will do a simpler validation, merely checking that all edges + // are actually part of the graph + weight_t M_w[num_vertices][num_vertices]; + edge_t M_edge_id[num_vertices][num_vertices]; + + for (int i = 0; i < num_vertices; ++i) + for (int j = 0; j < num_vertices; ++j) { + M_w[i][j] = 0.0; + M_edge_id[i][j] = -1; + } + + for (int i = 0; i < num_edges; ++i) { + M_w[src[i]][dst[i]] = wgt[i]; + M_edge_id[src[i]][dst[i]] = edge_ids[i]; + } + + for (int i = 0; (i < result_size) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + M_w[h_srcs[i]][h_dsts[i]] == h_weight[i], + "biased_neighbor_sample got edge that doesn't exist"); + TEST_ASSERT(test_ret_value, + M_edge_id[h_srcs[i]][h_dsts[i]] == h_edge_id[i], + "biased_neighbor_sample got edge that doesn't exist"); + } + + TEST_ASSERT( + test_ret_value, result_offsets_size == expected_size[rank], "incorrect number of results"); + + for (int i = 0; i < (result_offsets_size - 1) && (test_ret_value == 0); ++i) { + for (int j = h_result_offsets[i]; j < (h_result_offsets[i + 1] - 1) && (test_ret_value == 0); + ++j) { + TEST_ASSERT(test_ret_value, h_hops[j] <= h_hops[j + 1], "Results not sorted by hop id"); + } + } + + cugraph_sample_result_free(result); +#endif + + cugraph_sg_graph_free(graph); + cugraph_error_free(ret_error); +} + +int test_biased_neighbor_sample_dedupe_sources(const cugraph_resource_handle_t* handle) +{ + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + data_type_id_t edge_id_tid = INT32; + data_type_id_t edge_type_tid = INT32; + + size_t num_edges = 9; + size_t num_vertices = 6; + size_t fan_out_size = 3; + size_t num_starts = 2; + + vertex_t src[] = {0, 0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t dst[] = {1, 3, 3, 4, 0, 1, 3, 5, 5}; + edge_t edge_ids[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + weight_t weight[] = {0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9}; + int32_t edge_types[] = {8, 7, 6, 5, 4, 3, 2, 1, 0}; + vertex_t start[] = {2, 3}; + int start_labels[] = {6, 12}; + int fan_out[] = {-1, -1, -1}; + + int test_ret_value = 0; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error = NULL; + + bool_t with_replacement = FALSE; + bool_t return_hops = TRUE; + cugraph_prior_sources_behavior_t prior_sources_behavior = DEFAULT; + bool_t dedupe_sources = TRUE; + + return generic_biased_neighbor_sample_test(handle, + src, + dst, + weight, + edge_ids, + edge_types, + num_vertices, + num_edges, + start, + start_labels, + num_starts, + fan_out, + fan_out_size, + with_replacement, + return_hops, + prior_sources_behavior, + dedupe_sources); +} + +int test_biased_neighbor_sample_unique_sources(const cugraph_resource_handle_t* handle) +{ + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + data_type_id_t edge_id_tid = INT32; + data_type_id_t edge_type_tid = INT32; + + size_t num_edges = 9; + size_t num_vertices = 6; + size_t fan_out_size = 3; + size_t num_starts = 2; + + vertex_t src[] = {0, 0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t dst[] = {1, 2, 3, 4, 0, 1, 3, 5, 5}; + edge_t edge_ids[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + weight_t weight[] = {0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9}; + int32_t edge_types[] = {8, 7, 6, 5, 4, 3, 2, 1, 0}; + vertex_t start[] = {2, 3}; + int start_labels[] = {6, 12}; + int fan_out[] = {-1, -1, -1}; + + int test_ret_value = 0; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error = NULL; + + bool_t with_replacement = FALSE; + bool_t return_hops = TRUE; + cugraph_prior_sources_behavior_t prior_sources_behavior = EXCLUDE; + bool_t dedupe_sources = FALSE; + + return generic_biased_neighbor_sample_test(handle, + src, + dst, + weight, + edge_ids, + edge_types, + num_vertices, + num_edges, + start, + start_labels, + num_starts, + fan_out, + fan_out_size, + with_replacement, + return_hops, + prior_sources_behavior, + dedupe_sources); +} + +int test_biased_neighbor_sample_carry_over_sources(const cugraph_resource_handle_t* handle) +{ + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + data_type_id_t edge_id_tid = INT32; + data_type_id_t edge_type_tid = INT32; + + size_t num_edges = 9; + size_t num_vertices = 6; + size_t fan_out_size = 3; + size_t num_starts = 2; + + vertex_t src[] = {0, 0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t dst[] = {1, 2, 3, 4, 0, 1, 3, 5, 5}; + edge_t edge_ids[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + weight_t weight[] = {0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9}; + int32_t edge_types[] = {8, 7, 6, 5, 4, 3, 2, 1, 0}; + vertex_t start[] = {2, 3}; + int start_labels[] = {6, 12}; + int fan_out[] = {-1, -1, -1}; + + int test_ret_value = 0; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error = NULL; + + bool_t with_replacement = FALSE; + bool_t return_hops = TRUE; + cugraph_prior_sources_behavior_t prior_sources_behavior = CARRY_OVER; + bool_t dedupe_sources = FALSE; + + return generic_biased_neighbor_sample_test(handle, + src, + dst, + weight, + edge_ids, + edge_types, + num_vertices, + num_edges, + start, + start_labels, + num_starts, + fan_out, + fan_out_size, + with_replacement, + return_hops, + prior_sources_behavior, + dedupe_sources); +} + +/******************************************************************************/ + +int main(int argc, char** argv) +{ + void* raft_handle = create_mg_raft_handle(argc, argv); + cugraph_resource_handle_t* handle = cugraph_create_resource_handle(raft_handle); + + int result = 0; + result |= RUN_MG_TEST(test_biased_neighbor_sample, handle); + result |= RUN_MG_TEST(test_biased_neighbor_from_alex, handle); + // result |= RUN_MG_TEST(test_biased_neighbor_sample_alex_bug, handle); + result |= RUN_MG_TEST(test_biased_neighbor_sample_sort_by_hop, handle); + // result |= RUN_MG_TEST(test_biased_neighbor_sample_dedupe_sources, handle); + // result |= RUN_MG_TEST(test_biased_neighbor_sample_unique_sources, handle); + // result |= RUN_MG_TEST(test_biased_neighbor_sample_carry_over_sources, handle); + + cugraph_free_resource_handle(handle); + free_mg_raft_handle(raft_handle); + + return result; +} diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/properties.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/properties.pxd new file mode 100644 index 00000000000..2838de3d0ab --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/properties.pxd @@ -0,0 +1,29 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Have cython use python 3 syntax +# cython: language_level = 3 + +cdef extern from "cugraph_c/properties.h": + + ctypedef struct cugraph_vertex_property_t: + pass + + ctypedef struct cugraph_edge_property_t: + pass + + ctypedef struct cugraph_vertex_property_view_t: + pass + + ctypedef struct cugraph_edge_property_view_t: + pass diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/sampling_algorithms.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/sampling_algorithms.pxd index dbd3ef4b7e1..0f852d9cecd 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/sampling_algorithms.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/sampling_algorithms.pxd @@ -40,6 +40,10 @@ from pylibcugraph._cugraph_c.array cimport ( cugraph_type_erased_device_array_t, ) +from pylibcugraph._cugraph_c.properties cimport ( + cugraph_edge_property_view_t, +) + cdef extern from "cugraph_c/sampling_algorithms.h": ########################################################################### @@ -59,6 +63,23 @@ cdef extern from "cugraph_c/sampling_algorithms.h": cugraph_error_t** error ) + cdef cugraph_error_code_t cugraph_biased_neighbor_sample( + const cugraph_resource_handle_t* handle, + cugraph_graph_t* graph, + const cugraph_edge_property_view_t* edge_biases, + const cugraph_type_erased_device_array_view_t* start_vertices, + const cugraph_type_erased_device_array_view_t* start_vertex_labels, + const cugraph_type_erased_device_array_view_t* label_list, + const cugraph_type_erased_device_array_view_t* label_to_comm_rank, + const cugraph_type_erased_device_array_view_t* label_offsets, + const cugraph_type_erased_host_array_view_t* fan_out, + cugraph_rng_state_t* rng_state, + const cugraph_sampling_options_t* options, + bool_t do_expensive_check, + cugraph_sample_result_t** result, + cugraph_error_t** error + ) + cdef cugraph_error_code_t cugraph_test_uniform_neighborhood_sample_result_create( const cugraph_resource_handle_t* handle, const cugraph_type_erased_device_array_view_t* srcs, From a41d6b02cd16937992d8d60f07bcd3e14e88b9a1 Mon Sep 17 00:00:00 2001 From: Ralph Liu <137829296+nv-rliu@users.noreply.github.com> Date: Thu, 18 Jul 2024 20:14:23 -0400 Subject: [PATCH 07/11] Add Additional Check For SSSP Source Vertex & Fix SSSP Benchmark (#4541) Closes https://github.com/rapidsai/cugraph/issues/4511. Part of https://github.com/rapidsai/graph_dl/issues/513 ### Fixes - `bench_algos.py` will properly convert `source` vertex values returned from MG and SG graphs before calling the actual algorithm. --- *Other minor changes* - Fix white-space formatting of `pytest.ini` Authors: - Ralph Liu (https://github.com/nv-rliu) Approvers: - Rick Ratzel (https://github.com/rlratzel) URL: https://github.com/rapidsai/cugraph/pull/4541 --- .../cugraph/pytest-based/bench_algos.py | 11 +- benchmarks/pytest.ini | 106 +++++++++--------- 2 files changed, 63 insertions(+), 54 deletions(-) diff --git a/benchmarks/cugraph/pytest-based/bench_algos.py b/benchmarks/cugraph/pytest-based/bench_algos.py index 4a87b7086ea..04407d656d7 100644 --- a/benchmarks/cugraph/pytest-based/bench_algos.py +++ b/benchmarks/cugraph/pytest-based/bench_algos.py @@ -330,8 +330,17 @@ def bench_bfs(gpubenchmark, graph): def bench_sssp(gpubenchmark, graph): + if not graph.is_weighted(): + pytest.skip("Skipping: Unweighted Graphs are not supported by SSSP") + sssp = dask_cugraph.sssp if is_graph_distributed(graph) else cugraph.sssp - start = graph.edgelist.edgelist_df["src"][0] + + start_col = graph.select_random_vertices(num_vertices=1) + if is_graph_distributed(graph): + start_col = start_col.compute() + + start = start_col.to_arrow().to_pylist()[0] + gpubenchmark(sssp, graph, start) diff --git a/benchmarks/pytest.ini b/benchmarks/pytest.ini index b3d8a8bb36c..fe7fc31b6d6 100644 --- a/benchmarks/pytest.ini +++ b/benchmarks/pytest.ini @@ -1,67 +1,67 @@ [pytest] pythonpath = - shared/python + shared/python testpaths = - cugraph/pytest_based - cugraph-service/pytest_based + cugraph/pytest_based + cugraph-service/pytest_based addopts = - --benchmark-columns="min, max, mean, stddev, outliers" + --benchmark-columns="min, max, mean, stddev, outliers" markers = - managedmem_on: RMM managed memory enabled - managedmem_off: RMM managed memory disabled - poolallocator_on: RMM pool allocator enabled - poolallocator_off: RMM pool allocator disabled - tiny: tiny datasets - small: small datasets - medium: medium datasets - large: large datasets - directed: directed datasets - undirected: undirected datasets - matrix_types: inputs are matrices - nx_types: inputs are NetowrkX Graph objects - cugraph_types: inputs are cuGraph Graph objects - sg: single-GPU - mg: multi-GPU - snmg: single-node multi-GPU - mnmg: multi-node multi-GPU - local: local cugraph - remote: cugraph-service - batch_size_100: batch size of 100 for sampling algos - batch_size_500: batch size of 500 for sampling algos - batch_size_1000: batch size of 1000 for sampling algos - batch_size_2500: batch size of 2500 for sampling algos - batch_size_5000: batch size of 5000 for sampling algos - batch_size_10000: batch size of 10000 for sampling algos - batch_size_20000: batch size of 20000 for sampling algos - batch_size_30000: batch size of 30000 for sampling algos - batch_size_40000: batch size of 40000 for sampling algos - batch_size_50000: batch size of 50000 for sampling algos - batch_size_60000: batch size of 60000 for sampling algos - batch_size_70000: batch size of 70000 for sampling algos - batch_size_80000: batch size of 80000 for sampling algos - batch_size_90000: batch size of 90000 for sampling algos - batch_size_100000: batch size of 100000 for sampling algos - num_clients_2: start 2 cugraph-service clients - num_clients_4: start 4 cugraph-service clients - num_clients_8: start 8 cugraph-service clients - num_clients_16: start 16 cugraph-service clients - num_clients_32: start 32 cugraph-service clients - fanout_10_25: fanout [10, 25] for sampling algos - fanout_5_10_15: fanout [5, 10, 15] for sampling algos - rmat_data: RMAT-generated synthetic datasets - file_data: datasets from $RAPIDS_DATASET_ROOT_DIR + managedmem_on: RMM managed memory enabled + managedmem_off: RMM managed memory disabled + poolallocator_on: RMM pool allocator enabled + poolallocator_off: RMM pool allocator disabled + tiny: tiny datasets + small: small datasets + medium: medium datasets + large: large datasets + directed: directed datasets + undirected: undirected datasets + matrix_types: inputs are matrices + nx_types: inputs are NetowrkX Graph objects + cugraph_types: inputs are cuGraph Graph objects + sg: single-GPU + mg: multi-GPU + snmg: single-node multi-GPU + mnmg: multi-node multi-GPU + local: local cugraph + remote: cugraph-service + batch_size_100: batch size of 100 for sampling algos + batch_size_500: batch size of 500 for sampling algos + batch_size_1000: batch size of 1000 for sampling algos + batch_size_2500: batch size of 2500 for sampling algos + batch_size_5000: batch size of 5000 for sampling algos + batch_size_10000: batch size of 10000 for sampling algos + batch_size_20000: batch size of 20000 for sampling algos + batch_size_30000: batch size of 30000 for sampling algos + batch_size_40000: batch size of 40000 for sampling algos + batch_size_50000: batch size of 50000 for sampling algos + batch_size_60000: batch size of 60000 for sampling algos + batch_size_70000: batch size of 70000 for sampling algos + batch_size_80000: batch size of 80000 for sampling algos + batch_size_90000: batch size of 90000 for sampling algos + batch_size_100000: batch size of 100000 for sampling algos + num_clients_2: start 2 cugraph-service clients + num_clients_4: start 4 cugraph-service clients + num_clients_8: start 8 cugraph-service clients + num_clients_16: start 16 cugraph-service clients + num_clients_32: start 32 cugraph-service clients + fanout_10_25: fanout [10, 25] for sampling algos + fanout_5_10_15: fanout [5, 10, 15] for sampling algos + rmat_data: RMAT-generated synthetic datasets + file_data: datasets from $RAPIDS_DATASET_ROOT_DIR python_classes = - Bench* - Test* + Bench* + Test* python_files = - bench_* - test_* + bench_* + test_* python_functions = - bench_* - test_* + bench_* + test_* From 9c4585c2276606378014f8048c244f4fbc4d98bd Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 19 Jul 2024 17:24:16 -0400 Subject: [PATCH 08/11] Use workflow branch 24.08 again (#4544) After updating everything to CUDA 12.5.1, use `shared-workflows@branch-24.08` again. Contributes to https://github.com/rapidsai/build-planning/issues/73 Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - James Lamb (https://github.com/jameslamb) - https://github.com/jakirkham --- .github/workflows/build.yaml | 32 +++++++++++++------------- .github/workflows/pr.yaml | 44 ++++++++++++++++++------------------ .github/workflows/test.yaml | 18 +++++++-------- 3 files changed, 47 insertions(+), 47 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 704d632efe6..03d3162bdbf 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -38,7 +38,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -47,7 +47,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibcugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -83,7 +83,7 @@ jobs: wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -93,7 +93,7 @@ jobs: wheel-build-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -106,7 +106,7 @@ jobs: wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -116,7 +116,7 @@ jobs: wheel-build-nx-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -126,7 +126,7 @@ jobs: wheel-publish-nx-cugraph: needs: wheel-build-nx-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -136,7 +136,7 @@ jobs: wheel-build-cugraph-dgl: needs: wheel-publish-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -146,7 +146,7 @@ jobs: wheel-publish-cugraph-dgl: needs: wheel-build-cugraph-dgl secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -156,7 +156,7 @@ jobs: wheel-build-cugraph-pyg: needs: wheel-publish-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -166,7 +166,7 @@ jobs: wheel-publish-cugraph-pyg: needs: wheel-build-cugraph-pyg secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -175,7 +175,7 @@ jobs: package-name: cugraph-pyg wheel-build-cugraph-equivariant: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -185,7 +185,7 @@ jobs: wheel-publish-cugraph-equivariant: needs: wheel-build-cugraph-equivariant secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e41839bbf4f..94ed08d96d7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -34,29 +34,29 @@ jobs: - wheel-tests-cugraph-equivariant - devcontainer secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.08 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.08 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.08 with: build_type: pull-request node_type: cpu32 conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.08 with: build_type: pull-request conda-cpp-checks: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.08 with: build_type: pull-request enable_check_symbols: true @@ -64,19 +64,19 @@ jobs: conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08 with: build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 with: build_type: pull-request conda-notebook-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -86,7 +86,7 @@ jobs: docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -96,7 +96,7 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: pull-request script: ci/build_wheel_pylibcugraph.sh @@ -107,14 +107,14 @@ jobs: wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: build_type: pull-request script: ci/test_wheel_pylibcugraph.sh wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: pull-request script: ci/build_wheel_cugraph.sh @@ -124,35 +124,35 @@ jobs: wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: build_type: pull-request script: ci/test_wheel_cugraph.sh wheel-build-nx-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: pull-request script: ci/build_wheel_nx-cugraph.sh wheel-tests-nx-cugraph: needs: wheel-build-nx-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: build_type: pull-request script: ci/test_wheel_nx-cugraph.sh wheel-build-cugraph-dgl: needs: wheel-tests-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: pull-request script: ci/build_wheel_cugraph-dgl.sh wheel-tests-cugraph-dgl: needs: wheel-build-cugraph-dgl secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: build_type: pull-request script: ci/test_wheel_cugraph-dgl.sh @@ -160,35 +160,35 @@ jobs: wheel-build-cugraph-pyg: needs: wheel-tests-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: pull-request script: ci/build_wheel_cugraph-pyg.sh wheel-tests-cugraph-pyg: needs: wheel-build-cugraph-pyg secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: build_type: pull-request script: ci/test_wheel_cugraph-pyg.sh matrix_filter: map(select(.ARCH == "amd64")) wheel-build-cugraph-equivariant: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: pull-request script: ci/build_wheel_cugraph-equivariant.sh wheel-tests-cugraph-equivariant: needs: wheel-build-cugraph-equivariant secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: build_type: pull-request script: ci/test_wheel_cugraph-equivariant.sh matrix_filter: map(select(.ARCH == "amd64")) devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.08 with: arch: '["amd64"]' cuda: '["12.5"]' diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cab2063115a..03d8b73a412 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -26,7 +26,7 @@ jobs: symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel) conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -42,7 +42,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibcugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: script: ci/test_wheel_pylibcugraph.sh wheel-tests-cugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -60,7 +60,7 @@ jobs: script: ci/test_wheel_cugraph.sh wheel-tests-nx-cugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: script: ci/test_wheel_nx-cugraph.sh wheel-tests-cugraph-dgl: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -79,7 +79,7 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) wheel-tests-cugraph-pyg: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -89,7 +89,7 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) wheel-tests-cugraph-equivariant: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} From c9d28432ff5e05c117aaa10eb41025053d2eb230 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 19 Jul 2024 18:55:25 -0500 Subject: [PATCH 09/11] remove thriftpy2 ceiling (#4521) #4496 introduced a ceiling on `thriftpy2`. Context: https://github.com/rapidsai/cugraph/pull/4496#issuecomment-2192783353 The bug that ceiling was added to avoid was fixed in v0.5.2 of `thriftpy2`, which was just released (https://github.com/rapidsai/cugraph/pull/4478#issuecomment-2210762587). This removes that, adding `!=` constraints to skip the 2 versions that `cugraph` was not compatible with. ## Notes for Reviewers ### Why not a floor? I'm proposing adding `!=` constraints to skip v0.5.0 and v0.5.1 to maximize `cugraph`'s compatibility with other environments... that'd allow it to be used in environments with `thriftpy2<0.5.0` and in environments with `thriftpy2>0.5.2`. Let me know if you'd prefer the simplicity of a floor like `>=0.5.2` instead. Authors: - James Lamb (https://github.com/jameslamb) - Ralph Liu (https://github.com/nv-rliu) Approvers: - Bradley Dice (https://github.com/bdice) - Ralph Liu (https://github.com/nv-rliu) - Rick Ratzel (https://github.com/rlratzel) URL: https://github.com/rapidsai/cugraph/pull/4521 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-125_arch-x86_64.yaml | 2 +- conda/recipes/cugraph-service/meta.yaml | 4 ++-- dependencies.yaml | 4 +--- python/cugraph-service/client/pyproject.toml | 2 +- python/cugraph-service/server/pyproject.toml | 2 +- 6 files changed, 7 insertions(+), 9 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 5885d154aa5..5d2c942cd0c 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -71,7 +71,7 @@ dependencies: - sphinx-markdown-tables - sphinx<6 - sphinxcontrib-websupport -- thriftpy2<=0.5.0 +- thriftpy2!=0.5.0,!=0.5.1 - ucx-proc=*=gpu - ucx-py==0.39.*,>=0.0.0a0 - wget diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 8e7c2156b93..f8a95169ddd 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -76,7 +76,7 @@ dependencies: - sphinx-markdown-tables - sphinx<6 - sphinxcontrib-websupport -- thriftpy2<=0.5.0 +- thriftpy2!=0.5.0,!=0.5.1 - ucx-proc=*=gpu - ucx-py==0.39.*,>=0.0.0a0 - wget diff --git a/conda/recipes/cugraph-service/meta.yaml b/conda/recipes/cugraph-service/meta.yaml index 79b2837eb53..225f40fe2ec 100644 --- a/conda/recipes/cugraph-service/meta.yaml +++ b/conda/recipes/cugraph-service/meta.yaml @@ -32,7 +32,7 @@ outputs: - rapids-build-backend>=0.3.1,<0.4.0.dev0 run: - python - - thriftpy2 >=0.4.15 + - thriftpy2 >=0.4.15,!=0.5.0,!=0.5.1 - name: cugraph-service-server version: {{ version }} @@ -65,7 +65,7 @@ outputs: - numpy >=1.23,<2.0a0 - python - rapids-dask-dependency ={{ minor_version }} - - thriftpy2 >=0.4.15 + - thriftpy2 >=0.4.15,!=0.5.0,!=0.5.1 - ucx-py {{ ucx_py_version }} tests: diff --git a/dependencies.yaml b/dependencies.yaml index 1ef92274bde..c25dcf40cf3 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -563,9 +563,7 @@ dependencies: common: - output_types: [conda, pyproject] packages: - # this thriftpy2 entry can be removed entirely (or switched to a '!=') - # once a new release of that project resolves https://github.com/Thriftpy/thriftpy2/issues/281 - - &thrift thriftpy2<=0.5.0 + - &thrift thriftpy2!=0.5.0,!=0.5.1 python_run_cugraph_service_server: common: - output_types: [conda, pyproject] diff --git a/python/cugraph-service/client/pyproject.toml b/python/cugraph-service/client/pyproject.toml index e805a2aac95..53170e888ba 100644 --- a/python/cugraph-service/client/pyproject.toml +++ b/python/cugraph-service/client/pyproject.toml @@ -20,7 +20,7 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "thriftpy2<=0.5.0", + "thriftpy2!=0.5.0,!=0.5.1", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", diff --git a/python/cugraph-service/server/pyproject.toml b/python/cugraph-service/server/pyproject.toml index 2369c47b6d6..4a79e13e532 100644 --- a/python/cugraph-service/server/pyproject.toml +++ b/python/cugraph-service/server/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "numpy>=1.23,<2.0a0", "rapids-dask-dependency==24.8.*,>=0.0.0a0", "rmm==24.8.*,>=0.0.0a0", - "thriftpy2<=0.5.0", + "thriftpy2!=0.5.0,!=0.5.1", "ucx-py==0.39.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ From 522055b50f8a41b243b725bea355221a25baab64 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Mon, 22 Jul 2024 19:00:51 -0700 Subject: [PATCH 10/11] Add `-cuXX` suffixed versions of cugraph-service-client dependency to pyproject.toml's project.dependencies list (#4530) This change ensures RAPIDS build backend writes the correct `cugraph-service-client` package in the `cugraph-service-server` wheel's dependencies metadata. Authors: - Paul Taylor (https://github.com/trxcllnt) Approvers: - Mike Sarahan (https://github.com/msarahan) URL: https://github.com/rapidsai/cugraph/pull/4530 --- dependencies.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index c25dcf40cf3..9d0a5478a50 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -573,10 +573,6 @@ dependencies: - *numba - *numpy - *thrift - - output_types: pyproject - packages: - - *cugraph - - cugraph-service-client==24.8.*,>=0.0.0a0 - output_types: conda packages: - *ucx_py @@ -586,13 +582,19 @@ dependencies: - matrix: cuda: "11.*" packages: + - *cugraph_cu11 + - cugraph-service-client-cu11==24.8.*,>=0.0.0a0 - *ucx_py_cu11 - matrix: cuda: "12.*" packages: + - *cugraph_cu12 + - cugraph-service-client-cu12==24.8.*,>=0.0.0a0 - *ucx_py_cu12 - matrix: packages: + - *cugraph + - cugraph-service-client==24.8.*,>=0.0.0a0 - *ucx_py test_cpp: common: From 0900a06f78dd98864b7d9533c37f42e2ebe25f4b Mon Sep 17 00:00:00 2001 From: Joseph Nke <76006812+jnke2016@users.noreply.github.com> Date: Wed, 24 Jul 2024 10:04:14 +0100 Subject: [PATCH 11/11] Fix triangle count test bug (#4549) This PR count the total number of triangles before testing it against the ground truth. It also remove an unnecessary sorting before testing. Authors: - Joseph Nke (https://github.com/jnke2016) Approvers: - Rick Ratzel (https://github.com/rlratzel) URL: https://github.com/rapidsai/cugraph/pull/4549 --- .../tests/community/test_triangle_count.py | 30 +++++-------------- .../cugraph/tests/traversal/test_paths.py | 16 ++++++++-- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/python/cugraph/cugraph/tests/community/test_triangle_count.py b/python/cugraph/cugraph/tests/community/test_triangle_count.py index 449df32b52a..69cd5fd72e4 100644 --- a/python/cugraph/cugraph/tests/community/test_triangle_count.py +++ b/python/cugraph/cugraph/tests/community/test_triangle_count.py @@ -105,48 +105,32 @@ def test_triangles(input_combo): @pytest.mark.sg def test_triangles_int64(input_combo): Gnx = input_combo["Gnx"] - count_legacy_32 = cugraph.triangle_count(Gnx) + count_int32 = cugraph.triangle_count(Gnx)["counts"].sum() graph_file = input_combo["graph_file"] G = graph_file.get_graph() G.edgelist.edgelist_df = G.edgelist.edgelist_df.astype( {"src": "int64", "dst": "int64"} ) + count_int64 = cugraph.triangle_count(G)["counts"].sum() - count_exp_64 = ( - cugraph.triangle_count(G) - .sort_values("vertex") - .reset_index(drop=True) - .rename(columns={"counts": "exp_cugraph_counts"}) - ) - cugraph_exp_triangle_results = count_exp_64["exp_cugraph_counts"].sum() assert G.edgelist.edgelist_df["src"].dtype == "int64" assert G.edgelist.edgelist_df["dst"].dtype == "int64" - assert cugraph_exp_triangle_results == count_legacy_32 + assert count_int32 == count_int64 @pytest.mark.sg def test_triangles_no_weights(input_combo): G_weighted = input_combo["Gnx"] - count_legacy = ( - cugraph.triangle_count(G_weighted) - .sort_values("vertex") - .reset_index(drop=True) - .rename(columns={"counts": "exp_cugraph_counts"}) - ) + count_triangles_nx_graph = cugraph.triangle_count(G_weighted)["counts"].sum() graph_file = input_combo["graph_file"] G = graph_file.get_graph(ignore_weights=True) assert G.is_weighted() is False - triangle_count = ( - cugraph.triangle_count(G) - .sort_values("vertex") - .reset_index(drop=True) - .rename(columns={"counts": "exp_cugraph_counts"}) - ) - cugraph_exp_triangle_results = triangle_count["exp_cugraph_counts"].sum() - assert cugraph_exp_triangle_results == count_legacy + count_triangles = cugraph.triangle_count(G)["counts"].sum() + + assert count_triangles_nx_graph == count_triangles @pytest.mark.sg diff --git a/python/cugraph/cugraph/tests/traversal/test_paths.py b/python/cugraph/cugraph/tests/traversal/test_paths.py index 5ee22874f4a..4ef10da593c 100644 --- a/python/cugraph/cugraph/tests/traversal/test_paths.py +++ b/python/cugraph/cugraph/tests/traversal/test_paths.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -22,6 +22,7 @@ import cupy import cugraph from cugraph.testing import get_resultset, load_resultset +from cudf.testing.testing import assert_series_equal from cupyx.scipy.sparse import coo_matrix as cupy_coo_matrix @@ -204,7 +205,11 @@ def test_shortest_path_length_no_path(graphs): def test_shortest_path_length_no_target(graphs, load_traversal_results): cugraph_G, cupy_df = graphs - cugraph_path_1_to_all = cugraph.shortest_path_length(cugraph_G, 1) + cugraph_path_1_to_all = ( + cugraph.shortest_path_length(cugraph_G, 1) + .sort_values("vertex") + .reset_index(drop=True) + ) golden_path_1_to_all = get_resultset( resultset_name="traversal", algo="shortest_path_length", @@ -217,7 +222,12 @@ def test_shortest_path_length_no_target(graphs, load_traversal_results): # Cast networkx graph on cugraph vertex column type from str to int. # SSSP preserves vertex type, convert for comparison - assert cugraph_path_1_to_all == cupy_path_1_to_all + assert_series_equal( + cugraph_path_1_to_all["distance"], + cupy_path_1_to_all["distance"], + check_names=False, + check_dtype=False, + ) # results for vertex 8 and 9 are not returned assert cugraph_path_1_to_all.shape[0] == len(golden_path_1_to_all) + 2