Optimize opencl and make it default gpu feature.

filecoin-project · Mar 2, 2021 · 17364e7 · 17364e7
1 parent 64390b6
commit 17364e7
Show file tree

Hide file tree

Showing 9 changed files with 155 additions and 112 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -11,6 +11,11 @@ executors:
       - image: filecoin/rust:latest
     working_directory: /mnt/crate
     resource_class: 2xlarge+
+  gpu:
+    machine:
+      image: ubuntu-1604-cuda-10.1:201909-23
+    working_directory: ~/gpuci
+    resource_class: gpu.nvidia.medium
 
 setup-env: &setup-env
   FIL_PROOFS_PARAMETER_CACHE: "/root/filecoin-proof-parameters/"
@@ -27,6 +32,16 @@ jobs:
       - ensure_filecoin_parameters
       - save_parameter_cache
 
+  ensure_groth_parameters_and_keys_linux_gpu:
+    executor: gpu
+    environment: *setup-env
+    steps:
+      - checkout
+      - cargo_fetch_gpu
+      - restore_parameter_cache_gpu
+      - ensure_filecoin_parameters_gpu
+      - save_parameter_cache
+
   cargo_fetch:
     executor: default
     environment: *setup-env
@@ -57,6 +72,43 @@ jobs:
           paths:
             - /root/.cargo
             - /root/.rustup
+  cargo_fetch_gpu:
+    executor: gpu
+    environment: *setup-env
+    steps:
+      - checkout
+      - run: curl https://sh.rustup.rs -sSf | sh -s -- -y
+      - run: echo 'export PATH="$HOME:~/.cargo/bin:$PATH"' >> $BASH_ENV
+      - run: echo $BASH_ENV
+      - run: echo $HOME
+      - run: source $BASH_ENV
+      - run: cargo --version
+      - run: rustc --version
+      - run:
+          name: Calculate dependencies
+          command: cargo generate-lockfile
+          no_output_timeout: 30m
+      - restore_cache:
+          keys:
+            - cargo-v28-gpu-d-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - run: rustup install $(cat rust-toolchain)
+      - run: rustup default $(cat rust-toolchain)
+      - run: rustup install << pipeline.parameters.nightly-toolchain >>
+      - run: rustup component add rustfmt-preview
+      - run: rustup component add clippy
+      - run: cargo update
+      - run: cargo fetch
+      - run: rustc +$(cat rust-toolchain) --version
+      - run: rustup toolchain list --verbose
+      - persist_to_workspace:
+          root: ~/gpuci
+          paths:
+            - Cargo.lock
+      - save_cache:
+          key: cargo-v28-gpu-d-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+          paths:
+            - "~/.cargo"
+            - "~/.rustup"
   test:
     executor: default
     environment: *setup-env
@@ -152,7 +204,6 @@ jobs:
           environment:
             RUST_TEST_THREADS: 1
             FIL_PROOFS_USE_MULTICORE_SDR: true
-
       - run:
           name: Test with use_multicore_sdr and blst enabled
           command: |
@@ -165,44 +216,34 @@ jobs:
             RUST_TEST_THREADS: 1
             FIL_PROOFS_USE_MULTICORE_SDR: true
 
-  # Running with `use_multicore_sdr=true` should be integrated directly into the test code. For now we
-  # just re-run the lifecycle tests to exercise the use_multicore_sdr code path with that setting set.
-  test_multicore_sdr_gpu2:
-    executor: default
+  test_gpu_tree_building:
+    executor: gpu
     environment: *setup-env
     steps:
       - checkout
       - attach_workspace:
           at: "."
       - restore_cache:
           keys:
-            - cargo-v28-b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+            - cargo-v28-gpu-d-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - restore_parameter_cache
       - run:
-          name: Test with use_multicore_sdr pairing enabled
+          name: Test with GPU column and tree builders.
           command: |
+            sudo apt-get update -y
+            apt-cache search opencl
+            sudo apt install -y ocl-icd-opencl-dev
+            sudo apt install hwloc libhwloc-dev
             ulimit -n 20000
             ulimit -u 20000
             ulimit -n 20000
-            cargo +<< pipeline.parameters.nightly-toolchain >> -Zpackage-features test --all --verbose --release lifecycle -- --ignored --nocapture
-            cargo +<< pipeline.parameters.nightly-toolchain >> -Zpackage-features test -p storage-proofs-porep --features single-threaded --release checkout_cores -- --test-threads=1
+            cd filecoin-proofs
+            ~/.cargo/bin/cargo +<< pipeline.parameters.nightly-toolchain >> -Zpackage-features test --all --verbose --release lifecycle -- --ignored --nocapture
           no_output_timeout: 30m
           environment:
             RUST_TEST_THREADS: 1
-            FIL_PROOFS_USE_MULTICORE_SDR: true
-
-      - run:
-          name: Test with use_multicore_sdr and blst enabled
-          command: |
-            ulimit -n 20000
-            ulimit -u 20000
-            ulimit -n 20000
-            cargo +<< pipeline.parameters.nightly-toolchain >> -Zpackage-features test --all --no-default-features --features gpu2,blst --verbose --release  lifecycle -- --ignored --nocapture
-          no_output_timeout: 30m
-          environment:
-            RUST_TEST_THREADS: 1
-            FIL_PROOFS_USE_MULTICORE_SDR: true
-
+            FIL_PROOFS_USE_GPU_COLUMN_BUILDER: true
+            FIL_PROOFS_USE_GPU_TREE_BUILDER: true
 
   test_no_gpu:
     executor: default
@@ -226,7 +267,6 @@ jobs:
             cargo +<< pipeline.parameters.nightly-toolchain >> -Zpackage-features test --all --verbose --no-default-features --features blst
           no_output_timeout: 30m
 
-
   test_blst:
     executor: default
     environment: *setup-env
@@ -442,6 +482,18 @@ commands:
           name: Obtain filecoin groth parameters
           command: ~/paramcache.awesome --sector-sizes='2048,4096,16384,32768'
           no_output_timeout: 60m
+  ensure_filecoin_parameters_gpu:
+    steps:
+      - run:
+          name: Build paramcache if it doesn't already exist
+          command: |
+            set -x; test -f ~/paramcache.awesome \
+            || (~/.cargo/bin/cargo build --release --workspace && find . -type f -name paramcache | xargs -I {} mv {} ~/paramcache.awesome)
+      - run:
+          name: Obtain filecoin groth parameters
+          command: ~/paramcache.awesome --sector-sizes='2048,4096,16384,32768'
+          no_output_timeout: 60m
+
   save_parameter_cache:
     steps:
       - save_cache:
@@ -454,13 +506,29 @@ commands:
       - restore_cache:
          keys:
             - proof-params-v28-b-{{ checksum "filecoin-proofs/parameters.json" }}-{{ arch }}
+  save_parameter_cache_gpu:
+    steps:
+      - save_cache:
+          key: proof-params-v28-gpu-{{ checksum "filecoin-proofs/parameters.json" }}-{{ arch }}
+          paths:
+            - "~/paramcache.awesome"
+            - "~/filecoin-proof-parameters/"
+  restore_parameter_cache_gpu:
+    steps:
+      - restore_cache:
+         keys:
+            - proof-params-v28-gpu-{{ checksum "filecoin-proofs/parameters.json" }}-{{ arch }}
 
 workflows:
   version: 2.1
   test_all:
     jobs:
       - ensure_groth_parameters_and_keys_linux
+      - ensure_groth_parameters_and_keys_linux_gpu:
+          requires:
+            - cargo_fetch_gpu
       - cargo_fetch
+      - cargo_fetch_gpu
       - rustfmt:
           requires:
             - cargo_fetch
@@ -501,12 +569,12 @@ workflows:
             - cargo_fetch
             - ensure_groth_parameters_and_keys_linux
 
-      - test_multicore_sdr:
+      - test_gpu_tree_building:
           requires:
-            - cargo_fetch
-            - ensure_groth_parameters_and_keys_linux
+            - cargo_fetch_gpu
+            - ensure_groth_parameters_and_keys_linux_gpu
 
-      - test_multicore_sdr_gpu2:
+      - test_multicore_sdr:
           requires:
             - cargo_fetch
             - ensure_groth_parameters_and_keys_linux

diff --git a/fil-proofs-param/Cargo.toml b/fil-proofs-param/Cargo.toml
@@ -68,6 +68,5 @@ heap-profile = ["gperftools/heap"]
 simd = ["storage-proofs-core/simd"]
 asm = ["storage-proofs-core/asm"]
 gpu = ["storage-proofs-core/gpu", "storage-proofs-porep/gpu", "storage-proofs-post/gpu", "bellperson/gpu"]
-gpu2 = ["storage-proofs-core/gpu2", "storage-proofs-porep/gpu2", "storage-proofs-post/gpu2", "bellperson/gpu"]
 pairing = ["storage-proofs-core/pairing", "storage-proofs-porep/pairing", "storage-proofs-post/pairing", "bellperson/pairing"]
 blst = ["storage-proofs-core/blst", "storage-proofs-porep/blst", "storage-proofs-post/blst", "bellperson/blst"]
diff --git a/fil-proofs-tooling/Cargo.toml b/fil-proofs-tooling/Cargo.toml
@@ -62,14 +62,6 @@ gpu = [
     "bellperson/gpu",
     "filecoin-hashers/gpu",
 ]
-gpu2 = [
-    "storage-proofs-core/gpu2",
-    "storage-proofs-porep/gpu2",
-    "storage-proofs-post/gpu2",
-    "filecoin-proofs/gpu2",
-    "bellperson/gpu",
-    "filecoin-hashers/gpu2",
-]
 measurements = ["storage-proofs-core/measurements"]
 profile = ["storage-proofs-core/profile", "measurements"]
 pairing = [

diff --git a/filecoin-hashers/Cargo.toml b/filecoin-hashers/Cargo.toml
@@ -26,8 +26,7 @@ hex = "0.4.2"
 [features]
 default = ["gpu", "pairing", "blake2s", "poseidon", "sha256"]
 
-gpu = ["bellperson/gpu", "neptune/gpu"]
-gpu2 = ["bellperson/gpu", "neptune/opencl"]
+gpu = ["bellperson/gpu", "neptune/opencl"]
 
 pairing = ["bellperson/pairing", "neptune/pairing", "bellperson/pairing-serde"]
 blst = ["bellperson/blst", "neptune/blst", "bellperson/blst-serde"]

diff --git a/filecoin-proofs/Cargo.toml b/filecoin-proofs/Cargo.toml
@@ -62,14 +62,6 @@ gpu = [
     "filecoin-hashers/gpu",
     "fr32/gpu",
 ]
-gpu2 = [
-    "storage-proofs-core/gpu2",
-    "storage-proofs-porep/gpu2",
-    "storage-proofs-post/gpu2",
-    "bellperson/gpu",
-    "filecoin-hashers/gpu2",
-    "fr32/gpu",
-]
 pairing = [
     "storage-proofs-core/pairing",
     "storage-proofs-porep/pairing",

diff --git a/storage-proofs-core/Cargo.toml b/storage-proofs-core/Cargo.toml
@@ -63,8 +63,7 @@ big-sector-sizes-bench = []
 measurements = ["cpu-time", "gperftools"]
 profile = ["measurements"]
 
-gpu = ["bellperson/gpu", "neptune/gpu", "filecoin-hashers/gpu", "fr32/gpu"]
-gpu2 = ["bellperson/gpu", "neptune/opencl", "filecoin-hashers/gpu2", "fr32/gpu"]
+gpu = ["bellperson/gpu", "neptune/opencl", "filecoin-hashers/gpu", "fr32/gpu"]
 pairing = ["bellperson/pairing", "neptune/pairing", "bellperson/pairing-serde", "filecoin-hashers/pairing", "fr32/pairing"]
 blst = ["bellperson/blst", "neptune/blst", "bellperson/blst-serde", "filecoin-hashers/blst", "fr32/blst"]
 

diff --git a/storage-proofs-porep/Cargo.toml b/storage-proofs-porep/Cargo.toml
@@ -51,8 +51,7 @@ filecoin-hashers = { path = "../filecoin-hashers", version = "1.0.0", default-fe
 
 [features]
 default = ["pairing", "gpu"]
-gpu = ["storage-proofs-core/gpu", "filecoin-hashers/gpu", "neptune/gpu", "bellperson/gpu", "fr32/gpu"]
-gpu2 = ["storage-proofs-core/gpu2", "filecoin-hashers/gpu2", "neptune/opencl", "bellperson/gpu", "fr32/gpu"]
+gpu = ["storage-proofs-core/gpu", "filecoin-hashers/gpu", "neptune/opencl", "bellperson/gpu", "fr32/gpu"]
 pairing = ["storage-proofs-core/pairing", "bellperson/pairing", "neptune/pairing", "filecoin-hashers/pairing", "fr32/pairing"]
 blst = ["storage-proofs-core/blst", "bellperson/blst", "neptune/blst", "filecoin-hashers/blst", "fr32/blst"]
 single-threaded = []