Optimize opencl and make it default gpu feature.

filecoin-project · Mar 2, 2021 · 9ff7acb · 9ff7acb
1 parent 64390b6
commit 9ff7acb
Show file tree

Hide file tree

Showing 9 changed files with 73 additions and 107 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -11,6 +11,11 @@ executors:
       - image: filecoin/rust:latest
     working_directory: /mnt/crate
     resource_class: 2xlarge+
+  gpu:
+    machine:
+      image: ubuntu-1604-cuda-10.1:201909-23
+    working_directory: ~/gpuci
+    resource_class: gpu.nvidia.medium
 
 setup-env: &setup-env
   FIL_PROOFS_PARAMETER_CACHE: "/root/filecoin-proof-parameters/"
@@ -152,7 +157,6 @@ jobs:
           environment:
             RUST_TEST_THREADS: 1
             FIL_PROOFS_USE_MULTICORE_SDR: true
-
       - run:
           name: Test with use_multicore_sdr and blst enabled
           command: |
@@ -165,10 +169,8 @@ jobs:
             RUST_TEST_THREADS: 1
             FIL_PROOFS_USE_MULTICORE_SDR: true
 
-  # Running with `use_multicore_sdr=true` should be integrated directly into the test code. For now we
-  # just re-run the lifecycle tests to exercise the use_multicore_sdr code path with that setting set.
-  test_multicore_sdr_gpu2:
-    executor: default
+  test_gpu_tree_building:
+    executor: gpu
     environment: *setup-env
     steps:
       - checkout
@@ -179,7 +181,7 @@ jobs:
             - cargo-v28-b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - restore_parameter_cache
       - run:
-          name: Test with use_multicore_sdr pairing enabled
+          name: Test with GPU column and tree builders.
           command: |
             ulimit -n 20000
             ulimit -u 20000
@@ -189,20 +191,8 @@ jobs:
           no_output_timeout: 30m
           environment:
             RUST_TEST_THREADS: 1
-            FIL_PROOFS_USE_MULTICORE_SDR: true
-
-      - run:
-          name: Test with use_multicore_sdr and blst enabled
-          command: |
-            ulimit -n 20000
-            ulimit -u 20000
-            ulimit -n 20000
-            cargo +<< pipeline.parameters.nightly-toolchain >> -Zpackage-features test --all --no-default-features --features gpu2,blst --verbose --release  lifecycle -- --ignored --nocapture
-          no_output_timeout: 30m
-          environment:
-            RUST_TEST_THREADS: 1
-            FIL_PROOFS_USE_MULTICORE_SDR: true
-
+            FIL_PROOFS_USE_GPU_COLUMN_BUILDER: true
+            FIL_PROOFS_USE_GPU_TREE_BUILDER: true
 
   test_no_gpu:
     executor: default
@@ -226,7 +216,6 @@ jobs:
             cargo +<< pipeline.parameters.nightly-toolchain >> -Zpackage-features test --all --verbose --no-default-features --features blst
           no_output_timeout: 30m
 
-
   test_blst:
     executor: default
     environment: *setup-env
@@ -501,12 +490,14 @@ workflows:
             - cargo_fetch
             - ensure_groth_parameters_and_keys_linux
 
-      - test_multicore_sdr:
+      - test_gpu_tree_building:
+          name: test_ignored_release_filecoin_proofs_gpu_tree_builders
+          crate: "filecoin-proofs"
           requires:
             - cargo_fetch
             - ensure_groth_parameters_and_keys_linux
 
-      - test_multicore_sdr_gpu2:
+      - test_multicore_sdr:
           requires:
             - cargo_fetch
             - ensure_groth_parameters_and_keys_linux

diff --git a/fil-proofs-param/Cargo.toml b/fil-proofs-param/Cargo.toml
@@ -68,6 +68,5 @@ heap-profile = ["gperftools/heap"]
 simd = ["storage-proofs-core/simd"]
 asm = ["storage-proofs-core/asm"]
 gpu = ["storage-proofs-core/gpu", "storage-proofs-porep/gpu", "storage-proofs-post/gpu", "bellperson/gpu"]
-gpu2 = ["storage-proofs-core/gpu2", "storage-proofs-porep/gpu2", "storage-proofs-post/gpu2", "bellperson/gpu"]
 pairing = ["storage-proofs-core/pairing", "storage-proofs-porep/pairing", "storage-proofs-post/pairing", "bellperson/pairing"]
 blst = ["storage-proofs-core/blst", "storage-proofs-porep/blst", "storage-proofs-post/blst", "bellperson/blst"]
diff --git a/fil-proofs-tooling/Cargo.toml b/fil-proofs-tooling/Cargo.toml
@@ -62,14 +62,6 @@ gpu = [
     "bellperson/gpu",
     "filecoin-hashers/gpu",
 ]
-gpu2 = [
-    "storage-proofs-core/gpu2",
-    "storage-proofs-porep/gpu2",
-    "storage-proofs-post/gpu2",
-    "filecoin-proofs/gpu2",
-    "bellperson/gpu",
-    "filecoin-hashers/gpu2",
-]
 measurements = ["storage-proofs-core/measurements"]
 profile = ["storage-proofs-core/profile", "measurements"]
 pairing = [

diff --git a/filecoin-hashers/Cargo.toml b/filecoin-hashers/Cargo.toml
@@ -26,8 +26,7 @@ hex = "0.4.2"
 [features]
 default = ["gpu", "pairing", "blake2s", "poseidon", "sha256"]
 
-gpu = ["bellperson/gpu", "neptune/gpu"]
-gpu2 = ["bellperson/gpu", "neptune/opencl"]
+gpu = ["bellperson/gpu", "neptune/opencl"]
 
 pairing = ["bellperson/pairing", "neptune/pairing", "bellperson/pairing-serde"]
 blst = ["bellperson/blst", "neptune/blst", "bellperson/blst-serde"]

diff --git a/filecoin-proofs/Cargo.toml b/filecoin-proofs/Cargo.toml
@@ -62,14 +62,6 @@ gpu = [
     "filecoin-hashers/gpu",
     "fr32/gpu",
 ]
-gpu2 = [
-    "storage-proofs-core/gpu2",
-    "storage-proofs-porep/gpu2",
-    "storage-proofs-post/gpu2",
-    "bellperson/gpu",
-    "filecoin-hashers/gpu2",
-    "fr32/gpu",
-]
 pairing = [
     "storage-proofs-core/pairing",
     "storage-proofs-porep/pairing",

diff --git a/storage-proofs-core/Cargo.toml b/storage-proofs-core/Cargo.toml
@@ -63,8 +63,7 @@ big-sector-sizes-bench = []
 measurements = ["cpu-time", "gperftools"]
 profile = ["measurements"]
 
-gpu = ["bellperson/gpu", "neptune/gpu", "filecoin-hashers/gpu", "fr32/gpu"]
-gpu2 = ["bellperson/gpu", "neptune/opencl", "filecoin-hashers/gpu2", "fr32/gpu"]
+gpu = ["bellperson/gpu", "neptune/opencl", "filecoin-hashers/gpu", "fr32/gpu"]
 pairing = ["bellperson/pairing", "neptune/pairing", "bellperson/pairing-serde", "filecoin-hashers/pairing", "fr32/pairing"]
 blst = ["bellperson/blst", "neptune/blst", "bellperson/blst-serde", "filecoin-hashers/blst", "fr32/blst"]
 

diff --git a/storage-proofs-porep/Cargo.toml b/storage-proofs-porep/Cargo.toml
@@ -51,8 +51,7 @@ filecoin-hashers = { path = "../filecoin-hashers", version = "1.0.0", default-fe
 
 [features]
 default = ["pairing", "gpu"]
-gpu = ["storage-proofs-core/gpu", "filecoin-hashers/gpu", "neptune/gpu", "bellperson/gpu", "fr32/gpu"]
-gpu2 = ["storage-proofs-core/gpu2", "filecoin-hashers/gpu2", "neptune/opencl", "bellperson/gpu", "fr32/gpu"]
+gpu = ["storage-proofs-core/gpu", "filecoin-hashers/gpu", "neptune/opencl", "bellperson/gpu", "fr32/gpu"]
 pairing = ["storage-proofs-core/pairing", "bellperson/pairing", "neptune/pairing", "filecoin-hashers/pairing", "fr32/pairing"]
 blst = ["storage-proofs-core/blst", "bellperson/blst", "neptune/blst", "filecoin-hashers/blst", "fr32/blst"]
 single-threaded = []