Merge pull request #149 from filecoin-project/blstrs

feat: proofs-aip v5.3.0 (new blst backend and optimizations)
filecoin-project · Nov 2, 2020 · af37d8d · af37d8d
2 parents 0226d0b + d4ed4f7
commit af37d8d
Show file tree

Hide file tree

Showing 11 changed files with 450 additions and 419 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -32,10 +32,11 @@ jobs:
       - go/mod-download
       - go/install-golangci-lint:
           gobin: $HOME/.local/bin
-          version: 1.23.8
+          version: 1.32.0
       - run:
-          command: LD_LIBRARY_PATH="/tmp/__fil-hwloc/lib" make go-lint
-  build_and_test_linux_cgo_bindings:
+          command: make go-lint
+
+  build_and_test_linux_cgo_bindings_pairing:
     parameters:
       run_leak_detector:
         type: boolean
@@ -51,6 +52,25 @@ jobs:
       - save_parameter_cache
       - run_tests:
           run_leak_detector: << parameters.run_leak_detector >>
+
+  build_and_test_linux_cgo_bindings_blst:
+    parameters:
+      run_leak_detector:
+        type: boolean
+        default: false
+    executor: golang
+    working_directory: ~/go/src/github.com/filecoin-project/filecoin-ffi
+    steps:
+      - configure_environment_variables
+      - prepare
+      - build_project:
+          blst: true
+      - restore_parameter_cache
+      - obtain_filecoin_parameters
+      - save_parameter_cache
+      - run_tests:
+          run_leak_detector: << parameters.run_leak_detector >>
+
   build_darwin_cgo_bindings:
     macos:
       xcode: "10.0.0"
@@ -138,8 +158,11 @@ jobs:
       - run: cd rust && rustup component add clippy
       - run: cd rust && cargo fetch
       - run:
-          name: Run cargo clippy
-          command: cd rust && cargo +$(cat rust-toolchain) clippy --all-targets --all-features -- -D warnings
+          name: Run cargo clippy (pairing)
+          command: cd rust && cargo +$(cat rust-toolchain) clippy --all-targets -- -D warnings
+      - run:
+          name: Run cargo clippy (blst)
+          command: cd rust && cargo +$(cat rust-toolchain) clippy --all-targets --no-default-features --features blst -- -D warnings
 
 workflows:
   version: 2
@@ -157,18 +180,32 @@ workflows:
             - cargo_fetch
       - gofmt
       - go_lint
-      - build_and_test_linux_cgo_bindings:
+      - build_and_test_linux_cgo_bindings_pairing:
+          filters:
+            branches:
+              only:
+                - master
+          run_leak_detector: true
+      - build_and_test_linux_cgo_bindings_pairing:
+          filters:
+            branches:
+              ignore:
+                - master
+          run_leak_detector: false
+
+      - build_and_test_linux_cgo_bindings_blst:
           filters:
             branches:
               only:
                 - master
           run_leak_detector: true
-      - build_and_test_linux_cgo_bindings:
+      - build_and_test_linux_cgo_bindings_blst:
           filters:
             branches:
               ignore:
                 - master
           run_leak_detector: false
+
       - publish_linux_staticlib:
           filters:
             branches:
@@ -200,7 +237,7 @@ commands:
             - go/install-ssh
             - go/install: {package: git}
             - run: sudo apt-get update
-            - run: sudo apt-get install -y jq valgrind ocl-icd-opencl-dev clang libssl-dev
+            - run: sudo apt-get install -y jq valgrind ocl-icd-opencl-dev clang libssl-dev libhwloc-dev
             - run: curl https://sh.rustup.rs -sSf | sh -s -- -y
       - when:
           condition: << parameters.darwin >>
@@ -229,28 +266,47 @@ commands:
   publish_release:
     steps:
       - run:
-          name: Build and publish the standard release
+          name: Build and publish the standard release (pairing)
           command: |
             cd rust
 
             TARBALL_PATH="/tmp/${CIRCLE_PROJECT_REPONAME}-$(uname)-standard.tar.gz"
             RELEASE_NAME="${CIRCLE_PROJECT_REPONAME}-$(uname)-standard"
 
-            ./scripts/build-release.sh filcrypto $(cat ./rust-toolchain) --verbose --locked --all
+            ./scripts/build-release.sh filcrypto $(cat ./rust-toolchain) --verbose --locked --no-default-features --features pairing
             ./scripts/package-release.sh $TARBALL_PATH
             ./scripts/publish-release.sh $TARBALL_PATH $RELEASE_NAME
       - run:
-          name: Build and publish the optimized release
+          name: Build and publish the standard release (blst)
+          command: |
+            cd rust
+
+            TARBALL_PATH="/tmp/${CIRCLE_PROJECT_REPONAME}-$(uname)-standard-blst.tar.gz"
+            RELEASE_NAME="${CIRCLE_PROJECT_REPONAME}-$(uname)-standard-blst"
+
+            ./scripts/build-release.sh filcrypto $(cat ./rust-toolchain) --verbose --locked --no-default-features --features blst
+            ./scripts/package-release.sh $TARBALL_PATH
+            ./scripts/publish-release.sh $TARBALL_PATH $RELEASE_NAME
+      - run:
+          name: Build the optimized release (pairing)
+          command: |
+            cd rust
+
+            TARBALL_PATH="/tmp/${CIRCLE_PROJECT_REPONAME}-$(uname)-optimized.tar.gz"
+            RUSTFLAGS="-C target-feature=$(cat rustc-target-features-optimized.json | jq -r '.[].rustc_target_feature' | tr '\n' ',')"
+
+            ./scripts/build-release.sh filcrypto $(cat ./rust-toolchain) --verbose --locked --no-default-features --features pairing
+            ./scripts/package-release.sh $TARBALL_PATH
+      - run:
+          name: Build the optimized release (blst)
           command: |
             cd rust
 
             TARBALL_PATH="/tmp/${CIRCLE_PROJECT_REPONAME}-$(uname)-optimized.tar.gz"
-            RELEASE_NAME="${CIRCLE_PROJECT_REPONAME}-$(uname)-optimized"
             RUSTFLAGS="-C target-feature=$(cat rustc-target-features-optimized.json | jq -r '.[].rustc_target_feature' | tr '\n' ',')"
 
             ./scripts/build-release.sh filcrypto $(cat ./rust-toolchain) --verbose --locked --all
             ./scripts/package-release.sh $TARBALL_PATH
-            ./scripts/publish-release.sh $TARBALL_PATH $RELEASE_NAME
   configure_environment_variables:
     parameters:
       linux:
@@ -286,14 +342,31 @@ commands:
           GOPATH=/tmp GO111MODULE=off go get github.com/filecoin-project/go-paramfetch/paramfetch
           GOPATH=/tmp GO111MODULE=off go build -o go-paramfetch github.com/filecoin-project/go-paramfetch/paramfetch
           ./go-paramfetch 2048 "${DIR}/parameters.json"
+
   build_project:
+    parameters:
+      blst:
+        default: false
+        description: build with blst backend?
+        type: boolean
     steps:
-      - run:
-          name: Build project
-          command: make
+      - when:
+          condition: << parameters.blst >>
+          steps:
+            - run:
+                name: Build project (blst)
+                command: FFI_USE_BLST=1 make
+      - unless:
+          condition: << parameters.blst >>
+          steps:
+            - run:
+                name: Build project (pairing)
+                command: make
+
       - run:
           name: Build project without CGO
           command: env CGO_ENABLED=0 go build .
+
   ensure_generated_cgo_up_to_date:
     steps:
       - run:
@@ -312,15 +385,15 @@ commands:
           steps:
             - run:
                 name: Run leak detector
-                command: LD_LIBRARY_PATH="/tmp/__fil-hwloc/lib" make cgo-leakdetect
+                command: make cgo-leakdetect
                 no_output_timeout: 60m
       - run:
           name: Run the Rust tests
-          command: cd rust && FIL_PROOFS_PARAMETER_CACHE="${HOME}/filecoin-proof-parameters/" RUSTFLAGS="-L/tmp/__fil-hwloc/lib -lhwloc" RUST_LOG=info LD_LIBRARY_PATH="/tmp/__fil-hwloc/lib" cargo test --all --release && cd ..
+          command: cd rust && FIL_PROOFS_PARAMETER_CACHE="${HOME}/filecoin-proof-parameters/" RUST_LOG=info cargo test --all --release && cd ..
           no_output_timeout: 60m
       - run:
           name: Run the Go tests
-          command: CGO_LDFLAGS="-L/tmp/__fil-hwloc/lib -lhwloc" LD_LIBRARY_PATH="/tmp/__fil-hwloc/lib" GODEBUG=cgocheck=2 RUST_LOG=info go test -p 1 -timeout 60m
+          command: GODEBUG=cgocheck=2 RUST_LOG=info go test -p 1 -timeout 60m
           no_output_timeout: 60m
   compile_tests:
     steps:

diff --git a/build.sh b/build.sh
@@ -9,6 +9,6 @@ cargo update -p "filecoin-proofs-api"
 cargo install cbindgen
 cbindgen --clean --config cbindgen.toml --crate filcrypto --output ../include/filcrypto.h
 cd ..
-FFI_BUILD_FROM_SOURCE=1 make
+FFI_BUILD_FROM_SOURCE=1 FFI_USE_BLST=1 make
 make cgo-gen
 go mod tidy
diff --git a/filcrypto.yml b/filcrypto.yml
@@ -8,8 +8,8 @@ GENERATOR:
   Includes:
     - ../filcrypto.h
   FlagGroups:
-    - {name: LDFLAGS, flags: ["-L${SRCDIR}/.. -lfilcrypto -L/tmp/__fil-hwloc/lib -lhwloc"]}
-    - {name: pkg-config, flags: ["${SRCDIR}/../filcrypto.pc --static"]}
+    - {name: LDFLAGS, flags: ["-L${SRCDIR}/.."]}
+    - {name: pkg-config, flags: ["${SRCDIR}/../filcrypto.pc"]}
 
 PARSER:
   Defines:

diff --git a/generated/cgo_helpers.go b/generated/cgo_helpers.go
diff --git a/generated/const.go b/generated/const.go
diff --git a/generated/generated.go b/generated/generated.go
diff --git a/generated/types.go b/generated/types.go
diff --git a/install-filcrypto b/install-filcrypto
@@ -76,12 +76,12 @@ download_release_tarball() {
     local __release_tag="${__release_sha1:0:16}"
     local __release_tag_url="https://api.github.com/repos/filecoin-project/${__repo_name}/releases/tags/${__release_tag}"
 
-    # FIXME: Disable optimized release download, as it can't support properly support all native cpu flag detections
-    #if [ -z $release_flags ]; then
+    # Download either the non-optimized standard or standard-blst release.
+    if [ "${FFI_USE_BLST}" == "1" ]; then
+        release_flag_name="standard-blst"
+    else
         release_flag_name="standard"
-    #else
-    #    release_flag_name="optimized"
-    #fi
+    fi
 
     # TODO: This function shouldn't make assumptions about how these releases'
     # names are constructed. Marginally less-bad would be to require that this
@@ -131,23 +131,6 @@ build_from_source() {
     local __repo_sha1=$(git rev-parse HEAD)
     local __repo_sha1_truncated="${__repo_sha1:0:16}"
 
-    if [ "$(uname)" = "Linux" ]; then
-        # Install hwloc2 on Linux dependency from source.
-        HWLOC_PACKAGE="hwloc-2.3.0"
-        HWLOC_DOWNLOAD_URL="https://download.open-mpi.org/release/hwloc/v2.3/hwloc-2.3.0.tar.gz"
-        HWLOC_INSTALL_DIR="/tmp/__fil-hwloc"
-        rm -rf ${HWLOC_INSTALL_DIR}
-
-        curl ${HWLOC_DOWNLOAD_URL} --location --output /tmp/${HWLOC_PACKAGE}.tar.gz
-        pushd /tmp/
-        tar -xf ${HWLOC_PACKAGE}.tar.gz
-        rm -f /tmp/${HWLOC_PACKAGE}.tar.gz
-        cd ${HWLOC_PACKAGE}
-        ./configure --prefix=${HWLOC_INSTALL_DIR} && make && make install
-        popd
-        rm -rf /tmp/${HWLOC_PACKAGE}
-    fi
-
     (>&2 echo "building from source @ ${__repo_sha1_truncated}")
 
     if ! [ -x "$(command -v cargo)" ]; then
@@ -166,10 +149,17 @@ build_from_source() {
 
     cargo --version
 
+    # Add feature specific rust flags as needed here.
+    if [ "${FFI_USE_BLST}" == "1" ]; then
+        additional_flags="--no-default-features --features blst"
+    else
+        additional_flags="--no-default-features --features pairing"
+    fi
+
     if [ -n "${__release_flags}" ]; then
-        RUSTFLAGS="-C target-feature=${__release_flags}" ./scripts/build-release.sh "${__library_name}" "$(cat rust-toolchain)"
+        RUSTFLAGS="-C target-feature=${__release_flags}" ./scripts/build-release.sh "${__library_name}" "$(cat rust-toolchain)" "${additional_flags}"
     else
-        ./scripts/build-release.sh "${__library_name}" "$(cat rust-toolchain)"
+        ./scripts/build-release.sh "${__library_name}" "$(cat rust-toolchain)" "${additional_flags}"
     fi
 
     popd

diff --git a/run_tests.sh b/run_tests.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 
-RUST_LOG=info go test --ldflags '-extldflags "-L/tmp/__fil-hwloc/lib -lhwloc"' -count=1 ./... && cd rust && cargo test --release --all && cd ..
+RUST_LOG=info go test -count=1 ./... && cd rust && cargo test --release --all && cd ..