refactor(ffi): consolidate bls and proofs FFI + CGO

Merge go-sectorbuilder, go-bls-sigs, rust-fil-proofs-ffi, and the FFI portion of bls-signatures into a single project which produces a single header file and static lib.
filecoin-project · Nov 27, 2019 · 3de4c15 · 3de4c15
1 parent a60c7f7
commit 3de4c15
Show file tree

Hide file tree

Showing 33 changed files with 2,157 additions and 240 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,137 +1,91 @@
 version: 2.1
+
 jobs:
-  obtain_parameters:
+  build_and_test_linux_cgo_bindings:
     docker:
-      - image: filecoin/rust:latest
-    working_directory: /mnt/crate
-    resource_class: 2xlarge+
+      - image: circleci/golang:1.12.1-stretch
+    working_directory: ~/go/src/github.com/filecoin-project/filecoin-ffi
+    resource_class: 2xlarge
     steps:
-      - configure_env
-      - install_rust
+      - configure_environment_variables
       - run:
-          name: Install paramcache from head of rust-fil-proofs master branch
+          name: Install Rust toolchain
           command: |
-            cargo install filecoin-proofs --bin=paramcache --force --git=https://github.com/filecoin-project/rust-fil-proofs.git --branch=master
-            which paramcache || { printf '%s\n' "missing paramcache binary" >&2; exit 1; }
-      - restore_parameter_cache
+            (sudo apt-get update && sudo apt-get install -y ocl-icd-opencl-dev clang libssl-dev && which cargo && which rustc) || (curl https://sh.rustup.rs -sSf | sh -s -- -y)
+            rustc --version
       - run:
-          name: Generate Groth parameters and verifying keys
-          command: paramcache
-          no_output_timeout: 1h
-      - save_parameter_cache
-  cargo_fetch:
-    docker:
-      - image: filecoin/rust:latest
-    working_directory: /mnt/crate
-    resource_class: xlarge
-    steps:
-      - configure_env
+          name: Install jq
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y jq
+            jq --version
       - checkout
-      - restore_cache:
-          keys:
-            - cargo-v0-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
-      - run: rustup install $(cat rust-toolchain)
-      - run: rustup default $(cat rust-toolchain)
-      - run: rustup component add rustfmt-preview
-      - run: rustup component add clippy-preview
-      - run: cargo fetch
-      - run: rustc +stable --version
-      - run: rustc +$(cat rust-toolchain) --version
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - Cargo.lock
-      - save_cache:
-          key: cargo-v0-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
-          paths:
-            - /root/.cargo
-            - /root/.rustup
-
-  test:
-    docker:
-      - image: filecoin/rust:latest
-    working_directory: /mnt/crate
-    resource_class: 2xlarge+
+      - build_project
+      - restore_parameter_cache
+      - obtain_filecoin_parameters
+      - save_parameter_cache
+      - build_and_run_tests
+  build_darwin_cgo_bindings:
+    macos:
+      xcode: "10.0.0"
+    working_directory: ~/go/src/github.com/filecoin-project/filecoin-ffi
+    resource_class: large
     steps:
-      - configure_env
-      - checkout
-      - attach_workspace:
-          at: "."
-      - restore_cache:
-          keys:
-            - cargo-v0-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
-      - restore_cache:
-          keys:
-            - parameter-cache-{{ .Revision }}
+      - configure_environment_variables
       - run:
-          name: Test (nightly)
-          command: cargo test --release --verbose --all
-          no_output_timeout: 15m
-
-  rustfmt:
-    docker:
-      - image: filecoin/rust:latest
-    working_directory: /mnt/crate
-    resource_class: xlarge
-    steps:
-      - configure_env
-      - checkout
-      - attach_workspace:
-          at: "."
-      - restore_cache:
-          keys:
-            - cargo-v0-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+          name: Install go
+          command: |
+            curl https://dl.google.com/go/go1.12.1.darwin-amd64.pkg -o /tmp/go.pkg && \
+            sudo installer -pkg /tmp/go.pkg -target /
+            go version
       - run:
-          name: Run cargo fmt
-          command: cargo fmt --all -- --check
-
-  clippy:
-    docker:
-      - image: filecoin/rust:latest
-    working_directory: /mnt/crate
-    resource_class: xlarge
-    steps:
-      - configure_env
-      - checkout
-      - attach_workspace:
-          at: "."
-      - restore_cache:
-          keys:
-            - cargo-v0-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+          name: Install pkg-config and md5sum
+          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config md5sha1sum
       - run:
-          name: Run cargo clippy
-          command: cargo clippy --all
-
-  build_linux_release:
+          name: Install Rust toolchain
+          command: |
+            curl https://sh.rustup.rs -sSf | sh -s -- -y
+            rustc --version
+      - run:
+          name: Install jq
+          command: |
+            HOMEBREW_NO_AUTO_UPDATE=1 brew install jq
+            jq --version
+      - checkout
+      - build_project
+      - restore_parameter_cache
+      - obtain_filecoin_parameters
+      - save_parameter_cache
+      - build_and_compile_tests
+  publish_linux_staticlib:
     docker:
       - image: filecoin/rust:latest
     working_directory: /mnt/crate
-    resource_class: xlarge
+    resource_class: 2xlarge
     steps:
-      - configure_env
+      - configure_environment_variables
       - checkout
       - attach_workspace:
           at: "."
       - restore_cache:
           keys:
-            - cargo-v0-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+            - cargo-v0-{{ checksum "rust/rust-toolchain" }}-{{ checksum "rust/Cargo.toml" }}-{{ checksum "rust/Cargo.lock" }}-{{ arch }}
       - run:
           name: Install jq
           command: apt-get install jq -yqq
       - run:
-          name: Build the release
-          command: ./scripts/build-release.sh $(cat ./rust-toolchain) --verbose --frozen --all
+          name: Build the Rust release
+          command: cd rust && ./scripts/build-release.sh filecoin $(cat ./rust-toolchain) --verbose --frozen --all
       - run:
           name: Publish release to GitHub
-          command: bash ./scripts/publish-release.sh
-
-  build_darwin_release:
+          command: cd rust && bash ./scripts/publish-release.sh
+  publish_darwin_staticlib:
     macos:
       xcode: "10.0.0"
     working_directory: ~/crate
     resource_class: large
     steps:
-      - configure_env
+      - configure_environment_variables
       - run:
           name: Configure environment variables
           command: |
@@ -142,9 +96,9 @@ jobs:
           name: Install Rust
           command: |
             curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - run: rustup install $(cat rust-toolchain)
-      - run: rustup default $(cat rust-toolchain)
-      - run: cargo fetch
+      - run: cd rust && rustup install $(cat rust-toolchain)
+      - run: cd rust && rustup default $(cat rust-toolchain)
+      - run: cd rust && cargo fetch
       - run:
           name: Install jq
           command: |
@@ -153,64 +107,98 @@ jobs:
             chmod +x $HOME/.bin/jq
       - run:
           name: Build the release
-          command: ./scripts/build-release.sh $(cat ./rust-toolchain) --verbose --frozen --all
+          command: cd rust && ./scripts/build-release.sh filecoin $(cat ./rust-toolchain) --verbose --frozen --all
       - run:
           name: Publish to GitHub
-          command: bash ./scripts/publish-release.sh
+          command: cd rust && bash ./scripts/publish-release.sh
+  cargo_fetch:
+    docker:
+      - image: filecoin/rust:latest
+    working_directory: /mnt/crate
+    resource_class: 2xlarge
+    steps:
+      - configure_environment_variables
+      - checkout
+      - restore_cache:
+          keys:
+            - cargo-v0-{{ checksum "rust/rust-toolchain" }}-{{ checksum "rust/Cargo.toml" }}-{{ checksum "rust/Cargo.lock" }}-{{ arch }}
+      - run: cd rust && rustup install $(cat rust-toolchain)
+      - run: cd rust && rustup default $(cat rust-toolchain)
+      - run: cd rust && rustup component add rustfmt-preview
+      - run: cd rust && rustup component add clippy-preview
+      - run: cd rust && cargo fetch
+      - run: cd rust && rustc +stable --version
+      - run: cd rust && rustc +$(cat rust-toolchain) --version
+      - persist_to_workspace:
+          root: "."
+          paths:
+            - rust/Cargo.lock
+      - save_cache:
+          key: cargo-v0-{{ checksum "rust/rust-toolchain" }}-{{ checksum "rust/Cargo.toml" }}-{{ checksum "rust/Cargo.lock" }}-{{ arch }}
+          paths:
+            - /root/.cargo
+            - /root/.rustup
 
 workflows:
   version: 2
   test_all:
     jobs:
-      - obtain_parameters
       - cargo_fetch
-      - rustfmt:
+      - build_and_test_linux_cgo_bindings
+      - publish_linux_staticlib:
           requires:
             - cargo_fetch
-      - clippy:
-          requires:
-            - cargo_fetch
-      - test:
-          requires:
-            - obtain_parameters
-            - cargo_fetch
-      - build_linux_release:
-          requires:
-            - cargo_fetch
-          filters:
-            branches:
-              only: master
-      - build_darwin_release:
-          requires:
-            - cargo_fetch
-          filters:
-            branches:
-              only: master
+      - build_darwin_cgo_bindings
+      - publish_darwin_staticlib
+
 commands:
-  configure_env:
+  configure_environment_variables:
     steps:
       - run:
           name: Configure environment variables
           command: |
+            echo 'export FIL_PROOFS_PARAMETER_CACHE="${HOME}/filecoin-proof-parameters/"' >> $BASH_ENV
+            echo 'export GO111MODULE=on' >> $BASH_ENV
+            echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
+            echo 'export PATH="/usr/local/go/bin:${HOME}/.cargo/bin:${PATH}:${HOME}/go/bin:${HOME}/.bin"' >> $BASH_ENV
             echo 'export RUST_LOG=info' >> $BASH_ENV
-            echo 'export FIL_PROOFS_PARAMETER_CACHE="/tmp/filecoin-parameter-cache"' >> $BASH_ENV
-            echo 'export PATH="${CARGO_HOME}/bin:${PATH}"' >> $BASH_ENV
-            source $BASH_ENV
+  obtain_filecoin_parameters:
+    steps:
+      - run:
+          name: Obtain filecoin groth parameters
+          command: ./paramcache --params-for-sector-sizes=1024
+          no_output_timeout: 30m
+  build_project:
+    steps:
+      - run:
+          name: Build project
+          command: make
+      - run:
+          name: Ensure paramcache is installed to project root
+          command: |
+            test -f ./paramcache \
+              || (rustup run --install nightly cargo install filecoin-proofs --force --git=https://github.com/filecoin-project/rust-fil-proofs.git --branch=bugs/v17-manifest --bin=paramcache --root=./ \
+                && mv ./bin/paramcache ./paramcache)
+          no_output_timeout: 60m
+  build_and_run_tests:
+    steps:
+      - run:
+          name: Test project
+          command: RUST_LOG=info go test -p 1 -timeout 60m
+          no_output_timeout: 60m
+  build_and_compile_tests:
+    steps:
+      - run:
+          name: Build project and tests, but don't actually run the tests (used to verify that build/link works with Darwin)
+          command: RUST_LOG=info go test -run=^$
   restore_parameter_cache:
     steps:
       - restore_cache:
-          key: v17-proof-params-{{ arch }}
-          paths:
-            - /tmp/filecoin-parameter-cache
+          keys:
+            - v17-proof-params-{{ arch }}
   save_parameter_cache:
     steps:
       - save_cache:
           key: v17-proof-params-{{ arch }}
           paths:
-            - /tmp/filecoin-parameter-cache
-  install_rust:
-    steps:
-      - run:
-          name: Install Rust
-          command: |
-            curl https://sh.rustup.rs -sSf | sh -s -- -y
+            - "~/filecoin-proof-parameters/"
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,8 @@
-/target
+**/target
 **/*.rs.bk
 *.pc
-/include
+**/include
+*.h
+*.a
+.install-filecoin
+**/paramcache
diff --git a/Makefile b/Makefile
@@ -0,0 +1,16 @@
+DEPS:=filecoin.h filecoin.pc libfilecoin.a
+
+all: $(DEPS)
+.PHONY: all
+
+
+$(DEPS): .install-filecoin  ;
+
+.install-filecoin: rust
+	./install-filecoin
+	@touch $@
+
+
+clean:
+	rm -rf $(DEPS) .install-filecoin
+.PHONY: clean
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # Filecoin Proofs FFI
 
-> C bindings for filecoin-proofs.
+> C and CGO bindings for Filecoin's Rust libraries
 
 ## License