From ae646ef579c4075b1bd95a086d339e8fda4d4103 Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Thu, 30 Mar 2023 11:59:02 +0200 Subject: [PATCH 1/6] add gnark Signed-off-by: Karim TAAM --- build.sh | 35 ++++- mimc/build.gradle | 131 ++++++++++++++++++ mimc/mimc-jni/go.mod | 10 ++ mimc/mimc-jni/go.sum | 6 + mimc/mimc-jni/mimc-jni.go | 23 +++ .../besu/nativelib/mimc/LibMimc.java | 57 ++++++++ .../besu/nativelib/mimc/LibMimcTest.java | 49 +++++++ settings.gradle | 1 + 8 files changed, 305 insertions(+), 7 deletions(-) create mode 100644 mimc/build.gradle create mode 100644 mimc/mimc-jni/go.mod create mode 100644 mimc/mimc-jni/go.sum create mode 100644 mimc/mimc-jni/mimc-jni.go create mode 100644 mimc/src/main/java/org/hyperledger/besu/nativelib/mimc/LibMimc.java create mode 100644 mimc/src/test/java/org/hyperledger/besu/nativelib/mimc/LibMimcTest.java diff --git a/build.sh b/build.sh index c6ce46fe..15b4116c 100755 --- a/build.sh +++ b/build.sh @@ -311,13 +311,34 @@ EOF } -build_blake2bf -build_secp256k1 -build_altbn128 -build_arithmetic -build_bls12_381 -build_ipa_multipoint -build_secp256r1 +build_mimc() { + cat < Date: Thu, 30 Mar 2023 17:30:47 +0200 Subject: [PATCH 2/6] add gnark Signed-off-by: Karim TAAM --- .github/workflows/build.yml | 42 +++++++++++++++++++ build.sh | 7 ++++ mimc/build.gradle | 3 +- mimc/mimc-jni/mimc-jni.go | 8 +++- .../besu/nativelib/mimc/LibMimc.java | 19 +-------- .../besu/nativelib/mimc/LibMimcTest.java | 10 +++-- 6 files changed, 65 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dfc3aed6..d5ca3df6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,11 +54,19 @@ jobs: with: name: ipa-multipoint native build artifacts path: ipa-multipoint/build/ + - uses: actions/upload-artifact@v3.1.0 + with: + name: mimc native build artifacts + path: mimc/build/ native-build-linux-arm64: runs-on: [self-hosted, Linux, ARM64] env: SKIP_GRADLE: true steps: + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: 1.19 - name: Checkout Repo uses: actions/checkout@v3 with: @@ -94,6 +102,10 @@ jobs: with: name: ipa-multipoint native build artifacts path: ipa-multipoint/build/ + - uses: actions/upload-artifact@v3.1.0 + with: + name: mimc native build artifacts + path: mimc/build/ native-build-macos: runs-on: macos-11 env: @@ -144,11 +156,28 @@ jobs: with: name: ipa-multipoint native build artifacts path: ipa-multipoint/build/ + - uses: actions/upload-artifact@v3.1.0 + with: + name: mimc native build artifacts + path: mimc/build/ native-build-m1: runs-on: [self-hosted, macOS, ARM64, MacStadium] env: SKIP_GRADLE: true steps: + - name: Prepare + run: | + # rust dependencies + export CARGO_HOME="$HOME/.cargo" + curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.62.1 + # install both x86 and arm64 toolchains + export PATH="$HOME/.cargo/bin:$PATH" + rustup target add x86_64-apple-darwin + rustup target add aarch64-apple-darwin + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: 1.19 - name: Checkout Repo uses: actions/checkout@v3 with: @@ -183,6 +212,10 @@ jobs: with: name: ipa-multipoint native build artifacts path: ipa-multipoint/build/ + - uses: actions/upload-artifact@v3.1.0 + with: + name: mimc native build artifacts + path: mimc/build/ final-assembly: runs-on: ubuntu-20.04 needs: @@ -228,6 +261,11 @@ jobs: with: name: ipa-multipoint native build artifacts path: ipa-multipoint/build/ + - name: Download mimc + uses: actions/download-artifact@v3 + with: + name: mimc native build artifacts + path: mimc/build/ - name: Set up Java uses: actions/setup-java@v2 with: @@ -266,6 +304,10 @@ jobs: with: name: jars path: ipa-multipoint/build/libs + - uses: actions/upload-artifact@v3.1.0 + with: + name: jars + path: mimc/build/libs - name: gradle publish uses: gradle/gradle-build-action@v2 if: contains('refs/heads/release-', github.ref) || github.ref == 'refs/heads/main' diff --git a/build.sh b/build.sh index 15b4116c..9faf35d6 100755 --- a/build.sh +++ b/build.sh @@ -338,6 +338,13 @@ EOF cp libmimc_jni.* "$SCRIPTDIR/mimc/build/${OSARCH}/lib" } +build_blake2bf +build_secp256k1 +build_altbn128 +build_arithmetic +build_bls12_381 +build_ipa_multipoint +build_secp256r1 build_mimc diff --git a/mimc/build.gradle b/mimc/build.gradle index bc1395e4..27fd9b28 100644 --- a/mimc/build.gradle +++ b/mimc/build.gradle @@ -21,7 +21,8 @@ plugins { dependencies { implementation 'net.java.dev.jna:jna:5.12.1' - testImplementation 'org.apache.tuweni:tuweni-bytes:2.2.0' + testImplementation 'net.java.dev.jna:jna:5.12.1' + implementation 'org.apache.tuweni:tuweni-bytes:2.2.0' testImplementation 'junit:junit:4.13.2' testImplementation 'org.assertj:assertj-core:3.22.0' testImplementation 'org.mockito:mockito-core:4.4.0' diff --git a/mimc/mimc-jni/mimc-jni.go b/mimc/mimc-jni/mimc-jni.go index 725c8e34..59aca839 100644 --- a/mimc/mimc-jni/mimc-jni.go +++ b/mimc/mimc-jni/mimc-jni.go @@ -1,5 +1,7 @@ package main + + import "C" import "unsafe" import ( @@ -14,10 +16,12 @@ func MiMCHash(b []byte) []byte { //export compute -func compute(input *C.char, inputLength C.int) *C.char { +func compute(input *C.char, inputLength C.int, output *C.char) C.int { inputSlice := C.GoBytes(unsafe.Pointer(input), inputLength) + outputSlice := (*[32]byte)(unsafe.Pointer(output))[:] hash := MiMCHash(inputSlice) - return C.CString(string(hash)) + copy(outputSlice, hash) + return C.int(len(hash)) } func main() {} \ No newline at end of file diff --git a/mimc/src/main/java/org/hyperledger/besu/nativelib/mimc/LibMimc.java b/mimc/src/main/java/org/hyperledger/besu/nativelib/mimc/LibMimc.java index c846ce82..7ff45eaf 100644 --- a/mimc/src/main/java/org/hyperledger/besu/nativelib/mimc/LibMimc.java +++ b/mimc/src/main/java/org/hyperledger/besu/nativelib/mimc/LibMimc.java @@ -15,9 +15,7 @@ */ package org.hyperledger.besu.nativelib.mimc; -import com.sun.jna.Memory; import com.sun.jna.Native; -import com.sun.jna.Pointer; /** * Java interface to mimc @@ -39,19 +37,6 @@ public class LibMimc { ENABLED = enabled; } - public static byte[] mimc(byte[] input){ - Pointer inputPointer = new Memory(input.length); - inputPointer.write(0, input, 0, input.length); - - Pointer output = compute(inputPointer, input.length); - - byte[] hash = output.getByteArray(0, 32); - - Native.free(Pointer.nativeValue(output)); - - return hash; - } - - public static native Pointer compute( - Pointer input , int i_len); + public static native int compute( + byte[] i, int i_len, byte[] o); } \ No newline at end of file diff --git a/mimc/src/test/java/org/hyperledger/besu/nativelib/mimc/LibMimcTest.java b/mimc/src/test/java/org/hyperledger/besu/nativelib/mimc/LibMimcTest.java index b3dd335e..5384da64 100644 --- a/mimc/src/test/java/org/hyperledger/besu/nativelib/mimc/LibMimcTest.java +++ b/mimc/src/test/java/org/hyperledger/besu/nativelib/mimc/LibMimcTest.java @@ -26,13 +26,15 @@ public class LibMimcTest { @Test public void testHashZero() { - byte[] output = LibMimc.mimc(Bytes32.ZERO.toArrayUnsafe()); + byte[] output = new byte[Bytes32.SIZE]; + LibMimc.compute(Bytes32.ZERO.toArrayUnsafe(), Bytes32.SIZE, output); assertThat(Bytes.wrap(output)).isEqualTo(Bytes.fromHexString("0x2c7298fd87d3039ffea208538f6b297b60b373a63792b4cd0654fdc88fd0d6ee")); } @Test public void testHashOne() { - byte[] output = LibMimc.mimc(Bytes32.leftPad(Bytes.of(1)).toArrayUnsafe()); + byte[] output = new byte[Bytes32.SIZE]; + LibMimc.compute(Bytes32.leftPad(Bytes.of(1)).toArrayUnsafe(), Bytes32.SIZE, output); assertThat(Bytes.wrap(output)).isEqualTo(Bytes.fromHexString("0x27e5458b666ef581475a9acddbc3524ca252185cae3936506e65cda9c358222b")); } @@ -42,8 +44,8 @@ public void testLongString() { for (int i = 0; i < 16; i++) { input.set(Bytes32.SIZE*(i+1)-1,(byte) i); } - byte[] output = LibMimc.mimc(input.toArrayUnsafe()); - + byte[] output = new byte[Bytes32.SIZE]; + LibMimc.compute(input.toArrayUnsafe(), input.size(), output); assertThat(Bytes.wrap(output)).isEqualTo(Bytes.fromHexString("0x145875dd085ea2fb9796333e55c9da80228eb321df0ca9a41ca64ba6fe90b167")); } } \ No newline at end of file From 890ffa11201579e0e9ffc249924afb8184d38804 Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Thu, 30 Mar 2023 17:48:23 +0200 Subject: [PATCH 3/6] fix build Signed-off-by: Karim TAAM --- .github/workflows/build.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d5ca3df6..cc17a979 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -62,11 +62,14 @@ jobs: runs-on: [self-hosted, Linux, ARM64] env: SKIP_GRADLE: true + + steps: - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: 1.19 + - name: Prepare + run: | + sudo apt update + sudo apt upgrade + sudo apt install golang-go - name: Checkout Repo uses: actions/checkout@v3 with: @@ -174,10 +177,8 @@ jobs: export PATH="$HOME/.cargo/bin:$PATH" rustup target add x86_64-apple-darwin rustup target add aarch64-apple-darwin - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: 1.19 + # install go + brew install go@1.20 - name: Checkout Repo uses: actions/checkout@v3 with: From d8ad61c86b00bbe1560b928853187ad787527350 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Thu, 30 Mar 2023 17:48:38 -0700 Subject: [PATCH 4/6] explicit download of golang 1.20.2 via wget, brew Signed-off-by: garyschulte --- .github/workflows/build.yml | 31 +++++++++++++++++++------------ build.sh | 3 ++- native-build.sh | 9 +++++++-- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc17a979..86b41d1c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,10 +16,18 @@ jobs: run: | # secp256k1 dependencies sudo apt-get update + sudo apt upgrade -y sudo apt-get install -y autoconf build-essential libtool automake patchelf + sudo apt autoremove -y + # golang dependencies + wget https://go.dev/dl/go1.20.2.linux-amd64.tar.gz + echo "4eaea32f59cde4dc635fbc42161031d13e1c780b87097f4b4234cfce671f1768 go1.20.2.linux-amd64.tar.gz" | sha256sum -c + tar -xzf go1.20.2.linux-amd64.tar.gz + export GOROOT=$(pwd)/go + export PATH=$GOROOT/bin:$PATH # rust dependencies export CARGO_HOME="$HOME/.cargo" - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.62.1 + curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.68.2 - name: Checkout Repo uses: actions/checkout@v3 with: @@ -62,14 +70,7 @@ jobs: runs-on: [self-hosted, Linux, ARM64] env: SKIP_GRADLE: true - - steps: - - name: Prepare - run: | - sudo apt update - sudo apt upgrade - sudo apt install golang-go - name: Checkout Repo uses: actions/checkout@v3 with: @@ -118,9 +119,12 @@ jobs: run: | # secp256k1 dependencies brew install autoconf automake libtool + # gnark dependencies + brew install go@1.20 || true + export PATH=/usr/local/Cellar/go/1.20.2/bin:$PATH # rust dependencies export CARGO_HOME="$HOME/.cargo" - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.62.1 + curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.68.2 # install both x86 and arm64 toolchains export PATH="$HOME/.cargo/bin:$PATH" rustup target add x86_64-apple-darwin @@ -170,15 +174,18 @@ jobs: steps: - name: Prepare run: | + # secp256k1, gnark dependencies + brew install autoconf automake libtool + # gnark dependencies + brew install go@1.20 || true + export PATH=/usr/local/Cellar/go/1.20.2/bin:$PATH # rust dependencies export CARGO_HOME="$HOME/.cargo" - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.62.1 + curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.68.2 # install both x86 and arm64 toolchains export PATH="$HOME/.cargo/bin:$PATH" rustup target add x86_64-apple-darwin rustup target add aarch64-apple-darwin - # install go - brew install go@1.20 - name: Checkout Repo uses: actions/checkout@v3 with: diff --git a/build.sh b/build.sh index 9faf35d6..81d120aa 100755 --- a/build.sh +++ b/build.sh @@ -41,12 +41,13 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then fi if [[ "$OSTYPE" == "darwin"* ]]; then - export CFLAGS="-arch x86_64 -arch arm64" CORE_COUNT=$(sysctl -n hw.ncpu) if [[ "`machine`" == "arm"* ]]; then arch_name="aarch64" + export CFLAGS="-arch arm64" else arch_name="x86-64" + export CFLAGS="-arch x86_64" fi OSARCH="darwin-$arch_name" fi diff --git a/native-build.sh b/native-build.sh index 7a96c2e9..157a4787 100755 --- a/native-build.sh +++ b/native-build.sh @@ -1,8 +1,13 @@ #!/bin/bash apt-get update -DEBIAN_FRONTEND=non-interactive apt-get install -y autoconf build-essential libtool automake patchelf curl openjdk-11-jre-headless git +DEBIAN_FRONTEND=non-interactive apt-get install -y autoconf build-essential libtool automake patchelf curl openjdk-11-jre-headless git wget +wget https://go.dev/dl/go1.20.2.linux-amd64.tar.gz +echo "78d632915bb75e9a6356a47a42625fd1a785c83a64a643fedd8f61e31b1b3bef go1.20.2.linux-arm64.tar.gz" | sha256sum -c +tar -xzf go1.20.2.linux-arm64.tar.gz -C $HOME +export GOROOT="$HOME/go" +export PATH="$GOROOT/bin":$PATH +curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.68.2 export CARGO_HOME="$HOME/.cargo" -curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.62.1 export PATH=$PATH:$CARGO_HOME/bin git config --global --add safe.directory /home/ubuntu/secp256r1/besu-native-ec git config --global --add safe.directory /home/ubuntu/secp256r1/besu-native-ec/openssl From a7d01a7622cc395509be970d77acfb2c49e30f74 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Thu, 30 Mar 2023 18:10:49 -0700 Subject: [PATCH 5/6] fail if sha check for go does not succeed Signed-off-by: garyschulte --- .github/workflows/build.yml | 2 +- native-build.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 86b41d1c..5ed93bbe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,7 +21,7 @@ jobs: sudo apt autoremove -y # golang dependencies wget https://go.dev/dl/go1.20.2.linux-amd64.tar.gz - echo "4eaea32f59cde4dc635fbc42161031d13e1c780b87097f4b4234cfce671f1768 go1.20.2.linux-amd64.tar.gz" | sha256sum -c + echo "4eaea32f59cde4dc635fbc42161031d13e1c780b87097f4b4234cfce671f1768 go1.20.2.linux-amd64.tar.gz" | sha256sum -c || exit 1 tar -xzf go1.20.2.linux-amd64.tar.gz export GOROOT=$(pwd)/go export PATH=$GOROOT/bin:$PATH diff --git a/native-build.sh b/native-build.sh index 157a4787..9dad20ae 100755 --- a/native-build.sh +++ b/native-build.sh @@ -1,13 +1,13 @@ #!/bin/bash apt-get update DEBIAN_FRONTEND=non-interactive apt-get install -y autoconf build-essential libtool automake patchelf curl openjdk-11-jre-headless git wget -wget https://go.dev/dl/go1.20.2.linux-amd64.tar.gz -echo "78d632915bb75e9a6356a47a42625fd1a785c83a64a643fedd8f61e31b1b3bef go1.20.2.linux-arm64.tar.gz" | sha256sum -c +wget https://go.dev/dl/go1.20.2.linux-arm64.tar.gz +echo "78d632915bb75e9a6356a47a42625fd1a785c83a64a643fedd8f61e31b1b3bef go1.20.2.linux-arm64.tar.gz" | sha256sum -c || exit 1 tar -xzf go1.20.2.linux-arm64.tar.gz -C $HOME export GOROOT="$HOME/go" export PATH="$GOROOT/bin":$PATH -curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.68.2 export CARGO_HOME="$HOME/.cargo" +curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.68.2 export PATH=$PATH:$CARGO_HOME/bin git config --global --add safe.directory /home/ubuntu/secp256r1/besu-native-ec git config --global --add safe.directory /home/ubuntu/secp256r1/besu-native-ec/openssl From c1dad7ef4ecac70d07565bfa5b18d7ed8b79a354 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Thu, 30 Mar 2023 18:25:47 -0700 Subject: [PATCH 6/6] set explicit GOPATH to avoid conflict with GOROOT Signed-off-by: garyschulte --- native-build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/native-build.sh b/native-build.sh index 9dad20ae..d713acd9 100755 --- a/native-build.sh +++ b/native-build.sh @@ -4,6 +4,8 @@ DEBIAN_FRONTEND=non-interactive apt-get install -y autoconf build-essential libt wget https://go.dev/dl/go1.20.2.linux-arm64.tar.gz echo "78d632915bb75e9a6356a47a42625fd1a785c83a64a643fedd8f61e31b1b3bef go1.20.2.linux-arm64.tar.gz" | sha256sum -c || exit 1 tar -xzf go1.20.2.linux-arm64.tar.gz -C $HOME +export GOPATH=$HOME/.go +mkdir -p $GOPATH export GOROOT="$HOME/go" export PATH="$GOROOT/bin":$PATH export CARGO_HOME="$HOME/.cargo"