build: make the build script portable, explicit dependency on Go & GC…

…C, test libcompress build (#621)

Signed-off-by franklin.delehelle@consensys.net

.github/workflows/gradle.yml

@@ -13,6 +13,12 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
+      - name: Install Go
+        uses: actions/setup-go@v4
+
+      - name: Set up GCC
+        uses: egor-tensin/setup-gcc@v1
+
       - name: Checkout repository
         uses: actions/checkout@v3
 
@@ -36,6 +42,12 @@ jobs:
   acceptanceTest:
     runs-on: ubuntu-latest
     steps:
+      - name: Install Go
+        uses: actions/setup-go@v4
+
+      - name: Set up GCC
+        uses: egor-tensin/setup-gcc@v1
+
       - name: Checkout repository
         uses: actions/checkout@v3
 
@@ -54,9 +66,39 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v3
         with:
-          name: test-report
+          name: acceptance-test-report
           path: acceptance-tests/build/reports/tests/
 
+  libCompressTest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v4
+
+      - name: Set up GCC
+        uses: egor-tensin/setup-gcc@v1
+
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: '17'
+          distribution: 'adopt'
+
+      - name: Run libcompress JNI tests
+        run: ./gradlew :native:compress:test
+        env:
+          JAVA_OPTS: -Dorg.gradle.daemon=false
+
+      - name: Upload test report
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: compress-test-report
+          path: compress/build/reports/tests/
+
   tests:
     runs-on: ubuntu-latest
     steps:
@@ -65,7 +107,6 @@ jobs:
           ssh-private-key: |
             ${{ secrets.CONSTRAINTS_SSH_KEY }}
 
-
       - name: Checkout repository
         uses: actions/checkout@v3
         with:
@@ -103,8 +144,8 @@ jobs:
       - name: Upload test report
         uses: actions/upload-artifact@v3
         with:
-          name: test-report
-          path: build/reports/tests/
+          name: unit-test-report
+          path: arithmetization/build/reports/tests/
 
   spotless:
     runs-on: ubuntu-latest

README.md

@@ -29,6 +29,10 @@ an [existing implementation in Go](https://github.com/Consensys/zk-evm/).
 brew install openjdk@17
 ```
 
+### Install the relevant CGo compiler for your platform
+
+### Install the Go toolchain
+
 ### Install Rust
 
 ```

native/build.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 
 # Initialize external vars - need this to get around unbound variable errors
 SKIP_GRADLE="$SKIP_GRADLE"
@@ -9,71 +9,25 @@ set -o nounset
 # Exit script if a statement returns a non-true return value.
 set -o errexit
 
-# Use the error status of the first failure, rather than that of the last item in a pipeline.
-set -o pipefail
-
-# Resolve the directory that contains this script. We have to jump through a few
-# hoops for this because the usual one-liners for this don't work if the script
-# is a symlink
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
-  DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
-  SOURCE="$(readlink "$SOURCE")"
-  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
-
-# Determine core count for parallel make
-if [[ "$OSTYPE" == "linux-gnu" ]];  then
-  CORE_COUNT=$(nproc)
-  OSARCH=${OSTYPE%%[0-9.]*}-`arch`
-fi
-
-if [[ "$OSTYPE" == "darwin"* ]];  then
-  CORE_COUNT=$(sysctl -n hw.ncpu)
-  if [[ "`machine`" == "arm"* ]]; then
-    arch_name="aarch64"
-    export CFLAGS="-arch arm64"
-  else
-    arch_name="x86-64"
-    export CFLAGS="-arch x86_64"
-  fi
-  OSARCH="darwin-$arch_name"
+SCRIPTDIR=$(dirname -- "$( readlink -f -- "$0")")
+OSTYPE=$(uname -o)
+
+# delete old build dir, if exists
+rm -rf "$SCRIPTDIR/compress/build/native" || true
+mkdir -p "$SCRIPTDIR/compress/build/native"
+
+if [ x"$OSTYPE" = x"msys" ]; then
+  LIBRARY_EXTENSION=dll
+elif [ x"$OSTYPE" = x"GNU/Linux" ]; then
+  LIBRARY_EXTENSION=so
+elif [ x"$OSTYPE" = x"Darwin" ]; then
+  LIBRARY_EXTENSION=dylib
+else
+  echo "*** Unknown OS: $OSTYPE"
+  exit 1
 fi
 
-# add to path cargo
-[ -f $HOME/.cargo/env ] && . $HOME/.cargo/env
-
-# add to path brew
-[ -f $HOME/.zprofile ] && . $HOME/.zprofile
-
-build_compress() {
-  cat <<EOF
-  ############################
-  ####### build compress #######
-  ############################
-EOF
-
-  cd "$SCRIPTDIR/compress/compress-jni"
-
-  # delete old build dir, if exists
-  rm -rf "$SCRIPTDIR/compress/build" || true
-  mkdir -p "$SCRIPTDIR/compress/build/lib"
-
-  if [[ "$OSTYPE" == "msys" ]]; then
-    	LIBRARY_EXTENSION=dll
-  elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
-    LIBRARY_EXTENSION=so
-  elif [[ "$OSTYPE" == "darwin"* ]]; then
-    LIBRARY_EXTENSION=dylib
-  fi
-
-  go build -buildmode=c-shared -o libcompress_jni.$LIBRARY_EXTENSION compress-jni.go
-
-  mkdir -p "$SCRIPTDIR/compress/build/${OSARCH}/lib"
-  cp libcompress_jni.* "$SCRIPTDIR/compress/build/${OSARCH}/lib"
-}
-
-build_compress
-
-exit
+cd "$SCRIPTDIR/compress/compress-jni"
+echo "Building Go module libcompress_jni.$LIBRARY_EXTENSION for $OSTYPE"
+CGO_ENABLED=1 go build -buildmode=c-shared -o libcompress_jni.$LIBRARY_EXTENSION compress-jni.go
+mv libcompress_jni.* "$SCRIPTDIR/compress/build/native"

native/compress/build.gradle

@@ -29,13 +29,13 @@ test {
     useJUnitPlatform()
 }
 
-tasks.register('callBuildSh', Exec) {
+tasks.register('buildJNI', Exec) {
     workingDir buildscript.sourceFile.parentFile
-    commandLine 'bash', '../build.sh'
+    commandLine 'sh', '-c', '../build.sh'
 }
 
 compileJava{
-    dependsOn callBuildSh
+    dependsOn buildJNI
 }
 
 dependencies {
@@ -50,25 +50,25 @@ dependencies {
 
 
 tasks.register('macArmLibCopy', Copy) {
-    from 'build/darwin-aarch64/lib/libcompress_jni.dylib'
+    from 'build/native/libcompress_jni.dylib'
     into 'build/resources/main/darwin-aarch64'
 }
 processResources.dependsOn macArmLibCopy
 
 tasks.register('macLibCopy', Copy) {
-    from 'build/darwin-x86-64/lib/libcompress_jni.dylib'
+    from 'build/native/libcompress_jni.dylib'
     into 'build/resources/main/darwin-x86-64'
 }
 processResources.dependsOn macLibCopy
 
 tasks.register('linuxLibCopy', Copy) {
-    from 'build/linux-gnu-x86_64/lib/libcompress_jni.so'
+    from 'build/native/libcompress_jni.so'
     into 'build/resources/main/linux-x86-64'
 }
 processResources.dependsOn linuxLibCopy
 
 tasks.register('linuxArm64LibCopy', Copy) {
-    from 'build/linux-gnu-aarch64/lib/libcompress_jni.so'
+    from 'build/native/libcompress_jni.so'
     into 'build/resources/main/linux-aarch64'
 }
 processResources.dependsOn linuxArm64LibCopy
@@ -87,7 +87,7 @@ jar {
     }
 }
 
-jar.dependsOn(callBuildSh)
+jar.dependsOn(buildJNI)
 
 sourceSets {
     main {

native/compress/src/test/java/net/consensys/linea/compress/LibCompressTest.java

@@ -20,7 +20,6 @@
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
 
 public class LibCompressTest {
-
     @Test
     public void testCompressZeroes() {
         byte[] zeroes = new byte[128];