C API for Ethereum BLS signatures (#228)

* [testsuite] Rework parallel test runner to buffer beyond 65536 chars and properly wait for process exit

* [testsuite] improve error reporting

* rework openArray[byte/char] for BLS signature C API

* Prepare for optimized library and bindings

* properly link to constantine

* Compiler fixes, global sanitizers, GCC bug with --opt:size

* workaround/fix #229: don't inline field reduction in Fp2

* fix clang running out of registers with LTO

* [C API] missed length parameters for ctt_eth_bls_fast_aggregate_verify

* double-precision asm is too large for inlining, try to fix Linux and MacOS woes at #228 (comment)

* Use FORTIFY_SOURCE for testing

* Fix #230 - gcc miscompiles Fp6 mul with LTO

* disable LTO for now, PR is too long

benchmarks/bench_ec_g1_msm_bls12_381.nim

@@ -32,7 +32,7 @@ const AvailableCurves = [
 ]
 
 # const testNumPoints = [10, 100, 1000, 10000, 100000]
-const testNumPoints = [8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192,
+const testNumPoints = [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192,
                        16384, 32768, 65536, 131072, 262144]
 
 proc main() =

benchmarks/bench_ec_g1_scalar_mul.nim

@@ -44,7 +44,7 @@ proc main() =
   separator()
   staticFor i, 0, AvailableCurves.len:
     const curve = AvailableCurves[i]
-    const bits = 64 # curve.getCurveOrderBitwidth()
+    const bits = curve.getCurveOrderBitwidth()
     scalarMulUnsafeDoubleAddBench(ECP_ShortW_Prj[Fp[curve], G1], bits, MulIters)
     scalarMulUnsafeDoubleAddBench(ECP_ShortW_Jac[Fp[curve], G1], bits, MulIters)
     separator()

benchmarks/bench_ec_g2_scalar_mul.nim

@@ -45,7 +45,7 @@ proc main() =
   separator()
   staticFor i, 0, AvailableCurves.len:
     const curve = AvailableCurves[i]
-    const bits = 64 # curve.getCurveOrderBitwidth()
+    const bits = curve.getCurveOrderBitwidth()
     scalarMulUnsafeDoubleAddBench(ECP_ShortW_Prj[Fp2[curve], G2], bits, MulIters)
     scalarMulUnsafeDoubleAddBench(ECP_ShortW_Jac[Fp2[curve], G2], bits, MulIters)
     separator()

benchmarks/bench_blssig_on_bls12_381_g2.nim → benchmarks/bench_ethereum_bls_signatures.nim

@@ -9,7 +9,7 @@
 import
   # Internals
   ../constantine/[
-    blssig_pop_on_bls12381_g2,
+    ethereum_bls_signatures,
     ethereum_eip2333_bls12381_key_derivation],
   ../constantine/math/arithmetic,
   # Helpers
@@ -33,37 +33,37 @@ template bench(op: string, curve: string, iters: int, body: untyped): untyped =
 proc demoKeyGen(): tuple[seckey: SecretKey, pubkey: PublicKey] =
   # Don't do this at home, this is for benchmarking purposes
   # The RNG is NOT cryptographically secure
-  # The API for keygen is not ready in blssig_pop_on_bls12381_g2
+  # The API for keygen is not ready in ethereum_bls_signatures
   let ikm = rng.random_byte_seq(32)
   doAssert cast[ptr BigInt[255]](result.seckey.addr)[].derive_master_secretKey(ikm)
-  let ok = result.pubkey.derive_public_key(result.seckey)
+  let ok = result.pubkey.derive_pubkey(result.seckey)
   doAssert ok == cttBLS_Success
 
 proc benchDeserPubkey*(iters: int) =
   let (sk, pk) = demoKeyGen()
   var pk_comp{.noInit.}: array[48, byte]
 
   # Serialize compressed
-  let ok = pk_comp.serialize_public_key_compressed(pk)
+  let ok = pk_comp.serialize_pubkey_compressed(pk)
   doAssert ok == cttBLS_Success
 
   var pk2{.noInit.}: PublicKey
 
   bench("Pubkey deserialization (full checks)", "BLS12_381 G1", iters):
-    let status = pk2.deserialize_public_key_compressed(pk_comp)
+    let status = pk2.deserialize_pubkey_compressed(pk_comp)
 
 proc benchDeserPubkeyUnchecked*(iters: int) =
   let (sk, pk) = demoKeyGen()
   var pk_comp{.noInit.}: array[48, byte]
 
   # Serialize compressed
-  let ok = pk_comp.serialize_public_key_compressed(pk)
+  let ok = pk_comp.serialize_pubkey_compressed(pk)
   doAssert ok == cttBLS_Success
 
   var pk2{.noInit.}: PublicKey
 
   bench("Pubkey deserialization (skip checks)", "BLS12_381 G1", iters):
-    let status = pk2.deserialize_public_key_compressed_unchecked(pk_comp)
+    let status = pk2.deserialize_pubkey_compressed_unchecked(pk_comp)
 
 proc benchDeserSig*(iters: int) =
   let (sk, pk) = demoKeyGen()
@@ -139,7 +139,7 @@ proc benchFastAggregateVerify*(numKeys, iters: int) =
     let status = sigs[i].sign(sk, msg)
     doAssert status == cttBLS_Success
 
-  aggSig.aggregate_signatures(sigs)
+  aggSig.aggregate_signatures_unstable_api(sigs)
 
   bench("BLS agg verif of 1 msg by " & $numKeys & " pubkeys", "BLS12_381", iters):
     let valid = validators.fast_aggregate_verify(msg, aggSig)

bindings/gen_bindings.nim → bindings_generators/gen_bindings.nim

@@ -18,8 +18,11 @@ export curves, curves_primitives
 # This files provides template for C bindings generation
 
 template genBindingsField*(Field: untyped) =
-  {.push cdecl, dynlib, exportc,  raises: [].} # No exceptions allowed
-
+  when appType == "lib":
+    {.push cdecl, dynlib, exportc,  raises: [].} # No exceptions allowed
+  else:
+    {.push cdecl, exportc,  raises: [].} # No exceptions allowed
+
   func `ctt _ Field _ unmarshalBE`(dst: var Field, src: openarray[byte]) =
     ## Deserialize
     unmarshalBE(dst, src)
@@ -77,7 +80,7 @@ template genBindingsField*(Field: untyped) =
 
   func `ctt _ Field _ mul_in_place`(a: var Field, b: Field) =
     a *= b
-  
+
   func `ctt _ Field _ square`(r: var Field, a: Field) =
     r.square(a)
 
@@ -86,10 +89,10 @@ template genBindingsField*(Field: untyped) =
   # --------------------------------------------------------------------------------------
   func `ctt _ Field _ div2`(a: var Field) =
     a.div2()
-  
+
   func `ctt _ Field _ inv`(r: var Field, a: Field) =
     r.inv(a)
-  
+
   func `ctt _ Field _ inv_in_place`(a: var Field) =
     a.inv()
   # --------------------------------------------------------------------------------------
@@ -98,10 +101,10 @@ template genBindingsField*(Field: untyped) =
 
   func `ctt _ Field _ cswap`(a, b: var Field, ctl: SecretBool) =
     a.cswap(b, ctl)
-  
+
   func `ctt _ Field _ cset_zero`(a: var Field, ctl: SecretBool) =
     a.csetZero(ctl)
-  
+
   func `ctt _ Field _ cset_one`(a: var Field, ctl: SecretBool) =
     a.csetOne(ctl)
 
@@ -118,7 +121,10 @@ template genBindingsField*(Field: untyped) =
 
 
 template genBindingsFieldSqrt*(Field: untyped) =
-  {.push cdecl, dynlib, exportc,  raises: [].} # No exceptions allowed
+  when appType == "lib":
+    {.push cdecl, dynlib, exportc,  raises: [].} # No exceptions allowed
+  else:
+    {.push cdecl, exportc,  raises: [].} # No exceptions allowed
 
   func `ctt _ Field _ is_square`(a: Field): SecretBool =
     a.isSquare()
@@ -148,7 +154,10 @@ template genBindingsFieldSqrt*(Field: untyped) =
 
 
 template genBindingsExtField*(Field: untyped) =
-  {.push cdecl, dynlib, exportc,  raises: [].} # No exceptions allowed
+  when appType == "lib":
+    {.push cdecl, dynlib, exportc,  raises: [].} # No exceptions allowed
+  else:
+    {.push cdecl, exportc,  raises: [].} # No exceptions allowed
 
   # --------------------------------------------------------------------------------------
   func `ctt _ Field _ is_eq`(a, b: Field): SecretBool =
@@ -195,13 +204,13 @@ template genBindingsExtField*(Field: untyped) =
 
   func `ctt _ Field _ conj`(r: var Field, a: Field) =
     r.conj(a)
-  
+
   func `ctt _ Field _ conj_in_place`(a: var Field) =
     a.conj()
 
   func `ctt _ Field _ conjneg`(r: var Field, a: Field) =
     r.conjneg(a)
-  
+
   func `ctt _ Field _ conjneg_in_place`(a: var Field) =
     a.conjneg()
 
@@ -211,7 +220,7 @@ template genBindingsExtField*(Field: untyped) =
 
   func `ctt _ Field _ mul_in_place`(a: var Field, b: Field) =
     a *= b
-  
+
   func `ctt _ Field _ square`(r: var Field, a: Field) =
     r.square(a)
 
@@ -220,10 +229,10 @@ template genBindingsExtField*(Field: untyped) =
   # --------------------------------------------------------------------------------------
   func `ctt _ Field _ div2`(a: var Field) =
     a.div2()
-  
+
   func `ctt _ Field _ inv`(r: var Field, a: Field) =
     r.inv(a)
-  
+
   func `ctt _ Field _ inv_in_place`(a: var Field) =
     a.inv()
   # --------------------------------------------------------------------------------------
@@ -232,7 +241,7 @@ template genBindingsExtField*(Field: untyped) =
 
   func `ctt _ Field _ cset_zero`(a: var Field, ctl: SecretBool) =
     a.csetZero(ctl)
-  
+
   func `ctt _ Field _ cset_one`(a: var Field, ctl: SecretBool) =
     a.csetOne(ctl)
 
@@ -248,7 +257,10 @@ template genBindingsExtField*(Field: untyped) =
   {.pop.}
 
 template genBindingsExtFieldSqrt*(Field: untyped) =
-  {.push cdecl, dynlib, exportc,  raises: [].} # No exceptions allowed
+  when appType == "lib":
+    {.push cdecl, dynlib, exportc,  raises: [].} # No exceptions allowed
+  else:
+    {.push cdecl, exportc,  raises: [].} # No exceptions allowed
 
   func `ctt _ Field _ is_square`(a: Field): SecretBool =
     a.isSquare()
@@ -262,12 +274,15 @@ template genBindingsExtFieldSqrt*(Field: untyped) =
   {.pop}
 
 template genBindings_EC_ShortW_Affine*(ECP, Field: untyped) =
-  {.push cdecl, dynlib, exportc,  raises: [].} # No exceptions allowed
+  when appType == "lib":
+    {.push cdecl, dynlib, exportc,  raises: [].} # No exceptions allowed
+  else:
+    {.push cdecl, exportc,  raises: [].} # No exceptions allowed
 
   # --------------------------------------------------------------------------------------
   func `ctt _ ECP _ is_eq`(P, Q: ECP): SecretBool =
     P == Q
-  
+
   func `ctt _ ECP _ is_inf`(P: ECP): SecretBool =
     P.isInf()
 
@@ -276,7 +291,7 @@ template genBindings_EC_ShortW_Affine*(ECP, Field: untyped) =
 
   func `ctt _ ECP _ ccopy`(P: var ECP, Q: ECP, ctl: SecretBool) =
     P.ccopy(Q, ctl)
-  
+
   func `ctt _ ECP _ is_on_curve`(x, y: Field): SecretBool =
     isOnCurve(x, y, ECP.G)
 
@@ -289,12 +304,15 @@ template genBindings_EC_ShortW_Affine*(ECP, Field: untyped) =
   {.pop.}
 
 template genBindings_EC_ShortW_NonAffine*(ECP, ECP_Aff, Field: untyped) =
-  {.push cdecl, dynlib, exportc,  raises: [].} # No exceptions allowed
+  when appType == "lib":
+    {.push cdecl, dynlib, exportc,  raises: [].} # No exceptions allowed
+  else:
+    {.push cdecl, exportc,  raises: [].} # No exceptions allowed
 
   # --------------------------------------------------------------------------------------
   func `ctt _ ECP _ is_eq`(P, Q: ECP): SecretBool =
     P == Q
-  
+
   func `ctt _ ECP _ is_inf`(P: ECP): SecretBool =
     P.isInf()
 
@@ -303,7 +321,7 @@ template genBindings_EC_ShortW_NonAffine*(ECP, ECP_Aff, Field: untyped) =
 
   func `ctt _ ECP _ ccopy`(P: var ECP, Q: ECP, ctl: SecretBool) =
     P.ccopy(Q, ctl)
-  
+
   func `ctt _ ECP _ neg`(P: var ECP, Q: ECP) =
     P.neg(Q)
 
@@ -327,7 +345,7 @@ template genBindings_EC_ShortW_NonAffine*(ECP, ECP_Aff, Field: untyped) =
 
   func `ctt _ ECP _ double_in_place`(P: var ECP) =
     P.double()
-  
+
   func `ctt _ ECP _ affine`(dst: var ECP_Aff, src: ECP) =
     dst.affine(src)
 

bindings/gen_header.nim → bindings_generators/gen_header.nim

@@ -16,14 +16,13 @@ import
 
 proc genHeaderLicense*(): string =
   """
-/*
- * Constantine
- * Copyright (c) 2018-2019    Status Research & Development GmbH
- * Copyright (c) 2020-Present Mamy André-Ratsimbazafy
- * Licensed and distributed under either of
- *   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
- *   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
- * at your option. This file may not be copied, modified, or distributed except according to those terms.
+/** Constantine
+ *  Copyright (c) 2018-2019    Status Research & Development GmbH
+ *  Copyright (c) 2020-Present Mamy André-Ratsimbazafy
+ *  Licensed and distributed under either of
+ *    * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+ *    * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+ *  at your option. This file may not be copied, modified, or distributed except according to those terms.
  */
 """
 
@@ -102,7 +101,7 @@ proc declNimMain*(libName: string): string =
   ## - the Nim runtime if seqs, strings or heap-allocated types are used,
   ##   this is the case only if Constantine is multithreaded.
   ## - runtime CPU features detection
-  ## 
+  ##
   ## Assumes library is compiled with --nimMainPrefix:ctt_{libName}_
   &"""
 
@@ -124,9 +123,9 @@ proc toCrettype(node: NimNode): string =
   node.expectKind({nnkEmpty, nnkSym})
   if node.kind == nnkEmpty:
     # align iwth secret_bool and secret_word
-    "void       "  
+    "void       "
   else:
-    TypeMap[$node] 
+    TypeMap[$node]
 
 proc toCtrivialParam(name: string, typ: NimNode): string =
   typ.expectKind({nnkVarTy, nnkSym})
@@ -181,24 +180,24 @@ macro collectBindings*(cBindingsStr: untyped, body: typed): untyped =
     for fnDef in generator:
       if fnDef.kind notin {nnkProcDef, nnkFuncDef}:
         continue
-    
+
       cBindings &= "\n"
-      # rettype name(pType0* pName0, pType1* pName1, ...);    
+      # rettype name(pType0* pName0, pType1* pName1, ...);
       cBindings &= fnDef.params[0].toCrettype()
       cBindings &= ' '
       cBindings &= $fnDef.name
       cBindings &= '('
       for i in 1 ..< fnDef.params.len:
         if i != 1: cBindings &= ", "
-        
+
         let paramDef = fnDef.params[i]
         paramDef.expectKind(nnkIdentDefs)
         let pType = paramDef[^2]
         # No default value
         paramDef[^1].expectKind(nnkEmpty)
 
         for j in 0 ..< paramDef.len - 2:
-          if j != 0: cBindings &= ", " 
+          if j != 0: cBindings &= ", "
           var name = $paramDef[j]
           cBindings &= toCparam(name.split('`')[0], pType)