bench: replace wall-clock timer with per-process CPU timer

.github/workflows/ci.yml

@@ -571,7 +571,7 @@ jobs:
         # Use the bash shell included with Git for Windows.
         shell: bash
         run: |
-          cd build/bin/RelWithDebInfo && file *tests.exe bench*.exe libsecp256k1-*.dll || true
+          cd build/bin/RelWithDebInfo && file *tests.exe *bench*.exe libsecp256k1-*.dll || true
 
       - name: Symbol check
         if: ${{ matrix.configuration.symbol_check }}
@@ -584,9 +584,9 @@ jobs:
       - name: Check
         run: |
           ctest -C RelWithDebInfo --test-dir build -j ([int]$env:NUMBER_OF_PROCESSORS + 1)
-          build\bin\RelWithDebInfo\bench_ecmult.exe
-          build\bin\RelWithDebInfo\bench_internal.exe
-          build\bin\RelWithDebInfo\bench.exe
+          build\bin\RelWithDebInfo\secp256k1_bench_ecmult.exe
+          build\bin\RelWithDebInfo\secp256k1_bench_internal.exe
+          build\bin\RelWithDebInfo\secp256k1_bench.exe
 
   win64-native-headers:
     name: "x64 (MSVC): C++ (public headers)"

.gitignore

@@ -1,17 +1,17 @@
-bench
-bench_ecmult
-bench_internal
-noverify_tests
-tests
-exhaustive_tests
+secp256k1_bench
+secp256k1_bench_ecmult
+secp256k1_bench_internal
+secp256k1_noverify_tests
+secp256k1_tests
+secp256k1_exhaustive_tests
 precompute_ecmult_gen
 precompute_ecmult
-ctime_tests
-ecdh_example
-ecdsa_example
-schnorr_example
-ellswift_example
-musig_example
+secp256k1_ctime_tests
+secp256k1_ecdsa_example
+secp256k1_ecdsa_example
+secp256k1_schnorr_example
+secp256k1_ellswift_example
+secp256k1_musig_example
 *.exe
 *.so
 *.a

Makefile.am

@@ -106,109 +106,109 @@ libsecp256k1_la_LDFLAGS = -no-undefined -version-info $(LIB_VERSION_CURRENT):$(L
 
 noinst_PROGRAMS =
 if USE_BENCHMARK
-noinst_PROGRAMS += bench bench_internal bench_ecmult
-bench_SOURCES = src/bench.c
-bench_LDADD = libsecp256k1.la
-bench_CPPFLAGS = $(SECP_CONFIG_DEFINES)
-bench_internal_SOURCES = src/bench_internal.c
-bench_internal_LDADD = $(COMMON_LIB) $(PRECOMPUTED_LIB)
-bench_internal_CPPFLAGS = $(SECP_CONFIG_DEFINES)
-bench_ecmult_SOURCES = src/bench_ecmult.c
-bench_ecmult_LDADD = $(COMMON_LIB) $(PRECOMPUTED_LIB)
-bench_ecmult_CPPFLAGS = $(SECP_CONFIG_DEFINES)
+noinst_PROGRAMS += secp256k1_bench secp256k1_bench_internal secp256k1_bench_ecmult
+secp256k1_bench_SOURCES = src/bench.c
+secp256k1_bench_LDADD = libsecp256k1.la
+secp256k1_bench_CPPFLAGS = $(SECP_CONFIG_DEFINES)
+secp256k1_bench_internal_SOURCES = src/bench_internal.c
+secp256k1_bench_internal_LDADD = $(COMMON_LIB) $(PRECOMPUTED_LIB)
+secp256k1_bench_internal_CPPFLAGS = $(SECP_CONFIG_DEFINES)
+secp256k1_bench_ecmult_SOURCES = src/bench_ecmult.c
+secp256k1_bench_ecmult_LDADD = $(COMMON_LIB) $(PRECOMPUTED_LIB)
+secp256k1_bench_ecmult_CPPFLAGS = $(SECP_CONFIG_DEFINES)
 endif
 
 TESTS =
 if USE_TESTS
-TESTS += noverify_tests
-noinst_PROGRAMS += noverify_tests
-noverify_tests_SOURCES = src/tests.c
-noverify_tests_CPPFLAGS = $(SECP_CONFIG_DEFINES) $(TEST_DEFINES)
-noverify_tests_LDADD = $(COMMON_LIB) $(PRECOMPUTED_LIB)
-noverify_tests_LDFLAGS = -static
+TESTS += secp256k1_noverify_tests
+noinst_PROGRAMS += secp256k1_noverify_tests
+secp256k1_noverify_tests_SOURCES = src/tests.c
+secp256k1_noverify_tests_CPPFLAGS = $(SECP_CONFIG_DEFINES) $(TEST_DEFINES)
+secp256k1_noverify_tests_LDADD = $(COMMON_LIB) $(PRECOMPUTED_LIB)
+secp256k1_noverify_tests_LDFLAGS = -static
 if !ENABLE_COVERAGE
-TESTS += tests
-noinst_PROGRAMS += tests
-tests_SOURCES  = $(noverify_tests_SOURCES)
-tests_CPPFLAGS = $(noverify_tests_CPPFLAGS) -DVERIFY
-tests_LDADD    = $(noverify_tests_LDADD)
-tests_LDFLAGS  = $(noverify_tests_LDFLAGS)
+TESTS += secp256k1_tests
+noinst_PROGRAMS += secp256k1_tests
+secp256k1_tests_SOURCES  = $(secp256k1_noverify_tests_SOURCES)
+secp256k1_tests_CPPFLAGS = $(secp256k1_noverify_tests_CPPFLAGS) -DVERIFY
+secp256k1_tests_LDADD    = $(secp256k1_noverify_tests_LDADD)
+secp256k1_tests_LDFLAGS  = $(secp256k1_noverify_tests_LDFLAGS)
 endif
 endif
 
 if USE_CTIME_TESTS
-noinst_PROGRAMS += ctime_tests
-ctime_tests_SOURCES = src/ctime_tests.c
-ctime_tests_LDADD = libsecp256k1.la
-ctime_tests_CPPFLAGS = $(SECP_CONFIG_DEFINES)
+noinst_PROGRAMS += secp256k1_ctime_tests
+secp256k1_ctime_tests_SOURCES = src/ctime_tests.c
+secp256k1_ctime_tests_LDADD = libsecp256k1.la
+secp256k1_ctime_tests_CPPFLAGS = $(SECP_CONFIG_DEFINES)
 endif
 
 if USE_EXHAUSTIVE_TESTS
-noinst_PROGRAMS += exhaustive_tests
-exhaustive_tests_SOURCES = src/tests_exhaustive.c
-exhaustive_tests_CPPFLAGS = $(SECP_CONFIG_DEFINES)
+noinst_PROGRAMS += secp256k1_exhaustive_tests
+secp256k1_exhaustive_tests_SOURCES = src/tests_exhaustive.c
+secp256k1_exhaustive_tests_CPPFLAGS = $(SECP_CONFIG_DEFINES)
 if !ENABLE_COVERAGE
-exhaustive_tests_CPPFLAGS += -DVERIFY
+secp256k1_exhaustive_tests_CPPFLAGS += -DVERIFY
 endif
 # Note: do not include $(PRECOMPUTED_LIB) in exhaustive_tests (it uses runtime-generated tables).
-exhaustive_tests_LDADD = $(COMMON_LIB)
-exhaustive_tests_LDFLAGS = -static
-TESTS += exhaustive_tests
+secp256k1_exhaustive_tests_LDADD = $(COMMON_LIB)
+secp256k1_exhaustive_tests_LDFLAGS = -static
+TESTS += secp256k1_exhaustive_tests
 endif
 
 if USE_EXAMPLES
-noinst_PROGRAMS += ecdsa_example
-ecdsa_example_SOURCES = examples/ecdsa.c
-ecdsa_example_CPPFLAGS = -I$(top_srcdir)/include -DSECP256K1_STATIC
-ecdsa_example_LDADD = libsecp256k1.la
-ecdsa_example_LDFLAGS = -static
+noinst_PROGRAMS += secp256k1_ecdsa_example
+secp256k1_ecdsa_example_SOURCES = examples/ecdsa.c
+secp256k1_ecdsa_example_CPPFLAGS = -I$(top_srcdir)/include -DSECP256K1_STATIC
+secp256k1_ecdsa_example_LDADD = libsecp256k1.la
+secp256k1_ecdsa_example_LDFLAGS = -static
 if BUILD_WINDOWS
-ecdsa_example_LDFLAGS += -lbcrypt
+secp256k1_ecdsa_example_LDFLAGS += -lbcrypt
 endif
-TESTS += ecdsa_example
+TESTS += secp256k1_ecdsa_example
 if ENABLE_MODULE_ECDH
-noinst_PROGRAMS += ecdh_example
-ecdh_example_SOURCES = examples/ecdh.c
-ecdh_example_CPPFLAGS = -I$(top_srcdir)/include -DSECP256K1_STATIC
-ecdh_example_LDADD = libsecp256k1.la
-ecdh_example_LDFLAGS = -static
+noinst_PROGRAMS += secp256k1_ecdh_example
+secp256k1_ecdh_example_SOURCES = examples/ecdh.c
+secp256k1_ecdh_example_CPPFLAGS = -I$(top_srcdir)/include -DSECP256K1_STATIC
+secp256k1_ecdh_example_LDADD = libsecp256k1.la
+secp256k1_ecdh_example_LDFLAGS = -static
 if BUILD_WINDOWS
-ecdh_example_LDFLAGS += -lbcrypt
+secp256k1_ecdh_example_LDFLAGS += -lbcrypt
 endif
-TESTS += ecdh_example
+TESTS += secp256k1_ecdh_example
 endif
 if ENABLE_MODULE_SCHNORRSIG
-noinst_PROGRAMS += schnorr_example
-schnorr_example_SOURCES = examples/schnorr.c
-schnorr_example_CPPFLAGS = -I$(top_srcdir)/include -DSECP256K1_STATIC
-schnorr_example_LDADD = libsecp256k1.la
-schnorr_example_LDFLAGS = -static
+noinst_PROGRAMS += secp256k1_schnorr_example
+secp256k1_schnorr_example_SOURCES = examples/schnorr.c
+secp256k1_schnorr_example_CPPFLAGS = -I$(top_srcdir)/include -DSECP256K1_STATIC
+secp256k1_schnorr_example_LDADD = libsecp256k1.la
+secp256k1_schnorr_example_LDFLAGS = -static
 if BUILD_WINDOWS
-schnorr_example_LDFLAGS += -lbcrypt
+secp256k1_schnorr_example_LDFLAGS += -lbcrypt
 endif
-TESTS += schnorr_example
+TESTS += secp256k1_schnorr_example
 endif
 if ENABLE_MODULE_ELLSWIFT
-noinst_PROGRAMS += ellswift_example
-ellswift_example_SOURCES = examples/ellswift.c
-ellswift_example_CPPFLAGS = -I$(top_srcdir)/include -DSECP256K1_STATIC
-ellswift_example_LDADD = libsecp256k1.la
-ellswift_example_LDFLAGS = -static
+noinst_PROGRAMS += secp256k1_ellswift_example
+secp256k1_ellswift_example_SOURCES = examples/ellswift.c
+secp256k1_ellswift_example_CPPFLAGS = -I$(top_srcdir)/include -DSECP256K1_STATIC
+secp256k1_ellswift_example_LDADD = libsecp256k1.la
+secp256k1_ellswift_example_LDFLAGS = -static
 if BUILD_WINDOWS
-ellswift_example_LDFLAGS += -lbcrypt
+secp256k1_ellswift_example_LDFLAGS += -lbcrypt
 endif
-TESTS += ellswift_example
+TESTS += secp256k1_ellswift_example
 endif
 if ENABLE_MODULE_MUSIG
-noinst_PROGRAMS += musig_example
-musig_example_SOURCES = examples/musig.c
-musig_example_CPPFLAGS = -I$(top_srcdir)/include -DSECP256K1_STATIC
-musig_example_LDADD = libsecp256k1.la
-musig_example_LDFLAGS = -static
+noinst_PROGRAMS += secp256k1_musig_example
+secp256k1_musig_example_SOURCES = examples/musig.c
+secp256k1_musig_example_CPPFLAGS = -I$(top_srcdir)/include -DSECP256K1_STATIC
+secp256k1_musig_example_LDADD = libsecp256k1.la
+secp256k1_musig_example_LDFLAGS = -static
 if BUILD_WINDOWS
-musig_example_LDFLAGS += -lbcrypt
+secp256k1_musig_example_LDFLAGS += -lbcrypt
 endif
-TESTS += musig_example
+TESTS += secp256k1_musig_example
 endif
 endif
 

README.md

@@ -157,6 +157,8 @@ Benchmark
 ------------
 If configured with `--enable-benchmark` (which is the default), binaries for benchmarking the libsecp256k1 functions will be present in the root directory after the build.
 
+For reliable benchmarks, it is strongly recommended to pin the process to a single CPU core and to disable CPU frequency scaling.
+
 To print the benchmark result to the command line:
 
     $ ./bench_name

ci/ci.sh

@@ -104,7 +104,7 @@ fi
 
 # Print information about binaries so that we can see that the architecture is correct
 file *tests* || true
-file bench* || true
+file *bench* || true
 file .libs/* || true
 
 if [ "$SYMBOL_CHECK" = "yes" ]
@@ -136,18 +136,18 @@ fi
 if [ "$BENCH" = "yes" ]
 then
     {
-        $EXEC ./bench_ecmult
-        $EXEC ./bench_internal
-        $EXEC ./bench
+        $EXEC ./secp256k1_bench_ecmult
+        $EXEC ./secp256k1_bench_internal
+        $EXEC ./secp256k1_bench
     } >> bench.log 2>&1
 fi
 
 if [ "$CTIMETESTS" = "yes" ]
 then
     if [ "$WITH_VALGRIND" = "yes" ]; then
-        ./libtool --mode=execute valgrind --error-exitcode=42 ./ctime_tests > ctime_tests.log 2>&1
+        ./libtool --mode=execute valgrind --error-exitcode=42 ./secp256k1_ctime_tests > secp256k1_ctime_tests.log 2>&1
     else
-        $EXEC ./ctime_tests > ctime_tests.log 2>&1
+        $EXEC ./secp256k1_ctime_tests > secp256k1_ctime_tests.log 2>&1
     fi
 fi
 

src/CMakeLists.txt

@@ -125,12 +125,26 @@ endif()
 unset(${PROJECT_NAME}_soversion)
 
 if(SECP256K1_BUILD_BENCHMARK)
-  add_executable(bench bench.c)
-  target_link_libraries(bench secp256k1)
-  add_executable(bench_internal bench_internal.c)
-  target_link_libraries(bench_internal secp256k1_precomputed secp256k1_asm)
-  add_executable(bench_ecmult bench_ecmult.c)
-  target_link_libraries(bench_ecmult secp256k1_precomputed secp256k1_asm)
+  find_library(RT_LIBRARY rt)
+  add_library(optional_rt INTERFACE)
+  if(RT_LIBRARY)
+    target_link_libraries(optional_rt INTERFACE rt)
+  endif()
+  add_executable(secp256k1_bench bench.c)
+  add_executable(secp256k1_bench_internal bench_internal.c)
+  add_executable(secp256k1_bench_ecmult bench_ecmult.c)
+  target_link_libraries(secp256k1_bench secp256k1 optional_rt)
+  target_link_libraries(secp256k1_bench_internal secp256k1_precomputed secp256k1_asm optional_rt)
+  target_link_libraries(secp256k1_bench_ecmult secp256k1_precomputed secp256k1_asm optional_rt)
+  add_test(NAME secp256k1_bench COMMAND secp256k1_bench)
+  add_test(NAME secp256k1_bench_internal COMMAND secp256k1_bench_internal)
+  add_test(NAME secp256k1_bench_ecmult COMMAND secp256k1_bench_ecmult)
+  set_tests_properties(secp256k1_bench secp256k1_bench_internal secp256k1_bench_ecmult
+    PROPERTIES
+      PROCESSORS 1
+      PROCESSOR_AFFINITY ON
+      RUN_SERIAL ON
+  )
 endif()
 
 if(SECP256K1_BUILD_TESTS)
@@ -144,30 +158,30 @@ if(SECP256K1_BUILD_TESTS)
     list(APPEND TEST_DEFINITIONS SUPPORTS_CONCURRENCY=1)
   endif()
 
-  add_executable(noverify_tests tests.c)
-  target_link_libraries(noverify_tests secp256k1_precomputed secp256k1_asm)
-  target_compile_definitions(noverify_tests PRIVATE ${TEST_DEFINITIONS})
-  add_test(NAME secp256k1_noverify_tests COMMAND noverify_tests)
+  add_executable(secp256k1_noverify_tests tests.c)
+  target_link_libraries(secp256k1_noverify_tests secp256k1_precomputed secp256k1_asm)
+  target_compile_definitions(secp256k1_noverify_tests PRIVATE ${TEST_DEFINITIONS})
+  add_test(NAME secp256k1_noverify_tests COMMAND secp256k1_noverify_tests)
   if(NOT CMAKE_BUILD_TYPE STREQUAL "Coverage")
-    add_executable(tests tests.c)
-    target_compile_definitions(tests PRIVATE VERIFY ${TEST_DEFINITIONS})
-    target_link_libraries(tests secp256k1_precomputed secp256k1_asm)
-    add_test(NAME secp256k1_tests COMMAND tests)
+    add_executable(secp256k1_tests tests.c)
+    target_compile_definitions(secp256k1_tests PRIVATE VERIFY ${TEST_DEFINITIONS})
+    target_link_libraries(secp256k1_tests secp256k1_precomputed secp256k1_asm)
+    add_test(NAME secp256k1_tests COMMAND secp256k1_tests)
   endif()
   unset(TEST_DEFINITIONS)
 endif()
 
 if(SECP256K1_BUILD_EXHAUSTIVE_TESTS)
   # Note: do not include secp256k1_precomputed in exhaustive_tests (it uses runtime-generated tables).
-  add_executable(exhaustive_tests tests_exhaustive.c)
-  target_link_libraries(exhaustive_tests secp256k1_asm)
-  target_compile_definitions(exhaustive_tests PRIVATE $<$<NOT:$<CONFIG:Coverage>>:VERIFY>)
-  add_test(NAME secp256k1_exhaustive_tests COMMAND exhaustive_tests)
+  add_executable(secp256k1_exhaustive_tests tests_exhaustive.c)
+  target_link_libraries(secp256k1_exhaustive_tests secp256k1_asm)
+  target_compile_definitions(secp256k1_exhaustive_tests PRIVATE $<$<NOT:$<CONFIG:Coverage>>:VERIFY>)
+  add_test(NAME secp256k1_exhaustive_tests COMMAND secp256k1_exhaustive_tests)
 endif()
 
 if(SECP256K1_BUILD_CTIME_TESTS)
-  add_executable(ctime_tests ctime_tests.c)
-  target_link_libraries(ctime_tests secp256k1)
+  add_executable(secp256k1_ctime_tests ctime_tests.c)
+  target_link_libraries(secp256k1_ctime_tests secp256k1)
 endif()
 
 if(SECP256K1_INSTALL)

src/bench.c

@@ -4,6 +4,8 @@
  * file COPYING or https://www.opensource.org/licenses/mit-license.php.*
  ***********************************************************************/
 
+#define _POSIX_C_SOURCE 199309L /* for clock_gettime() */
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -252,6 +254,7 @@ int main(int argc, char** argv) {
     data.pubkeylen = 33;
     CHECK(secp256k1_ec_pubkey_serialize(data.ctx, data.pubkey, &data.pubkeylen, &pubkey, SECP256K1_EC_COMPRESSED) == 1);
 
+    print_clock_info();
     print_output_table_header_row();
     if (d || have_flag(argc, argv, "ecdsa") || have_flag(argc, argv, "verify") || have_flag(argc, argv, "ecdsa_verify")) run_benchmark("ecdsa_verify", bench_verify, NULL, NULL, &data, 10, iters);
 

src/bench.h

@@ -85,9 +85,9 @@ static void run_benchmark(char *name, void (*benchmark)(void*, int), void (*setu
         if (setup != NULL) {
             setup(data);
         }
-        begin = gettime_i64();
+        begin = gettime_us();
         benchmark(data, iter);
-        total = gettime_i64() - begin;
+        total = gettime_us() - begin;
         if (teardown != NULL) {
             teardown(data, iter);
         }
@@ -156,6 +156,14 @@ static int get_iters(int default_iters) {
     }
 }
 
+static void print_clock_info(void) {
+#if defined(CLOCK_PROCESS_CPUTIME_ID)
+    printf("INFO: Using per-process CPU timer\n\n");
+#else
+    printf("WARN: Using global timer instead of per-process CPU timer.\n\n");
+#endif
+}
+
 static void print_output_table_header_row(void) {
     char* bench_str = "Benchmark";     /* left justified */
     char* min_str = "    Min(us)    "; /* center alignment */