Update Falcon & McEliece (adding AVX)

.CMake/alg_support.cmake

@@ -67,15 +67,45 @@ cmake_dependent_option(OQS_ENABLE_SIG_picnic3_L5 "" ON "OQS_ENABLE_SIG_PICNIC" O
 ##### OQS_COPY_FROM_UPSTREAM_FRAGMENT_ADD_ENABLE_BY_ALG_START
 option(OQS_ENABLE_KEM_CLASSIC_MCELIECE "" ON)
 cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_348864 "" ON "OQS_ENABLE_KEM_CLASSIC_MCELIECE" OFF)
+if(ARCH STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME MATCHES "Linux|Darwin" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_USE_POPCNT_INSTRUCTIONS)
+    cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_348864_avx "" ON "OQS_ENABLE_KEM_classic_mceliece_348864" OFF)
+endif()
 cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_348864f "" ON "OQS_ENABLE_KEM_CLASSIC_MCELIECE" OFF)
+if(ARCH STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME MATCHES "Linux|Darwin" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_USE_POPCNT_INSTRUCTIONS AND OQS_USE_BMI1_INSTRUCTIONS)
+    cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_348864f_avx "" ON "OQS_ENABLE_KEM_classic_mceliece_348864f" OFF)
+endif()
 cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_460896 "" ON "OQS_ENABLE_KEM_CLASSIC_MCELIECE" OFF)
+if(ARCH STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME MATCHES "Linux|Darwin" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_USE_POPCNT_INSTRUCTIONS)
+    cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_460896_avx "" ON "OQS_ENABLE_KEM_classic_mceliece_460896" OFF)
+endif()
 cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_460896f "" ON "OQS_ENABLE_KEM_CLASSIC_MCELIECE" OFF)
+if(ARCH STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME MATCHES "Linux|Darwin" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_USE_BMI1_INSTRUCTIONS AND OQS_USE_POPCNT_INSTRUCTIONS)
+    cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_460896f_avx "" ON "OQS_ENABLE_KEM_classic_mceliece_460896f" OFF)
+endif()
 cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_6688128 "" ON "OQS_ENABLE_KEM_CLASSIC_MCELIECE" OFF)
+if(ARCH STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME MATCHES "Linux|Darwin" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_USE_POPCNT_INSTRUCTIONS)
+    cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_6688128_avx "" ON "OQS_ENABLE_KEM_classic_mceliece_6688128" OFF)
+endif()
 cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_6688128f "" ON "OQS_ENABLE_KEM_CLASSIC_MCELIECE" OFF)
+if(ARCH STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME MATCHES "Linux|Darwin" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_USE_BMI1_INSTRUCTIONS AND OQS_USE_POPCNT_INSTRUCTIONS)
+    cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_6688128f_avx "" ON "OQS_ENABLE_KEM_classic_mceliece_6688128f" OFF)
+endif()
 cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_6960119 "" ON "OQS_ENABLE_KEM_CLASSIC_MCELIECE" OFF)
+if(ARCH STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME MATCHES "Linux|Darwin" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_USE_POPCNT_INSTRUCTIONS)
+    cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_6960119_avx "" ON "OQS_ENABLE_KEM_classic_mceliece_6960119" OFF)
+endif()
 cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_6960119f "" ON "OQS_ENABLE_KEM_CLASSIC_MCELIECE" OFF)
+if(ARCH STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME MATCHES "Linux|Darwin" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_USE_BMI1_INSTRUCTIONS AND OQS_USE_POPCNT_INSTRUCTIONS)
+    cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_6960119f_avx "" ON "OQS_ENABLE_KEM_classic_mceliece_6960119f" OFF)
+endif()
 cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_8192128 "" ON "OQS_ENABLE_KEM_CLASSIC_MCELIECE" OFF)
+if(ARCH STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME MATCHES "Linux|Darwin" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_USE_POPCNT_INSTRUCTIONS)
+    cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_8192128_avx "" ON "OQS_ENABLE_KEM_classic_mceliece_8192128" OFF)
+endif()
 cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_8192128f "" ON "OQS_ENABLE_KEM_CLASSIC_MCELIECE" OFF)
+if(ARCH STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME MATCHES "Linux|Darwin" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_USE_POPCNT_INSTRUCTIONS AND OQS_USE_BMI1_INSTRUCTIONS)
+    cmake_dependent_option(OQS_ENABLE_KEM_classic_mceliece_8192128f_avx "" ON "OQS_ENABLE_KEM_classic_mceliece_8192128f" OFF)
+endif()
 
 option(OQS_ENABLE_KEM_HQC "" ON)
 cmake_dependent_option(OQS_ENABLE_KEM_hqc_128 "" ON "OQS_ENABLE_KEM_HQC" OFF)

.CMake/compiler_opts.cmake

@@ -16,6 +16,8 @@ if(CMAKE_C_COMPILER_ID MATCHES "Clang")
     option(OQS_USE_CPU_EXTENSIONS "Enable compile and run-time support for CPU extensions such as AVX2, SSE, etc." ON)
     if(OQS_USE_CPU_EXTENSIONS)
         include(${CMAKE_CURRENT_LIST_DIR}/gcc_clang_intrinsics.cmake)
+        # to allow McEliece AVX code to compile with clang9:
+        add_compile_options(-Wno-language-extension-token)
     endif()
 
     if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")

docs/algorithms/kem/classic_mceliece.md

@@ -10,11 +10,11 @@ Classic McEliece
 Implementation
 --------------
 
-- **Source of implementation**: SUPERCOP-20191221, "vec" implementation
-- **Implementation version**: https://github.com/PQClean/PQClean/commit/3c8be3cb1f8ca0953f2df44ea665f118efb468d6
+- **Source of implementation**: SUPERCOP-20191221, "vec" implementation ("avx" implementation if OQS_USE_CPU_EXTENSIONS set)
+- **Implementation version**: https://github.com/PQClean/PQClean/commit/ebcc71c51a30b6e5db4f1fade22999b346fdafce
 - **License**: Public domain
 - **Constant-time**: Yes
-- **Optimizations**: Portable C
+- **Optimizations**: Portable C, wth AVX2, POPCNT, BMI1 instructions (if available at run-time)
 
 Parameter sets
 --------------

docs/algorithms/sig/falcon.md

@@ -10,8 +10,8 @@ Falcon
 Implementation
 --------------
 
-- **Source of implementation**: supercop-20201018 via https://github.com/jschanck/package-pqclean/tree/78831f03/falcon
-- **Implementation version**: https://github.com/PQClean/PQClean/commit/3c8be3cb1f8ca0953f2df44ea665f118efb468d6
+- **Source of implementation**: supercop-20201018 via https://github.com/jschanck/package-pqclean/tree/cea1fa5a/falcon
+- **Implementation version**: https://github.com/PQClean/PQClean/commit/ebcc71c51a30b6e5db4f1fade22999b346fdafce
 - **License**: CC0 1.0 Universal
 - **Constant-time**: Yes
 - **Optimizations**: Portable C, with AVX2 instructions (if available at runtime)

scripts/copy_from_upstream/copy_from_upstream.py

@@ -78,7 +78,7 @@ def replacer(filename, instructions, delimiter):
     file_put_contents(os.path.join(os.environ['LIBOQS_DIR'], filename), contents)
 
 def load_instructions():
-    subprocess_stdout = subprocess.STDOUT if DEBUG > 0 else subprocess.DEVNULL
+    subprocess_stdout = None if DEBUG > 0 else subprocess.DEVNULL
     instructions = file_get_contents(
         os.path.join(os.environ['LIBOQS_DIR'], 'scripts', 'copy_from_upstream', 'copy_from_upstream.yml'),
         encoding='utf-8')
@@ -203,13 +203,7 @@ def load_instructions():
             scheme['pqclean_scheme_c'] = scheme['pqclean_scheme'].replace('-', '')
             scheme['scheme_c'] = scheme['scheme'].replace('-', '')
             scheme['default_implementation'] = family['default_implementation']
-            # This is a temporary hack to work around the fact that
-            # the PQClean's META.ymls for the Dilithium AVX2 variants
-            # are not properly specified.
-            if scheme['pretty_name_full'].startswith('DILITHIUM_') and scheme['upstream_location'] == "pqclean":
-                scheme['metadata']['implementations'][1]['supported_platforms'][0]['operating_systems'] = ['Linux']
-                scheme['metadata']['implementations'][1]['supported_platforms'][0]['required_flags'] = ['avx2', 'bmi1',
-                                                                                                        'popcnt']
+
             for impl in scheme['metadata']['implementations']:
                 if 'common_dep' in impl:
                     cdeps_names = impl['common_dep'].split(" ")
@@ -376,7 +370,7 @@ def process_families(instructions, basedir, with_kat, with_generator):
                 srcs = handle_implementation(impl, family, scheme, basedir)
                 if DEBUG > 3:
                     print("SRCs found: %s" % (srcs))
-                if (scheme['sources']):
+                if ('sources' in scheme):
                     assert (len(scheme['sources']) == len(srcs))
                 # in any case: add 'sources' to implementation(s)
                 # Only retain this 1 implementation:
@@ -385,6 +379,14 @@ def process_families(instructions, basedir, with_kat, with_generator):
                 scheme['metadata']['implementations'][0]['sources'] = srcs
             else:
                 # If no scheme['implementation'] given, get the list from META.yml and add all implementations
+                # our code generator logic assumes only one default and one optimized implementation
+                # so, for mceliece, kill off "clean" and "sse" implementations until this is fixed TBD
+                if family['name'] == "classic_mceliece":
+                     mceimps = []
+                     for i in scheme['metadata']['implementations']:
+                        if i['name'] != "sse" and i['name'] != "clean":
+                           mceimps.append(i)
+                     scheme['metadata']['implementations'] = mceimps
                 for impl in scheme['metadata']['implementations']:
                     srcs = handle_implementation(impl['name'], family, scheme, basedir)
                     if DEBUG > 2:
@@ -402,6 +404,7 @@ def process_families(instructions, basedir, with_kat, with_generator):
                                 scheme['scheme'], str(ke), impl['name']))
                         pass
 
+
             if with_kat:
                 if family in instructions['kems']:
                     try:

scripts/copy_from_upstream/copy_from_upstream.yml

@@ -3,7 +3,7 @@ upstreams:
     name: pqclean
     git_url: https://github.com/PQClean/PQClean.git
     git_branch: master
-    git_commit: 3d7d2024fa892bd7f00dca3fff4122175f4a26dc
+    git_commit: ebcc71c51a30b6e5db4f1fade22999b346fdafce
     kem_meta_path: 'crypto_kem/{pqclean_scheme}/META.yml'
     sig_meta_path: 'crypto_sign/{pqclean_scheme}/META.yml'
     kem_scheme_path: 'crypto_kem/{pqclean_scheme}'
@@ -34,62 +34,42 @@ kems:
         scheme: "348864"
         pqclean_scheme: mceliece348864
         pretty_name_full: Classic-McEliece-348864
-        implementation: vec
-        sources: ['aes256ctr.c', 'benes.c', 'bm.c', 'controlbits.c', 'decrypt.c', 'encrypt.c', 'fft.c', 'fft_tr.c', 'gf.c', 'operations.c', 'pk_gen.c', 'sk_gen.c', 'transpose.c', 'util.c', 'vec.c']
       -
         scheme: "348864f"
         pqclean_scheme: mceliece348864f
         pretty_name_full: Classic-McEliece-348864f
-        implementation: vec
-        sources: ['aes256ctr.c', 'benes.c', 'bm.c', 'controlbits.c', 'decrypt.c', 'encrypt.c', 'fft.c', 'fft_tr.c', 'gf.c', 'operations.c', 'pk_gen.c', 'sk_gen.c', 'transpose.c', 'util.c', 'vec.c']
       -
         scheme: "460896"
         pqclean_scheme: mceliece460896
         pretty_name_full: Classic-McEliece-460896
-        implementation: vec
-        sources: ['aes256ctr.c', 'benes.c', 'bm.c', 'controlbits.c', 'decrypt.c', 'encrypt.c', 'fft.c', 'fft_tr.c', 'gf.c', 'operations.c', 'pk_gen.c', 'sk_gen.c', 'transpose.c', 'util.c', 'vec.c']
       -
         scheme: "460896f"
         pqclean_scheme: mceliece460896f
         pretty_name_full: Classic-McEliece-460896f
-        implementation: vec
-        sources: ['aes256ctr.c', 'benes.c', 'bm.c', 'controlbits.c', 'decrypt.c', 'encrypt.c', 'fft.c', 'fft_tr.c', 'gf.c', 'operations.c', 'pk_gen.c', 'sk_gen.c', 'transpose.c', 'util.c', 'vec.c']
       -
         scheme: "6688128"
         pqclean_scheme: mceliece6688128
         pretty_name_full: Classic-McEliece-6688128
-        implementation: vec
-        sources: ['aes256ctr.c', 'benes.c', 'bm.c', 'controlbits.c', 'decrypt.c', 'encrypt.c', 'fft.c', 'fft_tr.c', 'gf.c', 'operations.c', 'pk_gen.c', 'sk_gen.c', 'transpose.c', 'util.c', 'vec.c']
       -
         scheme: "6688128f"
         pqclean_scheme: mceliece6688128f
         pretty_name_full: Classic-McEliece-6688128f
-        implementation: vec
-        sources: ['aes256ctr.c', 'benes.c', 'bm.c', 'controlbits.c', 'decrypt.c', 'encrypt.c', 'fft.c', 'fft_tr.c', 'gf.c', 'operations.c', 'pk_gen.c', 'sk_gen.c', 'transpose.c', 'util.c', 'vec.c']
       -
         scheme: "6960119"
         pqclean_scheme: mceliece6960119
         pretty_name_full: Classic-McEliece-6960119
-        implementation: vec
-        sources: ['aes256ctr.c', 'benes.c', 'bm.c', 'controlbits.c', 'decrypt.c', 'encrypt.c', 'fft.c', 'fft_tr.c', 'gf.c', 'operations.c', 'pk_gen.c', 'sk_gen.c', 'transpose.c', 'util.c', 'vec.c']
       -
         scheme: "6960119f"
         pqclean_scheme: mceliece6960119f
         pretty_name_full: Classic-McEliece-6960119f
-        implementation: vec
-        sources: ['aes256ctr.c', 'benes.c', 'bm.c', 'controlbits.c', 'decrypt.c', 'encrypt.c', 'fft.c', 'fft_tr.c', 'gf.c', 'operations.c', 'pk_gen.c', 'sk_gen.c', 'transpose.c', 'util.c', 'vec.c']
       -
         scheme: "8192128"
         pqclean_scheme: mceliece8192128
         pretty_name_full: Classic-McEliece-8192128
-        implementation: vec
-        sources: ['aes256ctr.c', 'benes.c', 'bm.c', 'controlbits.c', 'decrypt.c', 'encrypt.c', 'fft.c', 'fft_tr.c', 'gf.c', 'operations.c', 'pk_gen.c', 'sk_gen.c', 'transpose.c', 'util.c', 'vec.c']
       -
         scheme: "8192128f"
         pqclean_scheme: mceliece8192128f
         pretty_name_full: Classic-McEliece-8192128f
-        implementation: vec
-        sources: ['aes256ctr.c', 'benes.c', 'bm.c', 'controlbits.c', 'decrypt.c', 'encrypt.c', 'fft.c', 'fft_tr.c', 'gf.c', 'operations.c', 'pk_gen.c', 'sk_gen.c', 'transpose.c', 'util.c', 'vec.c']
   -
     name: hqc
     default_implementation: clean

src/CMakeLists.txt

@@ -77,6 +77,10 @@ if(OQS_USE_OPENSSL)
     target_link_libraries(oqs PUBLIC ${OPENSSL_CRYPTO_LIBRARY})
     target_include_directories(oqs PUBLIC ${OPENSSL_INCLUDE_DIR})
 endif()
+if (CMAKE_SYSTEM_NAME MATCHES "Linux" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_ENABLE_KEM_CLASSIC_MCELIECE)
+        # hack to enable McEliece' AVX symbols to link on Linux
+target_link_options(oqs PRIVATE -Wl,-Bsymbolic)
+endif()
 
 set_target_properties(oqs
     PROPERTIES