From 4e73797006a107070d807544486ab885069cb1e5 Mon Sep 17 00:00:00 2001
From: sejunkim <sejun.kim@intel.com>
Date: Fri, 23 Sep 2022 16:01:19 -0700
Subject: [PATCH 1/9] * Add cpu_features to determine AVX512IFMA runtime -
 Added CMake dependency - Refactoring 3rd party configuring and building w/
 CMake

---
 CMakeLists.txt          |  3 +++
 cmake/cpufeatures.cmake | 31 +++++++++++++++++++++++++++++++
 cmake/gbenchmark.cmake  |  5 ++++-
 cmake/gtest.cmake       |  7 ++++++-
 ipcl/CMakeLists.txt     |  1 +
 5 files changed, 45 insertions(+), 2 deletions(-)
 create mode 100644 cmake/cpufeatures.cmake

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1fbadfd..9fee5ef 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -145,6 +145,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ipcl)
 include(ipcl-util)
 
 include(cmake/ippcrypto.cmake)
+include(cmake/cpufeatures.cmake)
 
 if(IPCL_TEST)
   include(cmake/gtest.cmake)
@@ -153,6 +154,8 @@ if(IPCL_BENCHMARK)
   include(cmake/gbenchmark.cmake)
 endif()
 
+
+
 add_subdirectory(ipcl)
 
 # unit-test and benchmarks
diff --git a/cmake/cpufeatures.cmake b/cmake/cpufeatures.cmake
new file mode 100644
index 0000000..cc54b1c
--- /dev/null
+++ b/cmake/cpufeatures.cmake
@@ -0,0 +1,31 @@
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+include(ExternalProject)
+include(GNUInstallDirs)
+
+set(CPUFEATURES_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ext_cpufeatures)
+set(CPUFEATURES_GIT_REPO_URL https://github.com/google/cpu_features.git)
+set(CPUFEATURES_GIT_LABEL v0.7.0)
+set(CPUFEATURES_C_FLAGS "${IPCL_FORWARD_CMAKE_ARGS} -fPIC")
+
+ExternalProject_Add(
+  ext_cpufeatures
+  PREFIX ${CPUFEATURES_PREFIX}
+  GIT_REPOSITORY ${CPUFEATURES_GIT_REPO_URL}
+  GIT_TAG ${CPUFEATURES_GIT_LABEL}
+  CMAKE_ARGS ${CPUFEATURES_C_FLAGS}
+            -DCMAKE_BUILD_TYPE=Release
+            -DBUILD_TESTING=OFF
+            -DCMAKE_INSTALL_PREFIX=${CPUFEATURES_PREFIX}
+  UPDATE_COMMAND ""
+  EXCLUDE_FROM_ALL TRUE)
+
+
+add_library(libcpu_features INTERFACE)
+add_dependencies(libcpu_features ext_cpufeatures)
+
+target_include_directories(libcpu_features SYSTEM
+                           INTERFACE ${CPUFEATURES_PREFIX}/include)
+target_link_libraries(libcpu_features
+                      INTERFACE ${CPUFEATURES_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libcpu_features.a)
diff --git a/cmake/gbenchmark.cmake b/cmake/gbenchmark.cmake
index 538c04c..f80a624 100644
--- a/cmake/gbenchmark.cmake
+++ b/cmake/gbenchmark.cmake
@@ -2,7 +2,11 @@
 # SPDX-License-Identifier: Apache-2.0
 
 include(ExternalProject)
+include(GNUInstallDirs)
 
+if (NOT ${CMAKE_INSTALL_PREFIX})
+  set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR})
+endif()
 set(GBENCHMARK_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ext_gbenchmark)
 
 set(GBENCHMARK_SRC_DIR ${GBENCHMARK_PREFIX}/src/ext_gbenchmark/)
@@ -32,7 +36,6 @@ ExternalProject_Add(
 add_library(libgbenchmark INTERFACE)
 add_dependencies(libgbenchmark ext_gbenchmark)
 
-ExternalProject_Get_Property(ext_gbenchmark SOURCE_DIR BINARY_DIR)
 target_link_libraries(libgbenchmark INTERFACE ${GBENCHMARK_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libbenchmark.a)
 
 target_include_directories(libgbenchmark SYSTEM
diff --git a/cmake/gtest.cmake b/cmake/gtest.cmake
index 7678d59..cd2efa8 100644
--- a/cmake/gtest.cmake
+++ b/cmake/gtest.cmake
@@ -2,14 +2,19 @@
 # SPDX-License-Identifier: Apache-2.0
 
 include(ExternalProject)
+include(GNUInstallDirs)
+if(NOT ${CMAKE_INSTALL_PREFIX})
+  set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR})
+endif()
 
+set(GTEST_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ext_gtest)
 set(GTEST_GIT_REPO_URL https://github.com/google/googletest.git)
 set(GTEST_GIT_LABEL release-1.10.0)
 set(GTEST_CXX_FLAGS "${IPCL_FORWARD_CMAKE_ARGS} -fPIC")
 
 ExternalProject_Add(
   ext_gtest
-  PREFIX ext_gtest
+  PREFIX ${GTEST_PREFIX}
   GIT_REPOSITORY ${GTEST_GIT_REPO_URL}
   GIT_TAG ${GTEST_GIT_LABEL}
   CMAKE_ARGS ${GTEST_CXX_FLAGS} -DCMAKE_BUILD_TYPE=Release
diff --git a/ipcl/CMakeLists.txt b/ipcl/CMakeLists.txt
index 7ab34b1..dbda08d 100644
--- a/ipcl/CMakeLists.txt
+++ b/ipcl/CMakeLists.txt
@@ -56,6 +56,7 @@ endif()
 
 if(IPCL_SHARED)
 	target_link_libraries(ipcl PRIVATE libippcrypto)
+	target_link_libraries(ipcl PRIVATE libcpu_features)
 	target_include_directories(ipcl PRIVATE ${IPPCRYPTO_INC_DIR})
 else()
 	ipcl_create_archive(ipcl libippcrypto::ippcp)

From f19cd19c051e2c831cb43f59fa7a9be06601f9d4 Mon Sep 17 00:00:00 2001
From: sejunkim <sejun.kim@intel.com>
Date: Mon, 26 Sep 2022 18:10:10 -0700
Subject: [PATCH 2/9] Add option to determine AVX512IFMA during runtime (#18) -
 Add CMake flag to enable runtime version (IPCL_DETECT_IFMA_RUNTIME) - Add
 ```cpu_features``` dependency - Add manual IFMA disabling with environment
 variable (IPCL_DISABLE_AVX512IFMA=ON)

---
 CMakeLists.txt             | 22 +++++++++------
 cmake/cpufeatures.cmake    | 24 ++++++++++++----
 ipcl/CMakeLists.txt        | 17 ++++++++++++
 ipcl/include/ipcl/util.hpp | 11 ++++++++
 ipcl/mod_exp.cpp           | 57 ++++++++++++++++++++++++++++++++++++++
 5 files changed, 117 insertions(+), 14 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9fee5ef..7e879ce 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -54,6 +54,7 @@ option(IPCL_ENABLE_OMP "Enable OpenMP testing/benchmarking" ON)
 option(IPCL_THREAD_COUNT "The max number of threads used by OpenMP(If the value is OFF/0, it is determined at runtime)" OFF)
 option(IPCL_DOCS "Enable document building" OFF)
 option(IPCL_SHARED "Build shared library" ON)
+option(IPCL_DETECT_IFMA_RUNTIME "Detect AVX512/IFMA instructions during runtime" OFF)
 option(IPCL_DEBUG_DISABLE_AVX512IFMA "(Debugging) Disable usage of AVX512IFMA instructions" OFF)
 if(IPCL_ENABLE_OMP)
 	add_compile_definitions(IPCL_USE_OMP)
@@ -84,6 +85,7 @@ else()
 endif()
 message(STATUS "IPCL_DOCS:                  ${IPCL_DOCS}")
 message(STATUS "IPCL_SHARED:                ${IPCL_SHARED}")
+message(STATUS "IPCL_DETECT_IFMA_RUNTIME:   ${IPCL_DETECT_IFMA_RUNTIME}")
 
 set(IPCL_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR})
 set(IPCL_SRC_DIR ${IPCL_ROOT_DIR}/ipcl)
@@ -101,17 +103,21 @@ set(IPCL_FORWARD_CMAKE_ARGS
     -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
 )
 
-# check whether cpu support avx512 flag
-if(IPCL_DEBUG_DISABLE_AVX512IFMA)
-  message(STATUS "Support AVX512IFMA: False")
+if(IPCL_DETECT_IFMA_RUNTIME)
+  add_compile_definitions(IPCL_RUNTIME_MOD_EXP)
 else()
-  set(CPU_AVX512_FLAG "avx512ifma")
-  execute_process(COMMAND lscpu COMMAND grep ${CPU_AVX512_FLAG}  OUTPUT_VARIABLE CPU_ENABLE_AVX512)
-  if("${CPU_ENABLE_AVX512}" STREQUAL "")
+  # check whether cpu support avx512 flag
+  if(IPCL_DEBUG_DISABLE_AVX512IFMA)
     message(STATUS "Support AVX512IFMA: False")
   else()
-    message(STATUS "Support AVX512IFMA: True")
-    add_compile_definitions(IPCL_CRYPTO_MB_MOD_EXP)
+    set(CPU_AVX512_FLAG "avx512ifma")
+    execute_process(COMMAND lscpu COMMAND grep ${CPU_AVX512_FLAG}  OUTPUT_VARIABLE CPU_ENABLE_AVX512)
+    if("${CPU_ENABLE_AVX512}" STREQUAL "")
+      message(STATUS "Support AVX512IFMA: False")
+    else()
+      message(STATUS "Support AVX512IFMA: True")
+      add_compile_definitions(IPCL_CRYPTO_MB_MOD_EXP)
+    endif()
   endif()
 endif()
 
diff --git a/cmake/cpufeatures.cmake b/cmake/cpufeatures.cmake
index cc54b1c..ff23413 100644
--- a/cmake/cpufeatures.cmake
+++ b/cmake/cpufeatures.cmake
@@ -22,10 +22,22 @@ ExternalProject_Add(
   EXCLUDE_FROM_ALL TRUE)
 
 
-add_library(libcpu_features INTERFACE)
-add_dependencies(libcpu_features ext_cpufeatures)
+set(CPUFEATURES_INC_DIR ${CPUFEATURES_PREFIX}/include)
 
-target_include_directories(libcpu_features SYSTEM
-                           INTERFACE ${CPUFEATURES_PREFIX}/include)
-target_link_libraries(libcpu_features
-                      INTERFACE ${CPUFEATURES_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libcpu_features.a)
+if(IPCL_SHARED)
+  add_library(libcpu_features INTERFACE)
+  add_dependencies(libcpu_features ext_cpufeatures)
+
+  target_include_directories(libcpu_features SYSTEM
+                            INTERFACE ${CPUFEATURES_PREFIX}/include)
+  target_link_libraries(libcpu_features
+                        INTERFACE ${CPUFEATURES_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libcpu_features.a)
+
+else()
+  add_library(libcpu_features STATIC IMPORTED GLOBAL)
+  add_dependencies(libcpu_features ext_cpufeatures)
+
+  set_target_properties(libcpu_features PROPERTIES
+      IMPORTED_LOCATION ${CPUFEATURES_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libcpu_features.a
+      INCLUDE_DIRECTORIES ${CPUFEATURES_PREFIX}/include)
+endif()
diff --git a/ipcl/CMakeLists.txt b/ipcl/CMakeLists.txt
index dbda08d..a5872eb 100644
--- a/ipcl/CMakeLists.txt
+++ b/ipcl/CMakeLists.txt
@@ -35,6 +35,20 @@ target_include_directories(ipcl
 	PUBLIC $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
 )
 
+if(IPCL_DETECT_IFMA_RUNTIME)
+	target_include_directories(ipcl
+		PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+		PUBLIC $<BUILD_INTERFACE:${CPUFEATURES_INC_DIR}>
+		PUBLIC $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+	)
+	install(DIRECTORY ${CPUFEATURES_INC_DIR}/
+		DESTINATION ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}
+		FILES_MATCHING
+		PATTERN "*.hpp"
+		PATTERN "*.h")
+
+endif()
+
 install(DIRECTORY ${IPCL_INC_DIR}/
         DESTINATION ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}
         FILES_MATCHING
@@ -58,10 +72,13 @@ if(IPCL_SHARED)
 	target_link_libraries(ipcl PRIVATE libippcrypto)
 	target_link_libraries(ipcl PRIVATE libcpu_features)
 	target_include_directories(ipcl PRIVATE ${IPPCRYPTO_INC_DIR})
+	target_include_directories(ipcl PRIVATE ${CPUFEATURES_INC_DIR})
 else()
 	ipcl_create_archive(ipcl libippcrypto::ippcp)
 	ipcl_create_archive(ipcl libippcrypto::crypto_mb)
+	ipcl_create_archive(ipcl libcpu_features)
 	target_include_directories(ipcl PRIVATE ${IPPCRYPTO_INC_DIR})
+	target_include_directories(ipcl PRIVATE ${CPUFEATURES_INC_DIR})
 endif()
 
 set_target_properties(ipcl PROPERTIES POSITION_INDEPENDENT_CODE ON)
diff --git a/ipcl/include/ipcl/util.hpp b/ipcl/include/ipcl/util.hpp
index b264816..937d0ba 100644
--- a/ipcl/include/ipcl/util.hpp
+++ b/ipcl/include/ipcl/util.hpp
@@ -4,6 +4,9 @@
 #ifndef IPCL_INCLUDE_IPCL_UTIL_HPP_
 #define IPCL_INCLUDE_IPCL_UTIL_HPP_
 
+#include <cpu_features/cpuinfo_x86.h>
+
+#include <cstdlib>
 #include <exception>
 #include <sstream>
 #include <string>
@@ -66,6 +69,14 @@ class OMPUtilities {
 
 #endif  // IPCL_USE_OMP
 
+#ifdef IPCL_RUNTIME_MOD_EXP
+static const bool disable_avx512ifma =
+    (std::getenv("IPCL_DISABLE_AVX512IFMA") != nullptr);
+static const cpu_features::X86Features features =
+    cpu_features::GetX86Info().features;
+static const bool has_avx512ifma = features.avx512ifma && !disable_avx512ifma;
+#endif  // IPCL_RUNTIME_MOD_EXP
+
 }  // namespace ipcl
 
 #endif  // IPCL_INCLUDE_IPCL_UTIL_HPP_
diff --git a/ipcl/mod_exp.cpp b/ipcl/mod_exp.cpp
index cdb729d..1f885a4 100644
--- a/ipcl/mod_exp.cpp
+++ b/ipcl/mod_exp.cpp
@@ -160,6 +160,61 @@ std::vector<BigNumber> ippModExp(const std::vector<BigNumber>& base,
   std::size_t v_size = base.size();
   std::vector<BigNumber> res(v_size);
 
+#ifdef IPCL_RUNTIME_MOD_EXP
+
+  // If there is only 1 big number, we don't need to use MBModExp
+  if (v_size == 1) {
+    res[0] = ippSBModExp(base[0], exp[0], mod[0]);
+    return res;
+  }
+
+  if (has_avx512ifma) {
+    std::size_t remainder = v_size % IPCL_CRYPTO_MB_SIZE;
+    std::size_t num_chunk =
+        (v_size + IPCL_CRYPTO_MB_SIZE - 1) / IPCL_CRYPTO_MB_SIZE;
+#ifdef IPCL_USE_OMP
+    int omp_remaining_threads = OMPUtilities::MaxThreads;
+#pragma omp parallel for num_threads( \
+    OMPUtilities::assignOMPThreads(omp_remaining_threads, num_chunk))
+#endif  // IPCL_USE_OMP
+    for (std::size_t i = 0; i < num_chunk; i++) {
+      std::size_t chunk_size = IPCL_CRYPTO_MB_SIZE;
+      if ((i == (num_chunk - 1)) && (remainder > 0)) chunk_size = remainder;
+
+      std::size_t chunk_offset = i * IPCL_CRYPTO_MB_SIZE;
+
+      auto base_start = base.begin() + chunk_offset;
+      auto base_end = base_start + chunk_size;
+
+      auto exp_start = exp.begin() + chunk_offset;
+      auto exp_end = exp_start + chunk_size;
+
+      auto mod_start = mod.begin() + chunk_offset;
+      auto mod_end = mod_start + chunk_size;
+
+      auto base_chunk = std::vector<BigNumber>(base_start, base_end);
+      auto exp_chunk = std::vector<BigNumber>(exp_start, exp_end);
+      auto mod_chunk = std::vector<BigNumber>(mod_start, mod_end);
+
+      auto tmp = ippMBModExp(base_chunk, exp_chunk, mod_chunk);
+      std::copy(tmp.begin(), tmp.end(), res.begin() + chunk_offset);
+    }
+
+    return res;
+
+  } else {
+#ifdef IPCL_USE_OMP
+    int omp_remaining_threads = OMPUtilities::MaxThreads;
+#pragma omp parallel for num_threads( \
+    OMPUtilities::assignOMPThreads(omp_remaining_threads, v_size))
+#endif  // IPCL_USE_OMP
+    for (int i = 0; i < v_size; i++)
+      res[i] = ippSBModExp(base[i], exp[i], mod[i]);
+    return res;
+  }
+
+#else
+
 #ifdef IPCL_CRYPTO_MB_MOD_EXP
 
   // If there is only 1 big number, we don't need to use MBModExp
@@ -214,6 +269,8 @@ std::vector<BigNumber> ippModExp(const std::vector<BigNumber>& base,
   return res;
 
 #endif  // IPCL_CRYPTO_MB_MOD_EXP
+
+#endif  // IPCL_RUNTIME_MOD_EXP
 }
 
 BigNumber ippModExp(const BigNumber& base, const BigNumber& exp,

From 94df8ca63c39bc44bce21e76aad87359bbdaccb1 Mon Sep 17 00:00:00 2001
From: sejunkim <sejun.kim@intel.com>
Date: Mon, 26 Sep 2022 23:20:02 -0700
Subject: [PATCH 3/9] Fix to include cpu_features header when specified

---
 ipcl/include/ipcl/util.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ipcl/include/ipcl/util.hpp b/ipcl/include/ipcl/util.hpp
index 937d0ba..202e277 100644
--- a/ipcl/include/ipcl/util.hpp
+++ b/ipcl/include/ipcl/util.hpp
@@ -4,7 +4,9 @@
 #ifndef IPCL_INCLUDE_IPCL_UTIL_HPP_
 #define IPCL_INCLUDE_IPCL_UTIL_HPP_
 
+#ifdef IPCL_RUNTIME_MOD_EXP
 #include <cpu_features/cpuinfo_x86.h>
+#endif  // IPCL_RUNTIME_MOD_EXP
 
 #include <cstdlib>
 #include <exception>

From f1985d2469fc4086dcd2408ccd9f176637ac26aa Mon Sep 17 00:00:00 2001
From: sejunkim <sejun.kim@intel.com>
Date: Tue, 27 Sep 2022 23:19:48 -0700
Subject: [PATCH 4/9] Fixed shared library build - Refactor cmake
 configurations - Added example code

---
 .pre-commit-config.yaml        |  8 +++----
 CMakeLists.txt                 | 20 +++++++++--------
 cmake/cpufeatures.cmake        | 24 ++++++++++++++------
 cmake/gbenchmark.cmake         |  3 ---
 cmake/gtest.cmake              | 15 ++++++++-----
 cmake/ipcl/IPCLConfig.cmake.in |  3 +++
 cmake/ippcrypto.cmake          | 25 +++++++++++++++------
 example/CMakeLists.txt         | 21 +++++++++++++++++
 example/test.cpp               | 41 ++++++++++++++++++++++++++++++++++
 example/test.h                 | 12 ++++++++++
 ipcl/CMakeLists.txt            | 17 +++++++++-----
 11 files changed, 149 insertions(+), 40 deletions(-)
 create mode 100644 example/CMakeLists.txt
 create mode 100644 example/test.cpp
 create mode 100644 example/test.h

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index e13a70f..574af0c 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -18,9 +18,9 @@ repos:
         - id: typos
     - repo: local
       hooks:
-        - id: clang-format
-          name: clang-format
-          entry: clang-format
+        - id: clang-format-11
+          name: clang-format-11
+          entry: clang-format-11
           language: system
           files: \.(c|cc|cxx|cpp|h|hpp|hxx|js|proto)$
           args:
@@ -43,7 +43,7 @@ repos:
           entry: cpplint
           language: system
           files: \.(c|cc|cxx|cpp|h|hpp|hxx)$
-          exclude: ipcl/bignum.cpp|ipcl/include/ipcl/bignum.h
+          exclude: ipcl/bignum.cpp|ipcl/include/ipcl/bignum.h|example/test.cpp
           args:
             - --recursive
             - --filter=-runtime/references,-whitespace/comments,-whitespace/indent
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7e879ce..54bd81b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -8,6 +8,7 @@ project(IPCL VERSION 1.1.4 LANGUAGES C CXX)
 include(CMakePackageConfigHelpers)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
+include(GNUInstallDirs)
 
 if(CMAKE_BUILD_TYPE)
   set(RELEASE_TYPES
@@ -34,16 +35,14 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 set(CMAKE_INSTALL_MESSAGE LAZY)
-set(CMAKE_INSTALL_RPATH "\$ORIGIN")
 
-set(CMAKE_C_FLAGS "-O2 -Wno-error=deprecated-declarations")
-set(CMAKE_CXX_FLAGS "-O2 -fpermissive -Wno-error=deprecated-declarations")
-
-if(NOT CMAKE_INSTALL_PREFIX)
-  set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR})
+if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+  set(CMAKE_INSTALL_PREFIX "/opt/intel/ipcl")
 endif()
 
-include(GNUInstallDirs)
+set(CMAKE_C_FLAGS "-O2 -Wno-error=deprecated-declarations")
+set(CMAKE_CXX_FLAGS "-O2 -fpermissive -Wno-error=deprecated-declarations")
+set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR};${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/ippcrypto")
 
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_INSTALL_LIBDIR})
 
@@ -105,6 +104,7 @@ set(IPCL_FORWARD_CMAKE_ARGS
 
 if(IPCL_DETECT_IFMA_RUNTIME)
   add_compile_definitions(IPCL_RUNTIME_MOD_EXP)
+  set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH};${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/cpufeatures")
 else()
   # check whether cpu support avx512 flag
   if(IPCL_DEBUG_DISABLE_AVX512IFMA)
@@ -139,7 +139,7 @@ else()
 endif()
 
 # find package for OpenSSL and Threads
-set(OPENSSL_USE_STATIC_LIBS TRUE)
+# set(OPENSSL_USE_STATIC_LIBS TRUE)
 find_package(Threads REQUIRED)
 find_package(OpenSSL REQUIRED)
 
@@ -151,7 +151,9 @@ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ipcl)
 include(ipcl-util)
 
 include(cmake/ippcrypto.cmake)
-include(cmake/cpufeatures.cmake)
+if(IPCL_DETECT_IFMA_RUNTIME)
+  include(cmake/cpufeatures.cmake)
+endif()
 
 if(IPCL_TEST)
   include(cmake/gtest.cmake)
diff --git a/cmake/cpufeatures.cmake b/cmake/cpufeatures.cmake
index ff23413..75fe59b 100644
--- a/cmake/cpufeatures.cmake
+++ b/cmake/cpufeatures.cmake
@@ -4,7 +4,9 @@
 include(ExternalProject)
 include(GNUInstallDirs)
 
+message(STATUS "configuring cpu_features")
 set(CPUFEATURES_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ext_cpufeatures)
+set(CPUFEATURES_DESTDIR ${CPUFEATURES_PREFIX}/cpufeatures_install)
 set(CPUFEATURES_GIT_REPO_URL https://github.com/google/cpu_features.git)
 set(CPUFEATURES_GIT_LABEL v0.7.0)
 set(CPUFEATURES_C_FLAGS "${IPCL_FORWARD_CMAKE_ARGS} -fPIC")
@@ -17,27 +19,35 @@ ExternalProject_Add(
   CMAKE_ARGS ${CPUFEATURES_C_FLAGS}
             -DCMAKE_BUILD_TYPE=Release
             -DBUILD_TESTING=OFF
-            -DCMAKE_INSTALL_PREFIX=${CPUFEATURES_PREFIX}
+            -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
   UPDATE_COMMAND ""
-  EXCLUDE_FROM_ALL TRUE)
+  EXCLUDE_FROM_ALL TRUE
+  INSTALL_COMMAND make DESTDIR=${CPUFEATURES_DESTDIR} install
+  )
 
 
-set(CPUFEATURES_INC_DIR ${CPUFEATURES_PREFIX}/include)
+set(CPUFEATURES_INC_DIR ${CPUFEATURES_DESTDIR}/${CMAKE_INSTALL_PREFIX}/include)
+set(CPUFEATURES_LIB_DIR ${CPUFEATURES_DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR})
 
 if(IPCL_SHARED)
   add_library(libcpu_features INTERFACE)
   add_dependencies(libcpu_features ext_cpufeatures)
 
   target_include_directories(libcpu_features SYSTEM
-                            INTERFACE ${CPUFEATURES_PREFIX}/include)
+                            INTERFACE ${CPUFEATURES_INC_DIR})
   target_link_libraries(libcpu_features
-                        INTERFACE ${CPUFEATURES_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libcpu_features.a)
+                        INTERFACE ${CPUFEATURES_LIB_DIR}/libcpu_features.a)
 
+  install(
+    DIRECTORY ${CPUFEATURES_LIB_DIR}/
+    DESTINATION "./${CMAKE_INSTALL_LIBDIR}/cpufeatures"
+    USE_SOURCE_PERMISSIONS
+  )
 else()
   add_library(libcpu_features STATIC IMPORTED GLOBAL)
   add_dependencies(libcpu_features ext_cpufeatures)
 
   set_target_properties(libcpu_features PROPERTIES
-      IMPORTED_LOCATION ${CPUFEATURES_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libcpu_features.a
-      INCLUDE_DIRECTORIES ${CPUFEATURES_PREFIX}/include)
+      IMPORTED_LOCATION ${CPUFEATURES_LIB_DIR}/libcpu_features.a
+      INCLUDE_DIRECTORIES ${CPUFEATURES_INC_DIR})
 endif()
diff --git a/cmake/gbenchmark.cmake b/cmake/gbenchmark.cmake
index f80a624..8c3e8c6 100644
--- a/cmake/gbenchmark.cmake
+++ b/cmake/gbenchmark.cmake
@@ -4,9 +4,6 @@
 include(ExternalProject)
 include(GNUInstallDirs)
 
-if (NOT ${CMAKE_INSTALL_PREFIX})
-  set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR})
-endif()
 set(GBENCHMARK_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ext_gbenchmark)
 
 set(GBENCHMARK_SRC_DIR ${GBENCHMARK_PREFIX}/src/ext_gbenchmark/)
diff --git a/cmake/gtest.cmake b/cmake/gtest.cmake
index cd2efa8..24b80a7 100644
--- a/cmake/gtest.cmake
+++ b/cmake/gtest.cmake
@@ -3,9 +3,6 @@
 
 include(ExternalProject)
 include(GNUInstallDirs)
-if(NOT ${CMAKE_INSTALL_PREFIX})
-  set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR})
-endif()
 
 set(GTEST_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ext_gtest)
 set(GTEST_GIT_REPO_URL https://github.com/google/googletest.git)
@@ -18,9 +15,17 @@ ExternalProject_Add(
   GIT_REPOSITORY ${GTEST_GIT_REPO_URL}
   GIT_TAG ${GTEST_GIT_LABEL}
   CMAKE_ARGS ${GTEST_CXX_FLAGS} -DCMAKE_BUILD_TYPE=Release
-  INSTALL_COMMAND ""
+    -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
   UPDATE_COMMAND ""
-  EXCLUDE_FROM_ALL TRUE)
+  EXCLUDE_FROM_ALL TRUE
+  INSTALL_COMMAND ""
+)
+
+# install(
+#   DIRECTORY ${GTEST_DESTDIR}/${CMAKE_INSTALL_PREFIX}/
+#   DESTINATION "."
+#   USE_SOURCE_PERMISSIONS
+# )
 
 ExternalProject_Get_Property(ext_gtest SOURCE_DIR BINARY_DIR)
 
diff --git a/cmake/ipcl/IPCLConfig.cmake.in b/cmake/ipcl/IPCLConfig.cmake.in
index fecf06e..340a5c4 100644
--- a/cmake/ipcl/IPCLConfig.cmake.in
+++ b/cmake/ipcl/IPCLConfig.cmake.in
@@ -9,6 +9,9 @@ include(${CMAKE_CURRENT_LIST_DIR}/IPCLTargets.cmake)
 if(TARGET IPCL::ipcl)
   set(IPCL_FOUND TRUE)
   message(STATUS "Intel Paillier Cryptosystem Library found")
+  find_package(OpenSSL REQUIRED)
+  find_package(Threads REQUIRED)
+  find_package(OpenMP REQUIRED)
 else()
   message(STATUS "Intel Paillier Cryptosystem Library not found")
 endif()
diff --git a/cmake/ippcrypto.cmake b/cmake/ippcrypto.cmake
index 3ada4c3..6ebcffd 100644
--- a/cmake/ippcrypto.cmake
+++ b/cmake/ippcrypto.cmake
@@ -2,8 +2,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
 include(ExternalProject)
-MESSAGE(STATUS "Configuring ipp-crypto")
+message(STATUS "Configuring ipp-crypto")
+
 set(IPPCRYPTO_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ext_ipp-crypto)
+set(IPPCRYPTO_DESTDIR ${IPPCRYPTO_PREFIX}/ippcrypto_install)
 set(IPPCRYPTO_GIT_REPO_URL https://github.com/intel/ipp-crypto.git)
 set(IPPCRYPTO_GIT_LABEL ippcp_2021.6)
 set(IPPCRYPTO_SRC_DIR ${IPPCRYPTO_PREFIX}/src/ext_ipp-crypto/)
@@ -29,20 +31,29 @@ ExternalProject_Add(
              -DARCH=${IPPCRYPTO_ARCH}
              -DCMAKE_ASM_NASM_COMPILER=nasm
              -DCMAKE_BUILD_TYPE=Release
+             -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
   UPDATE_COMMAND ""
+  INSTALL_COMMAND make DESTDIR=${IPPCRYPTO_DESTDIR} install
 )
 
-set(IPPCRYPTO_INC_DIR ${IPPCRYPTO_PREFIX}/include)
-
+set(IPPCRYPTO_INC_DIR ${IPPCRYPTO_DESTDIR}/${CMAKE_INSTALL_PREFIX}/include)
+set(IPPCRYPTO_LIB_DIR ${IPPCRYPTO_DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/${IPPCRYPTO_ARCH})
 if(IPCL_SHARED)
   add_library(libippcrypto INTERFACE)
   add_dependencies(libippcrypto ext_ipp-crypto)
 
   ExternalProject_Get_Property(ext_ipp-crypto SOURCE_DIR BINARY_DIR)
 
-  target_link_libraries(libippcrypto INTERFACE ${IPPCRYPTO_PREFIX}/lib/${IPPCRYPTO_ARCH}/libippcp.a ${IPPCRYPTO_PREFIX}/lib/${IPPCRYPTO_ARCH}/libcrypto_mb.a)
-  target_include_directories(libippcrypto SYSTEM INTERFACE ${IPPCRYPTO_PREFIX}/include)
+  target_link_libraries(libippcrypto INTERFACE
+  ${IPPCRYPTO_LIB_DIR}/libippcp.so
+  ${IPPCRYPTO_LIB_DIR}/libcrypto_mb.so)
+  target_include_directories(libippcrypto SYSTEM INTERFACE ${IPPCRYPTO_INC_DIR})
 
+  install(
+    DIRECTORY ${IPPCRYPTO_LIB_DIR}/
+    DESTINATION "./${CMAKE_INSTALL_LIBDIR}/ippcrypto"
+    USE_SOURCE_PERMISSIONS
+  )
 else()
 
   add_library(libippcrypto::ippcp STATIC IMPORTED GLOBAL)
@@ -54,12 +65,12 @@ else()
   find_package(OpenSSL REQUIRED)
 
   set_target_properties(libippcrypto::ippcp PROPERTIES
-            IMPORTED_LOCATION ${IPPCRYPTO_PREFIX}/lib/${IPPCRYPTO_ARCH}/libippcp.a
+            IMPORTED_LOCATION ${IPPCRYPTO_LIB_DIR}/libippcp.a
             INCLUDE_DIRECTORIES ${IPPCRYPTO_INC_DIR}
   )
 
   set_target_properties(libippcrypto::crypto_mb PROPERTIES
-            IMPORTED_LOCATION ${IPPCRYPTO_PREFIX}/lib/${IPPCRYPTO_ARCH}/libcrypto_mb.a
+            IMPORTED_LOCATION ${IPPCRYPTO_LIB_DIR}/libcrypto_mb.a
             INCLUDE_DIRECTORIES ${IPPCRYPTO_INC_DIR}
   )
 endif()
diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt
new file mode 100644
index 0000000..be83d5e
--- /dev/null
+++ b/example/CMakeLists.txt
@@ -0,0 +1,21 @@
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.15.1)
+
+project(ipcl_example LANGUAGES C CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+if (DEFINED ENV{IPCL_DIR})
+  set(IPCL_DIR $ENV{IPCL_DIR})
+else()
+  set(IPCL_DIR /opt/intel/ipcl)
+endif()
+
+find_package(IPCL REQUIRED PATHS ${IPCL_DIR})
+
+add_executable(test test.cpp)
+target_link_libraries(test PRIVATE IPCL::ipcl)
diff --git a/example/test.cpp b/example/test.cpp
new file mode 100644
index 0000000..48a233d
--- /dev/null
+++ b/example/test.cpp
@@ -0,0 +1,41 @@
+// Copyright (C) 2022 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#include <climits>
+#include <iostream>
+#include <random>
+#include <vector>
+
+#include "ipcl/ipcl.hpp"
+
+int main() {
+  const uint32_t num_values = 9;
+
+  ipcl::keyPair key = ipcl::generateKeypair(2048, true);
+
+  std::vector<uint32_t> exp_value(num_values);
+  ipcl::PlainText pt;
+  ipcl::CipherText ct;
+  ipcl::PlainText dt;
+
+  std::random_device dev;
+  std::mt19937 rng(dev());
+  std::uniform_int_distribution<std::mt19937::result_type> dist(0, UINT_MAX);
+
+  for (int i = 0; i < num_values; i++) {
+    exp_value[i] = dist(rng);
+  }
+
+  pt = ipcl::PlainText(exp_value);
+  ct = key.pub_key->encrypt(pt);
+  dt = key.priv_key->decrypt(ct);
+
+  for (int i = 0; i < num_values; i++) {
+    std::vector<uint32_t> v = dt.getElementVec(i);
+    bool chk = v[0] == exp_value[i];
+    std::cout << (chk ? "pass" : "fail") << std::endl;
+  }
+
+  delete key.pub_key;
+  delete key.priv_key;
+}
diff --git a/example/test.h b/example/test.h
new file mode 100644
index 0000000..e5543d8
--- /dev/null
+++ b/example/test.h
@@ -0,0 +1,12 @@
+// Copyright (C) 2022 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef EXAMPLE_TEST_H_
+#define EXAMPLE_TEST_H_
+
+#include <climits>
+#include <iostream>
+#include <random>
+#include <vector>
+
+#endif  // EXAMPLE_TEST_H_
diff --git a/ipcl/CMakeLists.txt b/ipcl/CMakeLists.txt
index a5872eb..1966aae 100644
--- a/ipcl/CMakeLists.txt
+++ b/ipcl/CMakeLists.txt
@@ -61,28 +61,35 @@ install(DIRECTORY ${IPPCRYPTO_INC_DIR}/
         PATTERN "*.hpp"
         PATTERN "*.h")
 
+find_package(OpenSSL REQUIRED)
+find_package(Threads REQUIRED)
 target_link_libraries(ipcl PUBLIC OpenSSL::SSL OpenSSL::Crypto Threads::Threads -lnuma)
 
 if(IPCL_ENABLE_OMP)
   find_package(OpenMP REQUIRED)
-  target_link_libraries(ipcl PRIVATE OpenMP::OpenMP_CXX)
+  target_link_libraries(ipcl PUBLIC OpenMP::OpenMP_CXX)
 endif()
 
 if(IPCL_SHARED)
 	target_link_libraries(ipcl PRIVATE libippcrypto)
-	target_link_libraries(ipcl PRIVATE libcpu_features)
+	if(IPCL_DETECT_IFMA_RUNTIME)
+		target_link_libraries(ipcl PRIVATE libcpu_features)
+		target_include_directories(ipcl PRIVATE ${CPUFEATURES_INC_DIR})
+	endif()
 	target_include_directories(ipcl PRIVATE ${IPPCRYPTO_INC_DIR})
-	target_include_directories(ipcl PRIVATE ${CPUFEATURES_INC_DIR})
 else()
 	ipcl_create_archive(ipcl libippcrypto::ippcp)
 	ipcl_create_archive(ipcl libippcrypto::crypto_mb)
-	ipcl_create_archive(ipcl libcpu_features)
+	if(IPCL_DETECT_IFMA_RUNTIME)
+		ipcl_create_archive(ipcl libcpu_features)
+		target_include_directories(ipcl PRIVATE ${CPUFEATURES_INC_DIR})
+	endif()
 	target_include_directories(ipcl PRIVATE ${IPPCRYPTO_INC_DIR})
-	target_include_directories(ipcl PRIVATE ${CPUFEATURES_INC_DIR})
 endif()
 
 set_target_properties(ipcl PROPERTIES POSITION_INDEPENDENT_CODE ON)
 set_target_properties(ipcl PROPERTIES VERSION ${IPCL_VERSION})
+
 if(IPCL_DEBUG)
 	set_target_properties(ipcl PROPERTIES OUTPUT_NAME "ipcl_debug")
 else()

From 66dee52480f0ef84eba8ecc566138bfacdf91039 Mon Sep 17 00:00:00 2001
From: sejunkim <sejun.kim@intel.com>
Date: Wed, 28 Sep 2022 02:18:47 -0700
Subject: [PATCH 5/9] Typo fix - Minor typo fixes in third-party dependencies -
 Removed example/test.h

---
 cmake/cpufeatures.cmake |  2 +-
 cmake/ippcrypto.cmake   |  2 +-
 example/test.h          | 12 ------------
 3 files changed, 2 insertions(+), 14 deletions(-)
 delete mode 100644 example/test.h

diff --git a/cmake/cpufeatures.cmake b/cmake/cpufeatures.cmake
index 75fe59b..6df3c12 100644
--- a/cmake/cpufeatures.cmake
+++ b/cmake/cpufeatures.cmake
@@ -40,7 +40,7 @@ if(IPCL_SHARED)
 
   install(
     DIRECTORY ${CPUFEATURES_LIB_DIR}/
-    DESTINATION "./${CMAKE_INSTALL_LIBDIR}/cpufeatures"
+    DESTINATION "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/cpufeatures"
     USE_SOURCE_PERMISSIONS
   )
 else()
diff --git a/cmake/ippcrypto.cmake b/cmake/ippcrypto.cmake
index 6ebcffd..3650b9a 100644
--- a/cmake/ippcrypto.cmake
+++ b/cmake/ippcrypto.cmake
@@ -51,7 +51,7 @@ if(IPCL_SHARED)
 
   install(
     DIRECTORY ${IPPCRYPTO_LIB_DIR}/
-    DESTINATION "./${CMAKE_INSTALL_LIBDIR}/ippcrypto"
+    DESTINATION "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/ippcrypto"
     USE_SOURCE_PERMISSIONS
   )
 else()
diff --git a/example/test.h b/example/test.h
deleted file mode 100644
index e5543d8..0000000
--- a/example/test.h
+++ /dev/null
@@ -1,12 +0,0 @@
-// Copyright (C) 2022 Intel Corporation
-// SPDX-License-Identifier: Apache-2.0
-
-#ifndef EXAMPLE_TEST_H_
-#define EXAMPLE_TEST_H_
-
-#include <climits>
-#include <iostream>
-#include <random>
-#include <vector>
-
-#endif  // EXAMPLE_TEST_H_

From da4eb085d5f3cfdb178b6982aa640e164fff72bd Mon Sep 17 00:00:00 2001
From: sejunkim <sejun.kim@intel.com>
Date: Wed, 28 Sep 2022 03:14:15 -0700
Subject: [PATCH 6/9] Set pre-commit to use clang-format

---
 .pre-commit-config.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 574af0c..2f41239 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -18,9 +18,9 @@ repos:
         - id: typos
     - repo: local
       hooks:
-        - id: clang-format-11
-          name: clang-format-11
-          entry: clang-format-11
+        - id: clang-format
+          name: clang-format
+          entry: clang-format
           language: system
           files: \.(c|cc|cxx|cpp|h|hpp|hxx|js|proto)$
           args:

From c07b13a7f1249563643c6b6a9ff7a63076ee8ad1 Mon Sep 17 00:00:00 2001
From: sejunkim <sejun.kim@intel.com>
Date: Wed, 28 Sep 2022 12:22:19 -0700
Subject: [PATCH 7/9] Added build and usage documentation - Fixed minor typos
 and updated flag names to be more consistent - Cleaned up example CMake file

---
 example/CMakeLists.txt |  8 +-------
 example/README.md      | 45 ++++++++++++++++++++++++++++++++++++++++++
 example/test.cpp       |  3 +--
 3 files changed, 47 insertions(+), 9 deletions(-)
 create mode 100644 example/README.md

diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt
index be83d5e..cb66be8 100644
--- a/example/CMakeLists.txt
+++ b/example/CMakeLists.txt
@@ -9,13 +9,7 @@ set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
-if (DEFINED ENV{IPCL_DIR})
-  set(IPCL_DIR $ENV{IPCL_DIR})
-else()
-  set(IPCL_DIR /opt/intel/ipcl)
-endif()
-
-find_package(IPCL REQUIRED PATHS ${IPCL_DIR})
+find_package(IPCL 1.1.4 REQUIRED HINTS ${IPCL_HINT_DIR})
 
 add_executable(test test.cpp)
 target_link_libraries(test PRIVATE IPCL::ipcl)
diff --git a/example/README.md b/example/README.md
new file mode 100644
index 0000000..d427efe
--- /dev/null
+++ b/example/README.md
@@ -0,0 +1,45 @@
+# Example using Intel Paillier Cryptosystem Library
+
+The README provides an example program for using the Intel Paillier Cryptosystem Library in an external application.
+
+## Installation
+
+To install the library,
+```bash
+cmake -S . -B build -DCMAKE_INSTALL_PREFIX=/path/to/install/ -DCMAKE_BUILD_TYPE=Release -DIPCL_TEST=OFF -DIPCL_BENCHMARK=OFF
+cmake --build build -j
+cmake --build build --target install
+```
+
+If ```CMAKE_INSTALL_PREFIX``` is not explicitly set, the library will automatically set to ```/opt/intel/ipcl``` by default.
+
+For more details about the build configuration options, please refer to the build instructions provided in the [README.md](../README.md).
+
+## Linking and Running Applications
+Before proceeding after the library is installed, it is useful to setup an environment variable to point to the installation location.
+```bash
+export IPCL_DIR=/path/to/install/
+```
+
+### Using g++/clang compiler
+In order to directly use `g++` or `clang++` to compile an example code, it can be done by:
+```bash
+# gcc
+g++ test.cpp -o test -L${IPCL_DIR}/lib -I${IPCL_DIR}/include -lipcl -fopenmp -lnuma -lcrypto
+
+# clang
+clang++ test.cpp -o test -L${IPCL_DIR}/lib -I${IPCL_DIR}/include -lipcl -fopenmp -lnuma -lcrypto
+```
+
+### CMake
+A more convenient way to use the library is via the `find_package` functionality in `CMake`.
+In your external applications, add the following lines to your `CMakeLists.txt`.
+
+```bash
+find_package(IPCL 1.1.4
+    HINTS ${IPCL_HINT_DIR}
+    REQUIRED)
+target_link_libraries(${TARGET} IPCL::ipcl)
+```
+
+If the library is installed globally, `IPCL_DIR` or `IPCL_HINT_DIR` flag is not needed. If `IPCL_DIR` is properly set, `IPCL_HINT_DIR` is not needed as well. Otherwise `IPCL_HINT_DIR` should be the directory containing `IPCLCOnfig.cmake`, under `${CMAKE_INSTALL_PREFIX}/lib/cmake/ipcl-1.1.4/`
diff --git a/example/test.cpp b/example/test.cpp
index 48a233d..5ac7a31 100644
--- a/example/test.cpp
+++ b/example/test.cpp
@@ -3,11 +3,10 @@
 
 #include <climits>
 #include <iostream>
+#include <ipcl/ipcl.hpp>
 #include <random>
 #include <vector>
 
-#include "ipcl/ipcl.hpp"
-
 int main() {
   const uint32_t num_values = 9;
 

From cba129802a1aed450438fa63f0746e4945ea299a Mon Sep 17 00:00:00 2001
From: sejunkim <sejun.kim@intel.com>
Date: Wed, 28 Sep 2022 23:04:51 -0700
Subject: [PATCH 8/9] Added examples documentation - Build and install
 instruction - Linking and compiling instruction - Usage examples

---
 README.md         |  25 ++++++----
 example/README.md | 120 +++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 123 insertions(+), 22 deletions(-)

diff --git a/README.md b/README.md
index d120387..bf43f6f 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,7 @@ Intel Paillier Cryptosystem Library is an open-source library which provides acc
     - [Prerequisites](#prerequisites)
     - [Dependencies](#dependencies)
     - [Instructions](#instructions)
+    - [Installing and Using Example](#installing-and-using-example)
   - [Testing and Benchmarking](#testing-and-benchmarking)
 - [Python Extension](#python-extension)
 - [Standardization](#standardization)
@@ -72,21 +73,25 @@ sudo yum install numactl-devel
 ### Instructions
 The library can be built using the following commands:
 ```bash
-export IPCL_DIR=$(pwd)
+export IPCL_ROOT=$(pwd)
 cmake -S . -B build -DCMAKE_BUILD_TYPE=Release
 cmake --build build -j
 ```
 
 It is possible to pass additional options to enable more features. The following table contains the current CMake options, default values are in bold.
 
-| CMake options           | Values    | Default | Comments                            |
-|-------------------------|-----------|---------|-------------------------------------|
-|`IPCL_TEST`              | ON/OFF    | ON      | unit-test                           |
-|`IPCL_BENCHMARK`         | ON/OFF    | ON      | benchmark                           |
-|`IPCL_ENABLE_OMP`        | ON/OFF    | ON      | enables OpenMP functionalities      |
-|`IPCL_THREAD_COUNT`      | Integer   | OFF     | The max number of threads           |
-|`IPCL_DOCS`              | ON/OFF    | OFF     | build doxygen documentation         |
-|`IPCL_SHARED`            | ON/OFF    | ON      | build shared library                |
+| CMake options            | Values    | Default | Comments                            |
+|--------------------------|-----------|---------|-------------------------------------|
+|`IPCL_TEST`               | ON/OFF    | ON      | unit-test                           |
+|`IPCL_BENCHMARK`          | ON/OFF    | ON      | benchmark                           |
+|`IPCL_ENABLE_OMP`         | ON/OFF    | ON      | enables OpenMP functionalities      |
+|`IPCL_THREAD_COUNT`       | Integer   | OFF     | explicitly set max number of threads|
+|`IPCL_DOCS`               | ON/OFF    | OFF     | build doxygen documentation         |
+|`IPCL_SHARED`             | ON/OFF    | ON      | build shared library                |
+|`IPCL_DETECT_IFMA_RUNTIME`| ON/OFF    | OFF     | detects AVX512IFMA during runtime   |
+
+### Installing and Using Example
+For installing and using the library externally, see [example/README.md]](./example/README.md).
 
 ## Testing and Benchmarking
 To run a set of unit tests via [Googletest](https://github.com/google/googletest), configure and build library with `-DIPCL_TEST=ON` (see [Instructions](#instructions)).
@@ -102,7 +107,7 @@ cmake --build build --target benchmark
 ```
 Setting the CMake flag ```-DIPCL_ENABLE_OMP=ON``` during configuration will use OpenMP for acceleration. Setting the value of `-DIPCL_THREAD_COUNT` will limit the maximum number of threads used by OpenMP (If set to OFF or 0, its actual value will be determined at run time).
 
-The executables are located at `${IPCL_DIR}/build/test/unittest_ipcl` and `${IPCL_DIR}/build/benchmark/bench_ipcl`.
+The executables are located at `${IPCL_ROOT}/build/test/unittest_ipcl` and `${IPCL_ROOT}/build/benchmark/bench_ipcl`.
 
 # Python Extension
 Alongside the Intel Paillier Cryptosystem Library, we provide a Python extension package utilizing this library as a backend. For installation and usage detail, refer to [Intel Paillier Cryptosystem Library - Python](https://github.com/intel/pailliercryptolib_python).
diff --git a/example/README.md b/example/README.md
index d427efe..0bfe98c 100644
--- a/example/README.md
+++ b/example/README.md
@@ -1,9 +1,24 @@
 # Example using Intel Paillier Cryptosystem Library
+This document provides an example program for using the Intel Paillier Cryptosystem Library in an external application.
+
+## Contents
+- [Example using Intel Paillier Cryptosystem Library](#example-using-intel-paillier-cryptosystem-library)
+  - [Contents](#contents)
+  - [Installation](#installation)
+  - [Linking and Running Applications](#linking-and-running-applications)
+    - [Building with CMake](#building-with-cmake)
+    - [Manually Compiling](#manually-compiling)
+  - [Using Intel Paillier Cryptosystem Library](#using-intel-paillier-cryptosystem-library)
+    - [Data handling](#data-handling)
+      - [```ipcl::PlainText``` Constructor](#ipclplaintext-constructor)
+      - [Accessing data](#accessing-data)
+    - [Key Generation](#key-generation)
+    - [Encryption and Decryption](#encryption-and-decryption)
+    - [HE Operations](#he-operations)
 
-The README provides an example program for using the Intel Paillier Cryptosystem Library in an external application.
 
-## Installation
 
+## Installation
 To install the library,
 ```bash
 cmake -S . -B build -DCMAKE_INSTALL_PREFIX=/path/to/install/ -DCMAKE_BUILD_TYPE=Release -DIPCL_TEST=OFF -DIPCL_BENCHMARK=OFF
@@ -16,12 +31,26 @@ If ```CMAKE_INSTALL_PREFIX``` is not explicitly set, the library will automatica
 For more details about the build configuration options, please refer to the build instructions provided in the [README.md](../README.md).
 
 ## Linking and Running Applications
+
 Before proceeding after the library is installed, it is useful to setup an environment variable to point to the installation location.
 ```bash
 export IPCL_DIR=/path/to/install/
 ```
 
-### Using g++/clang compiler
+### Building with CMake
+A more convenient way to use the library is via the `find_package` functionality in `CMake`.
+In your external applications, add the following lines to your `CMakeLists.txt`.
+
+```bash
+find_package(IPCL 1.1.4
+    HINTS ${IPCL_HINT_DIR}
+    REQUIRED)
+target_link_libraries(${TARGET} IPCL::ipcl)
+```
+
+If the library is installed globally, `IPCL_DIR` or `IPCL_HINT_DIR` flag is not needed. If `IPCL_DIR` is properly set, `IPCL_HINT_DIR` is not needed as well. Otherwise `IPCL_HINT_DIR` should be the directory containing `IPCLCOnfig.cmake`, under `${CMAKE_INSTALL_PREFIX}/lib/cmake/ipcl-1.1.4/`
+
+### Manually Compiling
 In order to directly use `g++` or `clang++` to compile an example code, it can be done by:
 ```bash
 # gcc
@@ -31,15 +60,82 @@ g++ test.cpp -o test -L${IPCL_DIR}/lib -I${IPCL_DIR}/include -lipcl -fopenmp -ln
 clang++ test.cpp -o test -L${IPCL_DIR}/lib -I${IPCL_DIR}/include -lipcl -fopenmp -lnuma -lcrypto
 ```
 
-### CMake
-A more convenient way to use the library is via the `find_package` functionality in `CMake`.
-In your external applications, add the following lines to your `CMakeLists.txt`.
 
-```bash
-find_package(IPCL 1.1.4
-    HINTS ${IPCL_HINT_DIR}
-    REQUIRED)
-target_link_libraries(${TARGET} IPCL::ipcl)
+## Using Intel Paillier Cryptosystem Library
+
+### Data handling
+The library uses a container - ```ipcl::PlainText``` for encryption inputs and decryption outputs as well as plaintext HE operations.
+
+#### ```ipcl::PlainText``` Constructor
+
+```C++
+// construct w/ unsigned int32
+uint32_t val1;
+ipcl::PlainText pt1(val1);
+
+// construct w/ unsigned int32 vector
+std::vector<uint32_t> val2;
+ipcl::PlainText pt2(val2);
+
+// construct w/ IPP-Crypto BigNumber
+BigNumber val3;
+ipcl::PlainText pt3(val3);
+
+// construct w/ IPP-Crypto BigNumber vector
+std::vector<BigNumber> val4;
+ipcl::PlainText pt4(val4);
 ```
+For more details about the IPP-Crypto ```BigNumber``` class, please refer to [Intel IPP Developer Reference](https://www.intel.com/content/www/us/en/develop/documentation/ipp-crypto-reference/top/appendix-a-support-functions-and-classes/classes-and-functions-used-in-examples/bignumber-class.html).
 
-If the library is installed globally, `IPCL_DIR` or `IPCL_HINT_DIR` flag is not needed. If `IPCL_DIR` is properly set, `IPCL_HINT_DIR` is not needed as well. Otherwise `IPCL_HINT_DIR` should be the directory containing `IPCLCOnfig.cmake`, under `${CMAKE_INSTALL_PREFIX}/lib/cmake/ipcl-1.1.4/`
+#### Accessing data
+The library provides several methods to handle and access the data included in the ```ipcl::PlainText``` container.
+```C++
+ipcl::PlainText pt(raw_data);
+
+BigNumber ret1 = pt[1]; // idx
+std::vector<BigNumber> ret2 = pt; // convert
+pt.insert(pos, val); // insert BigNumber to position
+pt.remove(pos, length); // remove element (default length=1)
+ipcl::PlainText pt_copy = pt; // copy
+pt_copy.clear(); // empty the container
+```
+FOr more details, please refer to the [```base_text.hpp```](../ipcl/include/ipcl/base_text.hpp) and [```plaintext.hpp```](../ipcl/include/ipcl/plaintext.hpp).
+
+### Key Generation
+The public key and private key pair can be generated by using the ```ipcl::generateKeypair``` function.
+```C++
+// key.pub_key, key.priv_key
+ipcl::keyPair key = ipcl::generateKeypair(2048, true);
+// After computation, need to delete the key objects
+delete key.pub_key;
+delete key.priv_key;
+```
+
+### Encryption and Decryption
+The public key is used to encrypt ```ipcl::PlainText``` objects for ```ipcl::CipherText``` outputs.
+In the same way, the private key is used to decrypt ```ipcl::CipherText``` objects for ```ipcl::PlainText``` outputs.
+```C++
+ipcl::PlainText pt(raw_data);
+ipcl::CipherText ct = key.pub_key->encrypt(pt);
+ipcl::PlainText dec_pt = key.priv_key->decrypt(ct);
+```
+
+### HE Operations
+Since the Intel Paillier Cryptosystem Library being a partially homomorphic encryption scheme, while addition is supports both ciphertext operands, multiplication only supports single ciphertext operand.
+
+```C++
+// setup
+ipcl::PlainText a, b;
+
+ipcl::CipherText ct_a = key.pub_key->encrypt(a);
+ipcl::CipherText ct_b = key.pub_key->encrypt(b);
+
+// Addition (all three end up being same values after decryption)
+ipcl::CipherText ct_c1 = ct_a + ct_b; // ciphertext + ciphertext
+ipcl::CipherText ct_c2 = ct_a + b; // ciphertext + plaintext
+ipcl::CipherText ct_c3 = ct_b + a; // ciphertext + plaintext
+
+// Multiplication
+ipcl::CipherText ct_d1 = ct_a * b; // ciphertext * plaintext
+ipcl::CipherText ct_d2 = ct_b * a;
+```

From f395ced4bce6074badbfb274bef26d0755651f88 Mon Sep 17 00:00:00 2001
From: sejunkim <sejun.kim@intel.com>
Date: Wed, 28 Sep 2022 23:09:03 -0700
Subject: [PATCH 9/9] Updated README to include AVX512IFMA runtime detection
 option

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index bf43f6f..02a98a4 100644
--- a/README.md
+++ b/README.md
@@ -90,6 +90,8 @@ It is possible to pass additional options to enable more features. The following
 |`IPCL_SHARED`             | ON/OFF    | ON      | build shared library                |
 |`IPCL_DETECT_IFMA_RUNTIME`| ON/OFF    | OFF     | detects AVX512IFMA during runtime   |
 
+If ```IPCL_DETECT_IFMA_RUNTIME``` flag is set to ```ON```, it will determine whether the system supports the AVX512IFMA instructions on runtime. It is still possible to disable IFMA exclusive feature (multi-buffer modular exponentiation) during runtime by setting up the environment variable ```IPCL_DISABLE_AVX512IFMA=1```.
+
 ### Installing and Using Example
 For installing and using the library externally, see [example/README.md]](./example/README.md).