Add SHA256 and SHA512 fast implementations

Signed-off-by: Jan Kasiak <jan@cybojanek.net>

COPYRIGHT

@@ -19,6 +19,7 @@ notable exceptions and their respective licenses include:
   * AES Implementation: module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.gladman
   * AES Implementation: module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.openssl
   * PBKDF2 Implementation: lib/libzfs/THIRDPARTYLICENSE.openssl
+  * SHA2 Implementation: module/icp/asm-x86_64/sha2/THIRDPARTYLICENSE.intel
   * SPL Implementation: module/os/linux/spl/THIRDPARTYLICENSE.gplv2
   * GCM Implementation: module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.cryptogams
   * GCM Implementation: module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.openssl

Makefile.am

@@ -43,6 +43,8 @@ EXTRA_DIST += module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.cryptogams
 EXTRA_DIST += module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.cryptogams.descrip
 EXTRA_DIST += module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.openssl
 EXTRA_DIST += module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.openssl.descrip
+EXTRA_DIST += module/icp/asm-x86_64/sha2/THIRDPARTYLICENSE.intel
+EXTRA_DIST += module/icp/asm-x86_64/sha2/THIRDPARTYLICENSE.intel.descrip
 EXTRA_DIST += module/os/linux/spl/THIRDPARTYLICENSE.gplv2
 EXTRA_DIST += module/os/linux/spl/THIRDPARTYLICENSE.gplv2.descrip
 EXTRA_DIST += module/zfs/THIRDPARTYLICENSE.cityhash

lib/libicp/Makefile.am

@@ -19,7 +19,13 @@ ASM_SOURCES_AS = \
 	asm-x86_64/modes/ghash-x86_64.S \
 	asm-x86_64/sha1/sha1-x86_64.S \
 	asm-x86_64/sha2/sha256_impl.S \
-	asm-x86_64/sha2/sha512_impl.S
+	asm-x86_64/sha2/sha256_avx.S \
+	asm-x86_64/sha2/sha256_ssse3.S \
+	asm-x86_64/sha2/sha256_ni.S \
+	asm-x86_64/sha2/sha512_impl.S \
+	asm-x86_64/sha2/sha512_avx.S \
+	asm-x86_64/sha2/sha512_avx2.S \
+	asm-x86_64/sha2/sha512_ssse3.S
 else
 ASM_SOURCES_C =
 ASM_SOURCES_AS =

module/icp/Makefile.in

@@ -58,7 +58,13 @@ $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/modes/aesni-gcm-x86_64.o
 $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/modes/ghash-x86_64.o
 $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha1/sha1-x86_64.o
 $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha2/sha256_impl.o
+$(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha2/sha256_avx.o
+$(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha2/sha256_ssse3.o
+$(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha2/sha256_ni.o
 $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha2/sha512_impl.o
+$(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha2/sha512_avx.o
+$(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha2/sha512_avx2.o
+$(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha2/sha512_ssse3.o
 
 $(MODULE)-$(CONFIG_X86) += algs/modes/gcm_pclmulqdq.o
 $(MODULE)-$(CONFIG_X86) += algs/aes/aes_impl_aesni.o

module/icp/algs/sha2/sha2.c

@@ -57,6 +57,8 @@ static void Encode64(uint8_t *, uint64_t *, size_t);
 /* userspace only supports the generic version */
 #if	defined(_KERNEL)
 
+#include <sys/simd.h>
+
 typedef void (*sha256_block_f)(uint32_t *state, const void *in, size_t num);
 typedef void (*sha512_block_f)(uint64_t *state, const void *in, size_t num);
 
@@ -90,12 +92,57 @@ static alg_impl_ops_t sha256_x86_64 = {
 	sha256_x86_64_transform, alg_impl_will_always_work, 1, "x86_64"};
 #endif
 
+#if defined(__amd64) && defined(HAVE_AVX)
+static boolean_t
+sha256_avx_will_work(void)
+{
+	return (kfpu_allowed() && zfs_avx_available());
+}
+
+extern void sha256_avx_transform(SHA2_CTX *ctx, const void *in, size_t num);
+static alg_impl_ops_t sha256_avx = {
+	sha256_avx_transform, sha256_avx_will_work, 10, "sha-avx"};
+#endif
+
+#if defined(__amd64) && defined(HAVE_SSSE3)
+static boolean_t
+sha256_ssse3_will_work(void)
+{
+	return (kfpu_allowed() && zfs_ssse3_available());
+}
+
+extern void sha256_ssse3_transform(SHA2_CTX *ctx, const void *in, size_t num);
+static alg_impl_ops_t sha256_ssse3 = {
+	sha256_ssse3_transform, sha256_ssse3_will_work, 30, "sha-ssse3"};
+#endif
+
+#if defined(__amd64) && defined(HAVE_SHA)
+static boolean_t
+sha256_ni_will_work(void)
+{
+	return (kfpu_allowed() && zfs_sha_available());
+}
+
+extern void sha256_ni_transform(SHA2_CTX *ctx, const void *in, size_t num);
+static alg_impl_ops_t sha256_ni = {
+	sha256_ni_transform, sha256_ni_will_work, 40, "sha-ni"};
+#endif
+
 /* All compiled in implementations */
 static const alg_impl_ops_t *sha256_all_impl[] = {
 	&sha256_impl_generic,
 #if defined(__amd64)
 	&sha256_x86_64,
 #endif
+#if defined(__amd64) && defined(HAVE_AVX)
+	&sha256_avx,
+#endif
+#if defined(__amd64) && defined(HAVE_SSSE3)
+	&sha256_ssse3,
+#endif
+#if defined(__amd64) && defined(HAVE_SHA)
+	&sha256_ni,
+#endif
 };
 
 static alg_impl_ops_t *sha256_supp_impl[ARRAY_SIZE(sha256_all_impl)];
@@ -139,12 +186,57 @@ static alg_impl_ops_t sha512_x86_64 = {
 	sha512_x86_64_transform, alg_impl_will_always_work, 1, "x86_64"};
 #endif
 
+#if defined(__amd64) && defined(HAVE_AVX)
+static boolean_t
+sha512_avx_will_work(void)
+{
+	return (kfpu_allowed() && zfs_avx_available());
+}
+
+extern void sha512_avx_transform(SHA2_CTX *ctx, const void *in, size_t num);
+static alg_impl_ops_t sha512_avx = {
+	sha512_avx_transform, sha512_avx_will_work, 10, "sha-avx"};
+#endif
+
+#if defined(__amd64) && defined(HAVE_AVX2)
+static boolean_t
+sha512_avx2_will_work(void)
+{
+	return (kfpu_allowed() && zfs_avx2_available());
+}
+
+extern void sha512_avx2_transform(SHA2_CTX *ctx, const void *in, size_t num);
+static alg_impl_ops_t sha512_avx2 = {
+	sha512_avx2_transform, sha512_avx2_will_work, 20, "sha-avx2"};
+#endif
+
+#if defined(__amd64) && defined(HAVE_SSSE3)
+static boolean_t
+sha512_ssse3_will_work(void)
+{
+	return (kfpu_allowed() && zfs_ssse3_available());
+}
+
+extern void sha512_ssse3_transform(SHA2_CTX *ctx, const void *in, size_t num);
+static alg_impl_ops_t sha512_ssse3 = {
+	sha512_ssse3_transform, sha512_ssse3_will_work, 30, "sha-ssse3"};
+#endif
+
 /* All compiled in implementations */
 static const alg_impl_ops_t *sha512_all_impl[] = {
 	&sha512_impl_generic,
 #if defined(__amd64)
 	&sha512_x86_64,
 #endif
+#if defined(__amd64) && defined(HAVE_AVX)
+	&sha512_avx,
+#endif
+#if defined(__amd64) && defined(HAVE_AVX2)
+	&sha512_avx2,
+#endif
+#if defined(__amd64) && defined(HAVE_SSSE3)
+	&sha512_ssse3,
+#endif
 };
 
 static alg_impl_ops_t *sha512_supp_impl[ARRAY_SIZE(sha512_all_impl)];

module/icp/asm-x86_64/sha2/THIRDPARTYLICENSE.intel

@@ -0,0 +1,118 @@
+ ---------------------------------------------------------------------------
+
+ Implement fast SHA-256 with AVX1 instructions. (x86_64)
+ Implement fast SHA-256 with AVX2 instructions. (x86_64)
+ Implement fast SHA-256 with SSSE3 instructions. (x86_64)
+
+ Copyright (C) 2013 Intel Corporation.
+
+ Authors:
+     James Guilford <james.guilford@intel.com>
+     Kirk Yap <kirk.s.yap@intel.com>
+     Tim Chen <tim.c.chen@linux.intel.com>
+
+ This software is available to you under a choice of one of two
+ licenses.  You may choose to be licensed under the terms of the GNU
+ General Public License (GPL) Version 2, available from the file
+ COPYING in the main directory of this source tree, or the
+ OpenIB.org BSD license below:
+
+     Redistribution and use in source and binary forms, with or
+     without modification, are permitted provided that the following
+     conditions are met:
+
+      - Redistributions of source code must retain the above
+        copyright notice, this list of conditions and the following
+        disclaimer.
+
+      - Redistributions in binary form must reproduce the above
+        copyright notice, this list of conditions and the following
+        disclaimer in the documentation and/or other materials
+        provided with the distribution.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+
+ ---------------------------------------------------------------------------
+
+ Implement fast SHA-256 with NI instructions. (x86_64)
+
+ BSD LICENSE
+
+ Copyright(c) 2015 Intel Corporation.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+  * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+  * Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+  * Neither the name of Intel Corporation nor the names of its
+    contributors may be used to endorse or promote products derived
+    from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ ---------------------------------------------------------------------------
+
+ Implement fast SHA-512 with AVX instructions. (x86_64)
+ Implement fast SHA-512 with AVX2 instructions. (x86_64)
+ Implement fast SHA-512 with SSSE3 instructions. (x86_64)
+
+ Copyright (C) 2013 Intel Corporation.
+
+ Authors:
+     James Guilford <james.guilford@intel.com>
+     Kirk Yap <kirk.s.yap@intel.com>
+     David Cote <david.m.cote@intel.com>
+     Tim Chen <tim.c.chen@linux.intel.com>
+
+ This software is available to you under a choice of one of two
+ licenses.  You may choose to be licensed under the terms of the GNU
+ General Public License (GPL) Version 2, available from the file
+ COPYING in the main directory of this source tree, or the
+ OpenIB.org BSD license below:
+
+     Redistribution and use in source and binary forms, with or
+     without modification, are permitted provided that the following
+     conditions are met:
+
+      - Redistributions of source code must retain the above
+        copyright notice, this list of conditions and the following
+        disclaimer.
+
+      - Redistributions in binary form must reproduce the above
+        copyright notice, this list of conditions and the following
+        disclaimer in the documentation and/or other materials
+        provided with the distribution.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+
+ ---------------------------------------------------------------------------

module/icp/asm-x86_64/sha2/THIRDPARTYLICENSE.intel.descrip

@@ -0,0 +1 @@
+PORTIONS OF SHA2 FUNCTIONALITY