QAT support for AES-GCM

This patch adds support for acceleration of AES-GCM encryption
with Intel Quick Assist Technology.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Chengfeix Zhu <chengfeix.zhu@intel.com>
Signed-off-by: Weigang Li <weigang.li@intel.com>
Signed-off-by: Tom Caputi <tcaputi@datto.com>
Closes openzfs#7282

man/man5/zfs-module-parameters.5

@@ -2352,9 +2352,9 @@ Default value: \fB1\fR.
 \fBzfs_qat_disable\fR (int)
 .ad
 .RS 12n
-This tunable disables qat hardware acceleration for gzip compression.
-It is available only if qat acceleration is compiled in and qat driver
-is present.
+This tunable disables qat hardware acceleration for gzip compression and.
+AES-GCM encryption. It is available only if qat acceleration is compiled in
+and the qat driver is present.
 .sp
 Use \fB1\fR for yes and \fB0\fR for no (default).
 .RE

module/zfs/Makefile.in

@@ -132,7 +132,9 @@ $(MODULE)-objs += zrlock.o
 $(MODULE)-objs += zvol.o
 $(MODULE)-objs += dsl_destroy.o
 $(MODULE)-objs += dsl_userhold.o
+$(MODULE)-objs += qat.o
 $(MODULE)-objs += qat_compress.o
+$(MODULE)-objs += qat_crypt.o
 
 # Suppress incorrect warnings from versions of objtool which are not
 # aware of x86 EVEX prefix instructions used for AVX512.

module/zfs/gzip.c

@@ -28,7 +28,7 @@
 
 #include <sys/debug.h>
 #include <sys/types.h>
-#include "qat_compress.h"
+#include "qat.h"
 
 #ifdef _KERNEL
 
@@ -58,7 +58,7 @@ gzip_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n)
 	ASSERT(d_len <= s_len);
 
 	/* check if hardware accelerator can be used */
-	if (qat_use_accel(s_len)) {
+	if (qat_dc_use_accel(s_len)) {
 		if (qat_compress(QAT_COMPRESS, s_start,
 		    s_len, d_start, d_len, &dstlen) == CPA_STATUS_SUCCESS)
 			return ((size_t)dstlen);
@@ -85,7 +85,7 @@ gzip_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n)
 	ASSERT(d_len >= s_len);
 
 	/* check if hardware accelerator can be used */
-	if (qat_use_accel(d_len)) {
+	if (qat_dc_use_accel(d_len)) {
 		if (qat_compress(QAT_DECOMPRESS, s_start, s_len,
 		    d_start, d_len, &dstlen) == CPA_STATUS_SUCCESS)
 			return (0);

module/zfs/qat.c

@@ -0,0 +1,102 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+#if defined(_KERNEL) && defined(HAVE_QAT)
+#include <sys/zfs_context.h>
+#include "qat.h"
+
+qat_stats_t qat_stats = {
+	{ "comp_requests",			KSTAT_DATA_UINT64 },
+	{ "comp_total_in_bytes",		KSTAT_DATA_UINT64 },
+	{ "comp_total_out_bytes",		KSTAT_DATA_UINT64 },
+	{ "decomp_requests",			KSTAT_DATA_UINT64 },
+	{ "decomp_total_in_bytes",		KSTAT_DATA_UINT64 },
+	{ "decomp_total_out_bytes",		KSTAT_DATA_UINT64 },
+	{ "dc_fails",				KSTAT_DATA_UINT64 },
+	{ "encrypt_requests",			KSTAT_DATA_UINT64 },
+	{ "encrypt_total_in_bytes",		KSTAT_DATA_UINT64 },
+	{ "encrypt_total_out_bytes",		KSTAT_DATA_UINT64 },
+	{ "decrypt_requests",			KSTAT_DATA_UINT64 },
+	{ "decrypt_total_in_bytes",		KSTAT_DATA_UINT64 },
+	{ "decrypt_total_out_bytes",		KSTAT_DATA_UINT64 },
+	{ "crypt_fails",			KSTAT_DATA_UINT64 },
+};
+
+static kstat_t *qat_ksp = NULL;
+int zfs_qat_disable = 0;
+
+CpaStatus
+qat_mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes)
+{
+	*pp_mem_addr = kmalloc(size_bytes, GFP_KERNEL);
+	if (*pp_mem_addr == NULL)
+		return (CPA_STATUS_RESOURCE);
+	return (CPA_STATUS_SUCCESS);
+}
+
+void
+qat_mem_free_contig(void **pp_mem_addr)
+{
+	if (*pp_mem_addr != NULL) {
+		kfree(*pp_mem_addr);
+		*pp_mem_addr = NULL;
+	}
+}
+
+int
+qat_init(void)
+{
+	int ret;
+
+	ret = qat_dc_init();
+	if (ret != 0)
+		return (ret);
+
+	ret = qat_crypt_init();
+	if (ret != 0) {
+		qat_dc_fini();
+		return (ret);
+	}
+
+	qat_ksp = kstat_create("zfs", 0, "qat", "misc",
+	    KSTAT_TYPE_NAMED, sizeof (qat_stats) / sizeof (kstat_named_t),
+	    KSTAT_FLAG_VIRTUAL);
+	if (qat_ksp != NULL) {
+		qat_ksp->ks_data = &qat_stats;
+		kstat_install(qat_ksp);
+	}
+
+	return (0);
+}
+
+void
+qat_fini(void)
+{
+	if (qat_ksp != NULL) {
+		kstat_delete(qat_ksp);
+		qat_ksp = NULL;
+	}
+
+	qat_crypt_fini();
+	qat_dc_fini();
+}
+
+#endif

module/zfs/qat.h

@@ -0,0 +1,176 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+#ifndef	_SYS_QAT_H
+#define	_SYS_QAT_H
+
+typedef enum qat_compress_dir {
+	QAT_DECOMPRESS = 0,
+	QAT_COMPRESS = 1,
+} qat_compress_dir_t;
+
+typedef enum qat_encrypt_dir {
+	QAT_DECRYPT = 0,
+	QAT_ENCRYPT = 1,
+} qat_encrypt_dir_t;
+
+
+#if defined(_KERNEL) && defined(HAVE_QAT)
+#include <sys/zio.h>
+#include <sys/crypto/api.h>
+#include "cpa.h"
+#include "dc/cpa_dc.h"
+#include "lac/cpa_cy_sym.h"
+
+/*
+ * Timeout - no response from hardware after 0.5 seconds
+ */
+#define	QAT_TIMEOUT_MS		500
+
+/*
+ * The minimal and maximal buffer size, which are not restricted
+ * in the QAT hardware, but with the input buffer size between 4KB
+ * and 128KB, the hardware can provide the optimal performance.
+ */
+#define	QAT_MIN_BUF_SIZE	(4*1024)
+#define	QAT_MAX_BUF_SIZE	(128*1024)
+
+/*
+ * Used for qat kstat.
+ */
+typedef struct qat_stats {
+	/*
+	 * Number of jobs submitted to qat compression engine.
+	 */
+	kstat_named_t comp_requests;
+	/*
+	 * Total bytes sent to qat compression engine.
+	 */
+	kstat_named_t comp_total_in_bytes;
+	/*
+	 * Total bytes output from qat compression engine.
+	 */
+	kstat_named_t comp_total_out_bytes;
+	/*
+	 * Number of jobs submitted to qat de-compression engine.
+	 */
+	kstat_named_t decomp_requests;
+	/*
+	 * Total bytes sent to qat de-compression engine.
+	 */
+	kstat_named_t decomp_total_in_bytes;
+	/*
+	 * Total bytes output from qat de-compression engine.
+	 */
+	kstat_named_t decomp_total_out_bytes;
+	/*
+	 * Number of fails in the qat compression / decompression engine.
+	 * Note: when qat fail happens, it doesn't mean a critical hardware
+	 * issue. Sometimes it is because the output buffer is not big enough.
+	 * The compression job will be transfered to gzip software
+	 * implementation, so the functionality of ZFS is not impacted.
+	 */
+	kstat_named_t dc_fails;
+
+	/*
+	 * Number of jobs submitted to qat encryption engine.
+	 */
+	kstat_named_t encrypt_requests;
+	/*
+	 * Total bytes sent to qat encryption engine.
+	 */
+	kstat_named_t encrypt_total_in_bytes;
+	/*
+	 * Total bytes output from qat encryption engine.
+	 */
+	kstat_named_t encrypt_total_out_bytes;
+	/*
+	 * Number of jobs submitted to qat decryption engine.
+	 */
+	kstat_named_t decrypt_requests;
+	/*
+	 * Total bytes sent to qat decryption engine.
+	 */
+	kstat_named_t decrypt_total_in_bytes;
+	/*
+	 * Total bytes output from qat decryption engine.
+	 */
+	kstat_named_t decrypt_total_out_bytes;
+	/*
+	 * Number of fails in the qat encryption / decryption engine.
+	 * Note: when qat fail happens, it doesn't mean a critical hardware
+	 * issue. Sometimes it is because the output buffer is not big enough.
+	 * The encryption job will be transfered to the software implementation,
+	 * so the functionality of ZFS is not impacted.
+	 */
+	kstat_named_t crypt_fails;
+} qat_stats_t;
+
+#define	QAT_STAT_INCR(stat, val) \
+	atomic_add_64(&qat_stats.stat.value.ui64, (val))
+#define	QAT_STAT_BUMP(stat) \
+	QAT_STAT_INCR(stat, 1)
+
+extern qat_stats_t qat_stats;
+extern int zfs_qat_disable;
+
+/* inlined for performance */
+static inline struct page *
+qat_mem_to_page(void *addr)
+{
+	if (!is_vmalloc_addr(addr))
+		return (virt_to_page(addr));
+
+	return (vmalloc_to_page(addr));
+}
+
+CpaStatus qat_mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes);
+void qat_mem_free_contig(void **pp_mem_addr);
+#define	QAT_PHYS_CONTIG_ALLOC(pp_mem_addr, size_bytes)	\
+	qat_mem_alloc_contig((void *)(pp_mem_addr), (size_bytes))
+#define	QAT_PHYS_CONTIG_FREE(p_mem_addr)	\
+	qat_mem_free_contig((void *)&(p_mem_addr))
+
+extern int qat_dc_init(void);
+extern void qat_dc_fini(void);
+extern int qat_crypt_init(void);
+extern void qat_crypt_fini(void);
+extern int qat_init(void);
+extern void qat_fini(void);
+
+extern boolean_t qat_dc_use_accel(size_t s_len);
+extern boolean_t qat_crypt_use_accel(size_t s_len);
+extern int qat_compress(qat_compress_dir_t dir, char *src, int src_len,
+    char *dst, int dst_len, size_t *c_len);
+extern int qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
+    uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf,
+    crypto_key_t *key, uint64_t crypt, uint32_t enc_len);
+#else
+#define	CPA_STATUS_SUCCESS					0
+#define	qat_init()
+#define	qat_fini()
+#define	qat_dc_use_accel(s_len)					0
+#define	qat_crypt_use_accel(s_len)				0
+#define	qat_compress(dir, s, sl, d, dl, cl)			0
+#define	qat_crypt(dir, s, d, a, al, i, db, k, c, el)		0
+#endif
+
+#endif /* _SYS_QAT_H */