Skip to content

Commit

Permalink
Use fletcher_4 routines natively with abd_iterate_func()
Browse files Browse the repository at this point in the history 
This patch adds the necessary infrastructure for ABD to make use
of the vectorized fletcher 4 routines.

- export ABD compatible interface from fletcher_4
- add ABD fletcher_4 tests for data and metadata ABD types.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Original-patch-by: Gvozden Neskovic <neskovic@gmail.com>
Signed-off-by: David Quigley <david.quigley@intel.com>
Closes #5589
  • Loading branch information
@dpquigl @behlendorf
dpquigl authored and behlendorf committed Feb 1, 2017
1 parent 544b805 commit 2fe36b0
Show file tree
Hide file tree
Showing 4 changed files with 167 additions and 8 deletions.
28 changes: 28 additions & 0 deletions cmd/ztest/ztest.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5680,18 +5680,25 @@ ztest_fletcher(ztest_ds_t *zd, uint64_t id)
while (gethrtime() <= end) {
int run_count = 100;
void *buf;
struct abd *abd_data, *abd_meta;
uint32_t size;
int *ptr;
int i;
zio_cksum_t zc_ref;
zio_cksum_t zc_ref_byteswap;

size = ztest_random_blocksize();

buf = umem_alloc(size, UMEM_NOFAIL);
abd_data = abd_alloc(size, B_FALSE);
abd_meta = abd_alloc(size, B_TRUE);

for (i = 0, ptr = buf; i < size / sizeof (*ptr); i++, ptr++)
*ptr = ztest_random(UINT_MAX);

abd_copy_from_buf_off(abd_data, buf, 0, size);
abd_copy_from_buf_off(abd_meta, buf, 0, size);

VERIFY0(fletcher_4_impl_set("scalar"));
fletcher_4_native(buf, size, NULL, &zc_ref);
fletcher_4_byteswap(buf, size, NULL, &zc_ref_byteswap);
Expand All @@ -5707,9 +5714,30 @@ ztest_fletcher(ztest_ds_t *zd, uint64_t id)
VERIFY0(bcmp(&zc, &zc_ref, sizeof (zc)));
VERIFY0(bcmp(&zc_byteswap, &zc_ref_byteswap,
sizeof (zc_byteswap)));

/* Test ABD - data */
abd_fletcher_4_byteswap(abd_data, size, NULL,
&zc_byteswap);
abd_fletcher_4_native(abd_data, size, NULL, &zc);

VERIFY0(bcmp(&zc, &zc_ref, sizeof (zc)));
VERIFY0(bcmp(&zc_byteswap, &zc_ref_byteswap,
sizeof (zc_byteswap)));

/* Test ABD - metadata */
abd_fletcher_4_byteswap(abd_meta, size, NULL,
&zc_byteswap);
abd_fletcher_4_native(abd_meta, size, NULL, &zc);

VERIFY0(bcmp(&zc, &zc_ref, sizeof (zc)));
VERIFY0(bcmp(&zc_byteswap, &zc_ref_byteswap,
sizeof (zc_byteswap)));

}

umem_free(buf, size);
abd_free(abd_data);
abd_free(abd_meta);
}
}

Expand Down
27 changes: 27 additions & 0 deletions include/sys/zio_checksum.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

#include <sys/zio.h>
#include <zfeature_common.h>
#include <zfs_fletcher.h>

#ifdef __cplusplus
extern "C" {
Expand Down Expand Up @@ -57,6 +58,28 @@ typedef enum zio_checksum_flags {
ZCHECKSUM_FLAG_NOPWRITE = (1 << 5)
} zio_checksum_flags_t;

typedef enum {
ZIO_CHECKSUM_NATIVE,
ZIO_CHECKSUM_BYTESWAP
} zio_byteorder_t;

typedef struct zio_abd_checksum_data {
zio_byteorder_t acd_byteorder;
fletcher_4_ctx_t *acd_ctx;
zio_cksum_t *acd_zcp;
void *acd_private;
} zio_abd_checksum_data_t;

typedef void zio_abd_checksum_init_t(zio_abd_checksum_data_t *);
typedef void zio_abd_checksum_fini_t(zio_abd_checksum_data_t *);
typedef int zio_abd_checksum_iter_t(void *, size_t, void *);

typedef const struct zio_abd_checksum_func {
zio_abd_checksum_init_t *acf_init;
zio_abd_checksum_fini_t *acf_fini;
zio_abd_checksum_iter_t *acf_iter;
} zio_abd_checksum_func_t;

/*
* Information about each checksum function.
*/
Expand Down Expand Up @@ -99,6 +122,10 @@ extern zio_checksum_t abd_checksum_edonr_byteswap;
extern zio_checksum_tmpl_init_t abd_checksum_edonr_tmpl_init;
extern zio_checksum_tmpl_free_t abd_checksum_edonr_tmpl_free;

extern zio_abd_checksum_func_t fletcher_4_abd_ops;
extern zio_checksum_t abd_fletcher_4_native;
extern zio_checksum_t abd_fletcher_4_byteswap;

extern int zio_checksum_equal(spa_t *, blkptr_t *, enum zio_checksum,
void *, uint64_t, uint64_t, zio_bad_cksum_t *);
extern void zio_checksum_compute(zio_t *, enum zio_checksum,
Expand Down
87 changes: 85 additions & 2 deletions module/zcommon/zfs_fletcher.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@
#include <sys/zfs_context.h>
#include <zfs_fletcher.h>

#define FLETCHER_MIN_SIMD_SIZE 64

static void fletcher_4_scalar_init(fletcher_4_ctx_t *ctx);
static void fletcher_4_scalar_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp);
Expand Down Expand Up @@ -456,7 +457,7 @@ void
fletcher_4_native(const void *buf, uint64_t size,
const void *ctx_template, zio_cksum_t *zcp)
{
const uint64_t p2size = P2ALIGN(size, 64);
const uint64_t p2size = P2ALIGN(size, FLETCHER_MIN_SIMD_SIZE);

ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t)));

Expand Down Expand Up @@ -498,7 +499,7 @@ void
fletcher_4_byteswap(const void *buf, uint64_t size,
const void *ctx_template, zio_cksum_t *zcp)
{
const uint64_t p2size = P2ALIGN(size, 64);
const uint64_t p2size = P2ALIGN(size, FLETCHER_MIN_SIMD_SIZE);

ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t)));

Expand Down Expand Up @@ -778,6 +779,87 @@ fletcher_4_fini(void)
}
}

/* ABD adapters */

static void
abd_fletcher_4_init(zio_abd_checksum_data_t *cdp)
{
const fletcher_4_ops_t *ops = fletcher_4_impl_get();
cdp->acd_private = (void *) ops;

if (cdp->acd_byteorder == ZIO_CHECKSUM_NATIVE)
ops->init_native(cdp->acd_ctx);
else
ops->init_byteswap(cdp->acd_ctx);
}

static void
abd_fletcher_4_fini(zio_abd_checksum_data_t *cdp)
{
fletcher_4_ops_t *ops = (fletcher_4_ops_t *)cdp->acd_private;

ASSERT(ops);

if (cdp->acd_byteorder == ZIO_CHECKSUM_NATIVE)
ops->fini_native(cdp->acd_ctx, cdp->acd_zcp);
else
ops->fini_byteswap(cdp->acd_ctx, cdp->acd_zcp);
}

static void
abd_fletcher_4_simd2scalar(boolean_t native, void *data, size_t size,
zio_abd_checksum_data_t *cdp)
{
zio_cksum_t *zcp = cdp->acd_zcp;

ASSERT3U(size, <, FLETCHER_MIN_SIMD_SIZE);

abd_fletcher_4_fini(cdp);
cdp->acd_private = (void *)&fletcher_4_scalar_ops;

if (native)
fletcher_4_incremental_native(data, size, zcp);
else
fletcher_4_incremental_byteswap(data, size, zcp);
}

static int
abd_fletcher_4_iter(void *data, size_t size, void *private)
{
zio_abd_checksum_data_t *cdp = (zio_abd_checksum_data_t *)private;
fletcher_4_ctx_t *ctx = cdp->acd_ctx;
fletcher_4_ops_t *ops = (fletcher_4_ops_t *)cdp->acd_private;
boolean_t native = cdp->acd_byteorder == ZIO_CHECKSUM_NATIVE;
uint64_t asize = P2ALIGN(size, FLETCHER_MIN_SIMD_SIZE);

ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t)));

if (asize > 0) {
if (native)
ops->compute_native(ctx, data, asize);
else
ops->compute_byteswap(ctx, data, asize);

size -= asize;
data = (char *)data + asize;
}

if (size > 0) {
ASSERT3U(size, <, FLETCHER_MIN_SIMD_SIZE);
/* At this point we have to switch to scalar impl */
abd_fletcher_4_simd2scalar(native, data, size, cdp);
}

return (0);
}

zio_abd_checksum_func_t fletcher_4_abd_ops = {
.acf_init = abd_fletcher_4_init,
.acf_fini = abd_fletcher_4_fini,
.acf_iter = abd_fletcher_4_iter
};


#if defined(_KERNEL) && defined(HAVE_SPL)
#include <linux/mod_compat.h>

Expand Down Expand Up @@ -829,4 +911,5 @@ EXPORT_SYMBOL(fletcher_4_native_varsize);
EXPORT_SYMBOL(fletcher_4_byteswap);
EXPORT_SYMBOL(fletcher_4_incremental_native);
EXPORT_SYMBOL(fletcher_4_incremental_byteswap);
EXPORT_SYMBOL(fletcher_4_abd_ops);
#endif
33 changes: 27 additions & 6 deletions module/zfs/zio_checksum.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,24 +119,45 @@ abd_fletcher_2_byteswap(abd_t *abd, uint64_t size,
fletcher_2_incremental_byteswap, zcp);
}

static inline void
abd_fletcher_4_impl(abd_t *abd, uint64_t size, zio_abd_checksum_data_t *acdp)
{
fletcher_4_abd_ops.acf_init(acdp);
abd_iterate_func(abd, 0, size, fletcher_4_abd_ops.acf_iter, acdp);
fletcher_4_abd_ops.acf_fini(acdp);
}

/*ARGSUSED*/
void
abd_fletcher_4_native(abd_t *abd, uint64_t size,
const void *ctx_template, zio_cksum_t *zcp)
{
fletcher_init(zcp);
(void) abd_iterate_func(abd, 0, size,
fletcher_4_incremental_native, zcp);
fletcher_4_ctx_t ctx;

zio_abd_checksum_data_t acd = {
.acd_byteorder = ZIO_CHECKSUM_NATIVE,
.acd_zcp = zcp,
.acd_ctx = &ctx
};

abd_fletcher_4_impl(abd, size, &acd);

}

/*ARGSUSED*/
void
abd_fletcher_4_byteswap(abd_t *abd, uint64_t size,
const void *ctx_template, zio_cksum_t *zcp)
{
fletcher_init(zcp);
(void) abd_iterate_func(abd, 0, size,
fletcher_4_incremental_byteswap, zcp);
fletcher_4_ctx_t ctx;

zio_abd_checksum_data_t acd = {
.acd_byteorder = ZIO_CHECKSUM_BYTESWAP,
.acd_zcp = zcp,
.acd_ctx = &ctx
};

abd_fletcher_4_impl(abd, size, &acd);
}

zio_checksum_info_t zio_checksum_table[ZIO_CHECKSUM_FUNCTIONS] = {
Expand Down

0 comments on commit 2fe36b0

Please sign in to comment.