From 87c25d567fb7969b44c7d8af63990e704fc49dd3 Mon Sep 17 00:00:00 2001 From: Matthew Ahrens Date: Thu, 28 Feb 2019 17:52:55 -0800 Subject: [PATCH] abd_alloc should use scatter for >1K allocations abd_alloc() normally does scatter allocations, thus solving the problem that ABD originally set out to: the bulk of ZFS's allocations are single pages, which are faster to allocate and free, and don't suffer from internal fragmentation (and the inability to reclaim memory because some buffers in the slab are still allocated). However, the current code does linear allocations for 4KB and smaller allocations, defeating the purpose of ABD. Scatter ABD's use at least one page each, so sub-page allocations waste some space when allocated as scatter (e.g. 2KB scatter allocation wastes half of each page). Using linear ABD's for small allocations means that they will be put on slabs which contain many allocations. This can improve memory efficiency, but it also makes it much harder for ARC evictions to actually free pages, because all the buffers on one slab need to be freed in order for the slab (and underlying pages) to be freed. Typically, 512B and 1KB kmem caches have 16 buffers per slab, so it's possible for them to actually waste more memory than scatter (one page per buf = wasting 3/4 or 7/8th; one buf per slab = wasting 15/16th). Spill blocks are typically 512B and are heavily used on systems running selinux with the default dnode size and the `xattr=sa` property set. By default we will use linear allocations for 512B and 1KB, and scatter allocations for larger (1.5KB and up). Reviewed-by: George Melikov Reviewed-by: DHE Reviewed-by: Chunwei Chen Reviewed-by: Brian Behlendorf Reviewed-by: Don Brady Signed-off-by: Matthew Ahrens Closes #8455 --- man/man5/zfs-module-parameters.5 | 14 +++++++++++++- module/zfs/abd.c | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/man/man5/zfs-module-parameters.5 b/man/man5/zfs-module-parameters.5 index bdac6f3a76a2..c783a1e9b9b3 100644 --- a/man/man5/zfs-module-parameters.5 +++ b/man/man5/zfs-module-parameters.5 @@ -1,6 +1,6 @@ '\" te .\" Copyright (c) 2013 by Turbo Fredriksson . All rights reserved. -.\" Copyright (c) 2018 by Delphix. All rights reserved. +.\" Copyright (c) 2019 by Delphix. All rights reserved. .\" Copyright (c) 2019 Datto Inc. .\" The contents of this file are subject to the terms of the Common Development .\" and Distribution License (the "License"). You may not use this file except @@ -537,6 +537,18 @@ Min time before an active prefetch stream can be reclaimed Default value: \fB2\fR. .RE +.sp +.ne 2 +.na +\fBzfs_abd_scatter_min_size\fR (uint) +.ad +.RS 12n +This is the minimum allocation size that will use scatter (page-based) +ABD's. Smaller allocations will use linear ABD's. +.sp +Default value: \fB1536\fR (512B and 1KB allocations will be linear). +.RE + .sp .ne 2 .na diff --git a/module/zfs/abd.c b/module/zfs/abd.c index 34be760891b9..9041bd8b1841 100644 --- a/module/zfs/abd.c +++ b/module/zfs/abd.c @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2014 by Chunwei Chen. All rights reserved. - * Copyright (c) 2016 by Delphix. All rights reserved. + * Copyright (c) 2019 by Delphix. All rights reserved. */ /* @@ -209,6 +209,30 @@ static abd_stats_t abd_stats = { int zfs_abd_scatter_enabled = B_TRUE; unsigned zfs_abd_scatter_max_order = MAX_ORDER - 1; +/* + * zfs_abd_scatter_min_size is the minimum allocation size to use scatter + * ABD's. Smaller allocations will use linear ABD's which uses + * zio_[data_]buf_alloc(). + * + * Scatter ABD's use at least one page each, so sub-page allocations waste + * some space when allocated as scatter (e.g. 2KB scatter allocation wastes + * half of each page). Using linear ABD's for small allocations means that + * they will be put on slabs which contain many allocations. This can + * improve memory efficiency, but it also makes it much harder for ARC + * evictions to actually free pages, because all the buffers on one slab need + * to be freed in order for the slab (and underlying pages) to be freed. + * Typically, 512B and 1KB kmem caches have 16 buffers per slab, so it's + * possible for them to actually waste more memory than scatter (one page per + * buf = wasting 3/4 or 7/8th; one buf per slab = wasting 15/16th). + * + * Spill blocks are typically 512B and are heavily used on systems running + * selinux with the default dnode size and the `xattr=sa` property set. + * + * By default we use linear allocations for 512B and 1KB, and scatter + * allocations for larger (1.5KB and up). + */ +int zfs_abd_scatter_min_size = 512 * 3; + static kmem_cache_t *abd_cache = NULL; static kstat_t *abd_ksp; @@ -581,7 +605,8 @@ abd_free_struct(abd_t *abd) abd_t * abd_alloc(size_t size, boolean_t is_metadata) { - if (!zfs_abd_scatter_enabled || size <= PAGESIZE) + /* see the comment above zfs_abd_scatter_min_size */ + if (!zfs_abd_scatter_enabled || size < zfs_abd_scatter_min_size) return (abd_alloc_linear(size, is_metadata)); VERIFY3U(size, <=, SPA_MAXBLOCKSIZE); @@ -1532,6 +1557,9 @@ abd_scatter_bio_map_off(struct bio *bio, abd_t *abd, module_param(zfs_abd_scatter_enabled, int, 0644); MODULE_PARM_DESC(zfs_abd_scatter_enabled, "Toggle whether ABD allocations must be linear."); +module_param(zfs_abd_scatter_min_size, int, 0644); +MODULE_PARM_DESC(zfs_abd_scatter_min_size, + "Minimum size of scatter allocations."); /* CSTYLED */ module_param(zfs_abd_scatter_max_order, uint, 0644); MODULE_PARM_DESC(zfs_abd_scatter_max_order,