From 1e55718012aa0a7345469ea5af25ab8d83f90b33 Mon Sep 17 00:00:00 2001 From: Alexander Motin Date: Mon, 26 Oct 2020 21:45:41 -0400 Subject: [PATCH] Sleep periodically when rebuilding L2ARC. L2ARC devices of several terabytes filled with 4KB blocks may take 15 minutes to rebuild. Due to the way L2ARC log reading is implemented it is quite likely that for all that time rebuild thread will never sleep. At least on FreeBSD kernel threads have absolute priority and can not be preempted by threads with lower priorities. If some thread is also bound to that specific CPU it may not get any CPU time for all the 15 minutes. This patch adds small throttling to the rebuild process, putting it into sleep for 1 tick every 10 log block. On my system this means approximately 1ms every 10ms, reaching the single CPU usage of ~90%. Signed-off-by: Alexander Motin --- man/man5/zfs-module-parameters.5 | 12 ++++++++++++ module/zfs/arc.c | 16 +++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/man/man5/zfs-module-parameters.5 b/man/man5/zfs-module-parameters.5 index 469963750a05..9f9f3c70f980 100644 --- a/man/man5/zfs-module-parameters.5 +++ b/man/man5/zfs-module-parameters.5 @@ -325,6 +325,18 @@ has become somehow fragmented/unusable). Use \fB1\fR for yes (default) and \fB0\fR for no. .RE +.sp +.ne 2 +.na +\fBl2arc_rebuild_batch\fR (int) +.ad +.RS 12n +Number of L2ARC log blocks to process before sleeping for a tick to relieve +some CPU and let scheduler switch the context. +.sp +Default value: \fB10\fR. +.RE + .sp .ne 2 .na diff --git a/module/zfs/arc.c b/module/zfs/arc.c index 1e3d47693d37..727b67dcd707 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -942,6 +942,8 @@ unsigned long l2arc_trim_ahead = 0; * l2arc_rebuild_enabled : A ZFS module parameter that controls whether adding * an L2ARC device (either at pool import or later) will attempt * to rebuild L2ARC buffer contents. + * l2arc_rebuild_batch: Number of blocks to process before sleeping for 1/hz + * to relieve some CPU and let scheduler switch the context. * l2arc_rebuild_blocks_min_l2size : A ZFS module parameter that controls * whether log blocks are written to the L2ARC device. If the L2ARC * device is less than 1GB, the amount of data l2arc_evict() @@ -950,6 +952,7 @@ unsigned long l2arc_trim_ahead = 0; * not to waste space. */ int l2arc_rebuild_enabled = B_TRUE; +int l2arc_rebuild_batch = 10; unsigned long l2arc_rebuild_blocks_min_l2size = 1024 * 1024 * 1024; /* L2ARC persistence rebuild control routines. */ @@ -9767,7 +9770,7 @@ l2arc_rebuild(l2arc_dev_t *dev) { vdev_t *vd = dev->l2ad_vdev; spa_t *spa = vd->vdev_spa; - int err = 0; + int err = 0, batch = 0; l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr; l2arc_log_blk_phys_t *this_lb, *next_lb; zio_t *this_io = NULL, *next_io = NULL; @@ -9893,6 +9896,14 @@ l2arc_rebuild(l2arc_dev_t *dev) !dev->l2ad_first) goto out; + /* + * Release the CPU periodically to not monopolize it. + */ + if (++batch >= l2arc_rebuild_batch) { + batch = 0; + delay(1); + } + for (;;) { mutex_enter(&l2arc_rebuild_thr_lock); if (dev->l2ad_rebuild_cancel) { @@ -10719,6 +10730,9 @@ ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, meta_percent, INT, ZMOD_RW, ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_enabled, INT, ZMOD_RW, "Rebuild the L2ARC when importing a pool"); +ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_batch, INT, ZMOD_RW, + "Number of blocks to process before sleeping for 1/hz"); + ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_blocks_min_l2size, ULONG, ZMOD_RW, "Min size in bytes to write rebuild log blocks in L2ARC");