Sleep periodically when rebuilding L2ARC.

L2ARC devices of several terabytes filled with 4KB blocks may take 15
minutes to rebuild.  Due to the way L2ARC log reading is implemented
it is quite likely that for all that time rebuild thread will never
sleep.  At least on FreeBSD kernel threads have absolute priority and
can not be preempted by threads with lower priorities.  If some thread
is also bound to that specific CPU it may not get any CPU time for all
the 15 minutes.

This patch adds small throttling to the rebuild process, putting it
into sleep for 1 tick every 10 log block.  On my system this means
approximately 1ms every 10ms, reaching the single CPU usage of ~90%.

Signed-off-by: Alexander Motin <mav@FreeBSD.org>

man/man5/zfs-module-parameters.5

@@ -325,6 +325,19 @@ has become somehow fragmented/unusable).
 Use \fB1\fR for yes (default) and \fB0\fR for no.
 .RE
 
+.sp
+.ne 2
+.na
+\fBl2arc_rebuild_batch\fR (int)
+.ad
+.RS 12n
+Number of L2ARC log blocks to process before sleeping for a tick to relieve
+some CPU and let scheduler switch the context.
+Setting this value to 0 will disable the throttle.
+.sp
+Default value: \fB10\fR.
+.RE
+
 .sp
 .ne 2
 .na

module/zfs/arc.c

@@ -942,6 +942,8 @@ unsigned long l2arc_trim_ahead = 0;
  * l2arc_rebuild_enabled : A ZFS module parameter that controls whether adding
  * 		an L2ARC device (either at pool import or later) will attempt
  * 		to rebuild L2ARC buffer contents.
+ * l2arc_rebuild_batch: Number of blocks to process before sleeping for 1/hz
+ * 		to relieve some CPU and let scheduler switch the context.
  * l2arc_rebuild_blocks_min_l2size : A ZFS module parameter that controls
  * 		whether log blocks are written to the L2ARC device. If the L2ARC
  * 		device is less than 1GB, the amount of data l2arc_evict()
@@ -950,6 +952,7 @@ unsigned long l2arc_trim_ahead = 0;
  * 		not to waste space.
  */
 int l2arc_rebuild_enabled = B_TRUE;
+int l2arc_rebuild_batch = 10;
 unsigned long l2arc_rebuild_blocks_min_l2size = 1024 * 1024 * 1024;
 
 /* L2ARC persistence rebuild control routines. */
@@ -9767,7 +9770,7 @@ l2arc_rebuild(l2arc_dev_t *dev)
 {
 	vdev_t			*vd = dev->l2ad_vdev;
 	spa_t			*spa = vd->vdev_spa;
-	int			err = 0;
+	int			err = 0, batch = 0;
 	l2arc_dev_hdr_phys_t	*l2dhdr = dev->l2ad_dev_hdr;
 	l2arc_log_blk_phys_t	*this_lb, *next_lb;
 	zio_t			*this_io = NULL, *next_io = NULL;
@@ -9893,6 +9896,14 @@ l2arc_rebuild(l2arc_dev_t *dev)
 		    !dev->l2ad_first)
 			goto out;
 
+		/*
+		 * Release the CPU periodically to not monopolize it.
+		 */
+		if (l2arc_rebuild_batch && ++batch >= l2arc_rebuild_batch) {
+			batch = 0;
+			delay(1);
+		}
+
 		for (;;) {
 			mutex_enter(&l2arc_rebuild_thr_lock);
 			if (dev->l2ad_rebuild_cancel) {
@@ -10719,6 +10730,9 @@ ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, meta_percent, INT, ZMOD_RW,
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_enabled, INT, ZMOD_RW,
 	"Rebuild the L2ARC when importing a pool");
 
+ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_batch, INT, ZMOD_RW,
+	"Number of blocks to process before sleeping for 1/hz");
+
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_blocks_min_l2size, ULONG, ZMOD_RW,
 	"Min size in bytes to write rebuild log blocks in L2ARC");