OpenZFS 9284 - arc_reclaim_thread has 2 jobs

Following the fix for 9018 (Replace kmem_cache_reap_now() with
kmem_cache_reap_soon), the arc_reclaim_thread() no longer blocks
while reaping.  However, the code is still confusing and error-prone,
because this thread has two responsibilities.  We should instead
separate this into two threads each with their own responsibility:

 1. keep `arc_size` under `arc_c`, by calling `arc_adjust()`, which
    improves `arc_is_overflowing()`

 2. keep enough free memory in the system, by calling
    `arc_kmem_reap_now()` plus `arc_shrink()`, which improves
    `arc_available_memory()`.

Furthermore, we can use the zthr infrastructure to separate the
"should we do something" from "do it" parts of the logic, and
normalize the start up / shut down of the threads.

Authored by: Brad Lewis <brad.lewis@delphix.com>
Reviewed by: Matt Ahrens <mahrens@delphix.com>
Reviewed by: Serapheim Dimitropoulos <serapheim@delphix.com>
Reviewed by: Pavel Zakharov <pavel.zakharov@delphix.com>
Reviewed by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: Dan McDonald <danmcd@joyent.com>
Reviewed by: Tim Kordas <tim.kordas@joyent.com>
Reviewed by: Tim Chase <tim@chase2k.com>
Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
Ported-by:  Brad Lewis <brad.lewis@delphix.com>
Signed-off-by: Brad Lewis <brad.lewis@delphix.com>

OpenZFS-issue: https://www.illumos.org/issues/9284
OpenZFS-commit: openzfs/openzfs@de753e34f9
Closes openzfs#8165

include/spl/sys/kmem.h

@@ -163,6 +163,7 @@ extern unsigned int spl_kmem_alloc_max;
 #define	kmem_alloc(sz, fl)	spl_kmem_alloc((sz), (fl), __func__, __LINE__)
 #define	kmem_zalloc(sz, fl)	spl_kmem_zalloc((sz), (fl), __func__, __LINE__)
 #define	kmem_free(ptr, sz)	spl_kmem_free((ptr), (sz))
+#define	kmem_cache_reap_active	spl_kmem_cache_reap_active
 
 extern void *spl_kmem_alloc(size_t sz, int fl, const char *func, int line);
 extern void *spl_kmem_zalloc(size_t sz, int fl, const char *func, int line);
@@ -181,5 +182,6 @@ extern void spl_kmem_free_track(const void *buf, size_t size);
 
 extern int spl_kmem_init(void);
 extern void spl_kmem_fini(void);
+extern int spl_kmem_cache_reap_active(void);
 
 #endif	/* _SPL_KMEM_H */

include/sys/zfs_context.h

@@ -773,6 +773,7 @@ typedef int fstrans_cookie_t;
 extern fstrans_cookie_t spl_fstrans_mark(void);
 extern void spl_fstrans_unmark(fstrans_cookie_t);
 extern int __spl_pf_fstrans_check(void);
+extern int kmem_cache_reap_active(void);
 
 #define	____cacheline_aligned
 

include/sys/zthr.h

@@ -29,6 +29,7 @@ struct zthr {
 	kmutex_t	zthr_lock;
 	kcondvar_t	zthr_cv;
 	boolean_t	zthr_cancel;
+	hrtime_t	zthr_wait_time;
 
 	zthr_checkfunc_t	*zthr_checkfunc;
 	zthr_func_t	*zthr_func;
@@ -38,6 +39,9 @@ struct zthr {
 
 extern zthr_t *zthr_create(zthr_checkfunc_t checkfunc,
     zthr_func_t *func, void *arg);
+extern zthr_t *zthr_create_timer(zthr_checkfunc_t *checkfunc,
+    zthr_func_t *func, void *arg, hrtime_t nano_wait);
+
 extern void zthr_exit(zthr_t *t, int rc);
 extern void zthr_destroy(zthr_t *t);
 

lib/libzpool/kernel.c

@@ -1276,6 +1276,12 @@ __spl_pf_fstrans_check(void)
 	return (0);
 }
 
+int
+kmem_cache_reap_active(void)
+{
+	return (0);
+}
+
 void *zvol_tag = "zvol_tag";
 
 void

module/spl/spl-kmem-cache.c

@@ -1732,6 +1732,18 @@ spl_kmem_cache_reap_now(spl_kmem_cache_t *skc, int count)
 }
 EXPORT_SYMBOL(spl_kmem_cache_reap_now);
 
+/*
+ * This is stubbed out for code consistency with other platforms.  There
+ * is existing logic to prevent concurrent reaping so while this is ugly
+ * it should do no harm.
+ */
+int
+spl_kmem_cache_reap_active()
+{
+	return (0);
+}
+EXPORT_SYMBOL(spl_kmem_cache_reap_active);
+
 /*
  * Reap all free slabs from all registered caches.
  */