pam: implement a zfs_key pam module

currently the pam module does:
 * load a zfs key and mounts the dataset when a session opens
 * unmounts the dataset and unloads the key when the session closes
 * when the user is logged on and changes the password, the modules
   changes the encryption key.

Signed-off-by: Felix Dörre <felix@dogcraft.de>
Closes openzfs#9886

config/user-pam.m4

@@ -0,0 +1,31 @@
+AC_DEFUN([ZFS_AC_CONFIG_USER_PAM], [
+	AC_ARG_ENABLE(pam,
+		AC_HELP_STRING([--enable-pam],
+		[install pam_zfs_key module [[default: check]]]),
+		[enable_pam=$enableval],
+		[enable_pam=check])
+
+	AC_ARG_WITH(pammoduledir,
+		AC_HELP_STRING([--with-pammoduledir=DIR],
+		[install pam module in dir [[/lib/security]]]),
+		pammoduledir=$withval,pammoduledir=/lib/security)
+
+	AC_ARG_WITH(pamconfigsdir,
+		AC_HELP_STRING([--with-pamconfigsdir=DIR],
+		[install pam-config files in dir [[/usr/share/pamconfigs]]]),
+		pamconfigsdir=$withval,pamconfigsdir=/usr/share/pam-configs)
+
+        AS_IF([test "x$enable_pam" != "xno"], [
+		AC_CHECK_HEADERS([security/pam_modules.h security/pam_ext.h], [
+			enable_pam=yes
+		], [
+			AS_IF([test "x$enable_pam" == "xyes"], [
+			AC_MSG_FAILURE([*** security/pam_modules.h missing, libpam0g-dev package required])
+		])
+	])
+
+	])
+	AM_CONDITIONAL([PAM_ZFS_ENABLED], [test "x$enable_pam" = xyes])
+	AC_SUBST(pammoduledir)
+	AC_SUBST(pamconfigsdir)
+])

config/user.m4

@@ -15,6 +15,7 @@ AC_DEFUN([ZFS_AC_CONFIG_USER], [
 	ZFS_AC_CONFIG_USER_LIBUDEV
 	ZFS_AC_CONFIG_USER_LIBSSL
 	ZFS_AC_CONFIG_USER_LIBAIO
+	ZFS_AC_CONFIG_USER_PAM
 	ZFS_AC_CONFIG_USER_RUNSTATEDIR
 	ZFS_AC_CONFIG_USER_MAKEDEV_IN_SYSMACROS
 	ZFS_AC_CONFIG_USER_MAKEDEV_IN_MKDEV

configure.ac

@@ -94,6 +94,7 @@ AC_CONFIG_FILES([
 	contrib/initramfs/hooks/Makefile
 	contrib/initramfs/scripts/Makefile
 	contrib/initramfs/scripts/local-top/Makefile
+	contrib/pam_zfs_key/Makefile
 	contrib/pyzfs/Makefile
 	contrib/pyzfs/setup.py
 	contrib/zcp/Makefile

contrib/Makefile.am

@@ -1,2 +1,2 @@
-SUBDIRS = bash_completion.d bpftrace dracut initramfs pyzfs zcp
-DIST_SUBDIRS = bash_completion.d bpftrace dracut initramfs pyzfs zcp
+SUBDIRS = bash_completion.d bpftrace dracut initramfs pyzfs pam_zfs_key zcp
+DIST_SUBDIRS = bash_completion.d bpftrace dracut initramfs pyzfs pam_zfs_key zcp

contrib/pam_zfs_key/Makefile.am

@@ -0,0 +1,31 @@
+include $(top_srcdir)/config/Rules.am
+
+VPATH = \
+	$(top_srcdir)/module/icp \
+	$(top_srcdir)/module/zcommon \
+	$(top_srcdir)/lib/libzfs
+
+# Suppress unused but set variable warnings often due to ASSERTs
+AM_CFLAGS += $(NO_UNUSED_BUT_SET_VARIABLE)
+
+if PAM_ZFS_ENABLED
+
+pammodule_LTLIBRARIES = libpam_zfs_key.la
+
+#libpam_zfs_key_ladir = $(pammoduledir)
+libpam_zfs_key_la_SOURCES = pam_zfs_key.c
+
+libpam_zfs_key_la_LIBADD = \
+	$(top_builddir)/lib/libnvpair/libnvpair.la \
+	$(top_builddir)/lib/libuutil/libuutil.la \
+	$(top_builddir)/lib/libzfs/libzfs.la \
+	$(top_builddir)/lib/libzfs_core/libzfs_core.la
+
+libpam_zfs_key_la_LDFLAGS = -version-info 1:0:0
+
+libpam_zfs_key_la_LIBADD += -lm $(LIBSSL)
+
+pamconfigs_DATA = zfs_key
+
+
+endif

contrib/pam_zfs_key/pam-config-zfs_key

@@ -0,0 +1,13 @@
+Name: Unlock zfs datasets for user
+Default: yes
+Priority: 128
+Auth-Type: Additional
+Auth:
+	optional	pam_zfs_key.so
+Session-Interactive-Only: yes
+Session-Type: Additional
+Session:
+	optional	pam_zfs_key.so
+Password-Type: Additional
+Password:
+	optional	pam_zfs_key.so