pam: implement a zfs_key pam module

currently the pam module does:
 * load a zfs key and mounts the dataset when a session opens
 * unmounts the dataset and unloads the key when the session closes
 * when the user is logged on and changes the password, the modules
   changes the encryption key.

Signed-off-by: Felix Dörre <felix@dogcraft.de>
Closes openzfs#9886

config/always-pam.m4

@@ -0,0 +1,32 @@
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PAM], [
+	AC_ARG_ENABLE(pam,
+		AC_HELP_STRING([--enable-pam],
+		[install pam_zfs_key module [[default: check]]]),
+		[enable_pam=$enableval],
+		[enable_pam=check])
+
+	AC_ARG_WITH(pammoduledir,
+		AC_HELP_STRING([--with-pammoduledir=DIR],
+		[install pam module in dir [[/lib/security]]]),
+		pammoduledir=$withval,pammoduledir=/lib/security)
+
+	AC_ARG_WITH(pamconfigsdir,
+		AC_HELP_STRING([--with-pamconfigsdir=DIR],
+		[install pam-config files in dir [[/usr/share/pamconfigs]]]),
+		pamconfigsdir=$withval,pamconfigsdir=/usr/share/pam-configs)
+
+	AS_IF([test "x$enable_pam" != "xno"], [
+		AC_CHECK_HEADERS([security/pam_modules.h security/pam_ext.h], [
+			enable_pam=yes
+		], [
+			AS_IF([test "x$enable_pam" == "xyes"], [
+				AC_MSG_FAILURE([
+	*** security/pam_modules.h missing, libpam0g-dev package required
+				])
+			])
+		])
+	])
+	AM_CONDITIONAL([PAM_ZFS_ENABLED], [test "x$enable_pam" = xyes])
+	AC_SUBST(pammoduledir)
+	AC_SUBST(pamconfigsdir)
+])

config/zfs-build.m4

@@ -162,6 +162,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [
 	ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD
 	ZFS_AC_CONFIG_ALWAYS_SYSTEM
 	ZFS_AC_CONFIG_ALWAYS_ARCH
+	ZFS_AC_CONFIG_ALWAYS_PAM
 	ZFS_AC_CONFIG_ALWAYS_PYTHON
 	ZFS_AC_CONFIG_ALWAYS_PYZFS
 	ZFS_AC_CONFIG_ALWAYS_SED

configure.ac

@@ -94,6 +94,7 @@ AC_CONFIG_FILES([
 	contrib/initramfs/hooks/Makefile
 	contrib/initramfs/scripts/Makefile
 	contrib/initramfs/scripts/local-top/Makefile
+	contrib/pam_zfs_key/Makefile
 	contrib/pyzfs/Makefile
 	contrib/pyzfs/setup.py
 	contrib/zcp/Makefile

contrib/Makefile.am

@@ -1,2 +1,2 @@
-SUBDIRS = bash_completion.d bpftrace dracut initramfs pyzfs zcp
-DIST_SUBDIRS = bash_completion.d bpftrace dracut initramfs pyzfs zcp
+SUBDIRS = bash_completion.d bpftrace dracut initramfs pyzfs pam_zfs_key zcp
+DIST_SUBDIRS = bash_completion.d bpftrace dracut initramfs pyzfs pam_zfs_key zcp

contrib/pam_zfs_key/Makefile.am

@@ -0,0 +1,26 @@
+include $(top_srcdir)/config/Rules.am
+
+VPATH = \
+	$(top_srcdir)/module/icp \
+	$(top_srcdir)/module/zcommon \
+	$(top_srcdir)/lib/libzfs
+
+if PAM_ZFS_ENABLED
+
+pammodule_LTLIBRARIES = libpam_zfs_key.la
+
+libpam_zfs_key_la_SOURCES = pam_zfs_key.c
+
+libpam_zfs_key_la_LIBADD = \
+	$(top_builddir)/lib/libnvpair/libnvpair.la \
+	$(top_builddir)/lib/libuutil/libuutil.la \
+	$(top_builddir)/lib/libzfs/libzfs.la \
+	$(top_builddir)/lib/libzfs_core/libzfs_core.la
+
+libpam_zfs_key_la_LDFLAGS = -version-info 1:0:0
+
+libpam_zfs_key_la_LIBADD += -lm $(LIBSSL)
+
+pamconfigs_DATA = zfs_key
+
+endif