libzfs: add keylocation=https://, backed by fetch(3) or libcurl

Add support for http and https to the keylocation properly to allow encryption keys to be fetched from the specified URL. Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov> Reviewed-by: Ryan Moeller <ryan@ixsystems.com> Signed-off-by: Ahelenia Ziemiańska <nabijaczleweli@nabijaczleweli.xyz> Issue openzfs#9543 Closes openzfs#9947 Closes openzfs#11956
tonyhutter · Feb 10, 2022 · f33f121 · f33f121
1 parent 80f4460
commit f33f121
Show file tree

Hide file tree

Showing 29 changed files with 581 additions and 53 deletions.
diff --git a/config/Substfiles.am b/config/Substfiles.am
@@ -15,7 +15,9 @@ subst_sed_cmd = \
 	-e 's|@PYTHON[@]|$(PYTHON)|g' \
 	-e 's|@PYTHON_SHEBANG[@]|$(PYTHON_SHEBANG)|g' \
 	-e 's|@DEFAULT_INIT_NFS_SERVER[@]|$(DEFAULT_INIT_NFS_SERVER)|g' \
-	-e 's|@DEFAULT_INIT_SHELL[@]|$(DEFAULT_INIT_SHELL)|g'
+	-e 's|@DEFAULT_INIT_SHELL[@]|$(DEFAULT_INIT_SHELL)|g' \
+	-e 's|@LIBFETCH_DYNAMIC[@]|$(LIBFETCH_DYNAMIC)|g' \
+	-e 's|@LIBFETCH_SONAME[@]|$(LIBFETCH_SONAME)|g'
 
 SUBSTFILES =
 CLEANFILES = $(SUBSTFILES)

diff --git a/config/user-libfetch.m4 b/config/user-libfetch.m4
@@ -0,0 +1,71 @@
+dnl #
+dnl # Check for a libfetch - either fetch(3) or libcurl.
+dnl #
+dnl # There are two configuration dimensions:
+dnl #   * fetch(3) vs libcurl
+dnl #   * static vs dynamic
+dnl #
+dnl # fetch(3) is only dynamic.
+dnl # We use sover 6, which first appeared in FreeBSD 8.0-RELEASE.
+dnl #
+dnl # libcurl development packages include curl-config(1) – we want:
+dnl #   * HTTPS support
+dnl #   * version at least 7.16 (October 2006), for sover 4
+dnl #   * to decide if it's static or not
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_USER_LIBFETCH], [
+	AC_MSG_CHECKING([for libfetch])
+	LIBFETCH_LIBS=
+	LIBFETCH_IS_FETCH=0
+	LIBFETCH_IS_LIBCURL=0
+	LIBFETCH_DYNAMIC=0
+	LIBFETCH_SONAME=
+	have_libfetch=
+
+	saved_libs="$LIBS"
+	LIBS="$LIBS -lfetch"
+	AC_LINK_IFELSE([AC_LANG_PROGRAM([[
+		#include <sys/param.h>
+		#include <stdio.h>
+		#include <fetch.h>
+	]], [fetchGetURL("", "");])], [
+		have_libfetch=1
+		LIBFETCH_IS_FETCH=1
+		LIBFETCH_DYNAMIC=1
+		LIBFETCH_SONAME='"libfetch.so.6"'
+		LIBFETCH_LIBS="-ldl"
+		AC_MSG_RESULT([fetch(3)])
+	], [])
+	LIBS="$saved_libs"
+
+	if test -z "$have_libfetch"; then
+		if curl-config --protocols 2>/dev/null | grep -q HTTPS &&
+		    test "$(printf "%u" "0x$(curl-config --vernum)")" -ge "$(printf "%u" "0x071000")"; then
+			have_libfetch=1
+			LIBFETCH_IS_LIBCURL=1
+			if test "$(curl-config --built-shared)" = "yes"; then
+				LIBFETCH_DYNAMIC=1
+				LIBFETCH_SONAME='"libcurl.so.4"'
+				LIBFETCH_LIBS="-ldl"
+				AC_MSG_RESULT([libcurl])
+			else
+				LIBFETCH_LIBS="$(curl-config --libs)"
+				AC_MSG_RESULT([libcurl (static)])
+			fi
+
+			CCFLAGS="$CCFLAGS $(curl-config --cflags)"
+		fi
+	fi
+
+	if test -z "$have_libfetch"; then
+		AC_MSG_RESULT([none])
+	fi
+
+	AC_SUBST([LIBFETCH_LIBS])
+	AC_SUBST([LIBFETCH_DYNAMIC])
+	AC_SUBST([LIBFETCH_SONAME])
+	AC_DEFINE_UNQUOTED([LIBFETCH_IS_FETCH], [$LIBFETCH_IS_FETCH], [libfetch is fetch(3)])
+	AC_DEFINE_UNQUOTED([LIBFETCH_IS_LIBCURL], [$LIBFETCH_IS_LIBCURL], [libfetch is libcurl])
+	AC_DEFINE_UNQUOTED([LIBFETCH_DYNAMIC], [$LIBFETCH_DYNAMIC], [whether the chosen libfetch is to be loaded at run-time])
+	AC_DEFINE_UNQUOTED([LIBFETCH_SONAME], [$LIBFETCH_SONAME], [soname of chosen libfetch])
+])
diff --git a/config/user.m4 b/config/user.m4
@@ -22,6 +22,7 @@ AC_DEFUN([ZFS_AC_CONFIG_USER], [
 	ZFS_AC_CONFIG_USER_LIBCRYPTO
 	ZFS_AC_CONFIG_USER_LIBAIO
 	ZFS_AC_CONFIG_USER_LIBATOMIC
+	ZFS_AC_CONFIG_USER_LIBFETCH
 	ZFS_AC_CONFIG_USER_CLOCK_GETTIME
 	ZFS_AC_CONFIG_USER_PAM
 	ZFS_AC_CONFIG_USER_RUNSTATEDIR

diff --git a/contrib/dracut/90zfs/module-setup.sh.in b/contrib/dracut/90zfs/module-setup.sh.in
@@ -60,6 +60,11 @@ install() {
 		# Fallback: Guess the path and include all matches
 		dracut_install /usr/lib*/gcc/**/libgcc_s.so*
 	fi
+	if [ @LIBFETCH_DYNAMIC@ != 0 ]; then
+		for d in $libdirs; do
+			[ -e "$d"/@LIBFETCH_SONAME@ ] && dracut_install "$d"/@LIBFETCH_SONAME@
+		done
+	fi
 	dracut_install @mounthelperdir@/mount.zfs
 	dracut_install @udevdir@/vdev_id
 	dracut_install awk

diff --git a/contrib/dracut/90zfs/zfs-env-bootfs.service.in b/contrib/dracut/90zfs/zfs-env-bootfs.service.in
@@ -8,7 +8,7 @@ Before=zfs-import.target
 
 [Service]
 Type=oneshot
-ExecStart=/bin/sh -c "systemctl set-environment BOOTFS=$(@sbindir@/zpool list -H -o bootfs | grep -m1 -v '^-$')"
+ExecStart=/bin/sh -c "exec systemctl set-environment BOOTFS=$(@sbindir@/zpool list -H -o bootfs | grep -m1 -v '^-$')"
 
 [Install]
 WantedBy=zfs-import.target
diff --git a/contrib/dracut/90zfs/zfs-load-key.sh.in b/contrib/dracut/90zfs/zfs-load-key.sh.in
@@ -43,13 +43,14 @@ if [ "$(zpool list -H -o feature@encryption "${BOOTFS%%/*}")" = 'active' ]; then
 
         KEYLOCATION="$(zfs get -H -o value keylocation "${ENCRYPTIONROOT}")"
         if ! [ "${KEYLOCATION}" = "prompt" ]; then
+            if ! [ "${KEYLOCATION#http}" = "${KEYLOCATION}" ]; then
+                systemctl start network-online.target
+            fi
             zfs load-key "${ENCRYPTIONROOT}"
         else
             # decrypt them
-            TRY_COUNT=5
-            while [ $TRY_COUNT -gt 0 ]; do
+            for _ in 1 2 3 4 5; do
                 systemd-ask-password "Encrypted ZFS password for ${BOOTFS}" --no-tty | zfs load-key "${ENCRYPTIONROOT}" && break
-                TRY_COUNT=$((TRY_COUNT - 1))
             done
         fi
     fi

diff --git a/contrib/dracut/90zfs/zfs-rollback-bootfs.service.in b/contrib/dracut/90zfs/zfs-rollback-bootfs.service.in
@@ -10,5 +10,5 @@ ConditionKernelCommandLine=bootfs.rollback
 # ${BOOTFS} should have been set by zfs-env-bootfs.service
 Type=oneshot
 ExecStartPre=/bin/sh -c 'test -n "${BOOTFS}"'
-ExecStart=/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.rollback)"; @sbindir@/zfs rollback -Rf "${BOOTFS}@${SNAPNAME:-%v}"'
+ExecStart=/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.rollback)"; exec @sbindir@/zfs rollback -Rf "${BOOTFS}@${SNAPNAME:-%v}"'
 RemainAfterExit=yes
diff --git a/contrib/dracut/90zfs/zfs-snapshot-bootfs.service.in b/contrib/dracut/90zfs/zfs-snapshot-bootfs.service.in
@@ -10,5 +10,5 @@ ConditionKernelCommandLine=bootfs.snapshot
 # ${BOOTFS} should have been set by zfs-env-bootfs.service
 Type=oneshot
 ExecStartPre=/bin/sh -c 'test -n "${BOOTFS}"'
-ExecStart=-/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.snapshot)"; @sbindir@/zfs snapshot "${BOOTFS}@${SNAPNAME:-%v}"'
+ExecStart=-/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.snapshot)"; exec @sbindir@/zfs snapshot "${BOOTFS}@${SNAPNAME:-%v}"'
 RemainAfterExit=yes
diff --git a/contrib/initramfs/hooks/zfs.in b/contrib/initramfs/hooks/zfs.in
@@ -30,6 +30,13 @@ find /lib/ -type f -name "libgcc_s.so.[1-9]" | while read -r libgcc; do
 	copy_exec "$libgcc"
 done
 
+# shellcheck disable=SC2050
+if [ @LIBFETCH_DYNAMIC@ != 0 ]; then
+	find /lib/ -name @LIBFETCH_SONAME@ | while read -r libfetch; do
+		copy_exec "$libfetch"
+	done
+fi
+
 copy_file config "/etc/hostid"
 copy_file cache  "@sysconfdir@/zfs/zpool.cache"
 copy_file config "@initconfdir@/zfs"

diff --git a/contrib/initramfs/scripts/zfs b/contrib/initramfs/scripts/zfs
@@ -403,28 +403,25 @@ decrypt_fs()
 			KEYSTATUS="$(get_fs_value "${ENCRYPTIONROOT}" keystatus)"
 			# Continue only if the key needs to be loaded
 			[ "$KEYSTATUS" = "unavailable" ] || return 0
-			TRY_COUNT=3
 
-			# If key is stored in a file, do not prompt
+			# Do not prompt if key is stored noninteractively,
 			if ! [ "${KEYLOCATION}" = "prompt" ]; then
 				$ZFS load-key "${ENCRYPTIONROOT}"
 
 			# Prompt with plymouth, if active
-			elif [ -e /bin/plymouth ] && /bin/plymouth --ping 2>/dev/null; then
+			elif /bin/plymouth --ping 2>/dev/null; then
 				echo "plymouth" > /run/zfs_console_askpwd_cmd
-				while [ $TRY_COUNT -gt 0 ]; do
+				for _ in 1 2 3; do
 					plymouth ask-for-password --prompt "Encrypted ZFS password for ${ENCRYPTIONROOT}" | \
 						$ZFS load-key "${ENCRYPTIONROOT}" && break
-					TRY_COUNT=$((TRY_COUNT - 1))
 				done
 
 			# Prompt with systemd, if active
 			elif [ -e /run/systemd/system ]; then
 				echo "systemd-ask-password" > /run/zfs_console_askpwd_cmd
-				while [ $TRY_COUNT -gt 0 ]; do
+				for _ in 1 2 3; do
 					systemd-ask-password "Encrypted ZFS password for ${ENCRYPTIONROOT}" --no-tty | \
 						$ZFS load-key "${ENCRYPTIONROOT}" && break
-					TRY_COUNT=$((TRY_COUNT - 1))
 				done
 
 			# Prompt with ZFS tty, otherwise

diff --git a/include/libzfs_impl.h b/include/libzfs_impl.h
@@ -72,6 +72,8 @@ struct libzfs_handle {
 	boolean_t libzfs_prop_debug;
 	regex_t libzfs_urire;
 	uint64_t libzfs_max_nvlist;
+	void *libfetch;
+	char *libfetch_load_error;
 };
 
 struct zfs_handle {

diff --git a/lib/libzfs/Makefile.am b/lib/libzfs/Makefile.am
@@ -75,7 +75,7 @@ libzfs_la_LIBADD = \
 	$(abs_top_builddir)/lib/libnvpair/libnvpair.la \
 	$(abs_top_builddir)/lib/libuutil/libuutil.la
 
-libzfs_la_LIBADD += -lm $(LIBCRYPTO_LIBS) $(ZLIB_LIBS) $(LTLIBINTL)
+libzfs_la_LIBADD += -lm $(LIBCRYPTO_LIBS) $(ZLIB_LIBS) $(LIBFETCH_LIBS) $(LTLIBINTL)
 
 libzfs_la_LDFLAGS = -pthread
 

diff --git a/lib/libzfs/libzfs.abi b/lib/libzfs/libzfs.abi
@@ -551,10 +551,6 @@
       <parameter type-id='e1c52942'/>
       <return type-id='95e97e5e'/>
     </function-decl>
-    <function-decl name='unlink' visibility='default' binding='global' size-in-bits='64'>
-      <parameter type-id='80f4b756'/>
-      <return type-id='95e97e5e'/>
-    </function-decl>
   </abi-instr>
   <abi-instr address-size='64' path='os/linux/smb.c' language='LANG_C99'>
     <array-type-def dimensions='1' type-id='a84c031d' size-in-bits='2040' id='11641789'>
@@ -1373,7 +1369,7 @@
     <typedef-decl name='zpool_handle_t' type-id='67002a8a' id='b1efc708'/>
     <typedef-decl name='libzfs_handle_t' type-id='c8a9d9d8' id='95942d0c'/>
     <typedef-decl name='zfs_iter_f' type-id='5571cde4' id='d8e49ab9'/>
-    <class-decl name='libzfs_handle' size-in-bits='20224' is-struct='yes' visibility='default' id='c8a9d9d8'>
+    <class-decl name='libzfs_handle' size-in-bits='20352' is-struct='yes' visibility='default' id='c8a9d9d8'>
       <data-member access='public' layout-offset-in-bits='0'>
         <var-decl name='libzfs_error' type-id='95e97e5e' visibility='default'/>
       </data-member>
@@ -1434,6 +1430,12 @@
       <data-member access='public' layout-offset-in-bits='20160'>
         <var-decl name='libzfs_max_nvlist' type-id='9c313c2d' visibility='default'/>
       </data-member>
+      <data-member access='public' layout-offset-in-bits='20224'>
+        <var-decl name='libfetch' type-id='eaa32e2f' visibility='default'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='20288'>
+        <var-decl name='libfetch_load_error' type-id='26a90f95' visibility='default'/>
+      </data-member>
     </class-decl>
     <class-decl name='zfs_handle' size-in-bits='4928' is-struct='yes' visibility='default' id='f6ee4445'>
       <data-member access='public' layout-offset-in-bits='0'>
@@ -3190,6 +3192,19 @@
     <function-decl name='__ctype_b_loc' visibility='default' binding='global' size-in-bits='64'>
       <return type-id='c59e1ef0'/>
     </function-decl>
+    <function-decl name='dlopen' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='80f4b756'/>
+      <parameter type-id='95e97e5e'/>
+      <return type-id='eaa32e2f'/>
+    </function-decl>
+    <function-decl name='dlsym' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='1b7446cd'/>
+      <parameter type-id='9d26089a'/>
+      <return type-id='eaa32e2f'/>
+    </function-decl>
+    <function-decl name='dlerror' visibility='default' binding='global' size-in-bits='64'>
+      <return type-id='26a90f95'/>
+    </function-decl>
     <function-decl name='PKCS5_PBKDF2_HMAC_SHA1' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='80f4b756'/>
       <parameter type-id='95e97e5e'/>
@@ -3231,6 +3246,11 @@
       <parameter type-id='822cd80b'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='fdopen' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='95e97e5e'/>
+      <parameter type-id='80f4b756'/>
+      <return type-id='822cd80b'/>
+    </function-decl>
     <function-decl name='printf' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='80f4b756'/>
       <parameter is-variadic='yes'/>
@@ -3243,6 +3263,12 @@
       <parameter is-variadic='yes'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='asprintf' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='8c85230f'/>
+      <parameter type-id='9d26089a'/>
+      <parameter is-variadic='yes'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
     <function-decl name='fputc' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='95e97e5e'/>
       <parameter type-id='822cd80b'/>
@@ -3262,6 +3288,10 @@
       <parameter type-id='e75a27e9'/>
       <return type-id='b59d7dce'/>
     </function-decl>
+    <function-decl name='rewind' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='822cd80b'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
     <function-decl name='ferror' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='822cd80b'/>
       <return type-id='95e97e5e'/>
@@ -3285,6 +3315,10 @@
       <parameter type-id='b59d7dce'/>
       <return type-id='eaa32e2f'/>
     </function-decl>
+    <function-decl name='strdup' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='80f4b756'/>
+      <return type-id='26a90f95'/>
+    </function-decl>
     <function-decl name='strerror' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='95e97e5e'/>
       <return type-id='26a90f95'/>
@@ -3317,6 +3351,10 @@
       <parameter type-id='95e97e5e'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='unlink' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='80f4b756'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
     <function-type size-in-bits='64' id='ee076206'>
       <return type-id='48b5725f'/>
     </function-type>
@@ -4425,12 +4463,6 @@
       <parameter is-variadic='yes'/>
       <return type-id='95e97e5e'/>
     </function-decl>
-    <function-decl name='asprintf' visibility='default' binding='global' size-in-bits='64'>
-      <parameter type-id='8c85230f'/>
-      <parameter type-id='9d26089a'/>
-      <parameter is-variadic='yes'/>
-      <return type-id='95e97e5e'/>
-    </function-decl>
     <function-decl name='strtol' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='9d26089a'/>
       <parameter type-id='8c85230f'/>
@@ -4452,10 +4484,6 @@
       <parameter type-id='b59d7dce'/>
       <return type-id='26a90f95'/>
     </function-decl>
-    <function-decl name='strdup' visibility='default' binding='global' size-in-bits='64'>
-      <parameter type-id='80f4b756'/>
-      <return type-id='26a90f95'/>
-    </function-decl>
     <function-decl name='strrchr' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='80f4b756'/>
       <parameter type-id='95e97e5e'/>
@@ -4620,11 +4648,6 @@
       <parameter type-id='4051f5e7'/>
       <return type-id='95e97e5e'/>
     </function-decl>
-    <function-decl name='fdopen' visibility='default' binding='global' size-in-bits='64'>
-      <parameter type-id='95e97e5e'/>
-      <parameter type-id='80f4b756'/>
-      <return type-id='822cd80b'/>
-    </function-decl>
     <function-decl name='pipe2' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='7292109c'/>
       <parameter type-id='95e97e5e'/>
@@ -6705,6 +6728,12 @@
       <parameter type-id='95e97e5e'/>
       <return type-id='48b5725f'/>
     </function-decl>
+    <function-decl name='avl_insert' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='a3681dea'/>
+      <parameter type-id='eaa32e2f'/>
+      <parameter type-id='fba6cb51'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
     <function-decl name='nvlist_lookup_boolean' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='5ce45b60'/>
       <parameter type-id='80f4b756'/>
@@ -7235,6 +7264,10 @@
     <function-decl name='__ctype_toupper_loc' visibility='default' binding='global' size-in-bits='64'>
       <return type-id='24f95ba5'/>
     </function-decl>
+    <function-decl name='dlclose' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='eaa32e2f'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
     <function-decl name='regcomp' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='5c53ba29'/>
       <parameter type-id='9d26089a'/>