Cross-platform xattr user namespace compatibility

ZFS on Linux originally implemented xattr namespaces in a way that is
incompatible with other operating systems.  On illumos, xattrs do not
have namespaces.  Every xattr name is visible.  FreeBSD has two
universally defined namespaces: EXTATTR_NAMESPACE_USER and
EXTATTR_NAMESPACE_SYSTEM.  The system namespace is used for protected
FreeBSD-specific attributes such as MAC labels and pnfs state.  These
attributes have the namespace string "freebsd:system:" prefixed to the
name in the encoding scheme used by ZFS.  The user namespace is used
for general purpose user attributes and obeys normal access control
mechanisms.  These attributes have no namespace string prefixed, so
xattrs written on illumos are accessible in the user namespace on
FreeBSD, and xattrs written to the user namespace on FreeBSD are
accessible by the same name on illumos.

Linux has several xattr namespaces.  On Linux, ZFS encodes the
namespace in the xattr name for every namespace, including the user
namespace.  As a consequence, an xattr in the user namespace with the
name "foo" is stored by ZFS with the name "user.foo" and therefore
appears on FreeBSD and illumos to have the name "user.foo" rather than
"foo".  Conversely, none of the xattrs written on FreeBSD or illumos
are accessible on Linux unless the name happens to be prefixed with one
of the Linux xattr namespaces, in which case the namespace is stripped
from the name.  This makes xattrs entirely incompatible between Linux
and other platforms.

We want to make the encoding of user namespace xattrs compatible across
platforms.  A critical requirement of this compatibility is for xattrs
from existing pools from FreeBSD and illumos to be accessible by the
same names in the user namespace on Linux.  It is also necessary that
existing pools with xattrs written by Linux retain access to those
xattrs by the same names on Linux.  Making user namespace xattrs from
Linux accessible by the correct names on other platforms is important.
The handling of other namespaces is not required to be consistent.

Add a fallback mechanism for listing and getting xattrs to treat xattrs
as being in the user namespace if they do not match a known prefix.

When setting user namespace xattrs, do not prefix the namespace to the
name.  If the xattr is already present with the namespace prefix,
remove it so only the non-prefixed version persists.  This ensures
other platforms will be able to read the xattr with the correct name.

Do not allow setting or getting xattrs with a name that is prefixed
with one of the namespace names used by ZFS on supported platforms.

Make xattr namespace compatibility dependent on a new feature.  New
pools will use the compatible namespace encoding by default, and
existing pools will continue using the old Linux-specific encoding
until the feature is enabled.

Allow choosing between cross-platform compatability and legacy Linux
compatibility with a per-dataset property.  This facilitates
replication between hosts with different compatibility needs.

Add xattr_fallback property to skip fallbacks.

This property currently defaults to "on" so we do not miss xattrs in
datasets from porrls for which the encoding cannot be known or may even
be a mix of both.  It can be turned "off" for a performance boost when
it is known only the configured xattr_compat format is relevant.

Future work will investigate how we can change the default to "off" and
automatically force it to "on" when we cannot know whether skipping the
fallbacks is safe.

TODO:

* New tests should be added.
* Performance optimizations should be investigated.

Signed-off-by: Ryan Moeller <ryan@iXsystems.com>

include/os/freebsd/zfs/sys/zfs_vfsops_os.h

@@ -239,7 +239,10 @@ struct zfsvfs {
 	RW_WRITE_HELD(&(zfsvfs)->z_teardown_inactive_lock)
 #endif
 
-#define	ZSB_XATTR	0x0001		/* Enable user xattrs */
+#define	ZSB_XATTR		0x0001	/* Enable user xattrs */
+#define	ZSB_XATTR_COMPAT	0x0002	/* Enable cross-platform user xattrs */
+#define	ZSB_XATTR_FALLBACK	0x0004	/* Enable user xattr compat fallback */
+
 /*
  * Normal filesystems (those not under .zfs/snapshot) have a total
  * file ID size limited to 12 bytes (including the length field) due to

include/os/linux/zfs/sys/zfs_vfsops_os.h

@@ -170,7 +170,9 @@ struct zfsvfs {
 #define	ZFS_TEARDOWN_HELD(zfsvfs)		\
 	RRM_LOCK_HELD(&(zfsvfs)->z_teardown_lock)
 
-#define	ZSB_XATTR	0x0001		/* Enable user xattrs */
+#define	ZSB_XATTR		0x0001	/* Enable user xattrs */
+#define	ZSB_XATTR_COMPAT	0x0002	/* Enable cross-platform user xattrs */
+#define	ZSB_XATTR_FALLBACK	0x0004	/* Enable user xattr compat fallback */
 
 /*
  * Allow a maximum number of links.  While ZFS does not internally limit

include/sys/fs/zfs.h

@@ -186,6 +186,8 @@ typedef enum {
 	ZFS_PROP_IVSET_GUID,		/* not exposed to the user */
 	ZFS_PROP_REDACTED,
 	ZFS_PROP_REDACT_SNAPS,
+	ZFS_PROP_XATTR_COMPAT,
+	ZFS_PROP_XATTR_FALLBACK,
 	ZFS_NUM_PROPS
 } zfs_prop_t;
 
@@ -460,6 +462,11 @@ typedef enum zfs_key_location {
 #define	DEFAULT_PBKDF2_ITERATIONS 350000
 #define	MIN_PBKDF2_ITERATIONS 100000
 
+typedef enum zfs_xattr_compat {
+	ZFS_XATTR_COMPAT_LINUX = 0,
+	ZFS_XATTR_COMPAT_ALL,
+} zfs_xattr_compat_t;
+
 /*
  * On-disk version number.
  */
@@ -1616,7 +1623,6 @@ typedef enum {
 #define	ZFS_EV_HIST_DSID	"history_dsid"
 #define	ZFS_EV_RESILVER_TYPE	"resilver_type"
 
-
 /*
  * We currently support block sizes from 512 bytes to 16MB.
  * The benefits of larger blocks, and thus larger IO, need to be weighed
@@ -1638,7 +1644,6 @@ typedef enum {
 #define	SPA_OLD_MAXBLOCKSIZE	(1ULL << SPA_OLD_MAXBLOCKSHIFT)
 #define	SPA_MAXBLOCKSIZE	(1ULL << SPA_MAXBLOCKSHIFT)
 
-
 /* supported encryption algorithms */
 enum zio_encrypt {
 	ZIO_CRYPT_INHERIT = 0,
@@ -1656,6 +1661,35 @@ enum zio_encrypt {
 #define	ZIO_CRYPT_ON_VALUE	ZIO_CRYPT_AES_256_GCM
 #define	ZIO_CRYPT_DEFAULT	ZIO_CRYPT_OFF
 
+/*
+ * xattr namespace prefixes.  These are forbidden in xattr names.
+ *
+ * For cross-platform compatibility, xattrs in the user namespace should not be
+ * prefixed with the namespace name, but for backwards compatibility with older
+ * ZFS on Linux versions we do prefix the namespace.
+ */
+#define	ZFS_XA_NS_FREEBSD_PREFIX		"freebsd:"
+#define	ZFS_XA_NS_FREEBSD_PREFIX_LEN		strlen("freebsd:")
+#define	ZFS_XA_NS_LINUX_SECURITY_PREFIX		"security."
+#define	ZFS_XA_NS_LINUX_SECURITY_PREFIX_LEN	strlen("security.")
+#define	ZFS_XA_NS_LINUX_SYSTEM_PREFIX		"system."
+#define	ZFS_XA_NS_LINUX_SYSTEM_PREFIX_LEN	strlen("system.")
+#define	ZFS_XA_NS_LINUX_TRUSTED_PREFIX		"trusted."
+#define	ZFS_XA_NS_LINUX_TRUSTED_PREFIX_LEN	strlen("trusted.")
+#define	ZFS_XA_NS_LINUX_USER_PREFIX		"user."
+#define	ZFS_XA_NS_LINUX_USER_PREFIX_LEN		strlen("user.")
+
+/* BEGIN CSTYLED */
+#define	ZFS_XA_NS_PREFIX_MATCH(ns, name) \
+    (strncmp(name, ZFS_XA_NS_##ns##_PREFIX, ZFS_XA_NS_##ns##_PREFIX_LEN) == 0)
+
+#define	ZFS_XA_NS_PREFIX_FORBIDDEN(name) \
+    (ZFS_XA_NS_PREFIX_MATCH(FREEBSD, name) || \
+    ZFS_XA_NS_PREFIX_MATCH(LINUX_SECURITY, name) || \
+    ZFS_XA_NS_PREFIX_MATCH(LINUX_SYSTEM, name) || \
+    ZFS_XA_NS_PREFIX_MATCH(LINUX_TRUSTED, name) || \
+    ZFS_XA_NS_PREFIX_MATCH(LINUX_USER, name))
+/* END CSTYLED */
 
 #ifdef	__cplusplus
 }

include/zfeature_common.h

@@ -75,6 +75,7 @@ typedef enum spa_feature {
 	SPA_FEATURE_DEVICE_REBUILD,
 	SPA_FEATURE_ZSTD_COMPRESS,
 	SPA_FEATURE_DRAID,
+	SPA_FEATURE_XATTR_COMPAT,
 	SPA_FEATURES
 } spa_feature_t;
 

man/man7/zfsprops.7

@@ -1863,6 +1863,49 @@ are equivalent to the
 and
 .Sy noxattr
 mount options.
+.It Sy xattr_compat Ns = Ns Sy all Ns | Ns Sy linux
+Controls the preferred encoding of xattrs in the user namespace.
+When set to
+.Sy all
+(the default) with
+.Sy feature Ns @ Ns Ar xattr_compat
+enabled on the pool, xattrs written in the user namespace are stored in a
+format compatible across all supported platforms, and xattrs in the user
+namespace from all platforms are accessible.
+There is no notion of xattr namespaces on illumos, so all xattrs from
+illumos are presented in the user namespace on other platforms.
+The xattrs not in the user namespace are considered platform-specific and are
+not exposed on other platforms.
+Existing xattrs in the
+.Sy xattr_compat Ns = Ns Sy linux
+format are accessible and are replaced with the cross-platform compatible
+format when written.
+When
+.Sy feature Ns @ Ns Ar xattr_compat
+is disabled, xattrs behave as with
+.Sy xattr_compat Ns = Ns Sy linux
+on Linux and as with
+.Sy xattr_compat Ns = Ns Sy all
+elsewhere.
+When set to
+.Sy linux ,
+xattrs written in the user namespace are stored in a format that is compatible
+with ZFS on Linux prior to
+.Sy feature Ns @ Ns Ar xattr_compat
+but not compatible with ZFS on other platforms prior to this feature.
+See
+.Sy feature Ns @ Ns Ar xattr_compat
+in
+.Xr zpool-features 5
+for more information.
+.It Sy xattr_fallback Ns = Ns Sy on Ns | Ns Sy off
+Controls whether to fall back to the alternative encoding of xattrs in the
+user namespace for lookups.
+If accessing an xattr in the user namespace fails for the format configured by
+.Sy xattr_compat ,
+when set to
+.Sy on
+(the default), a second attempt will be made using the other xattr name format.
 .It Sy jailed Ns = Ns Sy off Ns | Ns Sy on
 Controls whether the dataset is managed from a jail.
 See

man/man7/zpool-features.7

@@ -782,6 +782,21 @@ by user and group.
 \*[instant-never]
 \*[remount-upgrade]
 .
+.feature com.ixsystems xattr_compat yes extensible_dataset enabled_txg
+This feature enables the use of a cross-platform compatible encoding for xattr
+names in the "user" namespace on Linux.
+.Pp
+This feature becomes
+.Sy active
+on a filesystem when an xattr is written to the "user" namespace after the
+feature has been enabled and
+.Sy xattr_compat Ns = Ns Sy all
+on Linux or
+.Sy xattr_compat Ns = Ns Sy linux
+on FreeBSD, and returns to
+.Sy enabled
+after all filesystems with the feature active have been destroyed.
+.
 .feature com.delphix zpool_checkpoint yes
 This feature enables the
 .Nm zpool Cm checkpoint

module/os/freebsd/zfs/zfs_vfsops.c

@@ -56,6 +56,7 @@
 #include <sys/sa_impl.h>
 #include <sys/policy.h>
 #include <sys/atomic.h>
+#include <sys/zfeature.h>
 #include <sys/zfs_ioctl.h>
 #include <sys/zfs_ctldir.h>
 #include <sys/zfs_fuid.h>
@@ -493,6 +494,42 @@ xattr_changed_cb(void *arg, uint64_t newval)
 	}
 }
 
+static void
+xattr_compat_changed_cb(void *arg, uint64_t newval)
+{
+	zfsvfs_t *zfsvfs = arg;
+
+	/*
+	 *  Force the old cross-platform compatible behavior if
+	 *  feature@xattr_compat is disabled.  This contrasts with
+	 *  Linux where the behavior prior to feature@xattr_compat
+	 *  was to use the incompatible Linux-only xattr format.
+	 */
+	if (!spa_feature_is_enabled(dmu_objset_spa(zfsvfs->z_os),
+	    SPA_FEATURE_XATTR_COMPAT))
+		newval = ZFS_XATTR_COMPAT_ALL;
+
+	switch (newval) {
+	case ZFS_XATTR_COMPAT_ALL:
+		zfsvfs->z_flags |= ZSB_XATTR_COMPAT;
+		break;
+	case ZFS_XATTR_COMPAT_LINUX:
+		zfsvfs->z_flags &= ~ZSB_XATTR_COMPAT;
+		break;
+	}
+}
+
+static void
+xattr_fallback_changed_cb(void *arg, uint64_t newval)
+{
+	zfsvfs_t *zfsvfs = arg;
+
+	if (newval)
+		zfsvfs->z_flags |= ZSB_XATTR_FALLBACK;
+	else
+		zfsvfs->z_flags &= ~ZSB_XATTR_FALLBACK;
+}
+
 static void
 blksz_changed_cb(void *arg, uint64_t newval)
 {
@@ -728,6 +765,12 @@ zfs_register_callbacks(vfs_t *vfsp)
 	    zfs_prop_to_name(ZFS_PROP_ATIME), atime_changed_cb, zfsvfs);
 	error = error ? error : dsl_prop_register(ds,
 	    zfs_prop_to_name(ZFS_PROP_XATTR), xattr_changed_cb, zfsvfs);
+	error = error ? error : dsl_prop_register(ds,
+	    zfs_prop_to_name(ZFS_PROP_XATTR_COMPAT), xattr_compat_changed_cb,
+	    zfsvfs);
+	error = error ? error : dsl_prop_register(ds,
+	    zfs_prop_to_name(ZFS_PROP_XATTR_FALLBACK),
+	    xattr_fallback_changed_cb, zfsvfs);
 	error = error ? error : dsl_prop_register(ds,
 	    zfs_prop_to_name(ZFS_PROP_RECORDSIZE), blksz_changed_cb, zfsvfs);
 	error = error ? error : dsl_prop_register(ds,
@@ -1242,6 +1285,14 @@ zfs_domount(vfs_t *vfsp, char *osname)
 		    "xattr", &pval, NULL)))
 			goto out;
 		xattr_changed_cb(zfsvfs, pval);
+		if ((error = dsl_prop_get_integer(osname,
+		    "xattr_compat", &pval, NULL)))
+			goto out;
+		xattr_compat_changed_cb(zfsvfs, pval);
+		if ((error = dsl_prop_get_integer(osname,
+		    "xattr_fallback", &pval, NULL)))
+			goto out;
+		xattr_fallback_changed_cb(zfsvfs, pval);
 		if ((error = dsl_prop_get_integer(osname,
 		    "acltype", &pval, NULL)))
 			goto out;