OpenZFS 8984 - fix for 6764 breaks ACL inheritance

Authored by: Dominik Hassler <hadfl@omniosce.org>
Reviewed by: Sam Zaydel <szaydel@racktopsystems.com>
Reviewed by: Paul B. Henson <henson@acm.org>
Reviewed by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
Approved by: Matthew Ahrens <mahrens@delphix.com>
Ported-by: Paul B. Henson <henson@acm.org>

OpenZFS-issue: https://www.illumos.org/issues/8984
OpenZFS-commit: openzfs/openzfs@e9bacc6d1a
Closes openzfs#10266

(cherry picked from commit 99495ba)

module/os/linux/zfs/zfs_acl.c

@@ -1646,7 +1646,7 @@ zfs_ace_can_use(umode_t obj_mode, uint16_t acep_flags)
  */
 static zfs_acl_t *
 zfs_acl_inherit(zfsvfs_t *zfsvfs, umode_t va_mode, zfs_acl_t *paclp,
-    uint64_t mode)
+    uint64_t mode, boolean_t *need_chmod)
 {
 	void		*pacep = NULL;
 	void		*acep;
@@ -1660,6 +1660,9 @@ zfs_acl_inherit(zfsvfs_t *zfsvfs, umode_t va_mode, zfs_acl_t *paclp,
 	size_t		data1sz, data2sz;
 	uint_t		aclinherit;
 	boolean_t	isdir = S_ISDIR(va_mode);
+	boolean_t	isreg = S_ISREG(va_mode);
+
+	*need_chmod = B_TRUE;
 
 	aclp = zfs_acl_alloc(paclp->z_version);
 	aclinherit = zfsvfs->z_acl_inherit;
@@ -1682,6 +1685,17 @@ zfs_acl_inherit(zfsvfs_t *zfsvfs, umode_t va_mode, zfs_acl_t *paclp,
 		    !zfs_ace_can_use(va_mode, iflags))
 			continue;
 
+		/*
+		 * If owner@, group@, or everyone@ inheritable
+		 * then zfs_acl_chmod() isn't needed.
+		 */
+		if ((aclinherit == ZFS_ACL_PASSTHROUGH ||
+		    aclinherit == ZFS_ACL_PASSTHROUGH_X) &&
+		    ((iflags & (ACE_OWNER|ACE_EVERYONE)) ||
+		    ((iflags & OWNING_GROUP) == OWNING_GROUP)) &&
+		    (isreg || (isdir && (iflags & ACE_DIRECTORY_INHERIT_ACE))))
+			*need_chmod = B_FALSE;
+
 		/*
 		 * Strip inherited execute permission from file if
 		 * not in mode
@@ -1769,6 +1783,7 @@ zfs_acl_ids_create(znode_t *dzp, int flag, vattr_t *vap, cred_t *cr,
 	zfsvfs_t	*zfsvfs = ZTOZSB(dzp);
 	zfs_acl_t	*paclp;
 	gid_t		gid = vap->va_gid;
+	boolean_t	need_chmod = B_TRUE;
 	boolean_t	trim = B_FALSE;
 	boolean_t	inherited = B_FALSE;
 
@@ -1862,7 +1877,7 @@ zfs_acl_ids_create(znode_t *dzp, int flag, vattr_t *vap, cred_t *cr,
 			VERIFY(0 == zfs_acl_node_read(dzp, B_TRUE,
 			    &paclp, B_FALSE));
 			acl_ids->z_aclp = zfs_acl_inherit(zfsvfs,
-			    vap->va_mode, paclp, acl_ids->z_mode);
+			    vap->va_mode, paclp, acl_ids->z_mode, &need_chmod);
 			inherited = B_TRUE;
 		} else {
 			acl_ids->z_aclp =
@@ -1872,15 +1887,18 @@ zfs_acl_ids_create(znode_t *dzp, int flag, vattr_t *vap, cred_t *cr,
 		mutex_exit(&dzp->z_lock);
 		mutex_exit(&dzp->z_acl_lock);
 
-		if (S_ISDIR(vap->va_mode))
-			acl_ids->z_aclp->z_hints |= ZFS_ACL_AUTO_INHERIT;
-
-		if (zfsvfs->z_acl_mode == ZFS_ACL_GROUPMASK &&
-		    zfsvfs->z_acl_inherit != ZFS_ACL_PASSTHROUGH &&
-		    zfsvfs->z_acl_inherit != ZFS_ACL_PASSTHROUGH_X)
-			trim = B_TRUE;
-		zfs_acl_chmod(S_ISDIR(vap->va_mode), acl_ids->z_mode, B_FALSE,
-		    trim, acl_ids->z_aclp);
+		if (need_chmod) {
+			if (S_ISDIR(vap->va_mode))
+				acl_ids->z_aclp->z_hints |=
+				    ZFS_ACL_AUTO_INHERIT;
+
+			if (zfsvfs->z_acl_mode == ZFS_ACL_GROUPMASK &&
+			    zfsvfs->z_acl_inherit != ZFS_ACL_PASSTHROUGH &&
+			    zfsvfs->z_acl_inherit != ZFS_ACL_PASSTHROUGH_X)
+				trim = B_TRUE;
+			zfs_acl_chmod(vap->va_mode, acl_ids->z_mode, B_FALSE,
+			    trim, acl_ids->z_aclp);
+		}
 	}
 
 	if (inherited || vsecp) {