Fix for openzfs#6845

The current on-disk format for encrypted datasets protects
not only the encrypted and authenticated blocks, but also
the order and interpretation of these blocks. In order to
make this work while maintaining the ability to do raw sends
the indirect bps maintain a secure checksum of all the MACs
in the block below it, along with a few other fields that
determine how the data is interpretted.

Unfortunately, the current on-disk format erroniously
includes the byteorder and compression of the blocks below,
which is not portable and thus cannot support raw sends.
Unfortunately, it is also not possible to easily work around
this issue due to a separate and much smaller bug which
causes indirect blocks for dnodes to not be compressed.

This patch zero's out the byteorder and compression when
computing the MAC (as they should have been) and registers
an errata for the on-disk format bug.

Signed-off-by: Tom Caputi <tcaputi@datto.com>

cmd/zpool/zpool_main.c

@@ -2119,6 +2119,17 @@ show_import(nvlist_t *config)
 				    "updating.\n"));
 				break;
 
+			case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
+				(void) printf(gettext(" action: existing "
+				    "encrypted datasets cannot be used with "
+				    "this\n\tversion of zfs due to an on-disk "
+				    "incompatibility which\n\tneeds to be "
+				    "corrected. Revert to an eariler"
+				    "version and\n\tbackup and destroy any "
+				    "existing encrypted datasets\n\tbefore "
+				    "updating.\n"));
+				break;
+
 			default:
 				/*
 				 * All errata must contain an action message.
@@ -6470,6 +6481,13 @@ status_callback(zpool_handle_t *zhp, void *data)
 			    "run 'zpool scrub'.\n"));
 			break;
 
+		case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
+			(void) printf(gettext("action: To correct the issue "
+			    "revert to an earlier version and backup\n\tand "
+			    "destroy any existing encrypted datasets before "
+			    "updating.\n"));
+			break;
+
 		default:
 			/*
 			 * All errata which allow the pool to be imported

include/sys/fs/zfs.h

@@ -890,6 +890,7 @@ typedef enum zpool_errata {
 	ZPOOL_ERRATA_NONE,
 	ZPOOL_ERRATA_ZOL_2094_SCRUB,
 	ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY,
+	ZPOOL_ERRATA_ZOL_6845_ENCRYPTION,
 } zpool_errata_t;
 
 /*

module/zfs/zio.c

@@ -401,6 +401,7 @@ zio_decrypt(zio_t *zio, abd_t *data, uint64_t size)
 	int ret;
 	void *tmp;
 	blkptr_t *bp = zio->io_bp;
+	spa_t *spa = zio->io_spa;
 	uint64_t lsize = BP_GET_LSIZE(bp);
 	dmu_object_type_t ot = BP_GET_TYPE(bp);
 	uint8_t salt[ZIO_DATA_SALT_LEN];
@@ -459,12 +460,12 @@ zio_decrypt(zio_t *zio, abd_t *data, uint64_t size)
 	 */
 	if (BP_IS_AUTHENTICATED(bp)) {
 		if (ot == DMU_OT_OBJSET) {
-			ret = spa_do_crypt_objset_mac_abd(B_FALSE, zio->io_spa,
+			ret = spa_do_crypt_objset_mac_abd(B_FALSE, spa,
 			    zio->io_bookmark.zb_objset, zio->io_abd, size,
 			    BP_SHOULD_BYTESWAP(bp));
 		} else {
 			zio_crypt_decode_mac_bp(bp, mac);
-			ret = spa_do_crypt_mac_abd(B_FALSE, zio->io_spa,
+			ret = spa_do_crypt_mac_abd(B_FALSE, spa,
 			    zio->io_bookmark.zb_objset, zio->io_abd, size, mac);
 		}
 		abd_copy(data, zio->io_abd, size);
@@ -485,7 +486,7 @@ zio_decrypt(zio_t *zio, abd_t *data, uint64_t size)
 		zio_crypt_decode_mac_bp(bp, mac);
 	}
 
-	ret = spa_do_crypt_abd(B_FALSE, zio->io_spa, zio->io_bookmark.zb_objset,
+	ret = spa_do_crypt_abd(B_FALSE, spa, zio->io_bookmark.zb_objset,
 	    bp, bp->blk_birth, size, data, zio->io_abd, iv, mac, salt,
 	    &no_crypt);
 	if (no_crypt)
@@ -508,8 +509,16 @@ zio_decrypt(zio_t *zio, abd_t *data, uint64_t size)
 		ret = SET_ERROR(EIO);
 		if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) {
 			zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION,
-			    zio->io_spa, NULL, &zio->io_bookmark, zio, 0, 0);
+			    spa, NULL, &zio->io_bookmark, zio, 0, 0);
 		}
+
+		/*
+		 * If this is a protected, indirect bp with a correct checksum
+		 * but a bad checksum-of-MACs we have found evidence of #6845
+		 * and we must report the errata.
+		 */
+		if (BP_HAS_INDIRECT_MAC_CKSUM(bp))
+			spa->spa_errata = ZPOOL_ERRATA_ZOL_6845_ENCRYPTION;
 	} else {
 		zio->io_error = ret;
 	}

module/zfs/zio_crypt.c

@@ -844,8 +844,10 @@ zio_crypt_copy_dnode_bonus(abd_t *src_abd, uint8_t *dst, uint_t datalen)
 static void
 zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp)
 {
+	BP_SET_BYTEORDER(bp, 0);
 	BP_SET_DEDUP(bp, 0);
 	BP_SET_CHECKSUM(bp, 0);
+	BP_SET_COMPRESS(bp, 0);
 
 	/*
 	 * psize cannot be set to zero or it will trigger asserts, but the