Compression, dedup, encryption

size_t
zio_get_compression_max_size(enum zio_compress compress, uint64_t gcd_alloc,
    uint64_t min_alloc, size_t s_len)
{
	size_t d_len;

	/* minimum 12.5% must be saved (legacy value, may be changed later) */
	d_len = s_len - (s_len >> 3);

	/* ZLE can't use exactly d_len bytes, it needs more, so ignore it */
	if (compress == ZIO_COMPRESS_ZLE)
		return (d_len);

	d_len = d_len - d_len % gcd_alloc;

	if (d_len < min_alloc)
		return (BPE_PAYLOAD_SIZE);
	return (d_len);
}

	/* If it's a compressed write that is not raw, compress the buffer. */
	if (compress != ZIO_COMPRESS_OFF &&
	    !(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) {
		abd_t *cabd = NULL;
		if (abd_cmp_zero(zio->io_abd, lsize) == 0)
			psize = 0;
		else if (compress == ZIO_COMPRESS_EMPTY)
			psize = lsize;
		else
			psize = zio_compress_data(compress, zio->io_abd, &cabd,
			    lsize,
			    zio_get_compression_max_size(compress,
			    spa->spa_gcd_alloc, spa->spa_min_alloc, lsize),
			    zp->zp_complevel);
		if (psize == 0) {
			compress = ZIO_COMPRESS_OFF;
		} else if (psize >= lsize) {
			compress = ZIO_COMPRESS_OFF;
			if (cabd != NULL)
				abd_free(cabd);
		} else if (psize <= BPE_PAYLOAD_SIZE && !zp->zp_encrypt &&
		    zp->zp_level == 0 && !DMU_OT_HAS_FILL(zp->zp_type) &&
		    spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA)) {
			void *cbuf = abd_borrow_buf_copy(cabd, lsize);
			encode_embedded_bp_compressed(bp,
			    cbuf, compress, lsize, psize);
			BPE_SET_ETYPE(bp, BP_EMBEDDED_TYPE_DATA);
			BP_SET_TYPE(bp, zio->io_prop.zp_type);
			BP_SET_LEVEL(bp, zio->io_prop.zp_level);
			abd_return_buf(cabd, cbuf, lsize);
			abd_free(cabd);
			BP_SET_LOGICAL_BIRTH(bp, zio->io_txg);
			zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
			ASSERT(spa_feature_is_active(spa,
			    SPA_FEATURE_EMBEDDED_DATA));
			return (zio);

		} else {
			/*
			 * Round compressed size up to the minimum allocation
			 * size of the smallest-ashift device, and zero the
			 * tail. This ensures that the compressed size of the
			 * BP (and thus compressratio property) are correct,
			 * in that we charge for the padding used to fill out
			 * the last sector.
			 */
			size_t rounded = (size_t)zio_roundup_alloc_size(spa,
			    psize);
			if (rounded >= lsize) {
				compress = ZIO_COMPRESS_OFF;
				abd_free(cabd);
				psize = lsize;
			} else {
				abd_zero_off(cabd, psize, rounded - psize);
				psize = rounded;
				zio_push_transform(zio, cabd,
				    psize, lsize, NULL);
			}
		}

	ddt_enter(ddt);
	/*
	 * Search DDT for matching entry.  Skip DVAs verification here, since
	 * they can go only from override, and once we get here the override
	 * pointer can't have "D" flag to be confused with pruned DDT entries.
	 */
	IMPLY(zio->io_bp_override, !BP_GET_DEDUP(zio->io_bp_override));
	dde = ddt_lookup(ddt, bp, B_FALSE);
	if (dde == NULL) {
		/* DDT size is over its quota so no new entries */
		ddt_exit(ddt);
		zp->zp_dedup = B_FALSE;
		BP_SET_DEDUP(bp, B_FALSE);
		if (zio->io_bp_override == NULL)
			zio->io_pipeline = ZIO_WRITE_PIPELINE;
		return (zio);
	}

	boolean_t valid = B_TRUE;

	if (dde->dde_type == DDT_TYPES &&
	    dde->dde_class == DDT_CLASSES &&
	    ddt_over_quota(spa)) {
		/* Over quota. If no one is waiting, clean up right now. */
		if (dde->dde_waiters == 0) {
			avl_remove(&ddt->ddt_tree, dde);
			ddt_free(ddt, dde);
			return (NULL);
		}

		/* Flag cleanup required */
		dde->dde_flags |= DDE_FLAG_OVERQUOTA;

		/*
		 * If there are enough DVAs in the entry to service our request,
		 * then we can just use them as-is.
		 */
		if (have_dvas >= need_dvas) {
			if (dde_io != NULL)
				mutex_exit(&dde_io->dde_io_lock);

			/*
			 * For rewrite operations, try preserving the original
			 * logical birth time.  If the result matches the
			 * original BP, this becomes a NOP.
			 */
			if (zp->zp_rewrite) {
				uint64_t orig_logical_birth =
				    BP_GET_LOGICAL_BIRTH(&zio->io_bp_orig);
				ddt_bp_fill(ddp, v, bp, orig_logical_birth);
				if (BP_EQUAL(bp, &zio->io_bp_orig)) {
					/* We can skip accounting. */
					ddt_exit(ddt);
					zio->io_flags |= ZIO_FLAG_NOPWRITE;
					return (zio);
				}
			}

			ddt_bp_fill(ddp, v, bp, txg);
			ddt_phys_addref(ddp, v);
			ddt_exit(ddt);
			return (zio);
		}

	/*
	 * Optimistically increment the refcount for this dedup write.
	 * If the write fails, zio_ddt_child_write_done() will decrement
	 * for all non-DDT-child parents.
	 */
	ddt_phys_addref(ddp, v);

	ddt_exit(ddt);

	zio_nowait(cio);

	return (zio);

 * ## Dedup log
 *
 * Historically, all entries modified on a txg were written back to dedup
 * storage objects at the end of every txg. This could cause significant
 * overheads, as each entry only takes up a tiny portion of a ZAP leaf node,
 * and so required reading the whole node, updating the entry, and writing it
 * back. On busy pools, this could add serious IO and memory overheads.
 *
 * To address this, the dedup log was added. If the "fast_dedup" feature is
 * enabled, at the end of each txg, modified entries will be copied to an
 * in-memory "log" object (ddt_log_t), and appended to an on-disk log. If the
 * same block is requested again, the in-memory object will be checked first,
 * and if its there, the entry inflated back onto the live tree without going
 * to storage. The on-disk log is only read at pool import time, to reload the
 * in-memory log.
 *
 * Each txg, some amount of the in-memory log will be flushed out to a DDT
 * storage object (ie ZAP) as normal. OpenZFS will try hard to flush enough to
 * keep up with the rate of change on dedup entries, but not so much that it
 * would impact overall throughput, and not using too much memory. See the
 * zfs_dedup_log_* tunables in zfs(4) for more details.

/*
 * The blkptr_t's of encrypted blocks also need to store the encryption
 * parameters so that the block can be decrypted. This layout is as follows:
 *
 *	64	56	48	40	32	24	16	8	0
 *	+-------+-------+-------+-------+-------+-------+-------+-------+
 * 0	|  pad  |	  vdev1         | pad   |	  ASIZE		|
 *	+-------+-------+-------+-------+-------+-------+-------+-------+
 * 1	|G|			 offset1				|
 *	+-------+-------+-------+-------+-------+-------+-------+-------+
 * 2	|  pad  |	  vdev2         | pad   |	  ASIZE		|
 *	+-------+-------+-------+-------+-------+-------+-------+-------+
 * 3	|G|			 offset2				|
 *	+-------+-------+-------+-------+-------+-------+-------+-------+
 * 4	|			salt					|
 *	+-------+-------+-------+-------+-------+-------+-------+-------+
 * 5	|			IV1					|
 *	+-------+-------+-------+-------+-------+-------+-------+-------+
 * 6	|BDX|lvl| type	| cksum |E| comp|    PSIZE	|     LSIZE	|
 *	+-------+-------+-------+-------+-------+-------+-------+-------+
 * 7	|R|			padding					|
 *	+-------+-------+-------+-------+-------+-------+-------+-------+
 * 8	|			padding					|
 *	+-------+-------+-------+-------+-------+-------+-------+-------+
 * 9	|			physical birth txg			|
 *	+-------+-------+-------+-------+-------+-------+-------+-------+
 * a	|			logical birth txg			|
 *	+-------+-------+-------+-------+-------+-------+-------+-------+
 * b	|		IV2		|	    fill count		|
 *	+-------+-------+-------+-------+-------+-------+-------+-------+
 * c	|			checksum[0]				|
 *	+-------+-------+-------+-------+-------+-------+-------+-------+
 * d	|			checksum[1]				|
 *	+-------+-------+-------+-------+-------+-------+-------+-------+
 * e	|			MAC[0]					|
 *	+-------+-------+-------+-------+-------+-------+-------+-------+
 * f	|			MAC[1]					|
 *	+-------+-------+-------+-------+-------+-------+-------+-------+

	/* indirect blocks only maintain a cksum of the lower level MACs */
	if (BP_GET_LEVEL(bp) > 0) {
		BP_SET_CRYPT(bp, B_TRUE);
		VERIFY0(zio_crypt_do_indirect_mac_checksum_abd(B_TRUE,
		    zio->io_orig_abd, BP_GET_LSIZE(bp), BP_SHOULD_BYTESWAP(bp),
		    mac));
		zio_crypt_encode_mac_bp(bp, mac);
		return (zio);
	}

	/*
	 * Objset blocks are a special case since they have 2 256-bit MACs
	 * embedded within them.
	 */
	if (ot == DMU_OT_OBJSET) {
		ASSERT0(DMU_OT_IS_ENCRYPTED(ot));
		ASSERT3U(BP_GET_COMPRESS(bp), ==, ZIO_COMPRESS_OFF);
		BP_SET_CRYPT(bp, B_TRUE);
		VERIFY0(spa_do_crypt_objset_mac_abd(B_TRUE, spa, dsobj,
		    zio->io_abd, psize, BP_SHOULD_BYTESWAP(bp)));
		return (zio);
	}

	/* unencrypted object types are only authenticated with a MAC */
	if (!DMU_OT_IS_ENCRYPTED(ot)) {
		BP_SET_CRYPT(bp, B_TRUE);
		VERIFY0(spa_do_crypt_mac_abd(B_TRUE, spa, dsobj,
		    zio->io_abd, psize, mac));
		zio_crypt_encode_mac_bp(bp, mac);
		return (zio);
	}

	/*
	 * All objects in an encrypted objset are protected from modification
	 * via a MAC. Encrypted objects store their IV and salt in the last DVA
	 * in the bp, so we cannot use all copies. Encrypted objects are also
	 * not subject to nopwrite since writing the same data will still
	 * result in a new ciphertext. Only encrypted blocks can be dedup'd
	 * to avoid ambiguity in the dedup code since the DDT does not store
	 * object types.
	 */
	if (os->os_encrypted && (wp & WP_NOFILL) == 0) {
		encrypt = B_TRUE;

		if (DMU_OT_IS_ENCRYPTED(type)) {
			copies = MIN(copies, SPA_DVAS_PER_BP - 1);
			gang_copies = MIN(gang_copies, SPA_DVAS_PER_BP - 1);
			nopwrite = B_FALSE;
		} else {
			dedup = B_FALSE;
		}

		if (level <= 0 &&
		    (type == DMU_OT_DNODE || type == DMU_OT_OBJSET)) {
			compress = ZIO_COMPRESS_EMPTY;
		}
	}

 * Message Authentication Code (MAC)
 * The MAC is an output of authenticated encryption modes such as AES-GCM and
 * AES-CCM. Its purpose is to ensure that an attacker cannot modify encrypted
 * data on disk and return garbage to the application. Effectively, it is a
 * checksum that can not be reproduced by an attacker. We store the MAC in the
 * second 128 bits of blk_cksum, leaving the first 128 bits for a truncated
 * regular checksum of the ciphertext which can be used for scrubbing.
 *
 * OBJECT AUTHENTICATION:
 * Some object types, such as DMU_OT_MASTER_NODE cannot be encrypted because
 * they contain some info that always needs to be readable. To prevent this
 * data from being altered, we authenticate this data using SHA512-HMAC. This
 * will produce a MAC (similar to the one produced via encryption) which can
 * be used to verify the object was not modified. HMACs do not require key
 * rotation or IVs, so we can keep up to the full 3 copies of authenticated
 * data.
 *
 * ZIL ENCRYPTION:
 * ZIL blocks have their bp written to disk ahead of the associated data, so we
 * cannot store the MAC there as we normally do. For these blocks the MAC is
 * stored in the embedded checksum within the zil_chain_t header. The salt and
 * IV are generated for the block on bp allocation instead of at encryption
 * time. In addition, ZIL blocks have some pieces that must be left in plaintext
 * for claiming even though all of the sensitive user data still needs to be
 * encrypted. The function zio_crypt_init_uios_zil() handles parsing which
 * pieces of the block need to be encrypted. All data that is not encrypted is
 * authenticated using the AAD mechanisms that the supported encryption modes
 * provide for. In order to preserve the semantics of the ZIL for encrypted
 * datasets, the ZIL is not protected at the objset level as described below.
 *
 * DNODE ENCRYPTION:
 * Similarly to ZIL blocks, the core part of each dnode_phys_t needs to be left
 * in plaintext for scrubbing and claiming, but the bonus buffers might contain
 * sensitive user data. The function zio_crypt_init_uios_dnode() handles parsing
 * which pieces of the block need to be encrypted. For more details about
 * dnode authentication and encryption, see zio_crypt_init_uios_dnode().

	/*
	 * MAC checksums are a special case since half of this checksum will
	 * actually be the encryption MAC. This will be verified by the
	 * decryption process, so we just check the truncated checksum now.
	 * Objset blocks use embedded MACs so we don't truncate the checksum
	 * for them.
	 */
	if (bp != NULL && BP_USES_CRYPT(bp) &&
	    BP_GET_TYPE(bp) != DMU_OT_OBJSET) {
		if (!(ci->ci_flags & ZCHECKSUM_FLAG_DEDUP)) {
			actual_cksum.zc_word[0] ^= actual_cksum.zc_word[2];
			actual_cksum.zc_word[1] ^= actual_cksum.zc_word[3];
		}

		actual_cksum.zc_word[2] = 0;
		actual_cksum.zc_word[3] = 0;
		expected_cksum.zc_word[2] = 0;
		expected_cksum.zc_word[3] = 0;
	}