blob: e07a37cf8b5fbe2ec696e1027c17788fdfa7d0a0 [file] [log] [blame]
// SPDX-License-Identifier: GPL-2.0
* Copyright 2019 Google LLC
* Refer to Documentation/block/inline-encryption.rst for detailed explanation.
#define pr_fmt(fmt) "blk-crypto: " fmt
#include <linux/blk-crypto.h>
#include <linux/blkdev.h>
#include <linux/keyslot-manager.h>
#include <linux/random.h>
#include <linux/siphash.h>
#include "blk-crypto-internal.h"
const struct blk_crypto_mode blk_crypto_modes[] = {
.cipher_str = "xts(aes)",
.keysize = 64,
.ivsize = 16,
.cipher_str = "essiv(cbc(aes),sha256)",
.keysize = 16,
.ivsize = 16,
.cipher_str = "adiantum(xchacha12,aes)",
.keysize = 32,
.ivsize = 32,
/* Check that all I/O segments are data unit aligned */
static int bio_crypt_check_alignment(struct bio *bio)
const unsigned int data_unit_size =
struct bvec_iter iter;
struct bio_vec bv;
bio_for_each_segment(bv, bio, iter) {
if (!IS_ALIGNED(bv.bv_len | bv.bv_offset, data_unit_size))
return -EIO;
return 0;
* blk_crypto_submit_bio - handle submitting bio for inline encryption
* @bio_ptr: pointer to original bio pointer
* If the bio doesn't have inline encryption enabled or the submitter already
* specified a keyslot for the target device, do nothing. Else, a raw key must
* have been provided, so acquire a device keyslot for it if supported. Else,
* use the crypto API fallback.
* When the crypto API fallback is used for encryption, blk-crypto may choose to
* split the bio into 2 - the first one that will continue to be processed and
* the second one that will be resubmitted via generic_make_request.
* A bounce bio will be allocated to encrypt the contents of the aforementioned
* "first one", and *bio_ptr will be updated to this bounce bio.
* Return: 0 if bio submission should continue; nonzero if bio_endio() was
* already called so bio submission should abort.
int blk_crypto_submit_bio(struct bio **bio_ptr)
struct bio *bio = *bio_ptr;
struct request_queue *q;
struct bio_crypt_ctx *bc = bio->bi_crypt_context;
int err;
if (!bc || !bio_has_data(bio))
return 0;
* When a read bio is marked for fallback decryption, its bi_iter is
* saved so that when we decrypt the bio later, we know what part of it
* was marked for fallback decryption (when the bio is passed down after
* blk_crypto_submit bio, it may be split or advanced so we cannot rely
* on the bi_iter while decrypting in blk_crypto_endio)
if (bio_crypt_fallback_crypted(bc))
return 0;
err = bio_crypt_check_alignment(bio);
if (err) {
bio->bi_status = BLK_STS_IOERR;
goto out;
q = bio->bi_disk->queue;
if (bc->bc_ksm) {
/* Key already programmed into device? */
if (q->ksm == bc->bc_ksm)
return 0;
/* Nope, release the existing keyslot. */
/* Get device keyslot if supported */
if (keyslot_manager_crypto_mode_supported(q->ksm,
bc->bc_key->is_hw_wrapped)) {
err = bio_crypt_ctx_acquire_keyslot(bc, q->ksm);
if (!err)
return 0;
pr_warn_once("Failed to acquire keyslot for %s (err=%d). Falling back to crypto API.\n",
bio->bi_disk->disk_name, err);
/* Fallback to crypto API */
err = blk_crypto_fallback_submit_bio(bio_ptr);
if (err)
goto out;
return 0;
return err;
* blk_crypto_endio - clean up bio w.r.t inline encryption during bio_endio
* @bio: the bio to clean up
* If blk_crypto_submit_bio decided to fallback to crypto API for this bio,
* we queue the bio for decryption into a workqueue and return false,
* and call bio_endio(bio) at a later time (after the bio has been decrypted).
* If the bio is not to be decrypted by the crypto API, this function releases
* the reference to the keyslot that blk_crypto_submit_bio got.
* Return: true if bio_endio should continue; false otherwise (bio_endio will
* be called again when bio has been decrypted).
bool blk_crypto_endio(struct bio *bio)
struct bio_crypt_ctx *bc = bio->bi_crypt_context;
if (!bc)
return true;
if (bio_crypt_fallback_crypted(bc)) {
* The only bios who's crypto is handled by the blk-crypto
* fallback when they reach here are those with
* bio_data_dir(bio) == READ, since WRITE bios that are
* encrypted by the crypto API fallback are handled by
* blk_crypto_encrypt_endio().
return !blk_crypto_queue_decrypt_bio(bio);
if (bc->bc_keyslot >= 0)
return true;
* blk_crypto_init_key() - Prepare a key for use with blk-crypto
* @blk_key: Pointer to the blk_crypto_key to initialize.
* @raw_key: Pointer to the raw key.
* @raw_key_size: Size of raw key. Must be at least the required size for the
* chosen @crypto_mode; see blk_crypto_modes[]. (It's allowed
* to be longer than the mode's actual key size, in order to
* support inline encryption hardware that accepts wrapped keys.
* @is_hw_wrapped has to be set for such keys)
* @is_hw_wrapped: Denotes @raw_key is wrapped.
* @crypto_mode: identifier for the encryption algorithm to use
* @dun_bytes: number of bytes that will be used to specify the DUN when this
* key is used
* @data_unit_size: the data unit size to use for en/decryption
* Return: The blk_crypto_key that was prepared, or an ERR_PTR() on error. When
* done using the key, it must be freed with blk_crypto_free_key().
int blk_crypto_init_key(struct blk_crypto_key *blk_key,
const u8 *raw_key, unsigned int raw_key_size,
bool is_hw_wrapped,
enum blk_crypto_mode_num crypto_mode,
unsigned int dun_bytes,
unsigned int data_unit_size)
const struct blk_crypto_mode *mode;
static siphash_key_t hash_key;
u32 hash;
memset(blk_key, 0, sizeof(*blk_key));
if (crypto_mode >= ARRAY_SIZE(blk_crypto_modes))
return -EINVAL;
mode = &blk_crypto_modes[crypto_mode];
if (is_hw_wrapped) {
if (raw_key_size < mode->keysize ||
return -EINVAL;
} else {
if (raw_key_size != mode->keysize)
return -EINVAL;
if (dun_bytes <= 0 || dun_bytes > BLK_CRYPTO_MAX_IV_SIZE)
return -EINVAL;
if (!is_power_of_2(data_unit_size))
return -EINVAL;
blk_key->crypto_mode = crypto_mode;
blk_key->data_unit_size = data_unit_size;
blk_key->data_unit_size_bits = ilog2(data_unit_size);
blk_key->size = raw_key_size;
blk_key->is_hw_wrapped = is_hw_wrapped;
memcpy(blk_key->raw, raw_key, raw_key_size);
* The keyslot manager uses the SipHash of the key to implement O(1) key
* lookups while avoiding leaking information about the keys. It's
* precomputed here so that it only needs to be computed once per key.
get_random_once(&hash_key, sizeof(hash_key));
hash = (u32)siphash(raw_key, raw_key_size, &hash_key);
blk_crypto_key_set_hash_and_dun_bytes(blk_key, hash, dun_bytes);
return 0;
* blk_crypto_start_using_mode() - Start using blk-crypto on a device
* @crypto_mode: the crypto mode that will be used
* @dun_bytes: number of bytes that will be used to specify the DUN
* @data_unit_size: the data unit size that will be used
* @is_hw_wrapped_key: whether the key will be hardware-wrapped
* @q: the request queue for the device
* Upper layers must call this function to ensure that either the hardware
* supports the needed crypto settings, or the crypto API fallback has
* transforms for the needed mode allocated and ready to go.
* Return: 0 on success; -ENOPKG if the hardware doesn't support the crypto
* settings and blk-crypto-fallback is either disabled or the needed
* algorithm is disabled in the crypto API; or another -errno code.
int blk_crypto_start_using_mode(enum blk_crypto_mode_num crypto_mode,
unsigned int dun_bytes,
unsigned int data_unit_size,
bool is_hw_wrapped_key,
struct request_queue *q)
if (keyslot_manager_crypto_mode_supported(q->ksm, crypto_mode,
dun_bytes, data_unit_size,
return 0;
if (is_hw_wrapped_key) {
pr_warn_once("hardware doesn't support wrapped keys\n");
return blk_crypto_fallback_start_using_mode(crypto_mode);
* blk_crypto_evict_key() - Evict a key from any inline encryption hardware
* it may have been programmed into
* @q: The request queue who's keyslot manager this key might have been
* programmed into
* @key: The key to evict
* Upper layers (filesystems) should call this function to ensure that a key
* is evicted from hardware that it might have been programmed into. This
* will call keyslot_manager_evict_key on the queue's keyslot manager, if one
* exists, and supports the crypto algorithm with the specified data unit size.
* Otherwise, it will evict the key from the blk-crypto-fallback's ksm.
* Return: 0 on success, -err on error.
int blk_crypto_evict_key(struct request_queue *q,
const struct blk_crypto_key *key)
if (q->ksm &&
keyslot_manager_crypto_mode_supported(q->ksm, key->crypto_mode,
return keyslot_manager_evict_key(q->ksm, key);
return blk_crypto_fallback_evict_key(key);