diff --git a/drivers/net/ovpn/crypto.h b/drivers/net/ovpn/crypto.h index 4ed44bab031f..454ccbaf157f 100644 --- a/drivers/net/ovpn/crypto.h +++ b/drivers/net/ovpn/crypto.h @@ -10,6 +10,7 @@ #ifndef _NET_OVPN_OVPNCRYPTO_H_ #define _NET_OVPN_OVPNCRYPTO_H_ +#include #include #include #include @@ -57,6 +58,8 @@ struct ovpn_peer_key_reset { struct ovpn_key_config key; }; +#define OVPN_CRYPTO_TX_ROTATE_PENDING 0 + /* state for one concrete AEAD key direction */ struct ovpn_key_ctx { u16 epoch; @@ -83,6 +86,7 @@ struct ovpn_crypto_key_slot { unsigned int pktid_size; unsigned int payload_offset; unsigned int tail_tag_size; + unsigned long flags; struct ovpn_epoch_key epoch_key_send; struct ovpn_epoch_key epoch_key_recv; diff --git a/drivers/net/ovpn/crypto_aead.c b/drivers/net/ovpn/crypto_aead.c index 4daf0314b046..033bfd4769a2 100644 --- a/drivers/net/ovpn/crypto_aead.c +++ b/drivers/net/ovpn/crypto_aead.c @@ -142,6 +142,160 @@ static struct scatterlist *ovpn_aead_crypto_req_sg(struct crypto_aead *aead, __alignof__(struct scatterlist)); } +/** + * ovpn_advance_encrypt_key - promote TX to a target epoch + * @ks: key slot containing TX epoch state + * @target_epoch: epoch to promote TX to + * @required: whether the caller must stop if promotion cannot complete + * + * TX promotion consumes prederived future keys up to @target_epoch. If another + * context is already updating TX state or refill has not caught up, optional + * promotions keep using the current key while required promotions fail in the + * caller. + * + * Return: 0 on success, -EINPROGRESS if the TX lock is busy, -EALREADY if TX + * is already at or beyond @target_epoch, -ENOKEY if the future ring cannot + * provide @target_epoch, or another negative error code otherwise. + */ +int ovpn_advance_encrypt_key(struct ovpn_crypto_key_slot *ks, u16 target_epoch, + bool required) +{ + struct ovpn_key_ctx *stale[OVPN_EPOCH_FUTURE_KEYS_COUNT]; + u16 stale_count = 0, advance, count, index, i; + struct ovpn_key_ctx *old_encrypt; + struct ovpn_key_ctx *new_encrypt; + bool lock_held; + + /* concurrent promotion or refill means another context is moving tx */ + if (unlikely(!spin_trylock_bh(&ks->tx_lock))) + return -EINPROGRESS; + + lock_held = lockdep_is_held(&ks->tx_lock); + old_encrypt = rcu_dereference_protected(ks->encrypt, + lock_held); + if (unlikely(old_encrypt->epoch >= target_epoch)) { + spin_unlock_bh(&ks->tx_lock); + return -EALREADY; + } + + count = ovpn_epoch_future_keys_count(&ks->future_tx_keys); + advance = target_epoch - old_encrypt->epoch; + /* refill can lag behind packet processing under pressure */ + if (unlikely(count < advance)) { + spin_unlock_bh(&ks->tx_lock); + ovpn_schedule_refill(ks, true); + return -ENOKEY; + } + + index = (ks->future_tx_keys.tail + advance - 1) % + OVPN_EPOCH_FUTURE_KEYS_COUNT; + new_encrypt = rcu_dereference_protected(ks->future_tx_keys.keys[index], + lock_held); + if (WARN_ON_ONCE(!new_encrypt || + new_encrypt->epoch != target_epoch)) { + spin_unlock_bh(&ks->tx_lock); + return -ENOKEY; + } + + for (i = 0; i < advance; i++) { + struct ovpn_key_ctx __rcu **slot; + struct ovpn_key_ctx *future; + u16 stale_index; + + /* drop skipped future keys when advancing past them */ + stale_index = (ks->future_tx_keys.tail + i) % + OVPN_EPOCH_FUTURE_KEYS_COUNT; + slot = &ks->future_tx_keys.keys[stale_index]; + future = rcu_dereference_protected(*slot, lock_held); + if (future != new_encrypt) + stale[stale_count++] = future; + RCU_INIT_POINTER(*slot, NULL); + } + + /* each concrete key carries fresh packet-ID state */ + rcu_assign_pointer(ks->encrypt, new_encrypt); + ks->future_tx_keys.tail = (ks->future_tx_keys.tail + advance) % + OVPN_EPOCH_FUTURE_KEYS_COUNT; + ks->future_tx_keys.full = false; + spin_unlock_bh(&ks->tx_lock); + + ovpn_key_ctx_put(old_encrypt); + for (i = 0; i < stale_count; i++) + ovpn_key_ctx_put(stale[i]); + + ovpn_schedule_refill(ks, true); + + return 0; +} + +static int ovpn_aead_encrypt_next_seq(struct ovpn_peer *peer, + struct ovpn_crypto_key_slot *ks, + struct ovpn_key_ctx *encrypt, + unsigned int pkt_len, u64 *seq) +{ + bool required = false; + bool pktid_notify; + u64 aead_blocks; + int ret; + + /* hard packet-ID exhaustion requires fresh key material */ + ret = ovpn_pktid_xmit_next(&encrypt->pid.xmit, seq); + if (unlikely(ret < 0)) { + if (!ks->epoch_format) + return ret; + required = true; + goto new_key; + } + pktid_notify = ret > 0; + + aead_blocks = ovpn_aead_limit_blocks(ks->cipher_alg, ks->aad_size, + pkt_len); + ret = ovpn_key_usage_xmit(&encrypt->usage, &ks->usage_limit, *seq, + aead_blocks, pktid_notify); + if (unlikely(ret < 0)) { + /* epoch keys rotate internally instead of asking userspace */ + if (!ks->epoch_format) + return ret; + required = true; + goto new_key; + } + if (unlikely(ret > 0)) { + if (!ks->epoch_format) { + ovpn_nl_key_swap_notify(peer, ks->key_id); + return 0; + } + /* epoch tx rotates locally when the soft limit is crossed */ + set_bit(OVPN_CRYPTO_TX_ROTATE_PENDING, &ks->flags); + } + + if (!ks->epoch_format) + return 0; + + /* rx can request a tx epoch bump without adding rx work to every + * tx packet + */ + if (unlikely(test_bit(OVPN_CRYPTO_TX_ROTATE_PENDING, &ks->flags))) + goto new_key; + + return 0; + +new_key: + /* keep enough epoch space for the future-key ring after promotion */ + if (unlikely(encrypt->epoch + 1 + OVPN_EPOCH_FUTURE_KEYS_COUNT >= + OVPN_MAX_EPOCH)) + return -ERANGE; + + ret = ovpn_advance_encrypt_key(ks, encrypt->epoch + 1, required); + if (unlikely(ret == -EINPROGRESS || ret == -ENOKEY)) + return required ? -EBUSY : 0; + if (unlikely(ret < 0 && ret != -EALREADY)) + return ret; + + clear_bit(OVPN_CRYPTO_TX_ROTATE_PENDING, &ks->flags); + + return -EAGAIN; +} + int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks, struct sk_buff *skb) { @@ -151,8 +305,7 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks, struct sk_buff *trailer; struct scatterlist *sg; unsigned int tag_size; - bool pktid_notify; - u64 aead_blocks; + bool retried = false; int nfrags, ret; u64 pktid, seq; void *tmp; @@ -175,6 +328,7 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks, * [ 8-byte packet ID ] */ +retry: ret = ovpn_key_ctx_get(&key, &ks->encrypt); if (unlikely(ret)) return ret; @@ -182,19 +336,20 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks, tag_size = crypto_aead_authsize(key->tfm); - ret = ovpn_pktid_xmit_next(&key->pid.xmit, &seq); - if (unlikely(ret < 0)) - return ret; - pktid_notify = ret > 0; - - aead_blocks = ovpn_aead_limit_blocks(ks->cipher_alg, ks->aad_size, - plaintext_len); - ret = ovpn_key_usage_xmit(&key->usage, &ks->usage_limit, seq, - aead_blocks, pktid_notify); + ret = ovpn_aead_encrypt_next_seq(peer, ks, key, plaintext_len, &seq); + if (unlikely(ret == -EAGAIN)) { + if (retried) + return -EBUSY; + + /* retry once after epoch promotion */ + ovpn_skb_cb(skb)->key = NULL; + ovpn_key_ctx_put(key); + key = NULL; + retried = true; + goto retry; + } if (unlikely(ret < 0)) return ret; - if (unlikely(ret > 0 && !ks->epoch_format)) - ovpn_nl_key_swap_notify(peer, ks->key_id); /* check that there's enough headroom in the skb for packet * encapsulation diff --git a/drivers/net/ovpn/crypto_aead.h b/drivers/net/ovpn/crypto_aead.h index 4f83a3aa37fd..c2c2fbd72cfa 100644 --- a/drivers/net/ovpn/crypto_aead.h +++ b/drivers/net/ovpn/crypto_aead.h @@ -15,6 +15,9 @@ #include #include +int ovpn_advance_encrypt_key(struct ovpn_crypto_key_slot *ks, u16 target_epoch, + bool required); + int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks, struct sk_buff *skb); int ovpn_aead_decrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks, diff --git a/drivers/net/ovpn/crypto_key.c b/drivers/net/ovpn/crypto_key.c index b4f40db4b30b..e7172a919bad 100644 --- a/drivers/net/ovpn/crypto_key.c +++ b/drivers/net/ovpn/crypto_key.c @@ -104,7 +104,8 @@ void ovpn_key_ctx_release(struct kref *kref) static struct ovpn_key_ctx * ovpn_key_ctx_new(const char *title, const char *alg_name, const u8 *cipher_key, unsigned int cipher_key_len, - const u8 *implicit_iv, u16 epoch, bool encrypt) + const u8 *implicit_iv, u16 epoch, bool epoch_format, + bool encrypt) { struct ovpn_limit pktid_limit; struct ovpn_key_ctx *key; @@ -134,7 +135,7 @@ ovpn_key_ctx_new(const char *title, const char *alg_name, /* initialize only the packet ID direction this context owns */ if (encrypt) { - ovpn_pktid_xmit_limit_init(&pktid_limit, false); + ovpn_pktid_xmit_limit_init(&pktid_limit, epoch_format); ovpn_pktid_xmit_init(&key->pid.xmit, &pktid_limit); } else { ovpn_pktid_recv_init(&key->pid.recv); @@ -158,7 +159,7 @@ ovpn_key_ctx_create_direct(bool encrypt, const char *alg_name, key = ovpn_key_ctx_new(encrypt ? "encrypt" : "decrypt", alg_name, dir->cipher_key, dir->cipher_key_size, - implicit_iv, 0, encrypt); + implicit_iv, 0, false, encrypt); memzero_explicit(implicit_iv, sizeof(implicit_iv)); return key; @@ -179,7 +180,7 @@ ovpn_key_ctx_create_epoch(bool encrypt, const char *alg_name, key = ovpn_key_ctx_new(encrypt ? "encrypt" : "decrypt", alg_name, cipher_key, epoch_key->cipher_key_len, - implicit_iv, epoch_key->epoch, encrypt); + implicit_iv, epoch_key->epoch, true, encrypt); memzero_explicit(cipher_key, sizeof(cipher_key)); memzero_explicit(implicit_iv, sizeof(implicit_iv)); diff --git a/drivers/net/ovpn/io.c b/drivers/net/ovpn/io.c index de145fb0b758..a8485a73e585 100644 --- a/drivers/net/ovpn/io.c +++ b/drivers/net/ovpn/io.c @@ -105,6 +105,200 @@ static void ovpn_netdev_write(struct ovpn_peer *peer, struct sk_buff *skb) local_bh_enable(); } +static void ovpn_rx_request_tx_rotation(struct ovpn_crypto_key_slot *ks, + const struct ovpn_key_ctx *decrypt) +{ + struct ovpn_key_ctx *current_decrypt; + struct ovpn_key_ctx *encrypt; + + if (unlikely(!ks->epoch_format)) + return; + + rcu_read_lock(); + current_decrypt = rcu_dereference(ks->decrypt); + encrypt = rcu_dereference(ks->encrypt); + /* only current rx key usage can ask the peer to move forward */ + if (decrypt == current_decrypt && + (!encrypt || decrypt->epoch >= encrypt->epoch)) + set_bit(OVPN_CRYPTO_TX_ROTATE_PENDING, &ks->flags); + rcu_read_unlock(); +} + +/** + * ovpn_advance_decrypt_key - promote RX to a target epoch + * @ks: key slot containing RX epoch state + * @target_epoch: authenticated epoch to promote RX to + * + * RX promotion consumes future keys up to @target_epoch, moves the closest + * previous epoch to the retiring slot for reordered packets, and schedules + * refill for the consumed future-key slots. Local TX is advanced as well so + * outbound packets signal the peer to leave older epochs. + * + * Return: 0 on success, -EINPROGRESS if the RX lock is busy, -EALREADY if RX + * is already at or beyond @target_epoch, or -EINVAL if the future ring cannot + * provide @target_epoch. + */ +static int ovpn_advance_decrypt_key(struct ovpn_crypto_key_slot *ks, + u16 target_epoch) +{ + struct ovpn_key_ctx *stale[OVPN_EPOCH_FUTURE_KEYS_COUNT]; + u16 stale_count = 0, advance, count, index, i; + struct ovpn_key_ctx __rcu **retire_slot; + struct ovpn_key_ctx *old_retiring; + struct ovpn_key_ctx *new_retiring; + struct ovpn_key_ctx *old_decrypt; + struct ovpn_key_ctx *new_decrypt; + u16 retire_index; + bool lock_held; + int ret; + + /* concurrent promotion or refill means another context is moving rx */ + if (unlikely(!spin_trylock_bh(&ks->rx_lock))) + return -EINPROGRESS; + + lock_held = lockdep_is_held(&ks->rx_lock); + old_retiring = rcu_dereference_protected(ks->retiring_key, + lock_held); + old_decrypt = rcu_dereference_protected(ks->decrypt, + lock_held); + /* current decrypt key is always installed while the slot is alive */ + if (unlikely(target_epoch <= old_decrypt->epoch)) { + spin_unlock_bh(&ks->rx_lock); + return -EALREADY; + } + + count = ovpn_epoch_future_keys_count(&ks->future_rx_keys); + advance = target_epoch - old_decrypt->epoch; + /* authenticated epoch must be available in the future ring */ + if (unlikely(count < advance)) { + spin_unlock_bh(&ks->rx_lock); + return -EINVAL; + } + + /* get the authenticated future key from the ring */ + index = (ks->future_rx_keys.tail + advance - 1) % + OVPN_EPOCH_FUTURE_KEYS_COUNT; + new_decrypt = rcu_dereference_protected(ks->future_rx_keys.keys[index], + lock_held); + if (WARN_ON_ONCE(!new_decrypt || + new_decrypt->epoch != target_epoch)) { + spin_unlock_bh(&ks->rx_lock); + return -EINVAL; + } + + new_retiring = old_decrypt; + if (unlikely(advance > 1)) { + /* keep target_epoch - 1 for reordered packets after jumps */ + retire_index = (ks->future_rx_keys.tail + advance - 2) % + OVPN_EPOCH_FUTURE_KEYS_COUNT; + retire_slot = &ks->future_rx_keys.keys[retire_index]; + new_retiring = rcu_dereference_protected(*retire_slot, + lock_held); + if (WARN_ON_ONCE(!new_retiring || + new_retiring->epoch != target_epoch - 1)) { + spin_unlock_bh(&ks->rx_lock); + return -EINVAL; + } + } + + for (i = 0; i < advance; i++) { + struct ovpn_key_ctx __rcu **slot; + struct ovpn_key_ctx *future; + u16 stale_index; + + /* drop skipped future keys when advancing past them */ + stale_index = (ks->future_rx_keys.tail + i) % + OVPN_EPOCH_FUTURE_KEYS_COUNT; + slot = &ks->future_rx_keys.keys[stale_index]; + future = rcu_dereference_protected(*slot, lock_held); + if (future != new_decrypt && future != new_retiring) + stale[stale_count++] = future; + RCU_INIT_POINTER(*slot, NULL); + } + + /* keep the nearest previous rx key for reordered packets */ + rcu_assign_pointer(ks->retiring_key, new_retiring); + rcu_assign_pointer(ks->decrypt, new_decrypt); + ks->future_rx_keys.tail = (ks->future_rx_keys.tail + advance) % + OVPN_EPOCH_FUTURE_KEYS_COUNT; + ks->future_rx_keys.full = false; + spin_unlock_bh(&ks->rx_lock); + + ovpn_schedule_refill(ks, false); + ovpn_key_ctx_put(old_retiring); + if (new_retiring != old_decrypt) + ovpn_key_ctx_put(old_decrypt); + for (i = 0; i < stale_count; i++) + ovpn_key_ctx_put(stale[i]); + + /* move local tx forward to signal the peer to advance as well */ + ret = ovpn_advance_encrypt_key(ks, target_epoch, false); + /* keep stale requests because the bit also carries tx soft-limit + * work + */ + if (!ret) + clear_bit(OVPN_CRYPTO_TX_ROTATE_PENDING, &ks->flags); + + return 0; +} + +/** + * ovpn_check_rotate_keys - promote RX after authenticating a future epoch + * @peer: peer that supplied the authenticated packet + * @ks: key slot containing RX epoch state + * @decrypt: key that authenticated the packet + * + * Current and retiring keys do not move RX state. A future key is promoted only + * after authentication has completed, which prevents unauthenticated packets + * from forcing epoch advancement. + * + * Return: 0 if no fatal rotation error occurred, -ERANGE if epoch space is + * exhausted and userspace must install fresh key material. + */ +static int ovpn_check_rotate_keys(struct ovpn_peer *peer, + struct ovpn_crypto_key_slot *ks, + struct ovpn_key_ctx *decrypt) +{ + struct ovpn_key_ctx *current_decrypt; + u16 target_epoch; + int ret; + + if (likely(!ks->epoch_format)) + return 0; + + /* current key means no promotion is needed */ + current_decrypt = rcu_access_pointer(ks->decrypt); + if (likely(decrypt == current_decrypt)) + return 0; + + /* take a stable reference to distinguish retiring from future keys */ + if (unlikely(ovpn_key_ctx_get(¤t_decrypt, &ks->decrypt))) + return 0; + /* retiring or older packets do not move rx forward */ + if (likely(decrypt->epoch <= current_decrypt->epoch)) { + ovpn_key_ctx_put(current_decrypt); + return 0; + } + ovpn_key_ctx_put(current_decrypt); + + target_epoch = decrypt->epoch; + /* keep enough epoch space for the future-key ring after promotion */ + if (unlikely(target_epoch + OVPN_EPOCH_FUTURE_KEYS_COUNT >= + OVPN_MAX_EPOCH)) + return -ERANGE; + + /* a future key is promoted only after successful authentication */ + ret = ovpn_advance_decrypt_key(ks, target_epoch); + if (unlikely(ret == -EALREADY || ret == -EINPROGRESS)) + return 0; + if (unlikely(ret)) + net_warn_ratelimited("%s: decrypt: cannot advance key to epoch %u\n", + netdev_name(peer->ovpn->dev), + target_epoch); + + return 0; +} + void ovpn_decrypt_post(void *data, int ret) { struct ovpn_crypto_key_slot *ks; @@ -113,8 +307,10 @@ void ovpn_decrypt_post(void *data, int ret) struct ovpn_key_ctx *key; struct ovpn_socket *sock; struct ovpn_peer *peer; + bool aead_notify; int payload_len; u64 aead_blocks; + bool aead_hard; u16 pkt_epoch; __be16 proto; u64 pktid; @@ -134,9 +330,12 @@ void ovpn_decrypt_post(void *data, int ret) kfree(ovpn_skb_cb(skb)->crypto_tmp); if (unlikely(ret == -EBADMSG)) { - if (key && unlikely(ovpn_aead_decrypt_failure_record(key)) && - likely(!ks->epoch_format)) - ovpn_nl_key_swap_notify(peer, ks->key_id); + if (key && unlikely(ovpn_aead_decrypt_failure_record(key))) { + if (!ks->epoch_format) + ovpn_nl_key_swap_notify(peer, ks->key_id); + else + ovpn_rx_request_tx_rotation(ks, key); + } goto drop; } @@ -166,12 +365,20 @@ void ovpn_decrypt_post(void *data, int ret) aead_blocks = ovpn_aead_limit_blocks(ks->cipher_alg, ks->aad_size, payload_len); - if (unlikely(ovpn_pktid_recv_update_aead(&key->pid.recv, &key->usage, - &ks->usage_limit, - aead_blocks) && - !ks->epoch_format)) + aead_notify = ovpn_pktid_recv_update_aead(&key->pid.recv, &key->usage, + &ks->usage_limit, aead_blocks, + &aead_hard); + if (unlikely(aead_hard && ks->epoch_format)) + ovpn_rx_request_tx_rotation(ks, key); + if (unlikely(aead_notify && !ks->epoch_format)) ovpn_nl_key_swap_notify(peer, ks->key_id); + if (unlikely(ovpn_check_rotate_keys(peer, ks, key) < 0)) { + if (ovpn_crypto_kill_key(&peer->crypto, ks->key_id)) + ovpn_nl_key_swap_notify(peer, ks->key_id); + goto drop; + } + if (unlikely(ks->tail_tag_size && pskb_trim(skb, skb->len - ks->tail_tag_size))) goto drop; diff --git a/drivers/net/ovpn/pktid.h b/drivers/net/ovpn/pktid.h index 62666017f051..9e6fb80e1f24 100644 --- a/drivers/net/ovpn/pktid.h +++ b/drivers/net/ovpn/pktid.h @@ -121,6 +121,7 @@ static inline int ovpn_pktid_xmit_next(struct ovpn_pktid_xmit *pid, u64 *pktid) * @usage: key usage state * @limit: key usage limits * @aead_blocks: AEAD usage blocks consumed by this packet + * @hard_exceeded: set to true if RX has crossed the hard usage limit * * RX AEAD accounting is only informational for the local userspace process. * The peer's packets that already passed authentication and replay checks are @@ -132,12 +133,15 @@ static inline bool ovpn_pktid_recv_update_aead(struct ovpn_pktid_recv *pr, struct ovpn_key_usage *usage, const struct ovpn_limit *limit, - u64 aead_blocks) + u64 aead_blocks, bool *hard_exceeded) { bool ret; spin_lock_bh(&pr->lock); ret = ovpn_key_usage_recv(usage, limit, pr->id, aead_blocks); + *hard_exceeded = + ovpn_key_usage_over_limit(limit->hard, pr->id, + atomic64_read(&usage->blocks)); spin_unlock_bh(&pr->lock); return ret;