@@ -10,6 +10,7 @@
#ifndef _NET_OVPN_OVPNCRYPTO_H_
#define _NET_OVPN_OVPNCRYPTO_H_
+#include <linux/bitops.h>
#include <linux/kref.h>
#include <linux/rcupdate.h>
#include <linux/workqueue_types.h>
@@ -57,6 +58,8 @@ struct ovpn_peer_key_reset {
struct ovpn_key_config key;
};
+#define OVPN_CRYPTO_TX_ROTATE_PENDING 0
+
/* state for one concrete AEAD key direction */
struct ovpn_key_ctx {
u16 epoch;
@@ -83,6 +86,7 @@ struct ovpn_crypto_key_slot {
unsigned int pktid_size;
unsigned int payload_offset;
unsigned int tail_tag_size;
+ unsigned long flags;
struct ovpn_epoch_key epoch_key_send;
struct ovpn_epoch_key epoch_key_recv;
@@ -142,6 +142,160 @@ static struct scatterlist *ovpn_aead_crypto_req_sg(struct crypto_aead *aead,
__alignof__(struct scatterlist));
}
+/**
+ * ovpn_advance_encrypt_key - promote TX to a target epoch
+ * @ks: key slot containing TX epoch state
+ * @target_epoch: epoch to promote TX to
+ * @required: whether the caller must stop if promotion cannot complete
+ *
+ * TX promotion consumes prederived future keys up to @target_epoch. If another
+ * context is already updating TX state or refill has not caught up, optional
+ * promotions keep using the current key while required promotions fail in the
+ * caller.
+ *
+ * Return: 0 on success, -EINPROGRESS if the TX lock is busy, -EALREADY if TX
+ * is already at or beyond @target_epoch, -ENOKEY if the future ring cannot
+ * provide @target_epoch, or another negative error code otherwise.
+ */
+int ovpn_advance_encrypt_key(struct ovpn_crypto_key_slot *ks, u16 target_epoch,
+ bool required)
+{
+ struct ovpn_key_ctx *stale[OVPN_EPOCH_FUTURE_KEYS_COUNT];
+ u16 stale_count = 0, advance, count, index, i;
+ struct ovpn_key_ctx *old_encrypt;
+ struct ovpn_key_ctx *new_encrypt;
+ bool lock_held;
+
+ /* concurrent promotion or refill means another context is moving tx */
+ if (unlikely(!spin_trylock_bh(&ks->tx_lock)))
+ return -EINPROGRESS;
+
+ lock_held = lockdep_is_held(&ks->tx_lock);
+ old_encrypt = rcu_dereference_protected(ks->encrypt,
+ lock_held);
+ if (unlikely(old_encrypt->epoch >= target_epoch)) {
+ spin_unlock_bh(&ks->tx_lock);
+ return -EALREADY;
+ }
+
+ count = ovpn_epoch_future_keys_count(&ks->future_tx_keys);
+ advance = target_epoch - old_encrypt->epoch;
+ /* refill can lag behind packet processing under pressure */
+ if (unlikely(count < advance)) {
+ spin_unlock_bh(&ks->tx_lock);
+ ovpn_schedule_refill(ks, true);
+ return -ENOKEY;
+ }
+
+ index = (ks->future_tx_keys.tail + advance - 1) %
+ OVPN_EPOCH_FUTURE_KEYS_COUNT;
+ new_encrypt = rcu_dereference_protected(ks->future_tx_keys.keys[index],
+ lock_held);
+ if (WARN_ON_ONCE(!new_encrypt ||
+ new_encrypt->epoch != target_epoch)) {
+ spin_unlock_bh(&ks->tx_lock);
+ return -ENOKEY;
+ }
+
+ for (i = 0; i < advance; i++) {
+ struct ovpn_key_ctx __rcu **slot;
+ struct ovpn_key_ctx *future;
+ u16 stale_index;
+
+ /* drop skipped future keys when advancing past them */
+ stale_index = (ks->future_tx_keys.tail + i) %
+ OVPN_EPOCH_FUTURE_KEYS_COUNT;
+ slot = &ks->future_tx_keys.keys[stale_index];
+ future = rcu_dereference_protected(*slot, lock_held);
+ if (future != new_encrypt)
+ stale[stale_count++] = future;
+ RCU_INIT_POINTER(*slot, NULL);
+ }
+
+ /* each concrete key carries fresh packet-ID state */
+ rcu_assign_pointer(ks->encrypt, new_encrypt);
+ ks->future_tx_keys.tail = (ks->future_tx_keys.tail + advance) %
+ OVPN_EPOCH_FUTURE_KEYS_COUNT;
+ ks->future_tx_keys.full = false;
+ spin_unlock_bh(&ks->tx_lock);
+
+ ovpn_key_ctx_put(old_encrypt);
+ for (i = 0; i < stale_count; i++)
+ ovpn_key_ctx_put(stale[i]);
+
+ ovpn_schedule_refill(ks, true);
+
+ return 0;
+}
+
+static int ovpn_aead_encrypt_next_seq(struct ovpn_peer *peer,
+ struct ovpn_crypto_key_slot *ks,
+ struct ovpn_key_ctx *encrypt,
+ unsigned int pkt_len, u64 *seq)
+{
+ bool required = false;
+ bool pktid_notify;
+ u64 aead_blocks;
+ int ret;
+
+ /* hard packet-ID exhaustion requires fresh key material */
+ ret = ovpn_pktid_xmit_next(&encrypt->pid.xmit, seq);
+ if (unlikely(ret < 0)) {
+ if (!ks->epoch_format)
+ return ret;
+ required = true;
+ goto new_key;
+ }
+ pktid_notify = ret > 0;
+
+ aead_blocks = ovpn_aead_limit_blocks(ks->cipher_alg, ks->aad_size,
+ pkt_len);
+ ret = ovpn_key_usage_xmit(&encrypt->usage, &ks->usage_limit, *seq,
+ aead_blocks, pktid_notify);
+ if (unlikely(ret < 0)) {
+ /* epoch keys rotate internally instead of asking userspace */
+ if (!ks->epoch_format)
+ return ret;
+ required = true;
+ goto new_key;
+ }
+ if (unlikely(ret > 0)) {
+ if (!ks->epoch_format) {
+ ovpn_nl_key_swap_notify(peer, ks->key_id);
+ return 0;
+ }
+ /* epoch tx rotates locally when the soft limit is crossed */
+ set_bit(OVPN_CRYPTO_TX_ROTATE_PENDING, &ks->flags);
+ }
+
+ if (!ks->epoch_format)
+ return 0;
+
+ /* rx can request a tx epoch bump without adding rx work to every
+ * tx packet
+ */
+ if (unlikely(test_bit(OVPN_CRYPTO_TX_ROTATE_PENDING, &ks->flags)))
+ goto new_key;
+
+ return 0;
+
+new_key:
+ /* keep enough epoch space for the future-key ring after promotion */
+ if (unlikely(encrypt->epoch + 1 + OVPN_EPOCH_FUTURE_KEYS_COUNT >=
+ OVPN_MAX_EPOCH))
+ return -ERANGE;
+
+ ret = ovpn_advance_encrypt_key(ks, encrypt->epoch + 1, required);
+ if (unlikely(ret == -EINPROGRESS || ret == -ENOKEY))
+ return required ? -EBUSY : 0;
+ if (unlikely(ret < 0 && ret != -EALREADY))
+ return ret;
+
+ clear_bit(OVPN_CRYPTO_TX_ROTATE_PENDING, &ks->flags);
+
+ return -EAGAIN;
+}
+
int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
struct sk_buff *skb)
{
@@ -151,8 +305,7 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
struct sk_buff *trailer;
struct scatterlist *sg;
unsigned int tag_size;
- bool pktid_notify;
- u64 aead_blocks;
+ bool retried = false;
int nfrags, ret;
u64 pktid, seq;
void *tmp;
@@ -175,6 +328,7 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
* [ 8-byte packet ID ]
*/
+retry:
ret = ovpn_key_ctx_get(&key, &ks->encrypt);
if (unlikely(ret))
return ret;
@@ -182,19 +336,20 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
tag_size = crypto_aead_authsize(key->tfm);
- ret = ovpn_pktid_xmit_next(&key->pid.xmit, &seq);
- if (unlikely(ret < 0))
- return ret;
- pktid_notify = ret > 0;
-
- aead_blocks = ovpn_aead_limit_blocks(ks->cipher_alg, ks->aad_size,
- plaintext_len);
- ret = ovpn_key_usage_xmit(&key->usage, &ks->usage_limit, seq,
- aead_blocks, pktid_notify);
+ ret = ovpn_aead_encrypt_next_seq(peer, ks, key, plaintext_len, &seq);
+ if (unlikely(ret == -EAGAIN)) {
+ if (retried)
+ return -EBUSY;
+
+ /* retry once after epoch promotion */
+ ovpn_skb_cb(skb)->key = NULL;
+ ovpn_key_ctx_put(key);
+ key = NULL;
+ retried = true;
+ goto retry;
+ }
if (unlikely(ret < 0))
return ret;
- if (unlikely(ret > 0 && !ks->epoch_format))
- ovpn_nl_key_swap_notify(peer, ks->key_id);
/* check that there's enough headroom in the skb for packet
* encapsulation
@@ -15,6 +15,9 @@
#include <asm/types.h>
#include <linux/skbuff.h>
+int ovpn_advance_encrypt_key(struct ovpn_crypto_key_slot *ks, u16 target_epoch,
+ bool required);
+
int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
struct sk_buff *skb);
int ovpn_aead_decrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
@@ -104,7 +104,8 @@ void ovpn_key_ctx_release(struct kref *kref)
static struct ovpn_key_ctx *
ovpn_key_ctx_new(const char *title, const char *alg_name,
const u8 *cipher_key, unsigned int cipher_key_len,
- const u8 *implicit_iv, u16 epoch, bool encrypt)
+ const u8 *implicit_iv, u16 epoch, bool epoch_format,
+ bool encrypt)
{
struct ovpn_limit pktid_limit;
struct ovpn_key_ctx *key;
@@ -134,7 +135,7 @@ ovpn_key_ctx_new(const char *title, const char *alg_name,
/* initialize only the packet ID direction this context owns */
if (encrypt) {
- ovpn_pktid_xmit_limit_init(&pktid_limit, false);
+ ovpn_pktid_xmit_limit_init(&pktid_limit, epoch_format);
ovpn_pktid_xmit_init(&key->pid.xmit, &pktid_limit);
} else {
ovpn_pktid_recv_init(&key->pid.recv);
@@ -158,7 +159,7 @@ ovpn_key_ctx_create_direct(bool encrypt, const char *alg_name,
key = ovpn_key_ctx_new(encrypt ? "encrypt" : "decrypt", alg_name,
dir->cipher_key, dir->cipher_key_size,
- implicit_iv, 0, encrypt);
+ implicit_iv, 0, false, encrypt);
memzero_explicit(implicit_iv, sizeof(implicit_iv));
return key;
@@ -179,7 +180,7 @@ ovpn_key_ctx_create_epoch(bool encrypt, const char *alg_name,
key = ovpn_key_ctx_new(encrypt ? "encrypt" : "decrypt", alg_name,
cipher_key, epoch_key->cipher_key_len,
- implicit_iv, epoch_key->epoch, encrypt);
+ implicit_iv, epoch_key->epoch, true, encrypt);
memzero_explicit(cipher_key, sizeof(cipher_key));
memzero_explicit(implicit_iv, sizeof(implicit_iv));
@@ -105,6 +105,200 @@ static void ovpn_netdev_write(struct ovpn_peer *peer, struct sk_buff *skb)
local_bh_enable();
}
+static void ovpn_rx_request_tx_rotation(struct ovpn_crypto_key_slot *ks,
+ const struct ovpn_key_ctx *decrypt)
+{
+ struct ovpn_key_ctx *current_decrypt;
+ struct ovpn_key_ctx *encrypt;
+
+ if (unlikely(!ks->epoch_format))
+ return;
+
+ rcu_read_lock();
+ current_decrypt = rcu_dereference(ks->decrypt);
+ encrypt = rcu_dereference(ks->encrypt);
+ /* only current rx key usage can ask the peer to move forward */
+ if (decrypt == current_decrypt &&
+ (!encrypt || decrypt->epoch >= encrypt->epoch))
+ set_bit(OVPN_CRYPTO_TX_ROTATE_PENDING, &ks->flags);
+ rcu_read_unlock();
+}
+
+/**
+ * ovpn_advance_decrypt_key - promote RX to a target epoch
+ * @ks: key slot containing RX epoch state
+ * @target_epoch: authenticated epoch to promote RX to
+ *
+ * RX promotion consumes future keys up to @target_epoch, moves the closest
+ * previous epoch to the retiring slot for reordered packets, and schedules
+ * refill for the consumed future-key slots. Local TX is advanced as well so
+ * outbound packets signal the peer to leave older epochs.
+ *
+ * Return: 0 on success, -EINPROGRESS if the RX lock is busy, -EALREADY if RX
+ * is already at or beyond @target_epoch, or -EINVAL if the future ring cannot
+ * provide @target_epoch.
+ */
+static int ovpn_advance_decrypt_key(struct ovpn_crypto_key_slot *ks,
+ u16 target_epoch)
+{
+ struct ovpn_key_ctx *stale[OVPN_EPOCH_FUTURE_KEYS_COUNT];
+ u16 stale_count = 0, advance, count, index, i;
+ struct ovpn_key_ctx __rcu **retire_slot;
+ struct ovpn_key_ctx *old_retiring;
+ struct ovpn_key_ctx *new_retiring;
+ struct ovpn_key_ctx *old_decrypt;
+ struct ovpn_key_ctx *new_decrypt;
+ u16 retire_index;
+ bool lock_held;
+ int ret;
+
+ /* concurrent promotion or refill means another context is moving rx */
+ if (unlikely(!spin_trylock_bh(&ks->rx_lock)))
+ return -EINPROGRESS;
+
+ lock_held = lockdep_is_held(&ks->rx_lock);
+ old_retiring = rcu_dereference_protected(ks->retiring_key,
+ lock_held);
+ old_decrypt = rcu_dereference_protected(ks->decrypt,
+ lock_held);
+ /* current decrypt key is always installed while the slot is alive */
+ if (unlikely(target_epoch <= old_decrypt->epoch)) {
+ spin_unlock_bh(&ks->rx_lock);
+ return -EALREADY;
+ }
+
+ count = ovpn_epoch_future_keys_count(&ks->future_rx_keys);
+ advance = target_epoch - old_decrypt->epoch;
+ /* authenticated epoch must be available in the future ring */
+ if (unlikely(count < advance)) {
+ spin_unlock_bh(&ks->rx_lock);
+ return -EINVAL;
+ }
+
+ /* get the authenticated future key from the ring */
+ index = (ks->future_rx_keys.tail + advance - 1) %
+ OVPN_EPOCH_FUTURE_KEYS_COUNT;
+ new_decrypt = rcu_dereference_protected(ks->future_rx_keys.keys[index],
+ lock_held);
+ if (WARN_ON_ONCE(!new_decrypt ||
+ new_decrypt->epoch != target_epoch)) {
+ spin_unlock_bh(&ks->rx_lock);
+ return -EINVAL;
+ }
+
+ new_retiring = old_decrypt;
+ if (unlikely(advance > 1)) {
+ /* keep target_epoch - 1 for reordered packets after jumps */
+ retire_index = (ks->future_rx_keys.tail + advance - 2) %
+ OVPN_EPOCH_FUTURE_KEYS_COUNT;
+ retire_slot = &ks->future_rx_keys.keys[retire_index];
+ new_retiring = rcu_dereference_protected(*retire_slot,
+ lock_held);
+ if (WARN_ON_ONCE(!new_retiring ||
+ new_retiring->epoch != target_epoch - 1)) {
+ spin_unlock_bh(&ks->rx_lock);
+ return -EINVAL;
+ }
+ }
+
+ for (i = 0; i < advance; i++) {
+ struct ovpn_key_ctx __rcu **slot;
+ struct ovpn_key_ctx *future;
+ u16 stale_index;
+
+ /* drop skipped future keys when advancing past them */
+ stale_index = (ks->future_rx_keys.tail + i) %
+ OVPN_EPOCH_FUTURE_KEYS_COUNT;
+ slot = &ks->future_rx_keys.keys[stale_index];
+ future = rcu_dereference_protected(*slot, lock_held);
+ if (future != new_decrypt && future != new_retiring)
+ stale[stale_count++] = future;
+ RCU_INIT_POINTER(*slot, NULL);
+ }
+
+ /* keep the nearest previous rx key for reordered packets */
+ rcu_assign_pointer(ks->retiring_key, new_retiring);
+ rcu_assign_pointer(ks->decrypt, new_decrypt);
+ ks->future_rx_keys.tail = (ks->future_rx_keys.tail + advance) %
+ OVPN_EPOCH_FUTURE_KEYS_COUNT;
+ ks->future_rx_keys.full = false;
+ spin_unlock_bh(&ks->rx_lock);
+
+ ovpn_schedule_refill(ks, false);
+ ovpn_key_ctx_put(old_retiring);
+ if (new_retiring != old_decrypt)
+ ovpn_key_ctx_put(old_decrypt);
+ for (i = 0; i < stale_count; i++)
+ ovpn_key_ctx_put(stale[i]);
+
+ /* move local tx forward to signal the peer to advance as well */
+ ret = ovpn_advance_encrypt_key(ks, target_epoch, false);
+ /* keep stale requests because the bit also carries tx soft-limit
+ * work
+ */
+ if (!ret)
+ clear_bit(OVPN_CRYPTO_TX_ROTATE_PENDING, &ks->flags);
+
+ return 0;
+}
+
+/**
+ * ovpn_check_rotate_keys - promote RX after authenticating a future epoch
+ * @peer: peer that supplied the authenticated packet
+ * @ks: key slot containing RX epoch state
+ * @decrypt: key that authenticated the packet
+ *
+ * Current and retiring keys do not move RX state. A future key is promoted only
+ * after authentication has completed, which prevents unauthenticated packets
+ * from forcing epoch advancement.
+ *
+ * Return: 0 if no fatal rotation error occurred, -ERANGE if epoch space is
+ * exhausted and userspace must install fresh key material.
+ */
+static int ovpn_check_rotate_keys(struct ovpn_peer *peer,
+ struct ovpn_crypto_key_slot *ks,
+ struct ovpn_key_ctx *decrypt)
+{
+ struct ovpn_key_ctx *current_decrypt;
+ u16 target_epoch;
+ int ret;
+
+ if (likely(!ks->epoch_format))
+ return 0;
+
+ /* current key means no promotion is needed */
+ current_decrypt = rcu_access_pointer(ks->decrypt);
+ if (likely(decrypt == current_decrypt))
+ return 0;
+
+ /* take a stable reference to distinguish retiring from future keys */
+ if (unlikely(ovpn_key_ctx_get(¤t_decrypt, &ks->decrypt)))
+ return 0;
+ /* retiring or older packets do not move rx forward */
+ if (likely(decrypt->epoch <= current_decrypt->epoch)) {
+ ovpn_key_ctx_put(current_decrypt);
+ return 0;
+ }
+ ovpn_key_ctx_put(current_decrypt);
+
+ target_epoch = decrypt->epoch;
+ /* keep enough epoch space for the future-key ring after promotion */
+ if (unlikely(target_epoch + OVPN_EPOCH_FUTURE_KEYS_COUNT >=
+ OVPN_MAX_EPOCH))
+ return -ERANGE;
+
+ /* a future key is promoted only after successful authentication */
+ ret = ovpn_advance_decrypt_key(ks, target_epoch);
+ if (unlikely(ret == -EALREADY || ret == -EINPROGRESS))
+ return 0;
+ if (unlikely(ret))
+ net_warn_ratelimited("%s: decrypt: cannot advance key to epoch %u\n",
+ netdev_name(peer->ovpn->dev),
+ target_epoch);
+
+ return 0;
+}
+
void ovpn_decrypt_post(void *data, int ret)
{
struct ovpn_crypto_key_slot *ks;
@@ -113,8 +307,10 @@ void ovpn_decrypt_post(void *data, int ret)
struct ovpn_key_ctx *key;
struct ovpn_socket *sock;
struct ovpn_peer *peer;
+ bool aead_notify;
int payload_len;
u64 aead_blocks;
+ bool aead_hard;
u16 pkt_epoch;
__be16 proto;
u64 pktid;
@@ -134,9 +330,12 @@ void ovpn_decrypt_post(void *data, int ret)
kfree(ovpn_skb_cb(skb)->crypto_tmp);
if (unlikely(ret == -EBADMSG)) {
- if (key && unlikely(ovpn_aead_decrypt_failure_record(key)) &&
- likely(!ks->epoch_format))
- ovpn_nl_key_swap_notify(peer, ks->key_id);
+ if (key && unlikely(ovpn_aead_decrypt_failure_record(key))) {
+ if (!ks->epoch_format)
+ ovpn_nl_key_swap_notify(peer, ks->key_id);
+ else
+ ovpn_rx_request_tx_rotation(ks, key);
+ }
goto drop;
}
@@ -166,12 +365,20 @@ void ovpn_decrypt_post(void *data, int ret)
aead_blocks = ovpn_aead_limit_blocks(ks->cipher_alg, ks->aad_size,
payload_len);
- if (unlikely(ovpn_pktid_recv_update_aead(&key->pid.recv, &key->usage,
- &ks->usage_limit,
- aead_blocks) &&
- !ks->epoch_format))
+ aead_notify = ovpn_pktid_recv_update_aead(&key->pid.recv, &key->usage,
+ &ks->usage_limit, aead_blocks,
+ &aead_hard);
+ if (unlikely(aead_hard && ks->epoch_format))
+ ovpn_rx_request_tx_rotation(ks, key);
+ if (unlikely(aead_notify && !ks->epoch_format))
ovpn_nl_key_swap_notify(peer, ks->key_id);
+ if (unlikely(ovpn_check_rotate_keys(peer, ks, key) < 0)) {
+ if (ovpn_crypto_kill_key(&peer->crypto, ks->key_id))
+ ovpn_nl_key_swap_notify(peer, ks->key_id);
+ goto drop;
+ }
+
if (unlikely(ks->tail_tag_size &&
pskb_trim(skb, skb->len - ks->tail_tag_size)))
goto drop;
@@ -121,6 +121,7 @@ static inline int ovpn_pktid_xmit_next(struct ovpn_pktid_xmit *pid, u64 *pktid)
* @usage: key usage state
* @limit: key usage limits
* @aead_blocks: AEAD usage blocks consumed by this packet
+ * @hard_exceeded: set to true if RX has crossed the hard usage limit
*
* RX AEAD accounting is only informational for the local userspace process.
* The peer's packets that already passed authentication and replay checks are
@@ -132,12 +133,15 @@ static inline bool
ovpn_pktid_recv_update_aead(struct ovpn_pktid_recv *pr,
struct ovpn_key_usage *usage,
const struct ovpn_limit *limit,
- u64 aead_blocks)
+ u64 aead_blocks, bool *hard_exceeded)
{
bool ret;
spin_lock_bh(&pr->lock);
ret = ovpn_key_usage_recv(usage, limit, pr->id, aead_blocks);
+ *hard_exceeded =
+ ovpn_key_usage_over_limit(limit->hard, pr->id,
+ atomic64_read(&usage->blocks));
spin_unlock_bh(&pr->lock);
return ret;