@@ -260,7 +260,8 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
struct ovpn_crypto_key_slot *ks, struct sk_buff *skb,
- struct sk_buff *gso_skb, unsigned int offset)
+ struct sk_buff *gso_skb, unsigned int offset,
+ u32 pktid)
{
const unsigned int dst_nents = skb_shinfo(gso_skb)->nr_frags + 4;
const unsigned int src_nents = skb_shinfo(skb)->nr_frags + 2;
@@ -268,7 +269,6 @@ int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
struct aead_request *req;
unsigned int payload_off;
int dst_idx, mapped, ret;
- u32 pktid;
u8 *aad, *iv;
/* each input records the shared peer and key for the common completion
@@ -285,9 +285,6 @@ int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
dst = src + src_nents;
aad = (u8 *)(dst + dst_nents);
- ret = ovpn_pktid_xmit_next(&ks->pid_xmit, &pktid);
- if (unlikely(ret < 0))
- return ret;
ovpn_aead_encrypt_header(peer, ks, pktid, iv, aad);
ret = skb_store_bits(gso_skb, offset, aad, OVPN_AAD_SIZE);
@@ -20,7 +20,7 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
struct ovpn_crypto_key_slot *ks,
struct sk_buff *skb, struct sk_buff *gso_skb,
- unsigned int offset);
+ unsigned int offset, u32 pktid);
int ovpn_aead_decrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
struct sk_buff *skb);
@@ -243,6 +243,16 @@ void ovpn_recv(struct ovpn_peer *peer, struct sk_buff *skb)
ovpn_decrypt_post(skb, ovpn_aead_decrypt(peer, ks, skb));
}
+static void ovpn_encrypt_key_exhausted(struct ovpn_peer *peer,
+ struct ovpn_crypto_key_slot *ks)
+{
+ netdev_warn(peer->ovpn->dev, "killing key %u for peer %u\n",
+ ks->key_id, peer->id);
+ if (ovpn_crypto_kill_key(&peer->crypto, ks->key_id))
+ /* let userspace know so that a new key must be negotiated */
+ ovpn_nl_key_swap_notify(peer, ks->key_id);
+}
+
void ovpn_encrypt_post(void *data, int ret)
{
unsigned int orig_len, packets = 1;
@@ -273,12 +283,7 @@ void ovpn_encrypt_post(void *data, int ret)
/* we ran out of IVs and we must kill the key as it can't be
* used anymore
*/
- netdev_warn(peer->ovpn->dev,
- "killing key %u for peer %u\n", ks->key_id,
- peer->id);
- if (ovpn_crypto_kill_key(&peer->crypto, ks->key_id))
- /* let userspace know so that a new key must be negotiated */
- ovpn_nl_key_swap_notify(peer, ks->key_id);
+ ovpn_encrypt_key_exhausted(peer, ks);
}
if (batch) {
@@ -392,6 +397,7 @@ static bool ovpn_encrypt_gso_queue(struct sk_buff_head *skbs,
unsigned int offset = 0, i, len;
struct ovpn_crypto_key_slot *ks;
struct sk_buff *skb;
+ u32 first_pktid;
/* acquire all shared state before removing the first input skb so that
* failure can leave the queue intact for the ordinary transmit path
@@ -400,6 +406,17 @@ static bool ovpn_encrypt_gso_queue(struct sk_buff_head *skbs,
if (unlikely(!ks))
return false;
+ /* reserve the complete range before dequeuing any input skb so that a
+ * failure leaves the ordinary transmit path available as a fallback
+ */
+ if (unlikely(ovpn_pktid_xmit_range(&ks->pid_xmit, segments,
+ &first_pktid))) {
+ ovpn_encrypt_key_exhausted(peer, ks);
+ ovpn_crypto_key_slot_put(ks);
+ ovpn_peer_put(peer);
+ return false;
+ }
+
/* the aggregate owns these references until every sync or async crypto
* completion has finished
*/
@@ -416,7 +433,8 @@ static bool ovpn_encrypt_gso_queue(struct sk_buff_head *skbs,
memset(ovpn_skb_cb(skb), 0, sizeof(struct ovpn_cb));
ovpn_skb_cb(skb)->batch = batch;
ovpn_encrypt_post(skb, ovpn_aead_encrypt_gso(peer, ks, skb,
- batch, offset));
+ batch, offset,
+ first_pktid + i));
offset += len;
}
@@ -68,6 +68,54 @@ static inline int ovpn_pktid_xmit_next(struct ovpn_pktid_xmit *pid, u32 *pktid)
return 0;
}
+/**
+ * ovpn_pktid_xmit_range - reserve consecutive packet IDs
+ * @pid: transmit packet-ID state
+ * @count: non-zero number of IDs to reserve
+ * @pktid: first reserved packet ID
+ *
+ * The reservation is all-or-nothing. If the requested range would cross the
+ * end of the 32-bit packet-ID space, exhaust the state without returning the
+ * unused tail. Packet IDs can be skipped but never reused as an AEAD IV.
+ *
+ * Return: 0 on success or -ERANGE when no complete range remains
+ */
+static inline int ovpn_pktid_xmit_range(struct ovpn_pktid_xmit *pid,
+ unsigned int count, u32 *pktid)
+{
+ int old = atomic_read(&pid->seq_num);
+ u32 first, next;
+ bool fits;
+
+ for (;;) {
+ first = (u32)old;
+ next = first + count;
+
+ /* space is already exhausted */
+ if (unlikely(!first))
+ return -ERANGE;
+
+ fits = count - 1 <= U32_MAX - first;
+ /* if it doesn't fit we still have to publish exhaustion (0)
+ * so concurrent allocators stop using this key
+ */
+ if (unlikely(!fits))
+ next = 0;
+
+ /* if another CPU changed the state, we retry using the new
+ * seq_num value
+ */
+ if (!atomic_try_cmpxchg(&pid->seq_num, &old, (int)next))
+ continue;
+
+ if (unlikely(!fits))
+ return -ERANGE;
+
+ *pktid = first;
+ return 0;
+ }
+}
+
/* Write 12-byte AEAD IV to dest */
static inline void ovpn_pktid_aead_write(const u32 pktid,
const u8 nt[],