Each encrypted GSO record currently builds its authenticated header in
request-local scratch storage, copies it into the final UDP GSO aggregate and
then maps that range again as destination AAD.
Map the final output range first and, when it is represented by one
base-page-contained scatterlist entry, construct the header directly there.
Use the same entry as source and destination AAD since the AEAD interface does
not write destination AAD and permits the two entries to alias.
Retain the scratch-and-copy path as a fallback when the output range cannot be
mapped suitably.
In a three-run ConnectX-5 hairpin test with AES-128-GCM, one inner TCP stream
over UDP, a 1408-byte tunnel MTU and a 1500-byte physical MTU, mean throughput
increased from 24.487 to 25.002 Gbit/s (+2.11%). Cycles per byte decreased
from 3.616 to 3.569 (-1.30%) and instructions per byte from 9.635 to 9.540
(-0.99%).
Signed-off-by: Ralf Lici <ralf@mandelbit.com>
---
New in v2.
drivers/net/ovpn/crypto_aead.c | 61 ++++++++++++++++++++++++++++------
1 file changed, 51 insertions(+), 10 deletions(-)
@@ -8,6 +8,7 @@
*/
#include <crypto/aead.h>
+#include <linux/highmem.h>
#include <linux/skbuff.h>
#include <linux/workqueue.h>
#include <net/ip.h>
@@ -173,6 +174,29 @@ static void ovpn_aead_encrypt_header(struct ovpn_peer *peer,
memcpy(data + OVPN_OPCODE_SIZE, iv, OVPN_NONCE_WIRE_SIZE);
}
+static bool ovpn_aead_encrypt_header_direct(struct ovpn_peer *peer,
+ struct ovpn_crypto_key_slot *ks,
+ u32 pktid, u8 *iv,
+ struct scatterlist *sg)
+{
+ unsigned int offset, page_offset;
+ struct page *page;
+ void *addr;
+
+ offset = sg->offset;
+ page_offset = offset_in_page(offset);
+ if (page_offset + OVPN_AAD_SIZE > PAGE_SIZE)
+ return false;
+
+ /* an SG offset can address a later base page in a compound page */
+ page = sg_page(sg) + (offset >> PAGE_SHIFT);
+ addr = kmap_local_page(page);
+ ovpn_aead_encrypt_header(peer, ks, pktid, iv, addr + page_offset);
+ kunmap_local(addr);
+
+ return true;
+}
+
int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
struct sk_buff *skb)
{
@@ -269,6 +293,7 @@ int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
struct aead_request *req;
unsigned int payload_off;
int dst_idx, mapped, ret;
+ bool direct_aad;
u8 *aad, *iv;
/* each input records the shared peer and key for the common completion
@@ -285,26 +310,42 @@ int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
dst = src + src_nents;
aad = (u8 *)(dst + dst_nents);
- ovpn_aead_encrypt_header(peer, ks, pktid, iv, aad);
-
- ret = skb_store_bits(gso_skb, offset, aad, OVPN_AAD_SIZE);
- if (unlikely(ret < 0))
- return ret;
-
/* encrypt out of place from the original segmented skb directly into
* its final range in the UDP GSO skb
*/
sg_init_table(src, src_nents);
- sg_set_buf(src, aad, OVPN_AAD_SIZE);
+ /* use the final output bytes as source AAD too when they can be safely
+ * written through one base-page mapping.
+ */
+ mapped = skb_to_sgvec_nomark(gso_skb, src, offset, OVPN_AAD_SIZE);
+ if (unlikely(mapped < 0))
+ return mapped;
+ direct_aad = mapped == 1 &&
+ ovpn_aead_encrypt_header_direct(peer, ks, pktid, iv, src);
+ if (unlikely(!direct_aad)) {
+ ovpn_aead_encrypt_header(peer, ks, pktid, iv, aad);
+ ret = skb_store_bits(gso_skb, offset, aad, OVPN_AAD_SIZE);
+ if (unlikely(ret < 0))
+ return ret;
+ sg_set_buf(src, aad, OVPN_AAD_SIZE);
+ }
+
mapped = skb_to_sgvec_nomark(skb, src + 1, 0, skb->len);
if (unlikely(mapped < 0))
return mapped;
sg_mark_end(&src[mapped]);
sg_init_table(dst, dst_nents);
- dst_idx = skb_to_sgvec_nomark(gso_skb, dst, offset, OVPN_AAD_SIZE);
- if (unlikely(dst_idx < 0))
- return dst_idx;
+ if (likely(direct_aad)) {
+ /* destination AAD is not written and may alias source AAD */
+ sg_set_page(dst, sg_page(src), src->length, src->offset);
+ dst_idx = 1;
+ } else {
+ dst_idx = skb_to_sgvec_nomark(gso_skb, dst, offset,
+ OVPN_AAD_SIZE);
+ if (unlikely(dst_idx < 0))
+ return dst_idx;
+ }
payload_off = offset + OVPN_AAD_SIZE + OVPN_AUTH_TAG_SIZE;
mapped = skb_to_sgvec_nomark(gso_skb, dst + dst_idx, payload_off,