Register UDP tunnel GRO callbacks for ovpn data sockets and coalesce
compatible DATA_V2 records from one transport flow. Keep each encrypted
record as a separate frag-list entry so the receive path can detach and
authenticate records independently.
Match the complete opcode/key/peer header and require compatible outer-
network and checksum state. Flush on short records, differing segment
geometry, existing GSO input, or the 64-record limit.
Export skb_gro_receive_list, which is already shared by the core UDP and
TCP frag-list GRO paths, so modular ovpn can use the same primitive
instead of maintaining a local copy.
Packet taps may clone a completed aggregate before UDP receives it.
Making the aggregate head private retains shared references to its child
skbs, so also clone and unshare each shared child before detaching the
list and handing its records to crypto.
On two directly connected 100-Gbit/s mlx5 ports, five interleaved iperf3
-t 60 -O 10 single-flow AES-128-GCM runs in each direction produced the
following throughput:
Forward Reverse
Without receive GRO 18.308 Gbit/s 19.233 Gbit/s
With frag-list GRO 22.087 Gbit/s 22.962 Gbit/s
The preceding UDP GSO transmit path and hardware UDP segmentation were
enabled in both cases. The equal-weight mean of the two directional
results increased from 18.770 to 22.524 Gbit/s, a 20.0% improvement.
Signed-off-by: Ralf Lici <ralf@mandelbit.com>
---
Changes since v3 https://lore.kernel.org/openvpn-devel/903e2f55a68a1653f15014a91b0d7ac35ccabf11.1789546917.git.ralf@mandelbit.com/
- Make each GRO record's skb header private before modifying it. If
cloning fails, drop only that record and continue draining the batch.
(Sashiko)
Changes since v2 https://lore.kernel.org/openvpn-devel/76cb811457ac17b519d219b13fb2a4317a6d5e74.1789540779.git.ralf@mandelbit.com/
- Unclone GRO aggregates to avoid shared skb corruption in
ovpn_udp_gro_detach. (Sashiko)
Changes since v1 https://lore.kernel.org/openvpn-devel/dd08a7a2fe0dfc88509115b5d7ee17825020c012.1789485693.git.ralf@mandelbit.com/
- Preserve the outer network offset in ovpn_udp_gro_receive_fraglist.
(Sashiko)
- Detach frag_list only from a UDP-tunnel GSO aggregate. (Sashiko)
drivers/net/ovpn/io.c | 7 +-
drivers/net/ovpn/udp.c | 214 ++++++++++++++++++++++++++++++++++++++++-
net/core/gro.c | 1 +
net/ipv4/udp_offload.c | 3 +-
4 files changed, 218 insertions(+), 7 deletions(-)
@@ -70,11 +70,10 @@ static void ovpn_netdev_write(struct ovpn_peer *peer, struct sk_buff *skb)
unsigned int pkt_len;
int ret;
- /*
- * GSO state from the transport layer is not valid for the tunnel/data
- * path. Reset all GSO fields to prevent any further GSO processing
- * from entering an inconsistent state.
+ /* the transport encapsulation and its GSO metadata do not describe the
+ * decrypted inner packet
*/
+ skb->encapsulation = 0;
skb_gso_reset(skb);
/* we can't guarantee the packet wasn't corrupted before entering the
@@ -11,8 +11,10 @@
#include <linux/skbuff.h>
#include <linux/socket.h>
#include <linux/udp.h>
+#include <linux/unaligned.h>
#include <net/addrconf.h>
#include <net/dst_cache.h>
+#include <net/gro.h>
#include <net/route.h>
#include <net/transp_v6.h>
#include <net/udp.h>
@@ -27,6 +29,209 @@
#include "socket.h"
#include "udp.h"
+/* like UDP and TCP frag-list GRO */
+#define OVPN_UDP_GRO_CNT_MAX 64
+
+static bool ovpn_udp_gro_header(struct sk_buff *skb, u32 *header)
+{
+ const unsigned int offset = skb_gro_offset(skb);
+
+ /* GRO replaces its frag0 pointer after holding an skb, so keep the
+ * openvpn header linear for later candidate comparisons
+ */
+ if (!pskb_may_pull(skb, offset + OVPN_OPCODE_SIZE))
+ return false;
+
+ *header = get_unaligned_be32(skb->data + offset);
+ return true;
+}
+
+static struct sk_buff *ovpn_udp_gro_receive_fraglist(struct sock *sk,
+ struct list_head *head,
+ struct sk_buff *skb)
+{
+ const unsigned int gso_size = skb_gro_len(skb);
+ struct sk_buff *p, *pp = NULL;
+ u32 header, header2;
+ int ret = 0, nhoff;
+ bool flush;
+
+ if (!ovpn_udp_gro_header(skb, &header) ||
+ FIELD_GET(OVPN_OPCODE_PKTTYPE_MASK, header) != OVPN_DATA_V2) {
+ NAPI_GRO_CB(skb)->flush = 1;
+ return NULL;
+ }
+
+ /* do not nest an existing GSO packet in the record list */
+ if (skb_is_gso(skb)) {
+ NAPI_GRO_CB(skb)->flush = 1;
+ return NULL;
+ }
+
+ list_for_each_entry(p, head, list) {
+ if (!NAPI_GRO_CB(p)->same_flow)
+ continue;
+
+ /* match opcode, key ID and peer ID */
+ if (!ovpn_udp_gro_header(p, &header2) || header != header2) {
+ NAPI_GRO_CB(p)->same_flow = 0;
+ continue;
+ }
+
+ /* GRO has already matched the outer addresses and UDP ports;
+ * check the remaining outer IP fields
+ */
+ nhoff = skb_transport_offset(p) -
+ NAPI_GRO_CB(p)->network_offset;
+ flush = __gro_receive_network_flush(udp_hdr(skb), udp_hdr(p), p,
+ nhoff, false);
+
+ /* The first record determines the nominal GSO size. A shorter
+ * final record may follow it, but a larger record cannot.
+ * Checksum metadata must also be uniform because the aggregate
+ * exposes only one checksum state.
+ */
+ if (gso_size > skb_shinfo(p)->gso_size || flush ||
+ skb->ip_summed != p->ip_summed ||
+ skb->csum_level != p->csum_level) {
+ pp = p;
+ } else {
+ /* skb_gro_receive_list pulls the headers already
+ * processed by GRO before linking this skb to the
+ * record list so we have to manually preserve the
+ * outer network header location for later handling
+ */
+ nhoff = NAPI_GRO_CB(skb)->network_offset;
+ skb_set_network_header(skb, nhoff);
+ ret = skb_gro_receive_list(p, skb);
+ }
+
+ /* complete the aggregate if the append failed, or after
+ * appending a shorter final record, or after reaching the
+ * record-count limit
+ */
+ if (ret || gso_size != skb_shinfo(p)->gso_size ||
+ NAPI_GRO_CB(p)->count >= OVPN_UDP_GRO_CNT_MAX)
+ pp = p;
+
+ return pp;
+ }
+
+ return NULL;
+}
+
+static int ovpn_udp_gro_complete(struct sock *sk, struct sk_buff *skb,
+ int nhoff)
+{
+ /* udp_gro_complete has already marked this as a UDP tunnel GSO packet.
+ * Keep that type so UDP passes the aggregate directly to the encap cb,
+ * where the original record skbs are detached.
+ */
+ skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
+
+ /* Each outer UDP checksum was either validated (or accepted in case of
+ * checksumless UDP) before its record was merged in
+ * skb_gro_checksum_validate_zero_check.
+ * The checksum in the aggregate cannot describe the concatenation of
+ * independent UDP payloads, so we preserve the validation result.
+ */
+ skb->ip_summed = CHECKSUM_UNNECESSARY;
+ skb->csum_level = 0;
+ skb->csum_valid = 0;
+
+ return 0;
+}
+
+/* skb_gro_receive_list keeps the first openvpn record in 'skb' and links the
+ * remaining records through fraglist. Here we segment by detaching that list
+ * before delivering the records individually, and remove the child skbs from
+ * the head skb's length and memory accounting so the head describes only the
+ * first record again.
+ */
+static struct sk_buff *ovpn_udp_gro_detach(struct sk_buff *skb)
+{
+ unsigned int data_len = 0, truesize = 0;
+ struct sk_buff *curr, *list;
+ int ret;
+
+ /* IP reassembly may also use fraglist, but it is not a record batch */
+ if (!skb_is_gso(skb) ||
+ !(skb_shinfo(skb)->gso_type &
+ (SKB_GSO_UDP_TUNNEL | SKB_GSO_UDP_TUNNEL_CSUM)))
+ return NULL;
+
+ if (unlikely(!skb_shinfo(skb)->frag_list))
+ return NULL;
+
+ /* packet taps may have cloned the aggregate before it reached UDP, so
+ * make shared info private before removing its fraglist
+ */
+ ret = skb_unclone(skb, GFP_ATOMIC);
+ if (unlikely(ret))
+ return ERR_PTR(ret);
+
+ list = skb_shinfo(skb)->frag_list;
+
+ for (curr = list; curr; curr = curr->next) {
+ data_len += curr->len;
+ truesize += curr->truesize;
+ }
+
+ skb_shinfo(skb)->frag_list = NULL;
+ skb->len -= data_len;
+ skb->data_len -= data_len;
+ skb->truesize -= truesize;
+
+ return list;
+}
+
+static void ovpn_udp_recv(struct ovpn_peer *peer, struct sk_buff *skb)
+{
+ struct sk_buff *list, *next;
+
+ list = ovpn_udp_gro_detach(skb);
+ if (IS_ERR(list)) {
+ ovpn_dev_dstats_rx_dropped(peer->ovpn->dev);
+ kfree_skb(skb);
+ ovpn_peer_put(peer);
+ return;
+ }
+ skb->next = list;
+
+ skb_list_walk_safe(skb, skb, next)
+ {
+ /* skb_unclone can leave fraglist children shared with a packet
+ * tap, so make each skb header private before changing its
+ * list pointer or control block. skb_cow_data makes the packet
+ * data writable later in the decrypt path.
+ */
+ skb = skb_share_check(skb, GFP_ATOMIC);
+ if (unlikely(!skb)) {
+ ovpn_dev_dstats_rx_dropped(peer->ovpn->dev);
+
+ /* carry the current record's peer reference to the
+ * next record or release it at the end of the list
+ */
+ if (!next)
+ ovpn_peer_put(peer);
+ continue;
+ }
+
+ skb_mark_not_on_list(skb);
+
+ /* keep the current reference alive for the next record before
+ * handing this one to crypto
+ */
+ if (next && unlikely(!ovpn_peer_hold(peer))) {
+ DEBUG_NET_WARN_ON_ONCE(1);
+ kfree_skb_list(next);
+ next = NULL;
+ }
+
+ ovpn_recv(peer, skb);
+ }
+}
+
/* Retrieve the corresponding ovpn object from a UDP socket
* rcu_read_lock must be held on entry
*/
@@ -121,8 +326,7 @@ static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
/* pop off outer UDP header */
__skb_pull(skb, sizeof(struct udphdr));
- skb_mark_not_on_list(skb);
- ovpn_recv(peer, skb);
+ ovpn_udp_recv(peer, skb);
return 0;
drop:
@@ -408,6 +612,8 @@ int ovpn_udp_socket_attach(struct ovpn_socket *ovpn_sock, struct socket *sock,
.encap_type = UDP_ENCAP_OVPNINUDP,
.encap_rcv = ovpn_udp_encap_recv,
.encap_destroy = ovpn_udp_encap_destroy,
+ .gro_receive = ovpn_udp_gro_receive_fraglist,
+ .gro_complete = ovpn_udp_gro_complete,
};
struct ovpn_socket *old_data;
int ret;
@@ -454,6 +660,8 @@ void ovpn_udp_socket_detach(struct ovpn_socket *ovpn_sock)
{
struct sock *sk = ovpn_sock->sk;
+ udp_tunnel_cleanup_gro(sk);
+
/* Re-enable multicast loopback */
inet_set_bit(MC_LOOP, sk);
/* Disable CHECKSUM_UNNECESSARY to CHECKSUM_COMPLETE conversion */
@@ -462,6 +670,8 @@ void ovpn_udp_socket_detach(struct ovpn_socket *ovpn_sock)
WRITE_ONCE(udp_sk(sk)->encap_type, 0);
WRITE_ONCE(udp_sk(sk)->encap_rcv, NULL);
WRITE_ONCE(udp_sk(sk)->encap_destroy, NULL);
+ WRITE_ONCE(udp_sk(sk)->gro_receive, NULL);
+ WRITE_ONCE(udp_sk(sk)->gro_complete, NULL);
rcu_assign_sk_user_data(sk, NULL);
}
@@ -262,6 +262,7 @@ int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
return 0;
}
+EXPORT_SYMBOL(skb_gro_receive_list);
static void gro_complete(struct gro_node *gro, struct sk_buff *skb)
{
@@ -40,7 +40,8 @@ struct udp_tunnel_type_entry {
#define UDP_MAX_TUNNEL_TYPES (IS_ENABLED(CONFIG_GENEVE) + \
IS_ENABLED(CONFIG_VXLAN) * 2 + \
IS_ENABLED(CONFIG_NET_FOU) * 2 + \
- IS_ENABLED(CONFIG_XFRM) * 2)
+ IS_ENABLED(CONFIG_XFRM) * 2 + \
+ IS_ENABLED(CONFIG_OVPN))
DEFINE_STATIC_CALL(udp_tunnel_gro_rcv, dummy_gro_rcv);
static DEFINE_STATIC_KEY_FALSE(udp_tunnel_static_call);