@@ -844,6 +844,14 @@ definitions:
entries:
- p2p
- mp
+ -
+ name: ovpn-udp-gro-mode
+ enum-name: ovpn-udp-gro-mode
+ name-prefix: ovpn-udp-gro-mode
+ type: enum
+ entries:
+ - full-stack
+ - direct
-
name: br-stp-mode
type: enum
@@ -2365,6 +2373,10 @@ attribute-sets:
name: mode
type: u8
enum: ovpn-mode
+ -
+ name: udp-gro-mode
+ type: u8
+ enum: ovpn-udp-gro-mode
sub-messages:
-
@@ -129,6 +129,9 @@ static const struct device_type ovpn_type = {
static const struct nla_policy ovpn_policy[IFLA_OVPN_MAX + 1] = {
[IFLA_OVPN_MODE] = NLA_POLICY_RANGE(NLA_U8, OVPN_MODE_P2P,
OVPN_MODE_MP),
+ [IFLA_OVPN_UDP_GRO_MODE] =
+ NLA_POLICY_RANGE(NLA_U8, OVPN_UDP_GRO_MODE_FULL_STACK,
+ OVPN_UDP_GRO_MODE_DIRECT),
};
/**
@@ -200,6 +203,7 @@ static int ovpn_newlink(struct net_device *dev,
struct rtnl_newlink_params *params,
struct netlink_ext_ack *extack)
{
+ enum ovpn_udp_gro_mode gro_mode = OVPN_UDP_GRO_MODE_FULL_STACK;
struct ovpn_priv *ovpn = netdev_priv(dev);
struct nlattr **data = params->data;
enum ovpn_mode mode = OVPN_MODE_P2P;
@@ -209,9 +213,14 @@ static int ovpn_newlink(struct net_device *dev,
mode = nla_get_u8(data[IFLA_OVPN_MODE]);
netdev_dbg(dev, "setting device mode: %u\n", mode);
}
+ if (data && data[IFLA_OVPN_UDP_GRO_MODE]) {
+ gro_mode = nla_get_u8(data[IFLA_OVPN_UDP_GRO_MODE]);
+ netdev_dbg(dev, "setting UDP GRO mode: %u\n", gro_mode);
+ }
ovpn->dev = dev;
ovpn->mode = mode;
+ ovpn->gro_mode = gro_mode;
spin_lock_init(&ovpn->lock);
INIT_DELAYED_WORK(&ovpn->keepalive_work, ovpn_peer_keepalive_work);
@@ -237,8 +246,8 @@ static int ovpn_newlink(struct net_device *dev,
static size_t ovpn_get_size(const struct net_device *dev)
{
- /* IFLA_OVPN_MODE */
- return nla_total_size(sizeof(u8));
+ /* IFLA_OVPN_MODE and IFLA_OVPN_UDP_GRO_MODE */
+ return nla_total_size(sizeof(u8)) + nla_total_size(sizeof(u8));
}
static int ovpn_fill_info(struct sk_buff *skb, const struct net_device *dev)
@@ -247,6 +256,8 @@ static int ovpn_fill_info(struct sk_buff *skb, const struct net_device *dev)
if (nla_put_u8(skb, IFLA_OVPN_MODE, ovpn->mode))
return -EMSGSIZE;
+ if (nla_put_u8(skb, IFLA_OVPN_UDP_GRO_MODE, ovpn->gro_mode))
+ return -EMSGSIZE;
return 0;
}
@@ -40,6 +40,7 @@ struct ovpn_peer_collection {
* struct ovpn_priv - per ovpn interface state
* @dev: the actual netdev representing the tunnel
* @mode: device operation mode (i.e. p2p, mp, ..)
+ * @gro_mode: whether UDP data follows the full stack or is decrypted from GRO
* @lock: protect this object
* @peers: data structures holding multi-peer references
* @peer: in P2P mode, this is the only remote peer
@@ -49,6 +50,7 @@ struct ovpn_peer_collection {
struct ovpn_priv {
struct net_device *dev;
enum ovpn_mode mode;
+ enum ovpn_udp_gro_mode gro_mode;
spinlock_t lock; /* protect writing to the ovpn_priv object */
struct ovpn_peer_collection *peers;
struct ovpn_peer __rcu *peer;
@@ -232,23 +232,12 @@ static struct ovpn_socket *ovpn_socket_from_udp_sock(struct sock *sk)
return ovpn_sock;
}
-/**
- * ovpn_udp_encap_recv - Start processing a received UDP packet.
- * @sk: socket over which the packet was received
- * @skb: the received packet
- *
- * If the first byte of the payload is:
- * - DATA_V2 the packet is accepted for further processing,
- * - DATA_V1 the packet is dropped as not supported,
- * - anything else the packet is forwarded to the UDP stack for
- * delivery to user space.
- *
- * Return:
- * 0 if skb was consumed or dropped
- * >0 if skb should be passed up to userspace as UDP (packet not consumed)
- * <0 if skb should be resubmitted as proto -N (packet not consumed)
+/* Process one packet after the caller has made its OpenVPN header visible at
+ * @payload_offset. A zero return means the skb was consumed. A positive return
+ * leaves a control packet for the UDP socket.
*/
-static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
+static int ovpn_udp_data_recv(struct sock *sk, struct sk_buff *skb,
+ unsigned int payload_offset)
{
struct ovpn_socket *ovpn_sock;
struct ovpn_priv *ovpn;
@@ -269,18 +258,16 @@ static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
goto drop_noovpn;
}
- /* Make sure the first 4 bytes of the skb data buffer after the UDP
- * header are accessible.
+ /* Make sure the first 4 bytes of the OpenVPN header are accessible.
* They are required to fetch the OP code, the key ID and the peer ID.
*/
- if (unlikely(!pskb_may_pull(skb, sizeof(struct udphdr) +
- OVPN_OPCODE_SIZE))) {
+ if (unlikely(!pskb_may_pull(skb, payload_offset + OVPN_OPCODE_SIZE))) {
net_dbg_ratelimited("%s: packet too small from UDP socket\n",
netdev_name(ovpn->dev));
goto drop;
}
- opcode = ovpn_opcode_from_skb(skb, sizeof(struct udphdr));
+ opcode = ovpn_opcode_from_skb(skb, payload_offset);
if (unlikely(opcode != OVPN_DATA_V2)) {
/* DATA_V1 is not supported */
if (opcode == OVPN_DATA_V1)
@@ -290,7 +277,7 @@ static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
return 1;
}
- peer_id = ovpn_peer_id_from_skb(skb, sizeof(struct udphdr));
+ peer_id = ovpn_peer_id_from_skb(skb, payload_offset);
/* some OpenVPN server implementations send data packets with the
* peer-id set to UNDEF. In this case we skip the peer lookup by peer-id
* and we try with the transport address
@@ -303,8 +290,10 @@ static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
if (unlikely(!peer))
goto drop;
- /* pop off outer UDP header */
- __skb_pull(skb, sizeof(struct udphdr));
+ /* the crypto receive path expects skb->data to begin at the OpenVPN
+ * header and takes ownership of the skb
+ */
+ __skb_pull(skb, payload_offset);
ovpn_udp_recv(peer, skb);
return 0;
@@ -315,6 +304,60 @@ static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
return 0;
}
+/* Consume DATA_V2 directly from UDP GRO. These packets deliberately bypass
+ * packet taps, TC ingress, the outer IP and netfilter receive paths, the final
+ * UDP lookup, and normal UDP accounting. Control packets are restored and
+ * continue through all of those layers normally.
+ */
+static struct sk_buff *ovpn_udp_gro_receive_direct(struct sock *sk,
+ struct list_head *head,
+ struct sk_buff *skb)
+{
+ unsigned int offset = skb_gro_offset(skb);
+
+ /* if the OpenVPN header is not accessible, leave validation and drop
+ * handling to the ordinary UDP receive path
+ */
+ if (unlikely(!pskb_pull(skb, offset)))
+ goto flush;
+
+ /* tell UDP GRO not to touch the skb if it was consumed by the direct
+ * receive path
+ */
+ if (likely(!ovpn_udp_data_recv(sk, skb, 0)))
+ return ERR_PTR(-EINPROGRESS);
+
+ /* control packets still belongs to the socket so we restore the data
+ * pointer because the normal receive path expects the outer headers
+ */
+ skb_push(skb, offset);
+
+flush:
+ NAPI_GRO_CB(skb)->same_flow = 0;
+ NAPI_GRO_CB(skb)->flush = 1;
+ return NULL;
+}
+
+/**
+ * ovpn_udp_encap_recv - Start processing a received UDP packet.
+ * @sk: socket over which the packet was received
+ * @skb: the received packet
+ *
+ * If the first byte of the payload is:
+ * - DATA_V2 the packet is accepted for further processing,
+ * - DATA_V1 the packet is dropped as not supported,
+ * - anything else the packet is forwarded to the UDP stack for
+ * delivery to user space.
+ *
+ * Return:
+ * 0 if @skb was consumed or dropped
+ * 1 if @skb should continue through normal UDP delivery
+ */
+static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
+{
+ return ovpn_udp_data_recv(sk, skb, sizeof(struct udphdr));
+}
+
/**
* ovpn_udp4_output - send IPv4 packet over udp socket
* @peer: the destination peer
@@ -591,7 +634,12 @@ int ovpn_udp_socket_attach(struct ovpn_socket *ovpn_sock, struct socket *sock,
.encap_type = UDP_ENCAP_OVPNINUDP,
.encap_rcv = ovpn_udp_encap_recv,
.encap_destroy = ovpn_udp_encap_destroy,
- .gro_receive = ovpn_udp_gro_receive_fraglist,
+ /* GRO mode cannot change after the interface is created so
+ * select the socket callback once at socket setup
+ */
+ .gro_receive = ovpn->gro_mode == OVPN_UDP_GRO_MODE_DIRECT ?
+ ovpn_udp_gro_receive_direct :
+ ovpn_udp_gro_receive_fraglist,
.gro_complete = ovpn_udp_gro_complete,
};
struct ovpn_socket *old_data;
@@ -2067,9 +2067,15 @@ enum ovpn_mode {
OVPN_MODE_MP,
};
+enum ovpn_udp_gro_mode {
+ OVPN_UDP_GRO_MODE_FULL_STACK,
+ OVPN_UDP_GRO_MODE_DIRECT,
+};
+
enum {
IFLA_OVPN_UNSPEC,
IFLA_OVPN_MODE,
+ IFLA_OVPN_UDP_GRO_MODE,
__IFLA_OVPN_MAX,
};
@@ -41,7 +41,7 @@ struct udp_tunnel_type_entry {
IS_ENABLED(CONFIG_VXLAN) * 2 + \
IS_ENABLED(CONFIG_NET_FOU) * 2 + \
IS_ENABLED(CONFIG_XFRM) * 2 + \
- IS_ENABLED(CONFIG_OVPN))
+ IS_ENABLED(CONFIG_OVPN) * 2)
DEFINE_STATIC_CALL(udp_tunnel_gro_rcv, dummy_gro_rcv);
static DEFINE_STATIC_KEY_FALSE(udp_tunnel_static_call);
@@ -37,6 +37,7 @@ TEST_PROGS := \
test-close-socket-tcp.sh \
test-close-socket.sh \
test-float.sh \
+ test-gro-direct.sh \
test-large-mtu.sh \
test-mark.sh \
test-symmetric-id-float.sh \
@@ -10,6 +10,7 @@ source "$OVPN_COMMON_DIR/../../kselftest/ktap_helpers.sh"
OVPN_UDP_PEERS_FILE=${OVPN_UDP_PEERS_FILE:-udp_peers.txt}
OVPN_TCP_PEERS_FILE=${OVPN_TCP_PEERS_FILE:-tcp_peers.txt}
OVPN_CLI=${OVPN_CLI:-${OVPN_COMMON_DIR}/ovpn-cli}
+OVPN_UDP_GRO_MODE=${OVPN_UDP_GRO_MODE:-FULL_STACK}
OVPN_YNL=${OVPN_YNL:-${OVPN_COMMON_DIR}/../../../../net/ynl/pyynl/cli.py}
OVPN_ALG=${OVPN_ALG:-aes}
OVPN_PROTO=${OVPN_PROTO:-UDP}
@@ -162,7 +163,8 @@ ovpn_setup_ns() {
done
fi
- ip netns exec "${peer}" ${OVPN_CLI} new_iface tun${1} $MODE
+ ip netns exec "${peer}" ${OVPN_CLI} new_iface tun${1} $MODE \
+ "${OVPN_UDP_GRO_MODE}"
ip -n "${peer}" addr add ${2} dev tun${1}
# add a secondary IP to peer 1, to test a LAN behind a client
if [ ${1} -eq 1 -a -n "${OVPN_LAN_IP}" ]; then
@@ -123,6 +123,8 @@ struct ovpn_ctx {
char ifname[IFNAMSIZ];
enum ovpn_mode mode;
bool mode_set;
+ enum ovpn_udp_gro_mode udp_gro_mode;
+ bool udp_gro_mode_set;
int socket;
int cli_sockets[MAX_PEERS];
@@ -1377,8 +1379,9 @@ static int ovpn_new_iface(struct ovpn_ctx *ovpn)
struct ovpn_link_req req = { 0 };
int ret = -1;
- fprintf(stdout, "Creating interface %s with mode %u\n", ovpn->ifname,
- ovpn->mode);
+ fprintf(stdout,
+ "Creating interface %s with mode %u and UDP GRO mode %u\n",
+ ovpn->ifname, ovpn->mode, ovpn->udp_gro_mode);
req.n.nlmsg_len = NLMSG_LENGTH(sizeof(req.i));
req.n.nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
@@ -1396,15 +1399,21 @@ static int ovpn_new_iface(struct ovpn_ctx *ovpn)
strlen(OVPN_FAMILY_NAME) + 1) < 0)
goto err;
- if (ovpn->mode_set) {
+ if (ovpn->mode_set || ovpn->udp_gro_mode_set) {
data = ovpn_nest_start(&req.n, sizeof(req), IFLA_INFO_DATA);
if (!data)
goto err;
- if (ovpn_addattr(&req.n, sizeof(req), IFLA_OVPN_MODE,
+ if (ovpn->mode_set &&
+ ovpn_addattr(&req.n, sizeof(req), IFLA_OVPN_MODE,
&ovpn->mode, sizeof(uint8_t)) < 0)
goto err;
+ if (ovpn->udp_gro_mode_set &&
+ ovpn_addattr(&req.n, sizeof(req), IFLA_OVPN_UDP_GRO_MODE,
+ &ovpn->udp_gro_mode, sizeof(uint8_t)) < 0)
+ goto err;
+
ovpn_nest_end(&req.n, data);
}
@@ -1666,11 +1675,16 @@ static void usage(const char *cmd)
cmd);
fprintf(stderr, "where <command> can be one of the following\n\n");
- fprintf(stderr, "* new_iface <iface> [mode]: create new ovpn interface\n");
+ fprintf(stderr,
+ "* new_iface <iface> [mode] [udp-gro-mode]: create new ovpn interface\n");
fprintf(stderr, "\tiface: ovpn interface name\n");
fprintf(stderr, "\tmode:\n");
fprintf(stderr, "\t\t- P2P for peer-to-peer mode (i.e. client)\n");
fprintf(stderr, "\t\t- MP for multi-peer mode (i.e. server)\n");
+ fprintf(stderr, "\tudp-gro-mode:\n");
+ fprintf(stderr, "\t\t- FULL_STACK for the normal receive stack\n");
+ fprintf(stderr,
+ "\t\t- DIRECT to decrypt data from the UDP GRO callback\n");
fprintf(stderr, "* del_iface <iface>: delete ovpn interface\n");
fprintf(stderr, "\tiface: ovpn interface name\n");
@@ -2206,6 +2220,20 @@ static int ovpn_parse_cmd_args(struct ovpn_ctx *ovpn, int argc, char *argv[])
return -1;
}
ovpn->mode_set = true;
+
+ if (argc < 5)
+ break;
+
+ if (!strcmp(argv[4], "FULL_STACK")) {
+ ovpn->udp_gro_mode = OVPN_UDP_GRO_MODE_FULL_STACK;
+ } else if (!strcmp(argv[4], "DIRECT")) {
+ ovpn->udp_gro_mode = OVPN_UDP_GRO_MODE_DIRECT;
+ } else {
+ fprintf(stderr, "Cannot parse UDP GRO mode: %s\n",
+ argv[4]);
+ return -1;
+ }
+ ovpn->udp_gro_mode_set = true;
break;
case CMD_DEL_IFACE:
break;
new file mode 100755
@@ -0,0 +1,10 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (C) 2020-2025 OpenVPN, Inc.
+#
+# Author: Ralf Lici <ralf@mandelbit.com>
+# Antonio Quartulli <antonio@openvpn.net>
+
+OVPN_UDP_GRO_MODE="DIRECT"
+
+source test.sh
@@ -113,6 +113,67 @@ ovpn_run_lan_traffic() {
ip netns exec ovpn_peer0 ping -qfc 100 -w 3 "${OVPN_LAN_IP}"
}
+ovpn_udp_gro_counter_add() {
+ [ "${OVPN_PROTO}" == "UDP" ] || return 0
+ # A custom tunnel MTU can fragment outer packets before UDP GRO.
+ [ -z "${MTU:-}" ] || return 0
+
+ # Enable UDP forwarding GRO on the receiving endpoint so this test
+ # exercises the configured ovpn callback.
+ ovpn_cmd_ok "enable UDP GRO on the iperf receive path" \
+ ip netns exec ovpn_peer0 ethtool -K veth1 gro on \
+ rx-udp-gro-forwarding on
+
+ ovpn_cmd_ok "create UDP GRO path counter table" \
+ ip netns exec ovpn_peer0 nft add table inet ovpn_gro_test
+ ovpn_cmd_ok "create UDP GRO path counter chain" \
+ ip netns exec ovpn_peer0 nft \
+ "add chain inet ovpn_gro_test prerouting { type filter hook \
+ prerouting priority filter; policy accept; }"
+
+ # Count only aggregated outer packets after they enter the normal
+ # receive stack in peer0. Direct GRO consumes those DATA_V2 aggregates
+ # before this hook, while small packets which bypass veth's GRO path
+ # are deliberately ignored.
+ ovpn_cmd_ok "add UDP GRO path counter" \
+ ip netns exec ovpn_peer0 nft add rule inet ovpn_gro_test \
+ prerouting iifname "veth1" meta length gt 1500 udp dport 1 \
+ counter
+}
+
+ovpn_udp_gro_counter_check() {
+ local packets
+
+ [ "${OVPN_PROTO}" == "UDP" ] || return 0
+ [ -z "${MTU:-}" ] || return 0
+
+ packets=$(ip netns exec ovpn_peer0 nft list chain inet ovpn_gro_test \
+ prerouting | sed -n \
+ 's/.*counter packets \([0-9][0-9]*\) bytes.*/\1/p')
+ ovpn_cmd_ok "remove UDP GRO path counter table" \
+ ip netns exec ovpn_peer0 nft delete table inet ovpn_gro_test
+
+ if [ -z "${packets}" ]; then
+ printf '%s\n' "unable to read UDP GRO path counter"
+ return 1
+ fi
+
+ if [ "${OVPN_UDP_GRO_MODE}" == "FULL_STACK" ]; then
+ if [ "${packets}" -eq 0 ]; then
+ printf '%s\n' \
+ "full-stack UDP GRO did not reach PRE_ROUTING"
+ return 1
+ fi
+ return 0
+ fi
+
+ if [ "${packets}" -ne 0 ]; then
+ printf '%s\n' \
+ "direct UDP GRO reached PRE_ROUTING ${packets} times"
+ return 1
+ fi
+}
+
ovpn_run_float_mode() {
local p
local peer_ns
@@ -134,12 +195,16 @@ ovpn_run_float_mode() {
ovpn_run_iperf() {
local iperf_pid
+ ovpn_udp_gro_counter_add
+
ovpn_run_bg iperf_pid ip netns exec ovpn_peer0 iperf3 -1 -s
sleep 1
ovpn_cmd_ok "run iperf throughput flow" \
ip netns exec ovpn_peer1 iperf3 -Z -t 3 -c 5.5.5.1
wait "${iperf_pid}" || return 1
+
+ ovpn_udp_gro_counter_check
}
ovpn_run_key_rollover() {