@@ -134,13 +134,17 @@ static struct scatterlist *ovpn_aead_crypto_req_sg(struct crypto_aead *aead,
static struct aead_request *ovpn_aead_request_alloc(struct crypto_aead *aead,
struct sk_buff *skb,
- unsigned int nents, u8 **iv)
+ unsigned int nents,
+ unsigned int extra, u8 **iv)
{
struct aead_request *req;
void *tmp;
- /* allocate IV, request and scatterlist entries in one block */
- tmp = kmalloc(ovpn_aead_crypto_tmp_size(aead, nents), GFP_ATOMIC);
+ /* allocate IV, request, scatterlist entries and caller scratch space
+ * in one block
+ */
+ tmp = kmalloc(ovpn_aead_crypto_tmp_size(aead, nents) + extra,
+ GFP_ATOMIC);
if (unlikely(!tmp))
return ERR_PTR(-ENOMEM);
@@ -217,7 +221,7 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
nfrags = 1;
}
- req = ovpn_aead_request_alloc(ks->encrypt, skb, nfrags + 2, &iv);
+ req = ovpn_aead_request_alloc(ks->encrypt, skb, nfrags + 2, 0, &iv);
if (IS_ERR(req))
return PTR_ERR(req);
sg = ovpn_aead_crypto_req_sg(ks->encrypt, req);
@@ -261,6 +265,79 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
return crypto_aead_encrypt(req);
}
+int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
+ struct ovpn_crypto_key_slot *ks, struct sk_buff *skb,
+ struct sk_buff *gso_skb, unsigned int offset)
+{
+ const unsigned int dst_nents = skb_shinfo(gso_skb)->nr_frags + 4;
+ const unsigned int src_nents = 2;
+ unsigned int nents, payload_off;
+ struct scatterlist *src, *dst;
+ struct aead_request *req;
+ int dst_idx, mapped, ret;
+ u8 *aad, *iv;
+
+ /* each input records the shared peer and key for the common completion
+ * path but their references remain owned by the output aggregate
+ */
+ ovpn_skb_cb(skb)->peer = peer;
+ ovpn_skb_cb(skb)->ks = ks;
+
+ if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
+ return -EINVAL;
+
+ nents = src_nents + dst_nents;
+ req = ovpn_aead_request_alloc(ks->encrypt, skb, nents, OVPN_AAD_SIZE,
+ &iv);
+ if (IS_ERR(req))
+ return PTR_ERR(req);
+ src = ovpn_aead_crypto_req_sg(ks->encrypt, req);
+ dst = src + src_nents;
+ aad = (u8 *)(dst + dst_nents);
+
+ ret = ovpn_aead_encrypt_header(peer, ks, iv, aad);
+ if (unlikely(ret < 0))
+ return ret;
+
+ ret = skb_store_bits(gso_skb, offset, aad, OVPN_AAD_SIZE);
+ if (unlikely(ret < 0))
+ return ret;
+
+ /* encrypt out of place from the original segmented skb directly into
+ * its final range in the UDP GSO skb
+ */
+ sg_init_table(src, src_nents);
+ sg_set_buf(src, aad, OVPN_AAD_SIZE);
+ sg_set_buf(src + 1, skb->data, skb->len);
+
+ sg_init_table(dst, dst_nents);
+ dst_idx = skb_to_sgvec_nomark(gso_skb, dst, offset, OVPN_AAD_SIZE);
+ if (unlikely(dst_idx < 0))
+ return dst_idx;
+
+ payload_off = offset + OVPN_AAD_SIZE + OVPN_AUTH_TAG_SIZE;
+ mapped = skb_to_sgvec_nomark(gso_skb, dst + dst_idx, payload_off,
+ skb->len);
+ if (unlikely(mapped < 0))
+ return mapped;
+ dst_idx += mapped;
+
+ mapped = skb_to_sgvec_nomark(gso_skb, dst + dst_idx,
+ offset + OVPN_AAD_SIZE,
+ OVPN_AUTH_TAG_SIZE);
+ if (unlikely(mapped < 0))
+ return mapped;
+ dst_idx += mapped;
+ sg_mark_end(&dst[dst_idx - 1]);
+
+ aead_request_set_tfm(req, ks->encrypt);
+ aead_request_set_callback(req, 0, ovpn_encrypt_post, skb);
+ aead_request_set_crypt(req, src, dst, skb->len, iv);
+ aead_request_set_ad(req, OVPN_AAD_SIZE);
+
+ return crypto_aead_encrypt(req);
+}
+
int ovpn_aead_decrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
struct sk_buff *skb)
{
@@ -17,6 +17,10 @@
int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
struct sk_buff *skb);
+int ovpn_aead_encrypt_gso(struct ovpn_peer *peer,
+ struct ovpn_crypto_key_slot *ks,
+ struct sk_buff *skb, struct sk_buff *gso_skb,
+ unsigned int offset);
int ovpn_aead_decrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
struct sk_buff *skb);
@@ -13,6 +13,8 @@
#include <net/gro_cells.h>
#include <net/gso.h>
#include <net/ip.h>
+#include <net/sock.h>
+#include <net/udp.h>
#include "ovpnpriv.h"
#include "peer.h"
@@ -32,6 +34,12 @@ const unsigned char ovpn_keepalive_message[OVPN_KEEPALIVE_SIZE] = {
0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48
};
+/* Leave room for the largest outer network header. The strict inequality in
+ * is_skb_forwardable also requires staying one byte below gso_max_size.
+ */
+#define OVPN_UDP_GSO_MAX_PAYLOAD (GSO_LEGACY_MAX_SIZE - \
+ sizeof(struct ipv6hdr) - \
+ sizeof(struct udphdr) - 1)
/**
* ovpn_is_keepalive - check if skb contains a keepalive message
* @skb: packet to check
@@ -237,11 +245,13 @@ void ovpn_recv(struct ovpn_peer *peer, struct sk_buff *skb)
void ovpn_encrypt_post(void *data, int ret)
{
+ unsigned int orig_len, packets = 1;
struct ovpn_crypto_key_slot *ks;
struct sk_buff *skb = data;
struct ovpn_socket *sock;
+ struct ovpn_cb *batch_cb;
struct ovpn_peer *peer;
- unsigned int orig_len;
+ struct sk_buff *batch;
/* encryption is happening asynchronously. This function will be
* called later by the crypto callback with a proper return value
@@ -249,15 +259,19 @@ void ovpn_encrypt_post(void *data, int ret)
if (unlikely(ret == -EINPROGRESS))
return;
- ks = ovpn_skb_cb(skb)->ks;
+ /* ordinary encryption leaves batch zeroed; a GSO input uses it to find
+ * the aggregate whose lifetime is shared by all segment requests
+ */
+ batch = ovpn_skb_cb(skb)->batch;
peer = ovpn_skb_cb(skb)->peer;
+ ks = ovpn_skb_cb(skb)->ks;
/* crypto is done, cleanup skb CB and its members */
kfree(ovpn_skb_cb(skb)->crypto_tmp);
if (unlikely(ret == -ERANGE)) {
/* we ran out of IVs and we must kill the key as it can't be
- * use anymore
+ * used anymore
*/
netdev_warn(peer->ovpn->dev,
"killing key %u for peer %u\n", ks->key_id,
@@ -265,8 +279,30 @@ void ovpn_encrypt_post(void *data, int ret)
if (ovpn_crypto_kill_key(&peer->crypto, ks->key_id))
/* let userspace know so that a new key must be negotiated */
ovpn_nl_key_swap_notify(peer, ks->key_id);
+ }
- goto err;
+ if (batch) {
+ batch_cb = ovpn_skb_cb(batch);
+ /* every segment publishes its result before releasing its
+ * pending count and only the final completion continues with
+ * the aggregate
+ */
+ if (unlikely(ret < 0))
+ atomic_set(&batch_cb->batch_state.failed, 1);
+
+ kfree_skb(skb);
+ if (!atomic_dec_and_test(&batch_cb->batch_state.pending))
+ return;
+
+ skb = batch;
+ packets = skb_shinfo(batch)->gso_segs;
+ if (unlikely(atomic_read(&batch_cb->batch_state.failed)))
+ goto err;
+
+ /* reaching the final callback with no sticky failure means
+ * every segment completed successfully
+ */
+ ret = 0;
}
if (unlikely(ret < 0))
@@ -292,7 +328,7 @@ void ovpn_encrypt_post(void *data, int ret)
goto err_unlock;
}
- ovpn_peer_stats_increment_tx(&peer->link_stats, orig_len);
+ ovpn_peer_stats_add_tx(&peer->link_stats, orig_len, packets);
/* keep track of last sent packet for keepalive */
WRITE_ONCE(peer->last_sent, ktime_get_boottime_seconds());
/* skb passed down the stack - don't free it */
@@ -301,7 +337,7 @@ void ovpn_encrypt_post(void *data, int ret)
rcu_read_unlock();
err:
if (unlikely(skb))
- ovpn_dev_dstats_tx_dropped(peer->ovpn->dev);
+ ovpn_dev_dstats_tx_dropped(peer->ovpn->dev, packets);
kfree_skb(skb);
if (likely(ks))
ovpn_crypto_key_slot_put(ks);
@@ -309,29 +345,124 @@ void ovpn_encrypt_post(void *data, int ret)
ovpn_peer_put(peer);
}
-static bool ovpn_encrypt_one(struct ovpn_peer *peer, struct sk_buff *skb)
+/* Hold the peer and its primary key for one encryption submission.
+ * The returned key and the peer each carry one reference which completion must
+ * release.
+ */
+static struct ovpn_crypto_key_slot *
+ovpn_encrypt_refs_get(struct ovpn_peer *peer)
{
struct ovpn_crypto_key_slot *ks;
/* get primary key to be used for encrypting data */
ks = ovpn_crypto_key_slot_primary(&peer->crypto);
if (unlikely(!ks))
- return false;
+ return NULL;
- /* take a reference to the peer because the crypto code may run async.
- * ovpn_encrypt_post() will release it upon completion
+ /* the caller already owns a peer reference, so failure indicates a
+ * broken reference lifetime elsewhere
*/
if (unlikely(!ovpn_peer_hold(peer))) {
DEBUG_NET_WARN_ON_ONCE(1);
ovpn_crypto_key_slot_put(ks);
- return false;
+ return NULL;
}
+ return ks;
+}
+
+static bool ovpn_encrypt_one(struct ovpn_peer *peer, struct sk_buff *skb)
+{
+ struct ovpn_crypto_key_slot *ks;
+
+ ks = ovpn_encrypt_refs_get(peer);
+ if (unlikely(!ks))
+ return false;
+
memset(ovpn_skb_cb(skb), 0, sizeof(struct ovpn_cb));
ovpn_encrypt_post(skb, ovpn_aead_encrypt(peer, ks, skb));
return true;
}
+static bool ovpn_encrypt_gso_queue(struct sk_buff_head *skbs,
+ struct ovpn_peer *peer,
+ struct sk_buff *batch,
+ unsigned int segments)
+{
+ unsigned int offset = 0, i, len;
+ struct ovpn_crypto_key_slot *ks;
+ struct sk_buff *skb;
+
+ /* acquire all shared state before removing the first input skb so that
+ * failure can leave the queue intact for the ordinary transmit path
+ */
+ ks = ovpn_encrypt_refs_get(peer);
+ if (unlikely(!ks))
+ return false;
+
+ /* the aggregate owns these references until every sync or async crypto
+ * completion has finished
+ */
+ memset(ovpn_skb_cb(batch), 0, sizeof(struct ovpn_cb));
+ ovpn_skb_cb(batch)->peer = peer;
+ ovpn_skb_cb(batch)->ks = ks;
+ atomic_set(&ovpn_skb_cb(batch)->batch_state.pending, segments);
+ atomic_set(&ovpn_skb_cb(batch)->batch_state.failed, 0);
+
+ for (i = 0; i < segments; i++) {
+ skb = __skb_dequeue(skbs);
+ len = skb->len + OVPN_DATA_V2_OVERHEAD;
+
+ memset(ovpn_skb_cb(skb), 0, sizeof(struct ovpn_cb));
+ ovpn_skb_cb(skb)->batch = batch;
+ ovpn_encrypt_post(skb, ovpn_aead_encrypt_gso(peer, ks, skb,
+ batch, offset));
+ offset += len;
+ }
+
+ return true;
+}
+
+static struct sk_buff *ovpn_udp_gso_alloc(const struct sk_buff *first_segment,
+ unsigned int batch_len,
+ unsigned int segments)
+{
+ struct sk_buff *gso_skb;
+ int ret;
+
+ gso_skb = alloc_skb_with_frags(OVPN_HEAD_ROOM, batch_len,
+ SKB_FRAG_PAGE_ORDER, &ret, GFP_ATOMIC);
+ if (unlikely(!gso_skb))
+ return NULL;
+
+ skb_reserve(gso_skb, OVPN_HEAD_ROOM);
+ gso_skb->len = batch_len;
+ gso_skb->data_len = batch_len;
+ gso_skb->priority = first_segment->priority;
+
+ /* Segments retain the originating socket so we keep its send-buffer
+ * accounting active until the UDP GSO is transmitted. This also
+ * preserves its cached TX queue.
+ */
+ if (first_segment->sk && is_skb_wmem(first_segment))
+ skb_set_owner_w(gso_skb, first_segment->sk);
+ skb_copy_hash(gso_skb, first_segment);
+#ifdef CONFIG_XPS
+ /* keep the aggregate on the TX queue selected for the original flow
+ * otherwise async crypto completion on another CPU could move the flow
+ * to a different queue and cause delay or reordering
+ */
+ gso_skb->sender_cpu = first_segment->sender_cpu;
+#endif
+
+ skb_shinfo(gso_skb)->gso_type = SKB_GSO_UDP_L4;
+ skb_shinfo(gso_skb)->gso_size = first_segment->len +
+ OVPN_DATA_V2_OVERHEAD;
+ skb_shinfo(gso_skb)->gso_segs = segments;
+
+ return gso_skb;
+}
+
/* send skb to connected peer, if any */
static void ovpn_send(struct ovpn_priv *ovpn, struct sk_buff *skb,
struct ovpn_peer *peer)
@@ -343,7 +474,7 @@ static void ovpn_send(struct ovpn_priv *ovpn, struct sk_buff *skb,
*/
skb_list_walk_safe(skb, curr, next) {
if (unlikely(!ovpn_encrypt_one(peer, curr))) {
- ovpn_dev_dstats_tx_dropped(ovpn->dev);
+ ovpn_dev_dstats_tx_dropped(ovpn->dev, 1);
kfree_skb(curr);
}
}
@@ -351,19 +482,96 @@ static void ovpn_send(struct ovpn_priv *ovpn, struct sk_buff *skb,
ovpn_peer_put(peer);
}
+/* encrypt fixed-size input segments into one or more UDP GSO aggregates */
+static void ovpn_send_gso(struct sk_buff_head *skbs, struct ovpn_peer *peer)
+{
+ unsigned int max_segs = 1, seg_len, batch_len, segs;
+ struct sk_buff *batch;
+
+ seg_len = skb_peek(skbs)->len + OVPN_DATA_V2_OVERHEAD;
+ max_segs = min_t(unsigned int, UDP_MAX_SEGMENTS,
+ OVPN_UDP_GSO_MAX_PAYLOAD / seg_len);
+
+ /* if even two encrypted skbs cannot fit, leave the whole queue for the
+ * ordinary transmit path
+ */
+ if (max_segs < 2)
+ return;
+
+ /* An input GSO skb might be prduce more than max_segs segments so we
+ * consume as many as we can for each iteration. A final single skb, or
+ * the whole remainder after an allocation failure, stays queued and
+ * fallback to the ordinary transmit path.
+ */
+ while (skb_queue_len(skbs) > 1) {
+ segs = min_t(unsigned int, skb_queue_len(skbs), max_segs);
+
+ /* all but the final input skb have the same length, so start
+ * with the full-size calculation and adjust only the final
+ * group below
+ */
+ batch_len = segs * (skb_peek(skbs)->len +
+ OVPN_DATA_V2_OVERHEAD);
+ if (segs == skb_queue_len(skbs))
+ batch_len -= skb_peek(skbs)->len -
+ skb_peek_tail(skbs)->len;
+
+ batch = ovpn_udp_gso_alloc(skb_peek(skbs), batch_len, segs);
+ if (unlikely(!batch))
+ return;
+
+ if (unlikely(!ovpn_encrypt_gso_queue(skbs, peer,
+ batch, segs))) {
+ kfree_skb(batch);
+ return;
+ }
+ }
+}
+
+static bool ovpn_peer_supports_udp_gso(struct ovpn_peer *peer)
+{
+ struct ovpn_socket *sock;
+ bool udp_gso;
+
+ rcu_read_lock();
+ sock = rcu_dereference(peer->sock);
+ /* UDP GSO requires checksums. These socket settings can change after
+ * we decide to batch, but an already-built batch remains checksummed.
+ * Linux's ordinary UDP GSO path makes the same choice.
+ */
+ udp_gso = sock && sock->sk->sk_protocol == IPPROTO_UDP &&
+ !sock->sk->sk_no_check_tx && !udp_get_no_check6_tx(sock->sk);
+ rcu_read_unlock();
+
+ return udp_gso;
+}
+
/* Send user data to the network
*/
netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev)
{
struct ovpn_priv *ovpn = netdev_priv(dev);
struct sk_buff *segments, *curr, *next;
+ const bool gso_in = skb_is_gso(skb);
struct sk_buff_head skb_list;
netdev_features_t features;
unsigned int tx_bytes = 0;
struct ovpn_peer *peer;
+ bool gso_out;
__be16 proto;
int ret;
+ /* A frag-list GSO skb already stores complete segments as child skbs.
+ * Keep those children on the ordinary in-place encryption path instead
+ * of copying them into a replacement UDP GSO skb.
+ *
+ * GSO_BY_FRAGS input must also remain on that path because its variable
+ * segment sizes cannot be represented by one UDP GSO output size.
+ */
+ gso_out = gso_in &&
+ !(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST) &&
+ skb_shinfo(skb)->gso_size != GSO_BY_FRAGS;
+
/* reset netfilter state */
nf_reset_ct(skb);
@@ -392,13 +600,14 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev)
/* dst was needed for peer selection - it can now be dropped */
skb_dst_drop(skb);
- if (skb_is_gso(skb)) {
- /* force software segmentation, but keep ovpn's non-GSO feature
- * bits so the generated segments can preserve non-linear skb
- * data where possible
+ if (gso_in) {
+ /* force software segmentation into linear skbs and calculate
+ * each checksum while copying the segment
*/
features = netif_skb_features(skb);
- segments = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);
+ features &= ~(NETIF_F_GSO_MASK | NETIF_F_SG |
+ NETIF_F_CSUM_MASK);
+ segments = skb_gso_segment(skb, features);
if (IS_ERR_OR_NULL(segments)) {
ret = PTR_ERR(segments);
net_err_ratelimited("%s: cannot segment payload packet: %d\n",
@@ -420,7 +629,8 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev)
if (unlikely(!curr)) {
net_err_ratelimited("%s: skb_share_check failed for payload packet\n",
netdev_name(dev));
- ovpn_dev_dstats_tx_dropped(ovpn->dev);
+ ovpn_dev_dstats_tx_dropped(ovpn->dev, 1);
+ gso_out = false;
continue;
}
@@ -429,8 +639,9 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev)
skb_checksum_help(curr) < 0)) {
net_err_ratelimited("%s: skb_checksum_help failed for payload packet\n",
netdev_name(dev));
- ovpn_dev_dstats_tx_dropped(ovpn->dev);
+ ovpn_dev_dstats_tx_dropped(ovpn->dev, 1);
kfree_skb(curr);
+ gso_out = false;
continue;
}
@@ -446,9 +657,14 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev)
ovpn_peer_put(peer);
return NETDEV_TX_OK;
}
- skb_list.prev->next = NULL;
ovpn_peer_stats_increment_tx(&peer->vpn_stats, tx_bytes);
+
+ if (gso_out && skb_queue_len(&skb_list) > 1 &&
+ ovpn_peer_supports_udp_gso(peer))
+ ovpn_send_gso(&skb_list, peer);
+
+ skb_list.prev->next = NULL;
ovpn_send(ovpn, skb_list.next, peer);
return NETDEV_TX_OK;
@@ -456,7 +672,7 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev)
drop:
ovpn_peer_put(peer);
drop_no_peer:
- ovpn_dev_dstats_tx_dropped(ovpn->dev);
+ ovpn_dev_dstats_tx_dropped(ovpn->dev, 1);
skb_tx_error(skb);
kfree_skb_list(skb);
return NETDEV_TX_OK;
@@ -10,6 +10,7 @@
#ifndef _NET_OVPN_SKB_H_
#define _NET_OVPN_SKB_H_
+#include <linux/atomic.h>
#include <linux/in.h>
#include <linux/in6.h>
#include <linux/ip.h>
@@ -20,19 +21,35 @@
/**
* struct ovpn_cb - ovpn skb control block
- * @peer: the peer this skb was received from/sent to
- * @ks: the crypto key slot used to encrypt/decrypt this skb
* @crypto_tmp: pointer to temporary memory used for crypto operations
* containing the IV, the scatter gather list and the aead request
+ * @peer: peer used by this crypto operation or owned by this aggregate
+ * @ks: crypto key slot used by this operation or owned by this aggregate
* @payload_offset: offset in the skb where the payload starts
* @nosignal: whether this skb should be sent with the MSG_NOSIGNAL flag (TCP)
+ * @batch: UDP GSO aggregate receiving this input skb's encrypted payload
+ * @batch_state: completion state owned by a UDP GSO aggregate
*/
struct ovpn_cb {
+ void *crypto_tmp;
struct ovpn_peer *peer;
struct ovpn_crypto_key_slot *ks;
- void *crypto_tmp;
- unsigned int payload_offset;
- bool nosignal;
+
+ /* Ordinary encryption leaves this union zeroed. Decryption and TCP use
+ * their ordinary fields, a UDP GSO input stores its output aggregate,
+ * and that aggregate uses the same space to coordinate its completions.
+ */
+ union {
+ struct {
+ unsigned int payload_offset;
+ bool nosignal;
+ };
+ struct sk_buff *batch;
+ struct {
+ atomic_t pending;
+ atomic_t failed;
+ } batch_state;
+ };
};
static inline struct ovpn_cb *ovpn_skb_cb(struct sk_buff *skb)
@@ -40,16 +40,26 @@ static inline void ovpn_peer_stats_increment_rx(struct ovpn_peer_stats *stats,
ovpn_peer_stats_increment(&stats->rx, n);
}
+static inline void ovpn_peer_stats_add_tx(struct ovpn_peer_stats *stats,
+ const unsigned int bytes,
+ unsigned int packets)
+{
+ atomic64_add(bytes, &stats->tx.bytes);
+ atomic64_add(packets, &stats->tx.packets);
+}
+
static inline void ovpn_peer_stats_increment_tx(struct ovpn_peer_stats *stats,
const unsigned int n)
{
- ovpn_peer_stats_increment(&stats->tx, n);
+ ovpn_peer_stats_add_tx(stats, n, 1);
}
-static inline void ovpn_dev_dstats_tx_dropped(struct net_device *dev)
+static inline void ovpn_dev_dstats_tx_dropped(struct net_device *dev,
+ unsigned int packets)
{
local_bh_disable();
- dev_dstats_tx_dropped(dev);
+ while (packets--)
+ dev_dstats_tx_dropped(dev);
local_bh_enable();
}
@@ -332,7 +332,7 @@ static void ovpn_tcp_send_sock_skb(struct ovpn_peer *peer, struct sock *sk,
ovpn_tcp_send_sock(peer, sk);
if (peer->tcp.out_msg.skb) {
- ovpn_dev_dstats_tx_dropped(peer->ovpn->dev);
+ ovpn_dev_dstats_tx_dropped(peer->ovpn->dev, 1);
kfree_skb(skb);
return;
}
@@ -354,7 +354,7 @@ void ovpn_tcp_send_skb(struct ovpn_peer *peer, struct sock *sk,
if (sock_owned_by_user(sk)) {
if (skb_queue_len(&peer->tcp.out_queue) >=
READ_ONCE(net_hotdata.max_backlog)) {
- ovpn_dev_dstats_tx_dropped(peer->ovpn->dev);
+ ovpn_dev_dstats_tx_dropped(peer->ovpn->dev, 1);
kfree_skb(skb);
goto unlock;
}
@@ -121,6 +121,7 @@ static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
/* pop off outer UDP header */
__skb_pull(skb, sizeof(struct udphdr));
+ skb_mark_not_on_list(skb);
ovpn_recv(peer, skb);
return 0;
@@ -196,9 +197,13 @@ static int ovpn_udp4_output(struct ovpn_peer *peer, struct ovpn_bind *bind,
dst_cache_set_ip4(cache, &rt->dst, fl.saddr);
transmit:
+ /* an already-built UDP GSO needs a checksum seed even if the socket's
+ * no-check option changed while encryption was in flight
+ */
udp_tunnel_xmit_skb(rt, sk, skb, fl.saddr, fl.daddr, 0,
ip4_dst_hoplimit(&rt->dst), 0, fl.fl4_sport,
- fl.fl4_dport, false, sk->sk_no_check_tx, 0);
+ fl.fl4_dport, false,
+ !skb_is_gso(skb) && sk->sk_no_check_tx, 0);
ret = 0;
err:
local_bh_enable();
@@ -271,9 +276,13 @@ static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind,
* udp_tunnel_xmit_skb()
*/
skb->ignore_df = 1;
+ /* keep checksum offload enabled for an in-flight UDP GSO batch even if
+ * the socket's no-check option has changed since batch creation
+ */
udp_tunnel6_xmit_skb(dst, sk, skb, skb->dev, &fl.saddr, &fl.daddr, 0,
ip6_dst_hoplimit(dst), 0, fl.fl6_sport,
- fl.fl6_dport, udp_get_no_check6_tx(sk), 0);
+ fl.fl6_dport,
+ !skb_is_gso(skb) && udp_get_no_check6_tx(sk), 0);
ret = 0;
err:
local_bh_enable();
@@ -344,8 +353,18 @@ void ovpn_udp_send_skb(struct ovpn_peer *peer, struct sock *sk,
skb->dev = peer->ovpn->dev;
skb->mark = READ_ONCE(sk->sk_mark);
- /* no checksum performed at this layer */
- skb->ip_summed = CHECKSUM_NONE;
+ if (skb_is_gso(skb)) {
+ /* udp_tunnel_xmit_skb installs the outer UDP header after this
+ * function returns: point CHECKSUM_PARTIAL at that future
+ * header so both hw and sw UDP GSO can complete the checksum.
+ */
+ skb->ip_summed = CHECKSUM_PARTIAL;
+ skb->csum_start = skb_headroom(skb) - sizeof(struct udphdr);
+ skb->csum_offset = offsetof(struct udphdr, check);
+ } else {
+ /* no checksum performed at this layer */
+ skb->ip_summed = CHECKSUM_NONE;
+ }
/* crypto layer -> transport (UDP) */
ret = ovpn_udp_output(peer, &peer->dst_cache, sk, skb);