Each packet holds its peer across packet processing and asynchronous
crypto completion. When traffic for one peer is spread across several
CPUs, the shared kref cache line therefore moves between those CPUs for
every packet.
Replace the peer kref with percpu_ref. Use the RCU-specific
live-reference helper in lookup paths, and kill the initial reference
exactly once after the peer has been removed from its lookup structures
and detached from its socket. Keep the existing RCU-delayed destruction
and release the percpu_ref storage in the final RCU callback.
percpu_ref_init can fail, so propagate allocation failure from peer
creation. This trades a per-CPU counter allocation for a cheaper live
data path on these long-lived, heavily shared objects. Specifically, in
terms of memory, this costs: 8 bytes per possible CPU for the counters,
a 56-byte control object, and 8 additional bytes in struct ovpn_peer.
In a 32-stream test using one peer and one key, perf c2c placed the peer
kref cacheline second among system-wide shared cachelines, with 37
sampled HITM loads attributed to the locked reference update. After this
change that shared reference line was no longer sampled, while the
untouched key-slot kref (addressed by the next commit) remained visible.
RX used a single receive queue, so only TX exercised CPU fan-out in this
test.
Signed-off-by: Ralf Lici <ralf@mandelbit.com>
---
drivers/net/ovpn/netlink.c | 6 +++---
drivers/net/ovpn/peer.c | 32 ++++++++++++++++++++------------
drivers/net/ovpn/peer.h | 34 ++++++++++++++++++++++++++++++----
3 files changed, 53 insertions(+), 19 deletions(-)
@@ -463,11 +463,11 @@ int ovpn_nl_peer_new_doit(struct sk_buff *skb, struct genl_info *info)
ovpn_socket_release(peer);
peer_release:
/* For UDP, the peer is unreachable until added to the hashtables, so
- * dropping the initial reference is enough. For TCP, the peer may be
+ * killing the initial reference is enough. For TCP, the peer may be
* concurrently reachable via sk_user_data->peer until
- * ovpn_socket_release() detaches; rely on the refcount.
+ * ovpn_socket_release() detaches; rely on the percpu reference.
*/
- ovpn_peer_put(peer);
+ ovpn_peer_kill(peer);
return ret;
}
@@ -33,7 +33,7 @@ static void unlock_ovpn(struct ovpn_priv *ovpn,
llist_for_each_entry_safe(peer, next, release_list->first,
release_entry) {
ovpn_socket_release(peer);
- ovpn_peer_put(peer);
+ ovpn_peer_kill(peer);
}
}
@@ -113,7 +113,6 @@ struct ovpn_peer *ovpn_peer_new(struct ovpn_priv *ovpn, u32 id)
RCU_INIT_POINTER(peer->bind, NULL);
ovpn_crypto_state_init(&peer->crypto);
spin_lock_init(&peer->lock);
- kref_init(&peer->refcount);
ovpn_peer_stats_init(&peer->vpn_stats);
ovpn_peer_stats_init(&peer->link_stats);
INIT_WORK(&peer->keepalive_work, ovpn_peer_keepalive_send);
@@ -127,6 +126,14 @@ struct ovpn_peer *ovpn_peer_new(struct ovpn_priv *ovpn, u32 id)
return ERR_PTR(ret);
}
+ ret = percpu_ref_init(&peer->refcount, ovpn_peer_release_ref, 0,
+ GFP_KERNEL);
+ if (ret < 0) {
+ dst_cache_destroy(&peer->dst_cache);
+ kfree(peer);
+ return ERR_PTR(ret);
+ }
+
netdev_hold(ovpn->dev, &peer->dev_tracker, GFP_KERNEL);
return peer;
@@ -348,6 +355,7 @@ static void ovpn_peer_release_rcu(struct rcu_head *head)
* perform it in the RCU callback, when all contexts are done
*/
dst_cache_destroy(&peer->dst_cache);
+ percpu_ref_exit(&peer->refcount);
kfree(peer);
}
@@ -366,12 +374,12 @@ static void ovpn_peer_release(struct ovpn_peer *peer)
}
/**
- * ovpn_peer_release_kref - callback for kref_put
- * @kref: the kref object belonging to the peer
+ * ovpn_peer_release_ref - callback for the peer percpu reference
+ * @ref: the percpu_ref object belonging to the peer
*/
-void ovpn_peer_release_kref(struct kref *kref)
+void ovpn_peer_release_ref(struct percpu_ref *ref)
{
- struct ovpn_peer *peer = container_of(kref, struct ovpn_peer, refcount);
+ struct ovpn_peer *peer = container_of(ref, struct ovpn_peer, refcount);
ovpn_peer_release(peer);
}
@@ -569,7 +577,7 @@ ovpn_peer_get_by_transp_addr_p2p(struct ovpn_priv *ovpn,
rcu_read_lock();
tmp = rcu_dereference(ovpn->peer);
if (likely(tmp && ovpn_peer_transp_match(tmp, ss) &&
- ovpn_peer_hold(tmp)))
+ ovpn_peer_hold_rcu(tmp)))
peer = tmp;
rcu_read_unlock();
@@ -610,7 +618,7 @@ struct ovpn_peer *ovpn_peer_get_by_transp_addr(struct ovpn_priv *ovpn,
if (!ovpn_peer_transp_match(tmp, &ss))
continue;
- if (!ovpn_peer_hold(tmp))
+ if (!ovpn_peer_hold_rcu(tmp))
continue;
peer = tmp;
@@ -641,7 +649,7 @@ static struct ovpn_peer *ovpn_peer_get_by_id_p2p(struct ovpn_priv *ovpn,
rcu_read_lock();
tmp = rcu_dereference(ovpn->peer);
- if (likely(tmp && tmp->id == peer_id && ovpn_peer_hold(tmp)))
+ if (likely(tmp && tmp->id == peer_id && ovpn_peer_hold_rcu(tmp)))
peer = tmp;
rcu_read_unlock();
@@ -671,7 +679,7 @@ struct ovpn_peer *ovpn_peer_get_by_id(struct ovpn_priv *ovpn, u32 peer_id)
if (tmp->id != peer_id)
continue;
- if (!ovpn_peer_hold(tmp))
+ if (!ovpn_peer_hold_rcu(tmp))
continue;
peer = tmp;
@@ -745,7 +753,7 @@ struct ovpn_peer *ovpn_peer_get_by_dst(struct ovpn_priv *ovpn,
if (ovpn->mode == OVPN_MODE_P2P) {
rcu_read_lock();
peer = rcu_dereference(ovpn->peer);
- if (unlikely(peer && !ovpn_peer_hold(peer)))
+ if (unlikely(peer && !ovpn_peer_hold_rcu(peer)))
peer = NULL;
rcu_read_unlock();
return peer;
@@ -763,7 +771,7 @@ struct ovpn_peer *ovpn_peer_get_by_dst(struct ovpn_priv *ovpn,
break;
}
- if (unlikely(peer && !ovpn_peer_hold(peer)))
+ if (unlikely(peer && !ovpn_peer_hold_rcu(peer)))
peer = NULL;
rcu_read_unlock();
@@ -10,6 +10,7 @@
#ifndef _NET_OVPN_OVPNPEER_H_
#define _NET_OVPN_OVPNPEER_H_
+#include <linux/percpu-refcount.h>
#include <net/dst_cache.h>
#include <net/strparser.h>
@@ -110,7 +111,7 @@ struct ovpn_peer {
struct ovpn_peer_stats link_stats;
enum ovpn_del_peer_reason delete_reason;
spinlock_t lock; /* protects bind and keepalive* */
- struct kref refcount;
+ struct percpu_ref refcount;
struct rcu_head rcu;
struct llist_node release_entry;
struct work_struct keepalive_work;
@@ -124,10 +125,23 @@ struct ovpn_peer {
*/
static inline bool ovpn_peer_hold(struct ovpn_peer *peer)
{
- return kref_get_unless_zero(&peer->refcount);
+ return percpu_ref_tryget_live(&peer->refcount);
}
-void ovpn_peer_release_kref(struct kref *kref);
+/**
+ * ovpn_peer_hold_rcu - acquire a live peer reference under RCU
+ * @peer: peer to acquire
+ *
+ * The caller must hold rcu_read_lock.
+ *
+ * Return: true if the reference was acquired, false if teardown has started
+ */
+static inline bool ovpn_peer_hold_rcu(struct ovpn_peer *peer)
+{
+ return percpu_ref_tryget_live_rcu(&peer->refcount);
+}
+
+void ovpn_peer_release_ref(struct percpu_ref *ref);
/**
* ovpn_peer_put - decrease reference counter
@@ -135,7 +149,19 @@ void ovpn_peer_release_kref(struct kref *kref);
*/
static inline void ovpn_peer_put(struct ovpn_peer *peer)
{
- kref_put(&peer->refcount, ovpn_peer_release_kref);
+ percpu_ref_put(&peer->refcount);
+}
+
+/**
+ * ovpn_peer_kill - drop the peer's initial reference
+ * @peer: peer which has been unpublished and is being destroyed
+ *
+ * This must be called exactly once, after the peer is no longer reachable
+ * through its owning ovpn instance.
+ */
+static inline void ovpn_peer_kill(struct ovpn_peer *peer)
+{
+ percpu_ref_kill(&peer->refcount);
}
struct ovpn_peer *ovpn_peer_new(struct ovpn_priv *ovpn, u32 id);