@@ -321,11 +321,7 @@ static bool ovpn_encrypt_one(struct ovpn_peer *peer, struct sk_buff *skb)
/* take a reference to the peer because the crypto code may run async.
* ovpn_encrypt_post() will release it upon completion
*/
- if (unlikely(!ovpn_peer_hold(peer))) {
- DEBUG_NET_WARN_ON_ONCE(1);
- ovpn_crypto_key_slot_put(ks);
- return false;
- }
+ ovpn_peer_hold(peer);
memset(ovpn_skb_cb(skb), 0, sizeof(struct ovpn_cb));
ovpn_encrypt_post(skb, ovpn_aead_encrypt(peer, ks, skb));
@@ -463,11 +463,11 @@ int ovpn_nl_peer_new_doit(struct sk_buff *skb, struct genl_info *info)
ovpn_socket_release(peer);
peer_release:
/* For UDP, the peer is unreachable until added to the hashtables, so
- * dropping the initial reference is enough. For TCP, the peer may be
+ * killing the initial reference is enough. For TCP, the peer may be
* concurrently reachable via sk_user_data->peer until
- * ovpn_socket_release() detaches; rely on the refcount.
+ * ovpn_socket_release() detaches; rely on the percpu reference.
*/
- ovpn_peer_put(peer);
+ ovpn_peer_kill(peer);
return ret;
}
@@ -22,6 +22,8 @@
#include "peer.h"
#include "socket.h"
+static void ovpn_peer_release_ref(struct percpu_ref *ref);
+
static void unlock_ovpn(struct ovpn_priv *ovpn,
struct llist_head *release_list)
__releases(&ovpn->lock)
@@ -33,7 +35,7 @@ static void unlock_ovpn(struct ovpn_priv *ovpn,
llist_for_each_entry_safe(peer, next, release_list->first,
release_entry) {
ovpn_socket_release(peer);
- ovpn_peer_put(peer);
+ ovpn_peer_kill(peer);
}
}
@@ -113,7 +115,6 @@ struct ovpn_peer *ovpn_peer_new(struct ovpn_priv *ovpn, u32 id)
RCU_INIT_POINTER(peer->bind, NULL);
ovpn_crypto_state_init(&peer->crypto);
spin_lock_init(&peer->lock);
- kref_init(&peer->refcount);
ovpn_peer_stats_init(&peer->vpn_stats);
ovpn_peer_stats_init(&peer->link_stats);
INIT_WORK(&peer->keepalive_work, ovpn_peer_keepalive_send);
@@ -127,6 +128,14 @@ struct ovpn_peer *ovpn_peer_new(struct ovpn_priv *ovpn, u32 id)
return ERR_PTR(ret);
}
+ ret = percpu_ref_init(&peer->refcount, ovpn_peer_release_ref, 0,
+ GFP_KERNEL);
+ if (ret < 0) {
+ dst_cache_destroy(&peer->dst_cache);
+ kfree(peer);
+ return ERR_PTR(ret);
+ }
+
netdev_hold(ovpn->dev, &peer->dev_tracker, GFP_KERNEL);
return peer;
@@ -348,6 +357,7 @@ static void ovpn_peer_release_rcu(struct rcu_head *head)
* perform it in the RCU callback, when all contexts are done
*/
dst_cache_destroy(&peer->dst_cache);
+ percpu_ref_exit(&peer->refcount);
kfree(peer);
}
@@ -366,12 +376,12 @@ static void ovpn_peer_release(struct ovpn_peer *peer)
}
/**
- * ovpn_peer_release_kref - callback for kref_put
- * @kref: the kref object belonging to the peer
+ * ovpn_peer_release_ref - callback for the peer percpu reference
+ * @ref: the percpu_ref object belonging to the peer
*/
-void ovpn_peer_release_kref(struct kref *kref)
+static void ovpn_peer_release_ref(struct percpu_ref *ref)
{
- struct ovpn_peer *peer = container_of(kref, struct ovpn_peer, refcount);
+ struct ovpn_peer *peer = container_of(ref, struct ovpn_peer, refcount);
ovpn_peer_release(peer);
}
@@ -569,7 +579,7 @@ ovpn_peer_get_by_transp_addr_p2p(struct ovpn_priv *ovpn,
rcu_read_lock();
tmp = rcu_dereference(ovpn->peer);
if (likely(tmp && ovpn_peer_transp_match(tmp, ss) &&
- ovpn_peer_hold(tmp)))
+ ovpn_peer_hold_rcu(tmp)))
peer = tmp;
rcu_read_unlock();
@@ -610,7 +620,7 @@ struct ovpn_peer *ovpn_peer_get_by_transp_addr(struct ovpn_priv *ovpn,
if (!ovpn_peer_transp_match(tmp, &ss))
continue;
- if (!ovpn_peer_hold(tmp))
+ if (!ovpn_peer_hold_rcu(tmp))
continue;
peer = tmp;
@@ -641,7 +651,7 @@ static struct ovpn_peer *ovpn_peer_get_by_id_p2p(struct ovpn_priv *ovpn,
rcu_read_lock();
tmp = rcu_dereference(ovpn->peer);
- if (likely(tmp && tmp->id == peer_id && ovpn_peer_hold(tmp)))
+ if (likely(tmp && tmp->id == peer_id && ovpn_peer_hold_rcu(tmp)))
peer = tmp;
rcu_read_unlock();
@@ -671,7 +681,7 @@ struct ovpn_peer *ovpn_peer_get_by_id(struct ovpn_priv *ovpn, u32 peer_id)
if (tmp->id != peer_id)
continue;
- if (!ovpn_peer_hold(tmp))
+ if (!ovpn_peer_hold_rcu(tmp))
continue;
peer = tmp;
@@ -745,7 +755,7 @@ struct ovpn_peer *ovpn_peer_get_by_dst(struct ovpn_priv *ovpn,
if (ovpn->mode == OVPN_MODE_P2P) {
rcu_read_lock();
peer = rcu_dereference(ovpn->peer);
- if (unlikely(peer && !ovpn_peer_hold(peer)))
+ if (unlikely(peer && !ovpn_peer_hold_rcu(peer)))
peer = NULL;
rcu_read_unlock();
return peer;
@@ -763,7 +773,7 @@ struct ovpn_peer *ovpn_peer_get_by_dst(struct ovpn_priv *ovpn,
break;
}
- if (unlikely(peer && !ovpn_peer_hold(peer)))
+ if (unlikely(peer && !ovpn_peer_hold_rcu(peer)))
peer = NULL;
rcu_read_unlock();
@@ -1369,8 +1379,7 @@ static time64_t ovpn_peer_keepalive_work_single(struct ovpn_peer *peer,
netdev_dbg(peer->ovpn->dev,
"sending keepalive to peer %u\n",
peer->id);
- if (WARN_ON(!ovpn_peer_hold(peer)))
- return 0;
+ ovpn_peer_hold(peer);
if (!queue_work(ovpn_wq, &peer->keepalive_work))
ovpn_peer_put(peer);
}
@@ -10,6 +10,7 @@
#ifndef _NET_OVPN_OVPNPEER_H_
#define _NET_OVPN_OVPNPEER_H_
+#include <linux/percpu-refcount.h>
#include <net/dst_cache.h>
#include <net/strparser.h>
@@ -110,7 +111,7 @@ struct ovpn_peer {
struct ovpn_peer_stats link_stats;
enum ovpn_del_peer_reason delete_reason;
spinlock_t lock; /* protects bind and keepalive* */
- struct kref refcount;
+ struct percpu_ref refcount;
struct rcu_head rcu;
struct llist_node release_entry;
struct work_struct keepalive_work;
@@ -120,14 +121,26 @@ struct ovpn_peer {
* ovpn_peer_hold - increase reference counter
* @peer: the peer whose counter should be increased
*
- * Return: true if the counter was increased or false if it was zero already
+ * The caller must already own a peer reference or otherwise guarantee that
+ * teardown cannot complete while the new reference is acquired.
*/
-static inline bool ovpn_peer_hold(struct ovpn_peer *peer)
+static inline void ovpn_peer_hold(struct ovpn_peer *peer)
{
- return kref_get_unless_zero(&peer->refcount);
+ percpu_ref_get(&peer->refcount);
}
-void ovpn_peer_release_kref(struct kref *kref);
+/**
+ * ovpn_peer_hold_rcu - acquire a live peer reference under RCU
+ * @peer: peer to acquire
+ *
+ * The caller must hold rcu_read_lock.
+ *
+ * Return: true if the reference was acquired, false if teardown has started
+ */
+static inline bool ovpn_peer_hold_rcu(struct ovpn_peer *peer)
+{
+ return percpu_ref_tryget_live_rcu(&peer->refcount);
+}
/**
* ovpn_peer_put - decrease reference counter
@@ -135,7 +148,19 @@ void ovpn_peer_release_kref(struct kref *kref);
*/
static inline void ovpn_peer_put(struct ovpn_peer *peer)
{
- kref_put(&peer->refcount, ovpn_peer_release_kref);
+ percpu_ref_put(&peer->refcount);
+}
+
+/**
+ * ovpn_peer_kill - drop the peer's initial reference
+ * @peer: peer which has been unpublished and is being destroyed
+ *
+ * This must be called exactly once, after the peer is no longer reachable
+ * through its owning ovpn instance.
+ */
+static inline void ovpn_peer_kill(struct ovpn_peer *peer)
+{
+ percpu_ref_kill(&peer->refcount);
}
struct ovpn_peer *ovpn_peer_new(struct ovpn_priv *ovpn, u32 id);
@@ -140,21 +140,18 @@ static void ovpn_tcp_rcv(struct strparser *strp, struct sk_buff *skb)
/* hold reference to peer as required by ovpn_recv().
*
* NOTE: in this context we should already be holding a reference to
- * this peer, therefore ovpn_peer_hold() is not expected to fail
+ * this peer
*/
- if (WARN_ON(!ovpn_peer_hold(peer)))
- goto err_nopeer;
+ ovpn_peer_hold(peer);
ovpn_recv(peer, skb);
return;
err:
- /* take reference for deferred peer deletion. should never fail */
- if (WARN_ON(!ovpn_peer_hold(peer)))
- goto err_nopeer;
+ /* take reference for deferred peer deletion */
+ ovpn_peer_hold(peer);
if (!queue_work(ovpn_wq, &peer->tcp.defer_del_work))
ovpn_peer_put(peer);
ovpn_dev_dstats_rx_dropped(peer->ovpn->dev);
-err_nopeer:
kfree_skb(skb);
}
@@ -168,11 +165,12 @@ static int ovpn_tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
rcu_read_lock();
sock = rcu_dereference_sk_user_data(sk);
- if (unlikely(!sock || !sock->peer || !ovpn_peer_hold(sock->peer))) {
+ if (unlikely(!sock || !sock->peer)) {
rcu_read_unlock();
return -EBADF;
}
peer = sock->peer;
+ ovpn_peer_hold(peer);
rcu_read_unlock();
skb = __skb_recv_datagram(sk, &peer->tcp.user_queue, flags, &off, &err);
@@ -385,10 +383,11 @@ static void ovpn_tcp_release(struct sock *sk)
/* during initialization this function is called before
* assigning sock->peer
*/
- if (unlikely(!peer || !ovpn_peer_hold(peer))) {
+ if (unlikely(!peer)) {
rcu_read_unlock();
return;
}
+ ovpn_peer_hold(peer);
rcu_read_unlock();
__skb_queue_head_init(&queue);
@@ -411,13 +410,14 @@ static int ovpn_tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
lock_sock(sk);
rcu_read_lock();
sock = rcu_dereference_sk_user_data(sk);
- if (unlikely(!sock || !sock->peer || !ovpn_peer_hold(sock->peer))) {
+ if (unlikely(!sock || !sock->peer)) {
rcu_read_unlock();
release_sock(sk);
return -EIO;
}
- rcu_read_unlock();
peer = sock->peer;
+ ovpn_peer_hold(peer);
+ rcu_read_unlock();
if (msg->msg_flags & ~(MSG_DONTWAIT | MSG_NOSIGNAL)) {
ret = -EOPNOTSUPP;
@@ -588,10 +588,12 @@ static void ovpn_tcp_close(struct sock *sk, long timeout)
}
peer = sock->peer;
- if (!peer || !ovpn_peer_hold(peer)) {
+ if (!peer) {
rcu_read_unlock();
return;
}
+ /* socket teardown waits for RCU readers before killing the peer */
+ ovpn_peer_hold(peer);
rcu_read_unlock();
ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_TRANSPORT_DISCONNECT);
@@ -604,7 +606,7 @@ static __poll_t ovpn_tcp_poll(struct file *file, struct socket *sock,
{
struct sk_buff_head *queue = &sock->sk->sk_receive_queue;
struct ovpn_socket *ovpn_sock;
- struct ovpn_peer *peer = NULL;
+ struct ovpn_peer *peer;
__poll_t mask;
rcu_read_lock();
@@ -619,16 +621,13 @@ static __poll_t ovpn_tcp_poll(struct file *file, struct socket *sock,
return 0;
}
- if (ovpn_peer_hold(ovpn_sock->peer)) {
- peer = ovpn_sock->peer;
- queue = &peer->tcp.user_queue;
- }
+ peer = ovpn_sock->peer;
+ ovpn_peer_hold(peer);
+ queue = &peer->tcp.user_queue;
rcu_read_unlock();
mask = datagram_poll_queue(file, sock, wait, queue);
-
- if (peer)
- ovpn_peer_put(peer);
+ ovpn_peer_put(peer);
return mask;
}