@@ -349,7 +349,7 @@ void ovpn_tcp_send_skb(struct ovpn_peer *peer, struct sock *sk,
*(__be16 *)__skb_push(skb, sizeof(u16)) = htons(len);
- spin_lock_nested(&sk->sk_lock.slock, OVPN_TCP_DEPTH_NESTING);
+ spin_lock_bh_nested(&sk->sk_lock.slock, OVPN_TCP_DEPTH_NESTING);
if (sock_owned_by_user(sk)) {
if (skb_queue_len(&peer->tcp.out_queue) >=
READ_ONCE(net_hotdata.max_backlog)) {
@@ -362,7 +362,7 @@ void ovpn_tcp_send_skb(struct ovpn_peer *peer, struct sock *sk,
ovpn_tcp_send_sock_skb(peer, sk, skb);
}
unlock:
- spin_unlock(&sk->sk_lock.slock);
+ spin_unlock_bh(&sk->sk_lock.slock);
}
static void ovpn_tcp_release(struct sock *sk)
@@ -220,6 +220,8 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
#ifdef CONFIG_DEBUG_LOCK_ALLOC
# define raw_spin_lock_nested(lock, subclass) \
_raw_spin_lock_nested(lock, subclass)
+# define raw_spin_lock_bh_nested(lock, subclass) \
+ _raw_spin_lock_bh_nested(lock, subclass)
# define raw_spin_lock_nest_lock(lock, nest_lock) \
do { \
@@ -234,6 +236,8 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
*/
# define raw_spin_lock_nested(lock, subclass) \
_raw_spin_lock(((void)(subclass), (lock)))
+# define raw_spin_lock_bh_nested(lock, subclass) \
+ _raw_spin_lock_bh(((void)(subclass), (lock)))
# define raw_spin_lock_nest_lock(lock, nest_lock) _raw_spin_lock(lock)
#endif
@@ -360,6 +364,12 @@ do { \
__release(spinlock_check(lock)); __acquire(lock); \
} while (0)
+#define spin_lock_bh_nested(lock, subclass) \
+do { \
+ raw_spin_lock_bh_nested(spinlock_check(lock), subclass); \
+ __release(spinlock_check(lock)); __acquire(lock); \
+} while (0)
+
#define spin_lock_nest_lock(lock, nest_lock) \
do { \
raw_spin_lock_nest_lock(spinlock_check(lock), nest_lock); \
@@ -22,6 +22,8 @@ int in_lock_functions(unsigned long addr);
void __lockfunc _raw_spin_lock(raw_spinlock_t *lock) __acquires(lock);
void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass)
__acquires(lock);
+void __lockfunc _raw_spin_lock_bh_nested(raw_spinlock_t *lock, int subclass)
+ __acquires(lock);
void __lockfunc
_raw_spin_lock_nest_lock(raw_spinlock_t *lock, struct lockdep_map *map)
__acquires(lock);
@@ -63,6 +63,8 @@
#define _raw_spin_lock(lock) __LOCK(lock)
#define _raw_spin_lock_nested(lock, subclass) __LOCK(lock)
+#define _raw_spin_lock_bh_nested(lock, subclass) \
+ __LOCK_BH(lock)
#define _raw_read_lock(lock) __LOCK(lock, shared)
#define _raw_write_lock(lock) __LOCK(lock)
#define _raw_write_lock_nested(lock, subclass) __LOCK(lock)
@@ -90,6 +90,13 @@ static __always_inline void spin_lock_bh(spinlock_t *lock)
rt_spin_lock(lock);
}
+static __always_inline void spin_lock_bh_nested(spinlock_t *lock, int subclass)
+ __acquires(lock)
+{
+ local_bh_disable();
+ __spin_lock_nested(lock, subclass);
+}
+
static __always_inline void spin_lock_irq(spinlock_t *lock)
__acquires(lock)
{
@@ -384,6 +384,14 @@ void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass)
}
EXPORT_SYMBOL(_raw_spin_lock_nested);
+void __lockfunc _raw_spin_lock_bh_nested(raw_spinlock_t *lock, int subclass)
+{
+ __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET);
+ spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
+ LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
+}
+EXPORT_SYMBOL(_raw_spin_lock_bh_nested);
+
unsigned long __lockfunc _raw_spin_lock_irqsave_nested(raw_spinlock_t *lock,
int subclass)
{
ovpn_tcp_send_skb takes sk_lock.slock directly and may run from a process-context crypto completion with BHs enabled. Since the same socket spinlock is also acquired from softirq-side TCP paths via bh_lock_sock, bottom halves must be disabled while holding sk_lock.slock to avoid local softirq self-deadlock. The acquisition is also nested, so the lockdep subclass annotation must be preserved. spin_lock_nested provides the required lockdep subclass annotation, but does not disable bottom halves. spin_lock_bh disables bottom halves, but does not allow specifying a lockdep subclass. Add spin_lock_bh_nested and the corresponding raw helper so callers can combine the existing _bh locking semantics with nested lockdep annotation. Implement the raw helper like the existing _raw_spin_lock_bh, but pass the requested subclass to spin_acquire. Use the new helper in ovpn_tcp_send_skb and pair it with spin_unlock_bh, preventing softirq re-entry while sk_lock.slock is held without losing the nested lockdep annotation. As a side note, include/linux/xarray.h already has an xa_lock_bh_nested wrapper which expands to spin_lock_bh_nested, but no caller currently exercises it. This change provides the missing core helper as a real implementation. Signed-off-by: Ralf Lici <ralf@mandelbit.com> --- drivers/net/ovpn/tcp.c | 4 ++-- include/linux/spinlock.h | 10 ++++++++++ include/linux/spinlock_api_smp.h | 2 ++ include/linux/spinlock_api_up.h | 2 ++ include/linux/spinlock_rt.h | 7 +++++++ kernel/locking/spinlock.c | 8 ++++++++ 6 files changed, 31 insertions(+), 2 deletions(-)