From: Gianmarco De Gregori <gianmarco@mandelbit.com>
When accept() fails with EMFILE, Linux and FreeBSD leave the pending
connection in the listen queue and the level-triggered listener is
reported again on every wait. The server keeps creating and destroying
an instance for it, never gets an empty event set, so the timers do not
run and hand-window never reaps the idle connections that hold the
descriptors: the server stays at 100% CPU, writes the failure to the log
in a loop and stops serving new TCP and UDP clients until the peer
closes its connections. A management client connecting while the
descriptors are exhausted causes the same loop on the management
listener.
When accept() fails for lack of resources, stop polling the TCP
listeners and, in server mode, the management listener for up to a
second. Clients may see failed connections meanwhile, so log a warning,
at most once a minute.
GitHub: OpenVPN/openvpn#1148
Change-Id: I978041ea4e6323f5a92d8721809f0fa44b9c2e03
Signed-off-by: Gianmarco De Gregori <gianmarco@mandelbit.com>
Acked-by: Razvan Cojocaru <razvanc@mailbox.org>
Gerrit URL: https://gerrit.openvpn.net/c/openvpn/+/2210
---
This change was reviewed on Gerrit and approved by at least one
developer. I request to merge it to master.
Gerrit URL: https://gerrit.openvpn.net/c/openvpn/+/2210
This mail reflects revision 3 of this Change.
Acked-by according to Gerrit (reflected above):
Razvan Cojocaru <razvanc@mailbox.org>
@@ -3292,6 +3292,18 @@
msg(M_CLIENT, ">PASSWORD:Auth-Token:%s", token);
}
+/**
+ * Track which event the management socket is registered for in a persistent
+ * event set, so that event_ctl() is only called when it changes.
+ *
+ * @param persistent The cached state, or NULL if the event set is rebuilt
+ * before every wait.
+ * @param n The new state: 1 listening, 2 waiting to read from the
+ * client, 3 waiting to write to the client, 4 listening
+ * while accept() is paused for lack of resources.
+ *
+ * @return true if the socket must be registered again with event_ctl().
+ */
static inline bool
man_persist_state(unsigned int *persistent, const int n)
{
@@ -3320,11 +3332,14 @@
switch (man->connection.state)
{
case MS_LISTEN:
- if (man_persist_state(persistent, 1))
+ {
+ const bool paused = (man->settings.flags & MF_SERVER) && socket_accept_paused();
+ if (man_persist_state(persistent, paused ? 4 : 1))
{
- event_ctl(es, ev, EVENT_READ, arg);
+ event_ctl(es, ev, paused ? 0 : EVENT_READ, arg);
}
break;
+ }
case MS_CC_WAIT_READ:
if (man_persist_state(persistent, 2))
@@ -3402,11 +3417,14 @@
switch (man->connection.state)
{
case MS_LISTEN:
- if (man_persist_state(persistent, 1))
+ {
+ const bool paused = (man->settings.flags & MF_SERVER) && socket_accept_paused();
+ if (man_persist_state(persistent, paused ? 4 : 1))
{
- event_ctl(es, man->connection.sd_top, EVENT_READ, arg);
+ event_ctl(es, man->connection.sd_top, paused ? 0 : EVENT_READ, arg);
}
break;
+ }
case MS_CC_WAIT_READ:
if (man_persist_state(persistent, 2))
@@ -187,10 +187,25 @@
}
else if (!tuntap_is_dco_win(m->top.c1.tuntap))
{
- socket_set_listen_persistent(sock, m->multi_io->es, &sock->ev_arg);
+ if (socket_accept_paused())
+ {
+ socket_unset_listen_persistent(sock, m->multi_io->es);
+ }
+ else
+ {
+ socket_set_listen_persistent(sock, m->multi_io->es, &sock->ev_arg);
+ }
}
}
+ const time_t delay = socket_accept_paused();
+ if (delay && m->top.c2.timeval.tv_sec >= delay)
+ {
+ m->top.c2.timeval.tv_sec = (tv_sec_t)delay;
+ m->top.c2.timeval.tv_usec = 0;
+ m->earliest_wakeup = NULL;
+ }
+
tun_set(m->top.c1.tuntap, m->multi_io->es, EVENT_READ, MULTI_IO_TUN, persistent);
#if defined(ENABLE_DCO)
dco_event_set(&m->top.c1.tuntap->dco, m->multi_io->es, MULTI_IO_DCO);
@@ -422,6 +422,38 @@
gc_free(&gc);
}
+/* accept() failed for lack of resources: no new accept() before this time */
+static time_t accept_resume = 0; /* GLOBAL */
+static time_t accept_pause_logged = 0; /* GLOBAL */
+
+static void
+accept_check_resources(const int err)
+{
+#ifdef _WIN32
+ const bool out_of_resources = (err == WSAEMFILE || err == WSAENOBUFS);
+#else
+ const bool out_of_resources =
+ (err == EMFILE || err == ENFILE || err == ENOBUFS || err == ENOMEM);
+#endif
+ if (out_of_resources)
+ {
+ accept_resume = now + 1;
+ if (now >= accept_pause_logged + 60)
+ {
+ msg(M_WARN, "WARNING: out of resources to accept new connections, not accepting any "
+ "for a second. New connections may fail meanwhile; check the open files "
+ "limit. This message is shown at most once a minute.");
+ accept_pause_logged = now;
+ }
+ }
+}
+
+time_t
+socket_accept_paused(void)
+{
+ return now < accept_resume ? accept_resume - now : 0;
+}
+
#if defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-conversion"
@@ -456,6 +488,10 @@
else
{
new_sd = accept(sd, &act->dest.addr.sa, &remote_len);
+ if (!socket_defined(new_sd))
+ {
+ accept_check_resources(openvpn_errno());
+ }
}
#if 0 /* For debugging only, test the effect of accept() failures */
@@ -2700,6 +2736,10 @@
* scripts don't have access to it */
set_cloexec(ret);
}
+ else
+ {
+ accept_check_resources(openvpn_errno());
+ }
return ret;
}
@@ -410,6 +410,9 @@
const char *bind_dev, struct addrinfo *bind_addr,
bool bind_ipv6_only, bool optional);
+/* Seconds left before accept() is retried after it failed for lack of resources */
+time_t socket_accept_paused(void);
+
socket_descriptor_t socket_do_accept(socket_descriptor_t sd, struct link_socket_actual *act,
const bool nowait);
@@ -846,6 +849,16 @@
}
static inline void
+socket_unset_listen_persistent(struct link_socket *sock, struct event_set *es)
+{
+ if (sock && sock->listen_persistent_queued)
+ {
+ event_del(es, socket_listen_event_handle(sock));
+ sock->listen_persistent_queued = false;
+ }
+}
+
+static inline void
socket_reset_listen_persistent(struct link_socket *sock)
{
#ifdef _WIN32