Events: support for EPOLLEXCLUSIVE.

This flag appeared in Linux 4.5 and is useful for avoiding thundering herd
problem.

The current Linux kernel implementation walks the list of exclusive waiters,
and queues an event to each epfd, until it finds the first waiter that has
threads blocked on it via epoll_wait().

diff --git a/auto/modules b/auto/modules
index 829079842c289d8177f95b4650dc5678c79118be..614037cb1fe063db2c65c7299999b66bcaf401bd 100644 (file)
--- a/auto/modules
+++ b/auto/modules
@@ -43,6 +43,7 @@ fi
 if [ $NGX_TEST_BUILD_EPOLL = YES ]; then
     have=NGX_HAVE_EPOLL . auto/have
     have=NGX_HAVE_EPOLLRDHUP . auto/have
+    have=NGX_HAVE_EPOLLEXCLUSIVE . auto/have
     have=NGX_HAVE_EVENTFD . auto/have
     have=NGX_TEST_BUILD_EPOLL . auto/have
     EVENT_MODULES="$EVENT_MODULES $EPOLL_MODULE"

diff --git a/auto/os/linux b/auto/os/linux
index 132ce3b427c10d34d2fd94a7fb56046737f86c36..fae8842c645476edf438bda95562b22f1a81056e 100644 (file)
--- a/auto/os/linux
+++ b/auto/os/linux
@@ -70,6 +70,22 @@ if [ $ngx_found = yes ]; then
                       ee.data.ptr = NULL;
                       epoll_ctl(efd, EPOLL_CTL_ADD, fd, &ee)"
     . auto/feature
+
+
+    # EPOLLEXCLUSIVE appeared in Linux 4.5, glibc 2.24
+
+    ngx_feature="EPOLLEXCLUSIVE"
+    ngx_feature_name="NGX_HAVE_EPOLLEXCLUSIVE"
+    ngx_feature_run=no
+    ngx_feature_incs="#include <sys/epoll.h>"
+    ngx_feature_path=
+    ngx_feature_libs=
+    ngx_feature_test="int efd = 0, fd = 0;
+                      struct epoll_event ee;
+                      ee.events = EPOLLIN|EPOLLEXCLUSIVE;
+                      ee.data.ptr = NULL;
+                      epoll_ctl(efd, EPOLL_CTL_ADD, fd, &ee)"
+    . auto/feature
 fi
 
 

diff --git a/src/event/modules/ngx_epoll_module.c b/src/event/modules/ngx_epoll_module.c
index dff778831615e6c92d3b6bb62e87e2c2d5cefdf5..c267fd6c7a83fd660ff3df0228ca77d8d5d2b191 100644 (file)
--- a/src/event/modules/ngx_epoll_module.c
+++ b/src/event/modules/ngx_epoll_module.c
@@ -27,6 +27,7 @@
 
 #define EPOLLRDHUP     0x2000
 
+#define EPOLLEXCLUSIVE 0x10000000
 #define EPOLLONESHOT   0x40000000
 #define EPOLLET        0x80000000
 
@@ -610,6 +611,12 @@ ngx_epoll_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
         op = EPOLL_CTL_ADD;
     }
 
+#if (NGX_HAVE_EPOLLEXCLUSIVE && NGX_HAVE_EPOLLRDHUP)
+    if (flags & NGX_EXCLUSIVE_EVENT) {
+        events &= ~EPOLLRDHUP;
+    }
+#endif
+
     ee.events = events | (uint32_t) flags;
     ee.data.ptr = (void *) ((uintptr_t) c | ev->instance);
 

diff --git a/src/event/ngx_event.c b/src/event/ngx_event.c
index 2f3a098dec25814efda6cac5735f4aeb586b9f0d..9d6c4c91b3a2263458195da9c5368dba04219c6a 100644 (file)
--- a/src/event/ngx_event.c
+++ b/src/event/ngx_event.c
@@ -822,15 +822,38 @@ ngx_event_process_init(ngx_cycle_t *cycle)
         rev->handler = (c->type == SOCK_STREAM) ? ngx_event_accept
                                                 : ngx_event_recvmsg;
 
-        if (ngx_use_accept_mutex
 #if (NGX_HAVE_REUSEPORT)
-            && !ls[i].reuseport
+
+        if (ls[i].reuseport) {
+            if (ngx_add_event(rev, NGX_READ_EVENT, 0) == NGX_ERROR) {
+                return NGX_ERROR;
+            }
+
+            continue;
+        }
+
 #endif
-           )
+
+        if (ngx_use_accept_mutex) {
+            continue;
+        }
+
+#if (NGX_HAVE_EPOLLEXCLUSIVE)
+
+        if ((ngx_event_flags & NGX_USE_EPOLL_EVENT)
+            && ccf->worker_processes > 1)
         {
+            if (ngx_add_event(rev, NGX_READ_EVENT, NGX_EXCLUSIVE_EVENT)
+                == NGX_ERROR)
+            {
+                return NGX_ERROR;
+            }
+
             continue;
         }
 
+#endif
+
         if (ngx_add_event(rev, NGX_READ_EVENT, 0) == NGX_ERROR) {
             return NGX_ERROR;
         }

diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h
index 394a0e8e57e76abe78c1126b17c133fd384c309c..27139ee8c8fdaceef4487ac4e56e51b8ab041900 100644 (file)
--- a/src/event/ngx_event.h
+++ b/src/event/ngx_event.h
@@ -367,6 +367,9 @@ extern ngx_uint_t            ngx_use_epoll_rdhup;
 #define NGX_ONESHOT_EVENT  EPOLLONESHOT
 #endif
 
+#if (NGX_HAVE_EPOLLEXCLUSIVE)
+#define NGX_EXCLUSIVE_EVENT  EPOLLEXCLUSIVE
+#endif
 
 #elif (NGX_HAVE_POLL)
 
@@ -395,6 +398,11 @@ extern ngx_uint_t            ngx_use_epoll_rdhup;
 #endif
 
 
+#if (NGX_TEST_BUILD_EPOLL)
+#define NGX_EXCLUSIVE_EVENT  0
+#endif
+
+
 #ifndef NGX_CLEAR_EVENT
 #define NGX_CLEAR_EVENT    0    /* dummy declaration */
 #endif