Core: idle connections now closed only once on exiting.

Iterating through all connections takes a lot of CPU time, especially
with large number of worker connections configured.  As a result
nginx processes used to consume CPU time during graceful shutdown.
To mitigate this we now only do a full scan for idle connections when
shutdown signal is received.

Transitions of connections to idle ones are now expected to be
avoided if the ngx_exiting flag is set.  The upstream keepalive module
was modified to follow this.

5 files changed, 31 insertions, 35 deletions

diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
index 00c5a08e6..9f2675b88 100644
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -1163,6 +1163,26 @@ ngx_drain_connections(void)
 }
 
 
+void
+ngx_close_idle_connections(ngx_cycle_t *cycle)
+{
+    ngx_uint_t         i;
+    ngx_connection_t  *c;
+
+    c = cycle->connections;
+
+    for (i = 0; i < cycle->connection_n; i++) {
+
+        /* THREAD: lock */
+
+        if (c[i].fd != -1 && c[i].idle) {
+            c[i].close = 1;
+            c[i].read->handler(c[i].read);
+        }
+    }
+}
+
+
 ngx_int_t
 ngx_connection_local_sockaddr(ngx_connection_t *c, ngx_str_t *s,
     ngx_uint_t port)
diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h
index a49aa9579..c3aca7f47 100644
--- a/src/core/ngx_connection.h
+++ b/src/core/ngx_connection.h
@@ -215,6 +215,7 @@ ngx_int_t ngx_open_listening_sockets(ngx_cycle_t *cycle);
 void ngx_configure_listening_sockets(ngx_cycle_t *cycle);
 void ngx_close_listening_sockets(ngx_cycle_t *cycle);
 void ngx_close_connection(ngx_connection_t *c);
+void ngx_close_idle_connections(ngx_cycle_t *cycle);
 ngx_int_t ngx_connection_local_sockaddr(ngx_connection_t *c, ngx_str_t *s,
     ngx_uint_t port);
 ngx_int_t ngx_connection_error(ngx_connection_t *c, ngx_err_t err, char *text);
diff --git a/src/http/modules/ngx_http_upstream_keepalive_module.c b/src/http/modules/ngx_http_upstream_keepalive_module.c
index 768881ee4..51887b4e8 100644
--- a/src/http/modules/ngx_http_upstream_keepalive_module.c
+++ b/src/http/modules/ngx_http_upstream_keepalive_module.c
@@ -302,6 +302,10 @@ ngx_http_upstream_free_keepalive_peer(ngx_peer_connection_t *pc, void *data,
         goto invalid;
     }
 
+    if (ngx_terminate || ngx_exiting) {
+        goto invalid;
+    }
+
     if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
         goto invalid;
     }
diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c
index c69932ecd..5da09116f 100644
--- a/src/os/unix/ngx_process_cycle.c
+++ b/src/os/unix/ngx_process_cycle.c
@@ -728,9 +728,6 @@ ngx_worker_process_cycle(ngx_cycle_t *cycle, void *data)
 {
     ngx_int_t worker = (intptr_t) data;
 
-    ngx_uint_t         i;
-    ngx_connection_t  *c;
-
     ngx_process = NGX_PROCESS_WORKER;
     ngx_worker = worker;
 
@@ -741,19 +738,6 @@ ngx_worker_process_cycle(ngx_cycle_t *cycle, void *data)
     for ( ;; ) {
 
         if (ngx_exiting) {
-
-            c = cycle->connections;
-
-            for (i = 0; i < cycle->connection_n; i++) {
-
-                /* THREAD: lock */
-
-                if (c[i].fd != -1 && c[i].idle) {
-                    c[i].close = 1;
-                    c[i].read->handler(c[i].read);
-                }
-            }
-
             ngx_event_cancel_timers();
 
             if (ngx_event_timer_rbtree.root == ngx_event_timer_rbtree.sentinel)
@@ -781,8 +765,9 @@ ngx_worker_process_cycle(ngx_cycle_t *cycle, void *data)
             ngx_setproctitle("worker process is shutting down");
 
             if (!ngx_exiting) {
-                ngx_close_listening_sockets(cycle);
                 ngx_exiting = 1;
+                ngx_close_listening_sockets(cycle);
+                ngx_close_idle_connections(cycle);
             }
         }
 
diff --git a/src/os/win32/ngx_process_cycle.c b/src/os/win32/ngx_process_cycle.c
index 203bece3f..06ff8b50e 100644
--- a/src/os/win32/ngx_process_cycle.c
+++ b/src/os/win32/ngx_process_cycle.c
@@ -761,10 +761,8 @@ failed:
 static ngx_thread_value_t __stdcall
 ngx_worker_thread(void *data)
 {
-    ngx_int_t          n;
-    ngx_uint_t         i;
-    ngx_cycle_t       *cycle;
-    ngx_connection_t  *c;
+    ngx_int_t     n;
+    ngx_cycle_t  *cycle;
 
     cycle = (ngx_cycle_t *) ngx_cycle;
 
@@ -780,19 +778,6 @@ ngx_worker_thread(void *data)
     while (!ngx_quit) {
 
         if (ngx_exiting) {
-
-            c = cycle->connections;
-
-            for (i = 0; i < cycle->connection_n; i++) {
-
-                /* THREAD: lock */
-
-                if (c[i].fd != (ngx_socket_t) -1 && c[i].idle) {
-                    c[i].close = 1;
-                    c[i].read->handler(c[i].read);
-                }
-            }
-
             ngx_event_cancel_timers();
 
             if (ngx_event_timer_rbtree.root
@@ -814,8 +799,9 @@ ngx_worker_thread(void *data)
             ngx_quit = 0;
 
             if (!ngx_exiting) {
-                ngx_close_listening_sockets(cycle);
                 ngx_exiting = 1;
+                ngx_close_listening_sockets(cycle);
+                ngx_close_idle_connections(cycle);
             }
         }