Upstream: re-resolvable servers.

Specifying the upstream server by a hostname together with the
"resolve" parameter will make the hostname to be periodically
resolved, and upstream servers added/removed as necessary.

This requires a "resolver" at the "http" configuration block.

The "resolver_timeout" parameter also affects when the failed
DNS requests will be attempted again.  Responses with NXDOMAIN
will be attempted again in 10 seconds.

Upstream has a configuration generation number that is incremented each
time servers are added/removed to the primary/backup list.  This number
is remembered by the peer.init method, and if peer.get detects a change
in configuration, it returns NGX_BUSY.

Each server has a reference counter.  It is incremented by peer.get and
decremented by peer.free.  When a server is removed, it is removed from
the list of servers and is marked as "zombie".  The memory allocated by
a zombie peer is freed only when its reference count becomes zero.

Co-authored-by: Roman Arutyunyan <arut@nginx.com>
Co-authored-by: Sergey Kandaurov <pluknet@nginx.com>
Co-authored-by: Vladimir Homutov <vl@nginx.com>

1 files changed, 32 insertions, 3 deletions

diff --git a/src/stream/ngx_stream_upstream_random_module.c b/src/stream/ngx_stream_upstream_random_module.c
index 402c1b298..71f76f8d8 100644
--- a/src/stream/ngx_stream_upstream_random_module.c
+++ b/src/stream/ngx_stream_upstream_random_module.c
@@ -17,6 +17,9 @@ typedef struct {
 
 typedef struct {
     ngx_uint_t                              two;
+#if (NGX_STREAM_UPSTREAM_ZONE)
+    ngx_uint_t                              config;
+#endif
     ngx_stream_upstream_random_range_t     *ranges;
 } ngx_stream_upstream_random_srv_conf_t;
 
@@ -125,6 +128,11 @@ ngx_stream_upstream_update_random(ngx_pool_t *pool,
 
     rcf = ngx_stream_conf_upstream_srv_conf(us,
                                             ngx_stream_upstream_random_module);
+    if (rcf->ranges) {
+        ngx_free(rcf->ranges);
+        rcf->ranges = NULL;
+    }
+
     peers = us->peer.data;
 
     size = peers->number * sizeof(ngx_stream_upstream_random_range_t);
@@ -186,11 +194,15 @@ ngx_stream_upstream_init_random_peer(ngx_stream_session_t *s,
     ngx_stream_upstream_rr_peers_rlock(rp->rrp.peers);
 
 #if (NGX_STREAM_UPSTREAM_ZONE)
-    if (rp->rrp.peers->shpool && rcf->ranges == NULL) {
+    if (rp->rrp.peers->config
+        && (rcf->ranges == NULL || rcf->config != *rp->rrp.peers->config))
+    {
         if (ngx_stream_upstream_update_random(NULL, us) != NGX_OK) {
             ngx_stream_upstream_rr_peers_unlock(rp->rrp.peers);
             return NGX_ERROR;
         }
+
+        rcf->config = *rp->rrp.peers->config;
     }
 #endif
 
@@ -220,11 +232,18 @@ ngx_stream_upstream_get_random_peer(ngx_peer_connection_t *pc, void *data)
 
     ngx_stream_upstream_rr_peers_rlock(peers);
 
-    if (rp->tries > 20 || peers->single) {
+    if (rp->tries > 20 || peers->number < 2) {
         ngx_stream_upstream_rr_peers_unlock(peers);
         return ngx_stream_upstream_get_round_robin_peer(pc, rrp);
     }
 
+#if (NGX_STREAM_UPSTREAM_ZONE)
+    if (peers->config && rrp->config != *peers->config) {
+        ngx_stream_upstream_rr_peers_unlock(peers);
+        return ngx_stream_upstream_get_round_robin_peer(pc, rrp);
+    }
+#endif
+
     pc->cached = 0;
     pc->connection = NULL;
 
@@ -274,6 +293,7 @@ ngx_stream_upstream_get_random_peer(ngx_peer_connection_t *pc, void *data)
     }
 
     rrp->current = peer;
+    ngx_stream_upstream_rr_peer_ref(peers, peer);
 
     if (now - peer->checked > peer->fail_timeout) {
         peer->checked = now;
@@ -314,10 +334,17 @@ ngx_stream_upstream_get_random2_peer(ngx_peer_connection_t *pc, void *data)
 
     ngx_stream_upstream_rr_peers_wlock(peers);
 
-    if (rp->tries > 20 || peers->single) {
+    if (rp->tries > 20 || peers->number < 2) {
+        ngx_stream_upstream_rr_peers_unlock(peers);
+        return ngx_stream_upstream_get_round_robin_peer(pc, rrp);
+    }
+
+#if (NGX_STREAM_UPSTREAM_ZONE)
+    if (peers->config && rrp->config != *peers->config) {
         ngx_stream_upstream_rr_peers_unlock(peers);
         return ngx_stream_upstream_get_round_robin_peer(pc, rrp);
     }
+#endif
 
     pc->cached = 0;
     pc->connection = NULL;
@@ -384,6 +411,7 @@ ngx_stream_upstream_get_random2_peer(ngx_peer_connection_t *pc, void *data)
     }
 
     rrp->current = peer;
+    ngx_stream_upstream_rr_peer_ref(peers, peer);
 
     if (now - peer->checked > peer->fail_timeout) {
         peer->checked = now;
@@ -467,6 +495,7 @@ ngx_stream_upstream_random(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
     uscf->peer.init_upstream = ngx_stream_upstream_init_random;
 
     uscf->flags = NGX_STREAM_UPSTREAM_CREATE
+                  |NGX_STREAM_UPSTREAM_MODIFY
                   |NGX_STREAM_UPSTREAM_WEIGHT
                   |NGX_STREAM_UPSTREAM_MAX_CONNS
                   |NGX_STREAM_UPSTREAM_MAX_FAILS