Upstream: re-resolvable servers.

Specifying the upstream server by a hostname together with the
"resolve" parameter will make the hostname to be periodically
resolved, and upstream servers added/removed as necessary.

This requires a "resolver" at the "http" configuration block.

The "resolver_timeout" parameter also affects when the failed
DNS requests will be attempted again.  Responses with NXDOMAIN
will be attempted again in 10 seconds.

Upstream has a configuration generation number that is incremented each
time servers are added/removed to the primary/backup list.  This number
is remembered by the peer.init method, and if peer.get detects a change
in configuration, it returns NGX_BUSY.

Each server has a reference counter.  It is incremented by peer.get and
decremented by peer.free.  When a server is removed, it is removed from
the list of servers and is marked as "zombie".  The memory allocated by
a zombie peer is freed only when its reference count becomes zero.

Co-authored-by: Roman Arutyunyan <arut@nginx.com>
Co-authored-by: Sergey Kandaurov <pluknet@nginx.com>
Co-authored-by: Vladimir Homutov <vl@nginx.com>

1 files changed, 80 insertions, 10 deletions

diff --git a/src/stream/ngx_stream_upstream_hash_module.c b/src/stream/ngx_stream_upstream_hash_module.c
index b764fcbe0..20fca6c0d 100644
--- a/src/stream/ngx_stream_upstream_hash_module.c
+++ b/src/stream/ngx_stream_upstream_hash_module.c
@@ -23,6 +23,9 @@ typedef struct {
 
 
 typedef struct {
+#if (NGX_STREAM_UPSTREAM_ZONE)
+    ngx_uint_t                            config;
+#endif
     ngx_stream_complex_value_t            key;
     ngx_stream_upstream_chash_points_t   *points;
 } ngx_stream_upstream_hash_srv_conf_t;
@@ -49,6 +52,8 @@ static ngx_int_t ngx_stream_upstream_get_hash_peer(ngx_peer_connection_t *pc,
 
 static ngx_int_t ngx_stream_upstream_init_chash(ngx_conf_t *cf,
     ngx_stream_upstream_srv_conf_t *us);
+static ngx_int_t ngx_stream_upstream_update_chash(ngx_pool_t *pool,
+    ngx_stream_upstream_srv_conf_t *us);
 static int ngx_libc_cdecl
     ngx_stream_upstream_chash_cmp_points(const void *one, const void *two);
 static ngx_uint_t ngx_stream_upstream_find_chash_point(
@@ -178,10 +183,17 @@ ngx_stream_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data)
 
     ngx_stream_upstream_rr_peers_rlock(hp->rrp.peers);
 
-    if (hp->tries > 20 || hp->rrp.peers->single || hp->key.len == 0) {
+    if (hp->tries > 20 || hp->rrp.peers->number < 2 || hp->key.len == 0) {
+        ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers);
+        return hp->get_rr_peer(pc, &hp->rrp);
+    }
+
+#if (NGX_STREAM_UPSTREAM_ZONE)
+    if (hp->rrp.peers->config && hp->rrp.config != *hp->rrp.peers->config) {
         ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers);
         return hp->get_rr_peer(pc, &hp->rrp);
     }
+#endif
 
     now = ngx_time();
 
@@ -261,6 +273,7 @@ ngx_stream_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data)
     }
 
     hp->rrp.current = peer;
+    ngx_stream_upstream_rr_peer_ref(hp->rrp.peers, peer);
 
     pc->sockaddr = peer->sockaddr;
     pc->socklen = peer->socklen;
@@ -285,6 +298,26 @@ static ngx_int_t
 ngx_stream_upstream_init_chash(ngx_conf_t *cf,
     ngx_stream_upstream_srv_conf_t *us)
 {
+    if (ngx_stream_upstream_init_round_robin(cf, us) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    us->peer.init = ngx_stream_upstream_init_chash_peer;
+
+#if (NGX_STREAM_UPSTREAM_ZONE)
+    if (us->shm_zone) {
+        return NGX_OK;
+    }
+#endif
+
+    return ngx_stream_upstream_update_chash(cf->pool, us);
+}
+
+
+static ngx_int_t
+ngx_stream_upstream_update_chash(ngx_pool_t *pool,
+    ngx_stream_upstream_srv_conf_t *us)
+{
     u_char                               *host, *port, c;
     size_t                                host_len, port_len, size;
     uint32_t                              hash, base_hash;
@@ -299,25 +332,33 @@ ngx_stream_upstream_init_chash(ngx_conf_t *cf,
         u_char                            byte[4];
     } prev_hash;
 
-    if (ngx_stream_upstream_init_round_robin(cf, us) != NGX_OK) {
-        return NGX_ERROR;
-    }
+    hcf = ngx_stream_conf_upstream_srv_conf(us,
+                                            ngx_stream_upstream_hash_module);
 
-    us->peer.init = ngx_stream_upstream_init_chash_peer;
+    if (hcf->points) {
+        ngx_free(hcf->points);
+        hcf->points = NULL;
+    }
 
     peers = us->peer.data;
     npoints = peers->total_weight * 160;
 
     size = sizeof(ngx_stream_upstream_chash_points_t)
-           + sizeof(ngx_stream_upstream_chash_point_t) * (npoints - 1);
+           - sizeof(ngx_stream_upstream_chash_point_t)
+           + sizeof(ngx_stream_upstream_chash_point_t) * npoints;
 
-    points = ngx_palloc(cf->pool, size);
+    points = pool ? ngx_palloc(pool, size) : ngx_alloc(size, ngx_cycle->log);
     if (points == NULL) {
         return NGX_ERROR;
     }
 
     points->number = 0;
 
+    if (npoints == 0) {
+        hcf->points = points;
+        return NGX_OK;
+    }
+
     for (peer = peers->peer; peer; peer = peer->next) {
         server = &peer->server;
 
@@ -401,8 +442,6 @@ ngx_stream_upstream_init_chash(ngx_conf_t *cf,
 
     points->number = i + 1;
 
-    hcf = ngx_stream_conf_upstream_srv_conf(us,
-                                            ngx_stream_upstream_hash_module);
     hcf->points = points;
 
     return NGX_OK;
@@ -483,7 +522,22 @@ ngx_stream_upstream_init_chash_peer(ngx_stream_session_t *s,
 
     ngx_stream_upstream_rr_peers_rlock(hp->rrp.peers);
 
-    hp->hash = ngx_stream_upstream_find_chash_point(hcf->points, hash);
+#if (NGX_STREAM_UPSTREAM_ZONE)
+    if (hp->rrp.peers->config
+        && (hcf->points == NULL || hcf->config != *hp->rrp.peers->config))
+    {
+        if (ngx_stream_upstream_update_chash(NULL, us) != NGX_OK) {
+            ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers);
+            return NGX_ERROR;
+        }
+
+        hcf->config = *hp->rrp.peers->config;
+    }
+#endif
+
+    if (hcf->points->number) {
+        hp->hash = ngx_stream_upstream_find_chash_point(hcf->points, hash);
+    }
 
     ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers);
 
@@ -518,6 +572,20 @@ ngx_stream_upstream_get_chash_peer(ngx_peer_connection_t *pc, void *data)
 
     pc->connection = NULL;
 
+    if (hp->rrp.peers->number == 0) {
+        pc->name = hp->rrp.peers->name;
+        ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers);
+        return NGX_BUSY;
+    }
+
+#if (NGX_STREAM_UPSTREAM_ZONE)
+    if (hp->rrp.peers->config && hp->rrp.config != *hp->rrp.peers->config) {
+        pc->name = hp->rrp.peers->name;
+        ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers);
+        return NGX_BUSY;
+    }
+#endif
+
     now = ngx_time();
     hcf = hp->conf;
 
@@ -596,6 +664,7 @@ ngx_stream_upstream_get_chash_peer(ngx_peer_connection_t *pc, void *data)
     }
 
     hp->rrp.current = best;
+    ngx_stream_upstream_rr_peer_ref(hp->rrp.peers, best);
 
     pc->sockaddr = best->sockaddr;
     pc->socklen = best->socklen;
@@ -663,6 +732,7 @@ ngx_stream_upstream_hash(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
     }
 
     uscf->flags = NGX_STREAM_UPSTREAM_CREATE
+                  |NGX_STREAM_UPSTREAM_MODIFY
                   |NGX_STREAM_UPSTREAM_WEIGHT
                   |NGX_STREAM_UPSTREAM_MAX_CONNS
                   |NGX_STREAM_UPSTREAM_MAX_FAILS