SSL: save sessions for upstream peers using a callback function.

In TLSv1.3, NewSessionTicket messages arrive after the handshake and
can come at any time.  Therefore we use a callback to save the session
when we know about it.  This approach works for < TLSv1.3 as well.
The callback function is set once per location on merge phase.

Since SSL_get_session() in BoringSSL returns an unresumable session for
TLSv1.3, peer save_session() methods have been updated as well to use a
session supplied within the callback.  To preserve API, the session is
cached in c->ssl->session.  It is preferably accessed in save_session()
methods by ngx_ssl_get_session() and ngx_ssl_get0_session() wrappers.

diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
index 04b092305cdebbe68e1c4114b496934a5f87ff04..96ee1054306dc5234f4c1b45f112a2f93386cf46 100644 (file)
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -24,6 +24,8 @@ static int ngx_ssl_verify_callback(int ok, X509_STORE_CTX *x509_store);
 static void ngx_ssl_info_callback(const ngx_ssl_conn_t *ssl_conn, int where,
     int ret);
 static void ngx_ssl_passwords_cleanup(void *data);
+static int ngx_ssl_new_client_session(ngx_ssl_conn_t *ssl_conn,
+    ngx_ssl_session_t *sess);
 static void ngx_ssl_handshake_handler(ngx_event_t *ev);
 static ngx_int_t ngx_ssl_handle_recv(ngx_connection_t *c, int n);
 static void ngx_ssl_write_handler(ngx_event_t *wev);
@@ -1161,6 +1163,42 @@ ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *name)
 }
 
 
+ngx_int_t
+ngx_ssl_client_session_cache(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_uint_t enable)
+{
+    if (!enable) {
+        return NGX_OK;
+    }
+
+    SSL_CTX_set_session_cache_mode(ssl->ctx,
+                                   SSL_SESS_CACHE_CLIENT
+                                   |SSL_SESS_CACHE_NO_INTERNAL);
+
+    SSL_CTX_sess_set_new_cb(ssl->ctx, ngx_ssl_new_client_session);
+
+    return NGX_OK;
+}
+
+
+static int
+ngx_ssl_new_client_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess)
+{
+    ngx_connection_t  *c;
+
+    c = ngx_ssl_get_connection(ssl_conn);
+
+    if (c->ssl->save_session) {
+        c->ssl->session = sess;
+
+        c->ssl->save_session(c);
+
+        c->ssl->session = NULL;
+    }
+
+    return 0;
+}
+
+
 ngx_int_t
 ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags)
 {
@@ -1210,6 +1248,31 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags)
 }
 
 
+ngx_ssl_session_t *
+ngx_ssl_get_session(ngx_connection_t *c)
+{
+#ifdef TLS1_3_VERSION
+    if (c->ssl->session) {
+        SSL_SESSION_up_ref(c->ssl->session);
+        return c->ssl->session;
+    }
+#endif
+
+    return SSL_get1_session(c->ssl->connection);
+}
+
+
+ngx_ssl_session_t *
+ngx_ssl_get0_session(ngx_connection_t *c)
+{
+    if (c->ssl->session) {
+        return c->ssl->session;
+    }
+
+    return SSL_get0_session(c->ssl->connection);
+}
+
+
 ngx_int_t
 ngx_ssl_set_session(ngx_connection_t *c, ngx_ssl_session_t *session)
 {

diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h
index 623d851dc4e6162a546ca84bb79168ab8e25f54d..22350d54bee1a313d2e7f10d1c52a211d6e692bd 100644 (file)
--- a/src/event/ngx_event_openssl.h
+++ b/src/event/ngx_event_openssl.h
@@ -77,6 +77,9 @@ struct ngx_ssl_connection_s {
 
     ngx_connection_handler_pt   handler;
 
+    ngx_ssl_session_t          *session;
+    ngx_connection_handler_pt   save_session;
+
     ngx_event_handler_pt        saved_read_handler;
     ngx_event_handler_pt        saved_write_handler;
 
@@ -168,6 +171,8 @@ RSA *ngx_ssl_rsa512_key_callback(ngx_ssl_conn_t *ssl_conn, int is_export,
 ngx_array_t *ngx_ssl_read_password_file(ngx_conf_t *cf, ngx_str_t *file);
 ngx_int_t ngx_ssl_dhparam(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file);
 ngx_int_t ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *name);
+ngx_int_t ngx_ssl_client_session_cache(ngx_conf_t *cf, ngx_ssl_t *ssl,
+    ngx_uint_t enable);
 ngx_int_t ngx_ssl_session_cache(ngx_ssl_t *ssl, ngx_str_t *sess_ctx,
     ssize_t builtin_session_cache, ngx_shm_zone_t *shm_zone, time_t timeout);
 ngx_int_t ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl,
@@ -178,7 +183,8 @@ ngx_int_t ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c,
 
 void ngx_ssl_remove_cached_session(SSL_CTX *ssl, ngx_ssl_session_t *sess);
 ngx_int_t ngx_ssl_set_session(ngx_connection_t *c, ngx_ssl_session_t *session);
-#define ngx_ssl_get_session(c)      SSL_get1_session(c->ssl->connection)
+ngx_ssl_session_t *ngx_ssl_get_session(ngx_connection_t *c);
+ngx_ssl_session_t *ngx_ssl_get0_session(ngx_connection_t *c);
 #define ngx_ssl_free_session        SSL_SESSION_free
 #define ngx_ssl_get_connection(ssl_conn)                                      \
     SSL_get_ex_data(ssl_conn, ngx_ssl_connection_index)

diff --git a/src/http/modules/ngx_http_grpc_module.c b/src/http/modules/ngx_http_grpc_module.c
index 0baa85f283b79973f25c6de6c3b570cc9e86ab4f..3bdf06a6aeb0c43c6bc0f0f4ea2c935944f19959 100644 (file)
--- a/src/http/modules/ngx_http_grpc_module.c
+++ b/src/http/modules/ngx_http_grpc_module.c
@@ -4627,6 +4627,13 @@ ngx_http_grpc_set_ssl(ngx_conf_t *cf, ngx_http_grpc_loc_conf_t *glcf)
         }
     }
 
+    if (ngx_ssl_client_session_cache(cf, glcf->upstream.ssl,
+                                     glcf->upstream.ssl_session_reuse)
+        != NGX_OK)
+    {
+        return NGX_ERROR;
+    }
+
 #ifdef TLSEXT_TYPE_application_layer_protocol_negotiation
 
     if (SSL_CTX_set_alpn_protos(glcf->upstream.ssl->ctx,

diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c
index e7f829d6535d17dbd2800eb7d5aafcf87e80d7e7..d92407999a82ea61a12bf8d55d495c75bafab755 100644 (file)
--- a/src/http/modules/ngx_http_proxy_module.c
+++ b/src/http/modules/ngx_http_proxy_module.c
@@ -4308,6 +4308,13 @@ ngx_http_proxy_set_ssl(ngx_conf_t *cf, ngx_http_proxy_loc_conf_t *plcf)
         }
     }
 
+    if (ngx_ssl_client_session_cache(cf, plcf->upstream.ssl,
+                                     plcf->upstream.ssl_session_reuse)
+        != NGX_OK)
+    {
+        return NGX_ERROR;
+    }
+
     return NGX_OK;
 }
 

diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c
index 238bcf8a3bb2850124293fbe2bda06ace4d5e4a0..07a4c080511830356731ff1352904ffe131af1e6 100644 (file)
--- a/src/http/modules/ngx_http_uwsgi_module.c
+++ b/src/http/modules/ngx_http_uwsgi_module.c
@@ -2391,6 +2391,13 @@ ngx_http_uwsgi_set_ssl(ngx_conf_t *cf, ngx_http_uwsgi_loc_conf_t *uwcf)
         }
     }
 
+    if (ngx_ssl_client_session_cache(cf, uwcf->upstream.ssl,
+                                     uwcf->upstream.ssl_session_reuse)
+        != NGX_OK)
+    {
+        return NGX_ERROR;
+    }
+
     return NGX_OK;
 }
 

diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
index aba1fbb078e69c710d4dee69f10d246d06f3c69a..63fa5d998f8c87555d1fd6446e2600303c97b562 100644 (file)
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -187,6 +187,7 @@ static void ngx_http_upstream_ssl_init_connection(ngx_http_request_t *,
 static void ngx_http_upstream_ssl_handshake_handler(ngx_connection_t *c);
 static void ngx_http_upstream_ssl_handshake(ngx_http_request_t *,
     ngx_http_upstream_t *u, ngx_connection_t *c);
+static void ngx_http_upstream_ssl_save_session(ngx_connection_t *c);
 static ngx_int_t ngx_http_upstream_ssl_name(ngx_http_request_t *r,
     ngx_http_upstream_t *u, ngx_connection_t *c);
 #endif
@@ -1675,6 +1676,8 @@ ngx_http_upstream_ssl_init_connection(ngx_http_request_t *r,
     }
 
     if (u->conf->ssl_session_reuse) {
+        c->ssl->save_session = ngx_http_upstream_ssl_save_session;
+
         if (u->peer.set_session(&u->peer, u->peer.data) != NGX_OK) {
             ngx_http_upstream_finalize_request(r, u,
                                                NGX_HTTP_INTERNAL_SERVER_ERROR);
@@ -1759,10 +1762,6 @@ ngx_http_upstream_ssl_handshake(ngx_http_request_t *r, ngx_http_upstream_t *u,
             }
         }
 
-        if (u->conf->ssl_session_reuse) {
-            u->peer.save_session(&u->peer, u->peer.data);
-        }
-
         c->write->handler = ngx_http_upstream_handler;
         c->read->handler = ngx_http_upstream_handler;
 
@@ -1782,6 +1781,27 @@ failed:
 }
 
 
+static void
+ngx_http_upstream_ssl_save_session(ngx_connection_t *c)
+{
+    ngx_http_request_t   *r;
+    ngx_http_upstream_t  *u;
+
+    if (c->idle) {
+        return;
+    }
+
+    r = c->data;
+
+    u = r->upstream;
+    c = r->connection;
+
+    ngx_http_set_log_request(c->log, r);
+
+    u->peer.save_session(&u->peer, u->peer.data);
+}
+
+
 static ngx_int_t
 ngx_http_upstream_ssl_name(ngx_http_request_t *r, ngx_http_upstream_t *u,
     ngx_connection_t *c)

diff --git a/src/http/ngx_http_upstream_round_robin.c b/src/http/ngx_http_upstream_round_robin.c
index f6051ae57eca9f317113fd27c65cf5808fbac1f3..f72de3ee66ed2386b928a16f98b7e45610cc2fa1 100644 (file)
--- a/src/http/ngx_http_upstream_round_robin.c
+++ b/src/http/ngx_http_upstream_round_robin.c
@@ -744,7 +744,7 @@ ngx_http_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc,
 
     if (peers->shpool) {
 
-        ssl_session = SSL_get0_session(pc->connection->ssl->connection);
+        ssl_session = ngx_ssl_get0_session(pc->connection);
 
         if (ssl_session == NULL) {
             return;

diff --git a/src/stream/ngx_stream_proxy_module.c b/src/stream/ngx_stream_proxy_module.c
index 792bb77585f2f2dbec0fbdb2aad6155d928fe31e..d0497f5714896a2c0feebce0d232d4d6686db9a6 100644 (file)
--- a/src/stream/ngx_stream_proxy_module.c
+++ b/src/stream/ngx_stream_proxy_module.c
@@ -92,6 +92,7 @@ static char *ngx_stream_proxy_ssl_password_file(ngx_conf_t *cf,
     ngx_command_t *cmd, void *conf);
 static void ngx_stream_proxy_ssl_init_connection(ngx_stream_session_t *s);
 static void ngx_stream_proxy_ssl_handshake(ngx_connection_t *pc);
+static void ngx_stream_proxy_ssl_save_session(ngx_connection_t *c);
 static ngx_int_t ngx_stream_proxy_ssl_name(ngx_stream_session_t *s);
 static ngx_int_t ngx_stream_proxy_set_ssl(ngx_conf_t *cf,
     ngx_stream_proxy_srv_conf_t *pscf);
@@ -1008,6 +1009,8 @@ ngx_stream_proxy_ssl_init_connection(ngx_stream_session_t *s)
     }
 
     if (pscf->ssl_session_reuse) {
+        pc->ssl->save_session = ngx_stream_proxy_ssl_save_session;
+
         if (u->peer.set_session(&u->peer, u->peer.data) != NGX_OK) {
             ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR);
             return;
@@ -1066,11 +1069,6 @@ ngx_stream_proxy_ssl_handshake(ngx_connection_t *pc)
             }
         }
 
-        if (pscf->ssl_session_reuse) {
-            u = s->upstream;
-            u->peer.save_session(&u->peer, u->peer.data);
-        }
-
         if (pc->write->timer_set) {
             ngx_del_timer(pc->write);
         }
@@ -1086,6 +1084,19 @@ failed:
 }
 
 
+static void
+ngx_stream_proxy_ssl_save_session(ngx_connection_t *c)
+{
+    ngx_stream_session_t   *s;
+    ngx_stream_upstream_t  *u;
+
+    s = c->data;
+    u = s->upstream;
+
+    u->peer.save_session(&u->peer, u->peer.data);
+}
+
+
 static ngx_int_t
 ngx_stream_proxy_ssl_name(ngx_stream_session_t *s)
 {
@@ -2051,6 +2062,12 @@ ngx_stream_proxy_set_ssl(ngx_conf_t *cf, ngx_stream_proxy_srv_conf_t *pscf)
         }
     }
 
+    if (ngx_ssl_client_session_cache(cf, pscf->ssl, pscf->ssl_session_reuse)
+        != NGX_OK)
+    {
+        return NGX_ERROR;
+    }
+
     return NGX_OK;
 }
 

diff --git a/src/stream/ngx_stream_upstream_round_robin.c b/src/stream/ngx_stream_upstream_round_robin.c
index 526de3a8059acacdf19d03fa1a47d61b2e61798d..36e2ec5cad16bf243b5069c732064e8e0d889939 100644 (file)
--- a/src/stream/ngx_stream_upstream_round_robin.c
+++ b/src/stream/ngx_stream_upstream_round_robin.c
@@ -776,7 +776,7 @@ ngx_stream_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc,
 
     if (peers->shpool) {
 
-        ssl_session = SSL_get0_session(pc->connection->ssl->connection);
+        ssl_session = ngx_ssl_get0_session(pc->connection);
 
         if (ssl_session == NULL) {
             return;