]> git.kaiwu.me - nginx.git/commitdiff
QUIC: using QUIC API introduced in OpenSSL 3.5.
authorSergey Kandaurov <pluknet@nginx.com>
Thu, 13 Feb 2025 13:00:56 +0000 (17:00 +0400)
committerRoman Arutyunyan <arutyunyan.roman@gmail.com>
Fri, 23 May 2025 11:00:47 +0000 (15:00 +0400)
Similarly to the QUIC API originated in BoringSSL, this API allows
to register custom TLS callbacks for an external QUIC implementation.
See the SSL_set_quic_tls_cbs manual page for details.

Due to a different approach used in OpenSSL 3.5, handling of CRYPTO
frames was streamlined to always write an incoming CRYPTO buffer to
the crypto context.  Using SSL_provide_quic_data(), this results in
transient allocation of chain links and buffers for CRYPTO frames
received in order.  Testing didn't reveal performance degradation of
QUIC handshakes, https://github.com/nginx/nginx/pull/646 provides
specific results.

auto/lib/openssl/conf
src/event/quic/ngx_event_quic.h
src/event/quic/ngx_event_quic_connection.h
src/event/quic/ngx_event_quic_ssl.c

index f4b00ebd6722345b8334171fa1ad3669ddd511d3..3068cae36b10245e86ee32d0c32fd43579014f26 100644 (file)
@@ -147,11 +147,17 @@ else
 
             if [ $USE_OPENSSL_QUIC = YES ]; then
 
-                ngx_feature="OpenSSL QUIC support"
+                ngx_feature="OpenSSL QUIC API"
                 ngx_feature_name="NGX_QUIC"
-                ngx_feature_test="SSL_set_quic_method(NULL, NULL)"
+                ngx_feature_test="SSL_set_quic_tls_cbs(NULL, NULL, NULL)"
                 . auto/feature
 
+                if [ $ngx_found = no ]; then
+                    ngx_feature="BoringSSL-like QUIC API"
+                    ngx_feature_test="SSL_set_quic_method(NULL, NULL)"
+                    . auto/feature
+                fi
+
                 if [ $ngx_found = no ]; then
                     ngx_feature="OpenSSL QUIC compatibility"
                     ngx_feature_test="SSL_CTX_add_custom_ext(NULL, 0, 0,
index 50a5c214e1561f77a86a34ab25d86b4522fad275..d95d3d85b19b18bd060497ddd91d17d1102239f6 100644 (file)
 #include <ngx_core.h>
 
 
-#ifdef SSL_R_MISSING_QUIC_TRANSPORT_PARAMETERS_EXTENSION
+#ifdef OSSL_RECORD_PROTECTION_LEVEL_NONE
+#define NGX_QUIC_OPENSSL_API                 1
+
+#elif (defined SSL_R_MISSING_QUIC_TRANSPORT_PARAMETERS_EXTENSION)
 #define NGX_QUIC_QUICTLS_API                 1
 
 #elif (defined OPENSSL_IS_BORINGSSL || defined LIBRESSL_VERSION_NUMBER)
index 856512118f91674f31bef1cc26659361dbc24166..33922cf801e6eed48619846d9a3d8f714d463937 100644 (file)
@@ -301,6 +301,11 @@ struct ngx_quic_connection_s {
     unsigned                          key_phase:1;
     unsigned                          validated:1;
     unsigned                          client_tp_done:1;
+
+#if (NGX_QUIC_OPENSSL_API)
+    unsigned                          read_level:2;
+    unsigned                          write_level:2;
+#endif
 };
 
 
index 6ce926c8187155e3b1da85829ba2308969059982..e961c80cd6048b9823a9a3ce4b12b63a276379a9 100644 (file)
 #define NGX_QUIC_MAX_BUFFERED    65535
 
 
+#if (NGX_QUIC_OPENSSL_API)
+
+static int ngx_quic_cbs_send(ngx_ssl_conn_t *ssl_conn,
+    const unsigned char *data, size_t len, size_t *consumed, void *arg);
+static int ngx_quic_cbs_recv_rcd(ngx_ssl_conn_t *ssl_conn,
+    const unsigned char **data, size_t *bytes_read, void *arg);
+static int ngx_quic_cbs_release_rcd(ngx_ssl_conn_t *ssl_conn,
+    size_t bytes_read, void *arg);
+static int ngx_quic_cbs_yield_secret(ngx_ssl_conn_t *ssl_conn, uint32_t level,
+    int direction, const unsigned char *secret, size_t secret_len, void *arg);
+static int ngx_quic_cbs_got_transport_params(ngx_ssl_conn_t *ssl_conn,
+    const unsigned char *params, size_t params_len, void *arg);
+static int ngx_quic_cbs_alert(ngx_ssl_conn_t *ssl_conn, unsigned char alert,
+    void *arg);
+
+#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */
+
 static ngx_inline ngx_uint_t ngx_quic_map_encryption_level(
     enum ssl_encryption_level_t ssl_level);
 
@@ -39,9 +56,270 @@ static int ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn,
 static int ngx_quic_flush_flight(ngx_ssl_conn_t *ssl_conn);
 static int ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn,
     enum ssl_encryption_level_t ssl_level, uint8_t alert);
+
+#endif
+
 static ngx_int_t ngx_quic_handshake(ngx_connection_t *c);
-static ngx_int_t ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out,
-    ngx_uint_t level);
+static ngx_int_t ngx_quic_crypto_provide(ngx_connection_t *c, ngx_uint_t level);
+
+
+#if (NGX_QUIC_OPENSSL_API)
+
+static int
+ngx_quic_cbs_send(ngx_ssl_conn_t *ssl_conn,
+    const unsigned char *data, size_t len, size_t *consumed, void *arg)
+{
+    ngx_connection_t  *c = arg;
+
+    ngx_chain_t            *out;
+    unsigned int            alpn_len;
+    ngx_quic_frame_t       *frame;
+    const unsigned char    *alpn_data;
+    ngx_quic_send_ctx_t    *ctx;
+    ngx_quic_connection_t  *qc;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_cbs_send len:%uz", len);
+
+    qc = ngx_quic_get_connection(c);
+
+    *consumed = 0;
+
+    SSL_get0_alpn_selected(ssl_conn, &alpn_data, &alpn_len);
+
+    if (alpn_len == 0) {
+        qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_NO_APPLICATION_PROTOCOL);
+        qc->error_reason = "missing ALPN extension";
+
+        ngx_log_error(NGX_LOG_INFO, c->log, 0,
+                      "quic missing ALPN extension");
+        return 1;
+    }
+
+    if (!qc->client_tp_done) {
+        /* RFC 9001, 8.2.  QUIC Transport Parameters Extension */
+        qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_MISSING_EXTENSION);
+        qc->error_reason = "missing transport parameters";
+
+        ngx_log_error(NGX_LOG_INFO, c->log, 0,
+                      "missing transport parameters");
+        return 1;
+    }
+
+    ctx = ngx_quic_get_send_ctx(qc, qc->write_level);
+
+    out = ngx_quic_copy_buffer(c, (u_char *) data, len);
+    if (out == NGX_CHAIN_ERROR) {
+        qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
+        return 1;
+    }
+
+    frame = ngx_quic_alloc_frame(c);
+    if (frame == NULL) {
+        qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
+        return 1;
+    }
+
+    frame->data = out;
+    frame->level = qc->write_level;
+    frame->type = NGX_QUIC_FT_CRYPTO;
+    frame->u.crypto.offset = ctx->crypto_sent;
+    frame->u.crypto.length = len;
+
+    ctx->crypto_sent += len;
+    *consumed = len;
+
+    ngx_quic_queue_frame(qc, frame);
+
+    return 1;
+}
+
+
+static int
+ngx_quic_cbs_recv_rcd(ngx_ssl_conn_t *ssl_conn,
+    const unsigned char **data, size_t *bytes_read, void *arg)
+{
+    ngx_connection_t  *c = arg;
+
+    ngx_buf_t              *b;
+    ngx_chain_t            *cl;
+    ngx_quic_send_ctx_t    *ctx;
+    ngx_quic_connection_t  *qc;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_cbs_recv_rcd");
+
+    qc = ngx_quic_get_connection(c);
+    ctx = ngx_quic_get_send_ctx(qc, qc->read_level);
+
+    for (cl = ctx->crypto.chain; cl; cl = cl->next) {
+        b = cl->buf;
+
+        if (b->sync) {
+            /* hole */
+
+            *bytes_read = 0;
+
+            break;
+        }
+
+        *data = b->pos;
+        *bytes_read = b->last - b->pos;
+
+        break;
+    }
+
+    return 1;
+}
+
+
+static int
+ngx_quic_cbs_release_rcd(ngx_ssl_conn_t *ssl_conn, size_t bytes_read, void *arg)
+{
+    ngx_connection_t  *c = arg;
+
+    ngx_chain_t            *cl;
+    ngx_quic_send_ctx_t    *ctx;
+    ngx_quic_connection_t  *qc;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_cbs_release_rcd len:%uz", bytes_read);
+
+    qc = ngx_quic_get_connection(c);
+    ctx = ngx_quic_get_send_ctx(qc, qc->read_level);
+
+    cl = ngx_quic_read_buffer(c, &ctx->crypto, bytes_read);
+    if (cl == NGX_CHAIN_ERROR) {
+        qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
+        return 1;
+    }
+
+    ngx_quic_free_chain(c, cl);
+
+    return 1;
+}
+
+
+static int
+ngx_quic_cbs_yield_secret(ngx_ssl_conn_t *ssl_conn, uint32_t ssl_level,
+    int direction, const unsigned char *secret, size_t secret_len, void *arg)
+{
+    ngx_connection_t  *c = arg;
+
+    ngx_uint_t              level;
+    const SSL_CIPHER       *cipher;
+    ngx_quic_connection_t  *qc;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_cbs_yield_secret() level:%uD", ssl_level);
+#ifdef NGX_QUIC_DEBUG_CRYPTO
+    ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic %s secret len:%uz %*xs",
+                   direction ? "write" : "read", secret_len,
+                   secret_len, secret);
+#endif
+
+    qc = ngx_quic_get_connection(c);
+    cipher = SSL_get_current_cipher(ssl_conn);
+
+    switch (ssl_level) {
+    case OSSL_RECORD_PROTECTION_LEVEL_NONE:
+        level = NGX_QUIC_ENCRYPTION_INITIAL;
+        break;
+    case OSSL_RECORD_PROTECTION_LEVEL_EARLY:
+        level = NGX_QUIC_ENCRYPTION_EARLY_DATA;
+        break;
+    case OSSL_RECORD_PROTECTION_LEVEL_HANDSHAKE:
+        level = NGX_QUIC_ENCRYPTION_HANDSHAKE;
+        break;
+    default: /* OSSL_RECORD_PROTECTION_LEVEL_APPLICATION */
+        level = NGX_QUIC_ENCRYPTION_APPLICATION;
+        break;
+    }
+
+    if (ngx_quic_keys_set_encryption_secret(c->log, direction, qc->keys, level,
+                                            cipher, secret, secret_len)
+        != NGX_OK)
+    {
+        qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
+        return 1;
+    }
+
+    if (direction) {
+        qc->write_level = level;
+
+    } else {
+        qc->read_level = level;
+    }
+
+    return 1;
+}
+
+
+static int
+ngx_quic_cbs_got_transport_params(ngx_ssl_conn_t *ssl_conn,
+    const unsigned char *params, size_t params_len, void *arg)
+{
+    ngx_connection_t  *c = arg;
+
+    u_char                 *p, *end;
+    ngx_quic_tp_t           ctp;
+    ngx_quic_connection_t  *qc;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_cbs_got_transport_params() len:%uz",
+                   params_len);
+
+    qc = ngx_quic_get_connection(c);
+
+    /* defaults for parameters not sent by client */
+    ngx_memcpy(&ctp, &qc->ctp, sizeof(ngx_quic_tp_t));
+
+    p = (u_char *) params;
+    end = p + params_len;
+
+    if (ngx_quic_parse_transport_params(p, end, &ctp, c->log) != NGX_OK) {
+        qc->error = NGX_QUIC_ERR_TRANSPORT_PARAMETER_ERROR;
+        qc->error_reason = "failed to process transport parameters";
+
+        return 1;
+    }
+
+    if (ngx_quic_apply_transport_params(c, &ctp) != NGX_OK) {
+        return 1;
+    }
+
+    qc->client_tp_done = 1;
+
+    return 1;
+}
+
+
+static int
+ngx_quic_cbs_alert(ngx_ssl_conn_t *ssl_conn, unsigned char alert, void *arg)
+{
+    ngx_connection_t  *c = arg;
+
+    ngx_quic_connection_t  *qc;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_cbs_alert() alert:%d", (int) alert);
+
+    /* already closed on regular shutdown */
+
+    qc = ngx_quic_get_connection(c);
+    if (qc == NULL) {
+        return 1;
+    }
+
+    qc->error = NGX_QUIC_ERR_CRYPTO(alert);
+    qc->error_reason = "handshake failed";
+
+    return 1;
+}
+
+
+#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */
 
 
 static ngx_inline ngx_uint_t
@@ -340,13 +618,14 @@ ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn,
     return 1;
 }
 
+#endif
+
 
 ngx_int_t
 ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt,
     ngx_quic_frame_t *frame)
 {
     uint64_t                  last;
-    ngx_chain_t              *cl;
     ngx_quic_send_ctx_t      *ctx;
     ngx_quic_connection_t    *qc;
     ngx_quic_crypto_frame_t  *f;
@@ -385,41 +664,18 @@ ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt,
         return NGX_OK;
     }
 
-    if (f->offset == ctx->crypto.offset) {
-        if (ngx_quic_crypto_provide(c, frame->data, pkt->level) != NGX_OK) {
-            return NGX_ERROR;
-        }
-
-        if (ngx_quic_handshake(c) != NGX_OK) {
-            return NGX_ERROR;
-        }
-
-        ngx_quic_skip_buffer(c, &ctx->crypto, last);
-
-    } else {
-        if (ngx_quic_write_buffer(c, &ctx->crypto, frame->data, f->length,
-                                  f->offset)
-            == NGX_CHAIN_ERROR)
-        {
-            return NGX_ERROR;
-        }
+    if (ngx_quic_write_buffer(c, &ctx->crypto, frame->data, f->length,
+                              f->offset)
+        == NGX_CHAIN_ERROR)
+    {
+        return NGX_ERROR;
     }
 
-    cl = ngx_quic_read_buffer(c, &ctx->crypto, (uint64_t) -1);
-
-    if (cl) {
-        if (ngx_quic_crypto_provide(c, cl, pkt->level) != NGX_OK) {
-            return NGX_ERROR;
-        }
-
-        if (ngx_quic_handshake(c) != NGX_OK) {
-            return NGX_ERROR;
-        }
-
-        ngx_quic_free_chain(c, cl);
+    if (ngx_quic_crypto_provide(c, pkt->level) != NGX_OK) {
+        return NGX_ERROR;
     }
 
-    return NGX_OK;
+    return ngx_quic_handshake(c);
 }
 
 
@@ -528,13 +784,24 @@ ngx_quic_handshake(ngx_connection_t *c)
 
 
 static ngx_int_t
-ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out,
-    ngx_uint_t level)
+ngx_quic_crypto_provide(ngx_connection_t *c, ngx_uint_t level)
 {
+#if (NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API)
+
     ngx_buf_t                    *b;
-    ngx_chain_t                  *cl;
+    ngx_chain_t                  *out, *cl;
+    ngx_quic_send_ctx_t          *ctx;
+    ngx_quic_connection_t        *qc;
     enum ssl_encryption_level_t   ssl_level;
 
+    qc = ngx_quic_get_connection(c);
+    ctx = ngx_quic_get_send_ctx(qc, level);
+
+    out = ngx_quic_read_buffer(c, &ctx->crypto, (uint64_t) -1);
+    if (out == NGX_CHAIN_ERROR) {
+        return NGX_ERROR;
+    }
+
     switch (level) {
     case NGX_QUIC_ENCRYPTION_INITIAL:
         ssl_level = ssl_encryption_initial;
@@ -562,6 +829,10 @@ ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out,
         }
     }
 
+    ngx_quic_free_chain(c, out);
+
+#endif
+
     return NGX_OK;
 }
 
@@ -569,14 +840,40 @@ ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out,
 ngx_int_t
 ngx_quic_init_connection(ngx_connection_t *c)
 {
-    u_char                  *p;
-    size_t                   clen;
-    ssize_t                  len;
-    ngx_str_t                dcid;
-    ngx_ssl_conn_t          *ssl_conn;
-    ngx_quic_socket_t       *qsock;
-    ngx_quic_connection_t   *qc;
-    static SSL_QUIC_METHOD   quic_method;
+    u_char                 *p;
+    size_t                  clen;
+    ssize_t                 len;
+    ngx_str_t               dcid;
+    ngx_ssl_conn_t         *ssl_conn;
+    ngx_quic_socket_t      *qsock;
+    ngx_quic_connection_t  *qc;
+
+#if (NGX_QUIC_OPENSSL_API)
+    static const OSSL_DISPATCH  qtdis[] = {
+
+        { OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_SEND,
+          (void (*)(void)) ngx_quic_cbs_send },
+
+        { OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_RECV_RCD,
+          (void (*)(void)) ngx_quic_cbs_recv_rcd },
+
+        { OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_RELEASE_RCD,
+          (void (*)(void)) ngx_quic_cbs_release_rcd },
+
+        { OSSL_FUNC_SSL_QUIC_TLS_YIELD_SECRET,
+          (void (*)(void)) ngx_quic_cbs_yield_secret },
+
+        { OSSL_FUNC_SSL_QUIC_TLS_GOT_TRANSPORT_PARAMS,
+          (void (*)(void)) ngx_quic_cbs_got_transport_params },
+
+        { OSSL_FUNC_SSL_QUIC_TLS_ALERT,
+          (void (*)(void)) ngx_quic_cbs_alert },
+
+        { 0, NULL }
+    };
+#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */
+    static SSL_QUIC_METHOD  quic_method;
+#endif
 
     qc = ngx_quic_get_connection(c);
 
@@ -588,6 +885,20 @@ ngx_quic_init_connection(ngx_connection_t *c)
 
     ssl_conn = c->ssl->connection;
 
+#if (NGX_QUIC_OPENSSL_API)
+
+    if (SSL_set_quic_tls_cbs(ssl_conn, qtdis, c) == 0) {
+        ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+                      "quic SSL_set_quic_tls_cbs() failed");
+        return NGX_ERROR;
+    }
+
+    if (SSL_CTX_get_max_early_data(qc->conf->ssl->ctx)) {
+        SSL_set_quic_tls_early_data_enabled(ssl_conn, 1);
+    }
+
+#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */
+
     if (!quic_method.send_alert) {
 #if (NGX_QUIC_BORINGSSL_API)
         quic_method.set_read_secret = ngx_quic_set_read_secret;
@@ -610,6 +921,8 @@ ngx_quic_init_connection(ngx_connection_t *c)
     if (SSL_CTX_get_max_early_data(qc->conf->ssl->ctx)) {
         SSL_set_quic_early_data_enabled(ssl_conn, 1);
     }
+#endif
+
 #endif
 
     qsock = ngx_quic_get_socket(c);
@@ -641,11 +954,19 @@ ngx_quic_init_connection(ngx_connection_t *c)
                    "quic transport parameters len:%uz %*xs", len, len, p);
 #endif
 
+#if (NGX_QUIC_OPENSSL_API)
+    if (SSL_set_quic_tls_transport_params(ssl_conn, p, len) == 0) {
+        ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+                      "quic SSL_set_quic_tls_transport_params() failed");
+        return NGX_ERROR;
+    }
+#else
     if (SSL_set_quic_transport_params(ssl_conn, p, len) == 0) {
         ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
                       "quic SSL_set_quic_transport_params() failed");
         return NGX_ERROR;
     }
+#endif
 
 #ifdef OPENSSL_IS_BORINGSSL
     if (SSL_set_quic_early_data_context(ssl_conn, p, clen) == 0) {