SSL: available bytes handling (ticket #1431).

Added code to track number of bytes available in the socket.
This makes it possible to avoid looping for a long time while
working with fast enough peer when data are added to the socket buffer
faster than we are able to read and process data.

When kernel does not provide number of bytes available, it is
retrieved using ioctl(FIONREAD) as long as a buffer is filled by
SSL_read().

It is assumed that number of bytes returned by SSL_read() is close
to the number of bytes read from the socket, as we do not use
SSL compression.  But even if it is not true for some reason, this
is not important, as we post an additional reading event anyway.

Note that data can be buffered at SSL layer, and it is not possible
to simply stop reading at some point and wait till the event will
be reported by the kernel again.  This can be only done when there
are no data in SSL buffers, and there is no good way to find out if
it's the case.

Instead of trying to figure out if SSL buffers are empty, this patch
introduces events posted for the next event loop iteration - such
events will be processed only on the next event loop iteration,
after going into the kernel and retrieving additional events.  This
seems to be simple and reliable approach.

diff --git a/src/event/ngx_event.c b/src/event/ngx_event.c
index 69c55d7a0602670cf94bf2f6d9ea5ee75ca9a599..6e19f311b61c02eb8e344a8cdb30ce9339d79e90 100644 (file)
--- a/src/event/ngx_event.c
+++ b/src/event/ngx_event.c
@@ -237,6 +237,12 @@ ngx_process_events_and_timers(ngx_cycle_t *cycle)
         }
     }
 
+    if (!ngx_queue_empty(&ngx_posted_next_events)) {
+        ngx_queue_add(&ngx_posted_events, &ngx_posted_next_events);
+        ngx_queue_init(&ngx_posted_next_events);
+        timer = 0;
+    }
+
     delta = ngx_current_msec;
 
     (void) ngx_process_events(cycle, timer, flags);
@@ -639,6 +645,7 @@ ngx_event_process_init(ngx_cycle_t *cycle)
 #endif
 
     ngx_queue_init(&ngx_posted_accept_events);
+    ngx_queue_init(&ngx_posted_next_events);
     ngx_queue_init(&ngx_posted_events);
 
     if (ngx_event_timer_init(cycle->log) == NGX_ERROR) {

diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
index 4e3eb391c8b4d064ba4741e8df221b40520b7e45..e9431b2d6a19148fe3ef4496662f387c234d4598 100644 (file)
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -43,6 +43,7 @@ static ssize_t ngx_ssl_recv_early(ngx_connection_t *c, u_char *buf,
 #endif
 static ngx_int_t ngx_ssl_handle_recv(ngx_connection_t *c, int n);
 static void ngx_ssl_write_handler(ngx_event_t *wev);
+static void ngx_ssl_next_read_handler(ngx_event_t *rev);
 #ifdef SSL_READ_EARLY_DATA_SUCCESS
 static ssize_t ngx_ssl_write_early(ngx_connection_t *c, u_char *data,
     size_t size);
@@ -2003,6 +2004,48 @@ ngx_ssl_recv(ngx_connection_t *c, u_char *buf, size_t size)
 
             if (size == 0) {
                 c->read->ready = 1;
+
+                if (c->read->available >= 0) {
+                    c->read->available -= bytes;
+
+                    /*
+                     * there can be data buffered at SSL layer,
+                     * so we post an event to continue reading on the next
+                     * iteration of the event loop
+                     */
+
+                    if (c->read->available < 0) {
+                        c->read->available = 0;
+                        c->read->ready = 0;
+
+                        if (c->ssl->next_read_handler == NULL) {
+                            c->ssl->next_read_handler = c->read->handler;
+                            c->read->handler = ngx_ssl_next_read_handler;
+                        }
+
+                        ngx_post_event(c->read, &ngx_posted_next_events);
+                    }
+
+                    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                                   "SSL_read: avail:%d", c->read->available);
+
+                } else {
+
+#if (NGX_HAVE_FIONREAD)
+
+                    if (ngx_socket_nread(c->fd, &c->read->available) == -1) {
+                        c->read->error = 1;
+                        ngx_connection_error(c, ngx_socket_errno,
+                                             ngx_socket_nread_n " failed");
+                        return NGX_ERROR;
+                    }
+
+                    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                                   "SSL_read: avail:%d", c->read->available);
+
+#endif
+                }
+
                 return bytes;
             }
 
@@ -2285,6 +2328,31 @@ ngx_ssl_write_handler(ngx_event_t *wev)
 }
 
 
+static void
+ngx_ssl_next_read_handler(ngx_event_t *rev)
+{
+    ngx_connection_t  *c;
+
+    c = rev->data;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL next read handler");
+
+    rev->handler = c->ssl->next_read_handler;
+    c->ssl->next_read_handler = NULL;
+
+    if (!rev->ready) {
+        rev->ready = 1;
+        rev->available = -1;
+    }
+
+    if (rev->posted) {
+        ngx_delete_posted_event(rev);
+    }
+
+    rev->handler(rev);
+}
+
+
 /*
  * OpenSSL has no SSL_writev() so we copy several bufs into our 16K buffer
  * before the SSL_write() call to decrease a SSL overhead.

diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h
index 61da0c5db6f2114adf863b48597bd0a7e0d8f509..71df9004526dbc8d800258ffb018fc113ea4d670 100644 (file)
--- a/src/event/ngx_event_openssl.h
+++ b/src/event/ngx_event_openssl.h
@@ -86,6 +86,7 @@ struct ngx_ssl_connection_s {
 
     ngx_event_handler_pt        saved_read_handler;
     ngx_event_handler_pt        saved_write_handler;
+    ngx_event_handler_pt        next_read_handler;
 
     u_char                      early_buf;
 

diff --git a/src/event/ngx_event_posted.c b/src/event/ngx_event_posted.c
index d851f3d14fc3c5b95720592a3efcf549f1081b1b..fd0b411c4c3da7ee1078aa4292fcc26733467e2c 100644 (file)
--- a/src/event/ngx_event_posted.c
+++ b/src/event/ngx_event_posted.c
@@ -11,6 +11,7 @@
 
 
 ngx_queue_t  ngx_posted_accept_events;
+ngx_queue_t  ngx_posted_next_events;
 ngx_queue_t  ngx_posted_events;
 
 

diff --git a/src/event/ngx_event_posted.h b/src/event/ngx_event_posted.h
index 145d30feac760af4d56e4b7fd21e93c643c4557f..bac5b35552642117f1dee1a760927476ecabdc1a 100644 (file)
--- a/src/event/ngx_event_posted.h
+++ b/src/event/ngx_event_posted.h
@@ -42,6 +42,7 @@ void ngx_event_process_posted(ngx_cycle_t *cycle, ngx_queue_t *posted);
 
 
 extern ngx_queue_t  ngx_posted_accept_events;
+extern ngx_queue_t  ngx_posted_next_events;
 extern ngx_queue_t  ngx_posted_events;