Refactored sendfile() AIO preload.

This reduces layering violation and simplifies the logic of AIO preread, since
it's now triggered by the send chain function itself without falling back to
the copy filter.  The context of AIO operation is now stored per file buffer,
which makes it possible to properly handle cases when multiple buffers come
from different locations, each with its own configuration.

1 files changed, 61 insertions, 16 deletions

diff --git a/src/os/unix/ngx_freebsd_sendfile_chain.c b/src/os/unix/ngx_freebsd_sendfile_chain.c
index 7199c8654..25790b6b6 100644
--- a/src/os/unix/ngx_freebsd_sendfile_chain.c
+++ b/src/os/unix/ngx_freebsd_sendfile_chain.c
@@ -32,19 +32,23 @@
 ngx_chain_t *
 ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 {
-    int              rc, flags;
-    off_t            send, prev_send, sent;
-    size_t           file_size;
-    ssize_t          n;
-    ngx_uint_t       eintr, eagain;
-    ngx_err_t        err;
-    ngx_buf_t       *file;
-    ngx_event_t     *wev;
-    ngx_chain_t     *cl;
-    ngx_iovec_t      header, trailer;
-    struct sf_hdtr   hdtr;
-    struct iovec     headers[NGX_IOVS_PREALLOCATE];
-    struct iovec     trailers[NGX_IOVS_PREALLOCATE];
+    int               rc, flags;
+    off_t             send, prev_send, sent;
+    size_t            file_size;
+    ssize_t           n;
+    ngx_uint_t        eintr, eagain;
+    ngx_err_t         err;
+    ngx_buf_t        *file;
+    ngx_event_t      *wev;
+    ngx_chain_t      *cl;
+    ngx_iovec_t       header, trailer;
+    struct sf_hdtr    hdtr;
+    struct iovec      headers[NGX_IOVS_PREALLOCATE];
+    struct iovec      trailers[NGX_IOVS_PREALLOCATE];
+#if (NGX_HAVE_AIO_SENDFILE)
+    ngx_uint_t        ebusy;
+    ngx_event_aio_t  *aio;
+#endif
 
     wev = c->write;
 
@@ -73,6 +77,11 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
     eagain = 0;
     flags = 0;
 
+#if (NGX_HAVE_AIO_SENDFILE && NGX_SUPPRESS_WARN)
+    aio = NULL;
+    file = NULL;
+#endif
+
     header.iovs = headers;
     header.nalloc = NGX_IOVS_PREALLOCATE;
 
@@ -81,6 +90,9 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 
     for ( ;; ) {
         eintr = 0;
+#if (NGX_HAVE_AIO_SENDFILE)
+        ebusy = 0;
+#endif
         prev_send = send;
 
         /* create the header iovec and coalesce the neighbouring bufs */
@@ -160,7 +172,8 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
             sent = 0;
 
 #if (NGX_HAVE_AIO_SENDFILE)
-            flags = c->aio_sendfile ? SF_NODISKIO : 0;
+            aio = file->file->aio;
+            flags = (aio && aio->preload_handler) ? SF_NODISKIO : 0;
 #endif
 
             rc = sendfile(file->file->fd, c->fd, file->file_pos,
@@ -180,7 +193,7 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 
 #if (NGX_HAVE_AIO_SENDFILE)
                 case NGX_EBUSY:
-                    c->busy_sendfile = file;
+                    ebusy = 1;
                     break;
 #endif
 
@@ -232,9 +245,41 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
         in = ngx_chain_update_sent(in, sent);
 
 #if (NGX_HAVE_AIO_SENDFILE)
-        if (c->busy_sendfile) {
+
+        if (ebusy) {
+            if (sent == 0) {
+                c->busy_count++;
+
+                if (c->busy_count > 2) {
+                    ngx_log_error(NGX_LOG_ALERT, c->log, 0,
+                                  "sendfile(%V) returned busy again",
+                                  &file->file->name);
+
+                    c->busy_count = 0;
+                    aio->preload_handler = NULL;
+
+                    send = prev_send;
+                    continue;
+                }
+
+            } else {
+                c->busy_count = 0;
+            }
+
+            rc = aio->preload_handler(file);
+
+            if (rc > 0) {
+                send = prev_send + sent;
+                continue;
+            }
+
             return in;
         }
+
+        if (flags == SF_NODISKIO) {
+            c->busy_count = 0;
+        }
+
 #endif
 
         if (eagain) {