Refactored sendfile() AIO preload.

This reduces layering violation and simplifies the logic of AIO preread, since
it's now triggered by the send chain function itself without falling back to
the copy filter.  The context of AIO operation is now stored per file buffer,
which makes it possible to properly handle cases when multiple buffers come
from different locations, each with its own configuration.

4 files changed, 112 insertions, 47 deletions

diff --git a/src/os/unix/ngx_file_aio_read.c b/src/os/unix/ngx_file_aio_read.c
index 0bb383de5..b11cf8a3d 100644
--- a/src/os/unix/ngx_file_aio_read.c
+++ b/src/os/unix/ngx_file_aio_read.c
@@ -36,6 +36,28 @@ static ssize_t ngx_file_aio_result(ngx_file_t *file, ngx_event_aio_t *aio,
 static void ngx_file_aio_event_handler(ngx_event_t *ev);
 
 
+ngx_int_t
+ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool)
+{
+    ngx_event_aio_t  *aio;
+
+    aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
+    if (aio == NULL) {
+        return NGX_ERROR;
+    }
+
+    aio->file = file;
+    aio->fd = file->fd;
+    aio->event.data = aio;
+    aio->event.ready = 1;
+    aio->event.log = file->log;
+
+    file->aio = aio;
+
+    return NGX_OK;
+}
+
+
 ssize_t
 ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
     ngx_pool_t *pool)
@@ -48,25 +70,11 @@ ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
         return ngx_read_file(file, buf, size, offset);
     }
 
-    aio = file->aio;
-
-    if (aio == NULL) {
-        aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
-        if (aio == NULL) {
-            return NGX_ERROR;
-        }
-
-        aio->file = file;
-        aio->fd = file->fd;
-        aio->event.data = aio;
-        aio->event.ready = 1;
-        aio->event.log = file->log;
-#if (NGX_HAVE_AIO_SENDFILE)
-        aio->last_offset = -1;
-#endif
-        file->aio = aio;
+    if (file->aio == NULL && ngx_file_aio_init(file, pool) != NGX_OK) {
+        return NGX_ERROR;
     }
 
+    aio = file->aio;
     ev = &aio->event;
 
     if (!ev->ready) {
diff --git a/src/os/unix/ngx_files.h b/src/os/unix/ngx_files.h
index a78ec9613..a046ee756 100644
--- a/src/os/unix/ngx_files.h
+++ b/src/os/unix/ngx_files.h
@@ -375,6 +375,7 @@ size_t ngx_fs_bsize(u_char *name);
 
 #if (NGX_HAVE_FILE_AIO)
 
+ngx_int_t ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool);
 ssize_t ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size,
     off_t offset, ngx_pool_t *pool);
 
diff --git a/src/os/unix/ngx_freebsd_sendfile_chain.c b/src/os/unix/ngx_freebsd_sendfile_chain.c
index 7199c8654..25790b6b6 100644
--- a/src/os/unix/ngx_freebsd_sendfile_chain.c
+++ b/src/os/unix/ngx_freebsd_sendfile_chain.c
@@ -32,19 +32,23 @@
 ngx_chain_t *
 ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 {
-    int              rc, flags;
-    off_t            send, prev_send, sent;
-    size_t           file_size;
-    ssize_t          n;
-    ngx_uint_t       eintr, eagain;
-    ngx_err_t        err;
-    ngx_buf_t       *file;
-    ngx_event_t     *wev;
-    ngx_chain_t     *cl;
-    ngx_iovec_t      header, trailer;
-    struct sf_hdtr   hdtr;
-    struct iovec     headers[NGX_IOVS_PREALLOCATE];
-    struct iovec     trailers[NGX_IOVS_PREALLOCATE];
+    int               rc, flags;
+    off_t             send, prev_send, sent;
+    size_t            file_size;
+    ssize_t           n;
+    ngx_uint_t        eintr, eagain;
+    ngx_err_t         err;
+    ngx_buf_t        *file;
+    ngx_event_t      *wev;
+    ngx_chain_t      *cl;
+    ngx_iovec_t       header, trailer;
+    struct sf_hdtr    hdtr;
+    struct iovec      headers[NGX_IOVS_PREALLOCATE];
+    struct iovec      trailers[NGX_IOVS_PREALLOCATE];
+#if (NGX_HAVE_AIO_SENDFILE)
+    ngx_uint_t        ebusy;
+    ngx_event_aio_t  *aio;
+#endif
 
     wev = c->write;
 
@@ -73,6 +77,11 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
     eagain = 0;
     flags = 0;
 
+#if (NGX_HAVE_AIO_SENDFILE && NGX_SUPPRESS_WARN)
+    aio = NULL;
+    file = NULL;
+#endif
+
     header.iovs = headers;
     header.nalloc = NGX_IOVS_PREALLOCATE;
 
@@ -81,6 +90,9 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 
     for ( ;; ) {
         eintr = 0;
+#if (NGX_HAVE_AIO_SENDFILE)
+        ebusy = 0;
+#endif
         prev_send = send;
 
         /* create the header iovec and coalesce the neighbouring bufs */
@@ -160,7 +172,8 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
             sent = 0;
 
 #if (NGX_HAVE_AIO_SENDFILE)
-            flags = c->aio_sendfile ? SF_NODISKIO : 0;
+            aio = file->file->aio;
+            flags = (aio && aio->preload_handler) ? SF_NODISKIO : 0;
 #endif
 
             rc = sendfile(file->file->fd, c->fd, file->file_pos,
@@ -180,7 +193,7 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 
 #if (NGX_HAVE_AIO_SENDFILE)
                 case NGX_EBUSY:
-                    c->busy_sendfile = file;
+                    ebusy = 1;
                     break;
 #endif
 
@@ -232,9 +245,41 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
         in = ngx_chain_update_sent(in, sent);
 
 #if (NGX_HAVE_AIO_SENDFILE)
-        if (c->busy_sendfile) {
+
+        if (ebusy) {
+            if (sent == 0) {
+                c->busy_count++;
+
+                if (c->busy_count > 2) {
+                    ngx_log_error(NGX_LOG_ALERT, c->log, 0,
+                                  "sendfile(%V) returned busy again",
+                                  &file->file->name);
+
+                    c->busy_count = 0;
+                    aio->preload_handler = NULL;
+
+                    send = prev_send;
+                    continue;
+                }
+
+            } else {
+                c->busy_count = 0;
+            }
+
+            rc = aio->preload_handler(file);
+
+            if (rc > 0) {
+                send = prev_send + sent;
+                continue;
+            }
+
             return in;
         }
+
+        if (flags == SF_NODISKIO) {
+            c->busy_count = 0;
+        }
+
 #endif
 
         if (eagain) {
diff --git a/src/os/unix/ngx_linux_aio_read.c b/src/os/unix/ngx_linux_aio_read.c
index 8273c13f9..b0a923604 100644
--- a/src/os/unix/ngx_linux_aio_read.c
+++ b/src/os/unix/ngx_linux_aio_read.c
@@ -24,6 +24,28 @@ io_submit(aio_context_t ctx, long n, struct iocb **paiocb)
 }
 
 
+ngx_int_t
+ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool)
+{
+    ngx_event_aio_t  *aio;
+
+    aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
+    if (aio == NULL) {
+        return NGX_ERROR;
+    }
+
+    aio->file = file;
+    aio->fd = file->fd;
+    aio->event.data = aio;
+    aio->event.ready = 1;
+    aio->event.log = file->log;
+
+    file->aio = aio;
+
+    return NGX_OK;
+}
+
+
 ssize_t
 ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
     ngx_pool_t *pool)
@@ -37,22 +59,11 @@ ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
         return ngx_read_file(file, buf, size, offset);
     }
 
-    aio = file->aio;
-
-    if (aio == NULL) {
-        aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
-        if (aio == NULL) {
-            return NGX_ERROR;
-        }
-
-        aio->file = file;
-        aio->fd = file->fd;
-        aio->event.data = aio;
-        aio->event.ready = 1;
-        aio->event.log = file->log;
-        file->aio = aio;
+    if (file->aio == NULL && ngx_file_aio_init(file, pool) != NGX_OK) {
+        return NGX_ERROR;
     }
 
+    aio = file->aio;
     ev = &aio->event;
 
     if (!ev->ready) {