Refactored sendfile() AIO preload.

This reduces layering violation and simplifies the logic of AIO preread, since
it's now triggered by the send chain function itself without falling back to
the copy filter.  The context of AIO operation is now stored per file buffer,
which makes it possible to properly handle cases when multiple buffers come
from different locations, each with its own configuration.

diff --git a/src/core/ngx_buf.h b/src/core/ngx_buf.h
index 13536a69a4c6e3ea9cba5d3b001aa6f8d8f93e49..219894ffc5fca87104023312b34decb34b40d737 100644 (file)
--- a/src/core/ngx_buf.h
+++ b/src/core/ngx_buf.h
@@ -94,6 +94,9 @@ struct ngx_output_chain_ctx_s {
     unsigned                     aio:1;
 
     ngx_output_chain_aio_pt      aio_handler;
+#if (NGX_HAVE_FILE_AIO)
+    ssize_t                    (*aio_preload)(ngx_buf_t *file);
+#endif
 #endif
 
     off_t                        alignment;

diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h
index ed14e6023e7ac5d822ea03340d60c30798558e76..143cab7b31f18b22731691e26aafc5cc33484740 100644 (file)
--- a/src/core/ngx_connection.h
+++ b/src/core/ngx_connection.h
@@ -181,9 +181,7 @@ struct ngx_connection_s {
 #endif
 
 #if (NGX_HAVE_AIO_SENDFILE)
-    unsigned            aio_sendfile:1;
     unsigned            busy_count:2;
-    ngx_buf_t          *busy_sendfile;
 #endif
 
 #if (NGX_THREADS)

diff --git a/src/core/ngx_output_chain.c b/src/core/ngx_output_chain.c
index 9d7a8460f9be973ab41d99f9fa17b7a03e4d66eb..ca390e254d9b0d6e218306a76b2d9bdc7015592e 100644 (file)
--- a/src/core/ngx_output_chain.c
+++ b/src/core/ngx_output_chain.c
@@ -29,6 +29,10 @@
 
 static ngx_inline ngx_int_t
     ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf);
+#if (NGX_HAVE_AIO_SENDFILE)
+static ngx_int_t ngx_output_chain_aio_setup(ngx_output_chain_ctx_t *ctx,
+    ngx_file_t *file);
+#endif
 static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool,
     ngx_chain_t **chain, ngx_chain_t *in);
 static ngx_int_t ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx,
@@ -252,6 +256,12 @@ ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf)
         buf->in_file = 0;
     }
 
+#if (NGX_HAVE_AIO_SENDFILE)
+    if (ctx->aio_preload && buf->in_file) {
+        (void) ngx_output_chain_aio_setup(ctx, buf->file);
+    }
+#endif
+
     if (ctx->need_in_memory && !ngx_buf_in_memory(buf)) {
         return 0;
     }
@@ -264,6 +274,28 @@ ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf)
 }
 
 
+#if (NGX_HAVE_AIO_SENDFILE)
+
+static ngx_int_t
+ngx_output_chain_aio_setup(ngx_output_chain_ctx_t *ctx, ngx_file_t *file)
+{
+    ngx_event_aio_t  *aio;
+
+    if (file->aio == NULL && ngx_file_aio_init(file, ctx->pool) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    aio = file->aio;
+
+    aio->data = ctx->filter_ctx;
+    aio->preload_handler = ctx->aio_preload;
+
+    return NGX_OK;
+}
+
+#endif
+
+
 static ngx_int_t
 ngx_output_chain_add_copy(ngx_pool_t *pool, ngx_chain_t **chain,
     ngx_chain_t *in)

diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h
index a1643a1343de8e2d4ad2b21df592e36d89f2f53e..555cda0f472f77afd5736f796f2458cfa4f18cce 100644 (file)
--- a/src/event/ngx_event.h
+++ b/src/event/ngx_event.h
@@ -168,6 +168,10 @@ struct ngx_event_aio_s {
     ngx_event_handler_pt       handler;
     ngx_file_t                *file;
 
+#if (NGX_HAVE_AIO_SENDFILE)
+    ssize_t                  (*preload_handler)(ngx_buf_t *file);
+#endif
+
     ngx_fd_t                   fd;
 
 #if (NGX_HAVE_EVENTFD)
@@ -181,10 +185,6 @@ struct ngx_event_aio_s {
     size_t                     nbytes;
 #endif
 
-#if (NGX_HAVE_AIO_SENDFILE)
-    off_t                      last_offset;
-#endif
-
     ngx_aiocb_t                aiocb;
     ngx_event_t                event;
 };

diff --git a/src/http/ngx_http_copy_filter_module.c b/src/http/ngx_http_copy_filter_module.c
index 3ad27b0425b30fb32b2b088febbe57a66ff4bd84..cdd7fceb619ed7241196f1ae8cab92c8bb467297 100644 (file)
--- a/src/http/ngx_http_copy_filter_module.c
+++ b/src/http/ngx_http_copy_filter_module.c
@@ -20,6 +20,7 @@ static void ngx_http_copy_aio_handler(ngx_output_chain_ctx_t *ctx,
     ngx_file_t *file);
 static void ngx_http_copy_aio_event_handler(ngx_event_t *ev);
 #if (NGX_HAVE_AIO_SENDFILE)
+static ssize_t ngx_http_copy_aio_sendfile_preload(ngx_buf_t *file);
 static void ngx_http_copy_aio_sendfile_event_handler(ngx_event_t *ev);
 #endif
 #endif
@@ -125,7 +126,9 @@ ngx_http_copy_filter(ngx_http_request_t *r, ngx_chain_t *in)
                 ctx->aio_handler = ngx_http_copy_aio_handler;
             }
 #if (NGX_HAVE_AIO_SENDFILE)
-            c->aio_sendfile = (clcf->aio == NGX_HTTP_AIO_SENDFILE);
+            if (clcf->aio == NGX_HTTP_AIO_SENDFILE) {
+                ctx->aio_preload = ngx_http_copy_aio_sendfile_preload;
+            }
 #endif
         }
 #endif
@@ -139,72 +142,19 @@ ngx_http_copy_filter(ngx_http_request_t *r, ngx_chain_t *in)
     ctx->aio = r->aio;
 #endif
 
-    for ( ;; ) {
-        rc = ngx_output_chain(ctx, in);
-
-        if (ctx->in == NULL) {
-            r->buffered &= ~NGX_HTTP_COPY_BUFFERED;
-
-        } else {
-            r->buffered |= NGX_HTTP_COPY_BUFFERED;
-        }
-
-        ngx_log_debug3(NGX_LOG_DEBUG_HTTP, c->log, 0,
-                       "http copy filter: %i \"%V?%V\"", rc, &r->uri, &r->args);
-
-#if (NGX_HAVE_FILE_AIO && NGX_HAVE_AIO_SENDFILE)
-
-        if (c->busy_sendfile) {
-            ssize_t                n;
-            off_t                  offset;
-            ngx_file_t            *file;
-            ngx_http_ephemeral_t  *e;
-
-            if (r->aio) {
-                c->busy_sendfile = NULL;
-                return rc;
-            }
-
-            file = c->busy_sendfile->file;
-            offset = c->busy_sendfile->file_pos;
-
-            if (file->aio) {
-                c->busy_count = (offset == file->aio->last_offset) ?
-                                c->busy_count + 1 : 0;
-                file->aio->last_offset = offset;
-
-                if (c->busy_count > 2) {
-                    ngx_log_error(NGX_LOG_ALERT, c->log, 0,
-                                  "sendfile(%V) returned busy again",
-                                  &file->name);
-                    c->aio_sendfile = 0;
-                }
-            }
-
-            c->busy_sendfile = NULL;
-            e = (ngx_http_ephemeral_t *) &r->uri_start;
-
-            n = ngx_file_aio_read(file, &e->aio_preload, 1, offset, r->pool);
-
-            if (n > 0) {
-                in = NULL;
-                continue;
-            }
+    rc = ngx_output_chain(ctx, in);
 
-            rc = n;
+    if (ctx->in == NULL) {
+        r->buffered &= ~NGX_HTTP_COPY_BUFFERED;
 
-            if (rc == NGX_AGAIN) {
-                file->aio->data = r;
-                file->aio->handler = ngx_http_copy_aio_sendfile_event_handler;
+    } else {
+        r->buffered |= NGX_HTTP_COPY_BUFFERED;
+    }
 
-                r->main->blocked++;
-                r->aio = 1;
-            }
-        }
-#endif
+    ngx_log_debug3(NGX_LOG_DEBUG_HTTP, c->log, 0,
+                   "http copy filter: %i \"%V?%V\"", rc, &r->uri, &r->args);
 
-        return rc;
-    }
+    return rc;
 }
 
 
@@ -244,6 +194,29 @@ ngx_http_copy_aio_event_handler(ngx_event_t *ev)
 
 #if (NGX_HAVE_AIO_SENDFILE)
 
+static ssize_t
+ngx_http_copy_aio_sendfile_preload(ngx_buf_t *file)
+{
+    ssize_t              n;
+    static u_char        buf[1];
+    ngx_event_aio_t     *aio;
+    ngx_http_request_t  *r;
+
+    n = ngx_file_aio_read(file->file, buf, 1, file->file_pos, NULL);
+
+    if (n == NGX_AGAIN) {
+        aio = file->file->aio;
+        aio->handler = ngx_http_copy_aio_sendfile_event_handler;
+
+        r = aio->data;
+        r->main->blocked++;
+        r->aio = 1;
+    }
+
+    return n;
+}
+
+
 static void
 ngx_http_copy_aio_sendfile_event_handler(ngx_event_t *ev)
 {

diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h
index cffab9a694dcd6bd264c42ad0baa50fa0e45b690..9be0c6e71283d22dbd4f05a1936238f73046ad69 100644 (file)
--- a/src/http/ngx_http_request.h
+++ b/src/http/ngx_http_request.h
@@ -574,9 +574,6 @@ struct ngx_http_request_s {
 
 typedef struct {
     ngx_http_posted_request_t         terminal_posted_request;
-#if (NGX_HAVE_AIO_SENDFILE)
-    u_char                            aio_preload;
-#endif
 } ngx_http_ephemeral_t;
 
 

diff --git a/src/os/unix/ngx_file_aio_read.c b/src/os/unix/ngx_file_aio_read.c
index 0bb383de5515f96a0b57324c1913599405ccc6bf..b11cf8a3d841eefabbe1e861bc781ac0f083d72e 100644 (file)
--- a/src/os/unix/ngx_file_aio_read.c
+++ b/src/os/unix/ngx_file_aio_read.c
@@ -36,6 +36,28 @@ static ssize_t ngx_file_aio_result(ngx_file_t *file, ngx_event_aio_t *aio,
 static void ngx_file_aio_event_handler(ngx_event_t *ev);
 
 
+ngx_int_t
+ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool)
+{
+    ngx_event_aio_t  *aio;
+
+    aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
+    if (aio == NULL) {
+        return NGX_ERROR;
+    }
+
+    aio->file = file;
+    aio->fd = file->fd;
+    aio->event.data = aio;
+    aio->event.ready = 1;
+    aio->event.log = file->log;
+
+    file->aio = aio;
+
+    return NGX_OK;
+}
+
+
 ssize_t
 ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
     ngx_pool_t *pool)
@@ -48,25 +70,11 @@ ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
         return ngx_read_file(file, buf, size, offset);
     }
 
-    aio = file->aio;
-
-    if (aio == NULL) {
-        aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
-        if (aio == NULL) {
-            return NGX_ERROR;
-        }
-
-        aio->file = file;
-        aio->fd = file->fd;
-        aio->event.data = aio;
-        aio->event.ready = 1;
-        aio->event.log = file->log;
-#if (NGX_HAVE_AIO_SENDFILE)
-        aio->last_offset = -1;
-#endif
-        file->aio = aio;
+    if (file->aio == NULL && ngx_file_aio_init(file, pool) != NGX_OK) {
+        return NGX_ERROR;
     }
 
+    aio = file->aio;
     ev = &aio->event;
 
     if (!ev->ready) {

diff --git a/src/os/unix/ngx_files.h b/src/os/unix/ngx_files.h
index a78ec961365621b6090bf7d130cc2f322809141a..a046ee75635557fd5d06b91679bc2cf8d018bdd1 100644 (file)
--- a/src/os/unix/ngx_files.h
+++ b/src/os/unix/ngx_files.h
@@ -375,6 +375,7 @@ size_t ngx_fs_bsize(u_char *name);
 
 #if (NGX_HAVE_FILE_AIO)
 
+ngx_int_t ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool);
 ssize_t ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size,
     off_t offset, ngx_pool_t *pool);
 

diff --git a/src/os/unix/ngx_freebsd_sendfile_chain.c b/src/os/unix/ngx_freebsd_sendfile_chain.c
index 7199c8654a4c4257cf15712cb2c83898e2af64ee..25790b6b62bb09ba59bfa23d318e6f0f6915c5e9 100644 (file)
--- a/src/os/unix/ngx_freebsd_sendfile_chain.c
+++ b/src/os/unix/ngx_freebsd_sendfile_chain.c
@@ -32,19 +32,23 @@
 ngx_chain_t *
 ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 {
-    int              rc, flags;
-    off_t            send, prev_send, sent;
-    size_t           file_size;
-    ssize_t          n;
-    ngx_uint_t       eintr, eagain;
-    ngx_err_t        err;
-    ngx_buf_t       *file;
-    ngx_event_t     *wev;
-    ngx_chain_t     *cl;
-    ngx_iovec_t      header, trailer;
-    struct sf_hdtr   hdtr;
-    struct iovec     headers[NGX_IOVS_PREALLOCATE];
-    struct iovec     trailers[NGX_IOVS_PREALLOCATE];
+    int               rc, flags;
+    off_t             send, prev_send, sent;
+    size_t            file_size;
+    ssize_t           n;
+    ngx_uint_t        eintr, eagain;
+    ngx_err_t         err;
+    ngx_buf_t        *file;
+    ngx_event_t      *wev;
+    ngx_chain_t      *cl;
+    ngx_iovec_t       header, trailer;
+    struct sf_hdtr    hdtr;
+    struct iovec      headers[NGX_IOVS_PREALLOCATE];
+    struct iovec      trailers[NGX_IOVS_PREALLOCATE];
+#if (NGX_HAVE_AIO_SENDFILE)
+    ngx_uint_t        ebusy;
+    ngx_event_aio_t  *aio;
+#endif
 
     wev = c->write;
 
@@ -73,6 +77,11 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
     eagain = 0;
     flags = 0;
 
+#if (NGX_HAVE_AIO_SENDFILE && NGX_SUPPRESS_WARN)
+    aio = NULL;
+    file = NULL;
+#endif
+
     header.iovs = headers;
     header.nalloc = NGX_IOVS_PREALLOCATE;
 
@@ -81,6 +90,9 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 
     for ( ;; ) {
         eintr = 0;
+#if (NGX_HAVE_AIO_SENDFILE)
+        ebusy = 0;
+#endif
         prev_send = send;
 
         /* create the header iovec and coalesce the neighbouring bufs */
@@ -160,7 +172,8 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
             sent = 0;
 
 #if (NGX_HAVE_AIO_SENDFILE)
-            flags = c->aio_sendfile ? SF_NODISKIO : 0;
+            aio = file->file->aio;
+            flags = (aio && aio->preload_handler) ? SF_NODISKIO : 0;
 #endif
 
             rc = sendfile(file->file->fd, c->fd, file->file_pos,
@@ -180,7 +193,7 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 
 #if (NGX_HAVE_AIO_SENDFILE)
                 case NGX_EBUSY:
-                    c->busy_sendfile = file;
+                    ebusy = 1;
                     break;
 #endif
 
@@ -232,9 +245,41 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
         in = ngx_chain_update_sent(in, sent);
 
 #if (NGX_HAVE_AIO_SENDFILE)
-        if (c->busy_sendfile) {
+
+        if (ebusy) {
+            if (sent == 0) {
+                c->busy_count++;
+
+                if (c->busy_count > 2) {
+                    ngx_log_error(NGX_LOG_ALERT, c->log, 0,
+                                  "sendfile(%V) returned busy again",
+                                  &file->file->name);
+
+                    c->busy_count = 0;
+                    aio->preload_handler = NULL;
+
+                    send = prev_send;
+                    continue;
+                }
+
+            } else {
+                c->busy_count = 0;
+            }
+
+            rc = aio->preload_handler(file);
+
+            if (rc > 0) {
+                send = prev_send + sent;
+                continue;
+            }
+
             return in;
         }
+
+        if (flags == SF_NODISKIO) {
+            c->busy_count = 0;
+        }
+
 #endif
 
         if (eagain) {

diff --git a/src/os/unix/ngx_linux_aio_read.c b/src/os/unix/ngx_linux_aio_read.c
index 8273c13f960645539d938cd16ee4e11c173eba10..b0a9236042166bb60dce4d7bf29caf26577a3dcb 100644 (file)
--- a/src/os/unix/ngx_linux_aio_read.c
+++ b/src/os/unix/ngx_linux_aio_read.c
@@ -24,6 +24,28 @@ io_submit(aio_context_t ctx, long n, struct iocb **paiocb)
 }
 
 
+ngx_int_t
+ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool)
+{
+    ngx_event_aio_t  *aio;
+
+    aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
+    if (aio == NULL) {
+        return NGX_ERROR;
+    }
+
+    aio->file = file;
+    aio->fd = file->fd;
+    aio->event.data = aio;
+    aio->event.ready = 1;
+    aio->event.log = file->log;
+
+    file->aio = aio;
+
+    return NGX_OK;
+}
+
+
 ssize_t
 ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
     ngx_pool_t *pool)
@@ -37,22 +59,11 @@ ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
         return ngx_read_file(file, buf, size, offset);
     }
 
-    aio = file->aio;
-
-    if (aio == NULL) {
-        aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
-        if (aio == NULL) {
-            return NGX_ERROR;
-        }
-
-        aio->file = file;
-        aio->fd = file->fd;
-        aio->event.data = aio;
-        aio->event.ready = 1;
-        aio->event.log = file->log;
-        file->aio = aio;
+    if (file->aio == NULL && ngx_file_aio_init(file, pool) != NGX_OK) {
+        return NGX_ERROR;
     }
 
+    aio = file->aio;
     ev = &aio->event;
 
     if (!ev->ready) {