aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/core/ngx_buf.h3
-rw-r--r--src/core/ngx_connection.h2
-rw-r--r--src/core/ngx_output_chain.c32
-rw-r--r--src/event/ngx_event.h8
-rw-r--r--src/http/ngx_http_copy_filter_module.c99
-rw-r--r--src/http/ngx_http_request.h3
-rw-r--r--src/os/unix/ngx_file_aio_read.c42
-rw-r--r--src/os/unix/ngx_files.h1
-rw-r--r--src/os/unix/ngx_freebsd_sendfile_chain.c77
-rw-r--r--src/os/unix/ngx_linux_aio_read.c39
10 files changed, 187 insertions, 119 deletions
diff --git a/src/core/ngx_buf.h b/src/core/ngx_buf.h
index 13536a69a..219894ffc 100644
--- a/src/core/ngx_buf.h
+++ b/src/core/ngx_buf.h
@@ -94,6 +94,9 @@ struct ngx_output_chain_ctx_s {
unsigned aio:1;
ngx_output_chain_aio_pt aio_handler;
+#if (NGX_HAVE_FILE_AIO)
+ ssize_t (*aio_preload)(ngx_buf_t *file);
+#endif
#endif
off_t alignment;
diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h
index ed14e6023..143cab7b3 100644
--- a/src/core/ngx_connection.h
+++ b/src/core/ngx_connection.h
@@ -181,9 +181,7 @@ struct ngx_connection_s {
#endif
#if (NGX_HAVE_AIO_SENDFILE)
- unsigned aio_sendfile:1;
unsigned busy_count:2;
- ngx_buf_t *busy_sendfile;
#endif
#if (NGX_THREADS)
diff --git a/src/core/ngx_output_chain.c b/src/core/ngx_output_chain.c
index 9d7a8460f..ca390e254 100644
--- a/src/core/ngx_output_chain.c
+++ b/src/core/ngx_output_chain.c
@@ -29,6 +29,10 @@
static ngx_inline ngx_int_t
ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf);
+#if (NGX_HAVE_AIO_SENDFILE)
+static ngx_int_t ngx_output_chain_aio_setup(ngx_output_chain_ctx_t *ctx,
+ ngx_file_t *file);
+#endif
static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool,
ngx_chain_t **chain, ngx_chain_t *in);
static ngx_int_t ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx,
@@ -252,6 +256,12 @@ ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf)
buf->in_file = 0;
}
+#if (NGX_HAVE_AIO_SENDFILE)
+ if (ctx->aio_preload && buf->in_file) {
+ (void) ngx_output_chain_aio_setup(ctx, buf->file);
+ }
+#endif
+
if (ctx->need_in_memory && !ngx_buf_in_memory(buf)) {
return 0;
}
@@ -264,6 +274,28 @@ ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf)
}
+#if (NGX_HAVE_AIO_SENDFILE)
+
+static ngx_int_t
+ngx_output_chain_aio_setup(ngx_output_chain_ctx_t *ctx, ngx_file_t *file)
+{
+ ngx_event_aio_t *aio;
+
+ if (file->aio == NULL && ngx_file_aio_init(file, ctx->pool) != NGX_OK) {
+ return NGX_ERROR;
+ }
+
+ aio = file->aio;
+
+ aio->data = ctx->filter_ctx;
+ aio->preload_handler = ctx->aio_preload;
+
+ return NGX_OK;
+}
+
+#endif
+
+
static ngx_int_t
ngx_output_chain_add_copy(ngx_pool_t *pool, ngx_chain_t **chain,
ngx_chain_t *in)
diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h
index a1643a134..555cda0f4 100644
--- a/src/event/ngx_event.h
+++ b/src/event/ngx_event.h
@@ -168,6 +168,10 @@ struct ngx_event_aio_s {
ngx_event_handler_pt handler;
ngx_file_t *file;
+#if (NGX_HAVE_AIO_SENDFILE)
+ ssize_t (*preload_handler)(ngx_buf_t *file);
+#endif
+
ngx_fd_t fd;
#if (NGX_HAVE_EVENTFD)
@@ -181,10 +185,6 @@ struct ngx_event_aio_s {
size_t nbytes;
#endif
-#if (NGX_HAVE_AIO_SENDFILE)
- off_t last_offset;
-#endif
-
ngx_aiocb_t aiocb;
ngx_event_t event;
};
diff --git a/src/http/ngx_http_copy_filter_module.c b/src/http/ngx_http_copy_filter_module.c
index 3ad27b042..cdd7fceb6 100644
--- a/src/http/ngx_http_copy_filter_module.c
+++ b/src/http/ngx_http_copy_filter_module.c
@@ -20,6 +20,7 @@ static void ngx_http_copy_aio_handler(ngx_output_chain_ctx_t *ctx,
ngx_file_t *file);
static void ngx_http_copy_aio_event_handler(ngx_event_t *ev);
#if (NGX_HAVE_AIO_SENDFILE)
+static ssize_t ngx_http_copy_aio_sendfile_preload(ngx_buf_t *file);
static void ngx_http_copy_aio_sendfile_event_handler(ngx_event_t *ev);
#endif
#endif
@@ -125,7 +126,9 @@ ngx_http_copy_filter(ngx_http_request_t *r, ngx_chain_t *in)
ctx->aio_handler = ngx_http_copy_aio_handler;
}
#if (NGX_HAVE_AIO_SENDFILE)
- c->aio_sendfile = (clcf->aio == NGX_HTTP_AIO_SENDFILE);
+ if (clcf->aio == NGX_HTTP_AIO_SENDFILE) {
+ ctx->aio_preload = ngx_http_copy_aio_sendfile_preload;
+ }
#endif
}
#endif
@@ -139,72 +142,19 @@ ngx_http_copy_filter(ngx_http_request_t *r, ngx_chain_t *in)
ctx->aio = r->aio;
#endif
- for ( ;; ) {
- rc = ngx_output_chain(ctx, in);
-
- if (ctx->in == NULL) {
- r->buffered &= ~NGX_HTTP_COPY_BUFFERED;
-
- } else {
- r->buffered |= NGX_HTTP_COPY_BUFFERED;
- }
-
- ngx_log_debug3(NGX_LOG_DEBUG_HTTP, c->log, 0,
- "http copy filter: %i \"%V?%V\"", rc, &r->uri, &r->args);
-
-#if (NGX_HAVE_FILE_AIO && NGX_HAVE_AIO_SENDFILE)
-
- if (c->busy_sendfile) {
- ssize_t n;
- off_t offset;
- ngx_file_t *file;
- ngx_http_ephemeral_t *e;
-
- if (r->aio) {
- c->busy_sendfile = NULL;
- return rc;
- }
-
- file = c->busy_sendfile->file;
- offset = c->busy_sendfile->file_pos;
-
- if (file->aio) {
- c->busy_count = (offset == file->aio->last_offset) ?
- c->busy_count + 1 : 0;
- file->aio->last_offset = offset;
-
- if (c->busy_count > 2) {
- ngx_log_error(NGX_LOG_ALERT, c->log, 0,
- "sendfile(%V) returned busy again",
- &file->name);
- c->aio_sendfile = 0;
- }
- }
-
- c->busy_sendfile = NULL;
- e = (ngx_http_ephemeral_t *) &r->uri_start;
-
- n = ngx_file_aio_read(file, &e->aio_preload, 1, offset, r->pool);
-
- if (n > 0) {
- in = NULL;
- continue;
- }
+ rc = ngx_output_chain(ctx, in);
- rc = n;
+ if (ctx->in == NULL) {
+ r->buffered &= ~NGX_HTTP_COPY_BUFFERED;
- if (rc == NGX_AGAIN) {
- file->aio->data = r;
- file->aio->handler = ngx_http_copy_aio_sendfile_event_handler;
+ } else {
+ r->buffered |= NGX_HTTP_COPY_BUFFERED;
+ }
- r->main->blocked++;
- r->aio = 1;
- }
- }
-#endif
+ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, c->log, 0,
+ "http copy filter: %i \"%V?%V\"", rc, &r->uri, &r->args);
- return rc;
- }
+ return rc;
}
@@ -244,6 +194,29 @@ ngx_http_copy_aio_event_handler(ngx_event_t *ev)
#if (NGX_HAVE_AIO_SENDFILE)
+static ssize_t
+ngx_http_copy_aio_sendfile_preload(ngx_buf_t *file)
+{
+ ssize_t n;
+ static u_char buf[1];
+ ngx_event_aio_t *aio;
+ ngx_http_request_t *r;
+
+ n = ngx_file_aio_read(file->file, buf, 1, file->file_pos, NULL);
+
+ if (n == NGX_AGAIN) {
+ aio = file->file->aio;
+ aio->handler = ngx_http_copy_aio_sendfile_event_handler;
+
+ r = aio->data;
+ r->main->blocked++;
+ r->aio = 1;
+ }
+
+ return n;
+}
+
+
static void
ngx_http_copy_aio_sendfile_event_handler(ngx_event_t *ev)
{
diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h
index cffab9a69..9be0c6e71 100644
--- a/src/http/ngx_http_request.h
+++ b/src/http/ngx_http_request.h
@@ -574,9 +574,6 @@ struct ngx_http_request_s {
typedef struct {
ngx_http_posted_request_t terminal_posted_request;
-#if (NGX_HAVE_AIO_SENDFILE)
- u_char aio_preload;
-#endif
} ngx_http_ephemeral_t;
diff --git a/src/os/unix/ngx_file_aio_read.c b/src/os/unix/ngx_file_aio_read.c
index 0bb383de5..b11cf8a3d 100644
--- a/src/os/unix/ngx_file_aio_read.c
+++ b/src/os/unix/ngx_file_aio_read.c
@@ -36,6 +36,28 @@ static ssize_t ngx_file_aio_result(ngx_file_t *file, ngx_event_aio_t *aio,
static void ngx_file_aio_event_handler(ngx_event_t *ev);
+ngx_int_t
+ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool)
+{
+ ngx_event_aio_t *aio;
+
+ aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
+ if (aio == NULL) {
+ return NGX_ERROR;
+ }
+
+ aio->file = file;
+ aio->fd = file->fd;
+ aio->event.data = aio;
+ aio->event.ready = 1;
+ aio->event.log = file->log;
+
+ file->aio = aio;
+
+ return NGX_OK;
+}
+
+
ssize_t
ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
ngx_pool_t *pool)
@@ -48,25 +70,11 @@ ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
return ngx_read_file(file, buf, size, offset);
}
- aio = file->aio;
-
- if (aio == NULL) {
- aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
- if (aio == NULL) {
- return NGX_ERROR;
- }
-
- aio->file = file;
- aio->fd = file->fd;
- aio->event.data = aio;
- aio->event.ready = 1;
- aio->event.log = file->log;
-#if (NGX_HAVE_AIO_SENDFILE)
- aio->last_offset = -1;
-#endif
- file->aio = aio;
+ if (file->aio == NULL && ngx_file_aio_init(file, pool) != NGX_OK) {
+ return NGX_ERROR;
}
+ aio = file->aio;
ev = &aio->event;
if (!ev->ready) {
diff --git a/src/os/unix/ngx_files.h b/src/os/unix/ngx_files.h
index a78ec9613..a046ee756 100644
--- a/src/os/unix/ngx_files.h
+++ b/src/os/unix/ngx_files.h
@@ -375,6 +375,7 @@ size_t ngx_fs_bsize(u_char *name);
#if (NGX_HAVE_FILE_AIO)
+ngx_int_t ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool);
ssize_t ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size,
off_t offset, ngx_pool_t *pool);
diff --git a/src/os/unix/ngx_freebsd_sendfile_chain.c b/src/os/unix/ngx_freebsd_sendfile_chain.c
index 7199c8654..25790b6b6 100644
--- a/src/os/unix/ngx_freebsd_sendfile_chain.c
+++ b/src/os/unix/ngx_freebsd_sendfile_chain.c
@@ -32,19 +32,23 @@
ngx_chain_t *
ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
{
- int rc, flags;
- off_t send, prev_send, sent;
- size_t file_size;
- ssize_t n;
- ngx_uint_t eintr, eagain;
- ngx_err_t err;
- ngx_buf_t *file;
- ngx_event_t *wev;
- ngx_chain_t *cl;
- ngx_iovec_t header, trailer;
- struct sf_hdtr hdtr;
- struct iovec headers[NGX_IOVS_PREALLOCATE];
- struct iovec trailers[NGX_IOVS_PREALLOCATE];
+ int rc, flags;
+ off_t send, prev_send, sent;
+ size_t file_size;
+ ssize_t n;
+ ngx_uint_t eintr, eagain;
+ ngx_err_t err;
+ ngx_buf_t *file;
+ ngx_event_t *wev;
+ ngx_chain_t *cl;
+ ngx_iovec_t header, trailer;
+ struct sf_hdtr hdtr;
+ struct iovec headers[NGX_IOVS_PREALLOCATE];
+ struct iovec trailers[NGX_IOVS_PREALLOCATE];
+#if (NGX_HAVE_AIO_SENDFILE)
+ ngx_uint_t ebusy;
+ ngx_event_aio_t *aio;
+#endif
wev = c->write;
@@ -73,6 +77,11 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
eagain = 0;
flags = 0;
+#if (NGX_HAVE_AIO_SENDFILE && NGX_SUPPRESS_WARN)
+ aio = NULL;
+ file = NULL;
+#endif
+
header.iovs = headers;
header.nalloc = NGX_IOVS_PREALLOCATE;
@@ -81,6 +90,9 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
for ( ;; ) {
eintr = 0;
+#if (NGX_HAVE_AIO_SENDFILE)
+ ebusy = 0;
+#endif
prev_send = send;
/* create the header iovec and coalesce the neighbouring bufs */
@@ -160,7 +172,8 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
sent = 0;
#if (NGX_HAVE_AIO_SENDFILE)
- flags = c->aio_sendfile ? SF_NODISKIO : 0;
+ aio = file->file->aio;
+ flags = (aio && aio->preload_handler) ? SF_NODISKIO : 0;
#endif
rc = sendfile(file->file->fd, c->fd, file->file_pos,
@@ -180,7 +193,7 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
#if (NGX_HAVE_AIO_SENDFILE)
case NGX_EBUSY:
- c->busy_sendfile = file;
+ ebusy = 1;
break;
#endif
@@ -232,9 +245,41 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
in = ngx_chain_update_sent(in, sent);
#if (NGX_HAVE_AIO_SENDFILE)
- if (c->busy_sendfile) {
+
+ if (ebusy) {
+ if (sent == 0) {
+ c->busy_count++;
+
+ if (c->busy_count > 2) {
+ ngx_log_error(NGX_LOG_ALERT, c->log, 0,
+ "sendfile(%V) returned busy again",
+ &file->file->name);
+
+ c->busy_count = 0;
+ aio->preload_handler = NULL;
+
+ send = prev_send;
+ continue;
+ }
+
+ } else {
+ c->busy_count = 0;
+ }
+
+ rc = aio->preload_handler(file);
+
+ if (rc > 0) {
+ send = prev_send + sent;
+ continue;
+ }
+
return in;
}
+
+ if (flags == SF_NODISKIO) {
+ c->busy_count = 0;
+ }
+
#endif
if (eagain) {
diff --git a/src/os/unix/ngx_linux_aio_read.c b/src/os/unix/ngx_linux_aio_read.c
index 8273c13f9..b0a923604 100644
--- a/src/os/unix/ngx_linux_aio_read.c
+++ b/src/os/unix/ngx_linux_aio_read.c
@@ -24,6 +24,28 @@ io_submit(aio_context_t ctx, long n, struct iocb **paiocb)
}
+ngx_int_t
+ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool)
+{
+ ngx_event_aio_t *aio;
+
+ aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
+ if (aio == NULL) {
+ return NGX_ERROR;
+ }
+
+ aio->file = file;
+ aio->fd = file->fd;
+ aio->event.data = aio;
+ aio->event.ready = 1;
+ aio->event.log = file->log;
+
+ file->aio = aio;
+
+ return NGX_OK;
+}
+
+
ssize_t
ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
ngx_pool_t *pool)
@@ -37,22 +59,11 @@ ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
return ngx_read_file(file, buf, size, offset);
}
- aio = file->aio;
-
- if (aio == NULL) {
- aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
- if (aio == NULL) {
- return NGX_ERROR;
- }
-
- aio->file = file;
- aio->fd = file->fd;
- aio->event.data = aio;
- aio->event.ready = 1;
- aio->event.log = file->log;
- file->aio = aio;
+ if (file->aio == NULL && ngx_file_aio_init(file, pool) != NGX_OK) {
+ return NGX_ERROR;
}
+ aio = file->aio;
ev = &aio->event;
if (!ev->ready) {
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-15 14:14:49 +0000