4 files changed, 249 insertions, 166 deletions

diff --git a/src/os/unix/ngx_freebsd_config.h b/src/os/unix/ngx_freebsd_config.h
index 73646fa0a..fa08f8f4f 100644
--- a/src/os/unix/ngx_freebsd_config.h
+++ b/src/os/unix/ngx_freebsd_config.h
@@ -15,7 +15,9 @@
 #include <sys/resource.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>        /* TCP_NOPUSH */
 #include <arpa/inet.h>
+#include <netdb.h>
 #include <osreldate.h>
 
 
diff --git a/src/os/unix/ngx_freebsd_init.h b/src/os/unix/ngx_freebsd_init.h
index e81942b16..cc450f914 100644
--- a/src/os/unix/ngx_freebsd_init.h
+++ b/src/os/unix/ngx_freebsd_init.h
@@ -13,6 +13,8 @@ ssize_t ngx_unix_recv(ngx_connection_t *c, char *buf, size_t size);
 ssize_t ngx_readv_chain(ngx_connection_t *c, ngx_chain_t *entry);
 /* */
 
+ngx_chain_t *ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in);
+
 
 extern int ngx_freebsd_kern_osreldate;
 extern int ngx_freebsd_hw_ncpu;
diff --git a/src/os/unix/ngx_freebsd_sendfile_chain.c b/src/os/unix/ngx_freebsd_sendfile_chain.c
index fc510b72f..bdb7250c9 100644
--- a/src/os/unix/ngx_freebsd_sendfile_chain.c
+++ b/src/os/unix/ngx_freebsd_sendfile_chain.c
@@ -1,20 +1,27 @@
 
 #include <ngx_config.h>
-
 #include <ngx_core.h>
-#include <ngx_types.h>
-#include <ngx_alloc.h>
-#include <ngx_array.h>
-#include <ngx_hunk.h>
-#include <ngx_connection.h>
-#include <ngx_sendv.h>
-#include <ngx_sendfile.h>
 #include <ngx_freebsd_init.h>
 
 
+/*
+   sendfile() often sends 4K pages over ethernet in 3 packets: 2x1460 and 1176
+   or in 6 packets: 5x1460 and 892.  Besides although sendfile() allows
+   to pass the header and the trailer it never sends the header or the trailer
+   with the part of the file in one packet.  So we use TCP_NOPUSH (similar
+   to Linux's TCP_CORK) to postpone the sending - it not only sends the header
+   and the first part of the file in one packet but also sends 4K pages
+   in the full packets.
+
+   The turning TCP_NOPUSH off flushes any pending data at least in FreeBSD 4.2,
+   although there's special fix in src/sys/netinet/tcp_usrreq.c just before
+   FreeBSD 4.5.
+*/
+
+
 ngx_chain_t *ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in)
 {
-    int              rc;
+    int              rc, eintr, tcp_nopush;
     char            *prev;
     size_t           hsize, size;
     off_t            sent;
@@ -23,160 +30,208 @@ ngx_chain_t *ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in)
     ngx_err_t        err;
     ngx_array_t      header, trailer;
     ngx_hunk_t      *file;
-    ngx_chain_t     *ce;
+    ngx_chain_t     *ce, *tail;
 
-    ce = in;
-    file = NULL;
-    hsize = 0;
+    tcp_nopush = 0;
 
-    ngx_init_array(header, c->pool, 10, sizeof(struct iovec), NGX_CHAIN_ERROR);
-    ngx_init_array(trailer, c->pool, 10, sizeof(struct iovec), NGX_CHAIN_ERROR);
+    do {
+        ce = in;
+        file = NULL;
+        hsize = 0;
+        eintr = 0;
 
-    /* create the header iovec */
-    if (ngx_hunk_in_memory_only(ce->hunk)) {
-        prev = NULL;
-        iov = NULL;
+        ngx_init_array(header, c->pool, 10, sizeof(struct iovec),
+                       NGX_CHAIN_ERROR);
+        ngx_init_array(trailer, c->pool, 10, sizeof(struct iovec),
+                       NGX_CHAIN_ERROR);
 
-        /* create the iovec and coalesce the neighbouring chain entries */
-        while (ce && ngx_hunk_in_memory_only(ce->hunk)) {
+        /* create the header iovec */
+        if (ngx_hunk_in_memory_only(ce->hunk)) {
+            prev = NULL;
+            iov = NULL;
 
-            if (prev == ce->hunk->pos) {
-                iov->iov_len += ce->hunk->last - ce->hunk->pos;
-                prev = ce->hunk->last;
+            /* create the iovec and coalesce the neighbouring chain entries */
+            while (ce && ngx_hunk_in_memory_only(ce->hunk)) {
 
-            } else {
-                ngx_test_null(iov, ngx_push_array(&header), NGX_CHAIN_ERROR);
-                iov->iov_base = ce->hunk->pos;
-                iov->iov_len = ce->hunk->last - ce->hunk->pos;
-                prev = ce->hunk->last;
-            }
+                if (prev == ce->hunk->pos) {
+                    iov->iov_len += ce->hunk->last - ce->hunk->pos;
+                    prev = ce->hunk->last;
+
+                } else {
+                    ngx_test_null(iov, ngx_push_array(&header),
+                                  NGX_CHAIN_ERROR);
+                    iov->iov_base = ce->hunk->pos;
+                    iov->iov_len = ce->hunk->last - ce->hunk->pos;
+                    prev = ce->hunk->last;
+                }
 
-            if (ngx_freebsd_sendfile_nbytes_bug) {
                 hsize += ce->hunk->last - ce->hunk->pos;
+
+                ce = ce->next;
             }
+        }
 
+        /* TODO: coalesce the neighbouring file hunks */
+        if (ce && (ce->hunk->type & NGX_HUNK_FILE)) {
+            file = ce->hunk;
             ce = ce->next;
         }
-    }
 
-    /* TODO: coalesce the neighbouring file hunks */
-    if (ce && (ce->hunk->type & NGX_HUNK_FILE)) {
-        file = ce->hunk;
-        ce = ce->next;
-    }
+        /* create the trailer iovec */
+        if (ce && ngx_hunk_in_memory_only(ce->hunk)) {
+            prev = NULL;
+            iov = NULL;
 
-    /* create the trailer iovec */
-    if (ce && ngx_hunk_in_memory_only(ce->hunk)) {
-        prev = NULL;
-        iov = NULL;
+            /* create the iovec and coalesce the neighbouring chain entries */
+            while (ce && ngx_hunk_in_memory_only(ce->hunk)) {
 
-        /* create the iovec and coalesce the neighbouring chain entries */
-        while (ce && ngx_hunk_in_memory_only(ce->hunk)) {
+                if (prev == ce->hunk->pos) {
+                    iov->iov_len += ce->hunk->last - ce->hunk->pos;
+                    prev = ce->hunk->last;
 
-            if (prev == ce->hunk->pos) {
-                iov->iov_len += ce->hunk->last - ce->hunk->pos;
-                prev = ce->hunk->last;
+                } else {
+                    ngx_test_null(iov, ngx_push_array(&trailer),
+                                  NGX_CHAIN_ERROR);
+                    iov->iov_base = ce->hunk->pos;
+                    iov->iov_len = ce->hunk->last - ce->hunk->pos;
+                    prev = ce->hunk->last;
+                }
 
-            } else {
-                ngx_test_null(iov, ngx_push_array(&trailer), NGX_CHAIN_ERROR);
-                iov->iov_base = ce->hunk->pos;
-                iov->iov_len = ce->hunk->last - ce->hunk->pos;
-                prev = ce->hunk->last;
+                ce = ce->next;
             }
-
-            ce = ce->next;
         }
-    }
 
-    if (file) {
-        hdtr.headers = (struct iovec *) header.elts;
-        hdtr.hdr_cnt = header.nelts;
-        hdtr.trailers = (struct iovec *) trailer.elts;
-        hdtr.trl_cnt = trailer.nelts;
+        tail = ce;
 
-        rc = sendfile(file->file->fd, c->fd, file->file_pos,
-                      (size_t) (file->file_last - file->file_pos) + hsize,
-                      &hdtr, &sent, 0);
+        if (file) {
 
-        if (rc == -1) {
-            err = ngx_errno;
-            if (err == NGX_EAGAIN || err == NGX_EINTR) {
-                ngx_log_error(NGX_LOG_INFO, c->log, err,
-                              "sendfile() sent only %qd bytes", sent);
+            if (tcp_nopush == 0) {
+                tcp_nopush = 1;
+                if (setsockopt(c->fd, IPPROTO_TCP, TCP_NOPUSH,
+                               (const void *) &tcp_nopush,
+                               sizeof(int)) == -1)
+                {
+                    ngx_log_error(NGX_LOG_CRIT, c->log, ngx_errno,
+                                  "setsockopt(TCP_NO_PUSH) failed");
+                    return NGX_CHAIN_ERROR;
+                }
+            }
 
-            } else {
-                ngx_log_error(NGX_LOG_CRIT, c->log, err, "sendfile() failed");
-                return NGX_CHAIN_ERROR;
+            hdtr.headers = (struct iovec *) header.elts;
+            hdtr.hdr_cnt = header.nelts;
+            hdtr.trailers = (struct iovec *) trailer.elts;
+            hdtr.trl_cnt = trailer.nelts;
+
+            if (ngx_freebsd_sendfile_nbytes_bug == 0) {
+                hsize = 0;
+            }
+
+            rc = sendfile(file->file->fd, c->fd, file->file_pos,
+                          (size_t) (file->file_last - file->file_pos) + hsize,
+                          &hdtr, &sent, 0);
+
+            if (rc == -1) {
+                err = ngx_errno;
+
+                if (err == NGX_EINTR) {
+                    eintr = 1;
+                }
+
+                if (err == NGX_EAGAIN || err == NGX_EINTR) {
+                    ngx_log_error(NGX_LOG_INFO, c->log, err,
+                                  "sendfile() sent only %qd bytes", sent);
+
+                } else {
+                    ngx_log_error(NGX_LOG_CRIT, c->log, err,
+                                  "sendfile() failed");
+                    return NGX_CHAIN_ERROR;
+                }
             }
-        }
 
 #if (NGX_DEBUG_WRITE_CHAIN)
-        ngx_log_debug(c->log, "sendfile: %d, @%qd %qd:%d" _
-                      rc _ file->file_pos _ sent _
-                      (size_t) (file->file_last - file->file_pos) + hsize);
+            ngx_log_debug(c->log, "sendfile: %d, @%qd %qd:%d" _
+                          rc _ file->file_pos _ sent _
+                          (size_t) (file->file_last - file->file_pos) + hsize);
 #endif
 
-    } else {
-        rc = writev(c->fd, (struct iovec *) header.elts, header.nelts);
+        } else {
+            rc = writev(c->fd, (struct iovec *) header.elts, header.nelts);
 
-        if (rc == -1) {
-            err = ngx_errno;
-            if (err == NGX_EAGAIN) {
-                ngx_log_error(NGX_LOG_INFO, c->log, err, "writev() EAGAIN");
+            if (rc == -1) {
+                err = ngx_errno;
+                if (err == NGX_EAGAIN) {
+                    ngx_log_error(NGX_LOG_INFO, c->log, err, "writev() EAGAIN");
 
-            } else if (err == NGX_EINTR) {
-                ngx_log_error(NGX_LOG_INFO, c->log, err, "writev() EINTR");
+                } else if (err == NGX_EINTR) {
+                    eintr = 1;
+                    ngx_log_error(NGX_LOG_INFO, c->log, err, "writev() EINTR");
 
-            } else {
-                ngx_log_error(NGX_LOG_CRIT, c->log, err, "writev() failed");
-                return NGX_CHAIN_ERROR;
+                } else {
+                    ngx_log_error(NGX_LOG_CRIT, c->log, err, "writev() failed");
+                    return NGX_CHAIN_ERROR;
+                }
             }
-        }
 
-        sent = rc > 0 ? rc : 0;
-    }
+            sent = rc > 0 ? rc : 0;
 
 #if (NGX_DEBUG_WRITE_CHAIN)
-    ngx_log_debug(c->log, "sendv: %qd" _ sent);
+            ngx_log_debug(c->log, "writev: %qd" _ sent);
 #endif
+        }
 
-    c->sent += sent;
+        c->sent += sent;
 
-    for (ce = in; ce && sent > 0; ce = ce->next) {
+        for (ce = in; ce && sent > 0; ce = ce->next) {
 
-        if (ce->hunk->type & NGX_HUNK_IN_MEMORY) {
-            size = ce->hunk->last - ce->hunk->pos;
-        } else {
-            size = ce->hunk->file_last - ce->hunk->file_pos;
-        }
+            if (ce->hunk->type & NGX_HUNK_IN_MEMORY) {
+                size = ce->hunk->last - ce->hunk->pos;
+            } else {
+                size = ce->hunk->file_last - ce->hunk->file_pos;
+            }
+
+            if (sent >= size) {
+                sent -= size;
 
-        if (sent >= size) {
-            sent -= size;
+                if (ce->hunk->type & NGX_HUNK_IN_MEMORY) {
+                    ce->hunk->pos = ce->hunk->last;
+                }
+
+                if (ce->hunk->type & NGX_HUNK_FILE) {
+                    ce->hunk->file_pos = ce->hunk->file_last;
+                }
+
+                continue;
+            }
 
             if (ce->hunk->type & NGX_HUNK_IN_MEMORY) {
-                ce->hunk->pos = ce->hunk->last;
+                ce->hunk->pos += sent;
             }
 
             if (ce->hunk->type & NGX_HUNK_FILE) {
-                ce->hunk->file_pos = ce->hunk->file_last;
+                ce->hunk->file_pos += sent;
             }
 
-            continue;
+            break;
         }
 
-        if (ce->hunk->type & NGX_HUNK_IN_MEMORY) {
-            ce->hunk->pos += sent;
-        }
+        ngx_destroy_array(&trailer);
+        ngx_destroy_array(&header);
 
-        if (ce->hunk->type & NGX_HUNK_FILE) {
-            ce->hunk->file_pos += sent;
-        }
+        in = ce;
 
-        break;
-    }
+    } while ((tail && tail == ce) || eintr);
 
-    ngx_destroy_array(&trailer);
-    ngx_destroy_array(&header);
+    if (tcp_nopush == 1) {
+        tcp_nopush = 0;
+        if (setsockopt(c->fd, IPPROTO_TCP, TCP_NOPUSH,
+                       (const void *) &tcp_nopush,
+                       sizeof(int)) == -1)
+        {
+            ngx_log_error(NGX_LOG_CRIT, c->log, ngx_errno,
+                          "setsockopt(!TCP_NO_PUSH) failed");
+            return NGX_CHAIN_ERROR;
+        }
+    }
 
     return ce;
 }
diff --git a/src/os/unix/ngx_recv.c b/src/os/unix/ngx_recv.c
index b2be905f7..98607d0c7 100644
--- a/src/os/unix/ngx_recv.c
+++ b/src/os/unix/ngx_recv.c
@@ -1,94 +1,118 @@
 
 #include <ngx_config.h>
 #include <ngx_core.h>
-#include <ngx_errno.h>
-#include <ngx_log.h>
-#include <ngx_recv.h>
-#include <ngx_connection.h>
 
 
+static int ngx_unix_recv_error(ngx_event_t *rev, ngx_err_t err);
+
+
+#if (HAVE_KQUEUE)
+
 ssize_t ngx_unix_recv(ngx_connection_t *c, char *buf, size_t size)
 {
     ssize_t       n;
-    ngx_err_t     err;
-    ngx_event_t  *ev;
+    ngx_event_t  *rev;
 
-    ev = c->read;
+    rev = c->read;
 
-#if (HAVE_KQUEUE) /* DEBUG */
     if (ngx_event_flags & NGX_HAVE_KQUEUE_EVENT) {
         ngx_log_debug(c->log, "recv: eof:%d, avail:%d, err:%d" _
-                      ev->eof _ ev->available _ ev->error);
+                      rev->eof _ rev->available _ rev->error);
+
+        if (rev->available == 0) {
+            if (rev->eof) {
+                if (rev->error) {
+                    rev->ready = 0;
+                    ngx_set_socket_errno(rev->error);
+                    return ngx_unix_recv_error(rev, rev->error);
+                }
+                return 0;
+
+            } else {
+                return NGX_AGAIN;
+            }
+        }
     }
-#endif
-
-#if (HAVE_KQUEUE)
 
-    if ((ngx_event_flags & NGX_HAVE_KQUEUE_EVENT)
-        && ev->eof && ev->available == 0) {
+    do {
+        n = recv(c->fd, buf, size, 0);
 
-        if (ev->error == 0) {
-            return 0;
-        }
+        ngx_log_debug(c->log, "recv: %d:%d" _ n _ size);
 
-        ngx_set_socket_errno(ev->error);
-        err = ev->error;
-        n = -1;
+        if (n >= 0) {
+            if (ngx_event_flags & NGX_HAVE_KQUEUE_EVENT) {
+                rev->available -= n;
+                if (rev->available == 0) {
+                    rev->ready = 0;
+                }
 
-    } else {
-        n = recv(c->fd, buf, size, 0);
+                return n;
+            }
 
-ngx_log_debug(c->log, "recv: read:%d:%d" _ n _ size);
+            if ((size_t) n < size) {
+                rev->ready = 0;
+            }
 
-        if (n == -1) {
-            err = ngx_socket_errno;
+            return n;
         }
-    }
 
-#else /* not kqueue */
+        rev->ready = 0;
+        n = ngx_unix_recv_error(rev, ngx_socket_errno);
+
+    } while (n == NGX_EINTR);
 
-    n = recv(c->fd, buf, size, 0);
+    return n;
+}
 
-ngx_log_debug(c->log, "recv: read:%d:%d" _ n _ size);
+#else /* ! NAVE_KQUEUE */
 
-    if (n == -1) {
-        err = ngx_socket_errno;
-    }
+ssize_t ngx_unix_recv(ngx_connection_t *c, char *buf, size_t size)
+{
+    ssize_t       n;
+    ngx_event_t  *rev;
 
-#endif
+    rev = c->read;
 
-    if (n == -1) {
-        ev->ready = 0;
+    do {
+        n = recv(c->fd, buf, size, 0);
 
-        if (err == NGX_ECONNRESET && ev->ignore_econnreset) {
-            return 0;
-        }
+        ngx_log_debug(c->log, "recv: %d:%d" _ n _ size);
 
-        if (err == NGX_EAGAIN) {
-            ngx_log_error(NGX_LOG_INFO, c->log, err, "recv() returned EAGAIN");
-            return NGX_AGAIN;
+        if (n >= 0) {
+            if ((size_t) n < size) {
+                rev->ready = 0;
+            }
+            return n;
         }
 
-        ngx_log_error(NGX_LOG_ERR, c->log, err, "recv() failed");
-        return NGX_ERROR;
-    }
+        rev->ready = 0;
+        n = ngx_unix_recv_error(rev, ngx_socket_errno);
 
-#if (HAVE_KQUEUE)
+    } while (n == NGX_EINTR);
+
+    return n;
+}
+
+#endif /* NAVE_KQUEUE */
 
-    if (ngx_event_flags & NGX_HAVE_KQUEUE_EVENT) {
-        ev->available -= n;
-        if (ev->available == 0) {
-            ev->ready = 0;
-        }
 
-        return n;
+static int ngx_unix_recv_error(ngx_event_t *rev, ngx_err_t err)
+{
+    if (err == NGX_ECONNRESET && rev->ignore_econnreset) {
+        return 0;
     }
 
-#endif
+    if (err == NGX_EAGAIN) {
+        ngx_log_error(NGX_LOG_INFO, rev->log, err, "recv() returned EAGAIN");
+        return NGX_AGAIN;
+    }
 
-    if ((size_t) n < size) {
-        ev->ready = 0;
+    if (err == NGX_EINTR) {
+        ngx_log_error(NGX_LOG_INFO, rev->log, err, "recv() returned EINTR");
+        return NGX_EINTR;
     }
 
-    return n;
+    ngx_log_error(NGX_LOG_ERR, rev->log, err, "recv() failed");
+
+    return NGX_ERROR;
 }