use POSIX semaphores in shmtx instead of sched_yield()

number of spinlock spins are increased twice

8 files changed, 249 insertions, 80 deletions

diff --git a/src/core/ngx_shmtx.c b/src/core/ngx_shmtx.c
index ead94e484..3b429c4f9 100644
--- a/src/core/ngx_shmtx.c
+++ b/src/core/ngx_shmtx.c
@@ -16,9 +16,166 @@ ngx_shmtx_create(ngx_shmtx_t *mtx, void *addr, u_char *name)
 {
     mtx->lock = addr;
 
+    if (mtx->spin == (ngx_uint_t) -1) {
+        return NGX_OK;
+    }
+
+    mtx->spin = 2048;
+
+#if (NGX_HAVE_POSIX_SEM)
+
+    if (sem_init(&mtx->sem, 1, 0) == -1) {
+        ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, ngx_errno,
+                      "sem_init() failed");
+    } else {
+        mtx->semaphore = 1;
+    }
+
+#endif
+
     return NGX_OK;
 }
 
+
+void
+ngx_shmtx_destory(ngx_shmtx_t *mtx)
+{
+#if (NGX_HAVE_POSIX_SEM)
+
+    if (mtx->semaphore) {
+        if (sem_destroy(&mtx->sem) == -1) {
+            ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, ngx_errno,
+                          "sem_destroy() failed");
+        }
+    }
+
+#endif
+}
+
+
+ngx_uint_t
+ngx_shmtx_trylock(ngx_shmtx_t *mtx)
+{
+    ngx_atomic_uint_t  val;
+
+    val = *mtx->lock;
+
+    return ((val & 0x80000000) == 0
+            && ngx_atomic_cmp_set(mtx->lock, val, val | 0x80000000));
+}
+
+
+void
+ngx_shmtx_lock(ngx_shmtx_t *mtx)
+{
+    ngx_uint_t         i, n;
+    ngx_atomic_uint_t  val;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0, "shmtx lock");
+
+    for ( ;; ) {
+
+        val = *mtx->lock;
+
+        if ((val & 0x80000000) == 0
+            && ngx_atomic_cmp_set(mtx->lock, val, val | 0x80000000))
+        {
+            return;
+        }
+
+        if (ngx_ncpu > 1) {
+
+            for (n = 1; n < mtx->spin; n <<= 1) {
+
+                for (i = 0; i < n; i++) {
+                    ngx_cpu_pause();
+                }
+
+                val = *mtx->lock;
+
+                if ((val & 0x80000000) == 0
+                    && ngx_atomic_cmp_set(mtx->lock, val, val | 0x80000000))
+                {
+                    return;
+                }
+            }
+        }
+
+#if (NGX_HAVE_POSIX_SEM)
+
+        if (mtx->semaphore) {
+            val = *mtx->lock;
+
+            if ((val & 0x80000000)
+                && ngx_atomic_cmp_set(mtx->lock, val, val + 1))
+            {
+                ngx_log_debug1(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0,
+                               "shmtx wait %XA", val);
+
+                while (sem_wait(&mtx->sem) == -1) {
+                    ngx_err_t  err;
+
+                    err = ngx_errno;
+
+                    if (err != NGX_EINTR) {
+                        ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, err,
+                                   "sem_wait() failed while waiting on shmtx");
+                        break;
+                    }
+                }
+
+                ngx_log_debug0(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0,
+                               "shmtx awoke");
+            }
+
+            continue;
+        }
+
+#endif
+
+        ngx_sched_yield();
+    }
+}
+
+
+void
+ngx_shmtx_unlock(ngx_shmtx_t *mtx)
+{
+    ngx_atomic_uint_t  val, old, wait;
+
+    if (mtx->spin != (ngx_uint_t) -1) {
+        ngx_log_debug0(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0, "shmtx unlock");
+    }
+
+    for ( ;; ) {
+
+        old = *mtx->lock;
+        wait = old & 0x7fffffff;
+        val = wait ? wait - 1 : 0;
+
+        if (ngx_atomic_cmp_set(mtx->lock, old, val)) {
+            break;
+        }
+    }
+
+#if (NGX_HAVE_POSIX_SEM)
+
+    if (wait == 0 || !mtx->semaphore) {
+        return;
+    }
+
+    ngx_log_debug1(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0,
+                   "shmtx wake %XA", old);
+
+    if (sem_post(&mtx->sem) == -1) {
+        ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, ngx_errno,
+                      "sem_post() failed while wake shmtx");
+    }
+
+#endif
+}
+
+
 #else
 
 
@@ -65,4 +222,62 @@ ngx_shmtx_destory(ngx_shmtx_t *mtx)
 }
 
 
+ngx_uint_t
+ngx_shmtx_trylock(ngx_shmtx_t *mtx)
+{
+    ngx_err_t  err;
+
+    err = ngx_trylock_fd(mtx->fd);
+
+    if (err == 0) {
+        return 1;
+    }
+
+    if (err == NGX_EAGAIN) {
+        return 0;
+    }
+
+#if __osf__ /* Tru64 UNIX */
+
+    if (err == NGX_EACCESS) {
+        return 0;
+    }
+
+#endif
+
+    ngx_log_abort(err, ngx_trylock_fd_n " %s failed", mtx->name);
+
+    return 0;
+}
+
+
+void
+ngx_shmtx_lock(ngx_shmtx_t *mtx)
+{
+    ngx_err_t  err;
+
+    err = ngx_lock_fd(mtx->fd);
+
+    if (err == 0) {
+        return;
+    }
+
+    ngx_log_abort(err, ngx_lock_fd_n " %s failed", mtx->name);
+}
+
+
+void
+ngx_shmtx_unlock(ngx_shmtx_t *mtx)
+{
+    ngx_err_t  err;
+
+    err = ngx_unlock_fd(mtx->fd);
+
+    if (err == 0) {
+        return;
+    }
+
+    ngx_log_abort(err, ngx_unlock_fd_n " %s failed", mtx->name);
+}
+
 #endif
diff --git a/src/core/ngx_shmtx.h b/src/core/ngx_shmtx.h
index 57fe0b9f5..714f73aa6 100644
--- a/src/core/ngx_shmtx.h
+++ b/src/core/ngx_shmtx.h
@@ -15,95 +15,23 @@
 typedef struct {
 #if (NGX_HAVE_ATOMIC_OPS)
     ngx_atomic_t  *lock;
+#if (NGX_HAVE_POSIX_SEM)
+    ngx_uint_t     semaphore;
+    sem_t          sem;
+#endif
 #else
     ngx_fd_t       fd;
     u_char        *name;
 #endif
+    ngx_uint_t     spin;
 } ngx_shmtx_t;
 
 
 ngx_int_t ngx_shmtx_create(ngx_shmtx_t *mtx, void *addr, u_char *name);
-
-
-#if (NGX_HAVE_ATOMIC_OPS)
-
-static ngx_inline ngx_uint_t
-ngx_shmtx_trylock(ngx_shmtx_t *mtx)
-{
-    return (*mtx->lock == 0 && ngx_atomic_cmp_set(mtx->lock, 0, ngx_pid));
-}
-
-#define ngx_shmtx_lock(mtx)   ngx_spinlock((mtx)->lock, ngx_pid, 1024)
-
-#define ngx_shmtx_unlock(mtx) (void) ngx_atomic_cmp_set((mtx)->lock, ngx_pid, 0)
-
-#define ngx_shmtx_destory(mtx)
-
-
-#else
-
-static ngx_inline ngx_uint_t
-ngx_shmtx_trylock(ngx_shmtx_t *mtx)
-{
-    ngx_err_t  err;
-
-    err = ngx_trylock_fd(mtx->fd);
-
-    if (err == 0) {
-        return 1;
-    }
-
-    if (err == NGX_EAGAIN) {
-        return 0;
-    }
-
-#if __osf__ /* Tru64 UNIX */
-
-    if (err == NGX_EACCESS) {
-        return 0;
-    }
-
-#endif
-
-    ngx_log_abort(err, ngx_trylock_fd_n " %s failed", mtx->name);
-
-    return 0;
-}
-
-
-static ngx_inline void
-ngx_shmtx_lock(ngx_shmtx_t *mtx)
-{
-    ngx_err_t  err;
-
-    err = ngx_lock_fd(mtx->fd);
-
-    if (err == 0) {
-        return;
-    }
-
-    ngx_log_abort(err, ngx_lock_fd_n " %s failed", mtx->name);
-}
-
-
-static ngx_inline void
-ngx_shmtx_unlock(ngx_shmtx_t *mtx)
-{
-    ngx_err_t  err;
-
-    err = ngx_unlock_fd(mtx->fd);
-
-    if (err == 0) {
-        return;
-    }
-
-    ngx_log_abort(err, ngx_unlock_fd_n " %s failed", mtx->name);
-}
-
-
 void ngx_shmtx_destory(ngx_shmtx_t *mtx);
-
-#endif
+ngx_uint_t ngx_shmtx_trylock(ngx_shmtx_t *mtx);
+void ngx_shmtx_lock(ngx_shmtx_t *mtx);
+void ngx_shmtx_unlock(ngx_shmtx_t *mtx);
 
 
 #endif /* _NGX_SHMTX_H_INCLUDED_ */
diff --git a/src/event/ngx_event.c b/src/event/ngx_event.c
index 3922b0e20..c57d37ec4 100644
--- a/src/event/ngx_event.c
+++ b/src/event/ngx_event.c
@@ -519,6 +519,7 @@ ngx_event_module_init(ngx_cycle_t *cycle)
     shared = shm.addr;
 
     ngx_accept_mutex_ptr = (ngx_atomic_t *) shared;
+    ngx_accept_mutex.spin = (ngx_uint_t) -1;
 
     if (ngx_shmtx_create(&ngx_accept_mutex, shared, cycle->lock_file.data)
         != NGX_OK)
diff --git a/src/os/unix/ngx_darwin_config.h b/src/os/unix/ngx_darwin_config.h
index 39009768f..88aa6f502 100644
--- a/src/os/unix/ngx_darwin_config.h
+++ b/src/os/unix/ngx_darwin_config.h
@@ -56,6 +56,11 @@
 #include <ngx_auto_config.h>
 
 
+#if (NGX_HAVE_POSIX_SEM)
+#include <semaphore.h>
+#endif
+
+
 #if (NGX_HAVE_POLL)
 #include <poll.h>
 #endif
diff --git a/src/os/unix/ngx_freebsd_config.h b/src/os/unix/ngx_freebsd_config.h
index 20cb66640..ec7a375a8 100644
--- a/src/os/unix/ngx_freebsd_config.h
+++ b/src/os/unix/ngx_freebsd_config.h
@@ -68,6 +68,11 @@
 #include <ngx_auto_config.h>
 
 
+#if (NGX_HAVE_POSIX_SEM)
+#include <semaphore.h>
+#endif
+
+
 #if (NGX_HAVE_POLL)
 #include <poll.h>
 #endif
diff --git a/src/os/unix/ngx_linux_config.h b/src/os/unix/ngx_linux_config.h
index cf45e54a1..046095d1d 100644
--- a/src/os/unix/ngx_linux_config.h
+++ b/src/os/unix/ngx_linux_config.h
@@ -58,6 +58,11 @@
 #include <ngx_auto_config.h>
 
 
+#if (NGX_HAVE_POSIX_SEM)
+#include <semaphore.h>
+#endif
+
+
 #if (NGX_HAVE_SYS_PRCTL_H)
 #include <sys/prctl.h>
 #endif
diff --git a/src/os/unix/ngx_posix_config.h b/src/os/unix/ngx_posix_config.h
index 86a80d701..aec8a0a35 100644
--- a/src/os/unix/ngx_posix_config.h
+++ b/src/os/unix/ngx_posix_config.h
@@ -96,6 +96,11 @@
 #include <ngx_auto_config.h>
 
 
+#if (NGX_HAVE_POSIX_SEM)
+#include <semaphore.h>
+#endif
+
+
 #if (NGX_HAVE_POLL)
 #include <poll.h>
 #endif
diff --git a/src/os/unix/ngx_solaris_config.h b/src/os/unix/ngx_solaris_config.h
index ae8212d8a..6b3d42eaa 100644
--- a/src/os/unix/ngx_solaris_config.h
+++ b/src/os/unix/ngx_solaris_config.h
@@ -57,6 +57,11 @@
 #include <ngx_auto_config.h>
 
 
+#if (NGX_HAVE_POSIX_SEM)
+#include <semaphore.h>
+#endif
+
+
 #if (NGX_HAVE_POLL)
 #include <poll.h>
 #endif