aboutsummaryrefslogtreecommitdiff
path: root/src/os/unix/ngx_atomic.h
diff options
context:
space:
mode:
Diffstat (limited to 'src/os/unix/ngx_atomic.h')
-rw-r--r--src/os/unix/ngx_atomic.h327
1 files changed, 285 insertions, 42 deletions
diff --git a/src/os/unix/ngx_atomic.h b/src/os/unix/ngx_atomic.h
index 2de61cf81..c31396468 100644
--- a/src/os/unix/ngx_atomic.h
+++ b/src/os/unix/ngx_atomic.h
@@ -12,11 +12,14 @@
#include <ngx_core.h>
-#if ( __i386__ || __amd64__ )
+#if ( __i386__ )
#define NGX_HAVE_ATOMIC_OPS 1
-typedef volatile uint32_t ngx_atomic_t;
+typedef uint32_t ngx_atomic_int_t;
+typedef volatile ngx_atomic_int_t ngx_atomic_t;
+#define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1
+
#if (NGX_SMP)
#define NGX_SMP_LOCK "lock;"
@@ -24,57 +27,149 @@ typedef volatile uint32_t ngx_atomic_t;
#define NGX_SMP_LOCK
#endif
+/*
+ * the "=q" is any of the %eax, %ebx, %ecx, or %edx registers.
+ * the '"0" (1)' parameter preloads 1 into %0.
+ * the "cc" means that flags were changed.
+ */
-static ngx_inline uint32_t ngx_atomic_inc(ngx_atomic_t *value)
+static ngx_inline ngx_atomic_int_t
+ngx_atomic_inc(ngx_atomic_t *value)
{
- uint32_t old;
+ ngx_atomic_int_t old;
__asm__ volatile (
- NGX_SMP_LOCK
- " xaddl %0, %2; "
- " incl %0; "
+ NGX_SMP_LOCK
+ " xaddl %0, %2; "
+ " incl %0; "
- : "=q" (old) : "0" (1), "m" (*value));
+ : "=q" (old) : "0" (1), "m" (*value) : "cc", "memory");
return old;
}
-#if 0
-
-static ngx_inline uint32_t ngx_atomic_dec(ngx_atomic_t *value)
+static ngx_inline ngx_atomic_int_t
+ngx_atomic_dec(ngx_atomic_t *value)
{
- uint32_t old;
+ ngx_atomic_int_t old;
__asm__ volatile (
- NGX_SMP_LOCK
- " xaddl %0, %1; "
- " decl %0; "
+ NGX_SMP_LOCK
+ " xaddl %0, %2; "
+ " decl %0; "
- : "=q" (old) : "0" (-1), "m" (*value));
+ : "=q" (old) : "0" (-1), "m" (*value) : "cc", "memory");
return old;
}
+
+/*
+ * the "q" is any of the %eax, %ebx, %ecx, or %edx registers.
+ * the "=a" and "a" are the %eax register. Although we can return result
+ * in any register, we use %eax because it is used in cmpxchg anyway.
+ *
+ * "cmpxchg r, [m]":
+ *
+ * if (eax == [m]) {
+ * zf = 1;
+ * [m] = r;
+ * } else {
+ * zf = 0;
+ * eax = [m];
+ * }
+ */
+
+static ngx_inline ngx_atomic_int_t
+ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_int_t old,
+ ngx_atomic_int_t set)
+{
+ ngx_atomic_int_t res;
+
+ __asm__ volatile (
+
+ NGX_SMP_LOCK
+ " cmpxchgl %3, %1; "
+ " setz %b0; "
+ " movzbl %b0, %0; "
+
+ : "=a" (res) : "m" (*lock), "a" (old), "q" (set) : "cc", "memory");
+
+ return res;
+}
+
+
+#elif ( __amd64__ )
+
+#define NGX_HAVE_ATOMIC_OPS 1
+
+typedef int64_t ngx_atomic_int_t;
+typedef volatile ngx_atomic_int_t ngx_atomic_t;
+#define NGX_ATOMIC_T_LEN sizeof("-9223372036854775808") - 1
+
+
+#if (NGX_SMP)
+#define NGX_SMP_LOCK "lock;"
+#else
+#define NGX_SMP_LOCK
#endif
-static ngx_inline uint32_t ngx_atomic_cmp_set(ngx_atomic_t *lock,
- ngx_atomic_t old,
- ngx_atomic_t set)
+static ngx_inline ngx_atomic_int_t
+ngx_atomic_inc(ngx_atomic_t *value)
+{
+ ngx_atomic_int_t old;
+
+ __asm__ volatile (
+
+ NGX_SMP_LOCK
+ " xaddq %0, %2; "
+ " incq %0; "
+
+ : "=r" (old) : "0" (1), "m" (*value) : "cc", "memory");
+
+ return old;
+}
+
+
+/* the '"0" (-1LL)' parameter preloads -1 into the 64-bit %0 register */
+
+static ngx_inline ngx_atomic_int_t
+ngx_atomic_dec(ngx_atomic_t *value)
{
- uint32_t res;
+ ngx_atomic_int_t old;
__asm__ volatile (
- NGX_SMP_LOCK
- " cmpxchgl %3, %1; "
- " setz %%al; "
- " movzbl %%al, %0; "
+ NGX_SMP_LOCK
+ " xaddq %0, %2; "
+ " decq %0; "
+
+ : "=r" (old) : "0" (-1LL), "m" (*value) : "cc", "memory");
- : "=a" (res) : "m" (*lock), "a" (old), "q" (set));
+ return old;
+}
+
+
+/* the "=a" and "a" are the %rax register. */
+
+static ngx_inline ngx_atomic_int_t
+ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_int_t old,
+ ngx_atomic_int_t set)
+{
+ ngx_atomic_int_t res;
+
+ __asm__ volatile (
+
+ NGX_SMP_LOCK
+ " cmpxchgq %3, %1; "
+ " setz %b0; "
+ " movzbq %b0, %0; "
+
+ : "=a" (res) : "m" (*lock), "a" (old), "r" (set) : "cc", "memory");
return res;
}
@@ -84,12 +179,38 @@ static ngx_inline uint32_t ngx_atomic_cmp_set(ngx_atomic_t *lock,
#define NGX_HAVE_ATOMIC_OPS 1
-typedef volatile uint32_t ngx_atomic_t;
+#if (NGX_PTR_SIZE == 8)
+typedef uint64_t ngx_atomic_int_t;
+#define NGX_ATOMIC_T_LEN sizeof("-9223372036854775808") - 1
+#define NGX_CASXA "casxa"
+#else
+typedef uint32_t ngx_atomic_int_t;
+#define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1
+#define NGX_CASXA "casa"
+#endif
+
+typedef volatile ngx_atomic_int_t ngx_atomic_t;
+
+/*
+ * the "+r" means the general register used for both input and output.
+ *
+ * "casa [r1] 0x80, r2, r0" and
+ * "casxa [r1] 0x80, r2, r0" do the following:
+ *
+ * if ([r1] == r2) {
+ * swap(r0, [r1]);
+ * } else {
+ * r0 = [r1];
+ * }
+ *
+ * so "r0 == r2" means that the operation was successfull.
+ */
-static ngx_inline uint32_t ngx_atomic_inc(ngx_atomic_t *value)
+static ngx_inline ngx_atomic_int_t
+ngx_atomic_inc(ngx_atomic_t *value)
{
- uint32_t old, new, res;
+ ngx_atomic_int_t old, new, res;
old = *value;
@@ -100,9 +221,36 @@ static ngx_inline uint32_t ngx_atomic_inc(ngx_atomic_t *value)
__asm__ volatile (
- "casa [%1] 0x80, %2, %0"
+ NGX_CASXA " [%1] 0x80, %2, %0"
+
+ : "+r" (res) : "r" (value), "r" (old) : "memory");
+
+ if (res == old) {
+ return new;
+ }
+
+ old = res;
+ }
+}
+
+
+static ngx_inline ngx_atomic_int_t
+ngx_atomic_dec(ngx_atomic_t *value)
+{
+ ngx_atomic_int_t old, new, res;
+
+ old = *value;
+
+ for ( ;; ) {
+
+ new = old - 1;
+ res = new;
+
+ __asm__ volatile (
+
+ NGX_CASXA " [%1] 0x80, %2, %0"
- : "+r" (res) : "r" (value), "r" (old));
+ : "+r" (res) : "r" (value), "r" (old) : "memory");
if (res == old) {
return new;
@@ -113,33 +261,128 @@ static ngx_inline uint32_t ngx_atomic_inc(ngx_atomic_t *value)
}
-static ngx_inline uint32_t ngx_atomic_cmp_set(ngx_atomic_t *lock,
- ngx_atomic_t old,
- ngx_atomic_t set)
+static ngx_inline ngx_atomic_int_t
+ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_int_t old, ngx_atomic_int_t set)
+{
+ __asm__ volatile (
+
+ NGX_CASXA " [%1] 0x80, %2, %0"
+
+ : "+r" (set) : "r" (lock), "r" (old) : "memory");
+
+ return (set == old);
+}
+
+
+#elif ( __ppc__ )
+
+#define NGX_HAVE_ATOMIC_OPS 1
+
+#if (NGX_PTR_SIZE == 8)
+typedef uint64_t ngx_atomic_int_t;
+#define NGX_ATOMIC_T_LEN sizeof("-9223372036854775808") - 1
+#else
+#define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1
+typedef uint32_t ngx_atomic_int_t;
+#endif
+
+typedef volatile ngx_atomic_int_t ngx_atomic_t;
+
+
+/*
+ * the ppc assembler treats ";" as comment, so we have to use "\n".
+ * the minus in "bne-" is a hint for the branch prediction unit that
+ * this branch is unlikely to be taken.
+ *
+ * the "=&r" means that no input registers can be used.
+ * the "=&b" means that the base registers can be used only, i.e. any register
+ * except r0. the r0 register can not be used in "addi r0, r0, 1".
+ * the "1b" means the nearest backward label "1" and the "1f" means
+ * the nearest forward label "1".
+ */
+
+static ngx_inline ngx_atomic_int_t
+ngx_atomic_inc(ngx_atomic_t *value)
+{
+ ngx_atomic_int_t res;
+
+ __asm__ volatile (
+
+ "1: lwarx %0, 0, %1 \n" /* load from [value] into "res" */
+ /* and store reservation */
+ " addi %0, %0, 1 \n" /* add "1" to "res" */
+ " stwcx. %0, 0, %1 \n" /* store "res" into [value] if reservation */
+ /* is not cleared */
+ " bne- 1b \n" /* try again if reservation was cleared */
+
+ : "=&b" (res) : "r" (value) : "cc", "memory");
+
+ return res;
+}
+
+
+static ngx_inline ngx_atomic_int_t
+ngx_atomic_dec(ngx_atomic_t *value)
{
- uint32_t res = (uint32_t) set;
+ ngx_atomic_int_t res;
__asm__ volatile (
- "casa [%1] 0x80, %2, %0"
+ "1: lwarx %0, 0, %1 \n" /* load from [value] into "res" */
+ /* and store reservation */
+ " addi %0, %0, -1 \n" /* sub "1" from "res" */
+ " stwcx. %0, 0, %1 \n" /* store "res" into [value] if reservation */
+ /* is not cleared */
+ " bne- 1b \n" /* try again if reservation was cleared */
+
+ : "=&b" (res) : "r" (value) : "cc", "memory");
+
+ return res;
+}
+
+
+static ngx_inline ngx_atomic_int_t
+ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_int_t old,
+ ngx_atomic_int_t set)
+{
+ ngx_atomic_int_t res, temp;
- : "+r" (res) : "r" (lock), "r" (old));
+ __asm__ volatile (
- return (res == old);
+ " li %0, 0 \n" /* preset "0" to "res" */
+ " lwarx %1, 0, %2 \n" /* load from [lock] into "temp" */
+ /* and store reservation */
+ " cmpw %1, %3 \n" /* compare "temp" and "old" */
+ " bne- 1f \n" /* not equal */
+ " stwcx. %4, 0, %2 \n" /* store "set" into [lock] if reservation */
+ /* is not cleared */
+ " bne- 1f \n" /* the reservation was cleared */
+ " li %0, 1 \n" /* set "1" to "res" */
+ "1: \n"
+
+ : "=&r" (res), "=&r" (temp)
+ : "r" (lock), "r" (old), "r" (set)
+ : "cc", "memory");
+
+ return res;
}
+
#else
#define NGX_HAVE_ATOMIC_OPS 0
-typedef volatile uint32_t ngx_atomic_t;
+typedef uint32_t ngx_atomic_int_t;
+typedef volatile ngx_atomic_int_t ngx_atomic_t;
-#define ngx_atomic_inc(x) ++(*(x));
+#define ngx_atomic_inc(x) ++(*(x))
+#define ngx_atomic_dec(x) --(*(x))
-static ngx_inline uint32_t ngx_atomic_cmp_set(ngx_atomic_t *lock,
- ngx_atomic_t old,
- ngx_atomic_t set)
+static ngx_inline ngx_atomic_int_t
+ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_int_t old,
+ ngx_atomic_int_t set)
{
+ *lock = set;
return 1;
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-15 07:03:16 +0000