mitigate performance regression in libc-internal locks on x86_64

commit 3c43c0761e1725fd5f89a9c028cbf43250abb913 fixed missing
synchronization in the atomic store operation for i386 and x86_64, but
opted to use mfence for the barrier on x86_64 where it's always
available. however, in practice mfence is significantly slower than
the barrier approach used on i386 (a nop-like lock orl operation).
this commit changes x86_64 (and x32) to use the faster barrier.

diff --git a/arch/x32/atomic.h b/arch/x32/atomic.h
index 2ab1f7a26241a20179051923809d02b6b545af54..7690183d010c7f073a80ad1f1848ded488dfc593 100644 (file)
--- a/arch/x32/atomic.h
+++ b/arch/x32/atomic.h
@@ -83,7 +83,7 @@ static inline void a_dec(volatile int *x)
 
 static inline void a_store(volatile int *p, int x)
 {
-       __asm__( "mov %1, %0 ; mfence" : "=m"(*p) : "r"(x) : "memory" );
+       __asm__( "mov %1, %0 ; lock ; orl $0,(%%rsp)" : "=m"(*p) : "r"(x) : "memory" );
 }
 
 static inline void a_spin()

diff --git a/arch/x86_64/atomic.h b/arch/x86_64/atomic.h
index 2ab1f7a26241a20179051923809d02b6b545af54..7690183d010c7f073a80ad1f1848ded488dfc593 100644 (file)
--- a/arch/x86_64/atomic.h
+++ b/arch/x86_64/atomic.h
@@ -83,7 +83,7 @@ static inline void a_dec(volatile int *x)
 
 static inline void a_store(volatile int *p, int x)
 {
-       __asm__( "mov %1, %0 ; mfence" : "=m"(*p) : "r"(x) : "memory" );
+       __asm__( "mov %1, %0 ; lock ; orl $0,(%%rsp)" : "=m"(*p) : "r"(x) : "memory" );
 }
 
 static inline void a_spin()