From: Rich Felker <dalias@aerifal.cx>
Date: Sun, 17 Apr 2011 15:43:03 +0000 (-0400)
Subject: overhaul pthread cancellation
X-Git-Tag: v0.7.9~39
X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=feee98903cd8119d9a3db62589246a940f44a9f5;p=oweals%2Fmusl.git

overhaul pthread cancellation

this patch improves the correctness, simplicity, and size of
cancellation-related code. modulo any small errors, it should now be
completely conformant, safe, and resource-leak free.

the notion of entering and exiting cancellation-point context has been
completely eliminated and replaced with alternative syscall assembly
code for cancellable syscalls. the assembly is responsible for setting
up execution context information (stack pointer and address of the
syscall instruction) which the cancellation signal handler can use to
determine whether the interrupted code was in a cancellable state.

these changes eliminate race conditions in the previous generation of
cancellation handling code (whereby a cancellation request received
just prior to the syscall would not be processed, leaving the syscall
to block, potentially indefinitely), and remedy an issue where
non-cancellable syscalls made from signal handlers became cancellable
if the signal handler interrupted a cancellation point.

x86_64 asm is untested and may need a second try to get it right.
---

diff --git a/arch/i386/bits/pthread.h b/arch/i386/bits/pthread.h
index 7690ea39..c119dc8a 100644
--- a/arch/i386/bits/pthread.h
+++ b/arch/i386/bits/pthread.h
@@ -7,17 +7,17 @@ struct __ptcb {
 
 static inline void __pthread_register_cancel_2(struct __ptcb *__cb)
 {
-	__asm__ __volatile__( "call __pthread_register_cancel" : : "a"(__cb) );
+	__asm__ __volatile__( "call __pthread_register_cancel" : : "a"(__cb) : "ecx", "edx", "memory" );
 }
 
 static inline void __pthread_unregister_cancel_2(struct __ptcb *__cb)
 {
-	__asm__ __volatile__( "call __pthread_unregister_cancel" : : "a"(__cb) );
+	__asm__ __volatile__( "call __pthread_unregister_cancel" : : "a"(__cb) : "ecx", "edx", "memory" );
 }
 
 static inline void __pthread_unwind_next_2(struct __ptcb *__cb)
 {
-	__asm__ __volatile__( "call __pthread_unwind_next" : : "a"(__cb) );
+	__asm__ __volatile__( "call __pthread_unwind_next" : : "a"(__cb) : "ecx", "edx", "memory" );
 }
 
 #define __pthread_register_cancel __pthread_register_cancel_2
diff --git a/arch/i386/bits/syscall.h b/arch/i386/bits/syscall.h
index 519e2dcd..274f205c 100644
--- a/arch/i386/bits/syscall.h
+++ b/arch/i386/bits/syscall.h
@@ -122,7 +122,9 @@ static inline long __syscall6(long __n, long __a1, long __a2, long __a3, long __
 #define __SC_sendmsg     16
 #define __SC_recvmsg     17
 
-#define __socketcall(nm, a, b, c, d, e, f) syscall(SYS_socketcall, __SC_##nm, \
+#define __socketcall(nm,a,b,c,d,e,f) syscall(SYS_socketcall, __SC_##nm, \
+    ((long [6]){ (long)a, (long)b, (long)c, (long)d, (long)e, (long)f }))
+#define __socketcall_cp(nm,a,b,c,d,e,f) syscall_cp(SYS_socketcall, __SC_##nm, \
     ((long [6]){ (long)a, (long)b, (long)c, (long)d, (long)e, (long)f }))
 
 #define __NR_restart_syscall      0
diff --git a/arch/i386/pthread_arch.h b/arch/i386/pthread_arch.h
index 64d75cbc..b17dc87a 100644
--- a/arch/i386/pthread_arch.h
+++ b/arch/i386/pthread_arch.h
@@ -5,5 +5,5 @@ static inline struct pthread *__pthread_self()
 	return self;
 }
 
-#define PC_AT_SYS(c) \
-	(*(uint16_t *)(((ucontext_t *)(c))->uc_mcontext.__gregs[14])==0x80cd)
+#define CANCEL_REG_SP 7
+#define CANCEL_REG_IP 14
diff --git a/arch/x86_64/bits/syscall.h b/arch/x86_64/bits/syscall.h
index d18edece..21d4c23a 100644
--- a/arch/x86_64/bits/syscall.h
+++ b/arch/x86_64/bits/syscall.h
@@ -60,7 +60,8 @@ static inline long __syscall6(long __n, long __a1, long __a2, long __a3, long __
 	return __ret;
 }
 
-#define __socketcall(nm, a, b, c, d, e, f) syscall(__NR_##nm, a, b, c, d, e, f)
+#define __socketcall(nm,a,b,c,d,e,f) syscall(__NR_##nm, a, b, c, d, e, f)
+#define __socketcall_cp(nm,a,b,c,d,e,f) syscall_cp(__NR_##nm, a, b, c, d, e, f)
 
 #define __NR_read				0
 #define __NR_write				1
diff --git a/arch/x86_64/pthread_arch.h b/arch/x86_64/pthread_arch.h
index af7ae86e..c424493a 100644
--- a/arch/x86_64/pthread_arch.h
+++ b/arch/x86_64/pthread_arch.h
@@ -5,5 +5,5 @@ static inline struct pthread *__pthread_self()
 	return self;
 }
 
-#define PC_AT_SYS(c) \
-	(*(uint16_t *)(((ucontext_t *)(c))->uc_mcontext.__gregs[16])==0x050f)
+#define CANCEL_REG_SP 15
+#define CANCEL_REG_IP 16
diff --git a/src/fcntl/fcntl.c b/src/fcntl/fcntl.c
index ab0ebecf..2c9fb6f3 100644
--- a/src/fcntl/fcntl.c
+++ b/src/fcntl/fcntl.c
@@ -6,17 +6,14 @@
 
 int fcntl(int fd, int cmd, ...)
 {
-	int r;
 	long arg;
 	va_list ap;
 	va_start(ap, cmd);
 	arg = va_arg(ap, long);
 	va_end(ap);
 	if (cmd == F_SETFL) arg |= O_LARGEFILE;
-	if (cmd == F_SETLKW) CANCELPT_BEGIN;
-	r = syscall(SYS_fcntl, fd, cmd, arg);
-	if (cmd == F_SETLKW) CANCELPT_END;
-	return r;
+	if (cmd == F_SETLKW) return syscall_cp(SYS_fcntl, fd, cmd, arg);
+	return syscall(SYS_fcntl, fd, cmd, arg);
 }
 
 LFS64(fcntl);
diff --git a/src/fcntl/open.c b/src/fcntl/open.c
index 064d298c..31d6744c 100644
--- a/src/fcntl/open.c
+++ b/src/fcntl/open.c
@@ -6,16 +6,12 @@
 
 int open(const char *filename, int flags, ...)
 {
-	int r;
 	mode_t mode;
 	va_list ap;
 	va_start(ap, flags);
 	mode = va_arg(ap, mode_t);
 	va_end(ap);
-	CANCELPT_BEGIN;
-	r = syscall(SYS_open, filename, flags|O_LARGEFILE, mode);
-	CANCELPT_END;
-	return r;
+	return syscall_cp(SYS_open, filename, flags|O_LARGEFILE, mode);
 }
 
 LFS64(open);
diff --git a/src/fcntl/openat.c b/src/fcntl/openat.c
index 1a2d9535..bdecb8c8 100644
--- a/src/fcntl/openat.c
+++ b/src/fcntl/openat.c
@@ -6,16 +6,12 @@
 
 int openat(int fd, const char *filename, int flags, ...)
 {
-	int r;
 	mode_t mode;
 	va_list ap;
 	va_start(ap, flags);
 	mode = va_arg(ap, mode_t);
 	va_end(ap);
-	CANCELPT_BEGIN;
-	r = syscall(SYS_openat, fd, filename, flags|O_LARGEFILE, mode);
-	CANCELPT_END;
-	return r;
+	return syscall_cp(SYS_openat, fd, filename, flags|O_LARGEFILE, mode);
 }
 
 LFS64(openat);
diff --git a/src/internal/libc.h b/src/internal/libc.h
index c0039e77..3f1e55e5 100644
--- a/src/internal/libc.h
+++ b/src/internal/libc.h
@@ -6,7 +6,7 @@
 
 struct __libc {
 	int *(*errno_location)(void);
-	void (*cancelpt)(int);
+	void (*testcancel)(void);
 	void (*lock)(volatile int *);
 	void (*lockfile)(FILE *);
 	void (*fork_handler)(int);
@@ -40,12 +40,6 @@ void __lock(volatile int *);
 void __lockfile(FILE *);
 #define LOCK(x) (libc.threads_minus_1 ? (__lock(x),1) : ((void)(x),1))
 #define UNLOCK(x) (*(x)=0)
-#define CANCELPT(x) (libc.cancelpt ? libc.cancelpt((x)),0 : (void)(x),0)
-#define CANCELPT_BEGIN CANCELPT(1)
-#define CANCELPT_TRY CANCELPT(0)
-#define CANCELPT_END CANCELPT(-1)
-#define CANCELPT_INHIBIT CANCELPT(2)
-#define CANCELPT_RESUME CANCELPT(-2)
 
 int __rsyscall(int, long, long, long, long, long, long);
 
diff --git a/src/internal/pthread_impl.h b/src/internal/pthread_impl.h
index a6d90e9b..304bf98d 100644
--- a/src/internal/pthread_impl.h
+++ b/src/internal/pthread_impl.h
@@ -24,7 +24,8 @@ struct pthread {
 	unsigned long tlsdesc[4];
 	pid_t tid, pid;
 	int tsd_used, errno_val, *errno_ptr;
-	volatile int canceldisable, cancelasync, cancelpoint, cancel;
+	volatile uintptr_t cp_sp, cp_ip;
+	volatile int cancel, canceldisable, cancelasync;
 	unsigned char *map_base;
 	size_t map_size;
 	void *start_arg;
@@ -85,6 +86,7 @@ void __lock(volatile int *);
 void __unmapself(void *, size_t);
 
 int __timedwait(volatile int *, int, clockid_t, const struct timespec *, int);
+int __timedwait_cp(volatile int *, int, clockid_t, const struct timespec *, int);
 void __wait(volatile int *, volatile int *, int, int);
 void __wake(volatile int *, int, int);
 
diff --git a/src/internal/syscall.h b/src/internal/syscall.h
index 39efc42d..0c71eca6 100644
--- a/src/internal/syscall.h
+++ b/src/internal/syscall.h
@@ -6,5 +6,19 @@
 #include <sys/syscall.h>
 
 #define socketcall __socketcall
+#define socketcall_cp __socketcall_cp
+
+#define __syscall_cp0(n) __syscall_cp(n,0,0,0,0,0,0)
+#define __syscall_cp1(n,a) __syscall_cp(n,(long)(a),0,0,0,0,0)
+#define __syscall_cp2(n,a,b) __syscall_cp(n,(long)(a),(long)(b),0,0,0,0)
+#define __syscall_cp3(n,a,b,c) __syscall_cp(n,(long)(a),(long)(b),(long)(c),0,0,0)
+#define __syscall_cp4(n,a,b,c,d) __syscall_cp(n,(long)(a),(long)(b),(long)(c),(long)(d),0,0)
+#define __syscall_cp5(n,a,b,c,d,e) __syscall_cp(n,(long)(a),(long)(b),(long)(c),(long)(d),(long)(e),0)
+#define __syscall_cp6(n,a,b,c,d,e,f) __syscall_cp(n,(long)(a),(long)(b),(long)(c),(long)(d),(long)(e),(long)(f))
+
+long (__syscall_cp)(long, long, long, long, long, long, long);
+
+#define __syscall_cp(...) __SYSCALL_DISP(__syscall_cp,__VA_ARGS__)
+#define syscall_cp(...) __syscall_ret(__syscall_cp(__VA_ARGS__))
 
 #endif
diff --git a/src/ipc/msgrcv.c b/src/ipc/msgrcv.c
index 13c282ee..ed4a448a 100644
--- a/src/ipc/msgrcv.c
+++ b/src/ipc/msgrcv.c
@@ -5,13 +5,9 @@
 
 ssize_t msgrcv(int q, void *m, size_t len, long type, int flag)
 {
-	ssize_t r;
-	CANCELPT_BEGIN;
 #ifdef SYS_msgrcv
-	r = syscall(SYS_msgrcv, q, m, len, type, flag);
+	return syscall_cp(SYS_msgrcv, q, m, len, type, flag);
 #else
-	r = syscall(SYS_ipc, IPCOP_msgrcv, q, len, flag, ((long[]){ (long)m, type }));
+	return syscall_cp(SYS_ipc, IPCOP_msgrcv, q, len, flag, ((long[]){ (long)m, type }));
 #endif
-	CANCELPT_END;
-	return r;
 }
diff --git a/src/ipc/msgsnd.c b/src/ipc/msgsnd.c
index 1e0b2fa5..23f4a4cb 100644
--- a/src/ipc/msgsnd.c
+++ b/src/ipc/msgsnd.c
@@ -5,13 +5,9 @@
 
 int msgsnd(int q, const void *m, size_t len, int flag)
 {
-	ssize_t r;
-	CANCELPT_BEGIN;
 #ifdef SYS_msgsnd
-	r = syscall(SYS_msgsnd, q, m, len, flag);
+	return syscall_cp(SYS_msgsnd, q, m, len, flag);
 #else
-	r = syscall(SYS_ipc, IPCOP_msgsnd, q, len, flag, m);
+	return syscall_cp(SYS_ipc, IPCOP_msgsnd, q, len, flag, m);
 #endif
-	CANCELPT_END;
-	return r;
 }
diff --git a/src/network/accept.c b/src/network/accept.c
index 46adff5c..f6b75ba4 100644
--- a/src/network/accept.c
+++ b/src/network/accept.c
@@ -4,9 +4,5 @@
 
 int accept(int fd, struct sockaddr *addr, socklen_t *len)
 {
-	int ret;
-	CANCELPT_BEGIN;
-	ret = socketcall(accept, fd, addr, len, 0, 0, 0);
-	CANCELPT_END;
-	return ret;
+	return socketcall_cp(accept, fd, addr, len, 0, 0, 0);
 }
diff --git a/src/network/connect.c b/src/network/connect.c
index 29bffbcb..57f01a1e 100644
--- a/src/network/connect.c
+++ b/src/network/connect.c
@@ -4,9 +4,5 @@
 
 int connect(int fd, const struct sockaddr *addr, socklen_t len)
 {
-	int ret;
-	CANCELPT_BEGIN;
-	ret = socketcall(connect, fd, addr, len, 0, 0, 0);
-	CANCELPT_END;
-	return ret;
+	return socketcall_cp(connect, fd, addr, len, 0, 0, 0);
 }
diff --git a/src/network/recvfrom.c b/src/network/recvfrom.c
index d5222932..035a15f8 100644
--- a/src/network/recvfrom.c
+++ b/src/network/recvfrom.c
@@ -4,9 +4,5 @@
 
 ssize_t recvfrom(int fd, void *buf, size_t len, int flags, struct sockaddr *addr, socklen_t *alen)
 {
-	ssize_t r;
-	CANCELPT_BEGIN;
-	r = socketcall(recvfrom, fd, buf, len, flags, addr, alen);
-	CANCELPT_END;
-	return r;
+	return socketcall_cp(recvfrom, fd, buf, len, flags, addr, alen);
 }
diff --git a/src/network/recvmsg.c b/src/network/recvmsg.c
index 65094fc8..4f526659 100644
--- a/src/network/recvmsg.c
+++ b/src/network/recvmsg.c
@@ -14,9 +14,7 @@ ssize_t recvmsg(int fd, struct msghdr *msg, int flags)
 		msg = &h;
 	}
 #endif
-	CANCELPT_BEGIN;
-	r = socketcall(recvmsg, fd, msg, flags, 0, 0, 0);
-	CANCELPT_END;
+	r = socketcall_cp(recvmsg, fd, msg, flags, 0, 0, 0);
 #if LONG_MAX > INT_MAX
 	if (orig) *orig = h;
 #endif
diff --git a/src/network/sendmsg.c b/src/network/sendmsg.c
index 047c0eff..164c28d7 100644
--- a/src/network/sendmsg.c
+++ b/src/network/sendmsg.c
@@ -5,7 +5,6 @@
 
 ssize_t sendmsg(int fd, const struct msghdr *msg, int flags)
 {
-	ssize_t r;
 #if LONG_MAX > INT_MAX
 	struct msghdr h;
 	if (msg) {
@@ -14,8 +13,5 @@ ssize_t sendmsg(int fd, const struct msghdr *msg, int flags)
 		msg = &h;
 	}
 #endif
-	CANCELPT_BEGIN;
-	r = socketcall(sendmsg, fd, msg, flags, 0, 0, 0);
-	CANCELPT_END;
-	return r;
+	return socketcall_cp(sendmsg, fd, msg, flags, 0, 0, 0);
 }
diff --git a/src/network/sendto.c b/src/network/sendto.c
index 1cfc621d..899eecff 100644
--- a/src/network/sendto.c
+++ b/src/network/sendto.c
@@ -4,9 +4,5 @@
 
 ssize_t sendto(int fd, const void *buf, size_t len, int flags, const struct sockaddr *addr, socklen_t alen)
 {
-	ssize_t r;
-	CANCELPT_BEGIN;
-	r = socketcall(sendto, fd, buf, len, flags, addr, alen);
-	CANCELPT_END;
-	return r;
+	return socketcall_cp(sendto, fd, buf, len, flags, addr, alen);
 }
diff --git a/src/process/waitid.c b/src/process/waitid.c
index 4fa7c024..c67feac3 100644
--- a/src/process/waitid.c
+++ b/src/process/waitid.c
@@ -4,10 +4,5 @@
 
 int waitid(idtype_t type, id_t id, siginfo_t *info, int options)
 {
-	int r;
-	CANCELPT_BEGIN;
-	r = syscall(SYS_waitid, type, id, info, options, 0);
-	if (r<0) CANCELPT_TRY;
-	CANCELPT_END;
-	return r;
+	return syscall_cp(SYS_waitid, type, id, info, options, 0);
 }
diff --git a/src/process/waitpid.c b/src/process/waitpid.c
index 5e0320f7..f75e31ef 100644
--- a/src/process/waitpid.c
+++ b/src/process/waitpid.c
@@ -4,10 +4,5 @@
 
 pid_t waitpid(pid_t pid, int *status, int options)
 {
-	int r;
-	CANCELPT_BEGIN;
-	r = syscall(SYS_wait4, pid, status, options, 0);
-	if (r<0) CANCELPT_TRY;
-	CANCELPT_END;
-	return r;
+	return syscall_cp(SYS_wait4, pid, status, options, 0);
 }
diff --git a/src/select/poll.c b/src/select/poll.c
index caceebaf..f1e73e82 100644
--- a/src/select/poll.c
+++ b/src/select/poll.c
@@ -4,9 +4,5 @@
 
 int poll(struct pollfd *fds, nfds_t n, int timeout)
 {
-	int r;
-	CANCELPT_BEGIN;
-	r = syscall(SYS_poll, fds, n, timeout);
-	CANCELPT_END;
-	return r;
+	return syscall_cp(SYS_poll, fds, n, timeout);
 }
diff --git a/src/select/pselect.c b/src/select/pselect.c
index 155a6eb0..f28887ff 100644
--- a/src/select/pselect.c
+++ b/src/select/pselect.c
@@ -4,13 +4,8 @@
 
 int pselect(int n, fd_set *rfds, fd_set *wfds, fd_set *efds, const struct timespec *ts, const sigset_t *mask)
 {
-	int r;
 	long data[2] = { (long)mask, 8 };
 	struct timespec ts_tmp;
 	if (ts) ts_tmp = *ts;
-	CANCELPT_BEGIN;
-	r = syscall(SYS_pselect6, n, rfds, wfds, efds, ts ? &ts_tmp : 0, data);
-	CANCELPT_TRY;
-	CANCELPT_END;
-	return r;
+	return syscall_cp(SYS_pselect6, n, rfds, wfds, efds, ts ? &ts_tmp : 0, data);
 }
diff --git a/src/select/select.c b/src/select/select.c
index b38e7fd2..696cb288 100644
--- a/src/select/select.c
+++ b/src/select/select.c
@@ -4,10 +4,5 @@
 
 int select(int n, fd_set *rfds, fd_set *wfds, fd_set *efds, struct timeval *tv)
 {
-	int r;
-	CANCELPT_BEGIN;
-	r = syscall(SYS_select, n, rfds, wfds, efds, tv);
-	CANCELPT_TRY;
-	CANCELPT_END;
-	return r;
+	return syscall_cp(SYS_select, n, rfds, wfds, efds, tv);
 }
diff --git a/src/signal/sigsuspend.c b/src/signal/sigsuspend.c
index cec5ddce..cd3a7b59 100644
--- a/src/signal/sigsuspend.c
+++ b/src/signal/sigsuspend.c
@@ -4,10 +4,5 @@
 
 int sigsuspend(const sigset_t *mask)
 {
-	int ret;
-	CANCELPT_BEGIN;
-	ret = syscall(SYS_rt_sigsuspend, mask, 8);
-	if (ret<0) CANCELPT_TRY;
-	CANCELPT_END;
-	return ret;
+	return syscall_cp(SYS_rt_sigsuspend, mask, 8);
 }
diff --git a/src/signal/sigtimedwait.c b/src/signal/sigtimedwait.c
index 1694cbe1..7eea58ab 100644
--- a/src/signal/sigtimedwait.c
+++ b/src/signal/sigtimedwait.c
@@ -6,11 +6,7 @@
 int sigtimedwait(const sigset_t *mask, siginfo_t *si, const struct timespec *timeout)
 {
 	int ret;
-	CANCELPT_BEGIN;
-	do {
-		ret = syscall(SYS_rt_sigtimedwait, mask, si, timeout, 8);
-		if (ret<0) CANCELPT_TRY;
-	} while (ret<0 && errno==EINTR);
-	CANCELPT_END;
+	do ret = syscall_cp(SYS_rt_sigtimedwait, mask, si, timeout, 8);
+	while (ret<0 && errno==EINTR);
 	return ret;
 }
diff --git a/src/termios/tcdrain.c b/src/termios/tcdrain.c
index 9ff1ecd5..6e43afb7 100644
--- a/src/termios/tcdrain.c
+++ b/src/termios/tcdrain.c
@@ -1,13 +1,9 @@
 #include <termios.h>
 #include <sys/ioctl.h>
 #include "libc.h"
+#include "syscall.h"
 
 int tcdrain(int fd)
 {
-	int ret;
-	CANCELPT_BEGIN;
-	ret = ioctl(fd, TCSBRK, 1);
-	CANCELPT_TRY;
-	CANCELPT_END;
-	return ret;
+	return syscall_cp(SYS_ioctl, fd, TCSBRK, 1);
 }
diff --git a/src/thread/__timedwait.c b/src/thread/__timedwait.c
index 5c84e80c..b1d3af23 100644
--- a/src/thread/__timedwait.c
+++ b/src/thread/__timedwait.c
@@ -2,7 +2,7 @@
 #include <errno.h>
 #include "futex.h"
 #include "syscall.h"
-#include <stdio.h>
+
 int __timedwait(volatile int *addr, int val, clockid_t clk, const struct timespec *at, int priv)
 {
 	int r;
diff --git a/src/thread/__timedwait_cp.c b/src/thread/__timedwait_cp.c
new file mode 100644
index 00000000..c2890985
--- /dev/null
+++ b/src/thread/__timedwait_cp.c
@@ -0,0 +1,23 @@
+#include <time.h>
+#include <errno.h>
+#include "futex.h"
+#include "syscall.h"
+
+int __timedwait_cp(volatile int *addr, int val, clockid_t clk, const struct timespec *at, int priv)
+{
+	int r;
+	struct timespec to;
+	if (at) {
+		clock_gettime(clk, &to);
+		to.tv_sec = at->tv_sec - to.tv_sec;
+		if ((to.tv_nsec = at->tv_nsec - to.tv_nsec) < 0) {
+			to.tv_sec--;
+			to.tv_nsec += 1000000000;
+		}
+		if (to.tv_sec < 0) return ETIMEDOUT;
+	}
+	if (priv) priv = 128; priv=0;
+	r = -__syscall_cp(SYS_futex, (long)addr, FUTEX_WAIT | priv, val, at ? (long)&to : 0);
+	if (r == ETIMEDOUT || r == EINTR) return r;
+	return 0;
+}
diff --git a/src/thread/cancel_dummy.c b/src/thread/cancel_dummy.c
new file mode 100644
index 00000000..a39117e7
--- /dev/null
+++ b/src/thread/cancel_dummy.c
@@ -0,0 +1,8 @@
+#include "pthread_impl.h"
+
+static long sccp(long nr, long u, long v, long w, long x, long y, long z)
+{
+	return (__syscall)(nr, u, v, w, x, y, z);
+}
+
+weak_alias(sccp, __syscall_cp);
diff --git a/src/thread/cancel_impl.c b/src/thread/cancel_impl.c
new file mode 100644
index 00000000..5ce545d7
--- /dev/null
+++ b/src/thread/cancel_impl.c
@@ -0,0 +1,70 @@
+#include "pthread_impl.h"
+
+long __syscall_cp_asm(volatile void *, long, long, long, long, long, long, long);
+
+long (__syscall_cp)(long nr, long u, long v, long w, long x, long y, long z)
+{
+	pthread_t self;
+	uintptr_t old_sp, old_ip;
+	long r;
+
+	if (!libc.lock || (self = __pthread_self())->canceldisable)
+		return __syscall(nr, u, v, w, x, y, z);
+
+	old_sp = self->cp_sp;
+	old_ip = self->cp_ip;
+	self->cp_sp = 0;
+	self->cp_ip = 0;
+	r = __syscall_cp_asm(&self->cp_sp, nr, u, v, w, x, y, z);
+	self->cp_sp = old_sp;
+	self->cp_ip = old_ip;
+	if (r == -EINTR && self->cancel) pthread_exit(PTHREAD_CANCELED);
+	return r;
+}
+
+static void cancel_handler(int sig, siginfo_t *si, void *ctx)
+{
+	pthread_t self = __pthread_self();
+	ucontext_t *uc = ctx;
+	uintptr_t sp = ((uintptr_t *)&uc->uc_mcontext)[CANCEL_REG_SP];
+	uintptr_t ip = ((uintptr_t *)&uc->uc_mcontext)[CANCEL_REG_IP];
+
+	if (!self->cancel || self->canceldisable) return;
+
+	if (self->cancelasync) pthread_exit(PTHREAD_CANCELED);
+
+	if (sp != self->cp_sp) {
+		if (!sp) return;
+		sigaddset(&uc->uc_sigmask, SIGCANCEL);
+		__syscall(SYS_tgkill, self->pid, self->tid, SIGCANCEL);
+		return;
+	}
+
+	if (ip <= self->cp_ip) pthread_exit(PTHREAD_CANCELED);
+}
+
+static void testcancel()
+{
+	pthread_t self = __pthread_self();
+	if (self->cancel && !self->canceldisable)
+		pthread_exit(PTHREAD_CANCELED);
+}
+
+static void init_cancellation()
+{
+	struct sigaction sa = {
+		.sa_flags = SA_SIGINFO | SA_RESTART,
+		.sa_sigaction = cancel_handler
+	};
+	sigfillset(&sa.sa_mask);
+	__libc_sigaction(SIGCANCEL, &sa, 0);
+	libc.testcancel = testcancel;
+}
+
+int pthread_cancel(pthread_t t)
+{
+	static pthread_once_t once;
+	pthread_once(&once, init_cancellation);
+	a_store(&t->cancel, 1);
+	return pthread_kill(t, SIGCANCEL);
+}
diff --git a/src/thread/i386/syscall_cp.s b/src/thread/i386/syscall_cp.s
new file mode 100644
index 00000000..6f98a779
--- /dev/null
+++ b/src/thread/i386/syscall_cp.s
@@ -0,0 +1,36 @@
+.text
+.global __syscall_cp_asm
+.type   __syscall_cp_asm,%function
+__syscall_cp_asm:
+	pushl %ebx
+	pushl %esi
+	pushl %edi
+	pushl %ebp
+	leal 20(%esp),%ebp
+	call 1f
+1:	popl %eax
+	movl (%ebp),%ecx
+	addl $[1f-1b],%eax
+	movl %eax,4(%ecx)
+	movl %esp,(%ecx)
+	movl 8(%ecx),%eax
+	testl %eax,%eax
+	jnz 2f
+	movl 4(%ebp),%eax
+	movl 8(%ebp),%ebx
+	movl 12(%ebp),%ecx
+	movl 16(%ebp),%edx
+	movl 20(%ebp),%esi
+	movl 24(%ebp),%edi
+	movl 28(%ebp),%ebp
+1:	int $128
+	popl %ebp
+	popl %edi
+	popl %esi
+	popl %ebx
+	ret
+2:	xorl %eax,%eax
+	movl %eax,4(%ecx)
+	movl %eax,(%ecx)
+	pushl $-1
+	call pthread_exit
diff --git a/src/thread/pthread_cancel.c b/src/thread/pthread_cancel.c
deleted file mode 100644
index c497dbe6..00000000
--- a/src/thread/pthread_cancel.c
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "pthread_impl.h"
-
-int pthread_cancel(pthread_t t)
-{
-	a_store(&t->cancel, 1);
-	return pthread_kill(t, SIGCANCEL);
-}
diff --git a/src/thread/pthread_cond_timedwait.c b/src/thread/pthread_cond_timedwait.c
index 7a19fc55..13728282 100644
--- a/src/thread/pthread_cond_timedwait.c
+++ b/src/thread/pthread_cond_timedwait.c
@@ -8,22 +8,19 @@ static void relock(void *m)
 int pthread_cond_timedwait(pthread_cond_t *c, pthread_mutex_t *m, const struct timespec *ts)
 {
 	int r, e=0;
-	CANCELPT_BEGIN;
-	CANCELPT_END;
+
+	pthread_testcancel();
 
 	pthread_cleanup_push(relock, m);
 	c->_c_block = 1;
 	if ((r=pthread_mutex_unlock(m))) return r;
 
-	CANCELPT_BEGIN;
-	do e = __timedwait(&c->_c_block, 1, c->_c_clock, ts, 0);
+	do e = __timedwait_cp(&c->_c_block, 1, c->_c_clock, ts, 0);
 	while (e == EINTR);
-	CANCELPT_END;
 
 	pthread_cleanup_pop(0);
 	if ((r=pthread_mutex_lock(m))) return r;
 
-	CANCELPT_BEGIN;
-	CANCELPT_END;
+	pthread_testcancel();
 	return e;
 }
diff --git a/src/thread/pthread_create.c b/src/thread/pthread_create.c
index a722a2d6..844d7bf9 100644
--- a/src/thread/pthread_create.c
+++ b/src/thread/pthread_create.c
@@ -19,6 +19,7 @@ weak_alias(dummy_1, __pthread_tsd_run_dtors);
 void __pthread_unwind_next(struct __ptcb *cb)
 {
 	pthread_t self;
+	int n;
 
 	if (cb->__next) longjmp((void *)cb->__next->__jb, 1);
 
@@ -31,8 +32,9 @@ void __pthread_unwind_next(struct __ptcb *cb)
 	/* Mark this thread dead before decrementing count */
 	self->dead = 1;
 
-	if (!a_fetch_add(&libc.threads_minus_1, -1))
-		exit(0);
+	do n = libc.threads_minus_1;
+	while (n && a_cas(&libc.threads_minus_1, n, n-1)!=n);
+	if (!n) exit(0);
 
 	if (self->detached && self->map_base) {
 		__syscall(SYS_rt_sigprocmask, SIG_BLOCK, (long)(uint64_t[1]){-1},0,8);
@@ -42,43 +44,16 @@ void __pthread_unwind_next(struct __ptcb *cb)
 	__syscall(SYS_exit, 0);
 }
 
-static void docancel(struct pthread *self)
-{
-	struct __ptcb cb = { .__next = self->cancelbuf };
-	self->canceldisable = 1;
-	self->cancelasync = 0;
-	__pthread_unwind_next(&cb);
-}
-
-static void cancel_handler(int sig, siginfo_t *si, void *ctx)
-{
-	struct pthread *self = __pthread_self();
-	if (self->cancel && !self->canceldisable &&
-	    (self->cancelasync || (self->cancelpoint==1 && PC_AT_SYS(ctx))))
-		docancel(self);
-}
-
-static void cancelpt(int x)
-{
-	struct pthread *self = __pthread_self();
-	if ((self->cancelpoint+=x)==1 && self->cancel
-		&& x<2U && !self->canceldisable) docancel(self);
-}
-
 static void init_threads()
 {
-	struct sigaction sa = { .sa_flags = SA_SIGINFO | SA_RESTART };
+	sigset_t set;
 	libc.lock = __lock;
 	libc.lockfile = __lockfile;
-	libc.cancelpt = cancelpt;
-
-	sigemptyset(&sa.sa_mask);
-	sa.sa_sigaction = cancel_handler;
-	__libc_sigaction(SIGCANCEL, &sa, 0);
 
-	sigaddset(&sa.sa_mask, SIGSYSCALL);
-	sigaddset(&sa.sa_mask, SIGCANCEL);
-	__libc_sigprocmask(SIG_UNBLOCK, &sa.sa_mask, 0);
+	sigemptyset(&set);
+	sigaddset(&set, SIGSYSCALL);
+	sigaddset(&set, SIGCANCEL);
+	__libc_sigprocmask(SIG_UNBLOCK, &set, 0);
 }
 
 static int start(void *p)
@@ -159,6 +134,9 @@ int pthread_create(pthread_t *res, const pthread_attr_t *attr, void *(*entry)(vo
 void pthread_exit(void *result)
 {
 	struct pthread *self = pthread_self();
+	struct __ptcb cb = { .__next = self->cancelbuf };
 	self->result = result;
-	docancel(self);
+	self->canceldisable = 1;
+	self->cancelasync = 0;
+	__pthread_unwind_next(&cb);
 }
diff --git a/src/thread/pthread_join.c b/src/thread/pthread_join.c
index 5210ed48..ba7bb7d5 100644
--- a/src/thread/pthread_join.c
+++ b/src/thread/pthread_join.c
@@ -3,9 +3,7 @@
 int pthread_join(pthread_t t, void **res)
 {
 	int tmp = t->tid;
-	CANCELPT_BEGIN;
-	if (tmp) __wait(&t->tid, 0, tmp, 1);
-	CANCELPT_END;
+	if (tmp) __timedwait_cp(&t->tid, tmp, 0, 0, 1);
 	if (res) *res = t->result;
 	if (t->map_base) munmap(t->map_base, t->map_size);
 	return 0;
diff --git a/src/thread/pthread_testcancel.c b/src/thread/pthread_testcancel.c
index 774b7068..c6b250b2 100644
--- a/src/thread/pthread_testcancel.c
+++ b/src/thread/pthread_testcancel.c
@@ -2,6 +2,5 @@
 
 void pthread_testcancel()
 {
-	CANCELPT_BEGIN;
-	CANCELPT_END;
+	if (libc.testcancel) libc.testcancel();
 }
diff --git a/src/thread/sem_timedwait.c b/src/thread/sem_timedwait.c
index 4f45c172..db05b417 100644
--- a/src/thread/sem_timedwait.c
+++ b/src/thread/sem_timedwait.c
@@ -21,19 +21,16 @@ int sem_timedwait(sem_t *sem, const struct timespec *at)
 	a_inc(sem->__val+1);
 	pthread_cleanup_push(cleanup, sem->__val+1)
 
-	CANCELPT_BEGIN;
 	for (;;) {
 		r = 0;
 		if (!sem_trywait(sem)) break;
-		r = __timedwait(sem->__val, 0, CLOCK_REALTIME, at, 0);
+		r = __timedwait_cp(sem->__val, 0, CLOCK_REALTIME, at, 0);
 		if (r) {
 			errno = r;
 			r = -1;
 			break;
 		}
-		CANCELPT_TRY;
 	}
-	CANCELPT_END;
 
 	pthread_cleanup_pop(1);
 
diff --git a/src/thread/syscall_cp.c b/src/thread/syscall_cp.c
new file mode 100644
index 00000000..e69de29b
diff --git a/src/thread/x86_64/syscall_cp.s b/src/thread/x86_64/syscall_cp.s
new file mode 100644
index 00000000..1894ce19
--- /dev/null
+++ b/src/thread/x86_64/syscall_cp.s
@@ -0,0 +1,24 @@
+.text
+.global __syscall_cp_asm
+.type   __syscall_cp_asm,%function
+__syscall_cp_asm:
+	lea 1f(%rip),%rax
+	mov %rax,8(%rdi)
+	mov %rsp,(%rdi)
+	mov 16(%rdi),%eax
+	test %eax,%eax
+	jnz 2f
+	mov %rsi,%rax
+	mov %rdx,%rdi
+	mov %rcx,%rsi
+	mov %r8,%rdx
+	mov %r9,%r10
+	mov 8(%rsp),%r8
+	mov 16(%rsp),%r9
+1:	syscall
+	ret
+2:	xor %edi,%edi
+	mov %rdi,8(%r10)
+	mov %rdi,(%r10)
+	dec %rdi
+	jmp pthread_exit
diff --git a/src/time/clock_nanosleep.c b/src/time/clock_nanosleep.c
index 4667725b..ec87b9e3 100644
--- a/src/time/clock_nanosleep.c
+++ b/src/time/clock_nanosleep.c
@@ -4,9 +4,5 @@
 
 int clock_nanosleep(clockid_t clk, int flags, const struct timespec *req, struct timespec *rem)
 {
-	int ret;
-	CANCELPT_BEGIN;
-	ret = -__syscall(SYS_clock_nanosleep, clk, flags, req, rem);
-	CANCELPT_END;
-	return ret;
+	return -__syscall_cp(SYS_clock_nanosleep, clk, flags, req, rem);
 }
diff --git a/src/time/nanosleep.c b/src/time/nanosleep.c
index 0e0753f3..c8878b11 100644
--- a/src/time/nanosleep.c
+++ b/src/time/nanosleep.c
@@ -5,10 +5,5 @@
 
 int nanosleep(const struct timespec *req, struct timespec *rem)
 {
-	int ret;
-	CANCELPT_BEGIN;
-	ret = syscall(SYS_nanosleep, req, rem);
-	CANCELPT_TRY;
-	CANCELPT_END;
-	return ret;
+	return syscall_cp(SYS_nanosleep, req, rem);
 }
diff --git a/src/unistd/close.c b/src/unistd/close.c
index f52c0ef3..231f79ef 100644
--- a/src/unistd/close.c
+++ b/src/unistd/close.c
@@ -4,8 +4,7 @@
 
 int close(int fd)
 {
-	int ret = syscall(SYS_close, fd);
-	CANCELPT_BEGIN;
-	CANCELPT_END;
+	int ret = syscall_cp(SYS_close, fd);
+	if (libc.testcancel) libc.testcancel();
 	return ret;
 }
diff --git a/src/unistd/pause.c b/src/unistd/pause.c
index 57ed25e5..f7ed17d1 100644
--- a/src/unistd/pause.c
+++ b/src/unistd/pause.c
@@ -4,9 +4,5 @@
 
 int pause(void)
 {
-	int r;
-	CANCELPT_BEGIN;
-	r = syscall(SYS_pause);
-	CANCELPT_END;
-	return r;
+	return syscall_cp(SYS_pause);
 }
diff --git a/src/unistd/pread.c b/src/unistd/pread.c
index 0a045013..1bf0c754 100644
--- a/src/unistd/pread.c
+++ b/src/unistd/pread.c
@@ -4,11 +4,7 @@
 
 ssize_t pread(int fd, void *buf, size_t size, off_t ofs)
 {
-	ssize_t r;
-	CANCELPT_BEGIN;
-	r = syscall(SYS_pread, fd, buf, size, __SYSCALL_LL(ofs));
-	CANCELPT_END;
-	return r;
+	return syscall_cp(SYS_pread, fd, buf, size, __SYSCALL_LL(ofs));
 }
 
 LFS64(pread);
diff --git a/src/unistd/pwrite.c b/src/unistd/pwrite.c
index f878bb63..224eacdd 100644
--- a/src/unistd/pwrite.c
+++ b/src/unistd/pwrite.c
@@ -4,11 +4,7 @@
 
 ssize_t pwrite(int fd, const void *buf, size_t size, off_t ofs)
 {
-	ssize_t r;
-	CANCELPT_BEGIN;
-	r = syscall(SYS_pwrite, fd, buf, size, __SYSCALL_LL(ofs));
-	CANCELPT_END;
-	return r;
+	return syscall_cp(SYS_pwrite, fd, buf, size, __SYSCALL_LL(ofs));
 }
 
 LFS64(pwrite);
diff --git a/src/unistd/read.c b/src/unistd/read.c
index 194b389e..eb882fcc 100644
--- a/src/unistd/read.c
+++ b/src/unistd/read.c
@@ -4,9 +4,5 @@
 
 ssize_t read(int fd, void *buf, size_t count)
 {
-	ssize_t r;
-	CANCELPT_BEGIN;
-	r = syscall(SYS_read, fd, buf, count);
-	CANCELPT_END;
-	return r;
+	return syscall_cp(SYS_read, fd, buf, count);
 }
diff --git a/src/unistd/readv.c b/src/unistd/readv.c
index 9b87728e..e45cb484 100644
--- a/src/unistd/readv.c
+++ b/src/unistd/readv.c
@@ -4,9 +4,5 @@
 
 ssize_t readv(int fd, const struct iovec *iov, int count)
 {
-	ssize_t r;
-	CANCELPT_BEGIN;
-	r = syscall(SYS_readv, fd, iov, count);
-	CANCELPT_END;
-	return r;
+	return syscall_cp(SYS_readv, fd, iov, count);
 }
diff --git a/src/unistd/write.c b/src/unistd/write.c
index a8284b32..e2f7e1f2 100644
--- a/src/unistd/write.c
+++ b/src/unistd/write.c
@@ -4,9 +4,5 @@
 
 ssize_t write(int fd, const void *buf, size_t count)
 {
-	int r;
-	CANCELPT_BEGIN;
-	r = syscall(SYS_write, fd, buf, count);
-	CANCELPT_END;
-	return r;
+	return syscall_cp(SYS_write, fd, buf, count);
 }
diff --git a/src/unistd/writev.c b/src/unistd/writev.c
index a45afeb7..ef300ddf 100644
--- a/src/unistd/writev.c
+++ b/src/unistd/writev.c
@@ -4,9 +4,5 @@
 
 ssize_t writev(int fd, const struct iovec *iov, int count)
 {
-	ssize_t r;
-	CANCELPT_BEGIN;
-	r = syscall(SYS_writev, fd, iov, count);
-	CANCELPT_END;
-	return r;
+	return syscall_cp(SYS_writev, fd, iov, count);
 }