To support stack unwinding for cancellation points, add -fexceptions flag

for them, two functions _pthread_cancel_enter and _pthread_cancel_leave
are added to let thread enter and leave a cancellation point, it also
makes it possible that other functions can be cancellation points in
libraries without having to be rewritten in libthr.

lib/libc/Makefile

@@ -31,6 +31,11 @@ CLEANFILES+=tags
 INSTALL_PIC_ARCHIVE=
 PRECIOUSLIB=
 
+.ifndef NO_THREAD_STACK_UNWIND
+CANCELPOINTS_CFLAGS=-fexceptions
+CFLAGS+=${CANCELPOINTS_CFLAGS}
+.endif
+
 #
 # Only link with static libgcc.a (no libgcc_eh.a).
 #
@@ -145,3 +150,5 @@ CWARNFLAGS:=	${.IMPSRC:Ngdtoa_*.c:C/^.+$/${CWARNFLAGS}/:C/^$/-w/}
 SSP_CFLAGS:=	${SSP_CFLAGS:S/^-fstack-protector-all$/-fstack-protector/}
 # Disable stack protection for SSP symbols.
 SSP_CFLAGS:=	${.IMPSRC:N*/stack_protector.c:C/^.+$/${SSP_CFLAGS}/}
+# Generate stack unwinding tables for cancellation points
+CANCELPOINTS_CFLAGS:=	${.IMPSRC:Mcancelpoints_*:C/^.+$/${CANCELPOINTS_CFLAGS}/:C/^$//}

lib/libc/gen/Makefile.inc

@@ -25,7 +25,7 @@ SRCS+=  __getosreldate.c __xuname.c \
 	pause.c pmadvise.c popen.c posix_spawn.c \
 	psignal.c pututxline.c pw_scan.c pwcache.c \
 	raise.c readdir.c readpassphrase.c rewinddir.c \
-	scandir.c seed48.c seekdir.c sem.c sem_new.c semctl.c \
+	scandir.c seed48.c seekdir.c semctl.c \
 	setdomainname.c sethostname.c setjmperr.c setmode.c \
 	setproctitle.c setprogname.c siginterrupt.c siglist.c signal.c \
 	sigsetops.c sleep.c srand48.c statvfs.c stringlist.c strtofflags.c \
@@ -35,6 +35,14 @@ SRCS+=  __getosreldate.c __xuname.c \
 	usleep.c utime.c utxdb.c valloc.c vis.c wait.c wait3.c waitpid.c \
 	wordexp.c
 
+CANCELPOINTS_SRCS=sem.c sem_new.c
+.for src in ${CANCELPOINTS_SRCS}
+SRCS+=cancelpoints_${src}
+CLEANFILES+=cancelpoints_${src}
+cancelpoints_${src}:
+	ln -sf ${.CURDIR}/gen/${src} ${.TARGET}
+.endfor
+
 SYM_MAPS+=${.CURDIR}/gen/Symbol.map
 
 # machine-dependent gen sources

lib/libc/gen/Symbol.map

@@ -403,6 +403,8 @@ FBSDprivate_1.0 {
 	_pthread_attr_setstackaddr;
 	_pthread_attr_setstacksize;
 	_pthread_cancel;
+	_pthread_cancel_enter;
+	_pthread_cancel_leave;
 	_pthread_cleanup_pop;
 	_pthread_cleanup_push;
 	_pthread_cond_broadcast;

lib/libc/gen/_pthread_stubs.c

@@ -121,6 +121,8 @@ pthread_func_entry_t __thr_jtable[PJT_MAX] = {
 	{PJT_DUAL_ENTRY(stub_zero)},    /* PJT_TESTCANCEL */
 	{PJT_DUAL_ENTRY(stub_zero)},    /* PJT_CLEANUP_POP_IMP */
 	{PJT_DUAL_ENTRY(stub_zero)},    /* PJT_CLEANUP_PUSH_IMP */
+	{PJT_DUAL_ENTRY(stub_zero)},	/* PJT_CANCEL_ENTER */
+	{PJT_DUAL_ENTRY(stub_zero)},	/* PJT_CANCEL_LEAVE */
 };
 
 /*
@@ -269,6 +271,8 @@ STUB_FUNC2(pthread_setcanceltype, PJT_SETCANCELTYPE, int, int, void *)
 STUB_FUNC(pthread_testcancel, PJT_TESTCANCEL, void)
 STUB_FUNC1(__pthread_cleanup_pop_imp, PJT_CLEANUP_POP_IMP, int, int)
 STUB_FUNC2(__pthread_cleanup_push_imp, PJT_CLEANUP_PUSH_IMP, void, void*, void *);
+STUB_FUNC1(_pthread_cancel_enter, PJT_CANCEL_ENTER, int, int)
+STUB_FUNC1(_pthread_cancel_leave, PJT_CANCEL_LEAVE, int, int)
 
 static int
 stub_zero(void)

lib/libc/gen/sem.c

@@ -311,21 +311,6 @@ _libc_sem_unlink_compat(const char *name)
 	return (ksem_unlink(name));
 }
 
-static int
-enable_async_cancel(void)
-{
-	int old;
-
-	_pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old);
-	return (old);
-}
-
-static void
-restore_async_cancel(int val)
-{
-	_pthread_setcanceltype(val, NULL);
-}
-
 static int
 _umtx_wait_uint(volatile unsigned *mtx, unsigned id, const struct timespec *timeout)
 {
@@ -371,15 +356,15 @@ _libc_sem_timedwait_compat(sem_t * __restrict sem,
 	const struct timespec * __restrict abstime)
 {
 	struct timespec ts, ts2;
-	int val, retval, saved_cancel;
+	int val, retval;
 
 	if (sem_check_validity(sem) != 0)
 		return (-1);
 
 	if ((*sem)->syssem != 0) {
-		saved_cancel = enable_async_cancel();
-		retval = ksem_wait((*sem)->semid);
-		restore_async_cancel(saved_cancel);
+		_pthread_cancel_enter(1);
+		retval = ksem_wait((*sem)->semid); /* XXX no timeout */
+		_pthread_cancel_leave(retval == -1);
 		return (retval);
 	}
 
@@ -390,8 +375,10 @@ _libc_sem_timedwait_compat(sem_t * __restrict sem,
 			if (atomic_cmpset_acq_int(&(*sem)->count, val, val - 1))
 				return (0);
 		}
-		if (retval)
+		if (retval) {
+			_pthread_testcancel();
 			break;
+		}
 		if (abstime) {
 			if (abstime->tv_nsec >= 1000000000 || abstime->tv_nsec < 0) {
 				errno = EINVAL;
@@ -402,9 +389,9 @@ _libc_sem_timedwait_compat(sem_t * __restrict sem,
 		}
 		atomic_add_int(&(*sem)->nwaiters, 1);
 		pthread_cleanup_push(sem_cancel_handler, sem);
-		saved_cancel = enable_async_cancel();
+		_pthread_cancel_enter(1);
 		retval = _umtx_wait_uint(&(*sem)->count, 0, abstime ? &ts2 : NULL);
-		restore_async_cancel(saved_cancel);
+		_pthread_cancel_leave(0);
 		pthread_cleanup_pop(0);
 		atomic_add_int(&(*sem)->nwaiters, -1);
 	}

lib/libc/gen/sem_new.c

@@ -47,6 +47,7 @@
 #include <semaphore.h>
 #include <unistd.h>
 #include "un-namespace.h"
+#include "libc_private.h"
 
 __weak_reference(_sem_close, sem_close);
 __weak_reference(_sem_destroy, sem_destroy);
@@ -364,15 +365,6 @@ _sem_trywait(sem_t *sem)
 	return (-1);
 }
 
-static void
-sem_cancel_handler(void *arg)
-{
-	sem_t *sem = arg;
-
-	if (sem->_kern._has_waiters && sem->_kern._count)
-		usem_wake(&sem->_kern);
-}
-
 #define TIMESPEC_SUB(dst, src, val)                             \
         do {                                                    \
                 (dst)->tv_sec = (src)->tv_sec - (val)->tv_sec;  \
@@ -384,27 +376,12 @@ sem_cancel_handler(void *arg)
         } while (0)
 
 
-static __inline int
-enable_async_cancel(void)
-{
-	int old;
-
-	_pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old);
-	return (old);
-}
-
-static __inline void
-restore_async_cancel(int val)
-{
-	_pthread_setcanceltype(val, NULL);
-}
-
 int
 _sem_timedwait(sem_t * __restrict sem,
 	const struct timespec * __restrict abstime)
 {
 	struct timespec ts, ts2;
-	int val, retval, saved_cancel;
+	int val, retval;
 
 	if (sem_check_validity(sem) != 0)
 		return (-1);
@@ -416,8 +393,10 @@ _sem_timedwait(sem_t * __restrict sem,
 				return (0);
 		}
 
-		if (retval)
+		if (retval) {
+			_pthread_testcancel();
 			break;
+		}
 
 		/*
 		 * The timeout argument is only supposed to
@@ -431,11 +410,9 @@ _sem_timedwait(sem_t * __restrict sem,
 			clock_gettime(CLOCK_REALTIME, &ts);
 			TIMESPEC_SUB(&ts2, abstime, &ts);
 		}
-		pthread_cleanup_push(sem_cancel_handler, sem);
-		saved_cancel = enable_async_cancel();
+		_pthread_cancel_enter(1);
 		retval = usem_wait(&sem->_kern, abstime ? &ts2 : NULL);
-		restore_async_cancel(saved_cancel);
-		pthread_cleanup_pop(0);
+		_pthread_cancel_leave(0);
 	}
 	return (retval);
 }

lib/libc/include/libc_private.h

@@ -129,6 +129,8 @@ typedef enum {
 	PJT_TESTCANCEL,
 	PJT_CLEANUP_POP_IMP,
 	PJT_CLEANUP_PUSH_IMP,
+	PJT_CANCEL_ENTER,
+	PJT_CANCEL_LEAVE,
 	PJT_MAX
 } pjt_index_t;
 
@@ -217,4 +219,7 @@ int _elf_aux_info(int aux, void *buf, int buflen);
 struct dl_phdr_info;
 int __elf_phdr_match_addr(struct dl_phdr_info *, void *);
 
+void	_pthread_cancel_enter(int);
+void	_pthread_cancel_leave(int);
+
 #endif /* _LIBC_PRIVATE_H_ */

lib/librt/Makefile

@@ -3,14 +3,15 @@
 LIB=rt
 SHLIB_MAJOR= 1
 CFLAGS+=-I${.CURDIR}/../libc/include -I${.CURDIR}
+.ifndef NO_THREAD_STACK_UNWIND
+CFLAGS+=-fexceptions
+.endif
 CFLAGS+=-Winline -Wall -g
 DPADD=	${LIBPTHREAD}
 LDADD=	-lpthread
 
 WARNS?=	2
 
-#MAN=	libthr.3
-
 SRCS+= aio.c mq.c sigev_thread.c timer.c
 
 PRECIOUSLIB=

lib/librt/mq.c

@@ -39,6 +39,7 @@
 #include <signal.h>
 #include "sigev_thread.h"
 #include "un-namespace.h"
+#include "libc_private.h"
 
 extern int	__sys_kmq_notify(int, const struct sigevent *);
 extern int	__sys_kmq_open(const char *, int, mode_t,
@@ -200,12 +201,11 @@ ssize_t
 __mq_timedreceive_cancel(mqd_t mqd, char *buf, size_t len,
 	unsigned *prio, const struct timespec *timeout)
 {
-	int oldtype;
 	int ret;
 
-	_pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &oldtype);
+	_pthread_cancel_enter(1);
 	ret = __sys_kmq_timedreceive(mqd->oshandle, buf, len, prio, timeout);
-	_pthread_setcanceltype(oldtype, NULL);
+	_pthread_cancel_leave(ret == -1);
 	return (ret);
 }
 
@@ -219,12 +219,11 @@ __mq_receive(mqd_t mqd, char *buf, size_t len, unsigned *prio)
 ssize_t
 __mq_receive_cancel(mqd_t mqd, char *buf, size_t len, unsigned *prio)
 {
-	int oldtype;
 	int ret;
 
-	_pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &oldtype);
+	_pthread_cancel_enter(1);
 	ret = __sys_kmq_timedreceive(mqd->oshandle, buf, len, prio, NULL);
-	_pthread_setcanceltype(oldtype, NULL);
+	_pthread_cancel_leave(ret == -1);
 	return (ret);
 }
 ssize_t
@@ -239,12 +238,11 @@ ssize_t
 __mq_timedsend_cancel(mqd_t mqd, char *buf, size_t len,
 	unsigned prio, const struct timespec *timeout)
 {
-	int oldtype;
 	int ret;
 
-	_pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &oldtype);
+	_pthread_cancel_enter(1);
 	ret = __sys_kmq_timedsend(mqd->oshandle, buf, len, prio, timeout);
-	_pthread_setcanceltype(oldtype, NULL);
+	_pthread_cancel_leave(ret == -1);
 	return (ret);
 }
 
@@ -259,12 +257,11 @@ __mq_send(mqd_t mqd, char *buf, size_t len, unsigned prio)
 ssize_t
 __mq_send_cancel(mqd_t mqd, char *buf, size_t len, unsigned prio)
 {
-	int oldtype;
 	int ret;
 
-	_pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &oldtype);
+	_pthread_cancel_enter(1);
 	ret = __sys_kmq_timedsend(mqd->oshandle, buf, len, prio, NULL);
-	_pthread_setcanceltype(oldtype, NULL);
+	_pthread_cancel_leave(ret == -1);
 	return (ret);
 }
 

lib/libthr/Makefile

@@ -26,9 +26,7 @@ CFLAGS+=-I${.CURDIR}/../../libexec/rtld-elf/${MACHINE_CPUARCH}
 CFLAGS+=-I${.CURDIR}/../libthread_db
 CFLAGS+=-Winline
 
-LIBTHR_UNWIND_STACK=yes
-
-.ifdef LIBTHR_UNWIND_STACK
+.ifndef NO_THREAD_UNWIND_STACK
 CFLAGS+=-I${.CURDIR}/../../contrib/gcc -fexceptions 
 CFLAGS+=-D_PTHREAD_FORCED_UNWIND
 .endif

lib/libthr/pthread.map

@@ -246,6 +246,8 @@ FBSDprivate_1.0 {
 	_pthread_attr_setstackaddr;
 	_pthread_attr_setstacksize;
 	_pthread_cancel;
+	_pthread_cancel_enter;
+	_pthread_cancel_leave;
 	_pthread_cleanup_pop;
 	_pthread_cleanup_push;
 	_pthread_cond_broadcast;

lib/libthr/thread/thr_cancel.c

@@ -162,3 +162,15 @@ _thr_cancel_leave(struct pthread *curthread, int maycancel)
 	    !THR_IN_CRITICAL(curthread) && maycancel))
 		_pthread_exit(PTHREAD_CANCELED);
 }
+
+void
+_pthread_cancel_enter(int maycancel)
+{
+	_thr_cancel_enter2(_get_curthread(), maycancel);
+}
+
+void
+_pthread_cancel_leave(int maycancel)
+{
+	_thr_cancel_leave(_get_curthread(), maycancel);
+}

lib/libthr/thread/thr_init.c

@@ -248,7 +248,9 @@ static pthread_func_t jmp_table[][2] = {
 	{DUAL_ENTRY(_pthread_sigmask)},		/* PJT_SIGMASK */
 	{DUAL_ENTRY(_pthread_testcancel)},	/* PJT_TESTCANCEL */
 	{DUAL_ENTRY(__pthread_cleanup_pop_imp)},/* PJT_CLEANUP_POP_IMP */
-	{DUAL_ENTRY(__pthread_cleanup_push_imp)}/* PJT_CLEANUP_PUSH_IMP */
+	{DUAL_ENTRY(__pthread_cleanup_push_imp)},/* PJT_CLEANUP_PUSH_IMP */
+	{DUAL_ENTRY(_pthread_cancel_enter)},	/* PJT_CANCEL_ENTER */
+	{DUAL_ENTRY(_pthread_cancel_leave)}		/* PJT_CANCEL_LEAVE */
 };
 
 static int init_once = 0;

lib/libthr/thread/thr_private.h

@@ -716,7 +716,8 @@ int	_sched_yield(void);
 void	_pthread_cleanup_push(void (*)(void *), void *);
 void	_pthread_cleanup_pop(int);
 void	_pthread_exit_mask(void *status, sigset_t *mask) __dead2 __hidden;
-
+void	_pthread_cancel_enter(int maycancel);
+void 	_pthread_cancel_leave(int maycancel);
 
 /* #include <fcntl.h> */
 #ifdef  _SYS_FCNTL_H_