Add build system support for ASAN+UBSAN instrumentation

This adds two new options WITH_ASAN/WITH_UBSAN that can be set to
enable instrumentation of all binaries with AddressSanitizer and/or
UndefinedBehaviourSanitizer. This current patch is almost sufficient
to get a complete buildworld with sanitizer instrumentation but in
order to actually build and boot a system it depends on a few more
follow-up commits.

Reviewed By:	brooks, kib, markj
Differential Revision: https://reviews.freebsd.org/D31043

Makefile.inc1

@@ -718,6 +718,7 @@ BSARGS= 	DESTDIR= \
 		MK_HTML=no NO_LINT=yes MK_MAN=no MK_MAN_UTILS=yes \
 		-DNO_PIC MK_PROFILE=no -DNO_SHARED \
 		-DNO_CPU_CFLAGS MK_WERROR=no MK_CTF=no \
+		MK_ASAN=no MK_UBSAN=no \
 		MK_CLANG_EXTRAS=no MK_CLANG_FORMAT=no MK_CLANG_FULL=no \
 		MK_LLDB=no MK_RETPOLINE=no MK_TESTS=no \
 		MK_INCLUDES=yes
@@ -739,6 +740,7 @@ TMAKE=		\
 		SSP_CFLAGS= \
 		-DNO_LINT \
 		-DNO_CPU_CFLAGS MK_WERROR=no MK_CTF=no \
+		MK_ASAN=no MK_UBSAN=no \
 		MK_CLANG_EXTRAS=no MK_CLANG_FORMAT=no MK_CLANG_FULL=no \
 		MK_LLDB=no MK_RETPOLINE=no MK_TESTS=no
 
@@ -2844,6 +2846,16 @@ _prereq_libs= lib/libcompiler_rt
 .if ${MK_SSP} != "no"
 _prereq_libs+= lib/libssp_nonshared
 .endif
+.if ${MK_ASAN} != "no"
+_prereq_libs+=	lib/libclang_rt/asan
+_prereq_libs+=	lib/libclang_rt/asan-preinit
+_prereq_libs+=	lib/libclang_rt/asan_cxx
+.endif
+.if ${MK_UBSAN} != "no"
+_prereq_libs+=	lib/libclang_rt/ubsan_minimal
+_prereq_libs+=	lib/libclang_rt/ubsan_standalone
+_prereq_libs+=	lib/libclang_rt/ubsan_standalone_cxx
+.endif
 
 # These dependencies are not automatically generated:
 #

Makefile.libcompat

@@ -104,6 +104,7 @@ build${libcompat}: .PHONY
 	    OBJROOT='$${OBJTOP}/' \
 	    MAKEOBJDIRPREFIX= \
 	    DIRPRFX=${_dir}/ -DNO_LINT -DNO_CPU_CFLAGS \
+	    MK_ASAN=no MK_UBSAN=no \
 	    MK_CTF=no MK_RETPOLINE=no MK_WERROR=no \
 	    ${_t}
 .endfor

lib/Makefile

@@ -180,6 +180,8 @@ SUBDIR.${MK_STATS}+=	libstats
     ${MACHINE_CPUARCH} == "arm" || ${MACHINE_CPUARCH} == "i386" || \
     ${MACHINE_CPUARCH} == "powerpc")
 _libclang_rt=	libclang_rt
+.elif ${MK_ASAN} != "no" || ${MK_UBSAN} != "no"
+.error "Requested build with sanitizers but cannot build runtime libraries!"
 .endif
 
 .if ${MK_CXX} != "no"

lib/csu/Makefile.inc

@@ -3,6 +3,9 @@
 SSP_CFLAGS=
 
 NO_WMISSING_VARIABLE_DECLARATIONS=
+# Can't instrument these files since that breaks non-sanitized programs.
+MK_ASAN:=	no
+MK_UBSAN:=	no
 
 .include <src.opts.mk>
 

lib/libclang_rt/Makefile.inc

@@ -12,6 +12,8 @@ SHLIBDIR=	${LIBDIR}
 
 NO_PIC=
 MK_PROFILE=	no
+MK_ASAN:=	no
+MK_UBSAN:=	no
 
 WARNS?=		0
 

lib/libgcc_eh/Makefile.inc

@@ -20,9 +20,17 @@ SRCS_EXC+=	libunwind.cpp
 SRCS+=		${SRCS_EXC}
 .for file in ${SRCS_EXC:M*.c}
 CFLAGS.${file}+=	-fno-exceptions -funwind-tables
+.if ${MK_ASAN} != "no"
+# False-positives during stack unwinding
+CFLAGS.${file}+=	-fno-sanitize=address
+.endif
 .endfor
 .for file in ${SRCS_EXC:M*.cpp}
 CXXFLAGS.${file}+=	-fno-exceptions -funwind-tables
+.if ${MK_ASAN} != "no"
+# False-positives during stack unwinding
+CXXFLAGS.${file}+=	-fno-sanitize=address
+.endif
 .endfor
 
 CFLAGS+=	-I${UNWINDINCDIR}

lib/libgcc_s/Makefile

@@ -4,6 +4,9 @@ PACKAGE=	clibs
 SHLIB_NAME=	libgcc_s.so.1
 SHLIBDIR?=	/lib
 
+# Enabling UBSan triggers "undefined reference to vtable for __cxxabiv1::__function_type_info"
+MK_UBSAN:=	no
+
 .include <bsd.opts.mk>
 
 MK_SSP=		no

libexec/rtld-elf/Makefile

@@ -9,7 +9,10 @@ RTLD_ELF_DIR:=	${.PARSEDIR}
 .include <src.opts.mk>
 PACKAGE=	clibs
 MK_PIE=		no # Always position independent using local rules
+# Not compatible with sanitizer instrumentation or SSP.
+MK_ASAN=	no
 MK_SSP=		no
+MK_UBSAN=	no
 
 CONFS=		libmap.conf
 PROG?=		ld-elf.so.1

share/mk/bsd.README

@@ -118,7 +118,7 @@ The profiled libraries are no longer built in a different directory than
 the regular libraries.  A new suffix, ".po", is used to denote a profiled
 object, and ".pico" denotes a position-independent relocatable object.
 ".nossppico" denotes a position-independent relocatable object without
-stack smashing protection.
+stack smashing protection and without sanitizer instrumentation.
 
 =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
 

share/mk/bsd.compat.mk

@@ -162,6 +162,7 @@ LIBCOMPATCFLAGS+=	-B${LIBCOMPATTMP}/usr/lib${libcompat}
 LIBDIR_BASE:=	/usr/lib${libcompat}
 _LIB_OBJTOP=	${LIBCOMPAT_OBJTOP}
 LIBDESTDIR:=	${LIBCOMPATTMP}
+SYSROOT:=	${LIBCOMPATTMP}
 CFLAGS+=	${LIBCOMPATCFLAGS}
 LDFLAGS+=	${CFLAGS} ${LIBCOMPATLDFLAGS}
 MACHINE=	${LIBCOMPAT_MACHINE}

share/mk/bsd.lib.mk

@@ -6,6 +6,8 @@
 .include <bsd.compiler.mk>
 .include <bsd.linker.mk>
 
+__<bsd.lib.mk>__:
+
 .if defined(LIB_CXX) || defined(SHLIB_CXX)
 _LD=	${CXX}
 .else
@@ -106,6 +108,8 @@ CXXFLAGS+= -ftrivial-auto-var-init=pattern
 .endif
 .endif
 
+.include "bsd.sanitizer.mk"
+
 .if ${MK_DEBUG_FILES} != "no" && empty(DEBUG_FLAGS:M-g) && \
     empty(DEBUG_FLAGS:M-gdwarf*)
 CFLAGS+= ${DEBUG_FILES_CFLAGS}
@@ -147,7 +151,7 @@ PO_FLAG=-pg
 	${CTFCONVERT_CMD}
 
 .c.nossppico:
-	${CC} ${PICFLAG} -DPIC ${SHARED_CFLAGS:C/^-fstack-protector.*$//} ${CFLAGS:C/^-fstack-protector.*$//} -c ${.IMPSRC} -o ${.TARGET}
+	${CC} ${PICFLAG} -DPIC ${SHARED_CFLAGS:C/^-fstack-protector.*$//:C/^-fsanitize.*$//} ${CFLAGS:C/^-fstack-protector.*$//:C/^-fsanitize.*$//} -c ${.IMPSRC} -o ${.TARGET}
 	${CTFCONVERT_CMD}
 
 .c.pieo:
@@ -161,7 +165,7 @@ PO_FLAG=-pg
 	${CXX} ${PICFLAG} -DPIC ${SHARED_CXXFLAGS} ${CXXFLAGS} -c ${.IMPSRC} -o ${.TARGET}
 
 .cc.nossppico .C.nossppico .cpp.nossppico .cxx.nossppico:
-	${CXX} ${PICFLAG} -DPIC ${SHARED_CXXFLAGS:C/^-fstack-protector.*$//} ${CXXFLAGS:C/^-fstack-protector.*$//} -c ${.IMPSRC} -o ${.TARGET}
+	${CXX} ${PICFLAG} -DPIC ${SHARED_CXXFLAGS:C/^-fstack-protector.*$//:C/^-fsanitize.*$//} ${CXXFLAGS:C/^-fstack-protector.*$//:C/^-fsanitize.*$//} -c ${.IMPSRC} -o ${.TARGET}
 
 .cc.pieo .C.pieo .cpp.pieo .cxx.pieo:
 	${CXX} ${PIEFLAG} ${SHARED_CXXFLAGS} ${CXXFLAGS} -c ${.IMPSRC} -o ${.TARGET}

share/mk/bsd.opts.mk

@@ -69,6 +69,7 @@ __DEFAULT_YES_OPTIONS = \
     WERROR
 
 __DEFAULT_NO_OPTIONS = \
+    ASAN \
     BIND_NOW \
     CCACHE_BUILD \
     CTF \
@@ -77,7 +78,8 @@ __DEFAULT_NO_OPTIONS = \
     INSTALL_AS_USER \
     MANSPLITPKG \
     RETPOLINE \
-    STALE_STAGED
+    STALE_STAGED \
+    UBSAN
 
 __DEFAULT_DEPENDENT_OPTIONS = \
     MAKE_CHECK_USE_SANDBOX/TESTS \

share/mk/bsd.prog.mk

@@ -81,6 +81,8 @@ CXXFLAGS+= -ftrivial-auto-var-init=pattern
 .endif
 .endif
 
+.include "bsd.sanitizer.mk"
+
 .if ${MACHINE_CPUARCH} == "riscv" && ${LINKER_FEATURES:Mriscv-relaxations} == ""
 CFLAGS += -mno-relax
 .endif

share/mk/bsd.progs.mk

@@ -23,7 +23,7 @@ PROGS += ${PROGS_CXX}
 .if defined(PROG)
 # just one of many
 PROG_OVERRIDE_VARS +=	BINDIR BINGRP BINOWN BINMODE CSTD CXXSTD DPSRCS MAN \
-			NO_SHARED MK_WERROR PROGNAME SRCS STRIP WARNS
+			NO_SHARED MK_WERROR PROGNAME SRCS STRIP WARNS MK_ASAN MK_UBSAN
 PROG_VARS +=	CFLAGS CXXFLAGS DEBUG_FLAGS DPADD INTERNALPROG LDADD LIBADD \
 		LINKS LDFLAGS MLINKS ${PROG_OVERRIDE_VARS}
 .for v in ${PROG_VARS:O:u}

share/mk/bsd.sanitizer.mk

@@ -0,0 +1,43 @@
+.include <bsd.opts.mk>
+
+.include "../../lib/libclang_rt/compiler-rt-vars.mk"
+_use_sanitizers=	no
+# Add the necessary sanitizer flags if requested
+.if ${MK_ASAN} == "yes" && ${NO_SHARED:Uno:tl} == "no"
+SANITIZER_CFLAGS+=	-fsanitize=address -fPIC
+# TODO: remove this once all basic errors have been fixed:
+# https://github.com/google/sanitizers/wiki/AddressSanitizer#faq
+SANITIZER_CFLAGS+=	-fsanitize-recover=address
+SANITIZER_LDFLAGS+=	-fsanitize=address
+_use_sanitizers=	yes
+.endif # ${MK_ASAN} == "yes"
+
+.if ${MK_UBSAN} == "yes" && ${NO_SHARED:Uno:tl} == "no"
+# Unlike the other sanitizers, UBSan could also work for static libraries.
+# However, this currently results in linker errors (even with the
+# -fsanitize-minimal-runtime flag), so only enable it for dynamically linked
+# code for now.
+SANITIZER_CFLAGS+=	-fsanitize=undefined
+SANITIZER_CFLAGS+=	-fsanitize-recover=undefined
+SANITIZER_LDFLAGS+=	-fsanitize=undefined
+_use_sanitizers=	yes
+.endif # ${MK_UBSAN} == "yes"
+
+.if !defined(BOOTSTRAPPING) && ${_use_sanitizers} != 0 && \
+    ${COMPILER_TYPE} != "clang"
+.error "Sanitizer instrumentation currently only supported with clang"
+.endif
+
+# For libraries we only instrument the shared and PIE libraries by setting
+# SHARED_CFLAGS instead of CFLAGS. We do this since static executables are not
+# compatible with the santizers (interceptors do not work).
+.if ${_use_sanitizers} != "no"
+.if target(__<bsd.lib.mk>__)
+SHARED_CFLAGS+=	${SANITIZER_CFLAGS}
+SOLINKOPTS+=	${SANITIZER_LDFLAGS}
+LDFLAGS:=	${LDFLAGS:N-Wl,-no-undefined:N-Wl,--no-undefined}
+.else
+CFLAGS+=	${SANITIZER_CFLAGS}
+LDFLAGS+=	${SANITIZER_LDFLAGS}
+.endif
+.endif # ${_use_sanitizers} != "no"

share/mk/sys.mk

@@ -242,7 +242,7 @@ LFLAGS		?=
 # compiler driver flags (e.g. -mabi=*) that conflict with flags to LD.
 LD		?=	ld
 LDFLAGS		?=
-_LDFLAGS	=	${LDFLAGS:S/-Wl,//g:N-mabi=*:N-fuse-ld=*:N--ld-path=*}
+_LDFLAGS	=	${LDFLAGS:S/-Wl,//g:N-mabi=*:N-fuse-ld=*:N--ld-path=*:N-fsanitize=*:N-fno-sanitize=*}
 
 MAKE		?=	make