Allow bootstrapping llvm-tblgen on macOS and Linux

This is needed in order to build various LLVM binutils (e.g. addr2line)
as well as clang/lld/lldb.

Co-authored-by: Jessica Clarke <jrtc27@FreeBSD.org>
Test Plan:	Compiles on ubuntu 18.04 and macOS 11.4
Reviewed By:	dim
Differential Revision: https://reviews.freebsd.org/D31057

contrib/llvm-project/lld/tools/lld/lld.cpp

@@ -146,7 +146,8 @@ static Flavor parseFlavor(std::vector<const char *> &v) {
 static int lldMain(int argc, const char **argv, llvm::raw_ostream &stdoutOS,
                    llvm::raw_ostream &stderrOS, bool exitEarly = true) {
   std::vector<const char *> args(argv, argv + argc);
-#ifdef __FreeBSD__
+#if 1
+  /* On FreeBSD we only build the ELF linker. */
   return !elf::link(args, exitEarly, stdoutOS, stderrOS);
 #else
   switch (parseFlavor(args)) {

lib/clang/include/llvm/Config/config.h

@@ -29,7 +29,11 @@
 /* #undef HAVE_CRASHREPORTERCLIENT_H */
 
 /* can use __crashreporter_info__ */
+#if defined(__APPLE__)
+#define HAVE_CRASHREPORTER_INFO 1
+#else
 #define HAVE_CRASHREPORTER_INFO 0
+#endif
 
 /* Define to 1 if you have the declaration of `arc4random', and to 0 if you
    don't. */
@@ -125,28 +129,46 @@
 #define HAVE_PTHREAD_SETNAME_NP 1
 
 /* Define to 1 if you have the <link.h> header file. */
+#if __has_include(<link.h>)
 #define HAVE_LINK_H 1
+#else
+#define HAVE_LINK_H 0
+#endif
 
 /* Define to 1 if you have the `lseek64' function. */
-/* #undef HAVE_LSEEK64 */
+#if defined(__linux__)
+#define HAVE_LSEEK64 1
+#endif
 
 /* Define to 1 if you have the <mach/mach.h> header file. */
-/* #undef HAVE_MACH_MACH_H */
+#if __has_include(<mach/mach.h>)
+#define HAVE_MACH_MACH_H 1
+#endif
 
 /* Define to 1 if you have the `mallctl' function. */
+#if defined(__FreeBSD__)
 #define HAVE_MALLCTL 1
+#endif
 
 /* Define to 1 if you have the `mallinfo' function. */
-/* #undef HAVE_MALLINFO */
+#if defined(__linux__)
+#define HAVE_MALLINFO 1
+#endif
 
 /* Define to 1 if you have the <malloc/malloc.h> header file. */
-/* #undef HAVE_MALLOC_MALLOC_H */
+#if __has_include(<malloc/malloc.h>)
+#define HAVE_MALLOC_MALLOC_H 1
+#endif
 
 /* Define to 1 if you have the `malloc_zone_statistics' function. */
-/* #undef HAVE_MALLOC_ZONE_STATISTICS */
+#if defined(__APPLE__)
+#define HAVE_MALLOC_ZONE_STATISTICS 1
+#endif
 
 /* Define to 1 if you have the `posix_fallocate' function. */
+#if !defined(__APPLE__)
 #define HAVE_POSIX_FALLOCATE 1
+#endif
 
 /* Define to 1 if you have the `posix_spawn' function. */
 #define HAVE_POSIX_SPAWN 1
@@ -209,19 +231,31 @@
 #define HAVE_SYS_TIME_H 1
 
 /* Define to 1 if stat struct has st_mtimespec member .*/
+#if !defined(__linux__)
 #define HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC 1
+#endif
 
 /* Define to 1 if stat struct has st_mtim member. */
+#if !defined(__APPLE__)
 #define HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC 1
+#endif
 
 /* Define to 1 if you have the <sys/types.h> header file. */
 #define HAVE_SYS_TYPES_H 1
 
 /* Define if the setupterm() function is supported this platform. */
+#if defined(__FreeBSD__)
+/*
+ * This is only needed for terminalHasColors(). When disabled LLVM falls back
+ * to checking a list of TERM prefixes which is sufficient for a bootstrap tool.
+ */
 #define LLVM_ENABLE_TERMINFO 1
+#endif
 
 /* Define if the xar_open() function is supported this platform. */
-/* #undef HAVE_LIBXAR */
+#if defined(__APPLE__)
+#define HAVE_LIBXAR
+#endif
 
 /* Define to 1 if you have the <termios.h> header file. */
 #define HAVE_TERMIOS_H 1
@@ -239,7 +273,9 @@
 /* #undef HAVE__CHSIZE_S */
 
 /* Define to 1 if you have the `_Unwind_Backtrace' function. */
-/* #undef HAVE__UNWIND_BACKTRACE */
+#if !defined(__FreeBSD__)
+#define HAVE__UNWIND_BACKTRACE 1
+#endif
 
 /* Have host's __alloca */
 /* #undef HAVE___ALLOCA */
@@ -315,7 +351,11 @@
 /* #undef LLVM_LIBXML2_ENABLED */
 
 /* Define to the extension used for shared libraries, say, ".so". */
+#if defined(__APPLE__)
+#define LTDL_SHLIB_EXT ".dylib"
+#else
 #define LTDL_SHLIB_EXT ".so"
+#endif
 
 /* Define to the address where bug reports for this package should be sent. */
 #define PACKAGE_BUGREPORT "https://bugs.freebsd.org/submit/"
@@ -351,6 +391,10 @@
 /* #undef LLVM_GISEL_COV_PREFIX */
 
 /* Whether Timers signpost passes in Xcode Instruments */
+#if defined(__APPLE__)
+#define LLVM_SUPPORT_XCODE_SIGNPOSTS 1
+#else
 #define LLVM_SUPPORT_XCODE_SIGNPOSTS 0
+#endif
 
 #endif

lib/clang/libllvm/Makefile

@@ -717,7 +717,8 @@ SRCS_EXL+=	LTO/LTOModule.cpp
 SRCS_EXL+=	LTO/SummaryBasedOptimizations.cpp
 SRCS_EXL+=	LTO/ThinLTOCodeGenerator.cpp
 SRCS_MIN+=	LTO/UpdateCompilerUsed.cpp
-SRCS_MIN+=	LineEditor/LineEditor.cpp
+# Only needed for clangd/clang-query, uncomment once we build those.
+# SRCS_XDW+=	LineEditor/LineEditor.cpp
 SRCS_MIN+=	Linker/IRMover.cpp
 SRCS_MIN+=	Linker/LinkModules.cpp
 SRCS_MIN+=	MC/ConstantPools.cpp

lib/clang/llvm.build.mk

@@ -100,12 +100,24 @@ CFLAGS+=	-DLLVM_NATIVE_TARGETMC=LLVMInitialize${LLVM_NATIVE_ARCH}TargetMC
 
 CFLAGS+=	-ffunction-sections
 CFLAGS+=	-fdata-sections
+.if ${LINKER_TYPE} == "mac"
+LDFLAGS+=	-Wl,-dead_strip
+.else
 LDFLAGS+=	-Wl,--gc-sections
+.endif
 
 CXXSTD?=	c++14
 CXXFLAGS+=	-fno-exceptions
 CXXFLAGS+=	-fno-rtti
+.if ${.MAKE.OS} == "FreeBSD" || !defined(BOOTSTRAPPING)
+# Building on macOS/Linux needs the real sysctl() not the bootstrap tools stub.
+CFLAGS+=	-DBOOTSTRAPPING_WANT_NATIVE_SYSCTL
+.else
 CXXFLAGS.clang+= -stdlib=libc++
+.endif
+.if defined(BOOTSTRAPPING) && ${.MAKE.OS} == "Linux"
+LIBADD+=	dl
+.endif
 
 .if ${MACHINE_ARCH:Mmips64}
 STATIC_CFLAGS+= -mxgot

share/mk/src.opts.mk

@@ -360,14 +360,6 @@ __DEFAULT_YES_OPTIONS+=OPENMP
 __DEFAULT_NO_OPTIONS+=OPENMP
 .endif
 
-.if ${.MAKE.OS} != "FreeBSD"
-# Building the target compiler requires building tablegen on the host
-# which is (currently) not possible on non-FreeBSD.
-BROKEN_OPTIONS+=CLANG LLD LLDB
-# The same also applies to the bootstrap LLVM.
-BROKEN_OPTIONS+=CLANG_BOOTSTRAP LLD_BOOTSTRAP
-.endif
-
 .include <bsd.mkopt.mk>
 
 #

tools/build/cross-build/include/common/sys/sysctl.h

@@ -37,6 +37,11 @@
  */
 #pragma once
 
+#ifdef BOOTSTRAPPING_WANT_NATIVE_SYSCTL
+/* We need the real sysctl.h e.g. when bootstrapping the LLVM tools. */
+#include_next <sys/sysctl.h>
+#else
+/* Otherwise, avoid sysctls since they might not be supported on the host. */
 #include <sys/types.h>
 
 #define sysctlbyname __freebsd_sysctlbyname
@@ -44,3 +49,4 @@
 
 int sysctl(const int *, u_int, void *, size_t *, const void *, size_t);
 int sysctlbyname(const char *, void *, size_t *, const void *, size_t);
+#endif

tools/build/make.py

@@ -166,6 +166,9 @@ if __name__ == "__main__":
                              "needed if CC/CPP/CXX are not set). ")
     parser.add_argument("--debug", action="store_true",
                         help="Print information on inferred env vars")
+    parser.add_argument("--bootstrap-toolchain", action="store_true",
+                        help="Bootstrap the toolchain instead of using an "
+                             "external one (experimental and not recommended)")
     parser.add_argument("--clean", action="store_true",
                         help="Do a clean rebuild instead of building with "
                              "-DWITHOUT_CLEAN")
@@ -195,6 +198,7 @@ if __name__ == "__main__":
             if "universe" not in sys.argv and "tinderbox" not in sys.argv:
                 sys.exit("TARGET= and TARGET_ARCH= must be set explicitly "
                          "when building on non-FreeBSD")
+    if not parsed_args.bootstrap_toolchain:
         # infer values for CC/CXX/CPP
         if parsed_args.host_compiler_type == "gcc":
             default_cc, default_cxx, default_cpp = ("gcc", "g++", "cpp")

usr.bin/clang/clang.prog.mk

@@ -17,8 +17,10 @@ LDADD+=		${OBJTOP}/lib/clang/lib${lib}/lib${lib}.a
 
 PACKAGE=	clang
 
+.if ${.MAKE.OS} == "FreeBSD" || !defined(BOOTSTRAPPING)
 LIBADD+=	execinfo
 LIBADD+=	ncursesw
+.endif
 LIBADD+=	pthread
 
 .include <bsd.prog.mk>

usr.bin/clang/lld/Makefile

@@ -102,8 +102,10 @@ TGHDRS+=	${INCFILE}
 DPSRCS+=	${TGHDRS}
 CLEANFILES+=	${TGHDRS} ${TGHDRS:C/$/.d/}
 
+.if ${.MAKE.OS} == "FreeBSD" || !defined(BOOTSTRAPPING)
 LIBADD+=	execinfo
 LIBADD+=	ncursesw
+.endif
 LIBADD+=	pthread
 LIBADD+=	z
 

usr.bin/clang/llvm.prog.mk

@@ -22,8 +22,10 @@ LDADD+=		${OBJTOP}/lib/clang/lib${lib}/lib${lib}.a
 
 PACKAGE=	clang
 
+.if ${.MAKE.OS} == "FreeBSD" || !defined(BOOTSTRAPPING)
 LIBADD+=	execinfo
 LIBADD+=	ncursesw
+.endif
 LIBADD+=	pthread
 
 .include <bsd.prog.mk>