From 960b77be8ff51bd6ec899ef8640def27fa6905cb Mon Sep 17 00:00:00 2001 From: "Simon J. Gerraty" Date: Tue, 24 Nov 2020 21:45:38 +0000 Subject: [PATCH] Update dirdeps.mk et al to latest Move some local tweaks to local.*.mk Reviewed by: bdrewery MFC after: 1 week Differential Revision: https://reviews.freebsd.org/D27300 --- share/mk/bsd.init.mk | 8 +- share/mk/dirdeps-options.mk | 39 ++++++-- share/mk/dirdeps-targets.mk | 54 +++++++++-- share/mk/dirdeps.mk | 182 +++++++++++++++++++++-------------- share/mk/gendirdeps.mk | 47 ++++----- share/mk/local.autodep.mk | 3 + share/mk/local.dirdeps.mk | 2 - share/mk/local.gendirdeps.mk | 3 + share/mk/local.meta.sys.mk | 5 +- share/mk/meta.autodep.mk | 16 ++- share/mk/meta.stage.mk | 38 ++++---- share/mk/meta.subdir.mk | 10 +- share/mk/meta.sys.mk | 24 ++--- share/mk/meta2deps.py | 68 +++++++------ share/mk/meta2deps.sh | 49 ++++++---- 15 files changed, 343 insertions(+), 205 deletions(-) diff --git a/share/mk/bsd.init.mk b/share/mk/bsd.init.mk index 5cd435953bef..30eeb0b2a15b 100644 --- a/share/mk/bsd.init.mk +++ b/share/mk/bsd.init.mk @@ -55,9 +55,13 @@ $xGRP= ${_gid} # - make install is used without other targets. This is to avoid breaking # things like 'make all install' or 'make foo install'. # - non-build targets are called -.if ${MK_DIRDEPS_BUILD} == "yes" && ${.MAKE.LEVEL:U1} == 0 && \ - ${BUILD_AT_LEVEL0:Uyes:tl} == "no" && !make(clean*) && !make(*clean) +.if ${MK_DIRDEPS_BUILD} == "yes" && ${.MAKE.LEVEL} == 0 +# targets that are ok at level 0 +DIRDEPS_BUILD_LEVEL0_TARGETS += clean* destroy* +M_ListToSkip?= O:u:S,^,N,:ts: +.if ${.TARGETS:Uall:${DIRDEPS_BUILD_LEVEL0_TARGETS:${M_ListToSkip}}} != "" _SKIP_BUILD= not building at level 0 +.endif .elif !empty(.MAKEFLAGS:M-V${_V_DO_BUILD}) || \ ${.TARGETS:M*install*} == ${.TARGETS} || \ ${.TARGETS:Mclean*} == ${.TARGETS} || \ diff --git a/share/mk/dirdeps-options.mk b/share/mk/dirdeps-options.mk index d177900845a7..c59a5600923e 100644 --- a/share/mk/dirdeps-options.mk +++ b/share/mk/dirdeps-options.mk @@ -1,7 +1,7 @@ # $FreeBSD$ -# $Id: dirdeps-options.mk,v 1.9 2018/09/20 00:07:19 sjg Exp $ +# $Id: dirdeps-options.mk,v 1.17 2020/08/07 01:57:38 sjg Exp $ # -# @(#) Copyright (c) 2018, Simon J. Gerraty +# @(#) Copyright (c) 2018-2020, Simon J. Gerraty # # This file is provided in the hope that it will # be of use. There is absolutely NO WARRANTY. @@ -38,6 +38,11 @@ # to whatever applies for that dir, or it can rely on globals # set in local.dirdeps-options.mk # Either way, we will .undef DIRDEPS.* when done. +# +# In some cases the value of MK_FOO might depend on TARGET_SPEC +# so we qualify MK_FOO with .${TARGET_SPEC} and each component +# TARGET_SPEC_VAR (in reverse order) before using MK_FOO. +# # This should have been set by Makefile.depend.options # before including us @@ -48,21 +53,43 @@ DIRDEPS_OPTIONS ?= .if ${.MAKE.LEVEL} == 0 # :U below avoids potential errors when we := -.for o in ${DIRDEPS_OPTIONS:tu} -DIRDEPS += ${DIRDEPS.$o.${MK_$o:U}:U} +# some options can depend on TARGET_SPEC! +DIRDEPS_OPTIONS_QUALIFIER_LIST ?= \ + ${DEP_TARGET_SPEC:U${TARGET_SPEC}} \ + ${TARGET_SPEC_VARSr:U${TARGET_SPEC_VARS}:@v@${DEP_$v:U${$v}}@} +# note that we need to include $o in the variable _o$o +# to ensure correct evaluation. +.for o in ${DIRDEPS_OPTIONS} +.undef _o$o _v$o +.for x in ${DIRDEPS_OPTIONS_QUALIFIER_LIST} +.if defined(MK_$o.$x) +_o$o ?= MK_$o.$x +_v$o ?= ${MK_$o.$x} +.endif +.endfor +_v$o ?= ${MK_$o} +.if ${_debug_reldir:U0} +.info ${DEP_RELDIR:U${RELDIR}}.${DEP_TARGET_SPEC:U${TARGET_SPEC}}: o=$o ${_o$o:UMK_$o}=${_v$o:U} DIRDEPS += ${DIRDEPS.$o.${_v$o:U}:U} +.endif +DIRDEPS += ${DIRDEPS.$o.${_v$o:U}:U} .endfor DIRDEPS := ${DIRDEPS:O:u} +.if ${_debug_reldir:U0} +.info ${DEP_RELDIR:U${RELDIR}}: DIRDEPS=${DIRDEPS} +.endif # avoid cross contamination -.for o in ${DIRDEPS_OPTIONS:tu} +.for o in ${DIRDEPS_OPTIONS} .undef DIRDEPS.$o.yes .undef DIRDEPS.$o.no +.undef _o$o +.undef _v$o .endfor .else # whether options are enabled or not, # we want to filter out the relevant DIRDEPS.* # we should only be included by meta.autodep.mk # if dependencies are to be updated -.for o in ${DIRDEPS_OPTIONS:tu} +.for o in ${DIRDEPS_OPTIONS} .for d in ${DIRDEPS.$o.yes} ${DIRDEPS.$o.no} .if exists(${SRCTOP}/$d) GENDIRDEPS_FILTER += N$d* diff --git a/share/mk/dirdeps-targets.mk b/share/mk/dirdeps-targets.mk index 179c38b53fff..e65ad89742bd 100644 --- a/share/mk/dirdeps-targets.mk +++ b/share/mk/dirdeps-targets.mk @@ -1,16 +1,16 @@ # $FreeBSD$ # RCSid: -# $Id: dirdeps-targets.mk,v 1.9 2019/10/06 20:07:50 sjg Exp $ +# $Id: dirdeps-targets.mk,v 1.22 2020/08/15 18:00:11 sjg Exp $ # -# @(#) Copyright (c) 2019 Simon J. Gerraty +# @(#) Copyright (c) 2019-2020 Simon J. Gerraty # # This file is provided in the hope that it will # be of use. There is absolutely NO WARRANTY. # Permission to copy, redistribute or otherwise -# use this file is hereby granted provided that +# use this file is hereby granted provided that # the above copyright notice and this notice are -# left intact. -# +# left intact. +# # Please send copies of changes and bug-fixes to: # sjg@crufty.net # @@ -26,7 +26,16 @@ # We then search those dirs for any Makefile.depend* # Finally we select any that match conditions like REQUESTED_MACHINE # or TARGET_SPEC and initialize DIRDEPS accordingly. -# +# +# We will check each of the initial DIRDEPS for Makefile.dirdeps.options +# and include any found. +# This makes it feasible to tweak options like MK_DIRDEPS_CACHE +# for a specific target. +# +# If MK_STATIC_DIRDEPS_CACHE is defined we will check if the +# initial DIRDEPS has a static cache (Makefile.dirdeps.cache). +# This only makes sense for seriously expensive targets. +# .if ${.MAKE.LEVEL} == 0 # pickup customizations @@ -38,8 +47,11 @@ DIRDEPS_TARGETS_DIRS ?= targets targets/pseudo # they need to be stripped when looking for target dirs DIRDEPS_TARGETS_PREFIX_LIST ?= pkg- build- +# some .TARGETS need filtering +DIRDEPS_TARGETS_FILTER += Nall + # matching target dirs if any -tdirs := ${.TARGETS:Nall:${DIRDEPS_TARGETS_PREFIX_LIST:@p@S,^$p,,@:ts:}:@t@${DIRDEPS_TARGETS_DIRS:@d@$d/$t@}@:@d@${exists(${SRCTOP}/$d):?$d:}@} +tdirs := ${.TARGETS:${DIRDEPS_TARGETS_FILTER:ts:}:${DIRDEPS_TARGETS_PREFIX_LIST:@p@S,^$p,,@:ts:}:@t@${DIRDEPS_TARGETS_DIRS:@d@$d/$t@}@:@d@${exists(${SRCTOP}/$d):?$d:}@} .if !empty(DEBUG_DIRDEPS_TARGETS) .info tdirs=${tdirs} @@ -123,12 +135,38 @@ DIRDEPS := ${DIRDEPS:O:u} .endif # if we got DIRDEPS get to work .if !empty(DIRDEPS) +DIRDEPS.dirs := ${DIRDEPS:S,^,${SRCTOP}/,:@d@${exists($d):?$d:${d:R}}@} +# some targets what to tweak options we might want to process now +.for m in ${DIRDEPS.dirs:S,$,/Makefile.dirdeps.options,} +.-include <$m> +.endfor +.if defined(MK_STATIC_DIRDEPS_CACHE) +# some targets are very expensive to compute dirdeps for +# so we may have a static cache +.for c in ${DIRDEPS.dirs:S,$,/Makefile.dirdeps.cache,} +.if exists($c) +STATIC_DIRDEPS_CACHE ?= $c +.if ${MK_STATIC_DIRDEPS_CACHE} == "yes" +DIRDEPS_CACHE ?= $c +MK_DIRDEPS_CACHE = yes +.endif +.endif +.endfor +.if defined(STATIC_DIRDEPS_CACHE) +.export STATIC_DIRDEPS_CACHE +.endif +.endif + +# allow a top-level makefile to do other stuff +# before including dirdeps.mk +.if ${MK_DIRDEPS_TARGETS_INCLUDE_DIRDEPS:Uyes} == "yes" .include +.endif DIRDEPS_TARGETS_SKIP += all clean* destroy* .for t in ${.TARGETS:${DIRDEPS_TARGETS_SKIP:${M_ListToSkip}}} $t: dirdeps -.endfor +.endfor .endif .endif diff --git a/share/mk/dirdeps.mk b/share/mk/dirdeps.mk index 68ddd2155647..c7842b49d16b 100644 --- a/share/mk/dirdeps.mk +++ b/share/mk/dirdeps.mk @@ -1,7 +1,8 @@ # $FreeBSD$ -# $Id: dirdeps.mk,v 1.100 2019/11/12 06:47:58 sjg Exp $ +# $Id: dirdeps.mk,v 1.130 2020/11/02 00:34:30 sjg Exp $ -# Copyright (c) 2010-2013, Juniper Networks, Inc. +# Copyright (c) 2010-2020, Simon J. Gerraty +# Copyright (c) 2010-2018, Juniper Networks, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -41,7 +42,7 @@ # or . suffix (see TARGET_SPEC_VARS below), # for example to force building something for the pseudo # machines "host" or "common" regardless of current ${MACHINE}. -# +# # All unqualified entries end up being qualified with .${TARGET_SPEC} # and partially qualified (if TARGET_SPEC_VARS has multiple # entries) are also expanded to a full .. @@ -50,29 +51,22 @@ # # The fully qualified directory entries are used to construct a # dependency graph that will drive the build later. -# +# # Also, for each fully qualified directory target, we will search # using ${.MAKE.DEPENDFILE_PREFERENCE} to find additional # dependencies. We use Makefile.depend (default value for # .MAKE.DEPENDFILE_PREFIX) to refer to these makefiles to # distinguish them from others. -# +# # Before each Makefile.depend file is read, we set # DEP_RELDIR to be the RELDIR (path relative to SRCTOP) for # its directory, and DEP_MACHINE etc according to the . # represented by the suffix of the corresponding target. -# +# # Since each Makefile.depend file includes dirdeps.mk, this # processing is recursive and results in .MAKE.LEVEL 0 learning the # dependencies of the tree wrt the initial directory (_DEP_RELDIR). # -# BUILD_AT_LEVEL0 -# Indicates whether .MAKE.LEVEL 0 builds anything: -# if "no" sub-makes are used to build everything, -# if "yes" sub-makes are only used to build for other machines. -# It is best to use "no", but this can require fixing some -# makefiles to not do anything at .MAKE.LEVEL 0. -# # TARGET_SPEC_VARS # The default value is just MACHINE, and for most environments # this is sufficient. The _DIRDEP_USE target actually sets @@ -113,12 +107,12 @@ # # make sure we know what TARGET_SPEC is # # as we may need it to find Makefile.depend* # TARGET_SPEC = ${TARGET_SPEC_VARS:@v@${$v:U}@:ts,} -# +# # The following variables can influence the initial DIRDEPS # computation with regard to the TARGET_SPECs that will be # built. # Most should also be considered by init.mk -# +# # ONLY_TARGET_SPEC_LIST # Defines a list of TARGET_SPECs for which the current # directory can be built. @@ -137,6 +131,19 @@ # A list of MACHINEs the current directory should not be # built for. # +# _build_xtra_dirs +# local.dirdeps.mk can add targets to this variable. +# They will be hooked into the build, but independent of +# any other DIRDEP. +# +# This allows for adding TESTS to the build, such that the build +# if any test fails, but without the risk of introducing +# circular dependencies. + +now_utc ?= ${%s:L:gmtime} +.if !defined(start_utc) +start_utc := ${now_utc} +.endif .if !target(bootstrap) && (make(bootstrap) || \ make(bootstrap-this) || \ @@ -157,11 +164,6 @@ _DIRDEP_USE_LEVEL?= 0 _CURDIR ?= ${.CURDIR} _OBJDIR ?= ${.OBJDIR} -now_utc = ${%s:L:gmtime} -.if !defined(start_utc) -start_utc := ${now_utc} -.endif - .if ${MAKEFILE:T} == ${.PARSEFILE} && empty(DIRDEPS) && ${.TARGETS:Uall:M*/*} != "" # This little trick let's us do # @@ -208,12 +210,9 @@ DEP_$v ?= ${$v} # so we need to construct a set of modifiers to fill in the gaps. .if ${MAKE_VERSION} >= 20170130 _tspec_x := ${TARGET_SPEC_VARS:range} -.elif ${TARGET_SPEC_VARS:[#]} > 10 -# seriously? better have jot(1) or equivalent to produce suitable sequence -_tspec_x := ${${JOT:Ujot} ${TARGET_SPEC_VARS:[#]}:L:sh} .else -# we can provide the sequence ourselves -_tspec_x := ${1 2 3 4 5 6 7 8 9 10:L:[1..${TARGET_SPEC_VARS:[#]}]} +# do it the hard way +_tspec_x := ${TARGET_SPEC_VARS:[#]:@x@i=1;while [ $$i -le $x ]; do echo $$i; i=$$((i + 1)); done;@:sh} .endif # this handles unqualified entries M_dep_qual_fixes = C;(/[^/.,]+)$$;\1.$${DEP_TARGET_SPEC}; @@ -223,6 +222,7 @@ _tspec_m$i := ${TARGET_SPEC_VARS:[2..$i]:@w@[^,]+@:ts,} _tspec_a$i := ,${TARGET_SPEC_VARS:[$i..-1]:@v@$$$${DEP_$v}@:ts,} M_dep_qual_fixes += C;(\.${_tspec_m$i})$$;\1${_tspec_a$i}; .endfor +TARGET_SPEC_VARSr := ${TARGET_SPEC_VARS:[-1..1]} .else # A harmless? default. M_dep_qual_fixes = U @@ -307,6 +307,7 @@ DEP_MACHINE := ${_DEP_TARGET_SPEC} # reset each time through _build_all_dirs = +_build_xtra_dirs = # the first time we are included the _DIRDEP_USE target will not be defined # we can use this as a clue to do initialization and other one time things. @@ -337,7 +338,18 @@ BUILD_DIRDEPS ?= yes .if ${MK_DIRDEPS_CACHE} == "yes" # this is where we will cache all our work -DIRDEPS_CACHE ?= ${_OBJDIR:tA}/dirdeps.cache${.TARGETS:Nall:O:u:ts-:S,/,_,g:S,^,.,:N.} +DIRDEPS_CACHE ?= ${_OBJDIR:tA}/dirdeps.cache${_TARGETS:U${.TARGETS}:Nall:O:u:ts-:S,/,_,g:S,^,.,:N.} +.endif + +.if ${DEBUG_DIRDEPS:@x@${DEP_RELDIR:M$x}${${DEP_RELDIR}.${DEP_MACHINE}:L:M$x}@} != "" +_debug_reldir = 1 +.else +_debug_reldir = 0 +.endif +.if ${DEBUG_DIRDEPS:@x@${DEP_RELDIR:M$x}${${DEP_RELDIR}.depend:L:M$x}@} != "" +_debug_search = 1 +.else +_debug_search = 0 .endif # pickup customizations @@ -382,6 +394,18 @@ DIRDEPS_FILTER += M${_DEP_RELDIR} DIRDEP_MAKE ?= ${.MAKE} DIRDEP_DIR ?= ${.TARGET:R} +# if you want us to report load averages during build +# DIRDEP_USE_PRELUDE += ${DIRDEP_LOADAVG_REPORT}; + +DIRDEP_LOADAVG_CMD ?= ${UPTIME:Uuptime} | sed 's,.*\(load\),\1,' +DIRDEP_LOADAVG_LAST = 0 +# yes the expression here is a bit complicated, +# the trick is to only eval ${DIRDEP_LOADAVG_LAST::=${now_utc}} +# when we want to report. +DIRDEP_LOADAVG_REPORT = \ + test -z "${"${expr ${now_utc} - ${DIRDEP_LOADAVG_INTEVAL:U60} - ${DIRDEP_LOADAVG_LAST}:L:sh:N-*}":?yes${DIRDEP_LOADAVG_LAST::=${now_utc}}:}" || \ + echo "${TRACER}`${DIRDEP_LOADAVG_CMD}`" + # we suppress SUBDIR when visiting the leaves # we assume sys.mk will set MACHINE_ARCH # you can add extras to DIRDEP_USE_ENV @@ -389,7 +413,7 @@ DIRDEP_DIR ?= ${.TARGET:R} _DIRDEP_USE: .USE .MAKE @for m in ${.MAKE.MAKEFILE_PREFERENCE}; do \ test -s ${.TARGET:R}/$$m || continue; \ - echo "${TRACER}Checking ${.TARGET:R} for ${.TARGET:E} ..."; \ + echo "${TRACER}Checking ${.TARGET:S,${SRCTOP}/,,} for ${.TARGET:E} ..."; \ ${DIRDEP_USE_PRELUDE} \ MACHINE_ARCH= NO_SUBDIR=1 ${DIRDEP_USE_ENV} \ TARGET_SPEC=${.TARGET:E} \ @@ -468,10 +492,19 @@ dirdeps-cached: ${DIRDEPS_CACHE} .MAKE @MAKELEVEL=${.MAKE.LEVEL} ${.MAKE} -C ${_CURDIR} -f ${DIRDEPS_CACHE} \ dirdeps MK_DIRDEPS_CACHE=no BUILD_DIRDEPS=no +# leaf makefiles rarely work for building DIRDEPS_CACHE +.if ${RELDIR} != "." +BUILD_DIRDEPS_MAKEFILE ?= -f dirdeps.mk +.endif + # these should generally do -BUILD_DIRDEPS_MAKEFILE ?= ${MAKEFILE} +BUILD_DIRDEPS_MAKEFILE ?= BUILD_DIRDEPS_TARGETS ?= ${.TARGETS} +.if ${DIRDEPS_CACHE} != ${STATIC_DIRDEPS_CACHE:Uno} && ${DIRDEPS_CACHE:M${SRCTOP}/*} == "" +# export this for dirdeps-cache-update.mk +DYNAMIC_DIRDEPS_CACHE := ${DIRDEPS_CACHE} +.export DYNAMIC_DIRDEPS_CACHE # we need the .meta file to ensure we update if # any of the Makefile.depend* changed. # We do not want to compare the command line though. @@ -483,40 +516,34 @@ ${DIRDEPS_CACHE}: .META .NOMETA_CMP +@MAKELEVEL=${.MAKE.LEVEL} DIRDEPS_CACHE=${DIRDEPS_CACHE} \ DIRDEPS="${DIRDEPS}" \ TARGET_SPEC=${TARGET_SPEC} \ - MAKEFLAGS= ${.MAKE} -C ${_CURDIR} -f ${BUILD_DIRDEPS_MAKEFILE} \ + MAKEFLAGS= ${DIRDEP_CACHE_MAKE:U${.MAKE}} -C ${_CURDIR} \ + ${BUILD_DIRDEPS_MAKEFILE} \ ${BUILD_DIRDEPS_TARGETS} BUILD_DIRDEPS_CACHE=yes \ .MAKE.DEPENDFILE=.none \ + ${"${DEBUG_DIRDEPS:Nno}":?DEBUG_DIRDEPS='${DEBUG_DIRDEPS}':} \ ${.MAKEFLAGS:tW:S,-D ,-D,g:tw:M*WITH*} \ - 3>&1 1>&2 | sed 's,${SRCTOP},$${SRCTOP},g' >> ${.TARGET}.new && \ + ${.MAKEFLAGS:tW:S,-d ,-d,g:tw:M-d*} \ + 3>&1 1>&2 | sed 's,${SRCTOP},$${SRCTOP},g;s,_{,$${,g' >> ${.TARGET}.new && \ mv ${.TARGET}.new ${.TARGET} +.endif .endif .elif !target(_count_dirdeps) # we want to capture the dirdeps count in the cache .END: _count_dirdeps _count_dirdeps: .NOMETA - @echo '.info $${.newline}$${TRACER}Makefiles read: total=${.MAKE.MAKEFILES:[#]} depend=${.MAKE.MAKEFILES:M*depend*:[#]} dirdeps=${.ALLTARGETS:M${SRCTOP}*:O:u:[#]}' >&3 + @{ echo; echo '.info $${.newline}$${TRACER}Makefiles read: total=${.MAKE.MAKEFILES:[#]} depend=${.MAKE.MAKEFILES:M*depend*:[#]} dirdeps=${.ALLTARGETS:M${SRCTOP}*:O:u:[#]} ${DIRDEP_INFO_XTRAS}'; } >&3 .endif .elif !make(dirdeps) && !target(_count_dirdeps) beforedirdeps: _count_dirdeps _count_dirdeps: .NOMETA - @echo "${TRACER}Makefiles read: total=${.MAKE.MAKEFILES:[#]} depend=${.MAKE.MAKEFILES:M*depend*:[#]} dirdeps=${.ALLTARGETS:M${SRCTOP}*:O:u:[#]} seconds=`expr ${now_utc} - ${start_utc}`" + @echo "${TRACER}Makefiles read: total=${.MAKE.MAKEFILES:[#]} depend=${.MAKE.MAKEFILES:M*depend*:[#]} dirdeps=${.ALLTARGETS:M${SRCTOP}*:O:u:[#]} ${DIRDEP_INFO_XTRAS} seconds=`expr ${now_utc} - ${start_utc}`" .endif .endif .if ${BUILD_DIRDEPS} == "yes" -.if ${DEBUG_DIRDEPS:@x@${DEP_RELDIR:M$x}${${DEP_RELDIR}.${DEP_MACHINE}:L:M$x}@} != "" -_debug_reldir = 1 -.else -_debug_reldir = 0 -.endif -.if ${DEBUG_DIRDEPS:@x@${DEP_RELDIR:M$x}${${DEP_RELDIR}.depend:L:M$x}@} != "" -_debug_search = 1 -.else -_debug_search = 0 -.endif # the rest is done repeatedly for every Makefile.depend we read. # if we are anything but the original dir we care only about the @@ -568,19 +595,7 @@ _build_dirs = .if ${DEP_RELDIR} == ${_DEP_RELDIR} # pickup other machines for this dir if necessary -.if ${BUILD_AT_LEVEL0:Uyes} == "no" _build_dirs += ${_machines:@m@${_CURDIR}.$m@} -.else -_build_dirs += ${_machines:N${DEP_TARGET_SPEC}:@m@${_CURDIR}.$m@} -.if ${DEP_TARGET_SPEC} == ${TARGET_SPEC} -# pickup local dependencies now -.if ${MAKE_VERSION} < 20160220 -.-include <.depend> -.else -.dinclude <.depend> -.endif -.endif -.endif .endif .if ${_debug_reldir} @@ -616,6 +631,7 @@ __qual_depdirs += ${__hostdpadd} .endif .if ${_debug_reldir} +.info DEP_DIRDEPS_FILTER=${DEP_DIRDEPS_FILTER:ts:} .info depdirs=${__depdirs} .info qualified=${__qual_depdirs} .info unqualified=${__unqual_depdirs} @@ -633,7 +649,7 @@ _build_dirs := ${_build_dirs:${M_dep_qual_fixes:ts:}:O:u} .endif # empty DIRDEPS -_build_all_dirs += ${_build_dirs} +_build_all_dirs += ${_build_dirs} ${_build_xtra_dirs} _build_all_dirs := ${_build_all_dirs:O:u} # Normally if doing make -V something, @@ -642,19 +658,14 @@ _build_all_dirs := ${_build_all_dirs:O:u} .if ${.MAKEFLAGS:M-V${_V_READ_DIRDEPS}} == "" .if !empty(_build_all_dirs) .if ${BUILD_DIRDEPS_CACHE} == "yes" -# guard against _build_all_dirs being too big for a single command line -# first get list of dirs that need _DIRDEP_USE -# then export that and _build_all_dirs +x!= echo; { echo; echo '\# ${DEP_RELDIR}.${DEP_TARGET_SPEC}'; } >&3 +# guard against _new_dirdeps being too big for a single command line _new_dirdeps := ${_build_all_dirs:@x@${target($x):?:$x}@} -.export _new_dirdeps _build_all_dirs -x!= echo; { echo '\# ${DEP_RELDIR}.${DEP_TARGET_SPEC}'; \ - echo "dirdeps: \\"; \ - for x in $$_build_all_dirs; do echo " $$x \\"; done; echo; \ - for x in $$_new_dirdeps; do echo "$$x: _DIRDEP_USE"; done; echo; } >&3 +.export _build_xtra_dirs _new_dirdeps .if !empty(DEP_EXPORT_VARS) # Discouraged, but there are always exceptions. # Handle it here rather than explain how. -x!= { echo; ${DEP_EXPORT_VARS:@v@echo '$v=${$v}';@} echo '.export ${DEP_EXPORT_VARS}'; echo; } >&3; echo +x!= echo; { echo; ${DEP_EXPORT_VARS:@v@echo '$v=${$v}';@} echo '.export ${DEP_EXPORT_VARS}'; echo; } >&3 .endif .else # this makes it all happen @@ -673,6 +684,10 @@ DEP_EXPORT_VARS= # this builds the dependency graph .for m in ${_machines} +.if ${BUILD_DIRDEPS_CACHE} == "yes" && !empty(_build_dirs) +x!= echo; { echo; echo 'DIRDEPS.${_this_dir}.$m = \'; } >&3 +_cache_deps = +.endif # it would be nice to do :N${.TARGET} .if !empty(__qual_depdirs) .for q in ${__qual_depdirs:${M_dep_qual_fixes:ts:}:E:O:u:N$m} @@ -680,10 +695,7 @@ DEP_EXPORT_VARS= .info ${DEP_RELDIR}.$m: graph: ${_build_dirs:M*.$q} .endif .if ${BUILD_DIRDEPS_CACHE} == "yes" -_cache_deps := ${_build_dirs:M*.$q} -.export _cache_deps -x!= echo; { echo "${_this_dir}.$m: \\"; \ - for x in $$_cache_deps; do echo " $$x \\"; done; echo; } >&3 +_cache_deps += ${_build_dirs:M*.$q} .else ${_this_dir}.$m: ${_build_dirs:M*.$q} .endif @@ -693,10 +705,17 @@ ${_this_dir}.$m: ${_build_dirs:M*.$q} .info ${DEP_RELDIR}.$m: graph: ${_build_dirs:M*.$m:N${_this_dir}.$m} .endif .if ${BUILD_DIRDEPS_CACHE} == "yes" -_cache_deps := ${_build_dirs:M*.$m:N${_this_dir}.$m} +.if !empty(_build_dirs) +_cache_deps += ${_build_dirs:M*.$m:N${_this_dir}.$m} +.if !empty(_cache_deps) .export _cache_deps -x!= echo; { echo "${_this_dir}.$m: \\"; \ - for x in $$_cache_deps; do echo " $$x \\"; done; echo; } >&3 +x!= echo; for x in $$_cache_deps; do echo " $$x \\"; done >&3 +.endif +x!= echo; { echo; echo '${_this_dir}.$m: $${DIRDEPS.${_this_dir}.$m}'; \ + echo; echo 'dirdeps: ${_this_dir}.$m \'; \ + for x in $$_build_xtra_dirs; do echo " $$x \\"; done; \ + echo; for x in $$_new_dirdeps; do echo "$$x: _DIRDEP_USE"; done; } >&3 +.endif .else ${_this_dir}.$m: ${_build_dirs:M*.$m:N${_this_dir}.$m} .endif @@ -766,6 +785,27 @@ _DEP_RELDIR := ${RELDIR} # This is a final opportunity to add/hook global rules. .-include +# skip _reldir_{finish,failed} if not included from Makefile.depend* +# or not in meta mode +.if !defined(WITHOUT_META_STATS) && ${.INCLUDEDFROMFILE:U:M${.MAKE.DEPENDFILE_PREFIX}*} != "" && ${.MAKE.MODE:Mmeta} != "" + +meta_stats= meta=${empty(.MAKE.META.FILES):?0:${.MAKE.META.FILES:[#]}} \ + created=${empty(.MAKE.META.CREATED):?0:${.MAKE.META.CREATED:[#]}} + +.if !target(_reldir_finish) +.END: _reldir_finish +_reldir_finish: .NOMETA + @echo "${TRACER}Finished ${RELDIR}.${TARGET_SPEC} seconds=$$(( ${now_utc} - ${start_utc} )) ${meta_stats}" +.endif + +.if !target(_reldir_failed) +.ERROR: _reldir_failed +_reldir_failed: .NOMETA + @echo "${TRACER}Failed ${RELDIR}.${TARGET_SPEC} seconds=$$(( ${now_utc} - ${start_utc} )) ${meta_stats}" +.endif + +.endif + # pickup local dependencies .if ${MAKE_VERSION} < 20160220 .-include <.depend> diff --git a/share/mk/gendirdeps.mk b/share/mk/gendirdeps.mk index 96681fc7d9d3..6d26b3d308b1 100644 --- a/share/mk/gendirdeps.mk +++ b/share/mk/gendirdeps.mk @@ -1,18 +1,19 @@ # $FreeBSD$ -# $Id: gendirdeps.mk,v 1.41 2019/11/21 23:50:40 sjg Exp $ +# $Id: gendirdeps.mk,v 1.46 2020/08/19 17:51:53 sjg Exp $ -# Copyright (c) 2010-2013, Juniper Networks, Inc. +# Copyright (c) 2011-2020, Simon J. Gerraty +# Copyright (c) 2010-2018, Juniper Networks, Inc. # All rights reserved. -# +# # Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: +# modification, are permitted provided that the following conditions +# are met: # 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. +# notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# +# documentation and/or other materials provided with the distribution. +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR @@ -23,7 +24,7 @@ # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # This makefile [re]generates ${.MAKE.DEPENDFILE} @@ -51,7 +52,7 @@ all: _CURDIR ?= ${.CURDIR} _OBJDIR ?= ${.OBJDIR} _OBJTOP ?= ${OBJTOP} -_OBJROOT ?= ${OBJROOT:U${_OBJTOP}} +_OBJROOT ?= ${OBJROOT:U${_OBJTOP:H}} .if ${_OBJROOT:M*/} _slash=/ .else @@ -104,7 +105,7 @@ GENDIRDEPS_FILTER += ${GENDIRDEPS_FILTER_DIR_VARS:@v@S,${$v},_{${v}},@} GENDIRDEPS_FILTER += ${GENDIRDEPS_FILTER_VARS:@v@S,/${$v}/,/_{${v}}/,@:NS,//,*:u} .endif -# this (*should* be set in meta.sys.mk) +# this (*should* be set in meta.sys.mk) # is the script that extracts what we want. META2DEPS ?= ${.PARSEDIR}/meta2deps.sh META2DEPS := ${META2DEPS} @@ -122,7 +123,7 @@ _py_d = .if ${META2DEPS:E} == "py" # we can afford to do this all the time. DPDEPS ?= no -META2DEPS_CMD = ${_time} ${PYTHON} ${META2DEPS} ${_py_d} +META2DEPS_CMD = ${_time} ${PYTHON} ${META2DEPS} ${_py_d} .if ${DPDEPS:tl} != "no" META2DEPS_CMD += -D ${DPDEPS} .endif @@ -155,8 +156,8 @@ M2D_OBJROOTS += ${STAGE_ROOT} # and tell it not to add machine qualifiers META2DEPS_ARGS += MACHINE=none .endif -.if defined(SB_BACKING_SB) -META2DEPS_CMD += -S ${SB_BACKING_SB}/src +.if defined(SB_BACKING_SB) +META2DEPS_CMD += -S ${SB_BACKING_SB}/src M2D_OBJROOTS += ${SB_BACKING_SB}/${SB_OBJPREFIX} .endif @@ -177,7 +178,7 @@ _meta_files := ${META_FILES:N\*.meta:O:u} # assume a big list _meta_files_arg= @meta.list .if empty(_meta_files) && ${META_FILES:M\*.meta} != "" -# XXX this should be considered a bad idea, +# XXX this should be considered a bad idea, # since we cannot ignore stale .meta x != cd ${_OBJDIR} && find . -name '*.meta' -print -o \( -type d ! -name . -prune \) | sed 's,^./,,' > meta.list; echo .elif ${_meta_files:[#]} > 500 @@ -234,10 +235,10 @@ dir_list += ${ddeps} # DIRDEPS represent things that had to have been built first # so they should all be undir OBJTOP. -# Note that ${_OBJTOP}/bsd/include/machine will get reported +# Note that ${_OBJTOP}/bsd/include/machine will get reported # to us as $SRCTOP/bsd/sys/$MACHINE_ARCH/include meaning we # will want to visit bsd/include -# so we add +# so we add # ${"${dir_list:M*bsd/sys/${MACHINE_ARCH}/include}":?bsd/include:} # to GENDIRDEPS_DIR_LIST_XTRAS _objtops = ${OBJTOP} ${_OBJTOP} ${_objtop} @@ -272,11 +273,11 @@ DIRDEPS = \ # We only consider things below $RELDIR/ if they have a makefile. # This is the same test that _DIRDEP_USE applies. -# We have do a double test with dirdep_list as it _may_ contain +# We have do a double test with dirdep_list as it _may_ contain # qualified dirs - if we got anything from a stage dir. # qualdir_list we know are all qualified. # It would be nice do peform this check for all of DIRDEPS, -# but we cannot assume that all of the tree is present, +# but we cannot assume that all of the tree is present, # in fact we can only assume that RELDIR is. DIRDEPS += \ ${dirdep_list:M${RELDIR}/*:@d@${.MAKE.MAKEFILE_PREFERENCE:@m@${exists(${SRCTOP}/$d/$m):?$d:${exists(${SRCTOP}/${d:R}/$m):?$d:}}@}@} \ @@ -309,7 +310,7 @@ SRC_DIRDEPS = \ SRC_DIRDEPS := ${SRC_DIRDEPS:${GENDIRDEPS_SRC_FILTER:UN/*:ts:}:C,//+,/,g:O:u} # if you want to capture SRC_DIRDEPS in .MAKE.DEPENDFILE put -# SRC_DIRDEPS_FILE = ${_DEPENDFILE} +# SRC_DIRDEPS_FILE = ${_DEPENDFILE} # in local.gendirdeps.mk .if ${SRC_DIRDEPS_FILE:Uno:tl} != "no" ECHO_SRC_DIRDEPS = echo 'SRC_DIRDEPS = \'; echo '${SRC_DIRDEPS:@d@ $d \\${.newline}@}'; echo; @@ -324,7 +325,7 @@ ${SRC_DIRDEPS_FILE}: ${META_FILES} ${_this} ${META2DEPS} .endif .endif .endif -_include_src_dirdeps ?= +_include_src_dirdeps ?= all: ${_DEPENDFILE} @@ -339,6 +340,8 @@ CAT_DEPEND ?= .depend .PHONY: ${_DEPENDFILE} .endif +LOCAL_DEPENDS_GUARD ?= _{.MAKE.LEVEL} > 0 + # 'cat .depend' should suffice, but if we are mixing build modes # .depend may contain things we don't want. # The sed command at the end of the stream, allows for the filters @@ -350,7 +353,7 @@ ${_DEPENDFILE}: .NOMETA ${CAT_DEPEND:M.depend} ${META_FILES:O:u:@m@${exists($m): ${_include_src_dirdeps} \ echo '.include '; \ echo; \ - echo '.if $${DEP_RELDIR} == $${_DEP_RELDIR}'; \ + echo '.if ${LOCAL_DEPENDS_GUARD}'; \ echo '# local dependencies - needed for -jN in clean tree'; \ [ -s ${CAT_DEPEND} ] && { grep : ${CAT_DEPEND} | grep -v '[/\\]'; }; \ echo '.endif' ) | sed 's,_\([{(]\),$$\1,g' > $@.new${.MAKE.PID} diff --git a/share/mk/local.autodep.mk b/share/mk/local.autodep.mk index 6d3b3b9c32b6..ee73b23aadcf 100644 --- a/share/mk/local.autodep.mk +++ b/share/mk/local.autodep.mk @@ -8,6 +8,9 @@ UPDATE_DEPENDFILE= no .endif .endif +NOSSPPICO?= .nossppico +OBJ_EXTENSIONS+= ${NOSSPPICO} + CFLAGS+= ${CFLAGS_LAST} CXXFLAGS+= ${CXXFLAGS_LAST} LDFLAGS+= ${LDFLAGS_LAST} diff --git a/share/mk/local.dirdeps.mk b/share/mk/local.dirdeps.mk index 6e1a56d56036..16c9ee5a55ec 100644 --- a/share/mk/local.dirdeps.mk +++ b/share/mk/local.dirdeps.mk @@ -179,8 +179,6 @@ DIRDEPS+= usr.bin/yacc.host .endif _DPADD= ${DPADD} ${_PROGS_DPADD} .if !empty(_DPADD) -# Taken from meta.autodep.mk (where it only does something with -# BUILD_AT_LEVEL0, which we don't use). # This only works for DPADD with full OBJ/SRC paths, which is mostly just # _INTERNALLIBS. _DP_DIRDEPS= \ diff --git a/share/mk/local.gendirdeps.mk b/share/mk/local.gendirdeps.mk index b19e5f64ee74..3ec2be4cc865 100644 --- a/share/mk/local.gendirdeps.mk +++ b/share/mk/local.gendirdeps.mk @@ -67,3 +67,6 @@ GENDIRDEPS_FILTER_VARS+= \ GENDIRDEPS_FILTER+= ${GENDIRDEPS_FILTER_DIR_VARS:@v@S,${$v},_{${v}},@} GENDIRDEPS_FILTER+= ${GENDIRDEPS_FILTER_VARS:@v@S,/${$v}/,/_{${v}}/,@:NS,//,*:u} + +# avoid churn for now +LOCAL_DEPENDS_GUARD= _{DEP_RELDIR} == _{_DEP_RELDIR} diff --git a/share/mk/local.meta.sys.mk b/share/mk/local.meta.sys.mk index e603b4f8091f..b590b37a1d65 100644 --- a/share/mk/local.meta.sys.mk +++ b/share/mk/local.meta.sys.mk @@ -99,13 +99,10 @@ OBJTOP := ${HOST_OBJTOP} .if ${.MAKE.LEVEL} == 0 || empty(PYTHON) PYTHON ?= /usr/local/bin/python .export PYTHON -# this works best if share/mk is ready for it. -BUILD_AT_LEVEL0= no # _SKIP_BUILD is not 100% as it requires wrapping all 'all:' targets to avoid # building in MAKELEVEL0. Just prohibit 'all' entirely in this case to avoid # problems. -.if ${MK_DIRDEPS_BUILD} == "yes" && \ - ${.MAKE.LEVEL} == 0 && ${BUILD_AT_LEVEL0:Uyes:tl} == "no" +.if ${MK_DIRDEPS_BUILD} == "yes" && ${.MAKE.LEVEL} == 0 .MAIN: dirdeps .if make(all) .error DIRDEPS_BUILD: Please run '${MAKE}' instead of '${MAKE} all'. diff --git a/share/mk/meta.autodep.mk b/share/mk/meta.autodep.mk index 7ebded4edc29..4dda9970774e 100644 --- a/share/mk/meta.autodep.mk +++ b/share/mk/meta.autodep.mk @@ -1,5 +1,5 @@ # $FreeBSD$ -# $Id: meta.autodep.mk,v 1.50 2018/06/08 01:25:31 sjg Exp $ +# $Id: meta.autodep.mk,v 1.53 2020/11/08 05:47:56 sjg Exp $ # # @(#) Copyright (c) 2010, Simon J. Gerraty @@ -22,11 +22,10 @@ __${_this}__: .NOTMAIN .-include PICO?= .pico -NOSSPPICO?= .nossppico .if defined(SRCS) # it would be nice to be able to query .SUFFIXES -OBJ_EXTENSIONS+= .o .po .lo ${PICO} ${NOSSPPICO} +OBJ_EXTENSIONS+= .o .po .lo ${PICO} # explicit dependencies help short-circuit .SUFFIX searches SRCS_DEP_FILTER+= N*.[hly] @@ -59,7 +58,7 @@ _OBJTOP ?= ${OBJTOP} _OBJROOT ?= ${OBJROOT:U${_OBJTOP}} _DEPENDFILE := ${_CURDIR}/${.MAKE.DEPENDFILE:T} -.if ${.MAKE.LEVEL} > 0 || ${BUILD_AT_LEVEL0:Uyes:tl} == "yes" +.if ${.MAKE.LEVEL} > 0 # do not allow auto update if we ever built this dir without filemon NO_FILEMON_COOKIE = .nofilemon CLEANFILES += ${NO_FILEMON_COOKIE} @@ -75,10 +74,8 @@ UPDATE_DEPENDFILE = NO .endif .if ${.MAKE.LEVEL} == 0 -.if ${BUILD_AT_LEVEL0:Uyes:tl} == "no" UPDATE_DEPENDFILE = NO .endif -.endif .if !exists(${_DEPENDFILE}) _bootstrap_dirdeps = yes .endif @@ -182,7 +179,8 @@ DEPEND_SUFFIXES += .c .h .cpp .hpp .cxx .hxx .cc .hh @case "${.MAKE.META.FILES:T:M*.po.*}" in \ *.po.*) mv $@.${.MAKE.PID} $@;; \ *) { cat $@.${.MAKE.PID}; \ - sed 's,\${NOSSPPICO}:,.o:,;s,\${PICO}:,.o:,;s,\.o:,.po:,' $@.${.MAKE.PID}; } | sort -u > $@; \ + sed ${OBJ_EXTENSIONS:N.o:N.po:@o@-e 's,\$o:,.o:,'@} \ + -e 's,\.o:,.po:,' $@.${.MAKE.PID}; } | sort -u > $@; \ rm -f $@.${.MAKE.PID};; \ esac .else @@ -285,9 +283,7 @@ ${_DEPENDFILE}: ${_depend} ${.PARSEDIR}/gendirdeps.mk ${META2DEPS} $${.MAKE.MET .endif .if ${_bootstrap_dirdeps} == "yes" -.if ${BUILD_AT_LEVEL0:Uno} == "no" DIRDEPS+= ${RELDIR}.${TARGET_SPEC:U${MACHINE}} -.endif # make sure this is included at least once .include .else @@ -314,7 +310,7 @@ _reldir_finish: .NOMETA _reldir_failed: .NOMETA @echo "${TIME_STAMP} Failed ${RELDIR}.${TARGET_SPEC} seconds=$$(( ${now_utc} - ${start_utc} )) ${meta_stats}" -.if defined(WITH_META_STATS) && ${.MAKE.LEVEL} > 0 +.if !defined(WITHOUT_META_STATS) && ${.MAKE.LEVEL} > 0 .END: _reldir_finish .ERROR: _reldir_failed .endif diff --git a/share/mk/meta.stage.mk b/share/mk/meta.stage.mk index f6eff154fba7..c362f0b590ad 100644 --- a/share/mk/meta.stage.mk +++ b/share/mk/meta.stage.mk @@ -1,15 +1,15 @@ # $FreeBSD$ -# $Id: meta.stage.mk,v 1.55 2017/10/27 01:17:09 sjg Exp $ +# $Id: meta.stage.mk,v 1.60 2020/08/19 17:51:53 sjg Exp $ # # @(#) Copyright (c) 2011-2017, Simon J. Gerraty # # This file is provided in the hope that it will # be of use. There is absolutely NO WARRANTY. # Permission to copy, redistribute or otherwise -# use this file is hereby granted provided that +# use this file is hereby granted provided that # the above copyright notice and this notice are -# left intact. -# +# left intact. +# # Please send copies of changes and bug-fixes to: # sjg@crufty.net # @@ -19,9 +19,11 @@ .if !target(__${.PARSEFILE}__) # the guard target is defined later +.-include + .if ${.MAKE.DEPENDFILE_PREFERENCE:U${.MAKE.DEPENDFILE}:M*.${MACHINE}} != "" # this is generally safer anyway -_dirdep ?= ${RELDIR}.${MACHINE} +_dirdep ?= ${RELDIR}.${TARGET_SPEC:U${MACHINE}} .else _dirdep ?= ${RELDIR} .endif @@ -68,7 +70,7 @@ LN_CP_SCRIPT = LnCp() { \ # a warning is handy when bootstapping different options. STAGE_CONFLICT?= ERROR .if ${STAGE_CONFLICT:tl} == "error" -STAGE_CONFLICT_ACTION= exit 1; +STAGE_CONFLICT_ACTION= exit 1 .else STAGE_CONFLICT_ACTION= .endif @@ -79,8 +81,10 @@ STAGE_DIRDEP_SCRIPT = ${LN_CP_SCRIPT}; StageDirdep() { \ t=$$1; \ if [ -s $$t.dirdep ]; then \ cmp -s .dirdep $$t.dirdep && return; \ - echo "${STAGE_CONFLICT}: $$t installed by `cat $$t.dirdep` not ${_dirdep}" >&2; \ - ${STAGE_CONFLICT_ACTION} \ + x=`cat $$t.dirdep`; \ + case "${RELDIR}:${_dirdep}" in $${x%.*}:$${x}*) ;; \ + *) echo "${STAGE_CONFLICT}: $$t installed by $$x not ${_dirdep}" >&2; \ + ${STAGE_CONFLICT_ACTION} ;; esac; \ fi; \ LnCp .dirdep $$t.dirdep || exit 1; } @@ -142,7 +146,7 @@ _STAGE_AS_BASENAME_USE: .USE .dirdep ${.TARGET:T} .if !empty(STAGE_INCSDIR) .if !empty(STAGE_INCS) -stage_incs: ${STAGE_INCS} +stage_incs: ${STAGE_INCS:N*\**} .endif .if target(stage_incs) || !empty(.ALLTARGETS:Mstage_includes) STAGE_TARGETS += stage_incs @@ -157,7 +161,7 @@ stage_incs: .dirdep .if !empty(STAGE_LIBDIR) .if !empty(STAGE_LIBS) -stage_libs: ${STAGE_LIBS} +stage_libs: ${STAGE_LIBS:N*\**} .endif .if target(stage_libs) STAGE_TARGETS += stage_libs @@ -192,7 +196,7 @@ CLEANFILES += ${STAGE_SETS:@s@stage*$s@} # some makefiles need to populate multiple directories .for s in ${STAGE_SETS:O:u} .if !empty(STAGE_FILES.$s) -stage_files.$s: ${STAGE_FILES.$s} +stage_files.$s: ${STAGE_FILES.$s:N*\**} .endif .if target(stage_files.$s) || target(stage_files${s:S,^,.,:N._default}) STAGE_TARGETS += stage_files @@ -206,7 +210,7 @@ stage_files.$s: .dirdep STAGE_FILES ?= ${.ALLSRC:N.dirdep:Nstage_*} stage_files: .dirdep .endif - @${STAGE_FILE_SCRIPT}; StageFiles ${FLAGS.$@} ${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}} ${STAGE_FILES.$s} + @${STAGE_FILE_SCRIPT}; StageFiles ${FLAGS.$@} ${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}} ${STAGE_FILES.$s:O} @touch $@ .endif .endif @@ -263,7 +267,7 @@ CLEANFILES += ${STAGE_AS_SETS:@s@stage*$s@} # both operations happen together .for s in ${STAGE_AS_SETS:O:u} .if !empty(STAGE_AS.$s) -stage_as.$s: ${STAGE_AS.$s} +stage_as.$s: ${STAGE_AS.$s:N*\**} .endif .if target(stage_as.$s) STAGE_TARGETS += stage_as @@ -272,13 +276,13 @@ STAGE_AS.$s ?= ${.ALLSRC:N.dirdep:Nstage_*} .stage_as.$s: stage_as: stage_as.$s stage_as.$s: .dirdep - @${STAGE_AS_SCRIPT}; StageAs ${FLAGS.$@} ${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}} ${STAGE_AS.$s:@f@$f ${STAGE_AS_${f:tA}:U${STAGE_AS_${f:T}:U${f:T}}}@} + @${STAGE_AS_SCRIPT}; StageAs ${FLAGS.$@} ${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}} ${STAGE_AS.$s:O:@f@$f ${STAGE_AS_${f:tA}:U${STAGE_AS_${f:T}:U${f:T}}}@} @touch $@ .endif .endif .if !empty(STAGE_AS_AND_SYMLINK.$s) -stage_as_and_symlink.$s: ${STAGE_AS_AND_SYMLINK.$s} +stage_as_and_symlink.$s: ${STAGE_AS_AND_SYMLINK.$s:N*\**} .endif .if target(stage_as_and_symlink.$s) STAGE_TARGETS += stage_as_and_symlink @@ -287,8 +291,8 @@ STAGE_AS_AND_SYMLINK.$s ?= ${.ALLSRC:N.dirdep:Nstage_*} .stage_as_and_symlink.$s: stage_as_and_symlink: stage_as_and_symlink.$s stage_as_and_symlink.$s: .dirdep - @${STAGE_AS_SCRIPT}; StageAs ${FLAGS.$@} ${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}} ${STAGE_AS_AND_SYMLINK.$s:@f@$f ${STAGE_AS_${f:tA}:U${STAGE_AS_${f:T}:U${f:T}}}@} - @${STAGE_LINKS_SCRIPT}; StageLinks -s ${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}} ${STAGE_AS_AND_SYMLINK.$s:@f@${STAGE_AS_${f:tA}:U${STAGE_AS_${f:T}:U${f:T}}} $f@} + @${STAGE_AS_SCRIPT}; StageAs ${FLAGS.$@} ${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}} ${STAGE_AS_AND_SYMLINK.$s:O:@f@$f ${STAGE_AS_${f:tA}:U${STAGE_AS_${f:T}:U${f:T}}}@} + @${STAGE_LINKS_SCRIPT}; StageLinks -s ${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}} ${STAGE_AS_AND_SYMLINK.$s:O:@f@${STAGE_AS_${f:tA}:U${STAGE_AS_${f:T}:U${f:T}}} $f@} @touch $@ .endif .endif diff --git a/share/mk/meta.subdir.mk b/share/mk/meta.subdir.mk index 966b6e0630f3..781a52856b64 100644 --- a/share/mk/meta.subdir.mk +++ b/share/mk/meta.subdir.mk @@ -1,5 +1,5 @@ # $FreeBSD$ -# $Id: meta.subdir.mk,v 1.11 2015/11/24 22:26:51 sjg Exp $ +# $Id: meta.subdir.mk,v 1.12 2020/08/19 17:51:53 sjg Exp $ # # @(#) Copyright (c) 2010, Simon J. Gerraty @@ -7,10 +7,10 @@ # This file is provided in the hope that it will # be of use. There is absolutely NO WARRANTY. # Permission to copy, redistribute or otherwise -# use this file is hereby granted provided that +# use this file is hereby granted provided that # the above copyright notice and this notice are -# left intact. -# +# left intact. +# # Please send copies of changes and bug-fixes to: # sjg@crufty.net # @@ -33,7 +33,7 @@ DIRDEPS = ${SUBDIR:N.WAIT:O:u:@d@${RELDIR}/$d@} .include .else # this is the cunning bit -# actually it is probably a bit risky +# actually it is probably a bit risky # since we may pickup subdirs which are not relevant # the alternative is a walk through the tree though # which is difficult without a sub-make. diff --git a/share/mk/meta.sys.mk b/share/mk/meta.sys.mk index 8c39757e15af..94e6ef862fdb 100644 --- a/share/mk/meta.sys.mk +++ b/share/mk/meta.sys.mk @@ -1,16 +1,16 @@ # $FreeBSD$ -# $Id: meta.sys.mk,v 1.34 2019/01/24 19:36:25 sjg Exp $ +# $Id: meta.sys.mk,v 1.38 2020/08/19 17:51:53 sjg Exp $ # -# @(#) Copyright (c) 2010, Simon J. Gerraty +# @(#) Copyright (c) 2010-2020, Simon J. Gerraty # # This file is provided in the hope that it will # be of use. There is absolutely NO WARRANTY. # Permission to copy, redistribute or otherwise -# use this file is hereby granted provided that +# use this file is hereby granted provided that # the above copyright notice and this notice are -# left intact. -# +# left intact. +# # Please send copies of changes and bug-fixes to: # sjg@crufty.net # @@ -129,9 +129,11 @@ META_NOECHO= : # ignore mtime of shell # and mtime of makefiles does not matter in meta mode .MAKE.META.IGNORE_PATHS += \ - ${MAKEFILE} \ - ${SHELL} \ - ${SYS_MK_DIR} + ${MAKEFILE} \ + ${MAKE_SHELL} \ + ${SHELL} \ + ${SYS_MK_DIR} \ + .if ${UPDATE_DEPENDFILE:Uyes:tl} != "no" .if ${.MAKEFLAGS:Uno:M-k} != "" @@ -154,12 +156,6 @@ dirdeps: # the first .MAIN: is what counts # by default dirdeps is all we want at level0 .MAIN: dirdeps -# tell dirdeps.mk what we want -BUILD_AT_LEVEL0 = no -.endif -.if ${.TARGETS:Nall} == "" -# it works best if we do everything via sub-makes -BUILD_AT_LEVEL0 ?= no .endif .endif diff --git a/share/mk/meta2deps.py b/share/mk/meta2deps.py index 700dc2073b8a..42e4bb353446 100755 --- a/share/mk/meta2deps.py +++ b/share/mk/meta2deps.py @@ -25,7 +25,7 @@ We only pay attention to a subset of the information in the 'W' files opened for write or read-write, for filemon V3 and earlier. - + 'E' files executed. 'L' files linked @@ -38,19 +38,20 @@ We only pay attention to a subset of the information in the """ RCSid: $FreeBSD$ - $Id: meta2deps.py,v 1.27 2017/05/24 00:04:04 sjg Exp $ + $Id: meta2deps.py,v 1.34 2020/10/02 03:11:17 sjg Exp $ - Copyright (c) 2011-2013, Juniper Networks, Inc. + Copyright (c) 2011-2020, Simon J. Gerraty + Copyright (c) 2011-2017, Juniper Networks, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: + modification, are permitted provided that the following conditions + are met: 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. + notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. + documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT @@ -62,8 +63,8 @@ RCSid: DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + """ import os, re, sys @@ -81,7 +82,11 @@ def resolve(path, cwd, last_dir=None, debug=0, debug_out=sys.stderr): if path.endswith('/.'): path = path[0:-2] if len(path) > 0 and path[0] == '/': - return path + if os.path.exists(path): + return path + if debug > 2: + print("skipping non-existent:", path, file=debug_out) + return None if path == '.': return cwd if path.startswith('./'): @@ -139,6 +144,8 @@ def abspath(path, cwd, last_dir=None, debug=0, debug_out=sys.stderr): rpath = resolve(path, cwd, last_dir, debug, debug_out) if rpath: path = rpath + elif len(path) > 0 and path[0] == '/': + return None if (path.find('/') < 0 or path.find('./') > 0 or path.endswith('/..')): @@ -158,7 +165,7 @@ def sort_unique(list, cmp=None, key=None, reverse=False): def add_trims(x): return ['/' + x + '/', - '/' + x, + '/' + x, x + '/', x] @@ -175,7 +182,7 @@ class MetaFile: obj_deps = [] src_deps = [] file_deps = [] - + def __init__(self, name, conf={}): """if name is set we will parse it now. conf can have the follwing keys: @@ -192,7 +199,7 @@ class MetaFile: TARGET_SPEC Sometimes MACHINE isn't enough. - + HOST_TARGET when we build for the pseudo machine 'host' the object tree uses HOST_TARGET rather than MACHINE. @@ -216,7 +223,7 @@ class MetaFile: debug_out open file to send debug output to (sys.stderr) """ - + self.name = name self.debug = getv(conf, 'debug', 0) self.debug_out = getv(conf, 'debug_out', sys.stderr) @@ -304,11 +311,11 @@ class MetaFile: self.obj_deps = [] self.src_deps = [] self.file_deps = [] - + def dirdeps(self, sep='\n'): """return DIRDEPS""" return sep.strip() + sep.join(self.obj_deps) - + def src_dirdeps(self, sep='\n'): """return SRC_DIRDEPS""" return sep.strip() + sep.join(self.src_deps) @@ -327,7 +334,7 @@ class MetaFile: def seenit(self, dir): """rememer that we have seen dir.""" self.seen[dir] = 1 - + def add(self, list, data, clue=''): """add data to list if it isn't already there.""" if data not in list: @@ -386,10 +393,10 @@ class MetaFile: # give a useful clue print('{}:{}: '.format(self.name, self.line), end=' ', file=sys.stderr) raise - + def parse(self, name=None, file=None): """A meta file looks like: - + # Meta data file "path" CMD "command-line" CWD "cwd" @@ -475,6 +482,10 @@ class MetaFile: continue elif w[0] == 'C': cwd = abspath(w[2], cwd, None, self.debug, self.debug_out) + if not cwd: + cwd = w[2] + if self.debug > 1: + print("missing cwd=", cwd, file=self.debug_out) if cwd.endswith('/.'): cwd = cwd[0:-2] self.last_dir = pid_last_dir[pid] = cwd @@ -491,13 +502,16 @@ class MetaFile: if w[0] in 'ML': # these are special, tread src as read and # target as write - self.parse_path(w[1].strip("'"), cwd, 'R', w) - self.parse_path(w[2].strip("'"), cwd, 'W', w) + self.parse_path(w[2].strip("'"), cwd, 'R', w) + self.parse_path(w[3].strip("'"), cwd, 'W', w) continue elif w[0] in 'ERWS': path = w[2] + if path == '.': + continue self.parse_path(path, cwd, w[0], w) + assert(version > 0) if not file: f.close() @@ -563,7 +577,7 @@ class MetaFile: print("ldir=", self.last_dir, file=self.debug_out) return - if op in 'ERW': + if op in 'ER': # finally, we get down to it if dir == self.cwd or dir == self.curdir: return @@ -591,13 +605,13 @@ class MetaFile: self.seenit(w[2]) self.seenit(dir) - + def main(argv, klass=MetaFile, xopts='', xoptf=None): """Simple driver for class MetaFile. Usage: script [options] [key=value ...] "meta" ... - + Options and key=value pairs contribute to the dictionary passed to MetaFile. @@ -605,7 +619,7 @@ def main(argv, klass=MetaFile, xopts='', xoptf=None): add "SRCTOP" to the "SRCTOPS" list. -C "CURDIR" - + -O "OBJROOT" add "OBJROOT" to the "OBJROOTS" list. @@ -616,7 +630,7 @@ def main(argv, klass=MetaFile, xopts='', xoptf=None): -H "HOST_TARGET" -D "DPDEPS" - + -d bumps debug level """ @@ -656,7 +670,7 @@ def main(argv, klass=MetaFile, xopts='', xoptf=None): debug = 0 output = True - + opts, args = getopt.getopt(argv[1:], 'a:dS:C:O:R:m:D:H:qT:X:' + xopts) for o, a in opts: if o == '-a': diff --git a/share/mk/meta2deps.sh b/share/mk/meta2deps.sh index d2536e6063ed..4767b83f4343 100755 --- a/share/mk/meta2deps.sh +++ b/share/mk/meta2deps.sh @@ -5,11 +5,11 @@ # # SYNOPSIS: # meta2deps.sh SB="SB" "meta" ... -# +# # DESCRIPTION: # This script looks each "meta" file and extracts the # information needed to deduce build and src dependencies. -# +# # To do this, we extract the 'CWD' record as well as all the # syscall traces which describe 'R'ead, 'C'hdir and 'E'xec # syscalls. @@ -76,20 +76,20 @@ # RCSid: # $FreeBSD$ -# $Id: meta2deps.sh,v 1.12 2016/12/13 20:44:16 sjg Exp $ +# $Id: meta2deps.sh,v 1.14 2020/10/02 03:11:17 sjg Exp $ # Copyright (c) 2010-2013, Juniper Networks, Inc. # All rights reserved. -# +# # Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: +# modification, are permitted provided that the following conditions +# are met: # 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. +# notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# +# documentation and/or other materials provided with the distribution. +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR @@ -100,14 +100,14 @@ # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. meta2src() { cat /dev/null "$@" | sed -n '/^R .*\.[chyl]$/s,^..[0-9]* ,,p' | sort -u } - + meta2dirs() { cat /dev/null "$@" | sed -n '/^R .*\/.*\.[a-z0-9][^\/]*$/s,^..[0-9]* \(.*\)/[^/]*$,\1,p' | @@ -142,6 +142,11 @@ _excludes_f() { egrep -v "$EXCLUDES" } +error() { + echo "ERROR: $@" >&2 + exit 1 +} + meta2deps() { DPDEPS= SRCTOPS=$SRCTOP @@ -168,7 +173,7 @@ meta2deps() { case "$MACHINE" in host) _ht=$HOST_TARGET;; esac - + for o in $OBJROOTS do case "$MACHINE,/$o/" in @@ -202,7 +207,7 @@ meta2deps() { obj_re= add_list '|' -s '/*' src_re $SRCTOPS add_list '|' -s '*' obj_re $OBJROOTS - + [ -z "$RELDIR" ] && unset DPDEPS tf=/tmp/m2d$$-$USER rm -f $tf.* @@ -235,8 +240,8 @@ meta2deps() { ;; *) cat /dev/null "$@";; esac 2> /dev/null | - sed -e 's,^CWD,C C,;/^[CREFLM] /!d' -e "s,',,g" | - $_excludes | + sed -e 's,^CWD,C C,;/^[CREFLMV] /!d' -e "s,',,g" | + $_excludes | ( version=no while read op pid path junk do : op=$op pid=$pid path=$path @@ -248,6 +253,12 @@ meta2deps() { SB=`echo $CWD | sed 's,/obj.*,,'` fi SRCTOP=${SRCTOP:-$SB/src} + case "$verion" in + no) ;; # ignore + 0) error "no filemon data";; + *) ;; + esac + version=0 continue ;; $pid,$pid) ;; @@ -262,6 +273,7 @@ meta2deps() { esac case "$op,$path" in + V,*) version=$path; continue;; W,*srcrel|*.dirdep) continue;; C,*) case "$path" in @@ -273,10 +285,10 @@ meta2deps() { eval cwd_$pid=$cwd continue ;; - F,*) # $path is new pid + F,*) # $path is new pid eval cwd_$path=$cwd ldir_$path=$ldir continue - ;; + ;; *) dir=${path%/*} case "$path" in $src_re|$obj_re) ;; @@ -367,6 +379,9 @@ meta2deps() { echo $dir;; esac done > $tf.dirdep + case "$version" in + 0) error "no filemon data";; + esac ) || exit 1 _nl=echo for f in $tf.dirdep $tf.qual $tf.srcdep do