Merge remote-tracking branch 'internal/freebsd/current/main' into hardened/current/master

Conflicts:
	share/mk/bsd.opts.mk (unresolved)

Makefile

@@ -521,9 +521,6 @@ worlds: .PHONY
 # Don't build rarely used, semi-supported architectures unless requested.
 #
 .if defined(EXTRA_TARGETS)
-# armv6's importance has waned enough to make building it the exception rather
-# than the rule.
-EXTRA_ARCHES_arm=	armv6
 # powerpcspe excluded from main list until clang fixed
 EXTRA_ARCHES_powerpc=	powerpcspe
 .endif
@@ -535,8 +532,7 @@ TARGET_ARCHES_${target}= ${MACHINE_ARCH_LIST_${target}}
 
 .if defined(USE_GCC_TOOLCHAINS)
 TOOLCHAINS_amd64=	amd64-gcc12
-TOOLCHAINS_arm=		armv6-gcc12 armv7-gcc12
+TOOLCHAINS_arm=		armv7-gcc12
-TOOLCHAIN_armv7=	armv7-gcc12
 TOOLCHAINS_arm64=	aarch64-gcc12
 TOOLCHAINS_i386=	i386-gcc12
 TOOLCHAINS_powerpc=	powerpc-gcc12 powerpc64-gcc12

Makefile.inc1

@@ -146,7 +146,6 @@ TARGET_TRIPLE_ABI?=	unknown
 TARGET_TRIPLE?=	${TARGET_ARCH:S/amd64/x86_64/}-${TARGET_TRIPLE_ABI}-freebsd${OS_REVISION}
 KNOWN_ARCHES?=	aarch64/arm64 \
 		amd64 \
-		armv6/arm \
 		armv7/arm \
 		i386 \
 		powerpc \

UPDATING

@@ -27,6 +27,9 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 15.x IS SLOW:
 	world, or to merely disable the most expensive debugging functionality
 	at runtime, run "ln -s 'abort:false,junk:false' /etc/malloc.conf".)
 
+20240712:
+	Support for armv6 has been disconnected and is being removed.
+
 20240617:
 	ifconfig now treats IPv4 addresses without a width or mask as an error.
 	Specify the desired mask or width along with the IP address on the

lib/libclang_rt/compiler-rt-vars.mk

@@ -5,7 +5,7 @@ SANITIZER_SHAREDIR=		${CLANGDIR}/share
 
 # armv[67] is a bit special since we allow a soft-floating version via
 # CPUTYPE matching *soft*. This variant may not actually work though.
-.if ${MACHINE_ARCH:Marmv[67]*} != "" && \
+.if ${MACHINE_CPUARCH} == "arm" && \
     (!defined(CPUTYPE) || ${CPUTYPE:M*soft*} == "")
 CRTARCH?=	armhf
 .else

lib/libipsec/pfkey.c

@@ -1677,7 +1677,8 @@ pfkey_align(struct sadb_msg *msg, caddr_t *mhp)
 
 		/* duplicate check */
 		/* XXX Are there duplication either KEY_AUTH or KEY_ENCRYPT ?*/
-		if (mhp[ext->sadb_ext_type] != NULL) {
+		if (mhp[ext->sadb_ext_type] != NULL &&
+		    ext->sadb_ext_type != SADB_X_EXT_IF_HW_OFFL /* XXXKIB */) {
 			__ipsec_errcode = EIPSEC_INVAL_EXTTYPE;
 			return -1;
 		}
@@ -1713,6 +1714,9 @@ pfkey_align(struct sadb_msg *msg, caddr_t *mhp)
 		case SADB_X_EXT_SA_REPLAY:
 		case SADB_X_EXT_NEW_ADDRESS_SRC:
 		case SADB_X_EXT_NEW_ADDRESS_DST:
+		case SADB_X_EXT_LFT_CUR_SW_OFFL:
+		case SADB_X_EXT_LFT_CUR_HW_OFFL:
+		case SADB_X_EXT_IF_HW_OFFL:
 			mhp[ext->sadb_ext_type] = (caddr_t)ext;
 			break;
 		default:

lib/libipsec/pfkey_dump.c

@@ -43,6 +43,7 @@
 #include <netinet/in.h>
 #include <arpa/inet.h>
 
+#include <stdbool.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include <stdio.h>
@@ -201,7 +202,7 @@ pfkey_sadump(struct sadb_msg *m)
 	caddr_t mhp[SADB_EXT_MAX + 1];
 	struct sadb_sa *m_sa;
 	struct sadb_x_sa2 *m_sa2;
-	struct sadb_lifetime *m_lftc, *m_lfth, *m_lfts;
+	struct sadb_lifetime *m_lftc, *m_lfth, *m_lfts, *m_lft_sw, *m_lft_hw;
 	struct sadb_address *m_saddr, *m_daddr, *m_paddr;
 	struct sadb_key *m_auth, *m_enc;
 	struct sadb_ident *m_sid, *m_did;
@@ -210,6 +211,10 @@ pfkey_sadump(struct sadb_msg *m)
 	struct sadb_x_nat_t_type *natt_type;
 	struct sadb_x_nat_t_port *natt_sport, *natt_dport;
 	struct sadb_address *natt_oai, *natt_oar;
+	struct sadb_x_if_hw_offl *if_hw_offl;
+	caddr_t p, ep;
+	struct sadb_ext *ext;
+	bool first;
 
 	/* check pfkey message. */
 	if (pfkey_align(m, mhp)) {
@@ -240,7 +245,9 @@ pfkey_sadump(struct sadb_msg *m)
 	natt_dport = (struct sadb_x_nat_t_port *)mhp[SADB_X_EXT_NAT_T_DPORT];
 	natt_oai = (struct sadb_address *)mhp[SADB_X_EXT_NAT_T_OAI];
 	natt_oar = (struct sadb_address *)mhp[SADB_X_EXT_NAT_T_OAR];
-
+	m_lft_sw = (struct sadb_lifetime *)mhp[SADB_X_EXT_LFT_CUR_SW_OFFL];
+	m_lft_hw = (struct sadb_lifetime *)mhp[SADB_X_EXT_LFT_CUR_HW_OFFL];
+	if_hw_offl = (struct sadb_x_if_hw_offl *)mhp[SADB_X_EXT_IF_HW_OFFL];
 
 	/* source address */
 	if (m_saddr == NULL) {
@@ -332,6 +339,27 @@ pfkey_sadump(struct sadb_msg *m)
 	GETMSGSTR(str_state, m_sa->sadb_sa_state);
 	printf("\n");
 
+	/* hw offload interface */
+	if (if_hw_offl != NULL) {
+		p = (caddr_t)m;
+		ep = p + PFKEY_UNUNIT64(m->sadb_msg_len);
+		p += sizeof(struct sadb_msg);
+		printf("\thw offl if: ");
+
+		for (first = true; p < ep; p += PFKEY_EXTLEN(ext)) {
+			ext = (struct sadb_ext *)p;
+			if (ext->sadb_ext_type != SADB_X_EXT_IF_HW_OFFL)
+				continue;
+			if_hw_offl = (struct sadb_x_if_hw_offl *)ext;
+			if (first)
+				first = false;
+			else
+				printf(",");
+			printf("%s", if_hw_offl->sadb_x_if_hw_offl_if);
+		}
+		printf("\n");
+	}
+
 	/* lifetime */
 	if (m_lftc != NULL) {
 		time_t tmp_time = time(0);
@@ -381,7 +409,23 @@ pfkey_sadump(struct sadb_msg *m)
 	/* XXX DEBUG */
 	printf("refcnt=%u\n", m->sadb_msg_reserved);
 
-	return;
+	if (m_lft_sw != NULL) {
+		printf("\tsw offl use: %s",
+		    str_time(m_lft_sw->sadb_lifetime_usetime));
+		printf("\tsw offl allocated: %lu",
+		    (unsigned long)m_lft_sw->sadb_lifetime_allocations);
+		str_lifetime_byte(m_lft_sw, "sw offl");
+		printf("\n");
+	}
+
+	if (m_lft_hw != NULL) {
+		printf("\thw offl use: %s",
+		    str_time(m_lft_hw->sadb_lifetime_usetime));
+		printf("\thw offl allocated: %lu",
+		    (unsigned long)m_lft_hw->sadb_lifetime_allocations);
+		str_lifetime_byte(m_lft_hw, "hw offl");
+		printf("\n");
+	}
 }
 
 void

sbin/ifconfig/ifconfig.c

@@ -2144,6 +2144,8 @@ static struct cmd basic_cmds[] = {
 	    setifcapnv),
 	DEF_CMD_SARG("-rxtls",	"-"IFCAP2_RXTLS4_NAME ",-" IFCAP2_RXTLS6_NAME,
 	    setifcapnv),
+	DEF_CMD_SARG("ipsec",	IFCAP2_IPSEC_OFFLOAD_NAME, setifcapnv),
+	DEF_CMD_SARG("-ipsec",	"-"IFCAP2_IPSEC_OFFLOAD_NAME, setifcapnv),
 	DEF_CMD("wol",		IFCAP_WOL,	setifcap),
 	DEF_CMD("-wol",		IFCAP_WOL,	clearifcap),
 	DEF_CMD("wol_ucast",	IFCAP_WOL_UCAST,	setifcap),

sbin/nvmecontrol/power.c

@@ -90,7 +90,7 @@ power_list(struct nvme_controller_data *cdata)
 	int i;
 
 	printf("\nPower States Supported: %d\n\n", cdata->npss + 1);
-	printf(" #   Max pwr  Enter Lat  Exit Lat RT RL WT WL Idle Pwr  Act Pwr Workloadd\n");
+	printf(" #   Max pwr  Enter Lat  Exit Lat RT RL WT WL Idle Pwr  Act Pwr Workload\n");
 	printf("--  --------  --------- --------- -- -- -- -- -------- -------- --\n");
 	for (i = 0; i <= cdata->npss; i++)
 		power_list_one(i, &cdata->power_state[i]);

sbin/setkey/parse.y

@@ -46,6 +46,7 @@
 
 #include <string.h>
 #include <unistd.h>
+#include <stdbool.h>
 #include <stdio.h>
 #include <stdint.h>
 #include <netdb.h>
@@ -68,6 +69,8 @@ u_int p_natt_type;
 struct addrinfo *p_natt_oai, *p_natt_oar;
 int p_natt_sport, p_natt_dport;
 int p_natt_fraglen;
+bool esn;
+vchar_t p_hwif;
 
 static int p_aiflags = 0, p_aifamily = PF_UNSPEC;
 
@@ -115,6 +118,7 @@ extern void yyerror(const char *);
 %token SPDADD SPDDELETE SPDDUMP SPDFLUSH
 %token F_POLICY PL_REQUESTS
 %token F_AIFLAGS F_NATT F_NATT_MTU
+%token F_ESN F_HWIF
 %token TAGGED
 
 %type <num> prefix protocol_spec upper_spec
@@ -539,12 +543,21 @@ extension
 		{
 			p_natt_fraglen = $2;
 		}
+	|	F_ESN
+		{
+			esn = true;
+			p_ext |= SADB_X_SAFLAGS_ESN;
+		}
+	|	F_HWIF STRING
+		{
+			p_hwif = $2;
+		}
 	;
 
 	/* definition about command for SPD management */
 	/* spdadd */
 spdadd_command
-	:	SPDADD ipaddropts STRING prefix portstr STRING prefix portstr upper_spec upper_misc_spec policy_spec EOT
+	:	SPDADD ipaddropts STRING prefix portstr STRING prefix portstr upper_spec upper_misc_spec policy_spec spd_hwif EOT
 		{
 			int status;
 			struct addrinfo *src, *dst;
@@ -648,6 +661,14 @@ ipaddropts
 	|	ipaddropts ipaddropt
 	;
 
+spd_hwif
+	:
+	|	F_HWIF STRING
+		{
+			p_hwif = $2;
+		}
+	;
+
 ipaddropt
 	:	F_AIFLAGS
 		{
@@ -831,6 +852,7 @@ setkeymsg_spdaddr(unsigned type, unsigned upper, vchar_t *policy,
 	char buf[BUFSIZ];
 	int l, l0;
 	struct sadb_address m_addr;
+	struct sadb_x_if_hw_offl m_if_hw;
 	struct addrinfo *s, *d;
 	int n;
 	int plen;
@@ -849,6 +871,20 @@ setkeymsg_spdaddr(unsigned type, unsigned upper, vchar_t *policy,
 	memcpy(buf + l, policy->buf, policy->len);
 	l += policy->len;
 
+	if (p_hwif.len != 0) {
+		l0 = sizeof(struct sadb_x_if_hw_offl);
+		m_if_hw.sadb_x_if_hw_offl_len = PFKEY_UNIT64(l0);
+		m_if_hw.sadb_x_if_hw_offl_exttype = SADB_X_EXT_IF_HW_OFFL;
+		m_if_hw.sadb_x_if_hw_offl_flags = 0;
+		memset(&m_if_hw.sadb_x_if_hw_offl_if[0], 0,
+		    sizeof(m_if_hw.sadb_x_if_hw_offl_if));
+		strlcpy(&m_if_hw.sadb_x_if_hw_offl_if[0], p_hwif.buf,
+		    sizeof(m_if_hw.sadb_x_if_hw_offl_if));
+
+		memcpy(buf + l, &m_if_hw, l0);
+		l += l0;
+	}
+
 	l0 = l;
 	n = 0;
 
@@ -1040,6 +1076,7 @@ setkeymsg_add(unsigned type, unsigned satype, struct addrinfo *srcs,
 	struct sadb_x_nat_t_type m_natt_type;
 	struct sadb_x_nat_t_port m_natt_port;
 	struct sadb_x_nat_t_frag m_natt_frag;
+	struct sadb_x_if_hw_offl m_if_hw;
 	int n;
 	int plen;
 	struct sockaddr *sa;
@@ -1256,6 +1293,20 @@ setkeymsg_add(unsigned type, unsigned satype, struct addrinfo *srcs,
 		}
 	}
 
+	if (p_hwif.len != 0) {
+		len = sizeof(struct sadb_x_if_hw_offl);
+		m_if_hw.sadb_x_if_hw_offl_len = PFKEY_UNIT64(len);
+		m_if_hw.sadb_x_if_hw_offl_exttype = SADB_X_EXT_IF_HW_OFFL;
+		m_if_hw.sadb_x_if_hw_offl_flags = 0;
+		memset(&m_if_hw.sadb_x_if_hw_offl_if[0], 0,
+		    sizeof(m_if_hw.sadb_x_if_hw_offl_if));
+		strlcpy(&m_if_hw.sadb_x_if_hw_offl_if[0], p_hwif.buf,
+		    sizeof(m_if_hw.sadb_x_if_hw_offl_if));
+
+		memcpy(buf + l, &m_if_hw, len);
+		l += len;
+	}
+
 	if (n == 0)
 		return -1;
 	else
@@ -1355,6 +1406,10 @@ parse_init(void)
 	p_natt_oai = p_natt_oar = NULL;
 	p_natt_sport = p_natt_dport = 0;
 	p_natt_fraglen = -1;
+
+	esn = false;
+	p_hwif.len = 0;
+	p_hwif.buf = NULL;
 }
 
 void

sbin/setkey/setkey.8

@@ -341,6 +341,8 @@ symbols are part of the syntax for the ports specification,
 not indication of the optional components.
 .It Fl natt_mtu Ar fragsize
 Configure NAT-T fragment size.
+.It Fl esn
+Enable Extended Sequence Number extension for this SA.
 .El
 .\"
 .Pp

sbin/setkey/token.l

@@ -187,6 +187,8 @@ nocyclic-seq	{ return(NOCYCLICSEQ); }
 {hyphen}ls	{ return(F_LIFETIME_SOFT); }
 {hyphen}natt	{ return(F_NATT); }
 {hyphen}natt_mtu { return(F_NATT_MTU); }
+{hyphen}esn	{ return(F_ESN); }
+{hyphen}hwif	{ return(F_HWIF); }
 
 	/* ... */
 any		{ return(ANY); }

share/mk/bsd.cpu.mk

@@ -315,13 +315,10 @@ MACHINE_CPU = riscv
 ########## arm
 .if ${MACHINE_CPUARCH} == "arm"
 MACHINE_CPU += arm
-. if ${MACHINE_ARCH:Marmv6*} != ""
-MACHINE_CPU += armv6
-. endif
 . if ${MACHINE_ARCH:Marmv7*} != ""
 MACHINE_CPU += armv7
 . endif
-# Normally armv6 and armv7 are hard float ABI from FreeBSD 11 onwards. However
+# Normally armv7 is hard float ABI from FreeBSD 11 onwards. However
 # when CPUTYPE has 'soft' in it, we use the soft-float ABI to allow building of
 # soft-float ABI libraries. In this case, we have to add the -mfloat-abi=softfp
 # to force that.

share/mk/bsd.opts.mk

@@ -87,6 +87,21 @@ __DEFAULT_DEPENDENT_OPTIONS = \
     STAGING_PROG/STAGING \
     STALE_STAGED/STAGING \
 
+<<<<<<< HEAD
+=======
+#
+# Default to disabling PIE on 32-bit architectures. The small address space
+# means that ASLR is of limited effectiveness, and it may cause issues with
+# some memory-hungry workloads.
+#
+.if ${MACHINE_ARCH} == "armv7" \
+    || ${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "powerpc" \
+    || ${MACHINE_ARCH} == "powerpcspe"
+__DEFAULT_NO_OPTIONS+= PIE
+.else
+__DEFAULT_YES_OPTIONS+=PIE
+.endif
+>>>>>>> internal/freebsd/current/main
 
 .if ${MACHINE_CPUARCH} != "aarch64"
 BROKEN_OPTIONS+=	BRANCH_PROTECTION

sys/amd64/conf/GENERIC

@@ -30,6 +30,7 @@ options 	VIMAGE			# Subsystem virtualization, e.g. VNET
 options 	INET			# InterNETworking
 options 	INET6			# IPv6 communications protocols
 options 	IPSEC_SUPPORT		# Allow kldload of ipsec and tcpmd5
+options 	IPSEC_OFFLOAD		# Inline ipsec offload infra
 options 	ROUTE_MPATH		# Multipath routing support
 options 	FIB_ALGO		# Modular fib lookups
 options 	TCP_OFFLOAD		# TCP offload

sys/arm/conf/RPI-B

@@ -1,109 +0,0 @@
-#
-# RPI-B -- Custom configuration for the Raspberry Pi
-#
-# For more information on this file, please read the config(5) manual page,
-# and/or the handbook section on Kernel Configuration Files:
-#
-#    https://docs.freebsd.org/en/books/handbook/kernelconfig/#kernelconfig-config
-#
-# The handbook is also available locally in /usr/share/doc/handbook
-# if you've installed the doc distribution, otherwise always see the
-# FreeBSD World Wide Web server (https://www.FreeBSD.org/) for the
-# latest information.
-#
-# An exhaustive list of options and more detailed explanations of the
-# device lines is also present in the ../../conf/NOTES and NOTES files.
-# If you are in doubt as to the purpose or necessity of a line, check first
-# in NOTES.
-#
-
-ident		RPI-B
-
-include 	"std.armv6"
-include 	"../broadcom/bcm2835/std.rpi"
-include 	"../broadcom/bcm2835/std.bcm2835"
-
-options 	SCHED_4BSD		# 4BSD scheduler
-options 	PLATFORM
-
-# NFS root from boopt/dhcp
-#options 	BOOTP
-#options 	BOOTP_NFSROOT
-#options 	BOOTP_COMPAT
-#options 	BOOTP_NFSV3
-#options 	BOOTP_WIRED_TO=ue0
-
-#options 	ROOTDEVNAME=\"ufs:mmcsd0s2\"
-
-# pseudo devices
-device		clk
-device		phy
-device		hwreset
-device		nvmem
-device		regulator
-device		syscon
-
-device		bpf
-device		loop
-device		ether
-device		uart
-device		pty
-device		snp
-device		pl011
-
-# Device mode support
-device		usb_template    	# Control of the gadget
-
-# Comment following lines for boot console on serial port
-device		vt
-device		kbdmux
-device		hkbd
-device		ukbd
-
-device		sdhci
-device		mmc
-device		mmcsd
-
-device		gpio
-device		gpioled
-
-# I2C
-device		iic
-device		iicbus
-device		bcm2835_bsc
-
-device		md
-
-# USB support
-device		usb
-device		dwcotg			# DWC OTG controller
-
-# USB storage support
-device		scbus
-device		da
-device		umass
-
-# USB ethernet support
-device		smscphy
-device		mii
-device		smsc
-
-# SPI
-device		spibus
-device		bcm2835_spi
-
-device		vchiq
-device		sound
-
-device		fdt_pinctrl
-
-# HID support
-device		hid			# Generic HID support
-
-# Flattened Device Tree
-options 	FDT			# Configure using FDT/DTB data
-# Note:  DTB is normally loaded and modified by RPi boot loader, then
-# handed to kernel via U-Boot and ubldr.
-#options 	FDT_DTB_STATIC
-#makeoptions	FDT_DTS_FILE=rpi.dts
-makeoptions	MODULES_EXTRA="dtb/rpi rpi_ft5406"

sys/arm/conf/std.armv6

@@ -1,81 +0,0 @@
-# Standard kernel config items for all ARMv6 systems.
-#
-
-options 	HZ=1000
-options 	PREEMPTION		# Enable kernel thread preemption
-options 	VIMAGE			# Subsystem virtualization, e.g. VNET
-options 	INET			# InterNETworking
-options 	INET6			# IPv6 communications protocols
-options 	CC_CUBIC		# include CUBIC congestion control
-options 	TCP_HHOOK		# hhook(9) framework for TCP
-device		crypto			# core crypto support
-options 	IPSEC_SUPPORT		# Allow kldload of ipsec and tcpmd5
-options 	NETLINK			# netlink(4) support
-options 	SCTP_SUPPORT		# Allow kldload of SCTP
-options 	FFS			# Berkeley Fast Filesystem
-options 	SOFTUPDATES		# Enable FFS soft updates support
-options 	UFS_ACL			# Support for access control lists
-options 	UFS_DIRHASH		# Improve performance on big directories
-options 	UFS_GJOURNAL		# Enable gjournal-based UFS journaling
-options 	QUOTA			# Enable disk quotas for UFS
-options 	NFSCL			# Network Filesystem Client
-options 	NFSLOCKD		# Network Lock Manager
-options 	NFS_ROOT		# NFS usable as /, requires NFSCL
-options 	MSDOSFS			# MSDOS Filesystem
-options 	CD9660			# ISO 9660 Filesystem
-options 	PROCFS			# Process filesystem (requires PSEUDOFS)
-options 	PSEUDOFS		# Pseudo-filesystem framework
-options 	TMPFS			# Efficient memory filesystem
-options 	GEOM_PART_GPT		# GUID Partition Tables
-options 	GEOM_PART_BSD		# BSD partition scheme
-options 	GEOM_PART_MBR		# MBR partition scheme
-options 	GEOM_LABEL		# Provides labelization
-options 	COMPAT_43		# Compatible with BSD 4.3 [KEEP THIS!]
-options 	SCSI_DELAY=5000		# Delay (in ms) before probing SCSI
-options 	KTRACE			# ktrace(1) support
-options 	STACK			# stack(9) support
-options 	SYSVSHM			# SYSV-style shared memory
-options 	SYSVMSG			# SYSV-style message queues
-options 	SYSVSEM			# SYSV-style semaphores
-options 	_KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
-options 	PRINTF_BUFR_SIZE=128	# Prevent printf output being interspersed.
-options 	KBD_INSTALL_CDEV	# install a CDEV entry in /dev
-options 	HWPMC_HOOKS		# Necessary kernel hooks for hwpmc(4)
-options 	CAPABILITY_MODE		# Capsicum capability mode
-options 	CAPABILITIES		# Capsicum capabilites
-options 	FREEBSD_BOOT_LOADER	# Process metadata passed from loader(8)
-options 	VFP			# Enable floating point hardware support
-options 	MAC			# Support for Mandatory Access Control (MAC)
-
-options 	COMPAT_FREEBSD10	# Compatible with FreeBSD10
-options 	COMPAT_FREEBSD11	# Compatible with FreeBSD11
-options 	COMPAT_FREEBSD12	# Compatible with FreeBSD12
-options 	COMPAT_FREEBSD13	# Compatible with FreeBSD13
-options 	COMPAT_FREEBSD14	# Compatible with FreeBSD14
-
-# DTrace support
-options 	KDTRACE_HOOKS		# Kernel DTrace hooks
-options 	DDB_CTF			# all architectures - kernel ELF linker loads CTF data
-makeoptions	WITH_CTF=1
-
-# Debugging support.  Always need this:
-makeoptions	DEBUG=-g		# Build kernel with gdb(1) debug symbols
-options 	KDB			# Enable kernel debugger support.
-options 	KDB_TRACE		# Print a stack trace for a panic.
-
-options		USB_DEBUG		# Enable usb debug support code
-
-# For full debugger support use (turn off in stable branch):
-include "std.debug"
-
-# Optional extras, never enabled by default:
-#options 	BOOTVERBOSE
-#options 	DEBUG			# May result in extreme spewage
-#options 	KTR
-#options 	KTR_COMPILE=KTR_ALL
-#options 	KTR_ENTRIES=16384
-#options 	KTR_MASK=(KTR_SPARE2)
-#options 	KTR_VERBOSE=0
-#options 	USB_REQ_DEBUG
-#options 	USB_VERBOSE
-

sys/arm64/conf/std.arm64

@@ -12,6 +12,7 @@ options 	INET			# InterNETworking
 options 	INET6			# IPv6 communications protocols
 options 	CC_CUBIC		# include CUBIC congestion control
 options 	IPSEC_SUPPORT		# Allow kldload of ipsec and tcpmd5
+options 	IPSEC_OFFLOAD		# Inline ipsec offload infra
 options 	ROUTE_MPATH		# Multipath routing support
 options 	FIB_ALGO		# Modular fib lookups
 options 	TCP_OFFLOAD		# TCP offload

sys/conf/files

@@ -4473,6 +4473,8 @@ netipsec/ipsec.c		optional ipsec inet | ipsec inet6
 netipsec/ipsec_input.c		optional ipsec inet | ipsec inet6
 netipsec/ipsec_mbuf.c		optional ipsec inet | ipsec inet6
 netipsec/ipsec_mod.c		optional ipsec inet | ipsec inet6
+netipsec/ipsec_offload.c	optional ipsec ipsec_offload inet | \
+	ipsec ipsec_offload inet6
 netipsec/ipsec_output.c		optional ipsec inet | ipsec inet6
 netipsec/ipsec_pcb.c		optional ipsec inet | ipsec inet6 | \
 	ipsec_support inet | ipsec_support inet6

sys/conf/options

@@ -466,6 +466,7 @@ IPFIREWALL_PMOD		opt_ipfw.h
 IPSEC			opt_ipsec.h
 IPSEC_DEBUG		opt_ipsec.h
 IPSEC_SUPPORT		opt_ipsec.h
+IPSEC_OFFLOAD		opt_ipsec.h
 IPSTEALTH
 KERN_TLS
 KRPC

sys/modules/ipsec/Makefile

@@ -2,8 +2,9 @@
 .PATH: ${SRCTOP}/sys/net ${SRCTOP}/sys/netipsec
 
 KMOD=	ipsec
-SRCS=	if_ipsec.c ipsec.c ipsec_input.c ipsec_mbuf.c ipsec_mod.c \
+SRCS=	if_ipsec.c ipsec.c ipsec_input.c ipsec_mbuf.c \
-	ipsec_output.c xform_ah.c xform_esp.c xform_ipcomp.c \
+	ipsec_mod.c ipsec_offload.c ipsec_output.c \
+	xform_ah.c xform_esp.c xform_ipcomp.c \
 	opt_inet.h opt_inet6.h opt_ipsec.h opt_kern_tls.h opt_sctp.h
 .if "${MK_INET}" != "no" || "${MK_INET6}" != "no"
 SRCS+=	udpencap.c

sys/net/if.c

@@ -2392,6 +2392,7 @@ const struct ifcap_nv_bit_name ifcap_nv_bit_names[] = {
 const struct ifcap_nv_bit_name ifcap2_nv_bit_names[] = {
 	CAP2NV(RXTLS4),
 	CAP2NV(RXTLS6),
+	CAP2NV(IPSEC_OFFLOAD),
 	{0, NULL}
 };
 #undef CAPNV
@@ -5149,6 +5150,12 @@ if_getl2com(if_t ifp)
 	return (ifp->if_l2com);
 }
 
+void
+if_setipsec_accel_methods(if_t ifp, const struct if_ipsec_accel_methods *m)
+{
+	ifp->if_ipsec_accel_m = m;
+}
+
 #ifdef DDB
 static void
 if_show_ifnet(struct ifnet *ifp)

sys/net/if.h

@@ -255,7 +255,8 @@ struct if_data {
 #define	IFCAP_B_TXTLS_RTLMT	31 /* can do TLS with rate limiting */
 #define	IFCAP_B_RXTLS4		32 /* can to TLS receive for TCP */
 #define	IFCAP_B_RXTLS6		33 /* can to TLS receive for TCP6 */
-#define	__IFCAP_B_SIZE		34
+#define	IFCAP_B_IPSEC_OFFLOAD	34 /* inline IPSEC offload */
+#define	__IFCAP_B_SIZE		35
 
 #define	IFCAP_B_MAX	(__IFCAP_B_MAX - 1)
 #define	IFCAP_B_SIZE	(__IFCAP_B_SIZE)
@@ -298,6 +299,7 @@ struct if_data {
 /* IFCAP2_* are integers, not bits. */
 #define	IFCAP2_RXTLS4		(IFCAP_B_RXTLS4 - 32)
 #define	IFCAP2_RXTLS6		(IFCAP_B_RXTLS6 - 32)
+#define	IFCAP2_IPSEC_OFFLOAD	(IFCAP_B_IPSEC_OFFLOAD - 32)
 
 #define	IFCAP2_BIT(x)		(1UL << (x))
 

sys/net/if_ipsec.c

@@ -415,12 +415,12 @@ ipsec_transmit(struct ifnet *ifp, struct mbuf *m)
 	switch (af) {
 #ifdef INET
 	case AF_INET:
-		error = ipsec4_process_packet(m, sp, NULL);
+		error = ipsec4_process_packet(ifp, m, sp, NULL, ifp->if_mtu);
 		break;
 #endif
 #ifdef INET6
 	case AF_INET6:
-		error = ipsec6_process_packet(m, sp, NULL);
+		error = ipsec6_process_packet(ifp, m, sp, NULL, ifp->if_mtu);
 		break;
 #endif
 	default:
@@ -901,8 +901,10 @@ ipsec_newpolicies(struct ipsec_softc *sc, struct secpolicy *sp[IPSEC_SPCOUNT],
 	}
 	return (0);
 fail:
-	for (i = 0; i < IPSEC_SPCOUNT; i++)
+	for (i = 0; i < IPSEC_SPCOUNT; i++) {
+		if (sp[i] != NULL)
 			key_freesp(&sp[i]);
+	}
 	return (ENOMEM);
 }
 

sys/net/if_private.h

@@ -138,6 +138,8 @@ struct ifnet {
 	int	(*if_requestencap)	/* make link header from request */
 		(struct ifnet *, struct if_encap_req *);
 
+	const struct if_ipsec_accel_methods *if_ipsec_accel_m;
+
 	/* Statistics. */
 	counter_u64_t	if_counters[IFCOUNTERS];
 

sys/net/if_strings.h

@@ -60,9 +60,11 @@
 #define	IFCAP_TXTLS_RTLMT_NAME		"TXTLS_RTLMT"
 #define	IFCAP_RXTLS4_NAME		"RXTLS4"
 #define	IFCAP_RXTLS6_NAME		"RXTLS6"
+#define	IFCAP_IPSEC_OFFLOAD_NAME	"IPSEC"
 
 #define	IFCAP2_RXTLS4_NAME	IFCAP_RXTLS4_NAME
 #define	IFCAP2_RXTLS6_NAME	IFCAP_RXTLS6_NAME
+#define	IFCAP2_IPSEC_OFFLOAD_NAME IFCAP_IPSEC_OFFLOAD_NAME
 
 static const char *ifcap_bit_names[] = {
 	IFCAP_RXCSUM_NAME,
@@ -99,6 +101,7 @@ static const char *ifcap_bit_names[] = {
 	IFCAP_TXTLS_RTLMT_NAME,
 	IFCAP_RXTLS4_NAME,
 	IFCAP_RXTLS6_NAME,
+	IFCAP_IPSEC_OFFLOAD_NAME,
 };
 
 #ifdef IFCAP_B_SIZE

sys/net/if_var.h

@@ -131,6 +131,25 @@ typedef void (*if_qflush_fn_t)(if_t);
 typedef int (*if_transmit_fn_t)(if_t, struct mbuf *);
 typedef	uint64_t (*if_get_counter_t)(if_t, ift_counter);
 typedef	void (*if_reassign_fn_t)(if_t, struct vnet *, char *);
+typedef int (*if_spdadd_fn_t)(if_t, void *sp, void *inp, void **priv);
+typedef int (*if_spddel_fn_t)(if_t, void *sp, void *priv);
+typedef int (*if_sa_newkey_fn_t)(if_t ifp, void *sav, u_int drv_spi,
+    void **privp);
+typedef int (*if_sa_deinstall_fn_t)(if_t ifp, u_int drv_spi, void *priv);
+struct seclifetime;
+#define	IF_SA_CNT_UPD	0x80000000
+enum IF_SA_CNT_WHICH {
+	IF_SA_CNT_IFP_HW_VAL = 1,
+	IF_SA_CNT_TOTAL_SW_VAL,
+	IF_SA_CNT_TOTAL_HW_VAL,
+	IF_SA_CNT_IFP_HW_UPD = IF_SA_CNT_IFP_HW_VAL | IF_SA_CNT_UPD,
+	IF_SA_CNT_TOTAL_SW_UPD = IF_SA_CNT_TOTAL_SW_VAL | IF_SA_CNT_UPD,
+	IF_SA_CNT_TOTAL_HW_UPD = IF_SA_CNT_TOTAL_HW_VAL | IF_SA_CNT_UPD,
+};
+typedef int (*if_sa_cnt_fn_t)(if_t ifp, void *sa,
+    uint32_t drv_spi, void *priv, struct seclifetime *lt);
+typedef int (*if_ipsec_hwassist_fn_t)(if_t ifp, void *sav,
+    u_int drv_spi,void *priv);
 
 struct ifnet_hw_tsomax {
 	u_int	tsomaxbytes;	/* TSO total burst length limit in bytes */
@@ -700,6 +719,20 @@ void if_setdebugnet_methods(if_t, struct debugnet_methods *);
 void if_setreassignfn(if_t ifp, if_reassign_fn_t);
 void if_setratelimitqueryfn(if_t ifp, if_ratelimit_query_t);
 
+/*
+ * NB: The interface is not yet stable, drivers implementing IPSEC
+ * offload need to be prepared to adapt to changes.
+ */
+struct if_ipsec_accel_methods {
+	if_spdadd_fn_t		if_spdadd;
+	if_spddel_fn_t		if_spddel;
+	if_sa_newkey_fn_t	if_sa_newkey;
+	if_sa_deinstall_fn_t	if_sa_deinstall;
+	if_sa_cnt_fn_t		if_sa_cnt;
+	if_ipsec_hwassist_fn_t	if_hwassist;
+};
+void if_setipsec_accel_methods(if_t ifp, const struct if_ipsec_accel_methods *);
+
 /* TSO */
 void if_hw_tsomax_common(if_t ifp, struct ifnet_hw_tsomax *);
 int if_hw_tsomax_update(if_t ifp, struct ifnet_hw_tsomax *);

sys/net/pfkeyv2.h

@@ -296,6 +296,13 @@ struct sadb_x_sa_replay {
 };
 _Static_assert(sizeof(struct sadb_x_sa_replay) == 8, "struct size mismatch");
 
+struct sadb_x_if_hw_offl {
+  u_int16_t sadb_x_if_hw_offl_len;
+  u_int16_t sadb_x_if_hw_offl_exttype;
+  u_int32_t sadb_x_if_hw_offl_flags;
+  u_int8_t sadb_x_if_hw_offl_if[32];	/* IF_NAMESIZE is 16, keep room */
+};
+
 #define SADB_EXT_RESERVED             0
 #define SADB_EXT_SA                   1
 #define SADB_EXT_LIFETIME_CURRENT     2
@@ -326,7 +333,10 @@ _Static_assert(sizeof(struct sadb_x_sa_replay) == 8, "struct size mismatch");
 #define SADB_X_EXT_SA_REPLAY          26	/* Replay window override. */
 #define	SADB_X_EXT_NEW_ADDRESS_SRC    27
 #define	SADB_X_EXT_NEW_ADDRESS_DST    28
-#define	SADB_EXT_MAX                  28
+#define	SADB_X_EXT_LFT_CUR_SW_OFFL    29
+#define	SADB_X_EXT_LFT_CUR_HW_OFFL    30
+#define	SADB_X_EXT_IF_HW_OFFL	      31
+#define	SADB_EXT_MAX                  31
 
 #define SADB_SATYPE_UNSPEC	0
 #define SADB_SATYPE_AH		2

sys/netinet/ip_output.c

@@ -673,7 +673,7 @@ sendit:
 			error = ENOBUFS;
 			goto bad;
 		}
-		if ((error = IPSEC_OUTPUT(ipv4, m, inp)) != 0) {
+		if ((error = IPSEC_OUTPUT(ipv4, ifp, m, inp, mtu)) != 0) {
 			if (error == EINPROGRESS)
 				error = 0;
 			goto done;

sys/netinet/tcp_input.c

@@ -3984,6 +3984,8 @@ tcp_mss(struct tcpcb *tp, int offer)
 		tp->t_tsomax = cap.tsomax;
 		tp->t_tsomaxsegcount = cap.tsomaxsegcount;
 		tp->t_tsomaxsegsize = cap.tsomaxsegsize;
+		if (cap.ipsec_tso)
+			tp->t_flags2 |= TF2_IPSEC_TSO;
 	}
 }
 

sys/netinet/tcp_output.c

@@ -201,9 +201,7 @@ tcp_default_output(struct tcpcb *tp)
 	struct tcphdr *th;
 	u_char opt[TCP_MAXOLEN];
 	unsigned ipoptlen, optlen, hdrlen, ulen;
-#if defined(IPSEC) || defined(IPSEC_SUPPORT)
 	unsigned ipsec_optlen = 0;
-#endif
 	int idle, sendalot, curticks;
 	int sack_rxmit, sack_bytes_rxmt;
 	struct sackhole *p;
@@ -553,15 +551,15 @@ after_sack_rexmit:
 				offsetof(struct ipoption, ipopt_list);
 	else
 		ipoptlen = 0;
-#if defined(IPSEC) || defined(IPSEC_SUPPORT)
 	ipoptlen += ipsec_optlen;
-#endif
 
 	if ((tp->t_flags & TF_TSO) && V_tcp_do_tso && len > tp->t_maxseg &&
 	    (tp->t_port == 0) &&
 	    ((tp->t_flags & TF_SIGNATURE) == 0) &&
 	    tp->rcv_numsacks == 0 && ((sack_rxmit == 0) || V_tcp_sack_tso) &&
-	    ipoptlen == 0 && !(flags & TH_SYN))
+	    (ipoptlen == 0 || (ipoptlen == ipsec_optlen &&
+	    (tp->t_flags2 & TF2_IPSEC_TSO) != 0)) &&
+	    !(flags & TH_SYN))
 		tso = 1;
 
 	if (SEQ_LT((sack_rxmit ? p->rxmit : tp->snd_nxt) + len,
@@ -917,7 +915,7 @@ send:
 			 * overflowing or exceeding the maximum length
 			 * allowed by the network interface:
 			 */
-			KASSERT(ipoptlen == 0,
+			KASSERT(ipoptlen ==  ipsec_optlen,
 			    ("%s: TSO can't do IP options", __func__));
 
 			/*
@@ -926,8 +924,8 @@ send:
 			 */
 			if (if_hw_tsomax != 0) {
 				/* compute maximum TSO length */
-				max_len = (if_hw_tsomax - hdrlen -
+				max_len = if_hw_tsomax - hdrlen -
-				    max_linkhdr);
+				    ipsec_optlen - max_linkhdr;
 				if (max_len <= 0) {
 					len = 0;
 				} else if (len > max_len) {
@@ -941,7 +939,7 @@ send:
 			 * fractional unless the send sockbuf can be
 			 * emptied:
 			 */
-			max_len = (tp->t_maxseg - optlen);
+			max_len = tp->t_maxseg - optlen - ipsec_optlen;
 			if (((uint32_t)off + (uint32_t)len) <
 			    sbavail(&so->so_snd)) {
 				moff = len % max_len;
@@ -1393,10 +1391,10 @@ send:
 	 * The TCP pseudo header checksum is always provided.
 	 */
 	if (tso) {
-		KASSERT(len > tp->t_maxseg - optlen,
+		KASSERT(len > tp->t_maxseg - optlen - ipsec_optlen,
 		    ("%s: len <= tso_segsz", __func__));
 		m->m_pkthdr.csum_flags |= CSUM_TSO;
-		m->m_pkthdr.tso_segsz = tp->t_maxseg - optlen;
+		m->m_pkthdr.tso_segsz = tp->t_maxseg - optlen - ipsec_optlen;
 	}
 
 	KASSERT(len + hdrlen == m_length(m, NULL),

sys/netinet/tcp_subr.c

@@ -3349,6 +3349,9 @@ tcp_maxmtu(struct in_conninfo *inc, struct tcp_ifcap *cap)
 				cap->tsomax = ifp->if_hw_tsomax;
 				cap->tsomaxsegcount = ifp->if_hw_tsomaxsegcount;
 				cap->tsomaxsegsize = ifp->if_hw_tsomaxsegsize;
+				/* XXXKIB IFCAP2_IPSEC_OFFLOAD_TSO */
+				cap->ipsec_tso =  (ifp->if_capenable2 &
+				    IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) != 0;
 			}
 		}
 	}
@@ -3388,6 +3391,7 @@ tcp_maxmtu6(struct in_conninfo *inc, struct tcp_ifcap *cap)
 				cap->tsomax = ifp->if_hw_tsomax;
 				cap->tsomaxsegcount = ifp->if_hw_tsomaxsegcount;
 				cap->tsomaxsegsize = ifp->if_hw_tsomaxsegsize;
+				cap->ipsec_tso = false; /* XXXKIB */
 			}
 		}
 	}

sys/netinet/tcp_var.h

@@ -844,6 +844,7 @@ tcp_packets_this_ack(struct tcpcb *tp, tcp_seq ack)
 #define	TF2_DONT_SACK_QUEUE	0x00040000 /* Don't wake on sack */
 #define	TF2_CANNOT_DO_ECN	0x00080000 /* The stack does not do ECN */
 #define TF2_PROC_SACK_PROHIBIT	0x00100000 /* Due to small MSS size do not process sack's */
+#define	TF2_IPSEC_TSO		0x00200000	/* IPSEC + TSO supported */
 
 /*
  * Structure to hold TCP options that are only used during segment
@@ -1430,6 +1431,7 @@ struct tcp_ifcap {
 	u_int	tsomax;
 	u_int	tsomaxsegcount;
 	u_int	tsomaxsegsize;
+	bool	ipsec_tso;
 };
 uint32_t tcp_maxmtu(struct in_conninfo *, struct tcp_ifcap *);
 uint32_t tcp_maxmtu6(struct in_conninfo *, struct tcp_ifcap *);

sys/netinet6/ip6_output.c

@@ -449,27 +449,6 @@ ip6_output(struct mbuf *m0, struct ip6_pktopts *opt,
 #endif
 	}
 
-#if defined(IPSEC) || defined(IPSEC_SUPPORT)
-	/*
-	 * IPSec checking which handles several cases.
-	 * FAST IPSEC: We re-injected the packet.
-	 * XXX: need scope argument.
-	 */
-	if (IPSEC_ENABLED(ipv6)) {
-		m = mb_unmapped_to_ext(m);
-		if (m == NULL) {
-			IP6STAT_INC(ip6s_odropped);
-			error = ENOBUFS;
-			goto bad;
-		}
-		if ((error = IPSEC_OUTPUT(ipv6, m, inp)) != 0) {
-			if (error == EINPROGRESS)
-				error = 0;
-			goto done;
-		}
-	}
-#endif /* IPSEC */
-
 	/* Source address validation. */
 	ip6 = mtod(m, struct ip6_hdr *);
 	if (IN6_IS_ADDR_UNSPECIFIED(&ip6->ip6_src) &&
@@ -806,6 +785,27 @@ nonh6lookup:
 	KASSERT((ifp != NULL), ("output interface must not be NULL"));
 	KASSERT((origifp != NULL), ("output address interface must not be NULL"));
 
+#if defined(IPSEC) || defined(IPSEC_SUPPORT)
+	/*
+	 * IPSec checking which handles several cases.
+	 * FAST IPSEC: We re-injected the packet.
+	 * XXX: need scope argument.
+	 */
+	if (IPSEC_ENABLED(ipv6)) {
+		m = mb_unmapped_to_ext(m);
+		if (m == NULL) {
+			IP6STAT_INC(ip6s_odropped);
+			error = ENOBUFS;
+			goto bad;
+		}
+		if ((error = IPSEC_OUTPUT(ipv6, ifp, m, inp, mtu)) != 0) {
+			if (error == EINPROGRESS)
+				error = 0;
+			goto done;
+		}
+	}
+#endif /* IPSEC */
+
 	if ((flags & IPV6_FORWARDING) == 0) {
 		/* XXX: the FORWARDING flag can be set for mrouting. */
 		in6_ifstat_inc(ifp, ifs6_out_request);

sys/netipsec/esp_var.h

@@ -74,6 +74,8 @@ struct espstat {
 #include <sys/counter.h>
 
 VNET_DECLARE(int, esp_enable);
+VNET_DECLARE(int, esp_ctr_compatibility);
+#define V_esp_ctr_compatibility VNET(esp_ctr_compatibility)
 VNET_PCPUSTAT_DECLARE(struct espstat, espstat);
 
 #define	ESPSTAT_ADD(name, val)	\

sys/netipsec/ipsec.c

@@ -85,6 +85,7 @@
 #ifdef INET6
 #include <netipsec/ipsec6.h>
 #endif
+#include <netipsec/ipsec_offload.h>
 #include <netipsec/ah_var.h>
 #include <netipsec/esp_var.h>
 #include <netipsec/ipcomp.h>		/*XXX*/
@@ -636,8 +637,16 @@ int
 ipsec4_in_reject(const struct mbuf *m, struct inpcb *inp)
 {
 	struct secpolicy *sp;
+#ifdef IPSEC_OFFLOAD
+	struct ipsec_accel_in_tag *tag;
+#endif
 	int result;
 
+#ifdef IPSEC_OFFLOAD
+	tag = ipsec_accel_input_tag_lookup(m);
+	if (tag != NULL)
+		return (0);
+#endif
 	sp = ipsec4_getpolicy(m, inp, IPSEC_DIR_INBOUND, 0);
 	result = ipsec_in_reject(sp, inp, m);
 	key_freesp(&sp);
@@ -802,8 +811,16 @@ int
 ipsec6_in_reject(const struct mbuf *m, struct inpcb *inp)
 {
 	struct secpolicy *sp;
+#ifdef IPSEC_OFFLOAD
+	struct ipsec_accel_in_tag *tag;
+#endif
 	int result;
 
+#ifdef IPSEC_OFFLOAD
+	tag = ipsec_accel_input_tag_lookup(m);
+	if (tag != NULL)
+		return (0);
+#endif
 	sp = ipsec6_getpolicy(m, inp, IPSEC_DIR_INBOUND, 0);
 	result = ipsec_in_reject(sp, inp, m);
 	key_freesp(&sp);

sys/netipsec/ipsec.h

@@ -71,6 +71,12 @@ struct ipsecrequest {
 	u_int level;		/* IPsec level defined below. */
 };
 
+struct ipsec_accel_adddel_sp_tq {
+	struct vnet *adddel_vnet;
+	struct task adddel_task;
+	int adddel_scheduled;
+};
+
 /* Security Policy Data Base */
 struct secpolicy {
 	TAILQ_ENTRY(secpolicy) chain;
@@ -102,6 +108,11 @@ struct secpolicy {
 	time_t lastused;	/* updated every when kernel sends a packet */
 	long lifetime;		/* duration of the lifetime of this policy */
 	long validtime;		/* duration this policy is valid without use */
+	CK_LIST_HEAD(, ifp_handle_sp) accel_ifps;
+	struct ipsec_accel_adddel_sp_tq accel_add_tq;
+	struct ipsec_accel_adddel_sp_tq accel_del_tq;
+	struct inpcb *ipsec_accel_add_sp_inp;
+	const char *accel_ifname;
 };
 
 /*
@@ -336,8 +347,9 @@ void ipsec_setspidx_inpcb(struct inpcb *, struct secpolicyindex *, u_int);
 void ipsec4_setsockaddrs(const struct mbuf *, union sockaddr_union *,
     union sockaddr_union *);
 int ipsec4_common_input_cb(struct mbuf *, struct secasvar *, int, int);
-int ipsec4_check_pmtu(struct mbuf *, struct secpolicy *, int);
+int ipsec4_check_pmtu(struct ifnet *, struct mbuf *, struct secpolicy *, int);
-int ipsec4_process_packet(struct mbuf *, struct secpolicy *, struct inpcb *);
+int ipsec4_process_packet(struct ifnet *, struct mbuf *, struct secpolicy *,
+    struct inpcb *, u_long);
 int ipsec_process_done(struct mbuf *, struct secpolicy *, struct secasvar *,
     u_int);
 

sys/netipsec/ipsec6.h

@@ -66,8 +66,9 @@ struct secpolicy *ipsec6_checkpolicy(const struct mbuf *,
 void ipsec6_setsockaddrs(const struct mbuf *, union sockaddr_union *,
     union sockaddr_union *);
 int ipsec6_common_input_cb(struct mbuf *, struct secasvar *, int, int);
-int ipsec6_check_pmtu(struct mbuf *, struct secpolicy *, int);
+int ipsec6_check_pmtu(struct ifnet *, struct mbuf *, struct secpolicy *, int);
-int ipsec6_process_packet(struct mbuf *, struct secpolicy *, struct inpcb *);
+int ipsec6_process_packet(struct ifnet *, struct mbuf *, struct secpolicy *,
+    struct inpcb *, u_long);
 
 int ip6_ipsec_filtertunnel(struct mbuf *);
 int ip6_ipsec_pcbctl(struct inpcb *, struct sockopt *);

sys/netipsec/ipsec_input.c

@@ -90,6 +90,7 @@
 #include <netipsec/esp.h>
 #include <netipsec/esp_var.h>
 #include <netipsec/ipcomp_var.h>
+#include <netipsec/ipsec_offload.h>
 
 #include <netipsec/key.h>
 #include <netipsec/keydb.h>
@@ -237,6 +238,11 @@ ipsec_common_input(struct mbuf *m, int skip, int protoff, int af, int sproto)
 int
 ipsec4_input(struct mbuf *m, int offset, int proto)
 {
+	int error;
+
+	error = ipsec_accel_input(m, offset, proto);
+	if (error != ENXIO)
+		return (error);
 
 	switch (proto) {
 	case IPPROTO_AH:
@@ -536,6 +542,11 @@ ipsec6_lasthdr(int proto)
 int
 ipsec6_input(struct mbuf *m, int offset, int proto)
 {
+	int error;
+
+	error = ipsec_accel_input(m, offset, proto);
+	if (error != ENXIO)
+		return (error);
 		
 	switch (proto) {
 	case IPPROTO_AH:

sys/netipsec/ipsec_offload.h

@@ -0,0 +1,194 @@
+/*-
+ * Copyright (c) 2021,2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _NETIPSEC_IPSEC_OFFLOAD_H_
+#define _NETIPSEC_IPSEC_OFFLOAD_H_
+
+#ifdef _KERNEL
+#include <sys/errno.h>
+#include <net/if.h>
+#include <net/if_var.h>
+
+struct secpolicy;
+struct secasvar;
+struct inpcb;
+
+struct ipsec_accel_out_tag {
+	struct m_tag tag;
+	uint16_t drv_spi;
+};
+
+struct ipsec_accel_in_tag {
+	struct m_tag tag;
+	uint16_t drv_spi;
+};
+
+#define	IPSEC_ACCEL_DRV_SPI_BYPASS	2
+#define	IPSEC_ACCEL_DRV_SPI_MIN		3
+#define	IPSEC_ACCEL_DRV_SPI_MAX		0xffff
+
+extern void (*ipsec_accel_sa_newkey_p)(struct secasvar *sav);
+extern void (*ipsec_accel_sa_install_input_p)(struct secasvar *sav,
+    const union sockaddr_union *dst_address, int sproto, uint32_t spi);
+extern void (*ipsec_accel_forget_sav_p)(struct secasvar *sav);
+extern void (*ipsec_accel_spdadd_p)(struct secpolicy *sp, struct inpcb *inp);
+extern void (*ipsec_accel_spddel_p)(struct secpolicy *sp);
+extern int (*ipsec_accel_sa_lifetime_op_p)(struct secasvar *sav,
+    struct seclifetime *lft_c, if_t ifp, enum IF_SA_CNT_WHICH op,
+    struct rm_priotracker *sahtree_trackerp);
+extern void (*ipsec_accel_sync_p)(void);
+extern bool (*ipsec_accel_is_accel_sav_p)(struct secasvar *sav);
+extern struct mbuf *(*ipsec_accel_key_setaccelif_p)(struct secasvar *sav);
+
+#ifdef IPSEC_OFFLOAD
+/*
+ * Have to use ipsec_accel_sa_install_input_p indirection because
+ * key.c is unconditionally included into the static kernel.
+ */
+static inline void
+ipsec_accel_sa_newkey(struct secasvar *sav)
+{
+	void (*p)(struct secasvar *sav);
+
+	p = atomic_load_ptr(&ipsec_accel_sa_newkey_p);
+	if (p != NULL)
+		p(sav);
+}
+
+static inline void
+ipsec_accel_forget_sav(struct secasvar *sav)
+{
+	void (*p)(struct secasvar *sav);
+
+	p = atomic_load_ptr(&ipsec_accel_forget_sav_p);
+	if (p != NULL)
+		p(sav);
+}
+
+static inline void
+ipsec_accel_spdadd(struct secpolicy *sp, struct inpcb *inp)
+{
+	void (*p)(struct secpolicy *sp, struct inpcb *inp);
+
+	p = atomic_load_ptr(&ipsec_accel_spdadd_p);
+	if (p != NULL)
+		p(sp, inp);
+}
+
+static inline void
+ipsec_accel_spddel(struct secpolicy *sp)
+{
+	void (*p)(struct secpolicy *sp);
+
+	p = atomic_load_ptr(&ipsec_accel_spddel_p);
+	if (p != NULL)
+		p(sp);
+}
+
+static inline int
+ipsec_accel_sa_lifetime_op(struct secasvar *sav,
+    struct seclifetime *lft_c, if_t ifp, enum IF_SA_CNT_WHICH op,
+    struct rm_priotracker *sahtree_trackerp)
+{
+	int (*p)(struct secasvar *sav, struct seclifetime *lft_c, if_t ifp,
+	    enum IF_SA_CNT_WHICH op, struct rm_priotracker *sahtree_trackerp);
+
+	p = atomic_load_ptr(&ipsec_accel_sa_lifetime_op_p);
+	if (p != NULL)
+		return (p(sav, lft_c, ifp, op, sahtree_trackerp));
+	return (ENOTSUP);
+}
+
+static inline void
+ipsec_accel_sync(void)
+{
+	void (*p)(void);
+
+	p = atomic_load_ptr(&ipsec_accel_sync_p);
+	if (p != NULL)
+		p();
+}
+
+static inline bool
+ipsec_accel_is_accel_sav(struct secasvar *sav)
+{
+	bool (*p)(struct secasvar *sav);
+
+	p = atomic_load_ptr(&ipsec_accel_is_accel_sav_p);
+	if (p != NULL)
+		return (p(sav));
+	return (false);
+}
+
+static inline struct mbuf *
+ipsec_accel_key_setaccelif(struct secasvar *sav)
+{
+	struct mbuf *(*p)(struct secasvar *sav);
+
+	p = atomic_load_ptr(&ipsec_accel_key_setaccelif_p);
+	if (p != NULL)
+		return (p(sav));
+	return (NULL);
+}
+
+
+#else
+#define	ipsec_accel_sa_newkey(a)
+#define	ipsec_accel_forget_sav(a)
+#define	ipsec_accel_spdadd(a, b)
+#define	ipsec_accel_spddel(a)
+#define	ipsec_accel_sa_lifetime_op(a, b, c, d, e)
+#define	ipsec_accel_sync()
+#define	ipsec_accel_is_accel_sav(a)
+#define	ipsec_accel_key_setaccelif(a)
+#endif
+
+void ipsec_accel_forget_sav_impl(struct secasvar *sav);
+void ipsec_accel_spdadd_impl(struct secpolicy *sp, struct inpcb *inp);
+void ipsec_accel_spddel_impl(struct secpolicy *sp);
+
+#ifdef IPSEC_OFFLOAD
+int ipsec_accel_input(struct mbuf *m, int offset, int proto);
+bool ipsec_accel_output(struct ifnet *ifp, struct mbuf *m,
+    struct inpcb *inp, struct secpolicy *sp, struct secasvar *sav, int af,
+    int mtu, int *hwassist);
+void ipsec_accel_forget_sav(struct secasvar *sav);
+#else
+#define	ipsec_accel_input(a, b, c) (ENXIO)
+#define	ipsec_accel_output(a, b, c, d, e, f, g, h) ({	\
+	*h = 0;						\
+	false;						\
+})
+#define	ipsec_accel_forget_sav(a)
+#endif
+
+struct ipsec_accel_in_tag *ipsec_accel_input_tag_lookup(const struct mbuf *);
+void ipsec_accel_on_ifdown(struct ifnet *ifp);
+void ipsec_accel_drv_sa_lifetime_update(struct secasvar *sav, if_t ifp,
+    u_int drv_spi, uint64_t octets, uint64_t allocs);
+
+#endif	/* _KERNEL */
+
+#endif	/* _NETIPSEC_IPSEC_OFFLOAD_H_ */

sys/netipsec/ipsec_output.c

@@ -84,6 +84,7 @@
 #include <netipsec/ipsec6.h>
 #endif
 #include <netipsec/ipsec_support.h>
+#include <netipsec/ipsec_offload.h>
 #include <netipsec/ah_var.h>
 #include <netipsec/esp_var.h>
 #include <netipsec/ipcomp_var.h>
@@ -110,7 +111,8 @@ static size_t ipsec_get_pmtu(struct secasvar *sav);
 
 #ifdef INET
 static struct secasvar *
-ipsec4_allocsa(struct mbuf *m, struct secpolicy *sp, u_int *pidx, int *error)
+ipsec4_allocsa(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
+    u_int *pidx, int *error)
 {
 	struct secasindex *saidx, tmpsaidx;
 	struct ipsecrequest *isr;
@@ -186,14 +188,15 @@ next:
  * IPsec output logic for IPv4.
  */
 static int
-ipsec4_perform_request(struct mbuf *m, struct secpolicy *sp,
+ipsec4_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
-    struct inpcb *inp, u_int idx)
+    struct inpcb *inp, u_int idx, u_long mtu)
 {
 	struct ipsec_ctx_data ctx;
 	union sockaddr_union *dst;
 	struct secasvar *sav;
 	struct ip *ip;
-	int error, i, off;
+	int error, hwassist, i, off;
+	bool accel;
 
 	IPSEC_ASSERT(idx < sp->tcount, ("Wrong IPsec request index %d", idx));
 
@@ -206,9 +209,11 @@ ipsec4_perform_request(struct mbuf *m, struct secpolicy *sp,
 	 * determine next transform. At the end of transform we can
 	 * release reference to SP.
 	 */
-	sav = ipsec4_allocsa(m, sp, &idx, &error);
+	sav = ipsec4_allocsa(ifp, m, sp, &idx, &error);
 	if (sav == NULL) {
 		if (error == EJUSTRETURN) { /* No IPsec required */
+			(void)ipsec_accel_output(ifp, m, inp, sp, NULL,
+			    AF_INET, mtu, &hwassist);
 			key_freesp(&sp);
 			return (error);
 		}
@@ -221,6 +226,30 @@ ipsec4_perform_request(struct mbuf *m, struct secpolicy *sp,
 	if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_OUT)) != 0)
 		goto bad;
 
+	hwassist = 0;
+	accel = ipsec_accel_output(ifp, m, inp, sp, sav, AF_INET, mtu,
+	    &hwassist);
+
+	/*
+	 * Do delayed checksums now because we send before
+	 * this is done in the normal processing path.
+	 */
+	if ((m->m_pkthdr.csum_flags & CSUM_DELAY_DATA & ~hwassist) != 0) {
+		in_delayed_cksum(m);
+		m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
+	}
+#if defined(SCTP) || defined(SCTP_SUPPORT)
+	if ((m->m_pkthdr.csum_flags & CSUM_SCTP & ~hwassist) != 0) {
+		struct ip *ip;
+
+		ip = mtod(m, struct ip *);
+		sctp_delayed_cksum(m, (uint32_t)(ip->ip_hl << 2));
+		m->m_pkthdr.csum_flags &= ~CSUM_SCTP;
+	}
+#endif
+	if (accel)
+		return (EJUSTRETURN);
+
 	ip = mtod(m, struct ip *);
 	dst = &sav->sah->saidx.dst;
 	/* Do the appropriate encapsulation, if necessary */
@@ -288,15 +317,16 @@ bad:
 }
 
 int
-ipsec4_process_packet(struct mbuf *m, struct secpolicy *sp,
+ipsec4_process_packet(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
-    struct inpcb *inp)
+    struct inpcb *inp, u_long mtu)
 {
 
-	return (ipsec4_perform_request(m, sp, inp, 0));
+	return (ipsec4_perform_request(ifp, m, sp, inp, 0, mtu));
 }
 
 int
-ipsec4_check_pmtu(struct mbuf *m, struct secpolicy *sp, int forwarding)
+ipsec4_check_pmtu(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
+    int forwarding)
 {
 	struct secasvar *sav;
 	struct ip *ip;
@@ -317,7 +347,7 @@ ipsec4_check_pmtu(struct mbuf *m, struct secpolicy *sp, int forwarding)
 
 setdf:
 	idx = sp->tcount - 1;
-	sav = ipsec4_allocsa(m, sp, &idx, &error);
+	sav = ipsec4_allocsa(ifp, m, sp, &idx, &error);
 	if (sav == NULL) {
 		key_freesp(&sp);
 		/*
@@ -368,7 +398,8 @@ setdf:
 }
 
 static int
-ipsec4_common_output(struct mbuf *m, struct inpcb *inp, int forwarding)
+ipsec4_common_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp,
+    int forwarding, u_long mtu)
 {
 	struct secpolicy *sp;
 	int error;
@@ -392,27 +423,9 @@ ipsec4_common_output(struct mbuf *m, struct inpcb *inp, int forwarding)
 	 * packets, and thus, even if they are forwarded, the replies will
 	 * return back to us.
 	 */
-	if (!forwarding) {
-		/*
-		 * Do delayed checksums now because we send before
-		 * this is done in the normal processing path.
-		 */
-		if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
-			in_delayed_cksum(m);
-			m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
-		}
-#if defined(SCTP) || defined(SCTP_SUPPORT)
-		if (m->m_pkthdr.csum_flags & CSUM_SCTP) {
-			struct ip *ip;
 
-			ip = mtod(m, struct ip *);
-			sctp_delayed_cksum(m, (uint32_t)(ip->ip_hl << 2));
-			m->m_pkthdr.csum_flags &= ~CSUM_SCTP;
-		}
-#endif
-	}
 	/* NB: callee frees mbuf and releases reference to SP */
-	error = ipsec4_check_pmtu(m, sp, forwarding);
+	error = ipsec4_check_pmtu(ifp, m, sp, forwarding);
 	if (error != 0) {
 		if (error == EJUSTRETURN)
 			return (0);
@@ -420,7 +433,7 @@ ipsec4_common_output(struct mbuf *m, struct inpcb *inp, int forwarding)
 		return (error);
 	}
 
-	error = ipsec4_process_packet(m, sp, inp);
+	error = ipsec4_process_packet(ifp, m, sp, inp, mtu);
 	if (error == EJUSTRETURN) {
 		/*
 		 * We had a SP with a level of 'use' and no SA. We
@@ -440,7 +453,7 @@ ipsec4_common_output(struct mbuf *m, struct inpcb *inp, int forwarding)
  * other values - mbuf consumed by IPsec.
  */
 int
-ipsec4_output(struct mbuf *m, struct inpcb *inp)
+ipsec4_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp, u_long mtu)
 {
 
 	/*
@@ -451,7 +464,7 @@ ipsec4_output(struct mbuf *m, struct inpcb *inp)
 	if (m_tag_find(m, PACKET_TAG_IPSEC_OUT_DONE, NULL) != NULL)
 		return (0);
 
-	return (ipsec4_common_output(m, inp, 0));
+	return (ipsec4_common_output(ifp, m, inp, 0, mtu));
 }
 
 /*
@@ -471,7 +484,7 @@ ipsec4_forward(struct mbuf *m)
 		m_freem(m);
 		return (EACCES);
 	}
-	return (ipsec4_common_output(m, NULL, 1));
+	return (ipsec4_common_output(NULL /* XXXKIB */, m, NULL, 1, 0));
 }
 #endif
 
@@ -491,7 +504,8 @@ in6_sa_equal_addrwithscope(const struct sockaddr_in6 *sa,
 }
 
 static struct secasvar *
-ipsec6_allocsa(struct mbuf *m, struct secpolicy *sp, u_int *pidx, int *error)
+ipsec6_allocsa(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
+    u_int *pidx, int *error)
 {
 	struct secasindex *saidx, tmpsaidx;
 	struct ipsecrequest *isr;
@@ -579,20 +593,23 @@ next:
  * IPsec output logic for IPv6.
  */
 static int
-ipsec6_perform_request(struct mbuf *m, struct secpolicy *sp,
+ipsec6_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
-    struct inpcb *inp, u_int idx)
+    struct inpcb *inp, u_int idx, u_long mtu)
 {
 	struct ipsec_ctx_data ctx;
 	union sockaddr_union *dst;
 	struct secasvar *sav;
 	struct ip6_hdr *ip6;
-	int error, i, off;
+	int error, hwassist, i, off;
+	bool accel;
 
 	IPSEC_ASSERT(idx < sp->tcount, ("Wrong IPsec request index %d", idx));
 
-	sav = ipsec6_allocsa(m, sp, &idx, &error);
+	sav = ipsec6_allocsa(ifp, m, sp, &idx, &error);
 	if (sav == NULL) {
 		if (error == EJUSTRETURN) { /* No IPsec required */
+			(void)ipsec_accel_output(ifp, m, inp, sp, NULL,
+			    AF_INET6, mtu, &hwassist);
 			key_freesp(&sp);
 			return (error);
 		}
@@ -607,6 +624,28 @@ ipsec6_perform_request(struct mbuf *m, struct secpolicy *sp,
 	if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_OUT)) != 0)
 		goto bad;
 
+	hwassist = 0;
+	accel = ipsec_accel_output(ifp, m, inp, sp, sav, AF_INET6, mtu,
+	    &hwassist);
+
+	/*
+	 * Do delayed checksums now because we send before
+	 * this is done in the normal processing path.
+	 */
+	if ((m->m_pkthdr.csum_flags & CSUM_DELAY_DATA_IPV6 & ~hwassist) != 0) {
+		in6_delayed_cksum(m, m->m_pkthdr.len -
+		    sizeof(struct ip6_hdr), sizeof(struct ip6_hdr));
+		m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA_IPV6;
+	}
+#if defined(SCTP) || defined(SCTP_SUPPORT)
+	if ((m->m_pkthdr.csum_flags & CSUM_SCTP_IPV6 & ~hwassist) != 0) {
+		sctp_delayed_cksum(m, sizeof(struct ip6_hdr));
+		m->m_pkthdr.csum_flags &= ~CSUM_SCTP_IPV6;
+	}
+#endif
+	if (accel)
+		return (EJUSTRETURN);
+
 	ip6 = mtod(m, struct ip6_hdr *); /* pfil can change mbuf */
 	dst = &sav->sah->saidx.dst;
 
@@ -671,18 +710,19 @@ bad:
 }
 
 int
-ipsec6_process_packet(struct mbuf *m, struct secpolicy *sp,
+ipsec6_process_packet(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
-    struct inpcb *inp)
+    struct inpcb *inp, u_long mtu)
 {
 
-	return (ipsec6_perform_request(m, sp, inp, 0));
+	return (ipsec6_perform_request(ifp, m, sp, inp, 0, mtu));
 }
 
 /*
  * IPv6 implementation is based on IPv4 implementation.
  */
 int
-ipsec6_check_pmtu(struct mbuf *m, struct secpolicy *sp, int forwarding)
+ipsec6_check_pmtu(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
+    int forwarding)
 {
 	struct secasvar *sav;
 	size_t hlen, pmtu;
@@ -699,7 +739,7 @@ ipsec6_check_pmtu(struct mbuf *m, struct secpolicy *sp, int forwarding)
 		return (0);
 
 	idx = sp->tcount - 1;
-	sav = ipsec6_allocsa(m, sp, &idx, &error);
+	sav = ipsec6_allocsa(ifp, m, sp, &idx, &error);
 	if (sav == NULL) {
 		key_freesp(&sp);
 		/*
@@ -745,7 +785,8 @@ ipsec6_check_pmtu(struct mbuf *m, struct secpolicy *sp, int forwarding)
 }
 
 static int
-ipsec6_common_output(struct mbuf *m, struct inpcb *inp, int forwarding)
+ipsec6_common_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp,
+    int forwarding, u_long mtu)
 {
 	struct secpolicy *sp;
 	int error;
@@ -761,25 +802,7 @@ ipsec6_common_output(struct mbuf *m, struct inpcb *inp, int forwarding)
 		return (0); /* No IPsec required. */
 	}
 
-	if (!forwarding) {
+	error = ipsec6_check_pmtu(ifp, m, sp, forwarding);
-		/*
-		 * Do delayed checksums now because we send before
-		 * this is done in the normal processing path.
-		 */
-		if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA_IPV6) {
-			in6_delayed_cksum(m, m->m_pkthdr.len -
-			    sizeof(struct ip6_hdr), sizeof(struct ip6_hdr));
-			m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA_IPV6;
-		}
-#if defined(SCTP) || defined(SCTP_SUPPORT)
-		if (m->m_pkthdr.csum_flags & CSUM_SCTP_IPV6) {
-			sctp_delayed_cksum(m, sizeof(struct ip6_hdr));
-			m->m_pkthdr.csum_flags &= ~CSUM_SCTP_IPV6;
-		}
-#endif
-	}
-
-	error = ipsec6_check_pmtu(m, sp, forwarding);
 	if (error != 0) {
 		if (error == EJUSTRETURN)
 			return (0);
@@ -788,7 +811,7 @@ ipsec6_common_output(struct mbuf *m, struct inpcb *inp, int forwarding)
 	}
 
 	/* NB: callee frees mbuf and releases reference to SP */
-	error = ipsec6_process_packet(m, sp, inp);
+	error = ipsec6_process_packet(ifp, m, sp, inp, mtu);
 	if (error == EJUSTRETURN) {
 		/*
 		 * We had a SP with a level of 'use' and no SA. We
@@ -808,7 +831,7 @@ ipsec6_common_output(struct mbuf *m, struct inpcb *inp, int forwarding)
  * other values - mbuf consumed by IPsec.
  */
 int
-ipsec6_output(struct mbuf *m, struct inpcb *inp)
+ipsec6_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp, u_long mtu)
 {
 
 	/*
@@ -819,7 +842,7 @@ ipsec6_output(struct mbuf *m, struct inpcb *inp)
 	if (m_tag_find(m, PACKET_TAG_IPSEC_OUT_DONE, NULL) != NULL)
 		return (0);
 
-	return (ipsec6_common_output(m, inp, 0));
+	return (ipsec6_common_output(ifp, m, inp, 0, mtu));
 }
 
 /*
@@ -839,7 +862,7 @@ ipsec6_forward(struct mbuf *m)
 		m_freem(m);
 		return (EACCES);
 	}
-	return (ipsec6_common_output(m, NULL, 1));
+	return (ipsec6_common_output(NULL /* XXXKIB */, m, NULL, 1, 0));
 }
 #endif /* INET6 */
 
@@ -853,6 +876,10 @@ ipsec_process_done(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	struct m_tag *mtag;
 	int error;
 
+	if (sav->state >= SADB_SASTATE_DEAD) {
+		error = ESRCH;
+		goto bad;
+	}
 	saidx = &sav->sah->saidx;
 	switch (saidx->dst.sa.sa_family) {
 #ifdef INET
@@ -916,14 +943,16 @@ ipsec_process_done(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 		case AF_INET:
 			key_freesav(&sav);
 			IPSECSTAT_INC(ips_out_bundlesa);
-			return (ipsec4_perform_request(m, sp, NULL, idx));
+			return (ipsec4_perform_request(NULL, m, sp, NULL,
+			    idx, 0));
 			/* NOTREACHED */
 #endif
 #ifdef INET6
 		case AF_INET6:
 			key_freesav(&sav);
 			IPSEC6STAT_INC(ips_out_bundlesa);
-			return (ipsec6_perform_request(m, sp, NULL, idx));
+			return (ipsec6_perform_request(NULL, m, sp, NULL,
+			    idx, 0));
 			/* NOTREACHED */
 #endif /* INET6 */
 		default:

sys/netipsec/ipsec_pcb.c

@@ -49,6 +49,7 @@
 #include <netipsec/ipsec_support.h>
 #include <netipsec/key.h>
 #include <netipsec/key_debug.h>
+#include <netipsec/ipsec_offload.h>
 
 MALLOC_DEFINE(M_IPSEC_INPCB, "inpcbpolicy", "inpcb-resident ipsec policy");
 
@@ -166,18 +167,26 @@ ipsec_init_pcbpolicy(struct inpcb *inp)
 int
 ipsec_delete_pcbpolicy(struct inpcb *inp)
 {
+	struct inpcbpolicy *inp_sp;
 
-	if (inp->inp_sp == NULL)
+	inp_sp = inp->inp_sp;
+	if (inp_sp == NULL)
 		return (0);
-
-	if (inp->inp_sp->sp_in != NULL)
-		key_freesp(&inp->inp_sp->sp_in);
-
-	if (inp->inp_sp->sp_out != NULL)
-		key_freesp(&inp->inp_sp->sp_out);
-
-	free(inp->inp_sp, M_IPSEC_INPCB);
 	inp->inp_sp = NULL;
+
+	if (inp_sp->sp_in != NULL) {
+		if ((inp_sp->flags & INP_INBOUND_POLICY) != 0)
+			ipsec_accel_spddel(inp_sp->sp_in);
+		key_freesp(&inp_sp->sp_in);
+	}
+
+	if (inp_sp->sp_out != NULL) {
+		if ((inp_sp->flags & INP_OUTBOUND_POLICY) != 0)
+			ipsec_accel_spddel(inp_sp->sp_out);
+		key_freesp(&inp_sp->sp_out);
+	}
+
+	free(inp_sp, M_IPSEC_INPCB);
 	return (0);
 }
 
@@ -248,20 +257,26 @@ ipsec_copy_pcbpolicy(struct inpcb *old, struct inpcb *new)
 		if (sp == NULL)
 			return (ENOBUFS);
 		ipsec_setspidx_inpcb(new, &sp->spidx, IPSEC_DIR_INBOUND);
-		if (new->inp_sp->sp_in != NULL)
+		if (new->inp_sp->sp_in != NULL) {
+			ipsec_accel_spddel(new->inp_sp->sp_in);
 			key_freesp(&new->inp_sp->sp_in);
+		}
 		new->inp_sp->sp_in = sp;
 		new->inp_sp->flags |= INP_INBOUND_POLICY;
+		ipsec_accel_spdadd(sp, new);
 	}
 	if (old->inp_sp->flags & INP_OUTBOUND_POLICY) {
 		sp = ipsec_deepcopy_pcbpolicy(old->inp_sp->sp_out);
 		if (sp == NULL)
 			return (ENOBUFS);
 		ipsec_setspidx_inpcb(new, &sp->spidx, IPSEC_DIR_OUTBOUND);
-		if (new->inp_sp->sp_out != NULL)
+		if (new->inp_sp->sp_out != NULL) {
+			ipsec_accel_spddel(new->inp_sp->sp_out);
 			key_freesp(&new->inp_sp->sp_out);
+		}
 		new->inp_sp->sp_out = sp;
 		new->inp_sp->flags |= INP_OUTBOUND_POLICY;
+		ipsec_accel_spdadd(sp, new);
 	}
 	return (0);
 }
@@ -339,8 +354,10 @@ ipsec_set_pcbpolicy(struct inpcb *inp, struct ucred *cred,
 		flags = INP_OUTBOUND_POLICY;
 	}
 	/* Clear old SP and set new SP. */
-	if (*spp != NULL)
+	if (*spp != NULL) {
+		ipsec_accel_spddel(*spp);
 		key_freesp(spp);
+	}
 	*spp = newsp;
 	KEYDBG(IPSEC_DUMP,
 	    printf("%s: new SP(%p)\n", __func__, newsp));
@@ -348,6 +365,7 @@ ipsec_set_pcbpolicy(struct inpcb *inp, struct ucred *cred,
 		inp->inp_sp->flags &= ~flags;
 	else {
 		inp->inp_sp->flags |= flags;
+		ipsec_accel_spdadd(newsp, inp);
 		KEYDBG(IPSEC_DUMP, kdebug_secpolicy(newsp));
 	}
 	INP_WUNLOCK(inp);

sys/netipsec/ipsec_support.h

@@ -29,6 +29,7 @@
 
 #ifdef _KERNEL
 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
+struct ifnet;
 struct mbuf;
 struct inpcb;
 struct tcphdr;
@@ -58,7 +59,7 @@ int ipsec4_in_reject(const struct mbuf *, struct inpcb *);
 int ipsec4_input(struct mbuf *, int, int);
 int ipsec4_forward(struct mbuf *);
 int ipsec4_pcbctl(struct inpcb *, struct sockopt *);
-int ipsec4_output(struct mbuf *, struct inpcb *);
+int ipsec4_output(struct ifnet *, struct mbuf *, struct inpcb *, u_long);
 int ipsec4_capability(struct mbuf *, u_int);
 int ipsec4_ctlinput(ipsec_ctlinput_param_t);
 #endif /* INET */
@@ -68,7 +69,7 @@ int ipsec6_input(struct mbuf *, int, int);
 int ipsec6_in_reject(const struct mbuf *, struct inpcb *);
 int ipsec6_forward(struct mbuf *);
 int ipsec6_pcbctl(struct inpcb *, struct sockopt *);
-int ipsec6_output(struct mbuf *, struct inpcb *);
+int ipsec6_output(struct ifnet *, struct mbuf *, struct inpcb *, u_long);
 int ipsec6_capability(struct mbuf *, u_int);
 int ipsec6_ctlinput(ipsec_ctlinput_param_t);
 #endif /* INET6 */
@@ -77,7 +78,8 @@ struct ipsec_methods {
 	int	(*input)(struct mbuf *, int, int);
 	int	(*check_policy)(const struct mbuf *, struct inpcb *);
 	int	(*forward)(struct mbuf *);
-	int	(*output)(struct mbuf *, struct inpcb *);
+	int	(*output)(struct ifnet *, struct mbuf *, struct inpcb *,
+		    u_long);
 	int	(*pcbctl)(struct inpcb *, struct sockopt *);
 	size_t	(*hdrsize)(struct inpcb *);
 	int	(*capability)(struct mbuf *, u_int);
@@ -187,8 +189,8 @@ int ipsec_kmod_input(struct ipsec_support * const, struct mbuf *, int, int);
 int ipsec_kmod_check_policy(struct ipsec_support * const, struct mbuf *,
     struct inpcb *);
 int ipsec_kmod_forward(struct ipsec_support * const, struct mbuf *);
-int ipsec_kmod_output(struct ipsec_support * const, struct mbuf *,
+int ipsec_kmod_output(struct ipsec_support * const, struct ifnet *,
-    struct inpcb *);
+    struct mbuf *, struct inpcb *, u_long);
 int ipsec_kmod_pcbctl(struct ipsec_support * const, struct inpcb *,
     struct sockopt *);
 int ipsec_kmod_capability(struct ipsec_support * const, struct mbuf *, u_int);

sys/netipsec/key.c

@@ -83,6 +83,7 @@
 #include <netipsec/key.h>
 #include <netipsec/keysock.h>
 #include <netipsec/key_debug.h>
+#include <netipsec/ipsec_offload.h>
 
 #include <netipsec/ipsec.h>
 #ifdef INET6
@@ -90,12 +91,26 @@
 #endif
 
 #include <netipsec/xform.h>
+#include <netipsec/ipsec_offload.h>
 #include <machine/in_cksum.h>
 #include <machine/stdarg.h>
 
 /* randomness */
 #include <sys/random.h>
 
+#ifdef IPSEC_OFFLOAD
+void (*ipsec_accel_sa_newkey_p)(struct secasvar *sav);
+void (*ipsec_accel_forget_sav_p)(struct secasvar *sav);
+void (*ipsec_accel_spdadd_p)(struct secpolicy *sp, struct inpcb *inp);
+void (*ipsec_accel_spddel_p)(struct secpolicy *sp);
+int (*ipsec_accel_sa_lifetime_op_p)(struct secasvar *sav,
+    struct seclifetime *lft_c, if_t ifp, enum IF_SA_CNT_WHICH op,
+    struct rm_priotracker *sahtree_trackerp);
+void (*ipsec_accel_sync_p)(void);
+bool (*ipsec_accel_is_accel_sav_p)(struct secasvar *sav);
+struct mbuf *(*ipsec_accel_key_setaccelif_p)(struct secasvar *sav);
+#endif
+
 #define FULLMASK	0xff
 #define	_BITS(bytes)	((bytes) << 3)
 
@@ -391,6 +406,9 @@ static const int minsize[] = {
 	[SADB_X_EXT_SA_REPLAY] = sizeof(struct sadb_x_sa_replay),
 	[SADB_X_EXT_NEW_ADDRESS_SRC] = sizeof(struct sadb_address),
 	[SADB_X_EXT_NEW_ADDRESS_DST] = sizeof(struct sadb_address),
+	[SADB_X_EXT_LFT_CUR_SW_OFFL] = sizeof(struct sadb_lifetime),
+	[SADB_X_EXT_LFT_CUR_HW_OFFL] = sizeof(struct sadb_lifetime),
+	[SADB_X_EXT_IF_HW_OFFL] = sizeof(struct sadb_x_if_hw_offl),
 };
 _Static_assert(nitems(minsize) == SADB_EXT_MAX + 1, "minsize size mismatch");
 
@@ -424,6 +442,9 @@ static const int maxsize[] = {
 	[SADB_X_EXT_SA_REPLAY] = sizeof(struct sadb_x_sa_replay),
 	[SADB_X_EXT_NEW_ADDRESS_SRC] = 0,
 	[SADB_X_EXT_NEW_ADDRESS_DST] = 0,
+	[SADB_X_EXT_LFT_CUR_SW_OFFL] = sizeof(struct sadb_lifetime),
+	[SADB_X_EXT_LFT_CUR_HW_OFFL] = sizeof(struct sadb_lifetime),
+	[SADB_X_EXT_IF_HW_OFFL] = sizeof(struct sadb_x_if_hw_offl),
 };
 _Static_assert(nitems(maxsize) == SADB_EXT_MAX + 1, "maxsize size mismatch");
 
@@ -622,7 +643,6 @@ static struct callout key_timer;
 
 static void key_unlink(struct secpolicy *);
 static void key_detach(struct secpolicy *);
-static struct secpolicy *key_do_allocsp(struct secpolicyindex *spidx, u_int dir);
 static struct secpolicy *key_getsp(struct secpolicyindex *);
 static struct secpolicy *key_getspbyid(u_int32_t);
 static struct mbuf *key_gather_mbuf(struct mbuf *,
@@ -662,7 +682,7 @@ static int key_updateaddresses(struct socket *, struct mbuf *,
     const struct sadb_msghdr *, struct secasvar *, struct secasindex *);
 
 static struct mbuf *key_setdumpsa(struct secasvar *, u_int8_t,
-	u_int8_t, u_int32_t, u_int32_t);
+	u_int8_t, u_int32_t, u_int32_t, struct rm_priotracker *);
 static struct mbuf *key_setsadbmsg(u_int8_t, u_int16_t, u_int8_t,
 	u_int32_t, pid_t, u_int16_t);
 static struct mbuf *key_setsadbsa(struct secasvar *);
@@ -1228,6 +1248,11 @@ key_freesp(struct secpolicy **spp)
 	KEYDBG(IPSEC_DATA, kdebug_secpolicy(sp));
 
 	*spp = NULL;
+#ifdef IPSEC_OFFLOAD
+	KASSERT(CK_LIST_EMPTY(&sp->accel_ifps),
+	    ("key_freesp: sp %p still offloaded", sp));
+	free(__DECONST(char *, sp->accel_ifname), M_IPSEC_MISC);
+#endif
 	while (sp->tcount > 0)
 		ipsec_delisr(sp->req[--sp->tcount]);
 	free(sp, M_IPSEC_SP);
@@ -1241,6 +1266,7 @@ key_unlink(struct secpolicy *sp)
 	SPTREE_WUNLOCK();
 	if (SPDCACHE_ENABLED())
 		spdcache_clear();
+	ipsec_accel_sync();
 	key_freesp(&sp);
 }
 
@@ -1259,6 +1285,7 @@ key_detach(struct secpolicy *sp)
 		return;
 	}
 	sp->state = IPSEC_SPSTATE_DEAD;
+	ipsec_accel_spddel(sp);
 	TAILQ_REMOVE(&V_sptree[sp->spidx.dir], sp, chain);
 	V_spd_size--;
 	LIST_REMOVE(sp, idhash);
@@ -1286,6 +1313,7 @@ done:
 	newsp->state = IPSEC_SPSTATE_ALIVE;
 	V_spd_size++;
 	V_sp_genid++;
+	ipsec_accel_spdadd(newsp, NULL);
 }
 
 /*
@@ -1330,6 +1358,7 @@ key_register_ifnet(struct secpolicy **spp, u_int count)
 		 */
 		LIST_INSERT_HEAD(SPHASH_HASH(spp[i]->id), spp[i], idhash);
 		spp[i]->state = IPSEC_SPSTATE_IFNET;
+		ipsec_accel_spdadd(spp[i], NULL);
 	}
 	SPTREE_WUNLOCK();
 	/*
@@ -1358,6 +1387,7 @@ key_unregister_ifnet(struct secpolicy **spp, u_int count)
 		if (spp[i]->state != IPSEC_SPSTATE_IFNET)
 			continue;
 		spp[i]->state = IPSEC_SPSTATE_DEAD;
+		ipsec_accel_spddel(spp[i]);
 		TAILQ_REMOVE(&V_sptree_ifnet[spp[i]->spidx.dir],
 		    spp[i], chain);
 		V_spd_size--;
@@ -1366,6 +1396,7 @@ key_unregister_ifnet(struct secpolicy **spp, u_int count)
 	SPTREE_WUNLOCK();
 	if (SPDCACHE_ENABLED())
 		spdcache_clear();
+	ipsec_accel_sync();
 
 	for (i = 0; i < count; i++) {
 		m = key_setdumpsp(spp[i], SADB_X_SPDDELETE, 0, 0);
@@ -1425,6 +1456,7 @@ key_unlinksav(struct secasvar *sav)
 	/* Unlink from SPI hash */
 	LIST_REMOVE(sav, spihash);
 	sav->state = SADB_SASTATE_DEAD;
+	ipsec_accel_forget_sav(sav);
 	sah = sav->sah;
 	SAHTREE_WUNLOCK();
 	key_freesav(&sav);
@@ -1822,6 +1854,9 @@ key_sp2msg(struct secpolicy *sp, void *request, size_t *len)
 	size_t xlen, ilen;
 	caddr_t p;
 	int error, i;
+#ifdef IPSEC_OFFLOAD
+	struct sadb_x_if_hw_offl *xif;
+#endif
 
 	IPSEC_ASSERT(sp != NULL, ("null policy"));
 
@@ -1877,6 +1912,18 @@ key_sp2msg(struct secpolicy *sp, void *request, size_t *len)
 		}
 	}
 	xpl->sadb_x_policy_len = PFKEY_UNIT64(xlen);
+#ifdef IPSEC_OFFLOAD
+	if (error == 0 && sp->accel_ifname != NULL) {
+		xif = (struct sadb_x_if_hw_offl *)(xpl + 1);
+		bzero(xif, sizeof(*xif));
+		xif->sadb_x_if_hw_offl_len = PFKEY_UNIT64(sizeof(*xif));
+		xif->sadb_x_if_hw_offl_exttype = SADB_X_EXT_IF_HW_OFFL;
+		xif->sadb_x_if_hw_offl_flags = 0;
+		strncpy(xif->sadb_x_if_hw_offl_if, sp->accel_ifname,
+		    sizeof(xif->sadb_x_if_hw_offl_if));
+		xlen += sizeof(*xif);
+	}
+#endif
 	if (error == 0)
 		*len = xlen;
 	else
@@ -2089,6 +2136,27 @@ key_spdadd(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
 	newsp->lifetime = lft ? lft->sadb_lifetime_addtime : 0;
 	newsp->validtime = lft ? lft->sadb_lifetime_usetime : 0;
 	bcopy(&spidx, &newsp->spidx, sizeof(spidx));
+#ifdef IPSEC_OFFLOAD
+	if (!SADB_CHECKHDR(mhp, SADB_X_EXT_IF_HW_OFFL) &&
+	    !SADB_CHECKLEN(mhp, SADB_X_EXT_IF_HW_OFFL)) {
+		struct sadb_x_if_hw_offl *xof;
+
+		xof = (struct sadb_x_if_hw_offl *)mhp->ext[
+		    SADB_X_EXT_IF_HW_OFFL];
+		newsp->accel_ifname = malloc(sizeof(xof->sadb_x_if_hw_offl_if),
+		    M_IPSEC_MISC, M_NOWAIT);
+		if (newsp->accel_ifname == NULL) {
+			ipseclog((LOG_DEBUG, "%s: cannot alloc accel_ifname.\n",
+			    __func__));
+			key_freesp(&newsp);
+			return (key_senderror(so, m, error));
+		}
+		strncpy(__DECONST(char *, newsp->accel_ifname),
+		    xof->sadb_x_if_hw_offl_if,
+		    sizeof(xof->sadb_x_if_hw_offl_if));
+	}
+
+#endif
 
 	SPTREE_WLOCK();
 	if ((newsp->id = key_getnewspid()) == 0) {
@@ -2096,6 +2164,7 @@ key_spdadd(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
 			key_detach(oldsp);
 		SPTREE_WUNLOCK();
 		if (oldsp != NULL) {
+			ipsec_accel_sync();
 			key_freesp(&oldsp); /* first for key_detach */
 			IPSEC_ASSERT(oldsp != NULL, ("null oldsp: refcount bug"));
 			key_freesp(&oldsp); /* second for our reference */
@@ -2110,6 +2179,7 @@ key_spdadd(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
 	key_insertsp(newsp);
 	SPTREE_WUNLOCK();
 	if (oldsp != NULL) {
+		ipsec_accel_sync();
 		key_freesp(&oldsp); /* first for key_detach */
 		IPSEC_ASSERT(oldsp != NULL, ("null oldsp: refcount bug"));
 		key_freesp(&oldsp); /* second for our reference */
@@ -2291,6 +2361,7 @@ key_spddelete(struct socket *so, struct mbuf *m,
 	KEYDBG(KEY_STAMP,
 	    printf("%s: SP(%p)\n", __func__, sp));
 	KEYDBG(KEY_DATA, kdebug_secpolicy(sp));
+	ipsec_accel_spddel(sp);
 	key_unlink(sp);
 	key_freesp(&sp);
 
@@ -2562,6 +2633,7 @@ key_spdflush(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
 	 */
 	TAILQ_FOREACH(sp, &drainq, chain) {
 		sp->state = IPSEC_SPSTATE_DEAD;
+		ipsec_accel_spddel(sp);
 		LIST_REMOVE(sp, idhash);
 	}
 	V_sp_genid++;
@@ -2765,6 +2837,10 @@ key_getspreqmsglen(struct secpolicy *sp)
 
 		tlen += PFKEY_ALIGN8(len);
 	}
+#ifdef IPSEC_OFFLOAD
+	if (sp->accel_ifname != NULL)
+		tlen += sizeof(struct sadb_x_if_hw_offl);
+#endif
 	return (tlen);
 }
 
@@ -3006,6 +3082,32 @@ key_newsav(const struct sadb_msghdr *mhp, struct secasindex *saidx,
 	sav->state = SADB_SASTATE_LARVAL;
 	sav->pid = (pid_t)mhp->msg->sadb_msg_pid;
 	SAV_INITREF(sav);
+#ifdef IPSEC_OFFLOAD
+	CK_LIST_INIT(&sav->accel_ifps);
+	sav->accel_forget_tq = 0;
+	sav->accel_lft_sw = uma_zalloc_pcpu(ipsec_key_lft_zone,
+	    M_NOWAIT | M_ZERO);
+	if (sav->accel_lft_sw == NULL) {
+		*errp = ENOBUFS;
+		goto done;
+	}
+	if (!SADB_CHECKHDR(mhp, SADB_X_EXT_IF_HW_OFFL) &&
+	    !SADB_CHECKLEN(mhp, SADB_X_EXT_IF_HW_OFFL)) {
+		struct sadb_x_if_hw_offl *xof;
+
+		xof = (struct sadb_x_if_hw_offl *)mhp->ext[
+		    SADB_X_EXT_IF_HW_OFFL];
+		sav->accel_ifname = malloc(sizeof(xof->sadb_x_if_hw_offl_if),
+		    M_IPSEC_MISC, M_NOWAIT);
+		if (sav->accel_ifname == NULL) {
+			*errp = ENOBUFS;
+			goto done;
+		}
+		strncpy(__DECONST(char *, sav->accel_ifname),
+		    xof->sadb_x_if_hw_offl_if,
+		    sizeof(xof->sadb_x_if_hw_offl_if));
+	}
+#endif
 again:
 	sah = key_getsah(saidx);
 	if (sah == NULL) {
@@ -3069,9 +3171,10 @@ again:
 		SAH_ADDREF(sah);
 	}
 	/* Link SAV with SAH */
-	if (sav->state == SADB_SASTATE_MATURE)
+	if (sav->state == SADB_SASTATE_MATURE) {
 		TAILQ_INSERT_HEAD(&sah->savtree_alive, sav, chain);
-	else
+		ipsec_accel_sa_newkey(sav);
+	} else
 		TAILQ_INSERT_HEAD(&sah->savtree_larval, sav, chain);
 	/* Add SAV into SPI hash */
 	LIST_INSERT_HEAD(SAVHASH_HASH(sav->spi), sav, spihash);
@@ -3086,6 +3189,13 @@ done:
 			}
 			if (sav->lft_c != NULL)
 				uma_zfree_pcpu(ipsec_key_lft_zone, sav->lft_c);
+#ifdef IPSEC_OFFLOAD
+			if (sav->accel_lft_sw != NULL)
+				uma_zfree_pcpu(ipsec_key_lft_zone,
+				    sav->accel_lft_sw);
+			free(__DECONST(char *, sav->accel_ifname),
+			    M_IPSEC_MISC);
+#endif
 			free(sav, M_IPSEC_SA), sav = NULL;
 		}
 		if (sah != NULL)
@@ -3154,6 +3264,10 @@ key_delsav(struct secasvar *sav)
 	    ("attempt to free non DEAD SA %p", sav));
 	IPSEC_ASSERT(sav->refcnt == 0, ("reference count %u > 0",
 	    sav->refcnt));
+#ifdef IPSEC_OFFLOAD
+	KASSERT(CK_LIST_EMPTY(&sav->accel_ifps),
+	    ("key_unlinksav: sav %p still offloaded", sav));
+#endif
 
 	/*
 	 * SA must be unlinked from the chain and hashtbl.
@@ -3166,6 +3280,11 @@ key_delsav(struct secasvar *sav)
 		free(sav->lock, M_IPSEC_MISC);
 		uma_zfree_pcpu(ipsec_key_lft_zone, sav->lft_c);
 	}
+#ifdef IPSEC_OFFLOAD
+	/* XXXKIB should this be moved to key_cleansav()? */
+	uma_zfree_pcpu(ipsec_key_lft_zone, sav->accel_lft_sw);
+	free(__DECONST(char *, sav->accel_ifname), M_IPSEC_MISC);
+#endif
 	free(sav, M_IPSEC_SA);
 }
 
@@ -3589,7 +3708,7 @@ fail:
  */
 static struct mbuf *
 key_setdumpsa(struct secasvar *sav, uint8_t type, uint8_t satype,
-    uint32_t seq, uint32_t pid)
+    uint32_t seq, uint32_t pid, struct rm_priotracker *sahtree_trackerp)
 {
 	struct seclifetime lft_c;
 	struct mbuf *result = NULL, *tres = NULL, *m;
@@ -3605,8 +3724,15 @@ key_setdumpsa(struct secasvar *sav, uint8_t type, uint8_t satype,
 		SADB_X_EXT_NAT_T_SPORT, SADB_X_EXT_NAT_T_DPORT,
 		SADB_X_EXT_NAT_T_OAI, SADB_X_EXT_NAT_T_OAR,
 		SADB_X_EXT_NAT_T_FRAG,
+#ifdef IPSEC_OFFLOAD
+		SADB_X_EXT_LFT_CUR_SW_OFFL, SADB_X_EXT_LFT_CUR_HW_OFFL,
+		SADB_X_EXT_IF_HW_OFFL,
+#endif
 	};
 	uint32_t replay_count;
+#ifdef IPSEC_OFFLOAD
+	int error;
+#endif
 
 	SECASVAR_RLOCK_TRACKER;
 
@@ -3753,6 +3879,44 @@ key_setdumpsa(struct secasvar *sav, uint8_t type, uint8_t satype,
 		case SADB_X_EXT_NAT_T_FRAG:
 			/* We do not (yet) support those. */
 			continue;
+#ifdef IPSEC_OFFLOAD
+		case SADB_X_EXT_LFT_CUR_SW_OFFL:
+			if (!ipsec_accel_is_accel_sav(sav))
+				continue;
+			SAV_ADDREF(sav);
+			error = ipsec_accel_sa_lifetime_op(sav, &lft_c,
+			    NULL, IF_SA_CNT_TOTAL_SW_VAL, sahtree_trackerp);
+			if (error != 0) {
+				m = NULL;
+				goto fail;
+			}
+			m = key_setlifetime(&lft_c, dumporder[i]);
+			if (m == NULL)
+				goto fail;
+			key_freesav(&sav);
+			if (sav == NULL) {
+				m_freem(m);
+				goto fail;
+			}
+			break;
+		case SADB_X_EXT_LFT_CUR_HW_OFFL:
+			if (!ipsec_accel_is_accel_sav(sav))
+				continue;
+			memset(&lft_c, 0, sizeof(lft_c));
+			lft_c.bytes = sav->accel_hw_octets;
+			lft_c.allocations = sav->accel_hw_allocs;
+			m = key_setlifetime(&lft_c, dumporder[i]);
+			if (m == NULL)
+				goto fail;
+			break;
+		case SADB_X_EXT_IF_HW_OFFL:
+			if (!ipsec_accel_is_accel_sav(sav))
+				continue;
+			m = ipsec_accel_key_setaccelif(sav);
+			if (m == NULL)
+				continue;	/* benigh */
+			break;
+#endif
 
 		case SADB_EXT_ADDRESS_PROXY:
 		case SADB_EXT_IDENTITY_SRC:
@@ -4503,6 +4667,7 @@ key_flush_spd(time_t now)
 		V_spd_size--;
 		LIST_REMOVE(sp, idhash);
 		sp->state = IPSEC_SPSTATE_DEAD;
+		ipsec_accel_spddel(sp);
 		sp = nextsp;
 	}
 	V_sp_genid++;
@@ -4626,6 +4791,7 @@ key_flush_sad(time_t now)
 		TAILQ_REMOVE(&sav->sah->savtree_larval, sav, chain);
 		LIST_REMOVE(sav, spihash);
 		sav->state = SADB_SASTATE_DEAD;
+		ipsec_accel_forget_sav(sav);
 		sav = nextsav;
 	}
 	/* Unlink all SAs with expired HARD lifetime */
@@ -4642,6 +4808,7 @@ key_flush_sad(time_t now)
 		TAILQ_REMOVE(&sav->sah->savtree_alive, sav, chain);
 		LIST_REMOVE(sav, spihash);
 		sav->state = SADB_SASTATE_DEAD;
+		ipsec_accel_forget_sav(sav);
 		sav = nextsav;
 	}
 	/* Mark all SAs with expired SOFT lifetime as DYING */
@@ -5240,6 +5407,30 @@ key_updateaddresses(struct socket *so, struct mbuf *m,
 	/* Clone SA's content into newsav */
 	SAV_INITREF(newsav);
 	bcopy(sav, newsav, offsetof(struct secasvar, chain));
+#ifdef IPSEC_OFFLOAD
+	CK_LIST_INIT(&newsav->accel_ifps);
+	newsav->accel_forget_tq = 0;
+	newsav->accel_lft_sw = uma_zalloc_pcpu(ipsec_key_lft_zone,
+	    M_NOWAIT | M_ZERO);
+	if (newsav->accel_lft_sw == NULL) {
+		error = ENOBUFS;
+		goto fail;
+	}
+	if (sav->accel_ifname != NULL) {
+		struct sadb_x_if_hw_offl xof;
+
+		newsav->accel_ifname = malloc(sizeof(xof.sadb_x_if_hw_offl_if),
+		    M_IPSEC_MISC, M_NOWAIT);
+		if (newsav->accel_ifname == NULL) {
+			error = ENOBUFS;
+			goto fail;
+		}
+		strncpy(__DECONST(char *, sav->accel_ifname),
+		    newsav->accel_ifname,
+		    sizeof(xof.sadb_x_if_hw_offl_if));
+	}
+#endif
+
 	/*
 	 * We create new NAT-T config if it is needed.
 	 * Old NAT-T config will be freed by key_cleansav() when
@@ -5270,6 +5461,7 @@ key_updateaddresses(struct socket *so, struct mbuf *m,
 	TAILQ_REMOVE(&sav->sah->savtree_alive, sav, chain);
 	LIST_REMOVE(sav, spihash);
 	sav->state = SADB_SASTATE_DEAD;
+	ipsec_accel_forget_sav(sav);
 
 	/*
 	 * Link new SA with SAH. Keep SAs ordered by
@@ -5327,6 +5519,10 @@ fail:
 	if (isnew != 0)
 		key_freesah(&sah);
 	if (newsav != NULL) {
+#ifdef IPSEC_OFFLOAD
+		uma_zfree_pcpu(ipsec_key_lft_zone, newsav->accel_lft_sw);
+		free(__DECONST(char *, newsav->accel_ifname), M_IPSEC_MISC);
+#endif
 		if (newsav->natt != NULL)
 			free(newsav->natt, M_IPSEC_MISC);
 		free(newsav, M_IPSEC_SA);
@@ -5541,6 +5737,7 @@ key_update(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
 	KEYDBG(KEY_STAMP,
 	    printf("%s: SA(%p)\n", __func__, sav));
 	KEYDBG(KEY_DATA, kdebug_secasv(sav));
+	ipsec_accel_sa_newkey(sav);
 	key_freesav(&sav);
 
     {
@@ -5693,6 +5890,7 @@ key_add(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
 	KEYDBG(KEY_STAMP,
 	    printf("%s: return SA(%p)\n", __func__, sav));
 	KEYDBG(KEY_DATA, kdebug_secasv(sav));
+	ipsec_accel_sa_newkey(sav);
 	/*
 	 * If SADB_ADD was in response to SADB_ACQUIRE, we need to schedule
 	 * ACQ for deletion.
@@ -6197,6 +6395,7 @@ key_delete_all(struct socket *so, struct mbuf *m,
 	/* Unlink all queued SAs from SPI hash */
 	TAILQ_FOREACH(sav, &drainq, chain) {
 		sav->state = SADB_SASTATE_DEAD;
+		ipsec_accel_forget_sav(sav);
 		LIST_REMOVE(sav, spihash);
 	}
 	SAHTREE_WUNLOCK();
@@ -6265,6 +6464,7 @@ key_delete_xform(const struct xformsw *xsp)
 	/* Unlink all queued SAs from SPI hash */
 	TAILQ_FOREACH(sav, &drainq, chain) {
 		sav->state = SADB_SASTATE_DEAD;
+		ipsec_accel_forget_sav(sav);
 		LIST_REMOVE(sav, spihash);
 	}
 	SAHTREE_WUNLOCK();
@@ -6373,7 +6573,7 @@ key_get(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
 
 	/* create new sadb_msg to reply. */
 	n = key_setdumpsa(sav, SADB_GET, satype, mhp->msg->sadb_msg_seq,
-	    mhp->msg->sadb_msg_pid);
+	    mhp->msg->sadb_msg_pid, NULL);
 
 	key_freesav(&sav);
 	if (!n)
@@ -7615,9 +7815,11 @@ key_flush(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
 			 */
 			TAILQ_FOREACH(sav, &sah->savtree_larval, chain) {
 				sav->state = SADB_SASTATE_DEAD;
+				ipsec_accel_forget_sav(sav);
 			}
 			TAILQ_FOREACH(sav, &sah->savtree_alive, chain) {
 				sav->state = SADB_SASTATE_DEAD;
+				ipsec_accel_forget_sav(sav);
 			}
 		}
 		SAHTREE_WUNLOCK();
@@ -7639,10 +7841,12 @@ key_flush(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
 			TAILQ_FOREACH(sav, &sah->savtree_larval, chain) {
 				LIST_REMOVE(sav, spihash);
 				sav->state = SADB_SASTATE_DEAD;
+				ipsec_accel_forget_sav(sav);
 			}
 			TAILQ_FOREACH(sav, &sah->savtree_alive, chain) {
 				LIST_REMOVE(sav, spihash);
 				sav->state = SADB_SASTATE_DEAD;
+				ipsec_accel_forget_sav(sav);
 			}
 			/* Add SAH into flushq */
 			TAILQ_INSERT_HEAD(&flushq, sah, chain);
@@ -7706,6 +7910,7 @@ key_dump(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
 
 	/* count sav entries to be sent to the userland. */
 	cnt = 0;
+	IFNET_RLOCK();
 	SAHTREE_RLOCK();
 	TAILQ_FOREACH(sah, &V_sahtree, chain) {
 		if (mhp->msg->sadb_msg_satype != SADB_SATYPE_UNSPEC &&
@@ -7720,6 +7925,7 @@ key_dump(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
 
 	if (cnt == 0) {
 		SAHTREE_RUNLOCK();
+		IFNET_RUNLOCK();
 		return key_senderror(so, m, ENOENT);
 	}
 
@@ -7732,30 +7938,34 @@ key_dump(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
 		/* map proto to satype */
 		if ((satype = key_proto2satype(sah->saidx.proto)) == 0) {
 			SAHTREE_RUNLOCK();
+			IFNET_RUNLOCK();
 			ipseclog((LOG_DEBUG, "%s: there was invalid proto in "
 			    "SAD.\n", __func__));
 			return key_senderror(so, m, EINVAL);
 		}
 		TAILQ_FOREACH(sav, &sah->savtree_larval, chain) {
 			n = key_setdumpsa(sav, SADB_DUMP, satype,
-			    --cnt, mhp->msg->sadb_msg_pid);
+			    --cnt, mhp->msg->sadb_msg_pid, &sahtree_tracker);
 			if (n == NULL) {
 				SAHTREE_RUNLOCK();
+				IFNET_RUNLOCK();
 				return key_senderror(so, m, ENOBUFS);
 			}
 			key_sendup_mbuf(so, n, KEY_SENDUP_ONE);
 		}
 		TAILQ_FOREACH(sav, &sah->savtree_alive, chain) {
 			n = key_setdumpsa(sav, SADB_DUMP, satype,
-			    --cnt, mhp->msg->sadb_msg_pid);
+			    --cnt, mhp->msg->sadb_msg_pid, &sahtree_tracker);
 			if (n == NULL) {
 				SAHTREE_RUNLOCK();
+				IFNET_RUNLOCK();
 				return key_senderror(so, m, ENOBUFS);
 			}
 			key_sendup_mbuf(so, n, KEY_SENDUP_ONE);
 		}
 	}
 	SAHTREE_RUNLOCK();
+	IFNET_RUNLOCK();
 	m_freem(m);
 	return (0);
 }
@@ -8176,6 +8386,11 @@ key_align(struct mbuf *m, struct sadb_msghdr *mhp)
 		case SADB_X_EXT_SA_REPLAY:
 		case SADB_X_EXT_NEW_ADDRESS_SRC:
 		case SADB_X_EXT_NEW_ADDRESS_DST:
+#ifdef IPSEC_OFFLOAD
+		case SADB_X_EXT_LFT_CUR_SW_OFFL:
+		case SADB_X_EXT_LFT_CUR_HW_OFFL:
+		case SADB_X_EXT_IF_HW_OFFL:
+#endif
 			/* duplicate check */
 			/*
 			 * XXX Are there duplication payloads of either
@@ -8484,9 +8699,11 @@ key_vnet_destroy(void *arg __unused)
 		sah->state = SADB_SASTATE_DEAD;
 		TAILQ_FOREACH(sav, &sah->savtree_larval, chain) {
 			sav->state = SADB_SASTATE_DEAD;
+			ipsec_accel_forget_sav(sav);
 		}
 		TAILQ_FOREACH(sav, &sah->savtree_alive, chain) {
 			sav->state = SADB_SASTATE_DEAD;
+			ipsec_accel_forget_sav(sav);
 		}
 	}
 	SAHTREE_WUNLOCK();
@@ -8634,6 +8851,32 @@ key_setkey(struct seckey *src, uint16_t exttype)
 	return m;
 }
 
+#ifdef IPSEC_OFFLOAD
+struct mbuf *
+key_setaccelif(const char *ifname)
+{
+	struct mbuf *m = NULL;
+	struct sadb_x_if_hw_offl *p;
+	int len = PFKEY_ALIGN8(sizeof(*p));
+
+	m = m_get2(len, M_NOWAIT, MT_DATA, 0);
+	if (m == NULL)
+		return (m);
+	m_align(m, len);
+	m->m_len = len;
+	p = mtod(m, struct sadb_x_if_hw_offl *);
+
+	bzero(p, len);
+	p->sadb_x_if_hw_offl_len = PFKEY_UNIT64(len);
+	p->sadb_x_if_hw_offl_exttype = SADB_X_EXT_IF_HW_OFFL;
+	p->sadb_x_if_hw_offl_flags = 0;
+	strncpy(p->sadb_x_if_hw_offl_if, ifname,
+	    sizeof(p->sadb_x_if_hw_offl_if));
+
+	return (m);
+}
+#endif
+
 /*
  * Take one of the kernel's lifetime data structures and convert it
  * into a PF_KEY structure within an mbuf, suitable for sending up to
@@ -8709,3 +8952,15 @@ comp_algorithm_lookup(int alg)
 			return (supported_calgs[i].xform);
 	return (NULL);
 }
+
+void
+ipsec_sahtree_runlock(struct rm_priotracker *sahtree_trackerp)
+{
+	rm_runlock(&sahtree_lock, sahtree_trackerp);
+}
+
+void
+ipsec_sahtree_rlock(struct rm_priotracker *sahtree_trackerp)
+{
+	rm_rlock(&sahtree_lock, sahtree_trackerp);
+}

sys/netipsec/key.h

@@ -36,6 +36,7 @@
 
 #ifdef _KERNEL
 
+struct mbuf;
 struct secpolicy;
 struct secpolicyindex;
 struct secasvar;
@@ -49,6 +50,7 @@ struct xformsw;
 
 struct secpolicy *key_newsp(void);
 struct secpolicy *key_allocsp(struct secpolicyindex *, u_int);
+struct secpolicy *key_do_allocsp(struct secpolicyindex *spidx, u_int dir);
 struct secpolicy *key_msg2sp(struct sadb_x_policy *, size_t, int *);
 int key_sp2msg(struct secpolicy *, void *, size_t *);
 void key_addref(struct secpolicy *);
@@ -59,6 +61,7 @@ int key_havesp_any(void);
 void key_bumpspgen(void);
 uint32_t key_getspgen(void);
 uint32_t key_newreqid(void);
+struct mbuf *key_setaccelif(const char *ifname);
 
 struct secasvar *key_allocsa(union sockaddr_union *, uint8_t, uint32_t);
 struct secasvar *key_allocsa_tunnel(union sockaddr_union *,
@@ -84,6 +87,10 @@ extern void key_sa_recordxfer(struct secasvar *, struct mbuf *);
 uint16_t key_portfromsaddr(struct sockaddr *);
 void key_porttosaddr(struct sockaddr *, uint16_t port);
 
+struct rm_priotracker;
+void ipsec_sahtree_runlock(struct rm_priotracker *);
+void ipsec_sahtree_rlock(struct rm_priotracker *);
+
 #ifdef MALLOC_DECLARE
 MALLOC_DECLARE(M_IPSEC_SA);
 MALLOC_DECLARE(M_IPSEC_SAH);

sys/netipsec/key_debug.c

@@ -155,6 +155,8 @@ kdebug_sadb_exttype(uint16_t type)
 	X_NAME(SA_REPLAY);
 	X_NAME(NEW_ADDRESS_SRC);
 	X_NAME(NEW_ADDRESS_DST);
+	X_NAME(LFT_CUR_SW_OFFL);
+	X_NAME(LFT_CUR_HW_OFFL);
 	default:
 		return ("UNKNOWN");
 	};
@@ -251,6 +253,9 @@ kdebug_sadb(struct sadb_msg *base)
 		case SADB_X_EXT_NAT_T_DPORT:
 			kdebug_sadb_x_natt(ext);
 			break;
+		case SADB_X_EXT_LFT_CUR_SW_OFFL:
+		case SADB_X_EXT_LFT_CUR_HW_OFFL:
+			kdebug_sadb_lifetime(ext);
 		default:
 			printf("%s: invalid ext_type %u\n", __func__,
 			    ext->sadb_ext_type);

sys/netipsec/keydb.h

@@ -36,9 +36,11 @@
 
 #ifdef _KERNEL
 #include <sys/counter.h>
+#include <sys/ck.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/rmlock.h>
+#include <sys/_task.h>
 
 #include <netipsec/key_var.h>
 #include <opencrypto/_cryptodev.h>
@@ -125,6 +127,7 @@ struct xformsw;
 struct enc_xform;
 struct auth_hash;
 struct comp_algo;
+struct ifp_handle_sav;
 
 /*
  * Security Association
@@ -185,8 +188,19 @@ struct secasvar {
 
 	uint64_t cntr;			/* counter for GCM and CTR */
 	volatile u_int refcnt;		/* reference count */
+	CK_LIST_HEAD(, ifp_handle_sav) accel_ifps;
+	uintptr_t	accel_forget_tq;
+	const char	*accel_ifname;
+	uint32_t	accel_flags;
+	counter_u64_t	accel_lft_sw;
+	uint64_t	accel_hw_allocs;
+	uint64_t	accel_hw_octets;
+	uint64_t	accel_firstused;
 };
 
+#define	SADB_KEY_ACCEL_INST	0x00000001
+#define	SADB_KEY_ACCEL_DEINST	0x00000002
+
 #define	SECASVAR_RLOCK_TRACKER		struct rm_priotracker _secas_tracker
 #define	SECASVAR_RLOCK(_sav)		rm_rlock((_sav)->lock, &_secas_tracker)
 #define	SECASVAR_RUNLOCK(_sav)		rm_runlock((_sav)->lock, &_secas_tracker)

sys/netipsec/subr_ipsec.c

@@ -368,9 +368,10 @@ IPSEC_KMOD_METHOD(int, ipsec_kmod_ctlinput, sc,
 	ipsec_ctlinput_param_t param), METHOD_ARGS(param)
 )
 
-IPSEC_KMOD_METHOD(int, ipsec_kmod_output, sc,
+IPSEC_KMOD_METHOD(int, ipsec_kmod_output, sc, output,
-    output, METHOD_DECL(struct ipsec_support * const sc, struct mbuf *m,
+    METHOD_DECL(struct ipsec_support * const sc, struct ifnet *ifp,
-	struct inpcb *inp), METHOD_ARGS(m, inp)
+    struct mbuf *m, struct inpcb *inp, u_long mtu),
+    METHOD_ARGS(ifp, m, inp, mtu)
 )
 
 IPSEC_KMOD_METHOD(int, ipsec_kmod_pcbctl, sc,

sys/netipsec/xform_esp.c

@@ -83,8 +83,7 @@
 #define SPI_SIZE	4
 
 VNET_DEFINE(int, esp_enable) = 1;
-VNET_DEFINE_STATIC(int, esp_ctr_compatibility) = 1;
+VNET_DEFINE(int, esp_ctr_compatibility) = 1;
-#define V_esp_ctr_compatibility VNET(esp_ctr_compatibility)
 VNET_PCPUSTAT_DEFINE(struct espstat, espstat);
 VNET_PCPUSTAT_SYSINIT(espstat);
 

sys/sys/mbuf.h

@@ -1386,6 +1386,8 @@ extern bool		mb_use_ext_pgs;	/* Use ext_pgs for sendfile */
 #define	PACKET_TAG_IPSEC_NAT_T_PORTS		29 /* two uint16_t */
 #define	PACKET_TAG_ND_OUTGOING			30 /* ND outgoing */
 #define	PACKET_TAG_PF_REASSEMBLED		31
+#define	PACKET_TAG_IPSEC_ACCEL_OUT		32  /* IPSEC accel out */
+#define	PACKET_TAG_IPSEC_ACCEL_IN		33  /* IPSEC accel in */
 
 /* Specific cookies and tags. */
 

sys/sys/param.h

@@ -73,7 +73,7 @@
  * cannot include sys/param.h and should only be updated here.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 1500019
+#define __FreeBSD_version 1500020
 
 /*
  * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,

usr.bin/time/time.1

@@ -1,3 +1,6 @@
+.\"-
+.\" SPDX-License-Identifier: BSD-3-Clause
+.\"
 .\" Copyright (c) 1980, 1991, 1993
 .\"	The Regents of the University of California.  All rights reserved.
 .\"
@@ -56,7 +59,7 @@ the time used to execute the
 process and the time consumed by system overhead.
 .Pp
 The following options are available:
-.Bl -tag -width indent
+.Bl -tag -width "-o file"
 .It Fl a
 If the
 .Fl o
@@ -143,8 +146,8 @@ file.
 Then execute the command again to make a new copy and add the result to the same
 file:
 .Bd -literal -offset indent
-$ /usr/bin/time -o times.txt cp FreeBSD-12.1-RELEASE-amd64-bootonly.iso copy1.iso
+$ /usr/bin/time -o times.txt cp source.iso copy1.iso
-$ /usr/bin/time -a -o times.txt cp FreeBSD-12.1-RELEASE-amd64-bootonly.iso copy2.iso
+$ /usr/bin/time -a -o times.txt cp source.iso copy2.iso
 .Ed
 .Pp
 The
@@ -190,7 +193,8 @@ sys 0.00
 .Sh STANDARDS
 The
 .Nm
-utility is expected to conform to ISO/IEC 9945-2:1993 (``POSIX'').
+utility is expected to conform to
+.St -iso9945-2-93
 .Sh HISTORY
 A
 .Nm