pflow: import from OpenBSD

pflow is a pseudo device to export flow accounting data over UDP.
It's compatible with netflow version 5 and IPFIX (10).

The data is extracted from the pf state table. States are exported once
they are removed.

Reviewed by:	melifaro
Obtained from:	OpenBSD
Sponsored by:	Rubicon Communications, LLC ("Netgate")
Differential Revision:	https://reviews.freebsd.org/D43106

sbin/Makefile

@@ -81,6 +81,7 @@ SUBDIR.${MK_NVME}+=	nvmecontrol
 SUBDIR.${MK_OPENSSL}+=	decryptcore
 SUBDIR.${MK_PF}+=	pfctl
 SUBDIR.${MK_PF}+=	pflogd
+SUBDIR.${MK_PF}+=	pflowctl
 SUBDIR.${MK_QUOTAS}+=	quotacheck
 SUBDIR.${MK_ROUTED}+=	routed
 SUBDIR.${MK_VERIEXEC}+=	veriexec

sbin/pflowctl/Makefile

@@ -0,0 +1,10 @@
+
+.include <src.opts.mk>
+
+PACKAGE=pf
+PROG=	pflowctl
+MAN=	pflowctl.8
+
+SRCS = pflowctl.c
+
+.include <bsd.prog.mk>

sbin/pflowctl/pflowctl.8

@@ -0,0 +1,91 @@
+.\" $OpenBSD: pflow.4,v 1.19 2014/03/29 11:26:03 florian Exp $
+.\"
+.\" Copyright (c) 2008 Henning Brauer <henning@openbsd.org>
+.\" Copyright (c) 2008 Joerg Goltermann <jg@osn.de>
+.\"
+.\" Permission to use, copy, modify, and distribute this software for any
+.\" purpose with or without fee is hereby granted, provided that the above
+.\" copyright notice and this permission notice appear in all copies.
+.\"
+.\" THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+.\" WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+.\" MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+.\" ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+.\" WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+.\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+.\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+.\"
+.Dd $Mdocdate: January 08 2024 $
+.Dt PFLOWCTL 8
+.Os
+.Sh NAME
+.Nm pflowctl
+.Nd control pflow data export
+.Sh SYNOPSIS
+.Nm pflowctl
+.Bk -words
+.Op Fl lc
+.Op Fl d Ar id
+.Op Fl s Ar id ...
+.Ek
+.Sh DESCRIPTION
+The
+.Nm
+utility creates, configures and deletes netflow accounting data export using the
+.Xr pflow 4
+subsystem.
+.Pp
+The
+.Nm
+utility provides several commands.
+The options are as follows:
+.Bl -tag -width Ds
+.It Fl c
+Create a new
+.Xr pflow 4
+exporter.
+.It Fl d Ar id
+Remove an existing
+.Xr pflow 4
+exporter.
+The
+.Ar id
+may be either numeric or the full pflowX name.
+.It Fl l
+List all existing
+.Xr pflow 4
+exporters.
+.It Fl s Ar id ...
+Configure an existing
+.Xr pflow 4
+exporter.
+This takes the following keywords:
+.Pp
+.Bl -tag -width xxxxxxxxxxxx -compact
+.It Cm src
+set the source IP address (and optionally port).
+.It Cm dst
+set the destination IP address (and optionally port).
+.It Cm proto
+set the protocol version.
+Valid values are 5 and 10.
+.El
+.Pp
+Multiple keywords may be passed in the same command invocation.
+.Pp
+For example, the following command sets 10.0.0.1 as the source
+and 10.0.0.2:1234 as destination:
+.Bd -literal -offset indent
+# pflowctl -s pflow0 src 10.0.0.1 dst 10.0.0.2:1234
+.Ed
+.Sh SEE ALSO
+.Xr netintro 4 ,
+.Xr pf 4 ,
+.Xr pflow 4 ,
+.Xr udp 4 ,
+.Xr pf.conf 5
+.Sh HISTORY
+The
+.Nm
+command first appeared in
+.Fx 15.0 .

sbin/pflowctl/pflowctl.c

@@ -0,0 +1,548 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Rubicon Communications, LLC (Netgate)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *    - Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *    - Redistributions in binary form must reproduce the above
+ *      copyright notice, this list of conditions and the following
+ *      disclaimer in the documentation and/or other materials provided
+ *      with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#include <sys/cdefs.h>
+
+#include <err.h>
+#include <errno.h>
+#include <netdb.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <net/pflow.h>
+
+#include <netlink/netlink.h>
+#include <netlink/netlink_generic.h>
+#include <netlink/netlink_snl.h>
+#include <netlink/netlink_snl_generic.h>
+#include <netlink/netlink_snl_route.h>
+
+static int get(int id);
+
+static void
+usage(void)
+{
+	extern char *__progname;
+
+	fprintf(stderr,
+"usage: %s [-la] [-d id]\n",
+	    __progname);
+
+	exit(1);
+}
+
+static int
+pflow_to_id(const char *name)
+{
+	int ret, id;
+
+	ret = sscanf(name, "pflow%d", &id);
+	if (ret == 1)
+		return (id);
+
+	ret = sscanf(name, "%d", &id);
+	if (ret == 1)
+		return (id);
+
+	return (-1);
+}
+
+struct pflowctl_list {
+	int id;
+};
+#define	_IN(_field)	offsetof(struct genlmsghdr, _field)
+#define	_OUT(_field)	offsetof(struct pflowctl_list, _field)
+static struct snl_attr_parser ap_list[] = {
+	{ .type = PFLOWNL_L_ID, .off = _OUT(id), .cb = snl_attr_get_int32 },
+};
+static struct snl_field_parser fp_list[] = {};
+#undef _IN
+#undef _OUT
+SNL_DECLARE_PARSER(list_parser, struct genlmsghdr, fp_list, ap_list);
+
+static int
+list(void)
+{
+	struct snl_state ss = {};
+	struct snl_errmsg_data e = {};
+	struct pflowctl_list l = {};
+	struct snl_writer nw;
+	struct nlmsghdr *hdr;
+	uint32_t seq_id;
+	int family_id;
+
+	snl_init(&ss, NETLINK_GENERIC);
+	family_id = snl_get_genl_family(&ss, PFLOWNL_FAMILY_NAME);
+	if (family_id == 0)
+		errx(1, "pflow.ko is not loaded.");
+
+	snl_init_writer(&ss, &nw);
+	hdr = snl_create_genl_msg_request(&nw, family_id, PFLOWNL_CMD_LIST);
+
+	hdr = snl_finalize_msg(&nw);
+	if (hdr == NULL)
+		return (ENOMEM);
+	seq_id = hdr->nlmsg_seq;
+
+	snl_send_message(&ss, hdr);
+
+	while ((hdr = snl_read_reply_multi(&ss, seq_id, &e)) != NULL) {
+		if (! snl_parse_nlmsg(&ss, hdr, &list_parser, &l))
+			continue;
+
+		get(l.id);
+	}
+
+	if (e.error)
+		errc(1, e.error, "failed to list");
+
+	return (0);
+}
+
+struct pflowctl_create {
+	int id;
+};
+#define	_IN(_field)	offsetof(struct genlmsghsdr, _field)
+#define	_OUT(_field)	offsetof(struct pflowctl_create, _field)
+static struct snl_attr_parser ap_create[] = {
+	{ .type = PFLOWNL_CREATE_ID, .off = _OUT(id), .cb = snl_attr_get_int32 },
+};
+static struct snl_field_parser pf_create[] = {};
+#undef _IN
+#undef _OUT
+SNL_DECLARE_PARSER(create_parser, struct genlmsghdr, pf_create, ap_create);
+
+static int
+create(void)
+{
+	struct snl_state ss = {};
+	struct snl_errmsg_data e = {};
+	struct pflowctl_create c = {};
+	struct snl_writer nw;
+	struct nlmsghdr *hdr;
+	uint32_t seq_id;
+	int family_id;
+
+	snl_init(&ss, NETLINK_GENERIC);
+	family_id = snl_get_genl_family(&ss, PFLOWNL_FAMILY_NAME);
+	if (family_id == 0)
+		errx(1, "pflow.ko is not loaded.");
+
+	snl_init_writer(&ss, &nw);
+	hdr = snl_create_genl_msg_request(&nw, family_id, PFLOWNL_CMD_CREATE);
+
+	hdr = snl_finalize_msg(&nw);
+	if (hdr == NULL)
+		return (ENOMEM);
+	seq_id = hdr->nlmsg_seq;
+
+	snl_send_message(&ss, hdr);
+
+	while ((hdr = snl_read_reply_multi(&ss, seq_id, &e)) != NULL) {
+		if (! snl_parse_nlmsg(&ss, hdr, &create_parser, &c))
+			continue;
+
+		printf("pflow%d\n", c.id);
+	}
+
+	if (e.error)
+		errc(1, e.error, "failed to create");
+
+	return (0);
+}
+
+static int
+del(char *idstr)
+{
+	struct snl_state ss = {};
+	struct snl_errmsg_data e = {};
+	struct snl_writer nw;
+	struct nlmsghdr *hdr;
+	int family_id;
+	int id;
+
+	id = pflow_to_id(idstr);
+	if (id < 0)
+		return (EINVAL);
+
+	snl_init(&ss, NETLINK_GENERIC);
+	family_id = snl_get_genl_family(&ss, PFLOWNL_FAMILY_NAME);
+	if (family_id == 0)
+		errx(1, "pflow.ko is not loaded.");
+
+	snl_init_writer(&ss, &nw);
+	hdr = snl_create_genl_msg_request(&nw, family_id, PFLOWNL_CMD_DEL);
+
+	snl_add_msg_attr_s32(&nw, PFLOWNL_DEL_ID, id);
+
+	hdr = snl_finalize_msg(&nw);
+	if (hdr == NULL)
+		return (ENOMEM);
+
+	snl_send_message(&ss, hdr);
+	snl_read_reply_code(&ss, hdr->nlmsg_seq, &e);
+
+	if (e.error)
+		errc(1, e.error, "failed to delete");
+
+	return (0);
+}
+
+struct pflowctl_sockaddr {
+	union {
+		struct sockaddr_in in;
+		struct sockaddr_in6 in6;
+		struct sockaddr_storage storage;
+	};
+};
+static bool
+pflowctl_post_sockaddr(struct snl_state* ss __unused, void *target)
+{
+	struct pflowctl_sockaddr *s = (struct pflowctl_sockaddr *)target;
+
+	if (s->storage.ss_family == AF_INET)
+		s->storage.ss_len = sizeof(struct sockaddr_in);
+	else if (s->storage.ss_family == AF_INET6)
+		s->storage.ss_len = sizeof(struct sockaddr_in6);
+	else
+		return (false);
+
+	return (true);
+}
+#define _OUT(_field)	offsetof(struct pflowctl_sockaddr, _field)
+static struct snl_attr_parser nla_p_sockaddr[] = {
+	{ .type = PFLOWNL_ADDR_FAMILY, .off = _OUT(in.sin_family), .cb = snl_attr_get_uint8 },
+	{ .type = PFLOWNL_ADDR_PORT, .off = _OUT(in.sin_port), .cb = snl_attr_get_uint16 },
+	{ .type = PFLOWNL_ADDR_IP, .off = _OUT(in.sin_addr), .cb = snl_attr_get_in_addr },
+	{ .type = PFLOWNL_ADDR_IP6, .off = _OUT(in6.sin6_addr), .cb = snl_attr_get_in6_addr },
+};
+SNL_DECLARE_ATTR_PARSER_EXT(sockaddr_parser, 0, nla_p_sockaddr, pflowctl_post_sockaddr);
+#undef _OUT
+
+struct pflowctl_get {
+	int id;
+	int version;
+	struct pflowctl_sockaddr src;
+	struct pflowctl_sockaddr dst;
+};
+#define	_IN(_field)	offsetof(struct genlmsghdr, _field)
+#define	_OUT(_field)	offsetof(struct pflowctl_get, _field)
+static struct snl_attr_parser ap_get[] = {
+	{ .type = PFLOWNL_GET_ID, .off = _OUT(id), .cb = snl_attr_get_int32 },
+	{ .type = PFLOWNL_GET_VERSION, .off = _OUT(version), .cb = snl_attr_get_int16 },
+	{ .type = PFLOWNL_GET_SRC, .off = _OUT(src), .arg = &sockaddr_parser, .cb = snl_attr_get_nested },
+	{ .type = PFLOWNL_GET_DST, .off = _OUT(dst), .arg = &sockaddr_parser, .cb = snl_attr_get_nested },
+};
+static struct snl_field_parser fp_get[] = {};
+#undef _IN
+#undef _OUT
+SNL_DECLARE_PARSER(get_parser, struct genlmsghdr, fp_get, ap_get);
+
+static void
+print_sockaddr(const char *prefix, const struct sockaddr_storage *s)
+{
+	char buf[INET6_ADDRSTRLEN];
+	int error;
+
+	if (s->ss_family != AF_INET && s->ss_family != AF_INET6)
+		return;
+
+	if (s->ss_family == AF_INET ||
+	    s->ss_family == AF_INET6) {
+		error = getnameinfo((const struct sockaddr *)s,
+		    s->ss_len, buf, sizeof(buf), NULL, 0,
+		    NI_NUMERICHOST);
+		if (error)
+			err(1, "sender: %s", gai_strerror(error));
+	}
+
+	printf("%s", prefix);
+	switch (s->ss_family) {
+	case AF_INET: {
+		const struct sockaddr_in *sin = (const struct sockaddr_in *)s;
+		if (sin->sin_addr.s_addr != INADDR_ANY) {
+			printf("%s", buf);
+			if (sin->sin_port != 0)
+				printf(":%u", ntohs(sin->sin_port));
+		}
+		break;
+	}
+	case AF_INET6: {
+		const struct sockaddr_in6 *sin6 = (const struct sockaddr_in6 *)s;
+		if (!IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) {
+			printf("[%s]", buf);
+			if (sin6->sin6_port != 0)
+				printf(":%u", ntohs(sin6->sin6_port));
+		}
+		break;
+	}
+	}
+}
+
+static int
+get(int id)
+{
+	struct snl_state ss = {};
+	struct snl_errmsg_data e = {};
+	struct pflowctl_get g = {};
+	struct snl_writer nw;
+	struct nlmsghdr *hdr;
+	uint32_t seq_id;
+	int family_id;
+
+	snl_init(&ss, NETLINK_GENERIC);
+	family_id = snl_get_genl_family(&ss, PFLOWNL_FAMILY_NAME);
+	if (family_id == 0)
+		errx(1, "pflow.ko is not loaded.");
+
+	snl_init_writer(&ss, &nw);
+	hdr = snl_create_genl_msg_request(&nw, family_id, PFLOWNL_CMD_GET);
+	snl_add_msg_attr_s32(&nw, PFLOWNL_GET_ID, id);
+
+	hdr = snl_finalize_msg(&nw);
+	if (hdr == NULL)
+		return (ENOMEM);
+	seq_id = hdr->nlmsg_seq;
+
+	snl_send_message(&ss, hdr);
+
+	while ((hdr = snl_read_reply_multi(&ss, seq_id, &e)) != NULL) {
+		if (! snl_parse_nlmsg(&ss, hdr, &get_parser, &g))
+			continue;
+
+		printf("pflow%d: version %d", g.id, g.version);
+		print_sockaddr(" src ", &g.src.storage);
+		print_sockaddr(" dst ", &g.dst.storage);
+		printf("\n");
+	}
+
+	if (e.error)
+		errc(1, e.error, "failed to get");
+
+	return (0);
+}
+
+struct pflowctl_set {
+	int id;
+	uint16_t version;
+	struct sockaddr_storage src;
+	struct sockaddr_storage dst;
+};
+static inline bool
+snl_add_msg_attr_sockaddr(struct snl_writer *nw, int attrtype, struct sockaddr_storage *s)
+{
+	int off = snl_add_msg_attr_nested(nw, attrtype);
+
+	snl_add_msg_attr_u8(nw, PFLOWNL_ADDR_FAMILY, s->ss_family);
+
+	switch (s->ss_family) {
+	case AF_INET: {
+		const struct sockaddr_in *in = (const struct sockaddr_in *)s;
+		snl_add_msg_attr_u16(nw, PFLOWNL_ADDR_PORT, in->sin_port);
+		snl_add_msg_attr_ip4(nw, PFLOWNL_ADDR_IP, &in->sin_addr);
+		break;
+	}
+	case AF_INET6: {
+		const struct sockaddr_in6 *in6 = (const struct sockaddr_in6 *)s;
+		snl_add_msg_attr_u16(nw, PFLOWNL_ADDR_PORT, in6->sin6_port);
+		snl_add_msg_attr_ip6(nw, PFLOWNL_ADDR_IP6, &in6->sin6_addr);
+		break;
+	}
+	default:
+		return (false);
+	}
+	snl_end_attr_nested(nw, off);
+
+	return (true);
+}
+
+static int
+do_set(struct pflowctl_set *s)
+{
+	struct snl_state ss = {};
+	struct snl_errmsg_data e = {};
+	struct snl_writer nw;
+	struct nlmsghdr *hdr;
+	int family_id;
+
+	snl_init(&ss, NETLINK_GENERIC);
+	family_id = snl_get_genl_family(&ss, PFLOWNL_FAMILY_NAME);
+	if (family_id == 0)
+		errx(1, "pflow.ko is not loaded.");
+
+	snl_init_writer(&ss, &nw);
+	snl_create_genl_msg_request(&nw, family_id, PFLOWNL_CMD_SET);
+
+	snl_add_msg_attr_s32(&nw, PFLOWNL_SET_ID, s->id);
+	if (s->version != 0)
+		snl_add_msg_attr_u16(&nw, PFLOWNL_SET_VERSION, s->version);
+	if (s->src.ss_len != 0)
+		snl_add_msg_attr_sockaddr(&nw, PFLOWNL_SET_SRC, &s->src);
+	if (s->dst.ss_len != 0)
+		snl_add_msg_attr_sockaddr(&nw, PFLOWNL_SET_DST, &s->dst);
+
+	hdr = snl_finalize_msg(&nw);
+	if (hdr == NULL)
+		return (1);
+
+	snl_send_message(&ss, hdr);
+	snl_read_reply_code(&ss, hdr->nlmsg_seq, &e);
+
+	if (e.error)
+		errc(1, e.error, "failed to set");
+
+	return (0);
+}
+
+static void
+pflowctl_addr(const char *val, struct sockaddr_storage *ss)
+{
+	struct addrinfo *res0;
+	int error;
+	bool flag;
+	char *ip, *port;
+	char buf[sysconf(_SC_HOST_NAME_MAX) + 1 + sizeof(":65535")];
+	struct addrinfo hints = {
+		.ai_family = AF_UNSPEC,
+		.ai_socktype = SOCK_DGRAM, /*dummy*/
+		.ai_flags = AI_NUMERICHOST,
+	};
+
+	if (strlcpy(buf, val, sizeof(buf)) >= sizeof(buf))
+		errx(1, "%s bad value", val);
+
+	port = NULL;
+	flag = *buf == '[';
+
+	for (char *cp = buf; *cp; ++cp) {
+		if (*cp == ']' && *(cp + 1) == ':' && flag) {
+			*cp = '\0';
+			*(cp + 1) = '\0';
+			port = cp + 2;
+			break;
+		}
+		if (*cp == ']' && *(cp + 1) == '\0' && flag) {
+			*cp = '\0';
+			port = NULL;
+			break;
+		}
+		if (*cp == ':' && !flag) {
+			*cp = '\0';
+			port = cp + 1;
+			break;
+		}
+	}
+
+	ip = buf;
+	if (flag)
+		ip++;
+
+	if ((error = getaddrinfo(ip, port, &hints, &res0)) != 0)
+		errx(1, "error in parsing address string: %s",
+		    gai_strerror(error));
+
+	memcpy(ss, res0->ai_addr, res0->ai_addr->sa_len);
+	freeaddrinfo(res0);
+}
+
+static int
+set(char *idstr, int argc, char *argv[])
+{
+	struct pflowctl_set s = {};
+
+	s.id = pflow_to_id(idstr);
+	if (s.id < 0)
+		return (EINVAL);
+
+	while (argc > 0) {
+		if (strcmp(argv[0], "src") == 0) {
+			if (argc < 2)
+				usage();
+
+			pflowctl_addr(argv[1], &s.src);
+
+			argc -= 2;
+			argv += 2;
+		} else if (strcmp(argv[0], "dst") == 0) {
+			if (argc < 2)
+				usage();
+
+			pflowctl_addr(argv[1], &s.dst);
+
+			argc -= 2;
+			argv += 2;
+		} else if (strcmp(argv[0], "proto") == 0) {
+			if (argc < 2)
+				usage();
+
+			s.version = strtol(argv[1], NULL, 10);
+
+			argc -= 2;
+			argv += 2;
+		} else {
+			usage();
+		}
+	}
+
+	return (do_set(&s));
+}
+
+static const struct snl_hdr_parser *all_parsers[] = {
+	&list_parser,
+	&get_parser,
+};
+
+int
+main(int argc, char *argv[])
+{
+	int ch;
+
+	SNL_VERIFY_PARSERS(all_parsers);
+
+	if (argc < 2)
+		usage();
+
+	while ((ch = getopt(argc, argv,
+	    "lcd:s:")) != -1) {
+		switch (ch) {
+		case 'l':
+			return (list());
+		case 'c':
+			return (create());
+		case 'd':
+			return (del(optarg));
+		case 's':
+			return (set(optarg, argc - optind, argv + optind));
+		}
+	}
+
+	return (0);
+}

share/man/man4/Makefile

@@ -433,6 +433,7 @@ MAN=	aac.4 \
 	pcm.4 \
 	${_pf.4} \
 	${_pflog.4} \
+	${_pflow.4} \
 	${_pfsync.4} \
 	pim.4 \
 	pms.4 \
@@ -968,6 +969,7 @@ _atf_test_case.4=	atf-test-case.4
 .if ${MK_PF} != "no"
 _pf.4=		pf.4
 _pflog.4=	pflog.4
+_pflow.4=	pflow.4
 _pfsync.4=	pfsync.4
 .endif
 

share/man/man4/pflow.4

@@ -0,0 +1,123 @@
+.\" $OpenBSD: pflow.4,v 1.19 2014/03/29 11:26:03 florian Exp $
+.\"
+.\" Copyright (c) 2008 Henning Brauer <henning@openbsd.org>
+.\" Copyright (c) 2008 Joerg Goltermann <jg@osn.de>
+.\"
+.\" Permission to use, copy, modify, and distribute this software for any
+.\" purpose with or without fee is hereby granted, provided that the above
+.\" copyright notice and this permission notice appear in all copies.
+.\"
+.\" THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+.\" WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+.\" MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+.\" ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+.\" WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+.\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+.\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+.\"
+.Dd $Mdocdate: January 08 2024 $
+.Dt PFLOW 4
+.Os
+.Sh NAME
+.Nm pflow
+.Nd kernel interface for pflow data export
+.Sh SYNOPSIS
+.Cd "pseudo-device pflow"
+.Sh DESCRIPTION
+The
+.Nm
+subsystem exports
+.Nm
+accounting data from the kernel using
+.Xr udp 4
+packets.
+.Nm
+is compatible with netflow version 5 and IPFIX (10).
+The data is extracted from the
+.Xr pf 4
+state table.
+.Pp
+Multiple
+.Nm
+interfaces can be created at runtime using the
+.Ic pflowctl Ns Ar N Ic -c
+command.
+Each interface must be configured with a flow receiver IP address
+and a flow receiver port number.
+.Pp
+Only states created by a rule marked with the
+.Ar pflow
+keyword are exported by
+.Nm .
+.Pp
+.Nm
+will attempt to export multiple
+.Nm
+records in one
+UDP packet, but will not hold a record for longer than 30 seconds.
+.Pp
+Each packet seen on this interface has one header and a variable number of
+flows.
+The header indicates the version of the protocol, number of
+flows in the packet, a unique sequence number, system time, and an engine
+ID and type.
+Header and flow structs are defined in
+.In net/pflow.h .
+.Pp
+The
+.Nm
+source and destination addresses are controlled by
+.Xr pflowctl 8 .
+.Cm src
+is the sender IP address of the UDP packet which can be used
+to identify the source of the data on the
+.Nm
+collector.
+.Cm dst
+defines the collector IP address and the port.
+The
+.Cm dst
+IP address and port must be defined to enable the export of flows.
+.Pp
+For example, the following command sets 10.0.0.1 as the source
+and 10.0.0.2:1234 as destination:
+.Bd -literal -offset indent
+# pflowctl -s pflow0 src 10.0.0.1 dst 10.0.0.2:1234
+.Ed
+.Pp
+The protocol is set to IPFIX with the following command:
+.Bd -literal -offset indent
+# pflowctl -s pflow0 proto 10
+.Ed
+.Sh SEE ALSO
+.Xr netintro 4 ,
+.Xr pf 4 ,
+.Xr udp 4 ,
+.Xr pf.conf 5 ,
+.Xr pflowctl 8 ,
+.Xr tcpdump 8
+.Sh STANDARDS
+.Rs
+.%A B. Claise
+.%D January 2008
+.%R RFC 5101
+.%T "Specification of the IP Flow Information Export (IPFIX) Protocol for the Exchange of IP Traffic Flow Information"
+.Re
+.Sh HISTORY
+The
+.Nm
+device first appeared in
+.Ox 4.5
+and was imported into
+FreeBSD 15.0 .
+.Sh BUGS
+A state created by
+.Xr pfsync 4
+can have a creation or expiration time before the machine came up.
+In this case,
+.Nm
+pretends such flows were created or expired when the machine came up.
+.Pp
+The IPFIX implementation is incomplete:
+The required transport protocol SCTP is not supported.
+Transport over TCP and DTLS protected flow export is also not supported.

sys/conf/files

@@ -4511,6 +4511,7 @@ netpfil/pf/pf_osfp.c		optional pf inet
 netpfil/pf/pf_ruleset.c		optional pf inet
 netpfil/pf/pf_syncookies.c	optional pf inet
 netpfil/pf/pf_table.c		optional pf inet
+netpfil/pf/pflow.c		optional pflow pf inet
 netpfil/pf/pfsync_nv.c		optional pfsync pf inet
 netpfil/pf/in4_cksum.c		optional pf inet
 netsmb/smb_conn.c		optional netsmb

sys/modules/Makefile

@@ -306,6 +306,7 @@ SUBDIR=	\
 	${_pcfclock} \
 	${_pf} \
 	${_pflog} \
+	${_pflow} \
 	${_pfsync} \
 	plip \
 	${_pms} \
@@ -611,6 +612,7 @@ _netgraph=	netgraph
 	${MK_INET6_SUPPORT} != "no")) || defined(ALL_MODULES)
 _pf=		pf
 _pflog=		pflog
+_pflow=		pflow
 .if ${MK_INET_SUPPORT} != "no"
 _pfsync=	pfsync
 .endif

sys/modules/pflow/Makefile

@@ -0,0 +1,16 @@
+.PATH: ${SRCTOP}/sys/netpfil/pf
+
+KMOD=	pflow
+SRCS=	pflow.c \
+	opt_pf.h opt_inet.h opt_inet6.h opt_global.h
+SRCS+=	bus_if.h device_if.h
+
+.if !defined(KERNBUILDDIR)
+.if defined(VIMAGE)
+opt_global.h:
+	echo "#define VIMAGE 1" >> ${.TARGET}
+CFLAGS+=	-include opt_global.h
+.endif
+.endif
+
+.include <bsd.kmod.mk>

sys/net/pflow.h

@@ -0,0 +1,333 @@
+/*	$OpenBSD: if_pflow.h,v 1.19 2022/11/23 15:12:27 mvs Exp $	*/
+
+/*
+ * Copyright (c) 2008 Henning Brauer <henning@openbsd.org>
+ * Copyright (c) 2008 Joerg Goltermann <jg@osn.de>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF MIND, USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+ * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef _NET_IF_PFLOW_H_
+#define _NET_IF_PFLOW_H_
+
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+
+#include <netinet/in.h>
+
+#ifdef _KERNEL
+#include <sys/param.h>
+#include <sys/lock.h>
+#include <sys/rmlock.h>
+#include <sys/interrupt.h>
+#include <net/if.h>
+#include <net/if_var.h>
+#include <net/if_private.h>
+#include <net/pfvar.h>
+
+#include <netinet/ip.h>
+#endif
+
+#define PFLOW_ID_LEN	sizeof(u_int64_t)
+
+#define PFLOW_MAXFLOWS 30
+#define PFLOW_ENGINE_TYPE 42
+#define PFLOW_ENGINE_ID 42
+#define PFLOW_MAXBYTES 0xffffffff
+#define PFLOW_TIMEOUT 30
+#define PFLOW_TMPL_TIMEOUT 30 /* rfc 5101 10.3.6 (p.40) recommends 600 */
+
+#define PFLOW_IPFIX_TMPL_SET_ID 2
+
+/* RFC 5102 Information Element Identifiers */
+
+#define PFIX_IE_octetDeltaCount			  1
+#define PFIX_IE_packetDeltaCount		  2
+#define PFIX_IE_protocolIdentifier		  4
+#define PFIX_IE_ipClassOfService		  5
+#define PFIX_IE_sourceTransportPort		  7
+#define PFIX_IE_sourceIPv4Address		  8
+#define PFIX_IE_ingressInterface		 10
+#define PFIX_IE_destinationTransportPort	 11
+#define PFIX_IE_destinationIPv4Address		 12
+#define PFIX_IE_egressInterface			 14
+#define PFIX_IE_flowEndSysUpTime		 21
+#define PFIX_IE_flowStartSysUpTime		 22
+#define PFIX_IE_sourceIPv6Address		 27
+#define PFIX_IE_destinationIPv6Address		 28
+#define PFIX_IE_flowStartMilliseconds		152
+#define PFIX_IE_flowEndMilliseconds		153
+
+struct pflow_flow {
+	u_int32_t	src_ip;
+	u_int32_t	dest_ip;
+	u_int32_t	nexthop_ip;
+	u_int16_t	if_index_in;
+	u_int16_t	if_index_out;
+	u_int32_t	flow_packets;
+	u_int32_t	flow_octets;
+	u_int32_t	flow_start;
+	u_int32_t	flow_finish;
+	u_int16_t	src_port;
+	u_int16_t	dest_port;
+	u_int8_t	pad1;
+	u_int8_t	tcp_flags;
+	u_int8_t	protocol;
+	u_int8_t	tos;
+	u_int16_t	src_as;
+	u_int16_t	dest_as;
+	u_int8_t	src_mask;
+	u_int8_t	dest_mask;
+	u_int16_t	pad2;
+} __packed;
+
+struct pflow_set_header {
+	u_int16_t	set_id;
+	u_int16_t	set_length; /* total length of the set,
+				       in octets, including the set header */
+} __packed;
+
+#define PFLOW_SET_HDRLEN sizeof(struct pflow_set_header)
+
+struct pflow_tmpl_hdr {
+	u_int16_t	tmpl_id;
+	u_int16_t	field_count;
+} __packed;
+
+struct pflow_tmpl_fspec {
+	u_int16_t	field_id;
+	u_int16_t	len;
+} __packed;
+
+/* update pflow_clone_create() when changing pflow_ipfix_tmpl_ipv4 */
+struct pflow_ipfix_tmpl_ipv4 {
+	struct pflow_tmpl_hdr	h;
+	struct pflow_tmpl_fspec	src_ip;
+	struct pflow_tmpl_fspec	dest_ip;
+	struct pflow_tmpl_fspec	if_index_in;
+	struct pflow_tmpl_fspec	if_index_out;
+	struct pflow_tmpl_fspec	packets;
+	struct pflow_tmpl_fspec	octets;
+	struct pflow_tmpl_fspec	start;
+	struct pflow_tmpl_fspec	finish;
+	struct pflow_tmpl_fspec	src_port;
+	struct pflow_tmpl_fspec	dest_port;
+	struct pflow_tmpl_fspec	tos;
+	struct pflow_tmpl_fspec	protocol;
+#define PFLOW_IPFIX_TMPL_IPV4_FIELD_COUNT 12
+#define PFLOW_IPFIX_TMPL_IPV4_ID 256
+} __packed;
+
+/* update pflow_clone_create() when changing pflow_ipfix_tmpl_v6 */
+struct pflow_ipfix_tmpl_ipv6 {
+	struct pflow_tmpl_hdr	h;
+	struct pflow_tmpl_fspec	src_ip;
+	struct pflow_tmpl_fspec	dest_ip;
+	struct pflow_tmpl_fspec	if_index_in;
+	struct pflow_tmpl_fspec	if_index_out;
+	struct pflow_tmpl_fspec	packets;
+	struct pflow_tmpl_fspec	octets;
+	struct pflow_tmpl_fspec	start;
+	struct pflow_tmpl_fspec	finish;
+	struct pflow_tmpl_fspec	src_port;
+	struct pflow_tmpl_fspec	dest_port;
+	struct pflow_tmpl_fspec	tos;
+	struct pflow_tmpl_fspec	protocol;
+#define PFLOW_IPFIX_TMPL_IPV6_FIELD_COUNT 12
+#define PFLOW_IPFIX_TMPL_IPV6_ID 257
+} __packed;
+
+struct pflow_ipfix_tmpl {
+	struct pflow_set_header	set_header;
+	struct pflow_ipfix_tmpl_ipv4	ipv4_tmpl;
+	struct pflow_ipfix_tmpl_ipv6	ipv6_tmpl;
+} __packed;
+
+struct pflow_ipfix_flow4 {
+	u_int32_t	src_ip;		/* sourceIPv4Address*/
+	u_int32_t	dest_ip;	/* destinationIPv4Address */
+	u_int32_t	if_index_in;	/* ingressInterface */
+	u_int32_t	if_index_out;	/* egressInterface */
+	u_int64_t	flow_packets;	/* packetDeltaCount */
+	u_int64_t	flow_octets;	/* octetDeltaCount */
+	int64_t		flow_start;	/* flowStartMilliseconds */
+	int64_t		flow_finish;	/* flowEndMilliseconds */
+	u_int16_t	src_port;	/* sourceTransportPort */
+	u_int16_t	dest_port;	/* destinationTransportPort */
+	u_int8_t	tos;		/* ipClassOfService */
+	u_int8_t	protocol;	/* protocolIdentifier */
+	/* XXX padding needed? */
+} __packed;
+
+struct pflow_ipfix_flow6 {
+	struct in6_addr src_ip;		/* sourceIPv6Address */
+	struct in6_addr dest_ip;	/* destinationIPv6Address */
+	u_int32_t	if_index_in;	/* ingressInterface */
+	u_int32_t	if_index_out;	/* egressInterface */
+	u_int64_t	flow_packets;	/* packetDeltaCount */
+	u_int64_t	flow_octets;	/* octetDeltaCount */
+	int64_t		flow_start;	/* flowStartMilliseconds */
+	int64_t		flow_finish;	/* flowEndMilliseconds */
+	u_int16_t	src_port;	/* sourceTransportPort */
+	u_int16_t	dest_port;	/* destinationTransportPort */
+	u_int8_t	tos;		/* ipClassOfService */
+	u_int8_t	protocol;	/* protocolIdentifier */
+	/* XXX padding needed? */
+} __packed;
+
+#ifdef _KERNEL
+
+struct pflow_softc {
+	int			 sc_id;
+
+	struct mtx		 sc_lock;
+
+	int			 sc_dying;	/* [N] */
+	struct vnet		*sc_vnet;
+
+	unsigned int		 sc_count;
+	unsigned int		 sc_count4;
+	unsigned int		 sc_count6;
+	unsigned int		 sc_maxcount;
+	unsigned int		 sc_maxcount4;
+	unsigned int		 sc_maxcount6;
+	u_int64_t		 sc_gcounter;
+	u_int32_t		 sc_sequence;
+	struct callout		 sc_tmo;
+	struct callout		 sc_tmo6;
+	struct callout		 sc_tmo_tmpl;
+	struct intr_event	*sc_swi_ie;
+	void			*sc_swi_cookie;
+	struct mbufq		 sc_outputqueue;
+	struct task		 sc_outputtask;
+	struct socket		*so;		/* [p] */
+	struct sockaddr		*sc_flowsrc;
+	struct sockaddr		*sc_flowdst;
+	struct pflow_ipfix_tmpl	 sc_tmpl_ipfix;
+	u_int8_t		 sc_version;
+	struct mbuf		*sc_mbuf;	/* current cumulative mbuf */
+	struct mbuf		*sc_mbuf6;	/* current cumulative mbuf */
+	CK_LIST_ENTRY(pflow_softc) sc_next;
+	struct epoch_context	 sc_epoch_ctx;
+};
+
+#endif /* _KERNEL */
+
+struct pflow_header {
+	u_int16_t	version;
+	u_int16_t	count;
+	u_int32_t	uptime_ms;
+	u_int32_t	time_sec;
+	u_int32_t	time_nanosec;
+	u_int32_t	flow_sequence;
+	u_int8_t	engine_type;
+	u_int8_t	engine_id;
+	u_int8_t	reserved1;
+	u_int8_t	reserved2;
+} __packed;
+
+#define PFLOW_HDRLEN sizeof(struct pflow_header)
+
+struct pflow_v10_header {
+	u_int16_t	version;
+	u_int16_t	length;
+	u_int32_t	time_sec;
+	u_int32_t	flow_sequence;
+	u_int32_t	observation_dom;
+} __packed;
+
+#define PFLOW_IPFIX_HDRLEN sizeof(struct pflow_v10_header)
+
+struct pflowstats {
+	u_int64_t	pflow_flows;
+	u_int64_t	pflow_packets;
+	u_int64_t	pflow_onomem;
+	u_int64_t	pflow_oerrors;
+};
+
+/* Supported flow protocols */
+#define PFLOW_PROTO_5	5	/* original pflow */
+#define PFLOW_PROTO_10	10	/* ipfix */
+#define PFLOW_PROTO_MAX	11
+
+#define PFLOW_PROTO_DEFAULT PFLOW_PROTO_5
+
+struct pflow_protos {
+	const char	*ppr_name;
+	u_int8_t	 ppr_proto;
+};
+
+#define PFLOW_PROTOS {                                 \
+		{ "5",	PFLOW_PROTO_5 },	       \
+		{ "10",	PFLOW_PROTO_10 },	       \
+}
+
+#define PFLOWNL_FAMILY_NAME	"pflow"
+
+enum {
+	PFLOWNL_CMD_UNSPEC = 0,
+	PFLOWNL_CMD_LIST = 1,
+	PFLOWNL_CMD_CREATE = 2,
+	PFLOWNL_CMD_DEL = 3,
+	PFLOWNL_CMD_SET = 4,
+	PFLOWNL_CMD_GET = 5,
+	__PFLOWNL_CMD_MAX,
+};
+#define PFLOWNL_CMD_MAX (__PFLOWNL_CMD_MAX - 1)
+
+enum pflow_list_type_t {
+	PFLOWNL_L_UNSPEC,
+	PFLOWNL_L_ID		= 1, /* u32 */
+};
+
+enum pflow_create_type_t {
+	PFLOWNL_CREATE_UNSPEC,
+	PFLOWNL_CREATE_ID	= 1, /* u32 */
+};
+
+enum pflow_del_type_t {
+	PFLOWNL_DEL_UNSPEC,
+	PFLOWNL_DEL_ID		= 1, /* u32 */
+};
+
+enum pflow_addr_type_t {
+	PFLOWNL_ADDR_UNSPEC,
+	PFLOWNL_ADDR_FAMILY	= 1, /* u8 */
+	PFLOWNL_ADDR_PORT	= 2, /* u16 */
+	PFLOWNL_ADDR_IP		= 3, /* struct in_addr */
+	PFLOWNL_ADDR_IP6	= 4, /* struct in6_addr */
+};
+
+enum pflow_get_type_t {
+	PFLOWNL_GET_UNSPEC,
+	PFLOWNL_GET_ID		= 1, /* u32 */
+	PFLOWNL_GET_VERSION	= 2, /* u16 */
+	PFLOWNL_GET_SRC		= 3, /* struct sockaddr_storage */
+	PFLOWNL_GET_DST		= 4, /* struct sockaddr_storage */
+};
+
+enum pflow_set_type_t {
+	PFLOWNL_SET_UNSPEC,
+	PFLOWNL_SET_ID		= 1, /* u32 */
+	PFLOWNL_SET_VERSION	= 2, /* u16 */
+	PFLOWNL_SET_SRC		= 3, /* struct sockaddr_storage */
+	PFLOWNL_SET_DST		= 4, /* struct sockaddr_storage */
+};
+
+#ifdef _KERNEL
+int export_pflow(struct pf_kstate *);
+int pflow_sysctl(int *, u_int,  void *, size_t *, void *, size_t);
+#endif /* _KERNEL */
+
+#endif /* _NET_IF_PFLOW_H_ */

sys/net/pfvar.h

@@ -1067,12 +1067,14 @@ struct pf_kstate {
 	struct pf_rule_actions	 act;
 	u_int16_t		 tag;
 	u_int8_t		 rt;
+	u_int16_t		 if_index_in;
+	u_int16_t		 if_index_out;
 };
 
 /*
  * Size <= fits 11 objects per page on LP64. Try to not grow the struct beyond that.
  */
-_Static_assert(sizeof(struct pf_kstate) <= 368, "pf_kstate size crosses 368 bytes");
+_Static_assert(sizeof(struct pf_kstate) <= 372, "pf_kstate size crosses 372 bytes");
 #endif
 
 /*

sys/netlink/netlink_message_parser.h

@@ -150,12 +150,16 @@ static const struct nlhdr_parser _name = {		\
 	.np_size = NL_ARRAY_LEN(_np),			\
 }
 
-#define	NL_DECLARE_ATTR_PARSER(_name, _np)		\
+#define	NL_DECLARE_ATTR_PARSER_EXT(_name, _np, _pp)	\
 static const struct nlhdr_parser _name = {		\
 	.np = &((_np)[0]),				\
 	.np_size = NL_ARRAY_LEN(_np),			\
+	.post_parse = (_pp)				\
 }
 
+#define	NL_DECLARE_ATTR_PARSER(_name, _np)		\
+	NL_DECLARE_ATTR_PARSER_EXT(_name, _np, NULL)
+
 #define	NL_ATTR_BMASK_SIZE	128
 BITSET_DEFINE(nlattr_bmask, NL_ATTR_BMASK_SIZE);
 

sys/netpfil/pf/pf.c

@@ -1226,6 +1226,7 @@ pf_state_key_attach(struct pf_state_key *skw, struct pf_state_key *sks,
 	struct pf_kstate	*si, *olds = NULL;
 	int idx;
 
+	NET_EPOCH_ASSERT();
 	KASSERT(s->refs == 0, ("%s: state not pristine", __func__));
 	KASSERT(s->key[PF_SK_WIRE] == NULL, ("%s: state has key", __func__));
 	KASSERT(s->key[PF_SK_STACK] == NULL, ("%s: state has key", __func__));
@@ -1392,6 +1393,8 @@ pf_detach_state(struct pf_kstate *s)
 	struct pf_state_key *sks = s->key[PF_SK_STACK];
 	struct pf_keyhash *kh;
 
+	NET_EPOCH_ASSERT();
+
 	pf_sctp_multihome_detach_addr(s);
 
 	if (sks != NULL) {
@@ -1491,6 +1494,8 @@ pf_state_insert(struct pfi_kkif *kif, struct pfi_kkif *orig_kif,
 	struct pf_kstate *cur;
 	int error;
 
+	NET_EPOCH_ASSERT();
+
 	KASSERT(TAILQ_EMPTY(&sks->states[0]) && TAILQ_EMPTY(&sks->states[1]),
 	    ("%s: sks not pristine", __func__));
 	KASSERT(TAILQ_EMPTY(&skw->states[0]) && TAILQ_EMPTY(&skw->states[1]),
@@ -1915,6 +1920,8 @@ pf_counter_u64_periodic_main(void)
 void
 pf_purge_thread(void *unused __unused)
 {
+	struct epoch_tracker	 et;
+
 	VNET_ITERATOR_DECL(vnet_iter);
 
 	sx_xlock(&pf_end_lock);
@@ -1922,6 +1929,7 @@ pf_purge_thread(void *unused __unused)
 		sx_sleep(pf_purge_thread, &pf_end_lock, 0, "pftm", pf_purge_thread_period);
 
 		VNET_LIST_RLOCK();
+		NET_EPOCH_ENTER(et);
 		VNET_FOREACH(vnet_iter) {
 			CURVNET_SET(vnet_iter);
 
@@ -1958,6 +1966,7 @@ pf_purge_thread(void *unused __unused)
 			}
 			CURVNET_RESTORE();
 		}
+		NET_EPOCH_EXIT(et);
 		VNET_LIST_RUNLOCK();
 	}
 
@@ -2097,6 +2106,7 @@ pf_unlink_state(struct pf_kstate *s)
 {
 	struct pf_idhash *ih = &V_pf_idhash[PF_IDHASH(s)];
 
+	NET_EPOCH_ASSERT();
 	PF_HASHROW_ASSERT(ih);
 
 	if (s->timeout == PFTM_UNLINKED) {
@@ -8434,6 +8444,13 @@ done:
 
 	SDT_PROBE4(pf, ip, test, done, action, reason, r, s);
 
+	if (s && action != PF_DROP) {
+		if (!s->if_index_in && dir == PF_IN)
+			s->if_index_in = ifp->if_index;
+		else if (!s->if_index_out && dir == PF_OUT)
+			s->if_index_out = ifp->if_index;
+	}
+
 	if (s)
 		PF_STATE_UNLOCK(s);
 
@@ -8986,6 +9003,13 @@ done:
 		break;
 	}
 
+	if (s && action != PF_DROP) {
+		if (!s->if_index_in && dir == PF_IN)
+			s->if_index_in = ifp->if_index;
+		else if (!s->if_index_out && dir == PF_OUT)
+			s->if_index_out = ifp->if_index;
+	}
+
 	if (s)
 		PF_STATE_UNLOCK(s);
 

sys/netpfil/pf/pf_ioctl.c

@@ -5784,9 +5784,11 @@ done:
 static void
 pf_clear_all_states(void)
 {
+	struct epoch_tracker	 et;
 	struct pf_kstate	*s;
 	u_int i;
 
+	NET_EPOCH_ENTER(et);
 	for (i = 0; i <= pf_hashmask; i++) {
 		struct pf_idhash *ih = &V_pf_idhash[i];
 relock:
@@ -5800,6 +5802,7 @@ relock:
 		}
 		PF_HASHROW_UNLOCK(ih);
 	}
+	NET_EPOCH_EXIT(et);
 }
 
 static int
@@ -5943,6 +5946,8 @@ pf_clear_states(const struct pf_kstate_kill *kill)
 	int		 idx;
 	unsigned int	 killed = 0, dir;
 
+	NET_EPOCH_ASSERT();
+
 	for (unsigned int i = 0; i <= pf_hashmask; i++) {
 		struct pf_idhash *ih = &V_pf_idhash[i];
 
@@ -6006,6 +6011,7 @@ pf_killstates(struct pf_kstate_kill *kill, unsigned int *killed)
 {
 	struct pf_kstate	*s;
 
+	NET_EPOCH_ASSERT();
 	if (kill->psk_pfcmp.id) {
 		if (kill->psk_pfcmp.creatorid == 0)
 			kill->psk_pfcmp.creatorid = V_pf_status.hostid;
@@ -6019,14 +6025,13 @@ pf_killstates(struct pf_kstate_kill *kill, unsigned int *killed)
 
 	for (unsigned int i = 0; i <= pf_hashmask; i++)
 		*killed += pf_killstates_row(kill, &V_pf_idhash[i]);
-
-	return;
 }
 
 static int
 pf_killstates_nv(struct pfioc_nv *nv)
 {
 	struct pf_kstate_kill	 kill;
+	struct epoch_tracker	 et;
 	nvlist_t		*nvl = NULL;
 	void			*nvlpacked = NULL;
 	int			 error = 0;
@@ -6053,7 +6058,9 @@ pf_killstates_nv(struct pfioc_nv *nv)
 	if (error)
 		ERROUT(error);
 
+	NET_EPOCH_ENTER(et);
 	pf_killstates(&kill, &killed);
+	NET_EPOCH_EXIT(et);
 
 	free(nvlpacked, M_NVLIST);
 	nvlpacked = NULL;
@@ -6085,6 +6092,7 @@ static int
 pf_clearstates_nv(struct pfioc_nv *nv)
 {
 	struct pf_kstate_kill	 kill;
+	struct epoch_tracker	 et;
 	nvlist_t		*nvl = NULL;
 	void			*nvlpacked = NULL;
 	int			 error = 0;
@@ -6111,7 +6119,9 @@ pf_clearstates_nv(struct pfioc_nv *nv)
 	if (error)
 		ERROUT(error);
 
+	NET_EPOCH_ENTER(et);
 	killed = pf_clear_states(&kill);
+	NET_EPOCH_EXIT(et);
 
 	free(nvlpacked, M_NVLIST);
 	nvlpacked = NULL;