hardenedbsd/HardenedBSD
1
0
Fork 0
mirror of https://git.hardenedbsd.org/hardenedbsd/HardenedBSD.git synced 2024-11-22 03:04:34 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

nvmecontrol: New commands to support Fabrics hosts

Browse Source 
- discover: Connects to a remote Discovery controller, fetches its
  Discovery Log Page, and enumerates the remote controllers described
  in the log page.

  The -v option can be used to display the Identify Controller data
  structure for the Discovery controller.  This is only really useful
  for debugging.

- connect: Connects to a remote I/O controller and establishes an
  association of an admin queue and a single I/O queue.  The
  association is handed off to the in-kernel host to create a new
  nvmeX device.

- connect-all: Connects to a Discovery controller and attempts to
  create an association with each I/O controller enumerated in the
  Discovery controller's Discovery Log Page.

- reconnect: Establishes a new association with a remote I/O
  controller for an existing nvmeX device.  This can be used to
  restore access to a remote I/O controller after the loss of a prior
  association due to a transport error, controller reboot, etc.

- disconnect: Deletes one or more nvmeX devices after detaching its
  namespaces and terminating any active associations.  The devices to
  delete can be identified by either a nvmeX device name or the NQN of
  the remote controller.

- disconnect-all: Deletes all active associations with remote
  controllers.

Reviewed by:	imp
Sponsored by:	Chelsio Communications
Differential Revision:	https://reviews.freebsd.org/D44715
This commit is contained in:
John Baldwin 2024-05-02 16:30:10 -07:00
parent a1eda74167
commit 1058c12197
8 changed files with 1563 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
sbin/nvmecontrol/Makefile
View File

@ -3,7 +3,11 @@
PACKAGE=nvme-tools PACKAGE=nvme-tools
PROG= nvmecontrol PROG= nvmecontrol
SRCS+= comnd.c SRCS+= comnd.c
SRCS+= connect.c
SRCS+= devlist.c SRCS+= devlist.c
SRCS+= disconnect.c
SRCS+= discover.c
SRCS+= fabrics.c
SRCS+= firmware.c SRCS+= firmware.c
SRCS+= format.c SRCS+= format.c
SRCS+= identify.c SRCS+= identify.c
@ -17,13 +21,15 @@ SRCS+= nvmecontrol.c
SRCS+= passthru.c SRCS+= passthru.c
SRCS+= perftest.c SRCS+= perftest.c
SRCS+= power.c SRCS+= power.c
SRCS+= reconnect.c
SRCS+= reset.c SRCS+= reset.c
SRCS+= resv.c SRCS+= resv.c
SRCS+= sanitize.c SRCS+= sanitize.c
SRCS+= selftest.c SRCS+= selftest.c
CFLAGS+= -I${SRCTOP}/lib/libnvmf
MAN= nvmecontrol.8 MAN= nvmecontrol.8
LDFLAGS+= -rdynamic LDFLAGS+= -rdynamic
LIBADD+= util LIBADD+= nvmf util
SUBDIR= modules SUBDIR= modules
HAS_TESTS= HAS_TESTS=
SUBDIR.${MK_TESTS}+= tests SUBDIR.${MK_TESTS}+= tests

283
sbin/nvmecontrol/connect.c Normal file
View File

@ -0,0 +1,283 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2023-2024 Chelsio Communications, Inc.
* Written by: John Baldwin <jhb@FreeBSD.org>
*/
#include <sys/socket.h>
#include <err.h>
#include <libnvmf.h>
#include <stdlib.h>
#include <string.h>
#include <sysexits.h>
#include <unistd.h>
#include "comnd.h"
#include "fabrics.h"
/*
* Settings that are currently hardcoded but could be exposed to the
* user via additional command line options:
*
* - ADMIN queue entries
* - MaxR2T
*/
static struct options {
const char *transport;
const char *address;
const char *cntlid;
const char *subnqn;
const char *hostnqn;
uint32_t kato;
uint16_t num_io_queues;
uint16_t queue_size;
bool data_digests;
bool flow_control;
bool header_digests;
} opt = {
.transport = "tcp",
.address = NULL,
.cntlid = "dynamic",
.subnqn = NULL,
.hostnqn = NULL,
.kato = NVMF_KATO_DEFAULT / 1000,
.num_io_queues = 1,
.queue_size = 0,
.data_digests = false,
.flow_control = false,
.header_digests = false,
};
static void
tcp_association_params(struct nvmf_association_params *params)
{
params->tcp.pda = 0;
params->tcp.header_digests = opt.header_digests;
params->tcp.data_digests = opt.data_digests;
/* XXX */
params->tcp.maxr2t = 1;
}
static int
connect_nvm_controller(enum nvmf_trtype trtype, int adrfam, const char *address,
const char *port, uint16_t cntlid, const char *subnqn)
{
struct nvme_controller_data cdata;
struct nvmf_association_params aparams;
struct nvmf_qpair *admin, **io;
int error;
memset(&aparams, 0, sizeof(aparams));
aparams.sq_flow_control = opt.flow_control;
switch (trtype) {
case NVMF_TRTYPE_TCP:
tcp_association_params(&aparams);
break;
default:
warnx("Unsupported transport %s", nvmf_transport_type(trtype));
return (EX_UNAVAILABLE);
}
io = calloc(opt.num_io_queues, sizeof(*io));
error = connect_nvm_queues(&aparams, trtype, adrfam, address, port,
cntlid, subnqn, opt.hostnqn, opt.kato, &admin, io,
opt.num_io_queues, opt.queue_size, &cdata);
if (error != 0)
return (error);
error = nvmf_handoff_host(admin, opt.num_io_queues, io, &cdata);
if (error != 0) {
warnc(error, "Failed to handoff queues to kernel");
return (EX_IOERR);
}
free(io);
return (0);
}
static void
connect_discovery_entry(struct nvme_discovery_log_entry *entry)
{
int adrfam;
switch (entry->trtype) {
case NVMF_TRTYPE_TCP:
switch (entry->adrfam) {
case NVMF_ADRFAM_IPV4:
adrfam = AF_INET;
break;
case NVMF_ADRFAM_IPV6:
adrfam = AF_INET6;
break;
default:
warnx("Skipping unsupported address family for %s",
entry->subnqn);
return;
}
switch (entry->tsas.tcp.sectype) {
case NVME_TCP_SECURITY_NONE:
break;
default:
warnx("Skipping unsupported TCP security type for %s",
entry->subnqn);
return;
}
break;
default:
warnx("Skipping unsupported transport %s for %s",
nvmf_transport_type(entry->trtype), entry->subnqn);
return;
}
/*
* XXX: Track portids and avoid duplicate connections for a
* given (subnqn,portid)?
*/
/* XXX: Should this make use of entry->aqsz in some way? */
connect_nvm_controller(entry->trtype, adrfam, entry->traddr,
entry->trsvcid, entry->cntlid, entry->subnqn);
}
static void
connect_discovery_log_page(struct nvmf_qpair *qp)
{
struct nvme_discovery_log *log;
int error;
error = nvmf_host_fetch_discovery_log_page(qp, &log);
if (error != 0)
errc(EX_IOERR, error, "Failed to fetch discovery log page");
for (u_int i = 0; i < log->numrec; i++)
connect_discovery_entry(&log->entries[i]);
free(log);
}
static void
discover_controllers(enum nvmf_trtype trtype, const char *address,
const char *port)
{
struct nvmf_qpair *qp;
qp = connect_discovery_adminq(trtype, address, port, opt.hostnqn);
connect_discovery_log_page(qp);
nvmf_free_qpair(qp);
}
static void
connect_fn(const struct cmd *f, int argc, char *argv[])
{
enum nvmf_trtype trtype;
const char *address, *port;
char *tofree;
u_long cntlid;
int error;
if (arg_parse(argc, argv, f))
return;
if (opt.num_io_queues <= 0)
errx(EX_USAGE, "Invalid number of I/O queues");
if (strcasecmp(opt.transport, "tcp") == 0) {
trtype = NVMF_TRTYPE_TCP;
} else
errx(EX_USAGE, "Unsupported or invalid transport");
nvmf_parse_address(opt.address, &address, &port, &tofree);
if (port == NULL)
errx(EX_USAGE, "Explicit port required");
cntlid = nvmf_parse_cntlid(opt.cntlid);
error = connect_nvm_controller(trtype, AF_UNSPEC, address, port, cntlid,
opt.subnqn);
if (error != 0)
exit(error);
free(tofree);
}
static void
connect_all_fn(const struct cmd *f, int argc, char *argv[])
{
enum nvmf_trtype trtype;
const char *address, *port;
char *tofree;
if (arg_parse(argc, argv, f))
return;
if (opt.num_io_queues <= 0)
errx(EX_USAGE, "Invalid number of I/O queues");
if (strcasecmp(opt.transport, "tcp") == 0) {
trtype = NVMF_TRTYPE_TCP;
} else
errx(EX_USAGE, "Unsupported or invalid transport");
nvmf_parse_address(opt.address, &address, &port, &tofree);
discover_controllers(trtype, address, port);
free(tofree);
}
static const struct opts connect_opts[] = {
#define OPT(l, s, t, opt, addr, desc) { l, s, t, &opt.addr, desc }
OPT("transport", 't', arg_string, opt, transport,
"Transport type"),
OPT("cntlid", 'c', arg_string, opt, cntlid,
"Controller ID"),
OPT("nr-io-queues", 'i', arg_uint16, opt, num_io_queues,
"Number of I/O queues"),
OPT("queue-size", 'Q', arg_uint16, opt, queue_size,
"Number of entries in each I/O queue"),
OPT("keep-alive-tmo", 'k', arg_uint32, opt, kato,
"Keep Alive timeout (in seconds)"),
OPT("hostnqn", 'q', arg_string, opt, hostnqn,
"Host NQN"),
OPT("flow_control", 'F', arg_none, opt, flow_control,
"Request SQ flow control"),
OPT("hdr_digests", 'g', arg_none, opt, header_digests,
"Enable TCP PDU header digests"),
OPT("data_digests", 'G', arg_none, opt, data_digests,
"Enable TCP PDU data digests"),
{ NULL, 0, arg_none, NULL, NULL }
};
#undef OPT
static const struct args connect_args[] = {
{ arg_string, &opt.address, "address" },
{ arg_string, &opt.subnqn, "SubNQN" },
{ arg_none, NULL, NULL },
};
static const struct args connect_all_args[] = {
{ arg_string, &opt.address, "address" },
{ arg_none, NULL, NULL },
};
static struct cmd connect_cmd = {
.name = "connect",
.fn = connect_fn,
.descr = "Connect to a fabrics controller",
.ctx_size = sizeof(opt),
.opts = connect_opts,
.args = connect_args,
};
static struct cmd connect_all_cmd = {
.name = "connect-all",
.fn = connect_all_fn,
.descr = "Discover and connect to fabrics controllers",
.ctx_size = sizeof(opt),
.opts = connect_opts,
.args = connect_all_args,
};
CMD_COMMAND(connect_cmd);
CMD_COMMAND(connect_all_cmd);

82
sbin/nvmecontrol/disconnect.c Normal file
View File

@ -0,0 +1,82 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2023-2024 Chelsio Communications, Inc.
* Written by: John Baldwin <jhb@FreeBSD.org>
*/
#include <err.h>
#include <libnvmf.h>
#include <stdlib.h>
#include <sysexits.h>
#include <unistd.h>
#include "nvmecontrol.h"
static struct options {
const char *dev;
} opt = {
.dev = NULL
};
static const struct args args[] = {
{ arg_string, &opt.dev, "controller-id|namespace-id|SubNQN" },
{ arg_none, NULL, NULL },
};
static void
disconnect(const struct cmd *f, int argc, char *argv[])
{
int error, fd;
char *path;
if (arg_parse(argc, argv, f))
return;
if (nvmf_nqn_valid(opt.dev)) {
error = nvmf_disconnect_host(opt.dev);
if (error != 0)
errc(EX_IOERR, error, "failed to disconnect from %s",
opt.dev);
} else {
open_dev(opt.dev, &fd, 1, 1);
get_nsid(fd, &path, NULL);
close(fd);
error = nvmf_disconnect_host(path);
if (error != 0)
errc(EX_IOERR, error, "failed to disconnect from %s",
path);
}
exit(0);
}
static void
disconnect_all(const struct cmd *f __unused, int argc __unused,
char *argv[] __unused)
{
int error;
error = nvmf_disconnect_all();
if (error != 0)
errc(EX_IOERR, error,
"failed to disconnect from remote controllers");
exit(0);
}
static struct cmd disconnect_cmd = {
.name = "disconnect",
.fn = disconnect,
.descr = "Disconnect from a fabrics controller",
.args = args,
};
static struct cmd disconnect_all_cmd = {
.name = "disconnect-all",
.fn = disconnect_all,
.descr = "Disconnect from all fabrics controllers",
};
CMD_COMMAND(disconnect_cmd);
CMD_COMMAND(disconnect_all_cmd);

300
sbin/nvmecontrol/discover.c Normal file
View File

@ -0,0 +1,300 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2023-2024 Chelsio Communications, Inc.
* Written by: John Baldwin <jhb@FreeBSD.org>
*/
#include <err.h>
#include <libnvmf.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sysexits.h>
#include "comnd.h"
#include "fabrics.h"
#include "nvmecontrol_ext.h"
static struct options {
const char *transport;
const char *address;
const char *hostnqn;
bool verbose;
} opt = {
.transport = "tcp",
.address = NULL,
.hostnqn = NULL,
.verbose = false,
};
static void
identify_controller(struct nvmf_qpair *qp)
{
struct nvme_controller_data cdata;
int error;
error = nvmf_host_identify_controller(qp, &cdata);
if (error != 0)
errc(EX_IOERR, error, "Failed to fetch controller data");
nvme_print_controller(&cdata);
}
static const char *
nvmf_address_family(uint8_t adrfam)
{
static char buf[8];
switch (adrfam) {
case NVMF_ADRFAM_IPV4:
return ("AF_INET");
case NVMF_ADRFAM_IPV6:
return ("AF_INET6");
case NVMF_ADRFAM_IB:
return ("InfiniBand");
case NVMF_ADRFAM_FC:
return ("Fibre Channel");
case NVMF_ADRFAM_INTRA_HOST:
return ("Intra-host");
default:
snprintf(buf, sizeof(buf), "0x%02x\n", adrfam);
return (buf);
}
}
static const char *
nvmf_subsystem_type(uint8_t subtype)
{
static char buf[8];
switch (subtype) {
case NVMF_SUBTYPE_DISCOVERY:
return ("Discovery");
case NVMF_SUBTYPE_NVME:
return ("NVMe");
default:
snprintf(buf, sizeof(buf), "0x%02x\n", subtype);
return (buf);
}
}
static const char *
nvmf_secure_channel(uint8_t treq)
{
switch (treq & 0x03) {
case NVMF_TREQ_SECURE_CHANNEL_NOT_SPECIFIED:
return ("Not specified");
case NVMF_TREQ_SECURE_CHANNEL_REQUIRED:
return ("Required");
case NVMF_TREQ_SECURE_CHANNEL_NOT_REQUIRED:
return ("Not required");
default:
return ("0x03");
}
}
static const char *
nvmf_controller_id(uint16_t cntlid)
{
static char buf[8];
switch (cntlid) {
case NVMF_CNTLID_DYNAMIC:
return ("Dynamic");
case NVMF_CNTLID_STATIC_ANY:
return ("Static");
default:
snprintf(buf, sizeof(buf), "%u", cntlid);
return (buf);
}
}
static const char *
nvmf_rdma_service_type(uint8_t qptype)
{
static char buf[8];
switch (qptype) {
case NVMF_RDMA_QPTYPE_RELIABLE_CONNECTED:
return ("Reliable connected");
case NVMF_RDMA_QPTYPE_RELIABLE_DATAGRAM:
return ("Reliable datagram");
default:
snprintf(buf, sizeof(buf), "0x%02x\n", qptype);
return (buf);
}
}
static const char *
nvmf_rdma_provider_type(uint8_t prtype)
{
static char buf[8];
switch (prtype) {
case NVMF_RDMA_PRTYPE_NONE:
return ("None");
case NVMF_RDMA_PRTYPE_IB:
return ("InfiniBand");
case NVMF_RDMA_PRTYPE_ROCE:
return ("RoCE (v1)");
case NVMF_RDMA_PRTYPE_ROCE2:
return ("RoCE (v2)");
case NVMF_RDMA_PRTYPE_IWARP:
return ("iWARP");
default:
snprintf(buf, sizeof(buf), "0x%02x\n", prtype);
return (buf);
}
}
static const char *
nvmf_rdma_cms(uint8_t cms)
{
static char buf[8];
switch (cms) {
case NVMF_RDMA_CMS_RDMA_CM:
return ("RDMA_IP_CM");
default:
snprintf(buf, sizeof(buf), "0x%02x\n", cms);
return (buf);
}
}
static const char *
nvmf_tcp_security_type(uint8_t sectype)
{
static char buf[8];
switch (sectype) {
case NVME_TCP_SECURITY_NONE:
return ("None");
case NVME_TCP_SECURITY_TLS_1_2:
return ("TLS 1.2");
case NVME_TCP_SECURITY_TLS_1_3:
return ("TLS 1.3");
default:
snprintf(buf, sizeof(buf), "0x%02x\n", sectype);
return (buf);
}
}
static void
print_discovery_entry(u_int i, struct nvme_discovery_log_entry *entry)
{
printf("Entry %02d\n", i + 1);
printf("========\n");
printf(" Transport type: %s\n",
nvmf_transport_type(entry->trtype));
printf(" Address family: %s\n",
nvmf_address_family(entry->adrfam));
printf(" Subsystem type: %s\n",
nvmf_subsystem_type(entry->subtype));
printf(" SQ flow control: %s\n",
(entry->treq & (1 << 2)) == 0 ? "required" : "optional");
printf(" Secure Channel: %s\n", nvmf_secure_channel(entry->treq));
printf(" Port ID: %u\n", entry->portid);
printf(" Controller ID: %s\n",
nvmf_controller_id(entry->cntlid));
printf(" Max Admin SQ Size: %u\n", entry->aqsz);
printf(" Sub NQN: %s\n", entry->subnqn);
printf(" Transport address: %s\n", entry->traddr);
printf(" Service identifier: %s\n", entry->trsvcid);
switch (entry->trtype) {
case NVMF_TRTYPE_RDMA:
printf(" RDMA Service Type: %s\n",
nvmf_rdma_service_type(entry->tsas.rdma.rdma_qptype));
printf(" RDMA Provider Type: %s\n",
nvmf_rdma_provider_type(entry->tsas.rdma.rdma_prtype));
printf(" RDMA CMS: %s\n",
nvmf_rdma_cms(entry->tsas.rdma.rdma_cms));
printf(" Partition key: %u\n",
entry->tsas.rdma.rdma_pkey);
break;
case NVMF_TRTYPE_TCP:
printf(" Security Type: %s\n",
nvmf_tcp_security_type(entry->tsas.tcp.sectype));
break;
}
}
static void
dump_discovery_log_page(struct nvmf_qpair *qp)
{
struct nvme_discovery_log *log;
int error;
error = nvmf_host_fetch_discovery_log_page(qp, &log);
if (error != 0)
errc(EX_IOERR, error, "Failed to fetch discovery log page");
printf("Discovery\n");
printf("=========\n");
if (log->numrec == 0) {
printf("No entries found\n");
} else {
for (u_int i = 0; i < log->numrec; i++)
print_discovery_entry(i, &log->entries[i]);
}
free(log);
}
static void
discover(const struct cmd *f, int argc, char *argv[])
{
enum nvmf_trtype trtype;
struct nvmf_qpair *qp;
const char *address, *port;
char *tofree;
if (arg_parse(argc, argv, f))
return;
if (strcasecmp(opt.transport, "tcp") == 0) {
trtype = NVMF_TRTYPE_TCP;
} else
errx(EX_USAGE, "Unsupported or invalid transport");
nvmf_parse_address(opt.address, &address, &port, &tofree);
qp = connect_discovery_adminq(trtype, address, port, opt.hostnqn);
free(tofree);
/* Use Identify to fetch controller data */
if (opt.verbose) {
identify_controller(qp);
printf("\n");
}
/* Fetch Log pages */
dump_discovery_log_page(qp);
nvmf_free_qpair(qp);
}
static const struct opts discover_opts[] = {
#define OPT(l, s, t, opt, addr, desc) { l, s, t, &opt.addr, desc }
OPT("transport", 't', arg_string, opt, transport,
"Transport type"),
OPT("hostnqn", 'q', arg_string, opt, hostnqn,
"Host NQN"),
OPT("verbose", 'v', arg_none, opt, verbose,
"Display the discovery controller's controller data"),
{ NULL, 0, arg_none, NULL, NULL }
};
#undef OPT
static const struct args discover_args[] = {
{ arg_string, &opt.address, "address" },
{ arg_none, NULL, NULL },
};
static struct cmd discover_cmd = {
.name = "discover",
.fn = discover,
.descr = "List discovery log pages from a fabrics controller",
.ctx_size = sizeof(opt),
.opts = discover_opts,
.args = discover_args,
};
CMD_COMMAND(discover_cmd);

520
sbin/nvmecontrol/fabrics.c Normal file
View File

@ -0,0 +1,520 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2023-2024 Chelsio Communications, Inc.
* Written by: John Baldwin <jhb@FreeBSD.org>
*/
#include <sys/socket.h>
#include <netinet/in.h>
#include <err.h>
#include <libnvmf.h>
#include <netdb.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sysexits.h>
#include <unistd.h>
#include "fabrics.h"
/*
* Subroutines shared by several Fabrics commands.
*/
static char nqn[NVMF_NQN_MAX_LEN];
static uint8_t hostid[16];
static bool hostid_initted = false;
static bool
init_hostid(void)
{
int error;
if (hostid_initted)
return (true);
error = nvmf_hostid_from_hostuuid(hostid);
if (error != 0) {
warnc(error, "Failed to generate hostid");
return (false);
}
error = nvmf_nqn_from_hostuuid(nqn);
if (error != 0) {
warnc(error, "Failed to generate host NQN");
return (false);
}
hostid_initted = true;
return (true);
}
void
nvmf_parse_address(const char *in_address, const char **address,
const char **port, char **tofree)
{
char *cp;
/*
* Accepts the following address formats:
*
* [IPv6 address]:port
* IPv4 address:port
* hostname:port
* [IPv6 address]
* IPv6 address
* IPv4 address
* hostname
*/
if (in_address[0] == '[') {
/* IPv6 address in square brackets. */
cp = strchr(in_address + 1, ']');
if (cp == NULL || cp == in_address + 1)
errx(EX_USAGE, "Invalid address %s", in_address);
*tofree = strndup(in_address + 1, cp - (in_address + 1));
*address = *tofree;
/* Skip over ']' */
cp++;
switch (*cp) {
case '\0':
*port = NULL;
return;
case ':':
if (cp[1] != '\0') {
*port = cp + 1;
return;
}
/* FALLTHROUGH */
default:
errx(EX_USAGE, "Invalid address %s", in_address);
}
}
/* Look for the first colon. */
cp = strchr(in_address, ':');
if (cp == NULL) {
*address = in_address;
*port = NULL;
*tofree = NULL;
return;
}
/* If there is another colon, assume this is an IPv6 address. */
if (strchr(cp + 1, ':') != NULL) {
*address = in_address;
*port = NULL;
*tofree = NULL;
return;
}
/* Both strings on either side of the colon must be non-empty. */
if (cp == in_address || cp[1] == '\0')
errx(EX_USAGE, "Invalid address %s", in_address);
*tofree = strndup(in_address, cp - in_address);
*address = *tofree;
/* Skip over ':' */
*port = cp + 1;
}
uint16_t
nvmf_parse_cntlid(const char *cntlid)
{
u_long value;
if (strcasecmp(cntlid, "dynamic") == 0)
return (NVMF_CNTLID_DYNAMIC);
else if (strcasecmp(cntlid, "static") == 0)
return (NVMF_CNTLID_STATIC_ANY);
else {
value = strtoul(cntlid, NULL, 0);
if (value > NVMF_CNTLID_STATIC_MAX)
errx(EX_USAGE, "Invalid controller ID");
return (value);
}
}
bool
tcp_qpair_params(struct nvmf_qpair_params *params, int adrfam,
const char *address, const char *port)
{
struct addrinfo hints, *ai, *list;
int error, s;
memset(&hints, 0, sizeof(hints));
hints.ai_family = adrfam;
hints.ai_protocol = IPPROTO_TCP;
error = getaddrinfo(address, port, &hints, &list);
if (error != 0) {
warnx("%s", gai_strerror(error));
return (false);
}
for (ai = list; ai != NULL; ai = ai->ai_next) {
s = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
if (s == -1)
continue;
if (connect(s, ai->ai_addr, ai->ai_addrlen) != 0) {
close(s);
continue;
}
params->tcp.fd = s;
freeaddrinfo(list);
return (true);
}
warn("Failed to connect to controller at %s:%s", address, port);
return (false);
}
static void
tcp_discovery_association_params(struct nvmf_association_params *params)
{
params->tcp.pda = 0;
params->tcp.header_digests = false;
params->tcp.data_digests = false;
params->tcp.maxr2t = 1;
}
struct nvmf_qpair *
connect_discovery_adminq(enum nvmf_trtype trtype, const char *address,
const char *port, const char *hostnqn)
{
struct nvmf_association_params aparams;
struct nvmf_qpair_params qparams;
struct nvmf_association *na;
struct nvmf_qpair *qp;
uint64_t cap, cc, csts;
int error, timo;
memset(&aparams, 0, sizeof(aparams));
aparams.sq_flow_control = false;
switch (trtype) {
case NVMF_TRTYPE_TCP:
/* 7.4.9.3 Default port for discovery */
if (port == NULL)
port = "8009";
tcp_discovery_association_params(&aparams);
break;
default:
errx(EX_UNAVAILABLE, "Unsupported transport %s",
nvmf_transport_type(trtype));
}
if (!init_hostid())
exit(EX_IOERR);
if (hostnqn != NULL) {
if (!nvmf_nqn_valid(hostnqn))
errx(EX_USAGE, "Invalid HostNQN %s", hostnqn);
} else
hostnqn = nqn;
na = nvmf_allocate_association(trtype, false, &aparams);
if (na == NULL)
err(EX_IOERR, "Failed to create discovery association");
memset(&qparams, 0, sizeof(qparams));
qparams.admin = true;
if (!tcp_qpair_params(&qparams, AF_UNSPEC, address, port))
exit(EX_NOHOST);
qp = nvmf_connect(na, &qparams, 0, NVME_MIN_ADMIN_ENTRIES, hostid,
NVMF_CNTLID_DYNAMIC, NVMF_DISCOVERY_NQN, hostnqn, 0);
if (qp == NULL)
errx(EX_IOERR, "Failed to connect to discovery controller: %s",
nvmf_association_error(na));
nvmf_free_association(na);
/* Fetch Controller Capabilities Property */
error = nvmf_read_property(qp, NVMF_PROP_CAP, 8, &cap);
if (error != 0)
errc(EX_IOERR, error, "Failed to fetch CAP");
/* Set Controller Configuration Property (CC.EN=1) */
error = nvmf_read_property(qp, NVMF_PROP_CC, 4, &cc);
if (error != 0)
errc(EX_IOERR, error, "Failed to fetch CC");
/* Clear known fields preserving any reserved fields. */
cc &= ~(NVMEM(NVME_CC_REG_SHN) | NVMEM(NVME_CC_REG_AMS) |
NVMEM(NVME_CC_REG_MPS) | NVMEM(NVME_CC_REG_CSS));
/* Leave AMS, MPS, and CSS as 0. */
cc |= NVMEF(NVME_CC_REG_EN, 1);
error = nvmf_write_property(qp, NVMF_PROP_CC, 4, cc);
if (error != 0)
errc(EX_IOERR, error, "Failed to set CC");
/* Wait for CSTS.RDY in Controller Status */
timo = NVME_CAP_LO_TO(cap);
for (;;) {
error = nvmf_read_property(qp, NVMF_PROP_CSTS, 4, &csts);
if (error != 0)
errc(EX_IOERR, error, "Failed to fetch CSTS");
if (NVMEV(NVME_CSTS_REG_RDY, csts) != 0)
break;
if (timo == 0)
errx(EX_IOERR, "Controller failed to become ready");
timo--;
usleep(500 * 1000);
}
return (qp);
}
/*
* XXX: Should this accept the admin queue size as a parameter rather
* than always using NVMF_MIN_ADMIN_MAX_SQ_SIZE?
*/
static int
connect_nvm_adminq(struct nvmf_association *na,
const struct nvmf_qpair_params *params, struct nvmf_qpair **qpp,
uint16_t cntlid, const char *subnqn, const char *hostnqn, uint32_t kato,
uint16_t *mqes)
{
struct nvmf_qpair *qp;
uint64_t cap, cc, csts;
u_int mps, mpsmin, mpsmax;
int error, timo;
qp = nvmf_connect(na, params, 0, NVMF_MIN_ADMIN_MAX_SQ_SIZE, hostid,
cntlid, subnqn, hostnqn, kato);
if (qp == NULL) {
warnx("Failed to connect to NVM controller %s: %s", subnqn,
nvmf_association_error(na));
return (EX_IOERR);
}
/* Fetch Controller Capabilities Property */
error = nvmf_read_property(qp, NVMF_PROP_CAP, 8, &cap);
if (error != 0) {
warnc(error, "Failed to fetch CAP");
nvmf_free_qpair(qp);
return (EX_IOERR);
}
/* Require the NVM command set. */
if (NVME_CAP_HI_CSS_NVM(cap >> 32) == 0) {
warnx("Controller %s does not support the NVM command set",
subnqn);
nvmf_free_qpair(qp);
return (EX_UNAVAILABLE);
}
*mqes = NVME_CAP_LO_MQES(cap);
/* Prefer native host page size if it fits. */
mpsmin = NVMEV(NVME_CAP_HI_REG_MPSMIN, cap >> 32);
mpsmax = NVMEV(NVME_CAP_HI_REG_MPSMAX, cap >> 32);
mps = ffs(getpagesize()) - 1;
if (mps < mpsmin + NVME_MPS_SHIFT)
mps = mpsmin;
else if (mps > mpsmax + NVME_MPS_SHIFT)
mps = mpsmax;
else
mps -= NVME_MPS_SHIFT;
/* Configure controller. */
error = nvmf_read_property(qp, NVMF_PROP_CC, 4, &cc);
if (error != 0) {
warnc(error, "Failed to fetch CC");
nvmf_free_qpair(qp);
return (EX_IOERR);
}
/* Clear known fields preserving any reserved fields. */
cc &= ~(NVMEM(NVME_CC_REG_IOCQES) | NVMEM(NVME_CC_REG_IOSQES) |
NVMEM(NVME_CC_REG_SHN) | NVMEM(NVME_CC_REG_AMS) |
NVMEM(NVME_CC_REG_MPS) | NVMEM(NVME_CC_REG_CSS));
cc |= NVMEF(NVME_CC_REG_IOCQES, 4); /* CQE entry size == 16 */
cc |= NVMEF(NVME_CC_REG_IOSQES, 6); /* SEQ entry size == 64 */
cc |= NVMEF(NVME_CC_REG_AMS, 0); /* AMS 0 (Round-robin) */
cc |= NVMEF(NVME_CC_REG_MPS, mps);
cc |= NVMEF(NVME_CC_REG_CSS, 0); /* NVM command set */
cc |= NVMEF(NVME_CC_REG_EN, 1); /* EN = 1 */
error = nvmf_write_property(qp, NVMF_PROP_CC, 4, cc);
if (error != 0) {
warnc(error, "Failed to set CC");
nvmf_free_qpair(qp);
return (EX_IOERR);
}
/* Wait for CSTS.RDY in Controller Status */
timo = NVME_CAP_LO_TO(cap);
for (;;) {
error = nvmf_read_property(qp, NVMF_PROP_CSTS, 4, &csts);
if (error != 0) {
warnc(error, "Failed to fetch CSTS");
nvmf_free_qpair(qp);
return (EX_IOERR);
}
if (NVMEV(NVME_CSTS_REG_RDY, csts) != 0)
break;
if (timo == 0) {
warnx("Controller failed to become ready");
nvmf_free_qpair(qp);
return (EX_IOERR);
}
timo--;
usleep(500 * 1000);
}
*qpp = qp;
return (0);
}
static void
shutdown_controller(struct nvmf_qpair *qp)
{
uint64_t cc;
int error;
error = nvmf_read_property(qp, NVMF_PROP_CC, 4, &cc);
if (error != 0) {
warnc(error, "Failed to fetch CC");
goto out;
}
cc |= NVMEF(NVME_CC_REG_SHN, NVME_SHN_NORMAL);
error = nvmf_write_property(qp, NVMF_PROP_CC, 4, cc);
if (error != 0) {
warnc(error, "Failed to set CC to trigger shutdown");
goto out;
}
out:
nvmf_free_qpair(qp);
}
/* Returns a value from <sysexits.h> */
int
connect_nvm_queues(const struct nvmf_association_params *aparams,
enum nvmf_trtype trtype, int adrfam, const char *address,
const char *port, uint16_t cntlid, const char *subnqn, const char *hostnqn,
uint32_t kato, struct nvmf_qpair **admin, struct nvmf_qpair **io,
u_int num_io_queues, u_int queue_size, struct nvme_controller_data *cdata)
{
struct nvmf_qpair_params qparams;
struct nvmf_association *na;
u_int queues;
int error;
uint16_t mqes;
switch (trtype) {
case NVMF_TRTYPE_TCP:
break;
default:
warnx("Unsupported transport %s", nvmf_transport_type(trtype));
return (EX_UNAVAILABLE);
}
if (!init_hostid())
return (EX_IOERR);
if (hostnqn != NULL) {
if (!nvmf_nqn_valid(hostnqn)) {
warnx("Invalid HostNQN %s", hostnqn);
return (EX_USAGE);
}
} else
hostnqn = nqn;
/* Association. */
na = nvmf_allocate_association(trtype, false, aparams);
if (na == NULL) {
warn("Failed to create association for %s", subnqn);
return (EX_IOERR);
}
/* Admin queue. */
memset(&qparams, 0, sizeof(qparams));
qparams.admin = true;
if (!tcp_qpair_params(&qparams, adrfam, address, port)) {
nvmf_free_association(na);
return (EX_NOHOST);
}
error = connect_nvm_adminq(na, &qparams, admin, cntlid, subnqn, hostnqn,
kato, &mqes);
if (error != 0) {
nvmf_free_association(na);
return (error);
}
/* Validate I/O queue size. */
if (queue_size == 0)
queue_size = mqes + 1;
else if (queue_size > mqes + 1) {
shutdown_controller(*admin);
nvmf_free_association(na);
warn("I/O queue size exceeds controller maximum (%u)",
mqes + 1);
return (EX_USAGE);
}
/* Fetch controller data. */
error = nvmf_host_identify_controller(*admin, cdata);
if (error != 0) {
shutdown_controller(*admin);
nvmf_free_association(na);
warnc(error, "Failed to fetch controller data for %s", subnqn);
return (EX_IOERR);
}
nvmf_update_assocation(na, cdata);
error = nvmf_host_request_queues(*admin, num_io_queues, &queues);
if (error != 0) {
shutdown_controller(*admin);
nvmf_free_association(na);
warnc(error, "Failed to request I/O queues");
return (EX_IOERR);
}
if (queues < num_io_queues) {
shutdown_controller(*admin);
nvmf_free_association(na);
warnx("Controller enabled fewer I/O queues (%u) than requested (%u)",
queues, num_io_queues);
return (EX_PROTOCOL);
}
/* I/O queues. */
memset(io, 0, sizeof(io) * num_io_queues);
for (u_int i = 0; i < num_io_queues; i++) {
memset(&qparams, 0, sizeof(qparams));
qparams.admin = false;
if (!tcp_qpair_params(&qparams, adrfam, address, port)) {
error = EX_NOHOST;
goto out;
}
io[i] = nvmf_connect(na, &qparams, i + 1, queue_size, hostid,
nvmf_cntlid(*admin), subnqn, hostnqn, 0);
if (io[i] == NULL) {
warnx("Failed to create I/O queue: %s",
nvmf_association_error(na));
error = EX_IOERR;
goto out;
}
}
nvmf_free_association(na);
return (0);
out:
for (u_int i = 0; i < num_io_queues; i++) {
if (io[i] == NULL)
break;
nvmf_free_qpair(io[i]);
}
shutdown_controller(*admin);
nvmf_free_association(na);
return (error);
}

41
sbin/nvmecontrol/fabrics.h Normal file
View File

@ -0,0 +1,41 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2023-2024 Chelsio Communications, Inc.
* Written by: John Baldwin <jhb@FreeBSD.org>
*/
#ifndef __FABRICS_H__
#define __FABRICS_H__
/*
* Splits 'in_address' into separate 'address' and 'port' strings. If
* a separate buffer for the address was allocated, 'tofree' is set to
* the allocated buffer, otherwise 'tofree' is set to NULL.
*/
void nvmf_parse_address(const char *in_address, const char **address,
const char **port, char **tofree);
uint16_t nvmf_parse_cntlid(const char *cntlid);
/* Returns true if able to open a connection. */
bool tcp_qpair_params(struct nvmf_qpair_params *params, int adrfam,
const char *address, const char *port);
/* Connect to a discovery controller and return the Admin qpair. */
struct nvmf_qpair *connect_discovery_adminq(enum nvmf_trtype trtype,
const char *address, const char *port, const char *hostnqn);
/*
* Connect to an NVM controller establishing an Admin qpair and one or
* more I/O qpairs. The controller's controller data is returned in
* *cdata on success. Returns a non-zero value from <sysexits.h> on
* failure.
*/
int connect_nvm_queues(const struct nvmf_association_params *aparams,
enum nvmf_trtype trtype, int adrfam, const char *address,
const char *port, uint16_t cntlid, const char *subnqn, const char *hostnqn,
uint32_t kato, struct nvmf_qpair **admin, struct nvmf_qpair **io,
u_int num_io_queues, u_int queue_size, struct nvme_controller_data *cdata);
#endif /* !__FABRICS_H__ */

165
sbin/nvmecontrol/nvmecontrol.8
View File

@ -205,9 +205,48 @@
.Ic io-passthru .Ic io-passthru
.Op args .Op args
.Aq Ar namespace-id .Aq Ar namespace-id
.Nm
.Ic discover
.Op Fl v
.Op Fl t Ar transport
.Op Fl q Ar HostNQN
.Nm
.Ic connect
.Op Fl FGg
.Op Fl c Ar cntl-id
.Op Fl i Ar queues
.Op Fl k Ar seconds
.Op Fl t Ar transport
.Op Fl q Ar HostNQN
.Op Fl Q Ar entries
.Aq Ar address
.Aq Ar SubNQN
.Nm
.Ic connect-all
.Op Fl FGg
.Op Fl i Ar queues
.Op Fl k Ar seconds
.Op Fl t Ar transport
.Op Fl q Ar HostNQN
.Op Fl Q Ar entries
.Aq Ar address
.Nm
.Ic disconnect
.Aq Ar device-id | Ar namespace-id | Ar SubNQN
.Nm
.Ic reconnect
.Op Fl FGg
.Op Fl i Ar queues
.Op Fl k Ar seconds
.Op Fl t Ar transport
.Op Fl q Ar HostNQN
.Op Fl Q Ar entries
.Aq Ar device-id
.Aq Ar address
.Sh DESCRIPTION .Sh DESCRIPTION
NVM Express (NVMe) is a storage protocol standard, for SSDs and other NVM Express (NVMe) is a storage protocol standard for SSDs and other
high-speed storage devices over PCI Express. high-speed storage devices over PCI Express as well as remote storage
devices accessed via a network fabric.
.Ss devlist .Ss devlist
List all NVMe controllers and namespaces along with their device nodes. List all NVMe controllers and namespaces along with their device nodes.
With the With the
@ -676,6 +715,97 @@ Commands either read data or write it, but not both.
Commands needing metadata are not supported by the Commands needing metadata are not supported by the
.Xr nvme 4 .Xr nvme 4
drive. drive.
.Ss discover
List the remote controllers advertised by a remote Discovery Controller:
.Bl -tag -width 6n
.It Fl t Ar transport
Transport to use.
The default is
.It Fl q Ar HostNQN
NVMe Qualified Name to use for this host.
By default an NQN is auto-generated from the current host's UUID.
.Ar tcp .
.It Fl v
Display the
.Dv IDENTIFY_CONTROLLER
data for the Discovery Controller.
.El
.Ss connect
Establish an association with the I/O controller named
.Ar SubNQN
at
.Ar address .
The address must include a port.
.Pp
An admin queue pair and one or more I/O queue pairs are created and handed
off to the kernel to create a new controller device.
.Bl -tag -width 6n
.It Fl c Ar cntl-id
Remote controller ID to request:
.Bl -tag
.It dynamic
Request a dynamic controller ID for controllers using the dynamic
controller model.
This is the default.
.It static
Request a dynamic controller ID for controllers using the static
controller model.
.It Ar number
Request a specific controller ID for controllers using the static
controller model.
.El
.It Fl F
Request submission queue flow control.
By default submission queue flow control is disabled unless the remote
controller requires it.
.It Fl g
Enable TCP PDU header digests.
.It Fl G
Enable TCP PDU data digests.
.It Fl i Ar queues
Number of I/O queue pairs to create.
The default is 1.
.It Fl k Ar seconds
Keep Alive timer duration in seconds.
The default is 120.
.It Fl t Ar transport
Transport to use.
The default is
.Ar tcp .
.It Fl q Ar HostNQN
NVMe Qualified Name to use for this host.
By default an NQN is auto-generated from the current host's UUID.
.It Fl Q Ar entries
Number of entries in each I/O queue.
By default the maximum queue size reported by the MQES field
of the remote host's CAP property is used.
.El
.Ss connect-all
Query the Discovery Controller at
.Ar address
and establish an association for each advertised I/O controller.
The
.Fl t
flag determines the transport used for the initial association with
the Discovery Controller and defaults to
.Ar tcp .
All other flags are used to control properties of each I/O assocation as
described above for the
.Cm connect
command.
.Ss disconnect
Delete the controller device associated with a remote I/O controller
including any active association and open queues.
.Ss reconnect
Reestablish an association for the remote I/O controller associated with
.Ar device-id
at
.Ar address .
The address must include a port.
The flags have the same meaning for the new association as described above
for the
.Cm connect
command.
.Sh DEVICE NAMES .Sh DEVICE NAMES
Where Where
.Aq Ar namespace-id .Aq Ar namespace-id
@ -705,6 +835,37 @@ A
of of
.Dq 0 .Dq 0
means query the drive itself. means query the drive itself.
.Sh FABRICS TRANSPORTS
The following NVM Express over Fabrics transports are supported for
accessing remote controllers:
.Bl -tag
.It tcp
TCP transport
.El
.Sh NETWORK ADDRESSES
Network addresses for remote controllers can use one of the following formats:
.Bl -bullet
.It
.Bq Ar IPv6 address
.Ns : Ns Ar port
.It
.Ar IPv4 address
.Ns : Ns Ar port
.It
.Ar hostname Ns : Ns Ar port
.It
.Bq Ar IPv6 address
.It
.Ar IPv6 address
.It
.Ar IPv4 address
.It
.Ar hostname
.El
.Pp
If a
.Ar port
is not provided, a default value is used if possible.
.Sh EXAMPLES .Sh EXAMPLES
.Dl nvmecontrol devlist .Dl nvmecontrol devlist
.Pp .Pp

167
sbin/nvmecontrol/reconnect.c Normal file
View File

@ -0,0 +1,167 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2023-2024 Chelsio Communications, Inc.
* Written by: John Baldwin <jhb@FreeBSD.org>
*/
#include <sys/socket.h>
#include <err.h>
#include <libnvmf.h>
#include <stdlib.h>
#include <string.h>
#include <sysexits.h>
#include <unistd.h>
#include "nvmecontrol.h"
#include "fabrics.h"
/*
* See comment about other possible settings in connect.c.
*/
static struct options {
const char *dev;
const char *transport;
const char *address;
const char *hostnqn;
uint32_t kato;
uint16_t num_io_queues;
uint16_t queue_size;
bool data_digests;
bool flow_control;
bool header_digests;
} opt = {
.dev = NULL,
.transport = "tcp",
.address = NULL,
.hostnqn = NULL,
.kato = NVMF_KATO_DEFAULT / 1000,
.num_io_queues = 1,
.queue_size = 0,
.data_digests = false,
.flow_control = false,
.header_digests = false,
};
static void
tcp_association_params(struct nvmf_association_params *params)
{
params->tcp.pda = 0;
params->tcp.header_digests = opt.header_digests;
params->tcp.data_digests = opt.data_digests;
/* XXX */
params->tcp.maxr2t = 1;
}
static int
reconnect_nvm_controller(int fd, enum nvmf_trtype trtype, int adrfam,
const char *address, const char *port)
{
struct nvme_controller_data cdata;
struct nvmf_association_params aparams;
struct nvmf_reconnect_params rparams;
struct nvmf_qpair *admin, **io;
int error;
error = nvmf_reconnect_params(fd, &rparams);
if (error != 0) {
warnc(error, "Failed to fetch reconnect parameters");
return (EX_IOERR);
}
memset(&aparams, 0, sizeof(aparams));
aparams.sq_flow_control = opt.flow_control;
switch (trtype) {
case NVMF_TRTYPE_TCP:
tcp_association_params(&aparams);
break;
default:
warnx("Unsupported transport %s", nvmf_transport_type(trtype));
return (EX_UNAVAILABLE);
}
io = calloc(opt.num_io_queues, sizeof(*io));
error = connect_nvm_queues(&aparams, trtype, adrfam, address, port,
rparams.cntlid, rparams.subnqn, opt.hostnqn, opt.kato, &admin, io,
opt.num_io_queues, opt.queue_size, &cdata);
if (error != 0)
return (error);
error = nvmf_reconnect_host(fd, admin, opt.num_io_queues, io, &cdata);
if (error != 0) {
warnc(error, "Failed to handoff queues to kernel");
return (EX_IOERR);
}
free(io);
return (0);
}
static void
reconnect_fn(const struct cmd *f, int argc, char *argv[])
{
enum nvmf_trtype trtype;
const char *address, *port;
char *tofree;
int error, fd;
if (arg_parse(argc, argv, f))
return;
if (strcasecmp(opt.transport, "tcp") == 0) {
trtype = NVMF_TRTYPE_TCP;
} else
errx(EX_USAGE, "Unsupported or invalid transport");
nvmf_parse_address(opt.address, &address, &port, &tofree);
open_dev(opt.dev, &fd, 1, 1);
if (port == NULL)
errx(EX_USAGE, "Explicit port required");
error = reconnect_nvm_controller(fd, trtype, AF_UNSPEC, address, port);
if (error != 0)
exit(error);
close(fd);
free(tofree);
}
static const struct opts reconnect_opts[] = {
#define OPT(l, s, t, opt, addr, desc) { l, s, t, &opt.addr, desc }
OPT("transport", 't', arg_string, opt, transport,
"Transport type"),
OPT("nr-io-queues", 'i', arg_uint16, opt, num_io_queues,
"Number of I/O queues"),
OPT("queue-size", 'Q', arg_uint16, opt, queue_size,
"Number of entries in each I/O queue"),
OPT("keep-alive-tmo", 'k', arg_uint32, opt, kato,
"Keep Alive timeout (in seconds)"),
OPT("hostnqn", 'q', arg_string, opt, hostnqn,
"Host NQN"),
OPT("flow_control", 'F', arg_none, opt, flow_control,
"Request SQ flow control"),
OPT("hdr_digests", 'g', arg_none, opt, header_digests,
"Enable TCP PDU header digests"),
OPT("data_digests", 'G', arg_none, opt, data_digests,
"Enable TCP PDU data digests"),
{ NULL, 0, arg_none, NULL, NULL }
};
#undef OPT
static const struct args reconnect_args[] = {
{ arg_string, &opt.dev, "controller-id" },
{ arg_string, &opt.address, "address" },
{ arg_none, NULL, NULL },
};
static struct cmd reconnect_cmd = {
.name = "reconnect",
.fn = reconnect_fn,
.descr = "Reconnect to a fabrics controller",
.ctx_size = sizeof(opt),
.opts = reconnect_opts,
.args = reconnect_args,
};
CMD_COMMAND(reconnect_cmd);