hardenedbsd/HardenedBSD
1
0
Fork 0
mirror of https://git.hardenedbsd.org/hardenedbsd/HardenedBSD.git synced 2024-11-11 04:42:16 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

irdma(4): Upgrade to 1.2.36-k

Browse Source 
Update Intel irdma driver to version 1.2.36-k.

Notable changes:

- Start using ib_sge directly instead of irdma_sge
- Turn off flush completion generator for libirdma
- Minor formatting changes

Signed-off-by: Bartosz Sobczak <bartosz.sobczak@intel.com>
Signed-off-by: Eric Joyner <erj@FreeBSD.org>

Reviewed by:	erj@
MFC after:	3 days
Sponsored by:	Intel Corporation
Differential Revision:	https://reviews.freebsd.org/D43567
This commit is contained in:
Bartosz Sobczak 2024-02-16 13:55:08 -08:00 committed by Eric Joyner
parent 66d37dbedf
commit 5b5f7d0e77
No known key found for this signature in database
GPG Key ID: 96F0C6FD61E05DE3
39 changed files with 426 additions and 1671 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
contrib/ofed/libirdma/abi.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (C) 2019 - 2022 Intel Corporation
* Copyright (C) 2019 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*$FreeBSD$*/
#ifndef PROVIDER_IRDMA_ABI_H
#define PROVIDER_IRDMA_ABI_H
@ -86,6 +85,7 @@ struct irdma_ucreate_qp {
struct ibv_create_qp ibv_cmd;
__aligned_u64 user_wqe_bufs;
__aligned_u64 user_compl_ctx;
__aligned_u64 comp_mask;
};
struct irdma_ucreate_qp_resp {
@ -98,6 +98,9 @@ struct irdma_ucreate_qp_resp {
__u8 lsmm;
__u8 rsvd;
__u32 qp_caps;
__aligned_u64 comp_mask;
__u8 start_wqe_idx;
__u8 rsvd2[7];
};
struct irdma_umodify_qp_resp {
@ -138,6 +141,8 @@ struct irdma_get_context_resp {
__u8 hw_rev;
__u8 rsvd2;
__aligned_u64 comp_mask;
__u16 min_hw_wq_size;
__u8 rsvd3[6];
};
struct irdma_ureg_mr {

1
contrib/ofed/libirdma/i40e_devids.h
View File

@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*$FreeBSD$*/
#ifndef I40E_DEVIDS_H
#define I40E_DEVIDS_H

3
contrib/ofed/libirdma/i40iw_hw.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2015 - 2022 Intel Corporation
* Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*$FreeBSD$*/
#ifndef I40IW_HW_H
#define I40IW_HW_H

1
contrib/ofed/libirdma/ice_devids.h
View File

@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*$FreeBSD$*/
#ifndef ICE_DEVIDS_H
#define ICE_DEVIDS_H

12
contrib/ofed/libirdma/irdma-abi.h
View File

@ -35,7 +35,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*$FreeBSD$*/
#ifndef IRDMA_ABI_H
#define IRDMA_ABI_H
@ -55,6 +54,11 @@ enum irdma_memreg_type {
enum {
IRDMA_ALLOC_UCTX_USE_RAW_ATTR = 1 << 0,
IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE = 1 << 1,
};
enum {
IRDMA_CREATE_QP_USE_START_WQE_IDX = 1 << 0,
};
struct irdma_alloc_ucontext_req {
@ -83,6 +87,8 @@ struct irdma_alloc_ucontext_resp {
__u8 hw_rev;
__u8 rsvd2;
__aligned_u64 comp_mask;
__u16 min_hw_wq_size;
__u8 rsvd3[6];
};
struct irdma_alloc_pd_resp {
@ -102,6 +108,7 @@ struct irdma_create_cq_req {
struct irdma_create_qp_req {
__aligned_u64 user_wqe_bufs;
__aligned_u64 user_compl_ctx;
__aligned_u64 comp_mask;
};
struct irdma_mem_reg_req {
@ -131,6 +138,9 @@ struct irdma_create_qp_resp {
__u8 lsmm;
__u8 rsvd;
__u32 qp_caps;
__aligned_u64 comp_mask;
__u8 start_wqe_idx;
__u8 rsvd2[7];
};
struct irdma_modify_qp_resp {

2
contrib/ofed/libirdma/irdma.h
View File

@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*$FreeBSD$*/
#ifndef IRDMA_H
#define IRDMA_H
@ -47,7 +46,6 @@ enum irdma_vers {
IRDMA_GEN_RSVD = 0,
IRDMA_GEN_1 = 1,
IRDMA_GEN_2 = 2,
IRDMA_GEN_MAX = 2,
};
struct irdma_uk_attrs {

14
contrib/ofed/libirdma/irdma_defs.h
View File

@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*$FreeBSD$*/
#ifndef IRDMA_DEFS_H
#define IRDMA_DEFS_H
@ -457,19 +456,6 @@ enum irdma_qp_wqe_size {
IRDMA_WQE_SIZE_256 = 256,
};
enum irdma_ws_op_type {
IRDMA_WS_OP_TYPE_NODE = 0,
IRDMA_WS_OP_TYPE_LEAF_NODE_GROUP,
};
enum irdma_ws_rate_limit_flags {
IRDMA_WS_RATE_LIMIT_FLAGS_VALID = 0x1,
IRDMA_WS_NO_RDMA_RATE_LIMIT = 0x2,
IRDMA_WS_LEAF_NODE_IS_PART_GROUP = 0x4,
IRDMA_WS_TREE_RATE_LIMITING = 0x8,
IRDMA_WS_PACING_CONTROL = 0x10,
};
/**
* set_64bit_val - set 64 bit value to hw wqe
* @wqe_words: wqe addr to write

108
contrib/ofed/libirdma/irdma_uk.c
View File

@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*$FreeBSD$*/
#include "osdep.h"
#include "irdma_defs.h"
@ -46,16 +45,16 @@
* @valid: The wqe valid
*/
static void
irdma_set_fragment(__le64 * wqe, u32 offset, struct irdma_sge *sge,
irdma_set_fragment(__le64 * wqe, u32 offset, struct ibv_sge *sge,
u8 valid)
{
if (sge) {
set_64bit_val(wqe, offset,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
FIELD_PREP(IRDMAQPSQ_VALID, valid) |
FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->len) |
FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->stag));
FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->length) |
FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->lkey));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
@ -72,14 +71,14 @@ irdma_set_fragment(__le64 * wqe, u32 offset, struct irdma_sge *sge,
*/
static void
irdma_set_fragment_gen_1(__le64 * wqe, u32 offset,
struct irdma_sge *sge, u8 valid)
struct ibv_sge *sge, u8 valid)
{
if (sge) {
set_64bit_val(wqe, offset,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->len) |
FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->stag));
FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->length) |
FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->lkey));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8, 0);
@ -210,8 +209,7 @@ irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 * wqe, u16 quanta,
if (IRDMA_RING_CURRENT_HEAD(qp->initial_ring) !=
IRDMA_RING_CURRENT_TAIL(qp->sq_ring) &&
!qp->push_mode) {
if (post_sq)
irdma_uk_qp_post_wr(qp);
irdma_uk_qp_post_wr(qp);
} else {
push = (__le64 *) ((uintptr_t)qp->push_wqe +
(wqe_idx & 0x7) * 0x20);
@ -339,7 +337,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
return EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
total_size += op_info->lo_sg_list[i].len;
total_size += op_info->lo_sg_list[i].length;
read_fence |= info->read_fence;
@ -358,7 +356,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
set_64bit_val(wqe, IRDMA_BYTE_16,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
if (info->imm_data_valid) {
set_64bit_val(wqe, IRDMA_BYTE_0,
@ -387,7 +385,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
++addl_frag_cnt;
}
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) |
@ -438,7 +436,7 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
return EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
total_size += op_info->lo_sg_list[i].len;
total_size += op_info->lo_sg_list[i].length;
ret_code = irdma_fragcnt_to_quanta_sq(op_info->num_lo_sges, &quanta);
if (ret_code)
@ -476,8 +474,8 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
++addl_frag_cnt;
}
set_64bit_val(wqe, IRDMA_BYTE_16,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
FIELD_PREP(IRDMAQPSQ_OPCODE,
@ -526,7 +524,7 @@ irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
return EINVAL;
for (i = 0; i < op_info->num_sges; i++)
total_size += op_info->sg_list[i].len;
total_size += op_info->sg_list[i].length;
if (info->imm_data_valid)
frag_cnt = op_info->num_sges + 1;
@ -621,15 +619,15 @@ irdma_set_mw_bind_wqe_gen_1(__le64 * wqe,
* @polarity: compatibility parameter
*/
static void
irdma_copy_inline_data_gen_1(u8 *wqe, struct irdma_sge *sge_list,
irdma_copy_inline_data_gen_1(u8 *wqe, struct ibv_sge *sge_list,
u32 num_sges, u8 polarity)
{
u32 quanta_bytes_remaining = 16;
u32 i;
for (i = 0; i < num_sges; i++) {
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off;
u32 sge_len = sge_list[i].len;
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
u32 sge_len = sge_list[i].length;
while (sge_len) {
u32 bytes_copied;
@ -684,7 +682,7 @@ irdma_set_mw_bind_wqe(__le64 * wqe,
* @polarity: polarity of wqe valid bit
*/
static void
irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list,
irdma_copy_inline_data(u8 *wqe, struct ibv_sge *sge_list,
u32 num_sges, u8 polarity)
{
u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;
@ -695,8 +693,8 @@ irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list,
wqe += 8;
for (i = 0; i < num_sges; i++) {
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off;
u32 sge_len = sge_list[i].len;
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
u32 sge_len = sge_list[i].length;
while (sge_len) {
u32 bytes_copied;
@ -776,7 +774,7 @@ irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
return EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
total_size += op_info->lo_sg_list[i].len;
total_size += op_info->lo_sg_list[i].length;
if (unlikely(total_size > qp->max_inline_data))
return EINVAL;
@ -789,9 +787,9 @@ irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
read_fence |= info->read_fence;
set_64bit_val(wqe, IRDMA_BYTE_16,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) |
@ -847,7 +845,7 @@ irdma_uk_inline_send(struct irdma_qp_uk *qp,
return EINVAL;
for (i = 0; i < op_info->num_sges; i++)
total_size += op_info->sg_list[i].len;
total_size += op_info->sg_list[i].length;
if (unlikely(total_size > qp->max_inline_data))
return EINVAL;
@ -912,7 +910,7 @@ irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
u64 hdr;
u32 wqe_idx;
bool local_fence = false;
struct irdma_sge sge = {0};
struct ibv_sge sge = {0};
u16 quanta = IRDMA_QP_WQE_MIN_QUANTA;
info->push_wqe = qp->push_db ? true : false;
@ -923,7 +921,7 @@ irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
if (!wqe)
return ENOSPC;
sge.stag = op_info->target_stag;
sge.lkey = op_info->target_stag;
qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, &sge, 0);
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
@ -1437,8 +1435,7 @@ irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
IRDMA_RING_MOVE_TAIL(cq->cq_ring);
set_64bit_val(cq->shadow_area, IRDMA_BYTE_0,
IRDMA_RING_CURRENT_HEAD(cq->cq_ring));
memset(info, 0,
sizeof(struct irdma_cq_poll_info));
memset(info, 0, sizeof(*info));
return irdma_uk_cq_poll_cmpl(cq, info);
}
}
@ -1511,7 +1508,6 @@ exit:
if (pring && IRDMA_RING_MORE_WORK(*pring))
move_cq_head = false;
}
if (move_cq_head) {
IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
if (!IRDMA_RING_CURRENT_HEAD(cq->cq_ring))
@ -1592,10 +1588,12 @@ irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
int
irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth)
{
u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift;
*sqdepth = irdma_round_up_wq((sq_size << shift) + IRDMA_SQ_RSVD);
if (*sqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
*sqdepth = uk_attrs->min_hw_wq_size << shift;
if (*sqdepth < min_size)
*sqdepth = min_size;
else if (*sqdepth > uk_attrs->max_hw_wq_quanta)
return EINVAL;
@ -1609,10 +1607,12 @@ irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *s
int
irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth)
{
u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift;
*rqdepth = irdma_round_up_wq((rq_size << shift) + IRDMA_RQ_RSVD);
if (*rqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
*rqdepth = uk_attrs->min_hw_wq_size << shift;
if (*rqdepth < min_size)
*rqdepth = min_size;
else if (*rqdepth > uk_attrs->max_hw_rq_quanta)
return EINVAL;
@ -1645,41 +1645,16 @@ irdma_setup_connection_wqes(struct irdma_qp_uk *qp,
{
u16 move_cnt = 1;
if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)
if (info->start_wqe_idx)
move_cnt = info->start_wqe_idx;
else if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)
move_cnt = 3;
qp->conn_wqes = move_cnt;
IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt);
IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt);
IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt);
}
/**
* irdma_uk_calc_shift_wq - calculate WQE shift for both SQ and RQ
* @ukinfo: qp initialization info
* @sq_shift: Returns shift of SQ
* @rq_shift: Returns shift of RQ
*/
void
irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,
u8 *rq_shift)
{
bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2 ? true : false;
irdma_get_wqe_shift(ukinfo->uk_attrs,
imm_support ? ukinfo->max_sq_frag_cnt + 1 :
ukinfo->max_sq_frag_cnt,
ukinfo->max_inline_data, sq_shift);
irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_rq_frag_cnt, 0,
rq_shift);
if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) {
if (ukinfo->abi_ver > 4)
*rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
}
}
/**
* irdma_uk_calc_depth_shift_sq - calculate depth and shift for SQ size.
* @ukinfo: qp initialization info
@ -1692,6 +1667,7 @@ irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,
{
bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2 ? true : false;
int status;
irdma_get_wqe_shift(ukinfo->uk_attrs,
imm_support ? ukinfo->max_sq_frag_cnt + 1 :
ukinfo->max_sq_frag_cnt,
@ -1786,6 +1762,8 @@ irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info)
qp->wqe_ops = iw_wqe_uk_ops_gen_1;
else
qp->wqe_ops = iw_wqe_uk_ops;
qp->start_wqe_idx = info->start_wqe_idx;
return ret_code;
}

5
contrib/ofed/libirdma/irdma_umain.c
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2021 - 2023 Intel Corporation
* Copyright (c) 2021 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*$FreeBSD$*/
#include <sys/mman.h>
@ -49,7 +48,7 @@
/**
* Driver version
*/
char libirdma_version[] = "1.2.17-k";
char libirdma_version[] = "1.2.36-k";
unsigned int irdma_dbg;

3
contrib/ofed/libirdma/irdma_umain.h
View File

@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*$FreeBSD$*/
#ifndef IRDMA_UMAIN_H
#define IRDMA_UMAIN_H
@ -97,7 +96,6 @@ struct irdma_cq_buf {
LIST_ENTRY(irdma_cq_buf) list;
struct irdma_cq_uk cq;
struct verbs_mr vmr;
size_t buf_size;
};
extern pthread_mutex_t sigusr1_wait_mutex;
@ -143,7 +141,6 @@ struct irdma_uqp {
struct ibv_recv_wr *pend_rx_wr;
struct irdma_qp_uk qp;
enum ibv_qp_type qp_type;
struct irdma_sge *recv_sges;
};
/* irdma_uverbs.c */

1
contrib/ofed/libirdma/irdma_uquery.h
View File

@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*$FreeBSD$*/
#ifndef IRDMA_UQUERY_H

197
contrib/ofed/libirdma/irdma_user.h
View File

@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*$FreeBSD$*/
#ifndef IRDMA_USER_H
#define IRDMA_USER_H
@ -51,7 +50,7 @@
#define irdma_access_privileges u32
#define irdma_physical_fragment u64
#define irdma_address_list u64 *
#define irdma_sgl struct irdma_sge *
#define irdma_sgl struct ibv_sge *
#define IRDMA_MAX_MR_SIZE 0x200000000000ULL
@ -81,96 +80,6 @@
#define IRDMA_OP_TYPE_REC_IMM 0x3f
#define IRDMA_FLUSH_MAJOR_ERR 1
#define IRDMA_SRQFLUSH_RSVD_MAJOR_ERR 0xfffe
/* Async Events codes */
#define IRDMA_AE_AMP_UNALLOCATED_STAG 0x0102
#define IRDMA_AE_AMP_INVALID_STAG 0x0103
#define IRDMA_AE_AMP_BAD_QP 0x0104
#define IRDMA_AE_AMP_BAD_PD 0x0105
#define IRDMA_AE_AMP_BAD_STAG_KEY 0x0106
#define IRDMA_AE_AMP_BAD_STAG_INDEX 0x0107
#define IRDMA_AE_AMP_BOUNDS_VIOLATION 0x0108
#define IRDMA_AE_AMP_RIGHTS_VIOLATION 0x0109
#define IRDMA_AE_AMP_TO_WRAP 0x010a
#define IRDMA_AE_AMP_FASTREG_VALID_STAG 0x010c
#define IRDMA_AE_AMP_FASTREG_MW_STAG 0x010d
#define IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS 0x010e
#define IRDMA_AE_AMP_FASTREG_INVALID_LENGTH 0x0110
#define IRDMA_AE_AMP_INVALIDATE_SHARED 0x0111
#define IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS 0x0112
#define IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS 0x0113
#define IRDMA_AE_AMP_MWBIND_VALID_STAG 0x0114
#define IRDMA_AE_AMP_MWBIND_OF_MR_STAG 0x0115
#define IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG 0x0116
#define IRDMA_AE_AMP_MWBIND_TO_MW_STAG 0x0117
#define IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS 0x0118
#define IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS 0x0119
#define IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT 0x011a
#define IRDMA_AE_AMP_MWBIND_BIND_DISABLED 0x011b
#define IRDMA_AE_PRIV_OPERATION_DENIED 0x011c
#define IRDMA_AE_AMP_INVALIDATE_TYPE1_MW 0x011d
#define IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW 0x011e
#define IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG 0x011f
#define IRDMA_AE_AMP_MWBIND_WRONG_TYPE 0x0120
#define IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH 0x0121
#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG 0x0132
#define IRDMA_AE_UDA_XMIT_BAD_PD 0x0133
#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT 0x0134
#define IRDMA_AE_UDA_L4LEN_INVALID 0x0135
#define IRDMA_AE_BAD_CLOSE 0x0201
#define IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE 0x0202
#define IRDMA_AE_CQ_OPERATION_ERROR 0x0203
#define IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO 0x0205
#define IRDMA_AE_STAG_ZERO_INVALID 0x0206
#define IRDMA_AE_IB_RREQ_AND_Q1_FULL 0x0207
#define IRDMA_AE_IB_INVALID_REQUEST 0x0208
#define IRDMA_AE_WQE_UNEXPECTED_OPCODE 0x020a
#define IRDMA_AE_WQE_INVALID_PARAMETER 0x020b
#define IRDMA_AE_WQE_INVALID_FRAG_DATA 0x020c
#define IRDMA_AE_IB_REMOTE_ACCESS_ERROR 0x020d
#define IRDMA_AE_IB_REMOTE_OP_ERROR 0x020e
#define IRDMA_AE_WQE_LSMM_TOO_LONG 0x0220
#define IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301
#define IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303
#define IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304
#define IRDMA_AE_DDP_UBE_INVALID_MO 0x0305
#define IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE 0x0306
#define IRDMA_AE_DDP_UBE_INVALID_QN 0x0307
#define IRDMA_AE_DDP_NO_L_BIT 0x0308
#define IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION 0x0311
#define IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE 0x0312
#define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313
#define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314
#define IRDMA_AE_ROCE_RSP_LENGTH_ERROR 0x0316
#define IRDMA_AE_ROCE_EMPTY_MCG 0x0380
#define IRDMA_AE_ROCE_BAD_MC_IP_ADDR 0x0381
#define IRDMA_AE_ROCE_BAD_MC_QPID 0x0382
#define IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH 0x0383
#define IRDMA_AE_INVALID_ARP_ENTRY 0x0401
#define IRDMA_AE_INVALID_TCP_OPTION_RCVD 0x0402
#define IRDMA_AE_STALE_ARP_ENTRY 0x0403
#define IRDMA_AE_INVALID_AH_ENTRY 0x0406
#define IRDMA_AE_LLP_CLOSE_COMPLETE 0x0501
#define IRDMA_AE_LLP_CONNECTION_RESET 0x0502
#define IRDMA_AE_LLP_FIN_RECEIVED 0x0503
#define IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH 0x0504
#define IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR 0x0505
#define IRDMA_AE_LLP_SEGMENT_TOO_SMALL 0x0507
#define IRDMA_AE_LLP_SYN_RECEIVED 0x0508
#define IRDMA_AE_LLP_TERMINATE_RECEIVED 0x0509
#define IRDMA_AE_LLP_TOO_MANY_RETRIES 0x050a
#define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b
#define IRDMA_AE_LLP_DOUBT_REACHABILITY 0x050c
#define IRDMA_AE_LLP_CONNECTION_ESTABLISHED 0x050e
#define IRDMA_AE_RESOURCE_EXHAUSTION 0x0520
#define IRDMA_AE_RESET_SENT 0x0601
#define IRDMA_AE_TERMINATE_SENT 0x0602
#define IRDMA_AE_RESET_NOT_SENT 0x0603
#define IRDMA_AE_LCE_QP_CATASTROPHIC 0x0700
#define IRDMA_AE_LCE_FUNCTION_CATASTROPHIC 0x0701
#define IRDMA_AE_LCE_CQ_CATASTROPHIC 0x0702
#define IRDMA_AE_QP_SUSPEND_COMPLETE 0x0900
enum irdma_device_caps_const {
IRDMA_WQE_SIZE = 4,
@ -202,8 +111,7 @@ enum irdma_device_caps_const {
IRDMA_MAX_OUTBOUND_MSG_SIZE = 65537,
/* 64K +1 */
IRDMA_MAX_INBOUND_MSG_SIZE = 65537,
IRDMA_MAX_PUSH_PAGE_COUNT = 1024,
IRDMA_MAX_PE_ENA_VF_COUNT = 32,
IRDMA_MAX_PE_ENA_VF_COUNT = 32,
IRDMA_MAX_VF_FPM_ID = 47,
IRDMA_MAX_SQ_PAYLOAD_SIZE = 2145386496,
IRDMA_MAX_INLINE_DATA_SIZE = 101,
@ -230,12 +138,7 @@ enum irdma_flush_opcode {
FLUSH_RETRY_EXC_ERR,
FLUSH_MW_BIND_ERR,
FLUSH_REM_INV_REQ_ERR,
};
enum irdma_qp_event_type {
IRDMA_QP_EVENT_CATASTROPHIC,
IRDMA_QP_EVENT_ACCESS_ERR,
IRDMA_QP_EVENT_REQ_ERR,
FLUSH_RNR_RETRY_EXC_ERR,
};
enum irdma_cmpl_status {
@ -283,12 +186,6 @@ struct irdma_cq_uk;
struct irdma_qp_uk_init_info;
struct irdma_cq_uk_init_info;
struct irdma_sge {
irdma_tagged_offset tag_off;
u32 len;
irdma_stag stag;
};
struct irdma_ring {
volatile u32 head;
volatile u32 tail; /* effective tail */
@ -320,13 +217,13 @@ struct irdma_post_rq_info {
struct irdma_rdma_write {
irdma_sgl lo_sg_list;
u32 num_lo_sges;
struct irdma_sge rem_addr;
struct ibv_sge rem_addr;
};
struct irdma_rdma_read {
irdma_sgl lo_sg_list;
u32 num_lo_sges;
struct irdma_sge rem_addr;
struct ibv_sge rem_addr;
};
struct irdma_bind_window {
@ -400,11 +297,6 @@ struct irdma_cq_poll_info {
} stat;
};
struct qp_err_code {
enum irdma_flush_opcode flush_code;
enum irdma_qp_event_type event_type;
};
int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info, bool post_sq);
int irdma_uk_inline_send(struct irdma_qp_uk *qp,
@ -427,9 +319,9 @@ int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
bool post_sq);
struct irdma_wqe_uk_ops {
void (*iw_copy_inline_data)(u8 *dest, struct irdma_sge *sge_list, u32 num_sges, u8 polarity);
void (*iw_copy_inline_data)(u8 *dest, struct ibv_sge *sge_list, u32 num_sges, u8 polarity);
u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct irdma_sge *sge,
void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct ibv_sge *sge,
u8 valid);
void (*iw_set_mw_bind_wqe)(__le64 *wqe,
struct irdma_bind_window *op_info);
@ -445,8 +337,6 @@ int irdma_uk_cq_init(struct irdma_cq_uk *cq,
struct irdma_cq_uk_init_info *info);
int irdma_uk_qp_init(struct irdma_qp_uk *qp,
struct irdma_qp_uk_init_info *info);
void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,
u8 *rq_shift);
int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,
u32 *sq_depth, u8 *sq_shift);
int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo,
@ -495,6 +385,7 @@ struct irdma_qp_uk {
u8 rwqe_polarity;
u8 rq_wqe_size;
u8 rq_wqe_size_multiplier;
u8 start_wqe_idx;
bool deferred_flag:1;
bool push_mode:1; /* whether the last post wqe was pushed */
bool push_dropped:1;
@ -542,6 +433,7 @@ struct irdma_qp_uk_init_info {
u32 sq_depth;
u32 rq_depth;
u8 first_sq_wq;
u8 start_wqe_idx;
u8 type;
u8 sq_shift;
u8 rq_shift;
@ -575,75 +467,4 @@ int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u3
void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta,
u32 wqe_idx, bool post_sq);
void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);
static inline struct qp_err_code irdma_ae_to_qp_err_code(u16 ae_id)
{
struct qp_err_code qp_err = { 0 };
switch (ae_id) {
case IRDMA_AE_AMP_BOUNDS_VIOLATION:
case IRDMA_AE_AMP_INVALID_STAG:
case IRDMA_AE_AMP_RIGHTS_VIOLATION:
case IRDMA_AE_AMP_UNALLOCATED_STAG:
case IRDMA_AE_AMP_BAD_PD:
case IRDMA_AE_AMP_BAD_QP:
case IRDMA_AE_AMP_BAD_STAG_KEY:
case IRDMA_AE_AMP_BAD_STAG_INDEX:
case IRDMA_AE_AMP_TO_WRAP:
case IRDMA_AE_PRIV_OPERATION_DENIED:
qp_err.flush_code = FLUSH_PROT_ERR;
qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
break;
case IRDMA_AE_UDA_XMIT_BAD_PD:
case IRDMA_AE_WQE_UNEXPECTED_OPCODE:
qp_err.flush_code = FLUSH_LOC_QP_OP_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT:
case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
case IRDMA_AE_UDA_L4LEN_INVALID:
case IRDMA_AE_DDP_UBE_INVALID_MO:
case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
qp_err.flush_code = FLUSH_LOC_LEN_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
case IRDMA_AE_IB_REMOTE_ACCESS_ERROR:
qp_err.flush_code = FLUSH_REM_ACCESS_ERR;
qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
break;
case IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS:
case IRDMA_AE_AMP_MWBIND_BIND_DISABLED:
case IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS:
case IRDMA_AE_AMP_MWBIND_VALID_STAG:
qp_err.flush_code = FLUSH_MW_BIND_ERR;
qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
break;
case IRDMA_AE_LLP_TOO_MANY_RETRIES:
qp_err.flush_code = FLUSH_RETRY_EXC_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_IB_INVALID_REQUEST:
qp_err.flush_code = FLUSH_REM_INV_REQ_ERR;
qp_err.event_type = IRDMA_QP_EVENT_REQ_ERR;
break;
case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
case IRDMA_AE_IB_REMOTE_OP_ERROR:
qp_err.flush_code = FLUSH_REM_OP_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_LCE_QP_CATASTROPHIC:
qp_err.flush_code = FLUSH_FATAL_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
default:
qp_err.flush_code = FLUSH_GENERAL_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
}
return qp_err;
}
#endif /* IRDMA_USER_H */

90
contrib/ofed/libirdma/irdma_uverbs.c
View File

@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*$FreeBSD$*/
#include <config.h>
#include <stdlib.h>
@ -153,6 +152,7 @@ irdma_ualloc_pd(struct ibv_context *context)
err_free:
free(iwupd);
errno = err;
return NULL;
}
@ -164,7 +164,6 @@ err_free:
int
irdma_ufree_pd(struct ibv_pd *pd)
{
struct irdma_uvcontext *iwvctx = container_of(pd->context, struct irdma_uvcontext, ibv_ctx);
struct irdma_upd *iwupd;
int ret;
@ -375,12 +374,12 @@ irdma_free_hw_buf(void *buf, size_t size)
* @cqe_64byte_ena: enable 64byte cqe
*/
static inline int
get_cq_size(int ncqe, u8 hw_rev, bool cqe_64byte_ena)
get_cq_size(int ncqe, u8 hw_rev)
{
ncqe++;
/* Completions with immediate require 1 extra entry */
if (!cqe_64byte_ena && hw_rev > IRDMA_GEN_1)
if (hw_rev > IRDMA_GEN_1)
ncqe *= 2;
if (ncqe < IRDMA_U_MINCQ_SIZE)
@ -389,11 +388,8 @@ get_cq_size(int ncqe, u8 hw_rev, bool cqe_64byte_ena)
return ncqe;
}
static inline size_t get_cq_total_bytes(u32 cq_size, bool cqe_64byte_ena){
if (cqe_64byte_ena)
return roundup(cq_size * sizeof(struct irdma_extended_cqe), IRDMA_HW_PAGE_SIZE);
else
return roundup(cq_size * sizeof(struct irdma_cqe), IRDMA_HW_PAGE_SIZE);
static inline size_t get_cq_total_bytes(u32 cq_size) {
return roundup(cq_size * sizeof(struct irdma_cqe), IRDMA_HW_PAGE_SIZE);
}
/**
@ -421,7 +417,6 @@ ucreate_cq(struct ibv_context *context,
u32 cq_pages;
int ret, ncqe;
u8 hw_rev;
bool cqe_64byte_ena;
iwvctx = container_of(context, struct irdma_uvcontext, ibv_ctx);
uk_attrs = &iwvctx->uk_attrs;
@ -455,11 +450,10 @@ ucreate_cq(struct ibv_context *context,
return NULL;
}
cqe_64byte_ena = uk_attrs->feature_flags & IRDMA_FEATURE_64_BYTE_CQE ? true : false;
info.cq_size = get_cq_size(attr_ex->cqe, hw_rev, cqe_64byte_ena);
info.cq_size = get_cq_size(attr_ex->cqe, hw_rev);
total_size = get_cq_total_bytes(info.cq_size);
iwucq->comp_vector = attr_ex->comp_vector;
LIST_INIT(&iwucq->resize_list);
total_size = get_cq_total_bytes(info.cq_size, cqe_64byte_ena);
cq_pages = total_size >> IRDMA_HW_PAGE_SHIFT;
if (!(uk_attrs->feature_flags & IRDMA_FEATURE_CQ_RESIZE))
@ -529,8 +523,6 @@ ucreate_cq(struct ibv_context *context,
info.cq_id = resp.cq_id;
/* Do not report the CQE's reserved for immediate and burned by HW */
iwucq->verbs_cq.cq.cqe = ncqe;
if (cqe_64byte_ena)
info.avoid_mem_cflct = true;
info.cqe_alloc_db = (u32 *)((u8 *)iwvctx->db + IRDMA_DB_CQ_OFFSET);
irdma_uk_cq_init(&iwucq->cq, &info);
return &iwucq->verbs_cq.cq_ex;
@ -586,7 +578,7 @@ static void
irdma_free_cq_buf(struct irdma_cq_buf *cq_buf)
{
ibv_cmd_dereg_mr(&cq_buf->vmr.ibv_mr);
irdma_free_hw_buf(cq_buf->cq.cq_base, cq_buf->buf_size);
irdma_free_hw_buf(cq_buf->cq.cq_base, get_cq_total_bytes(cq_buf->cq.cq_size));
free(cq_buf);
}
@ -1323,6 +1315,8 @@ irdma_vmapped_qp(struct irdma_uqp *iwuqp, struct ibv_pd *pd,
cmd.user_wqe_bufs = (__u64) ((uintptr_t)info->sq);
cmd.user_compl_ctx = (__u64) (uintptr_t)&iwuqp->qp;
cmd.comp_mask |= IRDMA_CREATE_QP_USE_START_WQE_IDX;
ret = ibv_cmd_create_qp(pd, &iwuqp->ibv_qp, attr, &cmd.ibv_cmd,
sizeof(cmd), &resp.ibv_resp,
sizeof(struct irdma_ucreate_qp_resp));
@ -1332,6 +1326,8 @@ irdma_vmapped_qp(struct irdma_uqp *iwuqp, struct ibv_pd *pd,
info->sq_size = resp.actual_sq_size;
info->rq_size = resp.actual_rq_size;
info->first_sq_wq = legacy_mode ? 1 : resp.lsmm;
if (resp.comp_mask & IRDMA_CREATE_QP_USE_START_WQE_IDX)
info->start_wqe_idx = resp.start_wqe_idx;
info->qp_caps = resp.qp_caps;
info->qp_id = resp.qp_id;
iwuqp->irdma_drv_opt = resp.irdma_drv_opt;
@ -1380,6 +1376,8 @@ irdma_ucreate_qp(struct ibv_pd *pd,
if (attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags ||
attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags ||
attr->cap.max_send_wr > uk_attrs->max_hw_wq_quanta ||
attr->cap.max_recv_wr > uk_attrs->max_hw_rq_quanta ||
attr->cap.max_inline_data > uk_attrs->max_hw_inline) {
errno = EINVAL;
return NULL;
@ -1431,18 +1429,12 @@ irdma_ucreate_qp(struct ibv_pd *pd,
attr->cap.max_recv_wr = info.rq_size;
}
iwuqp->recv_sges = calloc(attr->cap.max_recv_sge, sizeof(*iwuqp->recv_sges));
if (!iwuqp->recv_sges) {
status = errno; /* preserve errno */
goto err_destroy_lock;
}
info.wqe_alloc_db = (u32 *)iwvctx->db;
info.legacy_mode = iwvctx->legacy_mode;
info.sq_wrtrk_array = calloc(info.sq_depth, sizeof(*info.sq_wrtrk_array));
if (!info.sq_wrtrk_array) {
status = errno; /* preserve errno */
goto err_free_rsges;
goto err_destroy_lock;
}
info.rq_wrid_array = calloc(info.rq_depth, sizeof(*info.rq_wrid_array));
@ -1476,8 +1468,6 @@ err_free_rq_wrid:
free(info.rq_wrid_array);
err_free_sq_wrtrk:
free(info.sq_wrtrk_array);
err_free_rsges:
free(iwuqp->recv_sges);
err_destroy_lock:
pthread_spin_destroy(&iwuqp->lock);
err_free_qp:
@ -1636,7 +1626,6 @@ irdma_udestroy_qp(struct ibv_qp *qp)
free(iwuqp->qp.rq_wrid_array);
irdma_free_hw_buf(iwuqp->qp.sq_base, iwuqp->buf_size);
free(iwuqp->recv_sges);
free(iwuqp);
return 0;
@ -1646,26 +1635,6 @@ err:
return ret;
}
/**
* irdma_copy_sg_list - copy sg list for qp
* @sg_list: copied into sg_list
* @sgl: copy from sgl
* @num_sges: count of sg entries
* @max_sges: count of max supported sg entries
*/
static void
irdma_copy_sg_list(struct irdma_sge *sg_list, struct ibv_sge *sgl,
int num_sges)
{
int i;
for (i = 0; i < num_sges; i++) {
sg_list[i].tag_off = sgl[i].addr;
sg_list[i].len = sgl[i].length;
sg_list[i].stag = sgl[i].lkey;
}
}
/**
* calc_type2_mw_stag - calculate type 2 MW stag
* @rkey: desired rkey of the MW
@ -1744,7 +1713,7 @@ irdma_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr,
info.stag_to_inv = ib_wr->imm_data;
}
info.op.send.num_sges = ib_wr->num_sge;
info.op.send.sg_list = (struct irdma_sge *)ib_wr->sg_list;
info.op.send.sg_list = (struct ibv_sge *)ib_wr->sg_list;
if (ib_qp->qp_type == IBV_QPT_UD) {
struct irdma_uah *ah = container_of(ib_wr->wr.ud.ah,
struct irdma_uah, ibv_ah);
@ -1775,9 +1744,9 @@ irdma_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr,
info.op_type = IRDMA_OP_TYPE_RDMA_WRITE;
info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
info.op.rdma_write.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr;
info.op.rdma_write.rem_addr.stag = ib_wr->wr.rdma.rkey;
info.op.rdma_write.lo_sg_list = ib_wr->sg_list;
info.op.rdma_write.rem_addr.addr = ib_wr->wr.rdma.remote_addr;
info.op.rdma_write.rem_addr.lkey = ib_wr->wr.rdma.rkey;
if (ib_wr->send_flags & IBV_SEND_INLINE)
err = irdma_uk_inline_rdma_write(&iwuqp->qp, &info, false);
else
@ -1789,10 +1758,10 @@ irdma_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr,
break;
}
info.op_type = IRDMA_OP_TYPE_RDMA_READ;
info.op.rdma_read.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr;
info.op.rdma_read.rem_addr.stag = ib_wr->wr.rdma.rkey;
info.op.rdma_read.rem_addr.addr = ib_wr->wr.rdma.remote_addr;
info.op.rdma_read.rem_addr.lkey = ib_wr->wr.rdma.rkey;
info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list;
info.op.rdma_read.lo_sg_list = ib_wr->sg_list;
info.op.rdma_read.num_lo_sges = ib_wr->num_sge;
err = irdma_uk_rdma_read(&iwuqp->qp, &info, false, false);
break;
@ -1874,14 +1843,11 @@ irdma_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr,
struct ibv_recv_wr **bad_wr)
{
struct irdma_post_rq_info post_recv = {};
struct irdma_sge *sg_list;
struct irdma_uqp *iwuqp;
bool reflush = false;
int err = 0;
iwuqp = container_of(ib_qp, struct irdma_uqp, ibv_qp);
sg_list = iwuqp->recv_sges;
err = pthread_spin_lock(&iwuqp->lock);
if (err)
return err;
@ -1898,8 +1864,7 @@ irdma_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr,
}
post_recv.num_sges = ib_wr->num_sge;
post_recv.wr_id = ib_wr->wr_id;
irdma_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge);
post_recv.sg_list = sg_list;
post_recv.sg_list = ib_wr->sg_list;
err = irdma_uk_post_receive(&iwuqp->qp, &post_recv);
if (err) {
*bad_wr = ib_wr;
@ -2023,7 +1988,6 @@ irdma_uresize_cq(struct ibv_cq *cq, int cqe)
u32 cq_pages;
int cqe_needed;
int ret = 0;
bool cqe_64byte_ena;
iwucq = container_of(cq, struct irdma_ucq, verbs_cq.cq);
iwvctx = container_of(cq->context, struct irdma_uvcontext, ibv_ctx);
@ -2035,14 +1999,11 @@ irdma_uresize_cq(struct ibv_cq *cq, int cqe)
if (cqe < uk_attrs->min_hw_cq_size || cqe > uk_attrs->max_hw_cq_size - 1)
return EINVAL;
cqe_64byte_ena = uk_attrs->feature_flags & IRDMA_FEATURE_64_BYTE_CQE ? true : false;
cqe_needed = get_cq_size(cqe, uk_attrs->hw_rev, cqe_64byte_ena);
cqe_needed = get_cq_size(cqe, uk_attrs->hw_rev);
if (cqe_needed == iwucq->cq.cq_size)
return 0;
cq_size = get_cq_total_bytes(cqe_needed, cqe_64byte_ena);
cq_size = get_cq_total_bytes(cqe_needed);
cq_pages = cq_size >> IRDMA_HW_PAGE_SHIFT;
cq_base = irdma_alloc_hw_buf(cq_size);
if (!cq_base)
@ -2078,7 +2039,6 @@ irdma_uresize_cq(struct ibv_cq *cq, int cqe)
goto err_resize;
memcpy(&cq_buf->cq, &iwucq->cq, sizeof(cq_buf->cq));
cq_buf->buf_size = cq_size;
cq_buf->vmr = iwucq->vmr;
iwucq->vmr = new_mr;
irdma_uk_cq_resize(&iwucq->cq, cq_base, cqe_needed);

5
contrib/ofed/libirdma/osdep.h
View File

@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*$FreeBSD$*/
#ifndef _ICRDMA_OSDEP_H_
#define _ICRDMA_OSDEP_H_
@ -78,7 +77,6 @@
/* constants */
#define STATS_TIMER_DELAY 60000
/* a couple of linux size defines */
#define BIT_ULL(a) (1ULL << (a))
#define min(a, b) ((a) > (b) ? (b) : (a))
#ifndef likely
@ -91,9 +89,6 @@
#define __aligned_u64 uint64_t __aligned(8)
#define VLAN_PRIO_SHIFT 13
#if __FreeBSD_version < 1400000
#define IB_USER_VERBS_EX_CMD_MODIFY_QP IB_USER_VERBS_CMD_MODIFY_QP
#endif
/*
* debug definition section

3
sys/dev/irdma/fbsd_kcompat.c
View File

@ -662,8 +662,7 @@ static const struct irdma_sw_stats_tunable_info irdma_sws_list[] = {
"manage_hmc_pm_func_table", 0},
{IRDMA_OP_SUSPEND, "suspend", "suspend", 0},
{IRDMA_OP_RESUME, "resume", "resume", 0},
{IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP, "manage_vchnl_req_pble_bp",
"manage_vchnl_req_pble_bp", 0},
{25, "manage_vchnl_req_pble_bp", "manage_vchnl_req_pble_bp", 0},
{IRDMA_OP_QUERY_FPM_VAL, "query_fpm_val", "query_fpm_val", 0},
{IRDMA_OP_COMMIT_FPM_VAL, "commit_fpm_val", "commit_fpm_val", 0},
{IRDMA_OP_AH_CREATE, "ah_create", "ah_create", 0},

76
sys/dev/irdma/fbsd_kcompat.h
View File

@ -48,33 +48,23 @@
#define from_tasklet(var, callback_tasklet, tasklet_fieldname) \
container_of(callback_tasklet, typeof(*var), tasklet_fieldname)
#endif
#if __FreeBSD_version >= 1400000
#define IRDMA_SET_RDMA_OBJ_SIZE(ib_struct, drv_struct, member) \
(sizeof(struct drv_struct) + \
BUILD_BUG_ON_ZERO(offsetof(struct drv_struct, member)) + \
BUILD_BUG_ON_ZERO( \
!__same_type(((struct drv_struct *)NULL)->member, \
struct ib_struct)))
#endif /* __FreeBSD_version > 1400000 */
#define set_ibdev_dma_device(ibdev, dev) \
ibdev.dma_device = (dev)
#define set_max_sge(props, rf) \
((props)->max_sge = (rf)->sc_dev.hw_attrs.uk_attrs.max_hw_wq_frags)
#define rdma_query_gid(ibdev, port, index, gid) \
ib_get_cached_gid(ibdev, port, index, gid, NULL)
#define kmap(pg) page_address(pg)
#define kmap_local_page(pg) page_address(pg)
#define kunmap(pg)
#define kunmap_local(pg)
#define IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION IB_CQ_FLAGS_TIMESTAMP_COMPLETION
#if __FreeBSD_version < 1400026
#define kc_irdma_destroy_qp(ibqp, udata) irdma_destroy_qp(ibqp)
#else
#define kc_irdma_destroy_qp(ibqp, udata) irdma_destroy_qp(ibqp, udata)
#endif
#ifndef IB_QP_ATTR_STANDARD_BITS
#define IB_QP_ATTR_STANDARD_BITS GENMASK(20, 0)
#endif
@ -135,53 +125,23 @@ irdma_cmid_to_vnet(struct iw_cm_id *cm_id)
return rdma_id->route.addr.dev_addr.net;
}
#if __FreeBSD_version < 1400026
struct ib_cq *irdma_create_cq(struct ib_device *ibdev,
const struct ib_cq_init_attr *attr,
struct ib_ucontext *context,
struct ib_udata *udata);
#else
int irdma_create_cq(struct ib_cq *ibcq,
const struct ib_cq_init_attr *attr,
struct ib_udata *udata);
#endif
struct ib_qp *irdma_create_qp(struct ib_pd *ibpd,
struct ib_qp_init_attr *init_attr,
struct ib_udata *udata);
#if __FreeBSD_version >= 1400026
int irdma_create_ah(struct ib_ah *ib_ah,
struct ib_ah_attr *attr, u32 flags,
struct ib_udata *udata);
int irdma_create_ah_stub(struct ib_ah *ib_ah,
struct ib_ah_attr *attr, u32 flags,
struct ib_udata *udata);
#else
struct ib_ah *irdma_create_ah(struct ib_pd *ibpd,
struct ib_ah_attr *attr,
struct ib_udata *udata);
struct ib_ah *irdma_create_ah_stub(struct ib_pd *ibpd,
struct ib_ah_attr *attr,
struct ib_udata *udata);
#endif
void irdma_ether_copy(u8 *dmac, struct ib_ah_attr *attr);
#if __FreeBSD_version >= 1400026
void irdma_destroy_ah(struct ib_ah *ibah, u32 flags);
void irdma_destroy_ah_stub(struct ib_ah *ibah, u32 flags);
#else
int irdma_destroy_ah(struct ib_ah *ibah);
int irdma_destroy_ah_stub(struct ib_ah *ibah);
#endif
#if __FreeBSD_version < 1400026
int irdma_destroy_qp(struct ib_qp *ibqp);
#else
int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
#endif
#if __FreeBSD_version < 1400026
int irdma_dereg_mr(struct ib_mr *ib_mr);
#else
int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
#endif
int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u8 *speed, u8 *width);
enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev,
u8 port_num);
@ -208,10 +168,6 @@ int irdma_register_qset(struct irdma_sc_vsi *vsi,
void irdma_unregister_qset(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node);
void ib_unregister_device(struct ib_device *ibdev);
#if __FreeBSD_version < 1400026
int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma,
unsigned long pfn, unsigned long size, pgprot_t prot);
#endif
void irdma_disassociate_ucontext(struct ib_ucontext *context);
int kc_irdma_set_roce_cm_info(struct irdma_qp *iwqp,
struct ib_qp_attr *attr,
@ -245,13 +201,8 @@ int irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 len,
struct irdma_mr;
struct irdma_cq;
struct irdma_cq_buf;
#if __FreeBSD_version < 1400026
struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg);
#else
struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg, struct ib_udata *udata);
#endif
int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr,
u16 access);
struct ib_mr *irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len,
@ -264,8 +215,8 @@ void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq);
int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr,
struct irdma_device *iwdev);
void irdma_setup_virt_qp(struct irdma_device *iwdev,
struct irdma_qp *iwqp,
struct irdma_qp_init_info *init_info);
struct irdma_qp *iwqp,
struct irdma_qp_init_info *init_info);
int irdma_setup_kmode_qp(struct irdma_device *iwdev,
struct irdma_qp *iwqp,
struct irdma_qp_init_info *info,
@ -281,35 +232,14 @@ void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp,
struct irdma_qp_host_ctx_info *ctx_info);
int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp);
void irdma_dealloc_push_page(struct irdma_pci_f *rf,
struct irdma_sc_qp *qp);
struct irdma_qp *iwqp);
int irdma_process_resize_list(struct irdma_cq *iwcq, struct irdma_device *iwdev,
struct irdma_cq_buf *lcqe_buf);
#if __FreeBSD_version < 1400026
int irdma_destroy_cq(struct ib_cq *ib_cq);
#else
void irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);
#endif
#if __FreeBSD_version < 1400026
struct ib_ucontext *irdma_alloc_ucontext(struct ib_device *, struct ib_udata *);
#else
int irdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata);
#endif
#if __FreeBSD_version < 1400026
int irdma_dealloc_ucontext(struct ib_ucontext *);
#else
void irdma_dealloc_ucontext(struct ib_ucontext *context);
#endif
#if __FreeBSD_version < 1400026
struct ib_pd *irdma_alloc_pd(struct ib_device *, struct ib_ucontext *,
struct ib_udata *);
#else
int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);
#endif
#if __FreeBSD_version < 1400026
int irdma_dealloc_pd(struct ib_pd *);
#else
void irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
#endif
int irdma_add_gid(struct ib_device *, u8, unsigned int, const union ib_gid *,
const struct ib_gid_attr *, void **);
int irdma_del_gid(struct ib_device *, u8, unsigned int, void **);

4
sys/dev/irdma/icrdma.c
View File

@ -52,7 +52,7 @@
/**
* Driver version
*/
char irdma_driver_version[] = "1.2.17-k";
char irdma_driver_version[] = "1.2.36-k";
/**
* irdma_init_tunable - prepare tunables
@ -245,7 +245,7 @@ irdma_log_invalid_mtu(u16 mtu, struct irdma_sc_dev *dev)
mtu);
else if (mtu < IRDMA_MIN_MTU_IPV6)
irdma_dev_warn(to_ibdev(dev),
"MTU setting [%d] too low for RDMA traffic. Minimum MTU is 1280 for IPv6\\n",
"MTU setting [%d] too low for RDMA traffic. Minimum MTU is 1280 for IPv6\n",
mtu);
}

9
sys/dev/irdma/icrdma_hw.c
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2017 - 2022 Intel Corporation
* Copyright (c) 2017 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@ -76,7 +76,7 @@ static u64 icrdma_masks[IRDMA_MAX_MASKS] = {
ICRDMA_CQPSQ_CQ_CEQID,
ICRDMA_CQPSQ_CQ_CQID,
ICRDMA_COMMIT_FPM_CQCNT,
ICRDMA_CQPSQ_UPESD_HMCFNID
ICRDMA_CQPSQ_UPESD_HMCFNID,
};
static u8 icrdma_shifts[IRDMA_MAX_SHIFTS] = {
@ -86,7 +86,7 @@ static u8 icrdma_shifts[IRDMA_MAX_SHIFTS] = {
ICRDMA_CQPSQ_CQ_CEQID_S,
ICRDMA_CQPSQ_CQ_CQID_S,
ICRDMA_COMMIT_FPM_CQCNT_S,
ICRDMA_CQPSQ_UPESD_HMCFNID_S
ICRDMA_CQPSQ_UPESD_HMCFNID_S,
};
/**
@ -210,8 +210,6 @@ icrdma_init_hw(struct irdma_sc_dev *dev)
dev->hw_regs[i] = (u32 IOMEM *) (hw_addr + icrdma_regs[i]);
}
dev->hw_attrs.max_hw_vf_fpm_id = IRDMA_MAX_VF_FPM_ID;
dev->hw_attrs.first_hw_vf_fpm_id = IRDMA_FIRST_VF_FPM_ID;
for (i = 0; i < IRDMA_MAX_SHIFTS; ++i)
dev->hw_shifts[i] = icrdma_shifts[i];
@ -231,6 +229,7 @@ icrdma_init_hw(struct irdma_sc_dev *dev)
dev->hw_attrs.max_hw_ord = ICRDMA_MAX_ORD_SIZE;
dev->hw_attrs.max_stat_inst = ICRDMA_MAX_STATS_COUNT;
dev->hw_attrs.max_stat_idx = IRDMA_HW_STAT_INDEX_MAX_GEN_2;
dev->hw_attrs.max_hw_device_pages = ICRDMA_MAX_PUSH_PAGE_COUNT;
dev->hw_attrs.uk_attrs.max_hw_wq_frags = ICRDMA_MAX_WQ_FRAGMENT_COUNT;
dev->hw_attrs.uk_attrs.max_hw_read_sges = ICRDMA_MAX_SGE_RD;

3
sys/dev/irdma/icrdma_hw.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2017 - 2022 Intel Corporation
* Copyright (c) 2017 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@ -123,6 +123,7 @@ enum icrdma_device_caps_const {
ICRDMA_MAX_IRD_SIZE = 32,
ICRDMA_MAX_ORD_SIZE = 32,
ICRDMA_MIN_WQ_SIZE = 8 /* WQEs */,
ICRDMA_MAX_PUSH_PAGE_COUNT = 256,
};

11
sys/dev/irdma/irdma-abi.h
View File

@ -54,6 +54,11 @@ enum irdma_memreg_type {
enum {
IRDMA_ALLOC_UCTX_USE_RAW_ATTR = 1 << 0,
IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE = 1 << 1,
};
enum {
IRDMA_CREATE_QP_USE_START_WQE_IDX = 1 << 0,
};
struct irdma_alloc_ucontext_req {
@ -82,6 +87,8 @@ struct irdma_alloc_ucontext_resp {
__u8 hw_rev;
__u8 rsvd2;
__aligned_u64 comp_mask;
__u16 min_hw_wq_size;
__u8 rsvd3[6];
};
struct irdma_alloc_pd_resp {
@ -101,6 +108,7 @@ struct irdma_create_cq_req {
struct irdma_create_qp_req {
__aligned_u64 user_wqe_bufs;
__aligned_u64 user_compl_ctx;
__aligned_u64 comp_mask;
};
struct irdma_mem_reg_req {
@ -130,6 +138,9 @@ struct irdma_create_qp_resp {
__u8 lsmm;
__u8 rsvd;
__u32 qp_caps;
__aligned_u64 comp_mask;
__u8 start_wqe_idx;
__u8 rsvd2[7];
};
struct irdma_modify_qp_resp {

4
sys/dev/irdma/irdma.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2017 - 2023 Intel Corporation
* Copyright (c) 2017 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@ -183,7 +183,7 @@ enum irdma_vers {
IRDMA_GEN_RSVD = 0,
IRDMA_GEN_1 = 1,
IRDMA_GEN_2 = 2,
IRDMA_GEN_MAX = 2,
IRDMA_GEN_MAX = IRDMA_GEN_2,
};
struct irdma_uk_attrs {

49
sys/dev/irdma/irdma_cm.c
View File

@ -1079,14 +1079,14 @@ irdma_parse_mpa(struct irdma_cm_node *cm_node, u8 *buf, u32 *type,
*type = IRDMA_MPA_REQUEST_ACCEPT;
if (len < sizeof(struct ietf_mpa_v1)) {
if (len < sizeof(*mpa_frame)) {
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
"ietf buffer small (%x)\n", len);
return -EINVAL;
}
mpa_frame = (struct ietf_mpa_v1 *)buf;
mpa_hdr_len = sizeof(struct ietf_mpa_v1);
mpa_hdr_len = sizeof(*mpa_frame);
priv_data_len = ntohs(mpa_frame->priv_data_len);
if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) {
@ -1440,6 +1440,7 @@ irdma_send_syn(struct irdma_cm_node *cm_node, u32 sendack)
sizeof(struct option_base) + TCP_OPTIONS_PADDING];
struct irdma_kmem_info opts;
int optionssize = 0;
/* Sending MSS option */
union all_known_options *options;
@ -1615,7 +1616,8 @@ static u8 irdma_iw_get_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4)
}
/**
* irdma_netdev_vlan_ipv6 - Gets the netdev and mac
* irdma_get_vlan_mac_ipv6 - Get the vlan and mac for an IPv6
* address
* @addr: local IPv6 address
* @vlan_id: vlan id for the given IPv6 address
* @mac: mac address for the given IPv6 address
@ -1623,14 +1625,12 @@ static u8 irdma_iw_get_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4)
* Returns the net_device of the IPv6 address and also sets the
* vlan id and mac for that address.
*/
if_t
irdma_netdev_vlan_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, u8 *mac)
void
irdma_get_vlan_mac_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, u8 *mac)
{
if_t ip_dev = NULL;
struct in6_addr laddr6;
#ifdef VIMAGE
struct vnet *vnet = irdma_cmid_to_vnet(cm_id);
#endif
struct vnet *vnet = &init_net;
struct ifaddr *ifa;
u16 scope_id = 0;
@ -1645,10 +1645,9 @@ irdma_netdev_vlan_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, u8 *mac)
scope_id = ntohs(laddr6.__u6_addr.__u6_addr16[1]);
#ifdef VIMAGE
ip_dev = ip6_ifp_find(vnet, laddr6, scope_id);
#else
ip_dev = ip6_ifp_find(&init_net, laddr6, scope_id);
vnet = irdma_cmid_to_vnet(cm_id);
#endif
ip_dev = ip6_ifp_find(vnet, laddr6, scope_id);
if (ip_dev) {
if (vlan_id)
*vlan_id = rdma_vlan_dev_vlan_id(ip_dev);
@ -1656,8 +1655,6 @@ irdma_netdev_vlan_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, u8 *mac)
if (ifa && ifa->ifa_addr && mac)
ether_addr_copy(mac, if_getlladdr(ip_dev));
}
return ip_dev;
}
/**
@ -1668,16 +1665,13 @@ u16
irdma_get_vlan_ipv4(struct iw_cm_id *cm_id, u32 *addr)
{
if_t netdev;
#ifdef VIMAGE
struct vnet *vnet = irdma_cmid_to_vnet(cm_id);
#endif
struct vnet *vnet = &init_net;
u16 vlan_id = 0xFFFF;
#ifdef VIMAGE
netdev = ip_ifp_find(vnet, htonl(addr[0]));
#else
netdev = ip_ifp_find(&init_net, htonl(addr[0]));
vnet = irdma_cmid_to_vnet(cm_id);
#endif
netdev = ip_ifp_find(vnet, htonl(addr[0]));
if (netdev) {
vlan_id = rdma_vlan_dev_vlan_id(netdev);
dev_put(netdev);
@ -2546,7 +2540,7 @@ irdma_handle_syn_pkt(struct irdma_cm_node *cm_node,
u32 inc_sequence;
int optionsize;
optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr);
optionsize = (tcph->th_off << 2) - sizeof(*tcph);
inc_sequence = ntohl(tcph->th_seq);
switch (cm_node->state) {
@ -2613,7 +2607,7 @@ irdma_handle_synack_pkt(struct irdma_cm_node *cm_node,
u32 inc_sequence;
int optionsize;
optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr);
optionsize = (tcph->th_off << 2) - sizeof(*tcph);
inc_sequence = ntohl(tcph->th_seq);
switch (cm_node->state) {
case IRDMA_CM_STATE_SYN_SENT:
@ -2687,7 +2681,7 @@ irdma_handle_ack_pkt(struct irdma_cm_node *cm_node,
int optionsize;
u32 datasize = rbuf->datalen;
optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr);
optionsize = (tcph->th_off << 2) - sizeof(*tcph);
if (irdma_check_seq(cm_node, tcph))
return -EINVAL;
@ -3493,6 +3487,7 @@ irdma_free_lsmm_rsrc(struct irdma_qp *iwqp)
if (iwqp->ietf_mem.va) {
if (iwqp->lsmm_mr)
iwdev->ibdev.dereg_mr(iwqp->lsmm_mr, NULL);
irdma_free_dma_mem(iwdev->rf->sc_dev.hw,
&iwqp->ietf_mem);
iwqp->ietf_mem.va = NULL;
@ -3536,8 +3531,8 @@ irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
cm_node->vlan_id = irdma_get_vlan_ipv4(cm_id, cm_node->loc_addr);
} else {
cm_node->ipv4 = false;
irdma_netdev_vlan_ipv6(cm_id, cm_node->loc_addr,
&cm_node->vlan_id, NULL);
irdma_get_vlan_mac_ipv6(cm_id, cm_node->loc_addr, &cm_node->vlan_id,
NULL);
}
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "Accept vlan_id=%d\n",
cm_node->vlan_id);
@ -3744,7 +3739,7 @@ irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
raddr6->sin6_addr.__u6_addr.__u6_addr32);
cm_info.loc_port = ntohs(laddr6->sin6_port);
cm_info.rem_port = ntohs(raddr6->sin6_port);
irdma_netdev_vlan_ipv6(cm_id, cm_info.loc_addr, &cm_info.vlan_id, NULL);
irdma_get_vlan_mac_ipv6(cm_id, cm_info.loc_addr, &cm_info.vlan_id, NULL);
}
cm_info.cm_id = cm_id;
cm_info.qh_qpid = iwdev->vsi.ilq->qp_id;
@ -3873,8 +3868,8 @@ irdma_create_listen(struct iw_cm_id *cm_id, int backlog)
laddr6->sin6_addr.__u6_addr.__u6_addr32);
cm_info.loc_port = ntohs(laddr6->sin6_port);
if (!IN6_IS_ADDR_UNSPECIFIED(&laddr6->sin6_addr)) {
irdma_netdev_vlan_ipv6(cm_id, cm_info.loc_addr,
&cm_info.vlan_id, NULL);
irdma_get_vlan_mac_ipv6(cm_id, cm_info.loc_addr,
&cm_info.vlan_id, NULL);
} else {
cm_info.vlan_id = 0xFFFF;
wildcard = true;

100
sys/dev/irdma/irdma_ctrl.c
View File

@ -785,14 +785,14 @@ irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 * qp_ctx,
set_64bit_val(qp_ctx, IRDMA_BYTE_144,
FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx));
set_64bit_val(qp_ctx, IRDMA_BYTE_152,
FIELD_PREP(IRDMAQPC_MACADDRESS, irdma_mac_to_u64(roce_info->mac_addr)));
FIELD_PREP(IRDMAQPC_MACADDRESS,
irdma_mac_to_u64(roce_info->mac_addr)));
set_64bit_val(qp_ctx, IRDMA_BYTE_160,
FIELD_PREP(IRDMAQPC_ORDSIZE, roce_info->ord_size) |
FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(roce_info->ird_size)) |
FIELD_PREP(IRDMAQPC_WRRDRSPOK, roce_info->wr_rdresp_en) |
FIELD_PREP(IRDMAQPC_RDOK, roce_info->rd_en) |
FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) |
FIELD_PREP(IRDMAQPC_BINDEN, roce_info->bind_en) |
FIELD_PREP(IRDMAQPC_FASTREGEN, roce_info->fast_reg_en) |
FIELD_PREP(IRDMAQPC_DCQCNENABLE, roce_info->dcqcn_en) |
FIELD_PREP(IRDMAQPC_RCVNOICRC, roce_info->rcv_no_icrc) |
@ -1016,7 +1016,6 @@ irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 * qp_ctx,
FIELD_PREP(IRDMAQPC_WRRDRSPOK, iw->wr_rdresp_en) |
FIELD_PREP(IRDMAQPC_RDOK, iw->rd_en) |
FIELD_PREP(IRDMAQPC_SNDMARKERS, iw->snd_mark_en) |
FIELD_PREP(IRDMAQPC_BINDEN, iw->bind_en) |
FIELD_PREP(IRDMAQPC_FASTREGEN, iw->fast_reg_en) |
FIELD_PREP(IRDMAQPC_PRIVEN, iw->priv_mode_en) |
FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) |
@ -1466,6 +1465,15 @@ irdma_sc_gen_rts_ae(struct irdma_sc_qp *qp)
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
irdma_debug_buf(qp->dev, IRDMA_DEBUG_QP, "CONN EST WQE", wqe,
IRDMA_QP_WQE_MIN_SIZE);
if (qp->qp_uk.start_wqe_idx) {
wqe = qp_uk->sq_base[3].elem;
hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
FIELD_PREP(IRDMAQPSQ_LOCALFENCE, 1) |
FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
}
}
/**
@ -1831,7 +1839,7 @@ irdma_bld_terminate_hdr(struct irdma_sc_qp *qp,
if (copy_len)
irdma_memcpy(termhdr + 1, pkt, copy_len);
return sizeof(struct irdma_terminate_hdr) + copy_len;
return sizeof(*termhdr) + copy_len;
}
/**
@ -2124,10 +2132,6 @@ irdma_sc_gather_stats(struct irdma_sc_cqp *cqp,
IRDMA_CQP_WQE_SIZE * 8);
irdma_sc_cqp_post_sq(cqp);
irdma_debug(cqp->dev, IRDMA_DEBUG_STATS,
"CQP SQ head 0x%x tail 0x%x size 0x%x\n", cqp->sq_ring.head,
cqp->sq_ring.tail, cqp->sq_ring.size);
return 0;
}
@ -2607,17 +2611,18 @@ irdma_sc_cq_create(struct irdma_sc_cq *cq, u64 scratch,
set_64bit_val(wqe, IRDMA_BYTE_0, cq->cq_uk.cq_size);
set_64bit_val(wqe, IRDMA_BYTE_8, RS_64_1(cq, 1));
set_64bit_val(wqe, IRDMA_BYTE_16,
FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold));
set_64bit_val(wqe, IRDMA_BYTE_32, (cq->virtual_map ? 0 : cq->cq_pa));
FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD,
cq->shadow_read_threshold));
set_64bit_val(wqe, IRDMA_BYTE_32, cq->virtual_map ? 0 : cq->cq_pa);
set_64bit_val(wqe, IRDMA_BYTE_40, cq->shadow_area_pa);
set_64bit_val(wqe, IRDMA_BYTE_48,
FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX, (cq->virtual_map ? cq->first_pm_pbl_idx : 0)));
FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX,
cq->virtual_map ? cq->first_pm_pbl_idx : 0));
set_64bit_val(wqe, IRDMA_BYTE_56,
FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |
FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));
hdr = FLD_LS_64(cq->dev, cq->cq_uk.cq_id, IRDMA_CQPSQ_CQ_CQID) |
FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),
FLD_LS_64(cq->dev, cq->ceq_id_valid ? cq->ceq_id : 0,
IRDMA_CQPSQ_CQ_CEQID) |
FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) |
FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) |
@ -2935,10 +2940,12 @@ irdma_sc_parse_fpm_commit_buf(struct irdma_sc_dev *dev, __le64 * buf,
IRDMA_HMC_IW_HDR);
irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_152, info,
IRDMA_HMC_IW_MD);
irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_160, info,
IRDMA_HMC_IW_OOISC);
irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_168, info,
IRDMA_HMC_IW_OOISCFFL);
if (dev->cqp->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) {
irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_160, info,
IRDMA_HMC_IW_OOISC);
irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_168, info,
IRDMA_HMC_IW_OOISCFFL);
}
}
/* searching for the last object in HMC to find the size of the HMC area. */
@ -3071,15 +3078,18 @@ irdma_sc_parse_fpm_query_buf(struct irdma_sc_dev *dev, __le64 * buf,
irdma_sc_decode_fpm_query(buf, 144, obj_info, IRDMA_HMC_IW_HDR);
irdma_sc_decode_fpm_query(buf, 152, obj_info, IRDMA_HMC_IW_MD);
irdma_sc_decode_fpm_query(buf, 160, obj_info, IRDMA_HMC_IW_OOISC);
get_64bit_val(buf, IRDMA_BYTE_168, &temp);
obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp;
obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4;
hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp);
if (!hmc_fpm_misc->ooiscf_block_size &&
obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt)
return -EINVAL;
if (dev->cqp->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) {
irdma_sc_decode_fpm_query(buf, 160, obj_info, IRDMA_HMC_IW_OOISC);
get_64bit_val(buf, IRDMA_BYTE_168, &temp);
obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp;
obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4;
hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp);
if (!hmc_fpm_misc->ooiscf_block_size &&
obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt)
return -EINVAL;
}
return 0;
}
@ -3251,6 +3261,7 @@ irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err)
temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp->ena_vf_count) |
FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp->hmc_profile);
if (hw_rev >= IRDMA_GEN_2)
temp |= FIELD_PREP(IRDMA_CQPHC_EN_REM_ENDPOINT_TRK,
cqp->en_rem_endpoint_trk);
@ -3399,11 +3410,13 @@ irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp, bool free_hwcqp)
void
irdma_sc_ccq_arm(struct irdma_sc_cq *ccq)
{
unsigned long flags;
u64 temp_val;
u16 sw_cq_sel;
u8 arm_next_se;
u8 arm_seq_num;
spin_lock_irqsave(&ccq->dev->cqp_lock, flags);
get_64bit_val(ccq->cq_uk.shadow_area, IRDMA_BYTE_32, &temp_val);
sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
@ -3414,6 +3427,7 @@ irdma_sc_ccq_arm(struct irdma_sc_cq *ccq)
FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, 1);
set_64bit_val(ccq->cq_uk.shadow_area, IRDMA_BYTE_32, temp_val);
spin_unlock_irqrestore(&ccq->dev->cqp_lock, flags);
irdma_wmb(); /* make sure shadow area is updated before arming */
@ -3436,6 +3450,7 @@ irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
u32 error;
u8 polarity;
int ret_code = 0;
unsigned long flags;
if (ccq->cq_uk.avoid_mem_cflct)
cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(&ccq->cq_uk);
@ -3484,7 +3499,9 @@ irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
irdma_wmb(); /* make sure shadow area is updated before moving tail */
spin_lock_irqsave(&cqp->dev->cqp_lock, flags);
IRDMA_RING_MOVE_TAIL(cqp->sq_ring);
spin_unlock_irqrestore(&cqp->dev->cqp_lock, flags);
atomic64_inc(&cqp->completed_ops);
return ret_code;
@ -3809,7 +3826,6 @@ irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq)
if (ceq->reg_cq)
irdma_sc_remove_cq_ctx(ceq, ceq->dev->ccq);
cqp = ceq->dev->cqp;
cqp->process_cqp_sds = irdma_update_sds_noccq;
@ -3834,7 +3850,6 @@ irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch)
if (ret_code)
return ret_code;
}
ret_code = irdma_sc_ceq_create(ceq, scratch, true);
if (!ret_code)
return irdma_sc_cceq_create_done(ceq);
@ -3924,7 +3939,6 @@ irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq)
cq_idx = irdma_sc_find_reg_cq(ceq, cq);
spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
}
IRDMA_RING_MOVE_TAIL(ceq->ceq_ring);
if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring))
ceq->polarity ^= 1;
@ -4155,6 +4169,7 @@ irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq,
case IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST:
case IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
case IRDMA_AE_ROCE_REQ_LENGTH_ERROR:
case IRDMA_AE_INVALID_ARP_ENTRY:
case IRDMA_AE_INVALID_TCP_OPTION_RCVD:
case IRDMA_AE_STALE_ARP_ENTRY:
@ -4707,10 +4722,11 @@ static u32 irdma_est_sd(struct irdma_sc_dev *dev,
u64 sd;
int i;
for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++)
for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) {
if (i != IRDMA_HMC_IW_PBLE)
size += round_up(hmc_info->hmc_obj[i].cnt *
hmc_info->hmc_obj[i].size, 512);
}
pble_info = &hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE];
size += round_up(pble_info->cnt * pble_info->size, 512);
@ -4720,7 +4736,7 @@ static u32 irdma_est_sd(struct irdma_sc_dev *dev,
sd = size >> 21;
if (sd > 0xFFFFFFFF) {
irdma_debug(dev, IRDMA_DEBUG_HMC, "sd overflow[%ld]\n", sd);
sd = 0xFFFFFFFF - 1;
sd = 0xFFFFFFFE;
}
return (u32)sd;
@ -4776,10 +4792,9 @@ irdma_sc_query_rdma_features(struct irdma_sc_cqp *cqp,
int
irdma_get_rdma_features(struct irdma_sc_dev *dev)
{
int ret_code;
int ret_code, byte_idx, feat_type, feat_cnt, feat_idx;
struct irdma_dma_mem feat_buf;
u64 temp;
u16 byte_idx, feat_type, feat_cnt, feat_idx;
feat_buf.size = IRDMA_FEATURE_BUF_SIZE;
feat_buf.va = irdma_allocate_dma_mem(dev->hw, &feat_buf, feat_buf.size,
@ -4822,7 +4837,7 @@ irdma_get_rdma_features(struct irdma_sc_dev *dev)
irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "QUERY RDMA FEATURES", feat_buf.va,
feat_cnt * 8);
for (byte_idx = 0, feat_idx = 0; feat_idx < min(feat_cnt, (u16)IRDMA_MAX_FEATURES);
for (byte_idx = 0, feat_idx = 0; feat_idx < min(feat_cnt, IRDMA_MAX_FEATURES);
feat_idx++, byte_idx += 8) {
get_64bit_val(feat_buf.va, byte_idx, &temp);
feat_type = FIELD_GET(IRDMA_FEATURE_TYPE, temp);
@ -5409,6 +5424,7 @@ void
irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable)
{
u32 reg_val;
reg_val = FIELD_PREP(IRDMA_PFINT_AEQCTL_CAUSE_ENA, enable) |
FIELD_PREP(IRDMA_PFINT_AEQCTL_MSIX_INDX, idx) |
FIELD_PREP(IRDMA_PFINT_AEQCTL_ITR_INDX, IRDMA_IDX_NOITR);
@ -5423,14 +5439,16 @@ irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable)
void
sc_vsi_update_stats(struct irdma_sc_vsi *vsi)
{
struct irdma_gather_stats *gather_stats;
struct irdma_gather_stats *last_gather_stats;
struct irdma_dev_hw_stats *hw_stats = &vsi->pestat->hw_stats;
struct irdma_gather_stats *gather_stats =
vsi->pestat->gather_info.gather_stats_va;
struct irdma_gather_stats *last_gather_stats =
vsi->pestat->gather_info.last_gather_stats_va;
const struct irdma_hw_stat_map *map = vsi->dev->hw_stats_map;
u16 max_stat_idx = vsi->dev->hw_attrs.max_stat_idx;
gather_stats = vsi->pestat->gather_info.gather_stats_va;
last_gather_stats = vsi->pestat->gather_info.last_gather_stats_va;
irdma_update_stats(&vsi->pestat->hw_stats, gather_stats,
last_gather_stats, vsi->dev->hw_stats_map,
vsi->dev->hw_attrs.max_stat_idx);
irdma_update_stats(hw_stats, gather_stats, last_gather_stats,
map, max_stat_idx);
}
/**
@ -5500,7 +5518,6 @@ irdma_sc_dev_init(struct irdma_sc_dev *dev, struct irdma_device_init_info *info)
dev->hw_attrs.max_hw_outbound_msg_size = IRDMA_MAX_OUTBOUND_MSG_SIZE;
dev->hw_attrs.max_mr_size = IRDMA_MAX_MR_SIZE;
dev->hw_attrs.max_hw_inbound_msg_size = IRDMA_MAX_INBOUND_MSG_SIZE;
dev->hw_attrs.max_hw_device_pages = IRDMA_MAX_PUSH_PAGE_COUNT;
dev->hw_attrs.uk_attrs.max_hw_inline = IRDMA_MAX_INLINE_DATA_SIZE;
dev->hw_attrs.max_hw_wqes = IRDMA_MAX_WQ_ENTRIES;
dev->hw_attrs.max_qp_wr = IRDMA_MAX_QP_WRS(IRDMA_MAX_QUANTA_PER_WR);
@ -5528,7 +5545,6 @@ irdma_sc_dev_init(struct irdma_sc_dev *dev, struct irdma_device_init_info *info)
val, db_size);
return -ENODEV;
}
dev->db_addr = dev->hw->hw_addr + (uintptr_t)dev->hw_regs[IRDMA_DB_ADDR_OFFSET];
return ret_code;
}

3
sys/dev/irdma/irdma_defs.h
View File

@ -249,7 +249,6 @@ enum irdma_cqp_op_type {
IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE = 22,
IRDMA_OP_SUSPEND = 23,
IRDMA_OP_RESUME = 24,
IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP = 25,
IRDMA_OP_QUERY_FPM_VAL = 26,
IRDMA_OP_COMMIT_FPM_VAL = 27,
IRDMA_OP_AH_CREATE = 28,
@ -292,7 +291,6 @@ enum irdma_cqp_op_type {
#define IRDMA_CQP_OP_DEALLOC_STAG 0x0d
#define IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE 0x0e
#define IRDMA_CQP_OP_MANAGE_ARP 0x0f
#define IRDMA_CQP_OP_MANAGE_VCHNL_REQ_PBLE_BP 0x10
#define IRDMA_CQP_OP_MANAGE_PUSH_PAGES 0x11
#define IRDMA_CQP_OP_QUERY_RDMA_FEATURES 0x12
#define IRDMA_CQP_OP_UPLOAD_CONTEXT 0x13
@ -849,7 +847,6 @@ enum irdma_cqp_op_type {
#define IRDMA_CQPSQ_UCTX_RAWFORMAT BIT_ULL(61)
#define IRDMA_CQPSQ_UCTX_FREEZEQP_S 62
#define IRDMA_CQPSQ_UCTX_FREEZEQP BIT_ULL(62)
#define IRDMA_CQPSQ_MHMC_VFIDX_S 0
#define IRDMA_CQPSQ_MHMC_VFIDX GENMASK_ULL(15, 0)
#define IRDMA_CQPSQ_MHMC_FREEPMFN_S 62

46
sys/dev/irdma/irdma_hw.c
View File

@ -276,6 +276,7 @@ irdma_process_aeq(struct irdma_pci_f *rf)
switch (info->ae_id) {
struct irdma_cm_node *cm_node;
case IRDMA_AE_LLP_CONNECTION_ESTABLISHED:
cm_node = iwqp->cm_node;
if (cm_node->accept_pend) {
@ -393,6 +394,7 @@ irdma_process_aeq(struct irdma_pci_f *rf)
case IRDMA_AE_LLP_TOO_MANY_RETRIES:
case IRDMA_AE_LCE_QP_CATASTROPHIC:
case IRDMA_AE_LCE_FUNCTION_CATASTROPHIC:
case IRDMA_AE_LLP_TOO_MANY_RNRS:
case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
default:
irdma_dev_err(&iwdev->ibdev,
@ -489,7 +491,7 @@ irdma_save_msix_info(struct irdma_pci_f *rf)
{
struct irdma_qvlist_info *iw_qvlist;
struct irdma_qv_info *iw_qvinfo;
u32 ceq_idx;
u16 ceq_idx;
u32 i;
u32 size;
@ -499,8 +501,8 @@ irdma_save_msix_info(struct irdma_pci_f *rf)
}
size = sizeof(struct irdma_msix_vector) * rf->msix_count;
size += sizeof(struct irdma_qvlist_info);
size += sizeof(struct irdma_qv_info) * rf->msix_count - 1;
size += sizeof(*iw_qvlist);
size += sizeof(*iw_qvinfo) * rf->msix_count - 1;
rf->iw_msixtbl = kzalloc(size, GFP_KERNEL);
if (!rf->iw_msixtbl)
return -ENOMEM;
@ -599,6 +601,13 @@ irdma_destroy_irq(struct irdma_pci_f *rf,
dev->irq_ops->irdma_dis_irq(dev, msix_vec->idx);
irdma_free_irq(rf, msix_vec);
if (rf == dev_id) {
tasklet_kill(&rf->dpc_tasklet);
} else {
struct irdma_ceq *iwceq = (struct irdma_ceq *)dev_id;
tasklet_kill(&iwceq->dpc_tasklet);
}
}
/**
@ -963,13 +972,13 @@ irdma_create_cqp(struct irdma_pci_f *rf)
u16 maj_err, min_err;
int i, status;
cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests), GFP_KERNEL);
memset(cqp->cqp_requests, 0, sqsize * sizeof(*cqp->cqp_requests));
cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests),
GFP_KERNEL);
if (!cqp->cqp_requests)
return -ENOMEM;
cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), GFP_KERNEL);
memset(cqp->scratch_array, 0, sqsize * sizeof(*cqp->scratch_array));
cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array),
GFP_KERNEL);
if (!cqp->scratch_array) {
status = -ENOMEM;
goto err_scratch;
@ -1189,7 +1198,7 @@ fail_intr:
*/
static int
irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,
u32 ceq_id, struct irdma_msix_vector *msix_vec)
u16 ceq_id, struct irdma_msix_vector *msix_vec)
{
int status;
@ -1263,7 +1272,7 @@ irdma_cfg_aeq_vector(struct irdma_pci_f *rf)
*/
static int
irdma_create_ceq(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,
u32 ceq_id, struct irdma_sc_vsi *vsi)
u16 ceq_id, struct irdma_sc_vsi *vsi)
{
int status;
struct irdma_ceq_init_info info = {0};
@ -1380,7 +1389,7 @@ static int
irdma_setup_ceqs(struct irdma_pci_f *rf, struct irdma_sc_vsi *vsi)
{
u32 i;
u32 ceq_id;
u16 ceq_id;
struct irdma_ceq *iwceq;
struct irdma_msix_vector *msix_vec;
int status;
@ -1750,6 +1759,7 @@ void
irdma_rt_deinit_hw(struct irdma_device *iwdev)
{
struct irdma_sc_qp qp = {{0}};
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_INIT, "state = %d\n", iwdev->init_state);
switch (iwdev->init_state) {
@ -2011,6 +2021,7 @@ irdma_ctrl_init_hw(struct irdma_pci_f *rf)
{
struct irdma_sc_dev *dev = &rf->sc_dev;
int status;
do {
status = irdma_setup_init_state(rf);
if (status)
@ -2209,14 +2220,19 @@ irdma_cqp_ce_handler(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)
cqp_request = (struct irdma_cqp_request *)
(uintptr_t)info.scratch;
if (info.error && irdma_cqp_crit_err(dev, cqp_request->info.cqp_cmd,
if (info.error && irdma_cqp_crit_err(dev,
cqp_request->info.cqp_cmd,
info.maj_err_code,
info.min_err_code))
irdma_dev_err(&rf->iwdev->ibdev, "cqp opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n",
info.op_code, info.maj_err_code, info.min_err_code);
irdma_dev_err(&rf->iwdev->ibdev,
"cqp opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n",
info.op_code, info.maj_err_code,
info.min_err_code);
if (cqp_request) {
cqp_request->compl_info.maj_err_code = info.maj_err_code;
cqp_request->compl_info.min_err_code = info.min_err_code;
cqp_request->compl_info.maj_err_code =
info.maj_err_code;
cqp_request->compl_info.min_err_code =
info.min_err_code;
cqp_request->compl_info.op_ret_val = info.op_ret_val;
cqp_request->compl_info.error = info.error;
irdma_complete_cqp_request(&rf->cqp, cqp_request);

688
sys/dev/irdma/irdma_kcompat.c
View File

@ -96,7 +96,6 @@ irdma_del_gid(struct ib_device *device,
return 0;
}
#if __FreeBSD_version >= 1400026
/**
* irdma_alloc_mr - register stag for fast memory registration
* @pd: ibpd pointer
@ -108,18 +107,6 @@ struct ib_mr *
irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg, struct ib_udata *udata)
{
#else
/**
* irdma_alloc_mr - register stag for fast memory registration
* @pd: ibpd pointer
* @mr_type: memory for stag registrion
* @max_num_sg: man number of pages
*/
struct ib_mr *
irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg)
{
#endif
struct irdma_device *iwdev = to_iwdev(pd->device);
struct irdma_pble_alloc *palloc;
struct irdma_pbl *iwpbl;
@ -174,7 +161,6 @@ err:
#define IRDMA_ALLOC_UCTX_MIN_REQ_LEN offsetofend(struct irdma_alloc_ucontext_req, rsvd8)
#define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd)
#if __FreeBSD_version >= 1400026
/**
* irdma_alloc_ucontext - Allocate the user context data structure
* @uctx: context
@ -236,6 +222,8 @@ irdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size;
uresp.hw_rev = uk_attrs->hw_rev;
uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR;
uresp.min_hw_wq_size = uk_attrs->min_hw_wq_size;
uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE;
bar_off =
(uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
@ -269,119 +257,8 @@ ver_error:
req.userspace_ver, IRDMA_ABI_VER);
return -EINVAL;
}
#endif
#if __FreeBSD_version < 1400026
/**
* irdma_alloc_ucontext - Allocate the user context data structure
* @ibdev: ib device pointer
* @udata: user data
*
* This keeps track of all objects associated with a particular
* user-mode client.
*/
struct ib_ucontext *
irdma_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata)
{
struct irdma_device *iwdev = to_iwdev(ibdev);
struct irdma_alloc_ucontext_req req = {0};
struct irdma_alloc_ucontext_resp uresp = {0};
struct irdma_ucontext *ucontext;
struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs;
if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN ||
udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN)
return ERR_PTR(-EINVAL);
if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen)))
return ERR_PTR(-EINVAL);
if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER)
goto ver_error;
ucontext = kzalloc(sizeof(*ucontext), GFP_KERNEL);
if (!ucontext)
return ERR_PTR(-ENOMEM);
ucontext->iwdev = iwdev;
ucontext->abi_ver = req.userspace_ver;
if (req.comp_mask & IRDMA_ALLOC_UCTX_USE_RAW_ATTR)
ucontext->use_raw_attrs = true;
/* GEN_1 legacy support with libi40iw */
if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) {
if (uk_attrs->hw_rev != IRDMA_GEN_1) {
kfree(ucontext);
return ERR_PTR(-EOPNOTSUPP);
}
ucontext->legacy_mode = true;
uresp.max_qps = iwdev->rf->max_qp;
uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds;
uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2;
uresp.kernel_ver = req.userspace_ver;
if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen))) {
kfree(ucontext);
return ERR_PTR(-EFAULT);
}
} else {
u64 bar_off;
uresp.kernel_ver = IRDMA_ABI_VER;
uresp.feature_flags = uk_attrs->feature_flags;
uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags;
uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges;
uresp.max_hw_inline = uk_attrs->max_hw_inline;
uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta;
uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta;
uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk;
uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size;
uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size;
uresp.hw_rev = uk_attrs->hw_rev;
uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR;
bar_off =
(uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
spin_lock_init(&ucontext->mmap_tbl_lock);
ucontext->db_mmap_entry =
irdma_user_mmap_entry_add_hash(ucontext, bar_off,
IRDMA_MMAP_IO_NC,
&uresp.db_mmap_key);
if (!ucontext->db_mmap_entry) {
spin_lock_destroy(&ucontext->mmap_tbl_lock);
kfree(ucontext);
return ERR_PTR(-ENOMEM);
}
if (ib_copy_to_udata(udata, &uresp,
min(sizeof(uresp), udata->outlen))) {
irdma_user_mmap_entry_del_hash(ucontext->db_mmap_entry);
spin_lock_destroy(&ucontext->mmap_tbl_lock);
kfree(ucontext);
return ERR_PTR(-EFAULT);
}
}
INIT_LIST_HEAD(&ucontext->cq_reg_mem_list);
spin_lock_init(&ucontext->cq_reg_mem_list_lock);
INIT_LIST_HEAD(&ucontext->qp_reg_mem_list);
spin_lock_init(&ucontext->qp_reg_mem_list_lock);
INIT_LIST_HEAD(&ucontext->vma_list);
mutex_init(&ucontext->vma_list_mutex);
return &ucontext->ibucontext;
ver_error:
irdma_dev_err(&iwdev->ibdev,
"Invalid userspace driver version detected. Detected version %d, should be %d\n",
req.userspace_ver, IRDMA_ABI_VER);
return ERR_PTR(-EINVAL);
}
#endif
#if __FreeBSD_version >= 1400026
/**
* irdma_dealloc_ucontext - deallocate the user context data structure
* @context: user context created during alloc
@ -395,28 +272,9 @@ irdma_dealloc_ucontext(struct ib_ucontext *context)
return;
}
#endif
#if __FreeBSD_version < 1400026
/**
* irdma_dealloc_ucontext - deallocate the user context data structure
* @context: user context created during alloc
*/
int
irdma_dealloc_ucontext(struct ib_ucontext *context)
{
struct irdma_ucontext *ucontext = to_ucontext(context);
irdma_user_mmap_entry_del_hash(ucontext->db_mmap_entry);
spin_lock_destroy(&ucontext->mmap_tbl_lock);
kfree(ucontext);
return 0;
}
#endif
#define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd)
#if __FreeBSD_version >= 1400026
/**
* irdma_alloc_pd - allocate protection domain
* @pd: protection domain
@ -469,70 +327,8 @@ error:
return err;
}
#endif
#if __FreeBSD_version < 1400026
/**
* irdma_alloc_pd - allocate protection domain
* @ibdev: IB device
* @context: user context
* @udata: user data
*/
struct ib_pd *
irdma_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context, struct ib_udata *udata)
{
struct irdma_pd *iwpd;
struct irdma_device *iwdev = to_iwdev(ibdev);
struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
struct irdma_pci_f *rf = iwdev->rf;
struct irdma_alloc_pd_resp uresp = {0};
struct irdma_sc_pd *sc_pd;
u32 pd_id = 0;
int err;
err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id,
&rf->next_pd);
if (err)
return ERR_PTR(err);
iwpd = kzalloc(sizeof(*iwpd), GFP_KERNEL);
if (!iwpd) {
err = -ENOMEM;
goto free_res;
}
sc_pd = &iwpd->sc_pd;
if (udata) {
struct irdma_ucontext *ucontext = to_ucontext(context);
irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver);
uresp.pd_id = pd_id;
if (ib_copy_to_udata(udata, &uresp,
min(sizeof(uresp), udata->outlen))) {
err = -EFAULT;
goto error;
}
} else {
irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER);
}
spin_lock_init(&iwpd->udqp_list_lock);
INIT_LIST_HEAD(&iwpd->udqp_list);
return &iwpd->ibpd;
error:
kfree(iwpd);
free_res:
irdma_free_rsrc(rf, rf->allocated_pds, pd_id);
return ERR_PTR(err);
}
#endif
#if __FreeBSD_version >= 1400026
void
irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
{
@ -542,20 +338,7 @@ irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id);
}
#endif
#if __FreeBSD_version < 1400026
int
irdma_dealloc_pd(struct ib_pd *ibpd)
{
struct irdma_pd *iwpd = to_iwpd(ibpd);
struct irdma_device *iwdev = to_iwdev(ibpd->device);
irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id);
kfree(iwpd);
return 0;
}
#endif
/**
* irdma_find_qp_update_qs - update QS handle for UD QPs
@ -684,24 +467,40 @@ static int
irdma_create_ah_wait(struct irdma_pci_f *rf,
struct irdma_sc_ah *sc_ah, bool sleep)
{
int ret;
if (!sleep) {
int cnt = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms *
CQP_TIMEOUT_THRESHOLD;
struct irdma_cqp_request *cqp_request =
sc_ah->ah_info.cqp_request;
do {
irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
mdelay(1);
} while (!sc_ah->ah_info.ah_valid && --cnt);
} while (!READ_ONCE(cqp_request->request_done) && --cnt);
if (!cnt)
return -ETIMEDOUT;
if (cnt && !cqp_request->compl_info.op_ret_val) {
irdma_put_cqp_request(&rf->cqp, cqp_request);
sc_ah->ah_info.ah_valid = true;
} else {
ret = !cnt ? -ETIMEDOUT : -EINVAL;
irdma_dev_err(&rf->iwdev->ibdev, "CQP create AH error ret = %d opt_ret_val = %d",
ret, cqp_request->compl_info.op_ret_val);
irdma_put_cqp_request(&rf->cqp, cqp_request);
if (!cnt && !rf->reset) {
rf->reset = true;
rf->gen_ops.request_reset(rf);
}
return ret;
}
}
return 0;
}
#define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd)
#if __FreeBSD_version >= 1400026
/**
* irdma_create_ah - create address handle
* @ib_ah: ptr to AH
@ -787,17 +586,15 @@ irdma_create_ah(struct ib_ah *ib_ah,
goto err_gid_l2;
err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE,
sleep, irdma_gsi_ud_qp_ah_cb, sc_ah);
sleep, NULL, sc_ah);
if (err) {
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP-OP Create AH fail");
goto err_gid_l2;
}
err = irdma_create_ah_wait(rf, sc_ah, sleep);
if (err) {
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP create AH timed out");
if (err)
goto err_gid_l2;
}
if (udata) {
uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
@ -815,7 +612,6 @@ err_gid_l2:
return err;
}
#endif
void
irdma_ether_copy(u8 *dmac, struct ib_ah_attr *attr)
@ -823,162 +619,20 @@ irdma_ether_copy(u8 *dmac, struct ib_ah_attr *attr)
ether_addr_copy(dmac, attr->dmac);
}
#if __FreeBSD_version < 1400026
struct ib_ah *
irdma_create_ah_stub(struct ib_pd *ibpd,
struct ib_ah_attr *attr,
struct ib_udata *udata)
#else
int
irdma_create_ah_stub(struct ib_ah *ib_ah,
struct ib_ah_attr *attr, u32 flags,
struct ib_udata *udata)
#endif
{
#if __FreeBSD_version >= 1400026
return -ENOSYS;
#else
return ERR_PTR(-ENOSYS);
#endif
}
#if __FreeBSD_version >= 1400026
void
irdma_destroy_ah_stub(struct ib_ah *ibah, u32 flags)
{
return;
}
#else
int
irdma_destroy_ah_stub(struct ib_ah *ibah)
{
return -ENOSYS;
}
#endif
#if __FreeBSD_version < 1400026
/**
* irdma_create_ah - create address handle
* @ibpd: ptr to pd
* @attr: address handle attributes
* @udata: user data
*
* returns a pointer to an address handle
*/
struct ib_ah *
irdma_create_ah(struct ib_pd *ibpd,
struct ib_ah_attr *attr,
struct ib_udata *udata)
{
struct irdma_pd *pd = to_iwpd(ibpd);
struct irdma_device *iwdev = to_iwdev(ibpd->device);
struct irdma_ah *ah;
union ib_gid sgid;
struct ib_gid_attr sgid_attr;
struct irdma_pci_f *rf = iwdev->rf;
struct irdma_sc_ah *sc_ah;
u32 ah_id = 0;
struct irdma_ah_info *ah_info;
struct irdma_create_ah_resp uresp = {};
union irdma_sockaddr sgid_addr, dgid_addr;
int err;
u8 dmac[ETHER_ADDR_LEN];
bool sleep = udata ? true : false;
if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN)
return ERR_PTR(-EINVAL);
err = irdma_alloc_rsrc(rf, rf->allocated_ahs,
rf->max_ah, &ah_id, &rf->next_ah);
if (err)
return ERR_PTR(err);
ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
if (!ah) {
irdma_free_rsrc(rf, rf->allocated_ahs, ah_id);
return ERR_PTR(-ENOMEM);
}
ah->pd = pd;
sc_ah = &ah->sc_ah;
sc_ah->ah_info.ah_idx = ah_id;
sc_ah->ah_info.vsi = &iwdev->vsi;
irdma_sc_init_ah(&rf->sc_dev, sc_ah);
ah->sgid_index = attr->grh.sgid_index;
memcpy(&ah->dgid, &attr->grh.dgid, sizeof(ah->dgid));
rcu_read_lock();
err = ib_get_cached_gid(&iwdev->ibdev, attr->port_num,
attr->grh.sgid_index, &sgid, &sgid_attr);
rcu_read_unlock();
if (err) {
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
"GID lookup at idx=%d with port=%d failed\n",
attr->grh.sgid_index, attr->port_num);
err = -EINVAL;
goto err_gid_l2;
}
rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid);
rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid);
ah->av.attrs = *attr;
ah->av.net_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
if (sgid_attr.ndev)
dev_put(sgid_attr.ndev);
ah_info = &sc_ah->ah_info;
ah_info->ah_idx = ah_id;
ah_info->pd_idx = pd->sc_pd.pd_id;
ether_addr_copy(ah_info->mac_addr, if_getlladdr(iwdev->netdev));
if (attr->ah_flags & IB_AH_GRH) {
ah_info->flow_label = attr->grh.flow_label;
ah_info->hop_ttl = attr->grh.hop_limit;
ah_info->tc_tos = attr->grh.traffic_class;
}
if (udata)
ib_resolve_eth_dmac(ibpd->device, attr);
irdma_ether_copy(dmac, attr);
irdma_fill_ah_info(if_getvnet(iwdev->netdev), ah_info, &sgid_attr, &sgid_addr, &dgid_addr,
dmac, ah->av.net_type);
err = irdma_create_ah_vlan_tag(iwdev, pd, ah_info, &sgid_attr, dmac);
if (err)
goto err_gid_l2;
err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE,
sleep, irdma_gsi_ud_qp_ah_cb, sc_ah);
if (err) {
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "CQP-OP Create AH fail");
goto err_gid_l2;
}
err = irdma_create_ah_wait(rf, sc_ah, sleep);
if (err) {
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP create AH timed out");
goto err_gid_l2;
}
if (udata) {
uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen));
if (err) {
irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah,
IRDMA_OP_AH_DESTROY, false, NULL, ah);
goto err_gid_l2;
}
}
return &ah->ibah;
err_gid_l2:
kfree(ah);
irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id);
return ERR_PTR(err);
}
#endif
/**
* irdma_free_qp_rsrc - free up memory resources for qp
@ -992,7 +646,7 @@ irdma_free_qp_rsrc(struct irdma_qp *iwqp)
u32 qp_num = iwqp->ibqp.qp_num;
irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
irdma_dealloc_push_page(rf, &iwqp->sc_qp);
irdma_dealloc_push_page(rf, iwqp);
if (iwqp->sc_qp.vsi) {
irdma_qp_rem_qos(&iwqp->sc_qp);
iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi,
@ -1186,12 +840,17 @@ irdma_create_qp(struct ib_pd *ibpd,
if (udata) {
/* GEN_1 legacy support with libi40iw does not have expanded uresp struct */
if (udata->outlen < sizeof(uresp)) {
if (udata->outlen == IRDMA_CREATE_QP_MIN_RESP_LEN) {
uresp.lsmm = 1;
uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1;
} else {
if (rdma_protocol_iwarp(&iwdev->ibdev, 1))
if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) {
uresp.lsmm = 1;
if (qp->qp_uk.start_wqe_idx) {
uresp.comp_mask |= IRDMA_CREATE_QP_USE_START_WQE_IDX;
uresp.start_wqe_idx = qp->qp_uk.start_wqe_idx;
}
}
}
uresp.actual_sq_size = init_info.qp_uk_init_info.sq_size;
uresp.actual_rq_size = init_info.qp_uk_init_info.rq_size;
@ -1202,7 +861,7 @@ irdma_create_qp(struct ib_pd *ibpd,
min(sizeof(uresp), udata->outlen));
if (err_code) {
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy_to_udata failed\n");
kc_irdma_destroy_qp(&iwqp->ibqp, udata);
irdma_destroy_qp(&iwqp->ibqp, udata);
return ERR_PTR(err_code);
}
}
@ -1221,13 +880,8 @@ error:
* @ibqp: qp's ib pointer also to get to device's qp address
* @udata: user data
*/
#if __FreeBSD_version >= 1400026
int
irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
#else
int
irdma_destroy_qp(struct ib_qp *ibqp)
#endif
{
struct irdma_qp *iwqp = to_iwqp(ibqp);
struct irdma_device *iwdev = iwqp->iwdev;
@ -1270,31 +924,17 @@ free_rsrc:
* @attr: attributes for cq
* @udata: user data
*/
#if __FreeBSD_version >= 1400026
int
irdma_create_cq(struct ib_cq *ibcq,
const struct ib_cq_init_attr *attr,
struct ib_udata *udata)
#else
struct ib_cq *
irdma_create_cq(struct ib_device *ibdev,
const struct ib_cq_init_attr *attr,
struct ib_ucontext *context,
struct ib_udata *udata)
#endif
{
#define IRDMA_CREATE_CQ_MIN_REQ_LEN offsetofend(struct irdma_create_cq_req, user_cq_buf)
#define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size)
#if __FreeBSD_version >= 1400026
struct ib_device *ibdev = ibcq->device;
#endif
struct irdma_device *iwdev = to_iwdev(ibdev);
struct irdma_pci_f *rf = iwdev->rf;
#if __FreeBSD_version >= 1400026
struct irdma_cq *iwcq = to_iwcq(ibcq);
#else
struct irdma_cq *iwcq;
#endif
u32 cq_num = 0;
struct irdma_sc_cq *cq;
struct irdma_sc_dev *dev = &rf->sc_dev;
@ -1308,7 +948,6 @@ irdma_create_cq(struct ib_device *ibdev,
int entries = attr->cqe;
bool cqe_64byte_ena;
#if __FreeBSD_version >= 1400026
err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev);
if (err_code)
return err_code;
@ -1316,27 +955,10 @@ irdma_create_cq(struct ib_device *ibdev,
if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN ||
udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN))
return -EINVAL;
#else
err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev);
if (err_code)
return ERR_PTR(err_code);
if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN ||
udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN))
return ERR_PTR(-EINVAL);
iwcq = kzalloc(sizeof(*iwcq), GFP_KERNEL);
if (!iwcq)
return ERR_PTR(-ENOMEM);
#endif
err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num,
&rf->next_cq);
if (err_code)
#if __FreeBSD_version >= 1400026
return err_code;
#else
goto error;
#endif
cq = &iwcq->sc_cq;
cq->back_cq = iwcq;
atomic_set(&iwcq->refcnt, 1);
@ -1366,11 +988,7 @@ irdma_create_cq(struct ib_device *ibdev,
struct irdma_cq_mr *cqmr_shadow;
iwcq->user_mode = true;
#if __FreeBSD_version >= 1400026
ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
#else
ucontext = to_ucontext(context);
#endif
if (ib_copy_from_udata(&req, udata,
min(sizeof(req), udata->inlen))) {
@ -1499,11 +1117,7 @@ irdma_create_cq(struct ib_device *ibdev,
rf->cq_table[cq_num] = iwcq;
init_completion(&iwcq->free_cq);
#if __FreeBSD_version >= 1400026
return 0;
#else
return &iwcq->ibcq;
#endif
cq_destroy:
irdma_cq_wq_destroy(rf, cq);
cq_kmem_free:
@ -1513,13 +1127,7 @@ cq_kmem_free:
}
cq_free_rsrc:
irdma_free_rsrc(rf, rf->allocated_cqs, cq_num);
#if __FreeBSD_version >= 1400026
return err_code;
#else
error:
kfree(iwcq);
return ERR_PTR(err_code);
#endif
}
/**
@ -1569,7 +1177,6 @@ irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl,
* @ah_flags: destroy flags
*/
#if __FreeBSD_version >= 1400026
void
irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags)
{
@ -1582,33 +1189,10 @@ irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags)
irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs,
ah->sc_ah.ah_info.ah_idx);
}
#endif
#if __FreeBSD_version < 1400026
int
irdma_destroy_ah(struct ib_ah *ibah)
{
struct irdma_device *iwdev = to_iwdev(ibah->device);
struct irdma_ah *ah = to_iwah(ibah);
irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY,
false, NULL, ah);
irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs,
ah->sc_ah.ah_info.ah_idx);
kfree(ah);
return 0;
}
#endif
#if __FreeBSD_version >= 1400026
int
irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
#else
int
irdma_dereg_mr(struct ib_mr *ib_mr)
#endif
{
struct irdma_mr *iwmr = to_iwmr(ib_mr);
struct irdma_device *iwdev = to_iwdev(ib_mr->device);
@ -1618,15 +1202,9 @@ irdma_dereg_mr(struct ib_mr *ib_mr)
if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) {
if (iwmr->region) {
struct irdma_ucontext *ucontext;
#if __FreeBSD_version >= 1400026
ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
#else
struct ib_pd *ibpd = ib_mr->pd;
ucontext = to_ucontext(ibpd->uobject->context);
#endif
irdma_del_memlist(iwmr, ucontext);
}
goto done;
@ -1734,7 +1312,6 @@ kc_irdma_set_roce_cm_info(struct irdma_qp *iwqp, struct ib_qp_attr *attr,
return 0;
}
#if __FreeBSD_version >= 1400026
/**
* irdma_destroy_cq - destroy cq
* @ib_cq: cq pointer
@ -1769,90 +1346,6 @@ irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
irdma_cq_free_rsrc(iwdev->rf, iwcq);
}
#endif
#if __FreeBSD_version < 1400026
/**
* irdma_destroy_cq - destroy cq
* @ib_cq: cq pointer
*/
int
irdma_destroy_cq(struct ib_cq *ib_cq)
{
struct irdma_device *iwdev = to_iwdev(ib_cq->device);
struct irdma_cq *iwcq = to_iwcq(ib_cq);
struct irdma_sc_cq *cq = &iwcq->sc_cq;
struct irdma_sc_dev *dev = cq->dev;
struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id];
struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq);
unsigned long flags;
spin_lock_irqsave(&iwcq->lock, flags);
if (!list_empty(&iwcq->cmpl_generated))
irdma_remove_cmpls_list(iwcq);
if (!list_empty(&iwcq->resize_list))
irdma_process_resize_list(iwcq, iwdev, NULL);
spin_unlock_irqrestore(&iwcq->lock, flags);
irdma_cq_rem_ref(ib_cq);
wait_for_completion(&iwcq->free_cq);
irdma_cq_wq_destroy(iwdev->rf, cq);
spin_lock_irqsave(&iwceq->ce_lock, flags);
irdma_sc_cleanup_ceqes(cq, ceq);
spin_unlock_irqrestore(&iwceq->ce_lock, flags);
irdma_cq_free_rsrc(iwdev->rf, iwcq);
kfree(iwcq);
return 0;
}
#endif
/**
* irdma_alloc_mw - Allocate memory window
* @pd: Protection domain
* @type: Window type
* @udata: user data pointer
*/
struct ib_mw *
irdma_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
struct ib_udata *udata)
{
struct irdma_device *iwdev = to_iwdev(pd->device);
struct irdma_mr *iwmr;
int err_code;
u32 stag;
if (type != IB_MW_TYPE_1 && type != IB_MW_TYPE_2)
return ERR_PTR(-EINVAL);
iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
if (!iwmr)
return ERR_PTR(-ENOMEM);
stag = irdma_create_stag(iwdev);
if (!stag) {
kfree(iwmr);
return ERR_PTR(-ENOMEM);
}
iwmr->stag = stag;
iwmr->ibmw.rkey = stag;
iwmr->ibmw.pd = pd;
iwmr->ibmw.type = type;
iwmr->ibmw.device = pd->device;
err_code = irdma_hw_alloc_mw(iwdev, iwmr);
if (err_code) {
irdma_free_stag(iwdev, stag);
kfree(iwmr);
return ERR_PTR(err_code);
}
return &iwmr->ibmw;
}
/**
* kc_set_loc_seq_num_mss - Set local seq number and mss
* @cm_node: cm node info
@ -1875,113 +1368,6 @@ kc_set_loc_seq_num_mss(struct irdma_cm_node *cm_node)
(cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV6);
}
#if __FreeBSD_version < 1400026
struct irdma_vma_data {
struct list_head list;
struct vm_area_struct *vma;
struct mutex *vma_list_mutex; /* protect the vma_list */
};
/**
* irdma_vma_open -
* @vma: User VMA
*/
static void
irdma_vma_open(struct vm_area_struct *vma)
{
vma->vm_ops = NULL;
}
/**
* irdma_vma_close - Remove vma data from vma list
* @vma: User VMA
*/
static void
irdma_vma_close(struct vm_area_struct *vma)
{
struct irdma_vma_data *vma_data;
vma_data = vma->vm_private_data;
vma->vm_private_data = NULL;
vma_data->vma = NULL;
mutex_lock(vma_data->vma_list_mutex);
list_del(&vma_data->list);
mutex_unlock(vma_data->vma_list_mutex);
kfree(vma_data);
}
static const struct vm_operations_struct irdma_vm_ops = {
.open = irdma_vma_open,
.close = irdma_vma_close
};
/**
* irdma_set_vma_data - Save vma data in context list
* @vma: User VMA
* @context: ib user context
*/
static int
irdma_set_vma_data(struct vm_area_struct *vma,
struct irdma_ucontext *context)
{
struct list_head *vma_head = &context->vma_list;
struct irdma_vma_data *vma_entry;
vma_entry = kzalloc(sizeof(*vma_entry), GFP_KERNEL);
if (!vma_entry)
return -ENOMEM;
vma->vm_private_data = vma_entry;
vma->vm_ops = &irdma_vm_ops;
vma_entry->vma = vma;
vma_entry->vma_list_mutex = &context->vma_list_mutex;
mutex_lock(&context->vma_list_mutex);
list_add(&vma_entry->list, vma_head);
mutex_unlock(&context->vma_list_mutex);
return 0;
}
/**
* irdma_disassociate_ucontext - Disassociate user context
* @context: ib user context
*/
void
irdma_disassociate_ucontext(struct ib_ucontext *context)
{
struct irdma_ucontext *ucontext = to_ucontext(context);
struct irdma_vma_data *vma_data, *n;
struct vm_area_struct *vma;
mutex_lock(&ucontext->vma_list_mutex);
list_for_each_entry_safe(vma_data, n, &ucontext->vma_list, list) {
vma = vma_data->vma;
zap_vma_ptes(vma, vma->vm_start, PAGE_SIZE);
vma->vm_ops = NULL;
list_del(&vma_data->list);
kfree(vma_data);
}
mutex_unlock(&ucontext->vma_list_mutex);
}
int
rdma_user_mmap_io(struct ib_ucontext *context, struct vm_area_struct *vma,
unsigned long pfn, unsigned long size, pgprot_t prot)
{
if (io_remap_pfn_range(vma,
vma->vm_start,
pfn,
size,
prot))
return -EAGAIN;
return irdma_set_vma_data(vma, to_ucontext(context));
}
#else
/**
* irdma_disassociate_ucontext - Disassociate user context
* @context: ib user context
@ -1990,7 +1376,6 @@ void
irdma_disassociate_ucontext(struct ib_ucontext *context)
{
}
#endif
struct ib_device *
ib_device_get_by_netdev(if_t netdev, int driver_id)
@ -2032,7 +1417,7 @@ irdma_query_gid_roce(struct ib_device *ibdev, u8 port, int index,
{
int ret;
ret = rdma_query_gid(ibdev, port, index, gid);
ret = ib_get_cached_gid(ibdev, port, index, gid, NULL);
if (ret == -EAGAIN) {
memcpy(gid, &zgid, sizeof(*gid));
return 0;
@ -2144,7 +1529,7 @@ irdma_query_port(struct ib_device *ibdev, u8 port,
if (rdma_protocol_roce(ibdev, 1)) {
props->gid_tbl_len = 32;
props->port_cap_flags |= IB_PORT_IP_BASED_GIDS;
props->port_cap_flags |= IB_PORT_IP_BASED_GIDS;
props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ;
} else {
props->gid_tbl_len = 1;
@ -2325,9 +1710,6 @@ kc_set_rdma_uverbs_cmd_mask(struct irdma_device *iwdev)
BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) |
BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ) |
BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) |
BIT_ULL(IB_USER_VERBS_CMD_ALLOC_MW) |
BIT_ULL(IB_USER_VERBS_CMD_BIND_MW) |
BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_MW) |
BIT_ULL(IB_USER_VERBS_CMD_POST_RECV) |
BIT_ULL(IB_USER_VERBS_CMD_POST_SEND);
iwdev->ibdev.uverbs_ex_cmd_mask =

15
sys/dev/irdma/irdma_main.h
View File

@ -44,9 +44,7 @@
#include <netinet/if_ether.h>
#include <linux/slab.h>
#include <linux/rculist.h>
#if __FreeBSD_version >= 1400000
#include <rdma/uverbs_ioctl.h>
#endif
#include <rdma/ib_smi.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_pack.h>
@ -185,7 +183,7 @@ struct irdma_cqp_request {
void (*callback_fcn)(struct irdma_cqp_request *cqp_request);
void *param;
struct irdma_cqp_compl_info compl_info;
bool request_done; /* READ/WRITE_ONCE macros operate on it */
u8 request_done; /* READ/WRITE_ONCE macros operate on it */
bool waiting:1;
bool dynamic:1;
};
@ -236,7 +234,7 @@ struct irdma_msix_vector {
u32 idx;
u32 irq;
u32 cpu_affinity;
u32 ceq_id;
u16 ceq_id;
char name[IRDMA_IRQ_NAME_STR_LEN];
struct resource *res;
void *tag;
@ -378,7 +376,6 @@ struct irdma_device {
u32 roce_ackcreds;
u32 vendor_id;
u32 vendor_part_id;
u32 push_mode;
u32 rcv_wnd;
u16 mac_ip_table_idx;
u16 vsi_num;
@ -392,6 +389,7 @@ struct irdma_device {
bool override_ooo:1;
bool override_rd_fence_rate:1;
bool override_rtomin:1;
bool push_mode:1;
bool roce_mode:1;
bool roce_dcqcn_en:1;
bool dcb_vlan_mode:1;
@ -419,7 +417,6 @@ static inline struct irdma_ucontext *to_ucontext(struct ib_ucontext *ibucontext)
return container_of(ibucontext, struct irdma_ucontext, ibucontext);
}
#if __FreeBSD_version >= 1400026
static inline struct irdma_user_mmap_entry *
to_irdma_mmap_entry(struct rdma_user_mmap_entry *rdma_entry)
{
@ -427,7 +424,6 @@ to_irdma_mmap_entry(struct rdma_user_mmap_entry *rdma_entry)
rdma_entry);
}
#endif
static inline struct irdma_pd *to_iwpd(struct ib_pd *ibpd)
{
return container_of(ibpd, struct irdma_pd, ibpd);
@ -586,8 +582,8 @@ void irdma_gen_ae(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,
void irdma_copy_ip_ntohl(u32 *dst, __be32 *src);
void irdma_copy_ip_htonl(__be32 *dst, u32 *src);
u16 irdma_get_vlan_ipv4(struct iw_cm_id *cm_id, u32 *addr);
if_t irdma_netdev_vlan_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id,
u8 *mac);
void irdma_get_vlan_mac_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id,
u8 *mac);
struct ib_mr *irdma_reg_phys_mr(struct ib_pd *ib_pd, u64 addr, u64 size,
int acc, u64 *iova_start);
int irdma_upload_qp_context(struct irdma_qp *iwqp, bool freeze, bool raw);
@ -599,7 +595,6 @@ int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
bool wait,
void (*callback_fcn)(struct irdma_cqp_request *cqp_request),
void *cb_param);
void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request);
void irdma_udqp_qs_worker(struct work_struct *work);
bool irdma_cq_empty(struct irdma_cq *iwcq);
int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,

9
sys/dev/irdma/irdma_puda.c
View File

@ -47,6 +47,7 @@ static void irdma_ieq_tx_compl(struct irdma_sc_vsi *vsi, void *sqwrid);
static void
irdma_ilq_putback_rcvbuf(struct irdma_sc_qp *qp,
struct irdma_puda_buf *buf, u32 wqe_idx);
/**
* irdma_puda_get_listbuf - get buffer from puda list
* @list: list to use for buffers (ILQ or IEQ)
@ -181,7 +182,7 @@ irdma_puda_alloc_buf(struct irdma_sc_dev *dev,
struct irdma_puda_buf *buf;
struct irdma_virt_mem buf_mem;
buf_mem.size = sizeof(struct irdma_puda_buf);
buf_mem.size = sizeof(*buf);
buf_mem.va = kzalloc(buf_mem.size, GFP_KERNEL);
if (!buf_mem.va)
return NULL;
@ -935,6 +936,7 @@ irdma_puda_dele_rsrc(struct irdma_sc_vsi *vsi, enum puda_rsrc_type type,
struct irdma_sc_ceq *ceq;
ceq = vsi->dev->ceq[0];
switch (type) {
case IRDMA_PUDA_RSRC_TYPE_ILQ:
rsrc = vsi->ilq;
@ -1008,7 +1010,7 @@ irdma_puda_allocbufs(struct irdma_puda_rsrc *rsrc, u32 count)
bool virtdma = false;
unsigned long flags;
buf_mem.size = count * sizeof(struct irdma_puda_buf);
buf_mem.size = count * sizeof(*buf);
buf_mem.va = kzalloc(buf_mem.size, GFP_KERNEL);
if (!buf_mem.va) {
irdma_debug(rsrc->dev, IRDMA_DEBUG_PUDA,
@ -1103,7 +1105,7 @@ irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi,
struct irdma_virt_mem *vmem;
info->count = 1;
pudasize = sizeof(struct irdma_puda_rsrc);
pudasize = sizeof(*rsrc);
sqwridsize = info->sq_size * sizeof(struct irdma_sq_uk_wr_trk_info);
rqwridsize = info->rq_size * 8;
switch (info->type) {
@ -1703,6 +1705,7 @@ irdma_ieq_handle_exception(struct irdma_puda_rsrc *ieq,
struct irdma_pfpdu *pfpdu = &qp->pfpdu;
u32 *hw_host_ctx = (u32 *)qp->hw_host_ctx;
u32 rcv_wnd = hw_host_ctx[23];
/* first partial seq # in q2 */
u32 fps = *(u32 *)(qp->q2_buf + Q2_FPSN_OFFSET);
struct list_head *rxlist = &pfpdu->rxlist;

2
sys/dev/irdma/irdma_puda.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2015 - 2023 Intel Corporation
* Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU

29
sys/dev/irdma/irdma_type.h
View File

@ -42,6 +42,8 @@
#include "irdma_hmc.h"
#include "irdma_uda.h"
#include "irdma_ws.h"
#include "irdma_pble.h"
enum irdma_debug_flag {
IRDMA_DEBUG_NONE = 0x00000000,
IRDMA_DEBUG_ERR = 0x00000001,
@ -70,6 +72,8 @@ enum irdma_debug_flag {
IRDMA_DEBUG_ALL = 0xFFFFFFFF,
};
#define RSVD_OFFSET 0xFFFFFFFF
enum irdma_page_size {
IRDMA_PAGE_SIZE_4K = 0,
IRDMA_PAGE_SIZE_2M,
@ -472,7 +476,7 @@ struct irdma_sc_cq {
bool virtual_map:1;
bool check_overflow:1;
bool ceq_id_valid:1;
bool tph_en;
bool tph_en:1;
};
struct irdma_sc_qp {
@ -520,9 +524,9 @@ struct irdma_sc_qp {
};
struct irdma_stats_inst_info {
bool use_hmc_fcn_index;
u16 hmc_fn_id;
u16 stats_idx;
bool use_hmc_fcn_index:1;
};
struct irdma_up_info {
@ -570,7 +574,7 @@ struct irdma_qos {
u8 traffic_class;
u8 rel_bw;
u8 prio_type;
bool valid;
bool valid:1;
};
struct irdma_config_check {
@ -623,7 +627,6 @@ struct irdma_sc_dev {
__le64 *fpm_query_buf;
__le64 *fpm_commit_buf;
struct irdma_hw *hw;
u8 IOMEM *db_addr;
u32 IOMEM *wqe_alloc_db;
u32 IOMEM *cq_arm_db;
u32 IOMEM *aeq_alloc_db;
@ -649,8 +652,6 @@ struct irdma_sc_dev {
u32 debug_mask;
u16 num_vfs;
u16 hmc_fn_id;
u8 vf_id;
bool vchnl_up:1;
bool ceq_valid:1;
u8 pci_rev;
int (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri);
@ -666,7 +667,7 @@ struct irdma_modify_cq_info {
u8 pbl_chunk_size;
u32 first_pm_pbl_idx;
bool virtual_map:1;
bool check_overflow;
bool check_overflow:1;
bool cq_resize:1;
};
@ -676,7 +677,7 @@ struct irdma_create_qp_info {
bool cq_num_valid:1;
bool arp_cache_idx_valid:1;
bool mac_valid:1;
bool force_lpb;
bool force_lpb:1;
u8 next_iwarp_state;
};
@ -709,7 +710,7 @@ struct irdma_ccq_cqe_info {
u16 maj_err_code;
u16 min_err_code;
u8 op_code;
bool error;
bool error:1;
};
struct irdma_qos_tc_info {
@ -751,7 +752,7 @@ struct irdma_vsi_init_info {
struct irdma_vsi_stats_info {
struct irdma_vsi_pestat *pestat;
u8 fcn_id;
bool alloc_stats_inst;
bool alloc_stats_inst:1;
};
struct irdma_device_init_info {
@ -789,7 +790,7 @@ struct irdma_aeq_init_info {
u32 *aeqe_base;
void *pbl_list;
u32 elem_cnt;
bool virtual_map;
bool virtual_map:1;
u8 pbl_chunk_size;
u32 first_pm_pbl_idx;
u32 msix_idx;
@ -856,7 +857,6 @@ struct irdma_roce_offload_info {
bool dcqcn_en:1;
bool rcv_no_icrc:1;
bool wr_rdresp_en:1;
bool bind_en:1;
bool fast_reg_en:1;
bool priv_mode_en:1;
bool rd_en:1;
@ -888,7 +888,6 @@ struct irdma_iwarp_offload_info {
bool snd_mark_en:1;
bool rcv_mark_en:1;
bool wr_rdresp_en:1;
bool bind_en:1;
bool fast_reg_en:1;
bool priv_mode_en:1;
bool rd_en:1;
@ -1134,12 +1133,12 @@ struct irdma_add_arp_cache_entry_info {
u8 mac_addr[ETHER_ADDR_LEN];
u32 reach_max;
u16 arp_index;
bool permanent;
bool permanent:1;
};
struct irdma_apbvt_info {
u16 port;
bool add;
bool add:1;
};
struct irdma_qhash_table_info {

1
sys/dev/irdma/irdma_uda.h
View File

@ -43,6 +43,7 @@ struct irdma_sc_cqp;
struct irdma_ah_info {
struct irdma_sc_vsi *vsi;
struct irdma_cqp_request *cqp_request;
u32 pd_idx;
u32 dst_arpindex;
u32 dest_ip_addr[4];

80
sys/dev/irdma/irdma_uk.c
View File

@ -45,16 +45,16 @@
* @valid: The wqe valid
*/
static void
irdma_set_fragment(__le64 * wqe, u32 offset, struct irdma_sge *sge,
irdma_set_fragment(__le64 * wqe, u32 offset, struct ib_sge *sge,
u8 valid)
{
if (sge) {
set_64bit_val(wqe, offset,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
FIELD_PREP(IRDMAQPSQ_VALID, valid) |
FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->len) |
FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->stag));
FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->length) |
FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->lkey));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
@ -71,14 +71,14 @@ irdma_set_fragment(__le64 * wqe, u32 offset, struct irdma_sge *sge,
*/
static void
irdma_set_fragment_gen_1(__le64 * wqe, u32 offset,
struct irdma_sge *sge, u8 valid)
struct ib_sge *sge, u8 valid)
{
if (sge) {
set_64bit_val(wqe, offset,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->len) |
FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->stag));
FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->length) |
FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->lkey));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8, 0);
@ -209,8 +209,7 @@ irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 * wqe, u16 quanta,
if (IRDMA_RING_CURRENT_HEAD(qp->initial_ring) !=
IRDMA_RING_CURRENT_TAIL(qp->sq_ring) &&
!qp->push_mode) {
if (post_sq)
irdma_uk_qp_post_wr(qp);
irdma_uk_qp_post_wr(qp);
} else {
push = (__le64 *) ((uintptr_t)qp->push_wqe +
(wqe_idx & 0x7) * 0x20);
@ -338,7 +337,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
return -EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
total_size += op_info->lo_sg_list[i].len;
total_size += op_info->lo_sg_list[i].length;
read_fence |= info->read_fence;
@ -357,7 +356,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
set_64bit_val(wqe, IRDMA_BYTE_16,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
if (info->imm_data_valid) {
set_64bit_val(wqe, IRDMA_BYTE_0,
@ -386,7 +385,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
++addl_frag_cnt;
}
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) |
@ -437,7 +436,7 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
return -EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
total_size += op_info->lo_sg_list[i].len;
total_size += op_info->lo_sg_list[i].length;
ret_code = irdma_fragcnt_to_quanta_sq(op_info->num_lo_sges, &quanta);
if (ret_code)
@ -475,8 +474,8 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
++addl_frag_cnt;
}
set_64bit_val(wqe, IRDMA_BYTE_16,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
FIELD_PREP(IRDMAQPSQ_OPCODE,
@ -525,7 +524,7 @@ irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
return -EINVAL;
for (i = 0; i < op_info->num_sges; i++)
total_size += op_info->sg_list[i].len;
total_size += op_info->sg_list[i].length;
if (info->imm_data_valid)
frag_cnt = op_info->num_sges + 1;
@ -604,15 +603,15 @@ irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
* @polarity: compatibility parameter
*/
static void
irdma_copy_inline_data_gen_1(u8 *wqe, struct irdma_sge *sge_list,
irdma_copy_inline_data_gen_1(u8 *wqe, struct ib_sge *sge_list,
u32 num_sges, u8 polarity)
{
u32 quanta_bytes_remaining = 16;
u32 i;
for (i = 0; i < num_sges; i++) {
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off;
u32 sge_len = sge_list[i].len;
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
u32 sge_len = sge_list[i].length;
while (sge_len) {
u32 bytes_copied;
@ -651,7 +650,7 @@ static inline u16 irdma_inline_data_size_to_quanta_gen_1(u32 data_size) {
* @polarity: polarity of wqe valid bit
*/
static void
irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list,
irdma_copy_inline_data(u8 *wqe, struct ib_sge *sge_list,
u32 num_sges, u8 polarity)
{
u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;
@ -662,8 +661,8 @@ irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list,
wqe += 8;
for (i = 0; i < num_sges; i++) {
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off;
u32 sge_len = sge_list[i].len;
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
u32 sge_len = sge_list[i].length;
while (sge_len) {
u32 bytes_copied;
@ -743,7 +742,7 @@ irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
return -EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
total_size += op_info->lo_sg_list[i].len;
total_size += op_info->lo_sg_list[i].length;
if (unlikely(total_size > qp->max_inline_data))
return -EINVAL;
@ -756,9 +755,9 @@ irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
read_fence |= info->read_fence;
set_64bit_val(wqe, IRDMA_BYTE_16,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) |
@ -814,7 +813,7 @@ irdma_uk_inline_send(struct irdma_qp_uk *qp,
return -EINVAL;
for (i = 0; i < op_info->num_sges; i++)
total_size += op_info->sg_list[i].len;
total_size += op_info->sg_list[i].length;
if (unlikely(total_size > qp->max_inline_data))
return -EINVAL;
@ -879,7 +878,7 @@ irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
u64 hdr;
u32 wqe_idx;
bool local_fence = false;
struct irdma_sge sge = {0};
struct ib_sge sge = {0};
u16 quanta = IRDMA_QP_WQE_MIN_QUANTA;
info->push_wqe = qp->push_db ? true : false;
@ -890,7 +889,7 @@ irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
if (!wqe)
return -ENOSPC;
sge.stag = op_info->target_stag;
sge.lkey = op_info->target_stag;
qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, &sge, 0);
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
@ -1327,8 +1326,7 @@ irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
IRDMA_RING_MOVE_TAIL(cq->cq_ring);
set_64bit_val(cq->shadow_area, IRDMA_BYTE_0,
IRDMA_RING_CURRENT_HEAD(cq->cq_ring));
memset(info, 0,
sizeof(struct irdma_cq_poll_info));
memset(info, 0, sizeof(*info));
return irdma_uk_cq_poll_cmpl(cq, info);
}
}
@ -1484,10 +1482,12 @@ irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
int
irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth)
{
u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift;
*sqdepth = irdma_round_up_wq((sq_size << shift) + IRDMA_SQ_RSVD);
if (*sqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
*sqdepth = uk_attrs->min_hw_wq_size << shift;
if (*sqdepth < min_size)
*sqdepth = min_size;
else if (*sqdepth > uk_attrs->max_hw_wq_quanta)
return -EINVAL;
@ -1501,10 +1501,12 @@ irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *s
int
irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth)
{
u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift;
*rqdepth = irdma_round_up_wq((rq_size << shift) + IRDMA_RQ_RSVD);
if (*rqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
*rqdepth = uk_attrs->min_hw_wq_size << shift;
if (*rqdepth < min_size)
*rqdepth = min_size;
else if (*rqdepth > uk_attrs->max_hw_rq_quanta)
return -EINVAL;
@ -1535,9 +1537,10 @@ irdma_setup_connection_wqes(struct irdma_qp_uk *qp,
{
u16 move_cnt = 1;
if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)
if (info->start_wqe_idx)
move_cnt = info->start_wqe_idx;
else if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)
move_cnt = 3;
qp->conn_wqes = move_cnt;
IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt);
IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt);
@ -1582,6 +1585,7 @@ irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,
{
bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2 ? true : false;
int status;
irdma_get_wqe_shift(ukinfo->uk_attrs,
imm_support ? ukinfo->max_sq_frag_cnt + 1 :
ukinfo->max_sq_frag_cnt,
@ -1676,6 +1680,8 @@ irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info)
qp->wqe_ops = iw_wqe_uk_ops_gen_1;
else
qp->wqe_ops = iw_wqe_uk_ops;
qp->start_wqe_idx = info->start_wqe_idx;
return ret_code;
}

33
sys/dev/irdma/irdma_user.h
View File

@ -35,6 +35,8 @@
#ifndef IRDMA_USER_H
#define IRDMA_USER_H
#include <rdma/ib_verbs.h>
#define irdma_handle void *
#define irdma_adapter_handle irdma_handle
#define irdma_qp_handle irdma_handle
@ -48,7 +50,7 @@
#define irdma_access_privileges u32
#define irdma_physical_fragment u64
#define irdma_address_list u64 *
#define irdma_sgl struct irdma_sge *
#define irdma_sgl struct ib_sge *
#define IRDMA_MAX_MR_SIZE 0x200000000000ULL
@ -78,8 +80,6 @@
#define IRDMA_OP_TYPE_REC_IMM 0x3f
#define IRDMA_FLUSH_MAJOR_ERR 1
#define IRDMA_SRQFLUSH_RSVD_MAJOR_ERR 0xfffe
/* Async Events codes */
#define IRDMA_AE_AMP_UNALLOCATED_STAG 0x0102
#define IRDMA_AE_AMP_INVALID_STAG 0x0103
@ -140,6 +140,7 @@
#define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313
#define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314
#define IRDMA_AE_ROCE_RSP_LENGTH_ERROR 0x0316
#define IRDMA_AE_ROCE_REQ_LENGTH_ERROR 0x0318
#define IRDMA_AE_ROCE_EMPTY_MCG 0x0380
#define IRDMA_AE_ROCE_BAD_MC_IP_ADDR 0x0381
#define IRDMA_AE_ROCE_BAD_MC_QPID 0x0382
@ -160,6 +161,7 @@
#define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b
#define IRDMA_AE_LLP_DOUBT_REACHABILITY 0x050c
#define IRDMA_AE_LLP_CONNECTION_ESTABLISHED 0x050e
#define IRDMA_AE_LLP_TOO_MANY_RNRS 0x050f
#define IRDMA_AE_RESOURCE_EXHAUSTION 0x0520
#define IRDMA_AE_RESET_SENT 0x0601
#define IRDMA_AE_TERMINATE_SENT 0x0602
@ -199,8 +201,7 @@ enum irdma_device_caps_const {
IRDMA_MAX_OUTBOUND_MSG_SIZE = 65537,
/* 64K +1 */
IRDMA_MAX_INBOUND_MSG_SIZE = 65537,
IRDMA_MAX_PUSH_PAGE_COUNT = 1024,
IRDMA_MAX_PE_ENA_VF_COUNT = 32,
IRDMA_MAX_PE_ENA_VF_COUNT = 32,
IRDMA_MAX_VF_FPM_ID = 47,
IRDMA_MAX_SQ_PAYLOAD_SIZE = 2145386496,
IRDMA_MAX_INLINE_DATA_SIZE = 101,
@ -227,6 +228,7 @@ enum irdma_flush_opcode {
FLUSH_RETRY_EXC_ERR,
FLUSH_MW_BIND_ERR,
FLUSH_REM_INV_REQ_ERR,
FLUSH_RNR_RETRY_EXC_ERR,
};
enum irdma_qp_event_type {
@ -280,12 +282,6 @@ struct irdma_cq_uk;
struct irdma_qp_uk_init_info;
struct irdma_cq_uk_init_info;
struct irdma_sge {
irdma_tagged_offset tag_off;
u32 len;
irdma_stag stag;
};
struct irdma_ring {
volatile u32 head;
volatile u32 tail; /* effective tail */
@ -317,13 +313,13 @@ struct irdma_post_rq_info {
struct irdma_rdma_write {
irdma_sgl lo_sg_list;
u32 num_lo_sges;
struct irdma_sge rem_addr;
struct ib_sge rem_addr;
};
struct irdma_rdma_read {
irdma_sgl lo_sg_list;
u32 num_lo_sges;
struct irdma_sge rem_addr;
struct ib_sge rem_addr;
};
struct irdma_bind_window {
@ -422,9 +418,9 @@ int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
bool post_sq);
struct irdma_wqe_uk_ops {
void (*iw_copy_inline_data)(u8 *dest, struct irdma_sge *sge_list, u32 num_sges, u8 polarity);
void (*iw_copy_inline_data)(u8 *dest, struct ib_sge *sge_list, u32 num_sges, u8 polarity);
u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct irdma_sge *sge,
void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct ib_sge *sge,
u8 valid);
void (*iw_set_mw_bind_wqe)(__le64 *wqe,
struct irdma_bind_window *op_info);
@ -490,6 +486,7 @@ struct irdma_qp_uk {
u8 rwqe_polarity;
u8 rq_wqe_size;
u8 rq_wqe_size_multiplier;
u8 start_wqe_idx;
bool deferred_flag:1;
bool push_mode:1; /* whether the last post wqe was pushed */
bool push_dropped:1;
@ -537,6 +534,7 @@ struct irdma_qp_uk_init_info {
u32 sq_depth;
u32 rq_depth;
u8 first_sq_wq;
u8 start_wqe_idx;
u8 type;
u8 sq_shift;
u8 rq_shift;
@ -625,10 +623,15 @@ static inline struct qp_err_code irdma_ae_to_qp_err_code(u16 ae_id)
case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
case IRDMA_AE_ROCE_REQ_LENGTH_ERROR:
case IRDMA_AE_IB_REMOTE_OP_ERROR:
qp_err.flush_code = FLUSH_REM_OP_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_LLP_TOO_MANY_RNRS:
qp_err.flush_code = FLUSH_RNR_RETRY_EXC_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_LCE_QP_CATASTROPHIC:
qp_err.flush_code = FLUSH_FATAL_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;

52
sys/dev/irdma/irdma_utils.c
View File

@ -164,6 +164,7 @@ static const struct ae_desc ae_desc_list[] = {
"Connection error: Doubt reachability (usually occurs after the max number of retries has been reached)"},
{IRDMA_AE_LLP_CONNECTION_ESTABLISHED,
"iWARP event: Connection established"},
{IRDMA_AE_LLP_TOO_MANY_RNRS, "RoCEv2: Too many RNR NACKs"},
{IRDMA_AE_RESOURCE_EXHAUSTION,
"QP error: Resource exhaustion"},
{IRDMA_AE_RESET_SENT,
@ -437,11 +438,11 @@ static void
irdma_free_pending_cqp_request(struct irdma_cqp *cqp,
struct irdma_cqp_request *cqp_request)
{
if (cqp_request->waiting) {
cqp_request->compl_info.error = true;
WRITE_ONCE(cqp_request->request_done, true);
cqp_request->compl_info.error = true;
WRITE_ONCE(cqp_request->request_done, true);
if (cqp_request->waiting)
wake_up(&cqp_request->waitq);
}
wait_event_timeout(cqp->remove_wq,
atomic_read(&cqp_request->refcnt) == 1, 1000);
irdma_put_cqp_request(cqp, cqp_request);
@ -558,8 +559,6 @@ static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = {
[IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd",
[IRDMA_OP_SUSPEND] = "Suspend QP Cmd",
[IRDMA_OP_RESUME] = "Resume QP Cmd",
[IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP] =
"Manage Virtual Channel Requester Function PBLE Backing Pages Cmd",
[IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd",
[IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd",
[IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd",
@ -591,7 +590,7 @@ static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = {
{0xffff, 0x8007, "Modify QP Bad Close"},
{0xffff, 0x8009, "LLP Closed"},
{0xffff, 0x800a, "Reset Not Sent"},
{0xffff, 0x200, "Failover Pending"}
{0xffff, 0x0200, "Failover Pending"},
};
/**
@ -1055,15 +1054,16 @@ irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
/**
* irdma_dealloc_push_page - free a push page for qp
* @rf: RDMA PCI function
* @qp: hardware control qp
* @iwqp: QP pointer
*/
void
irdma_dealloc_push_page(struct irdma_pci_f *rf,
struct irdma_sc_qp *qp)
struct irdma_qp *iwqp)
{
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
int status;
struct irdma_sc_qp *qp = &iwqp->sc_qp;
if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)
return;
@ -1564,16 +1564,6 @@ irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi)
del_timer_sync(&devstat->stats_timer);
}
/**
* irdma_process_stats - Checking for wrap and update stats
* @pestat: stats structure pointer
*/
static inline void
irdma_process_stats(struct irdma_vsi_pestat *pestat)
{
sc_vsi_update_stats(pestat->vsi);
}
/**
* irdma_process_cqp_stats - Checking for wrap and update stats
* @cqp_request: cqp_request structure pointer
@ -1583,7 +1573,7 @@ irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request)
{
struct irdma_vsi_pestat *pestat = cqp_request->param;
irdma_process_stats(pestat);
sc_vsi_update_stats(pestat->vsi);
}
/**
@ -1619,7 +1609,7 @@ irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
cqp_request->callback_fcn = irdma_process_cqp_stats;
status = irdma_handle_cqp_op(rf, cqp_request);
if (wait)
irdma_process_stats(pestat);
sc_vsi_update_stats(pestat->vsi);
irdma_put_cqp_request(&rf->cqp, cqp_request);
return status;
@ -1814,6 +1804,10 @@ irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
cqp_info->cqp_cmd = cmd;
cqp_info->post_sq = 1;
if (cmd == IRDMA_OP_AH_CREATE) {
if (!wait)
irdma_get_cqp_request(cqp_request);
sc_ah->ah_info.cqp_request = cqp_request;
cqp_info->in.u.ah_create.info = sc_ah->ah_info;
cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request;
cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp;
@ -1948,21 +1942,6 @@ irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)
kfree(ah);
}
/**
* irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP
* @cqp_request: pointer to cqp_request of create AH
*/
void
irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request)
{
struct irdma_sc_ah *sc_ah = cqp_request->param;
if (!cqp_request->compl_info.op_ret_val)
sc_ah->ah_info.ah_valid = true;
else
sc_ah->ah_info.ah_valid = false;
}
/**
* irdma_prm_add_pble_mem - add moemory to pble resources
* @pprm: pble resource manager
@ -2010,6 +1989,7 @@ irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
struct list_head *chunk_entry = (&pprm->clist)->next;
u32 offset;
unsigned long flags;
*vaddr = NULL;
*fpm_addr = 0;

389
sys/dev/irdma/irdma_verbs.c
View File

@ -72,7 +72,6 @@ irdma_query_device(struct ib_device *ibdev,
props->max_cq = rf->max_cq - rf->used_cqs;
props->max_cqe = rf->max_cqe - 1;
props->max_mr = rf->max_mr - rf->used_mrs;
props->max_mw = props->max_mr;
props->max_pd = rf->max_pd - rf->used_pds;
props->max_sge_rd = hw_attrs->uk_attrs.max_hw_read_sges;
props->max_qp_rd_atom = hw_attrs->max_hw_ird;
@ -107,16 +106,10 @@ irdma_mmap_legacy(struct irdma_ucontext *ucontext,
pfn = ((uintptr_t)ucontext->iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET] +
pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT;
#if __FreeBSD_version >= 1400026
return rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, PAGE_SIZE,
pgprot_noncached(vma->vm_page_prot), NULL);
#else
return rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, PAGE_SIZE,
pgprot_noncached(vma->vm_page_prot));
#endif
}
#if __FreeBSD_version >= 1400026
static void
irdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
{
@ -149,104 +142,6 @@ irdma_user_mmap_entry_insert(struct irdma_ucontext *ucontext, u64 bar_offset,
return &entry->rdma_entry;
}
#else
static inline bool
find_key_in_mmap_tbl(struct irdma_ucontext *ucontext, u64 key)
{
struct irdma_user_mmap_entry *entry;
HASH_FOR_EACH_POSSIBLE(ucontext->mmap_hash_tbl, entry, hlist, key) {
if (entry->pgoff_key == key)
return true;
}
return false;
}
struct irdma_user_mmap_entry *
irdma_user_mmap_entry_add_hash(struct irdma_ucontext *ucontext, u64 bar_offset,
enum irdma_mmap_flag mmap_flag, u64 *mmap_offset)
{
struct irdma_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
unsigned long flags;
int retry_cnt = 0;
if (!entry)
return NULL;
entry->bar_offset = bar_offset;
entry->mmap_flag = mmap_flag;
entry->ucontext = ucontext;
do {
get_random_bytes(&entry->pgoff_key, sizeof(entry->pgoff_key));
/* The key is a page offset */
entry->pgoff_key >>= PAGE_SHIFT;
/* In the event of a collision in the hash table, retry a new key */
spin_lock_irqsave(&ucontext->mmap_tbl_lock, flags);
if (!find_key_in_mmap_tbl(ucontext, entry->pgoff_key)) {
HASH_ADD(ucontext->mmap_hash_tbl, &entry->hlist, entry->pgoff_key);
spin_unlock_irqrestore(&ucontext->mmap_tbl_lock, flags);
goto hash_add_done;
}
spin_unlock_irqrestore(&ucontext->mmap_tbl_lock, flags);
} while (retry_cnt++ < 10);
irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
"mmap table add failed: Cannot find a unique key\n");
kfree(entry);
return NULL;
hash_add_done:
/* libc mmap uses a byte offset */
*mmap_offset = entry->pgoff_key << PAGE_SHIFT;
return entry;
}
static struct irdma_user_mmap_entry *
irdma_find_user_mmap_entry(struct irdma_ucontext *ucontext,
struct vm_area_struct *vma)
{
struct irdma_user_mmap_entry *entry;
unsigned long flags;
if (vma->vm_end - vma->vm_start != PAGE_SIZE)
return NULL;
spin_lock_irqsave(&ucontext->mmap_tbl_lock, flags);
HASH_FOR_EACH_POSSIBLE(ucontext->mmap_hash_tbl, entry, hlist, vma->vm_pgoff) {
if (entry->pgoff_key == vma->vm_pgoff) {
spin_unlock_irqrestore(&ucontext->mmap_tbl_lock, flags);
return entry;
}
}
spin_unlock_irqrestore(&ucontext->mmap_tbl_lock, flags);
return NULL;
}
void
irdma_user_mmap_entry_del_hash(struct irdma_user_mmap_entry *entry)
{
struct irdma_ucontext *ucontext;
unsigned long flags;
if (!entry)
return;
ucontext = entry->ucontext;
spin_lock_irqsave(&ucontext->mmap_tbl_lock, flags);
HASH_DEL(ucontext->mmap_hash_tbl, &entry->hlist);
spin_unlock_irqrestore(&ucontext->mmap_tbl_lock, flags);
kfree(entry);
}
#endif
/**
* irdma_mmap - user memory map
* @context: context created during alloc
@ -255,9 +150,7 @@ irdma_user_mmap_entry_del_hash(struct irdma_user_mmap_entry *entry)
static int
irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
{
#if __FreeBSD_version >= 1400026
struct rdma_user_mmap_entry *rdma_entry;
#endif
struct irdma_user_mmap_entry *entry;
struct irdma_ucontext *ucontext;
u64 pfn;
@ -269,7 +162,6 @@ irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
if (ucontext->legacy_mode)
return irdma_mmap_legacy(ucontext, vma);
#if __FreeBSD_version >= 1400026
rdma_entry = rdma_user_mmap_entry_get(&ucontext->ibucontext, vma);
if (!rdma_entry) {
irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
@ -279,15 +171,6 @@ irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
}
entry = to_irdma_mmap_entry(rdma_entry);
#else
entry = irdma_find_user_mmap_entry(ucontext, vma);
if (!entry) {
irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
"pgoff[0x%lx] does not have valid entry\n",
vma->vm_pgoff);
return -EINVAL;
}
#endif
irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
"bar_offset [0x%lx] mmap_flag [%d]\n", entry->bar_offset,
entry->mmap_flag);
@ -297,24 +180,14 @@ irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
switch (entry->mmap_flag) {
case IRDMA_MMAP_IO_NC:
#if __FreeBSD_version >= 1400026
ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE,
pgprot_noncached(vma->vm_page_prot),
rdma_entry);
#else
ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE,
pgprot_noncached(vma->vm_page_prot));
#endif
break;
case IRDMA_MMAP_IO_WC:
#if __FreeBSD_version >= 1400026
ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE,
pgprot_writecombine(vma->vm_page_prot),
rdma_entry);
#else
ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE,
pgprot_writecombine(vma->vm_page_prot));
#endif
break;
default:
ret = -EINVAL;
@ -324,9 +197,7 @@ irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
"bar_offset [0x%lx] mmap_flag[%d] err[%d]\n",
entry->bar_offset, entry->mmap_flag, ret);
#if __FreeBSD_version >= 1400026
rdma_user_mmap_entry_put(rdma_entry);
#endif
return ret;
}
@ -428,19 +299,11 @@ void
irdma_remove_push_mmap_entries(struct irdma_qp *iwqp)
{
if (iwqp->push_db_mmap_entry) {
#if __FreeBSD_version >= 1400026
rdma_user_mmap_entry_remove(iwqp->push_db_mmap_entry);
#else
irdma_user_mmap_entry_del_hash(iwqp->push_db_mmap_entry);
#endif
iwqp->push_db_mmap_entry = NULL;
}
if (iwqp->push_wqe_mmap_entry) {
#if __FreeBSD_version >= 1400026
rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry);
#else
irdma_user_mmap_entry_del_hash(iwqp->push_wqe_mmap_entry);
#endif
iwqp->push_wqe_mmap_entry = NULL;
}
}
@ -458,36 +321,19 @@ irdma_setup_push_mmap_entries(struct irdma_ucontext *ucontext,
bar_off = irdma_compute_push_wqe_offset(iwdev, iwqp->sc_qp.push_idx);
#if __FreeBSD_version >= 1400026
iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext,
bar_off, IRDMA_MMAP_IO_WC,
push_wqe_mmap_key);
#else
iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_add_hash(ucontext, bar_off,
IRDMA_MMAP_IO_WC,
push_wqe_mmap_key);
#endif
if (!iwqp->push_wqe_mmap_entry)
return -ENOMEM;
/* push doorbell page */
bar_off += IRDMA_HW_PAGE_SIZE;
#if __FreeBSD_version >= 1400026
iwqp->push_db_mmap_entry = irdma_user_mmap_entry_insert(ucontext,
bar_off, IRDMA_MMAP_IO_NC,
push_db_mmap_key);
#else
iwqp->push_db_mmap_entry = irdma_user_mmap_entry_add_hash(ucontext, bar_off,
IRDMA_MMAP_IO_NC,
push_db_mmap_key);
#endif
if (!iwqp->push_db_mmap_entry) {
#if __FreeBSD_version >= 1400026
rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry);
#else
irdma_user_mmap_entry_del_hash(iwqp->push_wqe_mmap_entry);
#endif
return -ENOMEM;
}
@ -535,11 +381,7 @@ irdma_setup_umode_qp(struct ib_udata *udata,
struct irdma_qp_init_info *info,
struct ib_qp_init_attr *init_attr)
{
#if __FreeBSD_version >= 1400026
struct irdma_ucontext *ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
#else
struct irdma_ucontext *ucontext = to_ucontext(iwqp->iwpd->ibpd.uobject->context);
#endif
struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info;
struct irdma_create_qp_req req = {0};
unsigned long flags;
@ -598,6 +440,9 @@ irdma_setup_umode_qp(struct ib_udata *udata,
ukinfo->sq_size = ukinfo->sq_depth >> ukinfo->sq_shift;
ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift;
}
if (req.comp_mask & IRDMA_CREATE_QP_USE_START_WQE_IDX &&
iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)
ukinfo->start_wqe_idx = 4;
irdma_setup_virt_qp(iwdev, iwqp, info);
return 0;
@ -740,7 +585,6 @@ irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp,
roce_info->rd_en = true;
roce_info->wr_rdresp_en = true;
roce_info->bind_en = true;
roce_info->dcqcn_en = false;
roce_info->rtomin = iwdev->roce_rtomin;
@ -772,7 +616,6 @@ irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp,
ether_addr_copy(iwarp_info->mac_addr, if_getlladdr(iwdev->netdev));
iwarp_info->rd_en = true;
iwarp_info->wr_rdresp_en = true;
iwarp_info->bind_en = true;
iwarp_info->ecn_en = true;
iwarp_info->rtomin = 5;
@ -803,6 +646,8 @@ irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr,
if (init_attr->cap.max_inline_data > uk_attrs->max_hw_inline ||
init_attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags ||
init_attr->cap.max_send_wr > uk_attrs->max_hw_wq_quanta ||
init_attr->cap.max_recv_wr > uk_attrs->max_hw_rq_quanta ||
init_attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags)
return -EINVAL;
@ -857,8 +702,6 @@ irdma_get_ib_acc_flags(struct irdma_qp *iwqp)
}
if (iwqp->roce_info.rd_en)
acc_flags |= IB_ACCESS_REMOTE_READ;
if (iwqp->roce_info.bind_en)
acc_flags |= IB_ACCESS_MW_BIND;
} else {
if (iwqp->iwarp_info.wr_rdresp_en) {
acc_flags |= IB_ACCESS_LOCAL_WRITE;
@ -866,8 +709,6 @@ irdma_get_ib_acc_flags(struct irdma_qp *iwqp)
}
if (iwqp->iwarp_info.rd_en)
acc_flags |= IB_ACCESS_REMOTE_READ;
if (iwqp->iwarp_info.bind_en)
acc_flags |= IB_ACCESS_MW_BIND;
}
return acc_flags;
}
@ -1267,11 +1108,7 @@ irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
if (udata && udata->outlen && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
struct irdma_ucontext *ucontext;
#if __FreeBSD_version >= 1400026
ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
#else
ucontext = to_ucontext(ibqp->uobject->context);
#endif
if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX &&
!iwqp->push_wqe_mmap_entry &&
!irdma_setup_push_mmap_entries(ucontext, iwqp,
@ -1284,7 +1121,8 @@ irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
udata->outlen));
if (ret) {
irdma_remove_push_mmap_entries(iwqp);
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
irdma_debug(&iwdev->rf->sc_dev,
IRDMA_DEBUG_VERBS,
"copy_to_udata failed\n");
return ret;
}
@ -1517,11 +1355,7 @@ irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
struct irdma_ucontext *ucontext;
#if __FreeBSD_version >= 1400026
ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
#else
ucontext = to_ucontext(ibqp->uobject->context);
#endif
if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX &&
!iwqp->push_wqe_mmap_entry &&
!irdma_setup_push_mmap_entries(ucontext, iwqp,
@ -1660,11 +1494,7 @@ irdma_resize_cq(struct ib_cq *ibcq, int entries,
if (udata) {
struct irdma_resize_cq_req req = {};
struct irdma_ucontext *ucontext =
#if __FreeBSD_version >= 1400026
rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
#else
to_ucontext(ibcq->uobject->context);
#endif
/* CQ resize not supported with legacy GEN_1 libi40iw */
if (ucontext->legacy_mode)
@ -1758,8 +1588,10 @@ error:
/**
* irdma_get_mr_access - get hw MR access permissions from IB access flags
* @access: IB access flags
* @hw_rev: Hardware version
*/
static inline u16 irdma_get_mr_access(int access){
static inline u16 irdma_get_mr_access(int access, u8 hw_rev)
{
u16 hw_access = 0;
hw_access |= (access & IB_ACCESS_LOCAL_WRITE) ?
@ -1768,8 +1600,6 @@ static inline u16 irdma_get_mr_access(int access){
IRDMA_ACCESS_FLAGS_REMOTEWRITE : 0;
hw_access |= (access & IB_ACCESS_REMOTE_READ) ?
IRDMA_ACCESS_FLAGS_REMOTEREAD : 0;
hw_access |= (access & IB_ACCESS_MW_BIND) ?
IRDMA_ACCESS_FLAGS_BIND_WINDOW : 0;
hw_access |= (access & IB_ZERO_BASED) ?
IRDMA_ACCESS_FLAGS_ZERO_BASED : 0;
hw_access |= IRDMA_ACCESS_FLAGS_LOCALREAD;
@ -2004,81 +1834,6 @@ irdma_handle_q_mem(struct irdma_device *iwdev,
return err;
}
/**
* irdma_hw_alloc_mw - create the hw memory window
* @iwdev: irdma device
* @iwmr: pointer to memory window info
*/
int
irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr)
{
struct irdma_mw_alloc_info *info;
struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);
if (!cqp_request)
return -ENOMEM;
cqp_info = &cqp_request->info;
info = &cqp_info->in.u.mw_alloc.info;
memset(info, 0, sizeof(*info));
if (iwmr->ibmw.type == IB_MW_TYPE_1)
info->mw_wide = true;
info->page_size = PAGE_SIZE;
info->mw_stag_index = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S;
info->pd_id = iwpd->sc_pd.pd_id;
info->remote_access = true;
cqp_info->cqp_cmd = IRDMA_OP_MW_ALLOC;
cqp_info->post_sq = 1;
cqp_info->in.u.mw_alloc.dev = &iwdev->rf->sc_dev;
cqp_info->in.u.mw_alloc.scratch = (uintptr_t)cqp_request;
status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);
return status;
}
/**
* irdma_dealloc_mw - Dealloc memory window
* @ibmw: memory window structure.
*/
static int
irdma_dealloc_mw(struct ib_mw *ibmw)
{
struct ib_pd *ibpd = ibmw->pd;
struct irdma_pd *iwpd = to_iwpd(ibpd);
struct irdma_mr *iwmr = to_iwmr((struct ib_mr *)ibmw);
struct irdma_device *iwdev = to_iwdev(ibmw->device);
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
struct irdma_dealloc_stag_info *info;
cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);
if (!cqp_request)
return -ENOMEM;
cqp_info = &cqp_request->info;
info = &cqp_info->in.u.dealloc_stag.info;
memset(info, 0, sizeof(*info));
info->pd_id = iwpd->sc_pd.pd_id;
info->stag_idx = RS_64_1(ibmw->rkey, IRDMA_CQPSQ_STAG_IDX_S);
info->mr = false;
cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG;
cqp_info->post_sq = 1;
cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev;
cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request;
irdma_handle_cqp_op(iwdev->rf, cqp_request);
irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);
irdma_free_stag(iwdev, iwmr->stag);
kfree(iwmr);
return 0;
}
/**
* irdma_hw_alloc_stag - cqp command to allocate stag
* @iwdev: irdma device
@ -2199,7 +1954,8 @@ irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr,
stag_info->stag_key = (u8)iwmr->stag;
stag_info->total_len = iwmr->len;
stag_info->all_memory = (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) ? true : false;
stag_info->access_rights = irdma_get_mr_access(access);
stag_info->access_rights = irdma_get_mr_access(access,
iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev);
stag_info->pd_id = iwpd->sc_pd.pd_id;
if (stag_info->access_rights & IRDMA_ACCESS_FLAGS_ZERO_BASED)
stag_info->addr_type = IRDMA_ADDR_TYPE_ZERO_BASED;
@ -2273,14 +2029,18 @@ irdma_free_iwmr(struct irdma_mr *iwmr)
}
/*
* irdma_reg_user_mr_type_mem - Handle memory registration @iwmr - irdma mr @access - access rights
* irdma_reg_user_mr_type_mem - Handle memory registration
* @iwmr - irdma mr
* @access - access rights
* @create_stag - flag to create stag or not
*/
static int
irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access)
irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access,
bool create_stag)
{
struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
struct irdma_pbl *iwpbl = &iwmr->iwpbl;
u32 stag;
u32 stag = 0;
int err;
u8 lvl;
@ -2299,15 +2059,17 @@ irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access)
}
}
stag = irdma_create_stag(iwdev);
if (!stag) {
err = -ENOMEM;
goto free_pble;
}
if (create_stag) {
stag = irdma_create_stag(iwdev);
if (!stag) {
err = -ENOMEM;
goto free_pble;
}
iwmr->stag = stag;
iwmr->ibmr.rkey = stag;
iwmr->ibmr.lkey = stag;
iwmr->stag = stag;
iwmr->ibmr.rkey = stag;
iwmr->ibmr.lkey = stag;
}
iwmr->access = access;
err = irdma_hwreg_mr(iwdev, iwmr, access);
if (err)
@ -2316,7 +2078,8 @@ irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access)
return 0;
err_hwreg:
irdma_free_stag(iwdev, stag);
if (stag)
irdma_free_stag(iwdev, stag);
free_pble:
if (iwpbl->pble_alloc.level != PBLE_LEVEL_0 && iwpbl->pbl_allocated)
@ -2351,11 +2114,7 @@ irdma_reg_user_mr_type_qp(struct irdma_mem_reg_req req,
if (err)
return err;
#if __FreeBSD_version >= 1400026
ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
#else
ucontext = to_ucontext(iwmr->ibpd.pd->uobject->context);
#endif
spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
iwpbl->on_list = true;
@ -2390,11 +2149,7 @@ irdma_reg_user_mr_type_cq(struct irdma_mem_reg_req req,
if (err)
return err;
#if __FreeBSD_version >= 1400026
ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
#else
ucontext = to_ucontext(iwmr->ibmr.pd->uobject->context);
#endif
spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
iwpbl->on_list = true;
@ -2463,7 +2218,7 @@ irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
break;
case IRDMA_MEMREG_TYPE_MEM:
err = irdma_reg_user_mr_type_mem(iwmr, access);
err = irdma_reg_user_mr_type_mem(iwmr, access, true);
if (err)
goto error;
@ -2540,10 +2295,8 @@ irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len,
{
struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
struct irdma_pbl *iwpbl = &iwmr->iwpbl;
struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
struct ib_pd *pd = iwmr->ibmr.pd;
struct ib_umem *region;
u8 lvl;
int err;
region = ib_umem_get(pd->uobject->context, start, len, iwmr->access, 0);
@ -2564,35 +2317,14 @@ irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len,
iwmr->page_cnt = irdma_ib_umem_num_dma_blocks(region, iwmr->page_size,
virt);
lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0;
err = irdma_setup_pbles(iwdev->rf, iwmr, lvl);
err = irdma_reg_user_mr_type_mem(iwmr, iwmr->access, false);
if (err)
goto error;
if (lvl) {
err = irdma_check_mr_contiguous(palloc,
iwmr->page_size);
if (err) {
irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
iwpbl->pbl_allocated = false;
}
}
err = irdma_hwreg_mr(iwdev, iwmr, iwmr->access);
if (err)
goto error;
goto err;
return &iwmr->ibmr;
error:
if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) {
irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
iwpbl->pbl_allocated = false;
}
err:
ib_umem_release(region);
iwmr->region = NULL;
return ERR_PTR(err);
}
@ -2699,25 +2431,6 @@ irdma_del_memlist(struct irdma_mr *iwmr,
}
}
/**
* irdma_copy_sg_list - copy sg list for qp
* @sg_list: copied into sg_list
* @sgl: copy from sgl
* @num_sges: count of sg entries
*/
static void
irdma_copy_sg_list(struct irdma_sge *sg_list, struct ib_sge *sgl,
int num_sges)
{
unsigned int i;
for (i = 0; i < num_sges; i++) {
sg_list[i].tag_off = sgl[i].addr;
sg_list[i].len = sgl[i].length;
sg_list[i].stag = sgl[i].lkey;
}
}
/**
* irdma_post_send - kernel application wr
* @ibqp: qp ptr for wr
@ -2778,7 +2491,7 @@ irdma_post_send(struct ib_qp *ibqp,
}
info.op.send.num_sges = ib_wr->num_sge;
info.op.send.sg_list = (struct irdma_sge *)ib_wr->sg_list;
info.op.send.sg_list = ib_wr->sg_list;
if (iwqp->ibqp.qp_type == IB_QPT_UD ||
iwqp->ibqp.qp_type == IB_QPT_GSI) {
ah = to_iwah(ud_wr(ib_wr)->ah);
@ -2809,8 +2522,8 @@ irdma_post_send(struct ib_qp *ibqp,
info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
info.op.rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
info.op.rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey;
info.op.rdma_write.rem_addr.addr = rdma_wr(ib_wr)->remote_addr;
info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey;
if (ib_wr->send_flags & IB_SEND_INLINE)
err = irdma_uk_inline_rdma_write(ukqp, &info, false);
else
@ -2826,8 +2539,8 @@ irdma_post_send(struct ib_qp *ibqp,
break;
}
info.op_type = IRDMA_OP_TYPE_RDMA_READ;
info.op.rdma_read.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
info.op.rdma_read.rem_addr.stag = rdma_wr(ib_wr)->rkey;
info.op.rdma_read.rem_addr.addr = rdma_wr(ib_wr)->remote_addr;
info.op.rdma_read.rem_addr.lkey = rdma_wr(ib_wr)->rkey;
info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list;
info.op.rdma_read.num_lo_sges = ib_wr->num_sge;
err = irdma_uk_rdma_read(ukqp, &info, inv_stag, false);
@ -2845,7 +2558,9 @@ irdma_post_send(struct ib_qp *ibqp,
stag_info.signaled = info.signaled;
stag_info.read_fence = info.read_fence;
stag_info.access_rights = irdma_get_mr_access(reg_wr(ib_wr)->access);
stag_info.access_rights =
irdma_get_mr_access(reg_wr(ib_wr)->access,
dev->hw_attrs.uk_attrs.hw_rev);
stag_info.stag_key = reg_wr(ib_wr)->key & 0xff;
stag_info.stag_idx = reg_wr(ib_wr)->key >> 8;
stag_info.page_size = reg_wr(ib_wr)->mr->page_size;
@ -2907,7 +2622,6 @@ irdma_post_recv(struct ib_qp *ibqp,
struct irdma_qp *iwqp = to_iwqp(ibqp);
struct irdma_qp_uk *ukqp = &iwqp->sc_qp.qp_uk;
struct irdma_post_rq_info post_recv = {0};
struct irdma_sge *sg_list = iwqp->sg_list;
unsigned long flags;
int err = 0;
@ -2920,8 +2634,7 @@ irdma_post_recv(struct ib_qp *ibqp,
}
post_recv.num_sges = ib_wr->num_sge;
post_recv.wr_id = ib_wr->wr_id;
irdma_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge);
post_recv.sg_list = sg_list;
post_recv.sg_list = ib_wr->sg_list;
err = irdma_uk_post_receive(ukqp, &post_recv);
if (err) {
irdma_debug(&iwqp->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
@ -3336,7 +3049,7 @@ irdma_attach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid)
if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) {
irdma_copy_ip_ntohl(ip_addr,
sgid_addr.saddr_in6.sin6_addr.__u6_addr.__u6_addr32);
irdma_netdev_vlan_ipv6(iwqp->cm_id, ip_addr, &vlan_id, NULL);
irdma_get_vlan_mac_ipv6(iwqp->cm_id, ip_addr, &vlan_id, NULL);
ipv4 = false;
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
"qp_id=%d, IP6address=%x:%x:%x:%x\n", ibqp->qp_num,
@ -3539,9 +3252,7 @@ irdma_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr)
return 0;
}
static if_t
irdma_get_netdev(struct ib_device *ibdev, u8 port_num)
{
static if_t irdma_get_netdev(struct ib_device *ibdev, u8 port_num){
struct irdma_device *iwdev = to_iwdev(ibdev);
if (iwdev->netdev) {
@ -3557,7 +3268,6 @@ irdma_set_device_ops(struct ib_device *ibdev)
{
struct ib_device *dev_ops = ibdev;
#if __FreeBSD_version >= 1400000
dev_ops->ops.driver_id = RDMA_DRIVER_I40IW;
dev_ops->ops.size_ib_ah = IRDMA_SET_RDMA_OBJ_SIZE(ib_ah, irdma_ah, ibah);
dev_ops->ops.size_ib_cq = IRDMA_SET_RDMA_OBJ_SIZE(ib_cq, irdma_cq, ibcq);
@ -3566,15 +3276,12 @@ irdma_set_device_ops(struct ib_device *ibdev)
irdma_ucontext,
ibucontext);
#endif /* __FreeBSD_version >= 1400000 */
dev_ops->alloc_hw_stats = irdma_alloc_hw_stats;
dev_ops->alloc_mr = irdma_alloc_mr;
dev_ops->alloc_mw = irdma_alloc_mw;
dev_ops->alloc_pd = irdma_alloc_pd;
dev_ops->alloc_ucontext = irdma_alloc_ucontext;
dev_ops->create_cq = irdma_create_cq;
dev_ops->create_qp = irdma_create_qp;
dev_ops->dealloc_mw = irdma_dealloc_mw;
dev_ops->dealloc_pd = irdma_dealloc_pd;
dev_ops->dealloc_ucontext = irdma_dealloc_ucontext;
dev_ops->dereg_mr = irdma_dereg_mr;
@ -3587,9 +3294,7 @@ irdma_set_device_ops(struct ib_device *ibdev)
dev_ops->get_netdev = irdma_get_netdev;
dev_ops->map_mr_sg = irdma_map_mr_sg;
dev_ops->mmap = irdma_mmap;
#if __FreeBSD_version >= 1400026
dev_ops->mmap_free = irdma_mmap_free;
#endif
dev_ops->poll_cq = irdma_poll_cq;
dev_ops->post_recv = irdma_post_recv;
dev_ops->post_send = irdma_post_send;
@ -3607,6 +3312,7 @@ static void
irdma_set_device_mcast_ops(struct ib_device *ibdev)
{
struct ib_device *dev_ops = ibdev;
dev_ops->attach_mcast = irdma_attach_mcast;
dev_ops->detach_mcast = irdma_detach_mcast;
}
@ -3615,6 +3321,7 @@ static void
irdma_set_device_roce_ops(struct ib_device *ibdev)
{
struct ib_device *dev_ops = ibdev;
dev_ops->create_ah = irdma_create_ah;
dev_ops->destroy_ah = irdma_destroy_ah;
dev_ops->get_link_layer = irdma_get_link_layer;

26
sys/dev/irdma/irdma_verbs.h
View File

@ -48,13 +48,7 @@
struct irdma_ucontext {
struct ib_ucontext ibucontext;
struct irdma_device *iwdev;
#if __FreeBSD_version >= 1400026
struct rdma_user_mmap_entry *db_mmap_entry;
#else
struct irdma_user_mmap_entry *db_mmap_entry;
DECLARE_HASHTABLE(mmap_hash_tbl, 6);
spinlock_t mmap_tbl_lock; /* protect mmap hash table entries */
#endif
struct list_head cq_reg_mem_list;
spinlock_t cq_reg_mem_list_lock; /* protect CQ memory list */
struct list_head qp_reg_mem_list;
@ -215,13 +209,8 @@ struct irdma_qp {
struct irdma_cq *iwscq;
struct irdma_cq *iwrcq;
struct irdma_pd *iwpd;
#if __FreeBSD_version >= 1400026
struct rdma_user_mmap_entry *push_wqe_mmap_entry;
struct rdma_user_mmap_entry *push_db_mmap_entry;
#else
struct irdma_user_mmap_entry *push_wqe_mmap_entry;
struct irdma_user_mmap_entry *push_db_mmap_entry;
#endif
struct irdma_qp_host_ctx_info ctx_info;
union {
struct irdma_iwarp_offload_info iwarp_info;
@ -263,7 +252,7 @@ struct irdma_qp {
struct irdma_dma_mem host_ctx;
struct timer_list terminate_timer;
struct irdma_pbl *iwpbl;
struct irdma_sge *sg_list;
struct ib_sge *sg_list;
struct irdma_dma_mem q2_ctx_mem;
struct irdma_dma_mem ietf_mem;
struct completion free_qp;
@ -292,13 +281,7 @@ enum irdma_mmap_flag {
};
struct irdma_user_mmap_entry {
#if __FreeBSD_version >= 1400026
struct rdma_user_mmap_entry rdma_entry;
#else
struct irdma_ucontext *ucontext;
struct hlist_node hlist;
u64 pgoff_key; /* Used to compute offset (in bytes) returned to user libc's mmap */
#endif
u64 bar_offset;
u8 mmap_flag;
};
@ -399,16 +382,9 @@ static inline void irdma_mcast_mac_v6(u32 *ip_addr, u8 *mac)
ether_addr_copy(mac, mac6);
}
#if __FreeBSD_version >= 1400026
struct rdma_user_mmap_entry*
irdma_user_mmap_entry_insert(struct irdma_ucontext *ucontext, u64 bar_offset,
enum irdma_mmap_flag mmap_flag, u64 *mmap_offset);
#else
struct irdma_user_mmap_entry *
irdma_user_mmap_entry_add_hash(struct irdma_ucontext *ucontext, u64 bar_offset,
enum irdma_mmap_flag mmap_flag, u64 *mmap_offset);
void irdma_user_mmap_entry_del_hash(struct irdma_user_mmap_entry *entry);
#endif
int irdma_ib_register_device(struct irdma_device *iwdev);
void irdma_ib_unregister_device(struct irdma_device *iwdev);
void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event);

4
sys/dev/irdma/irdma_ws.c
View File

@ -57,7 +57,7 @@ irdma_alloc_node(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *node;
u16 node_index = 0;
ws_mem.size = sizeof(struct irdma_ws_node);
ws_mem.size = sizeof(*node);
ws_mem.va = kzalloc(ws_mem.size, GFP_KERNEL);
if (!ws_mem.va)
return NULL;
@ -109,7 +109,7 @@ irdma_free_node(struct irdma_sc_vsi *vsi,
irdma_free_ws_node_id(vsi->dev, node->index);
ws_mem.va = node;
ws_mem.size = sizeof(struct irdma_ws_node);
ws_mem.size = sizeof(*node);
kfree(ws_mem.va);
}

2
sys/dev/irdma/irdma_ws.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2015 - 2022 Intel Corporation
* Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU

8
sys/dev/irdma/osdep.h
View File

@ -85,7 +85,7 @@
#define STATS_TIMER_DELAY 60000
/* a couple of linux size defines */
#define SZ_128 128
#define SZ_128 128
#define SPEED_1000 1000
#define SPEED_10000 10000
#define SPEED_20000 20000
@ -95,17 +95,11 @@
#define irdma_mb() mb()
#define irdma_wmb() wmb()
#ifndef smp_mb
#define smp_mb() mb()
#endif
#define irdma_get_virt_to_phy vtophys
#define __aligned_u64 uint64_t __aligned(8)
#define VLAN_PRIO_SHIFT 13
#if __FreeBSD_version < 1400000
#define IB_USER_VERBS_EX_CMD_MODIFY_QP IB_USER_VERBS_CMD_MODIFY_QP
#endif
/*
* debug definition section