diff --git a/sys/contrib/ena-com/ena_admin_defs.h b/sys/contrib/ena-com/ena_admin_defs.h deleted file mode 100644 index 1d845c08256e..000000000000 --- a/sys/contrib/ena-com/ena_admin_defs.h +++ /dev/null @@ -1,1412 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of copyright holder nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _ENA_ADMIN_H_ -#define _ENA_ADMIN_H_ - -enum ena_admin_aq_opcode { - ENA_ADMIN_CREATE_SQ = 1, - - ENA_ADMIN_DESTROY_SQ = 2, - - ENA_ADMIN_CREATE_CQ = 3, - - ENA_ADMIN_DESTROY_CQ = 4, - - ENA_ADMIN_GET_FEATURE = 8, - - ENA_ADMIN_SET_FEATURE = 9, - - ENA_ADMIN_GET_STATS = 11, -}; - -enum ena_admin_aq_completion_status { - ENA_ADMIN_SUCCESS = 0, - - ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE = 1, - - ENA_ADMIN_BAD_OPCODE = 2, - - ENA_ADMIN_UNSUPPORTED_OPCODE = 3, - - ENA_ADMIN_MALFORMED_REQUEST = 4, - - /* Additional status is provided in ACQ entry extended_status */ - ENA_ADMIN_ILLEGAL_PARAMETER = 5, - - ENA_ADMIN_UNKNOWN_ERROR = 6, -}; - -enum ena_admin_aq_feature_id { - ENA_ADMIN_DEVICE_ATTRIBUTES = 1, - - ENA_ADMIN_MAX_QUEUES_NUM = 2, - - ENA_ADMIN_HW_HINTS = 3, - - ENA_ADMIN_RSS_HASH_FUNCTION = 10, - - ENA_ADMIN_STATELESS_OFFLOAD_CONFIG = 11, - - ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG = 12, - - ENA_ADMIN_MTU = 14, - - ENA_ADMIN_RSS_HASH_INPUT = 18, - - ENA_ADMIN_INTERRUPT_MODERATION = 20, - - ENA_ADMIN_AENQ_CONFIG = 26, - - ENA_ADMIN_LINK_CONFIG = 27, - - ENA_ADMIN_HOST_ATTR_CONFIG = 28, - - ENA_ADMIN_FEATURES_OPCODE_NUM = 32, -}; - -enum ena_admin_placement_policy_type { - /* descriptors and headers are in host memory */ - ENA_ADMIN_PLACEMENT_POLICY_HOST = 1, - - /* descriptors and headers are in device memory (a.k.a Low Latency - * Queue) - */ - ENA_ADMIN_PLACEMENT_POLICY_DEV = 3, -}; - -enum ena_admin_link_types { - ENA_ADMIN_LINK_SPEED_1G = 0x1, - - ENA_ADMIN_LINK_SPEED_2_HALF_G = 0x2, - - ENA_ADMIN_LINK_SPEED_5G = 0x4, - - ENA_ADMIN_LINK_SPEED_10G = 0x8, - - ENA_ADMIN_LINK_SPEED_25G = 0x10, - - ENA_ADMIN_LINK_SPEED_40G = 0x20, - - ENA_ADMIN_LINK_SPEED_50G = 0x40, - - ENA_ADMIN_LINK_SPEED_100G = 0x80, - - ENA_ADMIN_LINK_SPEED_200G = 0x100, - - ENA_ADMIN_LINK_SPEED_400G = 0x200, -}; - -enum ena_admin_completion_policy_type { - /* completion queue entry for each sq descriptor */ - ENA_ADMIN_COMPLETION_POLICY_DESC = 0, - - /* completion queue entry upon request in sq descriptor */ - ENA_ADMIN_COMPLETION_POLICY_DESC_ON_DEMAND = 1, - - /* current queue head pointer is updated in OS memory upon sq - * descriptor request - */ - ENA_ADMIN_COMPLETION_POLICY_HEAD_ON_DEMAND = 2, - - /* current queue head pointer is updated in OS memory for each sq - * descriptor - */ - ENA_ADMIN_COMPLETION_POLICY_HEAD = 3, -}; - -/* basic stats return ena_admin_basic_stats while extanded stats return a - * buffer (string format) with additional statistics per queue and per - * device id - */ -enum ena_admin_get_stats_type { - ENA_ADMIN_GET_STATS_TYPE_BASIC = 0, - - ENA_ADMIN_GET_STATS_TYPE_EXTENDED = 1, -}; - -enum ena_admin_get_stats_scope { - ENA_ADMIN_SPECIFIC_QUEUE = 0, - - ENA_ADMIN_ETH_TRAFFIC = 1, -}; - -struct ena_admin_aq_common_desc { - /* 11:0 : command_id - * 15:12 : reserved12 - */ - uint16_t command_id; - - /* as appears in ena_admin_aq_opcode */ - uint8_t opcode; - - /* 0 : phase - * 1 : ctrl_data - control buffer address valid - * 2 : ctrl_data_indirect - control buffer address - * points to list of pages with addresses of control - * buffers - * 7:3 : reserved3 - */ - uint8_t flags; -}; - -/* used in ena_admin_aq_entry. Can point directly to control data, or to a - * page list chunk. Used also at the end of indirect mode page list chunks, - * for chaining. - */ -struct ena_admin_ctrl_buff_info { - uint32_t length; - - struct ena_common_mem_addr address; -}; - -struct ena_admin_sq { - uint16_t sq_idx; - - /* 4:0 : reserved - * 7:5 : sq_direction - 0x1 - Tx; 0x2 - Rx - */ - uint8_t sq_identity; - - uint8_t reserved1; -}; - -struct ena_admin_aq_entry { - struct ena_admin_aq_common_desc aq_common_descriptor; - - union { - uint32_t inline_data_w1[3]; - - struct ena_admin_ctrl_buff_info control_buffer; - } u; - - uint32_t inline_data_w4[12]; -}; - -struct ena_admin_acq_common_desc { - /* command identifier to associate it with the aq descriptor - * 11:0 : command_id - * 15:12 : reserved12 - */ - uint16_t command; - - uint8_t status; - - /* 0 : phase - * 7:1 : reserved1 - */ - uint8_t flags; - - uint16_t extended_status; - - /* serves as a hint what AQ entries can be revoked */ - uint16_t sq_head_indx; -}; - -struct ena_admin_acq_entry { - struct ena_admin_acq_common_desc acq_common_descriptor; - - uint32_t response_specific_data[14]; -}; - -struct ena_admin_aq_create_sq_cmd { - struct ena_admin_aq_common_desc aq_common_descriptor; - - /* 4:0 : reserved0_w1 - * 7:5 : sq_direction - 0x1 - Tx, 0x2 - Rx - */ - uint8_t sq_identity; - - uint8_t reserved8_w1; - - /* 3:0 : placement_policy - Describing where the SQ - * descriptor ring and the SQ packet headers reside: - * 0x1 - descriptors and headers are in OS memory, - * 0x3 - descriptors and headers in device memory - * (a.k.a Low Latency Queue) - * 6:4 : completion_policy - Describing what policy - * to use for generation completion entry (cqe) in - * the CQ associated with this SQ: 0x0 - cqe for each - * sq descriptor, 0x1 - cqe upon request in sq - * descriptor, 0x2 - current queue head pointer is - * updated in OS memory upon sq descriptor request - * 0x3 - current queue head pointer is updated in OS - * memory for each sq descriptor - * 7 : reserved15_w1 - */ - uint8_t sq_caps_2; - - /* 0 : is_physically_contiguous - Described if the - * queue ring memory is allocated in physical - * contiguous pages or split. - * 7:1 : reserved17_w1 - */ - uint8_t sq_caps_3; - - /* associated completion queue id. This CQ must be created prior to - * SQ creation - */ - uint16_t cq_idx; - - /* submission queue depth in entries */ - uint16_t sq_depth; - - /* SQ physical base address in OS memory. This field should not be - * used for Low Latency queues. Has to be page aligned. - */ - struct ena_common_mem_addr sq_ba; - - /* specifies queue head writeback location in OS memory. Valid if - * completion_policy is set to completion_policy_head_on_demand or - * completion_policy_head. Has to be cache aligned - */ - struct ena_common_mem_addr sq_head_writeback; - - uint32_t reserved0_w7; - - uint32_t reserved0_w8; -}; - -enum ena_admin_sq_direction { - ENA_ADMIN_SQ_DIRECTION_TX = 1, - - ENA_ADMIN_SQ_DIRECTION_RX = 2, -}; - -struct ena_admin_acq_create_sq_resp_desc { - struct ena_admin_acq_common_desc acq_common_desc; - - uint16_t sq_idx; - - uint16_t reserved; - - /* queue doorbell address as an offset to PCIe MMIO REG BAR */ - uint32_t sq_doorbell_offset; - - /* low latency queue ring base address as an offset to PCIe MMIO - * LLQ_MEM BAR - */ - uint32_t llq_descriptors_offset; - - /* low latency queue headers' memory as an offset to PCIe MMIO - * LLQ_MEM BAR - */ - uint32_t llq_headers_offset; -}; - -struct ena_admin_aq_destroy_sq_cmd { - struct ena_admin_aq_common_desc aq_common_descriptor; - - struct ena_admin_sq sq; -}; - -struct ena_admin_acq_destroy_sq_resp_desc { - struct ena_admin_acq_common_desc acq_common_desc; -}; - -struct ena_admin_aq_create_cq_cmd { - struct ena_admin_aq_common_desc aq_common_descriptor; - - /* 4:0 : reserved5 - * 5 : interrupt_mode_enabled - if set, cq operates - * in interrupt mode, otherwise - polling - * 7:6 : reserved6 - */ - uint8_t cq_caps_1; - - /* 4:0 : cq_entry_size_words - size of CQ entry in - * 32-bit words, valid values: 4, 8. - * 7:5 : reserved7 - */ - uint8_t cq_caps_2; - - /* completion queue depth in # of entries. must be power of 2 */ - uint16_t cq_depth; - - /* msix vector assigned to this cq */ - uint32_t msix_vector; - - /* cq physical base address in OS memory. CQ must be physically - * contiguous - */ - struct ena_common_mem_addr cq_ba; -}; - -struct ena_admin_acq_create_cq_resp_desc { - struct ena_admin_acq_common_desc acq_common_desc; - - uint16_t cq_idx; - - /* actual cq depth in number of entries */ - uint16_t cq_actual_depth; - - uint32_t numa_node_register_offset; - - uint32_t cq_head_db_register_offset; - - uint32_t cq_interrupt_unmask_register_offset; -}; - -struct ena_admin_aq_destroy_cq_cmd { - struct ena_admin_aq_common_desc aq_common_descriptor; - - uint16_t cq_idx; - - uint16_t reserved1; -}; - -struct ena_admin_acq_destroy_cq_resp_desc { - struct ena_admin_acq_common_desc acq_common_desc; -}; - -/* ENA AQ Get Statistics command. Extended statistics are placed in control - * buffer pointed by AQ entry - */ -struct ena_admin_aq_get_stats_cmd { - struct ena_admin_aq_common_desc aq_common_descriptor; - - union { - /* command specific inline data */ - uint32_t inline_data_w1[3]; - - struct ena_admin_ctrl_buff_info control_buffer; - } u; - - /* stats type as defined in enum ena_admin_get_stats_type */ - uint8_t type; - - /* stats scope defined in enum ena_admin_get_stats_scope */ - uint8_t scope; - - uint16_t reserved3; - - /* queue id. used when scope is specific_queue */ - uint16_t queue_idx; - - /* device id, value 0xFFFF means mine. only privileged device can get - * stats of other device - */ - uint16_t device_id; -}; - -/* Basic Statistics Command. */ -struct ena_admin_basic_stats { - uint32_t tx_bytes_low; - - uint32_t tx_bytes_high; - - uint32_t tx_pkts_low; - - uint32_t tx_pkts_high; - - uint32_t rx_bytes_low; - - uint32_t rx_bytes_high; - - uint32_t rx_pkts_low; - - uint32_t rx_pkts_high; - - uint32_t rx_drops_low; - - uint32_t rx_drops_high; -}; - -struct ena_admin_acq_get_stats_resp { - struct ena_admin_acq_common_desc acq_common_desc; - - struct ena_admin_basic_stats basic_stats; -}; - -struct ena_admin_get_set_feature_common_desc { - /* 1:0 : select - 0x1 - current value; 0x3 - default - * value - * 7:3 : reserved3 - */ - uint8_t flags; - - /* as appears in ena_admin_aq_feature_id */ - uint8_t feature_id; - - uint16_t reserved16; -}; - -struct ena_admin_device_attr_feature_desc { - uint32_t impl_id; - - uint32_t device_version; - - /* bitmap of ena_admin_aq_feature_id */ - uint32_t supported_features; - - uint32_t reserved3; - - /* Indicates how many bits are used physical address access. */ - uint32_t phys_addr_width; - - /* Indicates how many bits are used virtual address access. */ - uint32_t virt_addr_width; - - /* unicast MAC address (in Network byte order) */ - uint8_t mac_addr[6]; - - uint8_t reserved7[2]; - - uint32_t max_mtu; -}; - -struct ena_admin_queue_feature_desc { - /* including LLQs */ - uint32_t max_sq_num; - - uint32_t max_sq_depth; - - uint32_t max_cq_num; - - uint32_t max_cq_depth; - - uint32_t max_llq_num; - - uint32_t max_llq_depth; - - uint32_t max_header_size; - - /* Maximum Descriptors number, including meta descriptor, allowed for - * a single Tx packet - */ - uint16_t max_packet_tx_descs; - - /* Maximum Descriptors number allowed for a single Rx packet */ - uint16_t max_packet_rx_descs; -}; - -struct ena_admin_set_feature_mtu_desc { - /* exclude L2 */ - uint32_t mtu; -}; - -struct ena_admin_set_feature_host_attr_desc { - /* host OS info base address in OS memory. host info is 4KB of - * physically contiguous - */ - struct ena_common_mem_addr os_info_ba; - - /* host debug area base address in OS memory. debug area must be - * physically contiguous - */ - struct ena_common_mem_addr debug_ba; - - /* debug area size */ - uint32_t debug_area_size; -}; - -struct ena_admin_feature_intr_moder_desc { - /* interrupt delay granularity in usec */ - uint16_t intr_delay_resolution; - - uint16_t reserved; -}; - -struct ena_admin_get_feature_link_desc { - /* Link speed in Mb */ - uint32_t speed; - - /* bit field of enum ena_admin_link types */ - uint32_t supported; - - /* 0 : autoneg - * 1 : duplex - Full Duplex - * 31:2 : reserved2 - */ - uint32_t flags; -}; - -struct ena_admin_feature_aenq_desc { - /* bitmask for AENQ groups the device can report */ - uint32_t supported_groups; - - /* bitmask for AENQ groups to report */ - uint32_t enabled_groups; -}; - -struct ena_admin_feature_offload_desc { - /* 0 : TX_L3_csum_ipv4 - * 1 : TX_L4_ipv4_csum_part - The checksum field - * should be initialized with pseudo header checksum - * 2 : TX_L4_ipv4_csum_full - * 3 : TX_L4_ipv6_csum_part - The checksum field - * should be initialized with pseudo header checksum - * 4 : TX_L4_ipv6_csum_full - * 5 : tso_ipv4 - * 6 : tso_ipv6 - * 7 : tso_ecn - */ - uint32_t tx; - - /* Receive side supported stateless offload - * 0 : RX_L3_csum_ipv4 - IPv4 checksum - * 1 : RX_L4_ipv4_csum - TCP/UDP/IPv4 checksum - * 2 : RX_L4_ipv6_csum - TCP/UDP/IPv6 checksum - * 3 : RX_hash - Hash calculation - */ - uint32_t rx_supported; - - uint32_t rx_enabled; -}; - -enum ena_admin_hash_functions { - ENA_ADMIN_TOEPLITZ = 1, - - ENA_ADMIN_CRC32 = 2, -}; - -struct ena_admin_feature_rss_flow_hash_control { - uint32_t keys_num; - - uint32_t reserved; - - uint32_t key[10]; -}; - -struct ena_admin_feature_rss_flow_hash_function { - /* 7:0 : funcs - bitmask of ena_admin_hash_functions */ - uint32_t supported_func; - - /* 7:0 : selected_func - bitmask of - * ena_admin_hash_functions - */ - uint32_t selected_func; - - /* initial value */ - uint32_t init_val; -}; - -/* RSS flow hash protocols */ -enum ena_admin_flow_hash_proto { - ENA_ADMIN_RSS_TCP4 = 0, - - ENA_ADMIN_RSS_UDP4 = 1, - - ENA_ADMIN_RSS_TCP6 = 2, - - ENA_ADMIN_RSS_UDP6 = 3, - - ENA_ADMIN_RSS_IP4 = 4, - - ENA_ADMIN_RSS_IP6 = 5, - - ENA_ADMIN_RSS_IP4_FRAG = 6, - - ENA_ADMIN_RSS_NOT_IP = 7, - - /* TCPv6 with extension header */ - ENA_ADMIN_RSS_TCP6_EX = 8, - - /* IPv6 with extension header */ - ENA_ADMIN_RSS_IP6_EX = 9, - - ENA_ADMIN_RSS_PROTO_NUM = 16, -}; - -/* RSS flow hash fields */ -enum ena_admin_flow_hash_fields { - /* Ethernet Dest Addr */ - ENA_ADMIN_RSS_L2_DA = BIT(0), - - /* Ethernet Src Addr */ - ENA_ADMIN_RSS_L2_SA = BIT(1), - - /* ipv4/6 Dest Addr */ - ENA_ADMIN_RSS_L3_DA = BIT(2), - - /* ipv4/6 Src Addr */ - ENA_ADMIN_RSS_L3_SA = BIT(3), - - /* tcp/udp Dest Port */ - ENA_ADMIN_RSS_L4_DP = BIT(4), - - /* tcp/udp Src Port */ - ENA_ADMIN_RSS_L4_SP = BIT(5), -}; - -struct ena_admin_proto_input { - /* flow hash fields (bitwise according to ena_admin_flow_hash_fields) */ - uint16_t fields; - - uint16_t reserved2; -}; - -struct ena_admin_feature_rss_hash_control { - struct ena_admin_proto_input supported_fields[ENA_ADMIN_RSS_PROTO_NUM]; - - struct ena_admin_proto_input selected_fields[ENA_ADMIN_RSS_PROTO_NUM]; - - struct ena_admin_proto_input reserved2[ENA_ADMIN_RSS_PROTO_NUM]; - - struct ena_admin_proto_input reserved3[ENA_ADMIN_RSS_PROTO_NUM]; -}; - -struct ena_admin_feature_rss_flow_hash_input { - /* supported hash input sorting - * 1 : L3_sort - support swap L3 addresses if DA is - * smaller than SA - * 2 : L4_sort - support swap L4 ports if DP smaller - * SP - */ - uint16_t supported_input_sort; - - /* enabled hash input sorting - * 1 : enable_L3_sort - enable swap L3 addresses if - * DA smaller than SA - * 2 : enable_L4_sort - enable swap L4 ports if DP - * smaller than SP - */ - uint16_t enabled_input_sort; -}; - -enum ena_admin_os_type { - ENA_ADMIN_OS_LINUX = 1, - - ENA_ADMIN_OS_WIN = 2, - - ENA_ADMIN_OS_DPDK = 3, - - ENA_ADMIN_OS_FREEBSD = 4, - - ENA_ADMIN_OS_IPXE = 5, -}; - -struct ena_admin_host_info { - /* defined in enum ena_admin_os_type */ - uint32_t os_type; - - /* os distribution string format */ - uint8_t os_dist_str[128]; - - /* OS distribution numeric format */ - uint32_t os_dist; - - /* kernel version string format */ - uint8_t kernel_ver_str[32]; - - /* Kernel version numeric format */ - uint32_t kernel_ver; - - /* 7:0 : major - * 15:8 : minor - * 23:16 : sub_minor - */ - uint32_t driver_version; - - /* features bitmap */ - uint32_t supported_network_features[4]; -}; - -struct ena_admin_rss_ind_table_entry { - uint16_t cq_idx; - - uint16_t reserved; -}; - -struct ena_admin_feature_rss_ind_table { - /* min supported table size (2^min_size) */ - uint16_t min_size; - - /* max supported table size (2^max_size) */ - uint16_t max_size; - - /* table size (2^size) */ - uint16_t size; - - uint16_t reserved; - - /* index of the inline entry. 0xFFFFFFFF means invalid */ - uint32_t inline_index; - - /* used for updating single entry, ignored when setting the entire - * table through the control buffer. - */ - struct ena_admin_rss_ind_table_entry inline_entry; -}; - -/* When hint value is 0, driver should use it's own predefined value */ -struct ena_admin_ena_hw_hints { - /* value in ms */ - uint16_t mmio_read_timeout; - - /* value in ms */ - uint16_t driver_watchdog_timeout; - - /* Per packet tx completion timeout. value in ms */ - uint16_t missing_tx_completion_timeout; - - uint16_t missed_tx_completion_count_threshold_to_reset; - - /* value in ms */ - uint16_t admin_completion_tx_timeout; - - uint16_t netdev_wd_timeout; - - uint16_t max_tx_sgl_size; - - uint16_t max_rx_sgl_size; - - uint16_t reserved[8]; -}; - -struct ena_admin_get_feat_cmd { - struct ena_admin_aq_common_desc aq_common_descriptor; - - struct ena_admin_ctrl_buff_info control_buffer; - - struct ena_admin_get_set_feature_common_desc feat_common; - - uint32_t raw[11]; -}; - -struct ena_admin_get_feat_resp { - struct ena_admin_acq_common_desc acq_common_desc; - - union { - uint32_t raw[14]; - - struct ena_admin_device_attr_feature_desc dev_attr; - - struct ena_admin_queue_feature_desc max_queue; - - struct ena_admin_feature_aenq_desc aenq; - - struct ena_admin_get_feature_link_desc link; - - struct ena_admin_feature_offload_desc offload; - - struct ena_admin_feature_rss_flow_hash_function flow_hash_func; - - struct ena_admin_feature_rss_flow_hash_input flow_hash_input; - - struct ena_admin_feature_rss_ind_table ind_table; - - struct ena_admin_feature_intr_moder_desc intr_moderation; - - struct ena_admin_ena_hw_hints hw_hints; - } u; -}; - -struct ena_admin_set_feat_cmd { - struct ena_admin_aq_common_desc aq_common_descriptor; - - struct ena_admin_ctrl_buff_info control_buffer; - - struct ena_admin_get_set_feature_common_desc feat_common; - - union { - uint32_t raw[11]; - - /* mtu size */ - struct ena_admin_set_feature_mtu_desc mtu; - - /* host attributes */ - struct ena_admin_set_feature_host_attr_desc host_attr; - - /* AENQ configuration */ - struct ena_admin_feature_aenq_desc aenq; - - /* rss flow hash function */ - struct ena_admin_feature_rss_flow_hash_function flow_hash_func; - - /* rss flow hash input */ - struct ena_admin_feature_rss_flow_hash_input flow_hash_input; - - /* rss indirection table */ - struct ena_admin_feature_rss_ind_table ind_table; - } u; -}; - -struct ena_admin_set_feat_resp { - struct ena_admin_acq_common_desc acq_common_desc; - - union { - uint32_t raw[14]; - } u; -}; - -struct ena_admin_aenq_common_desc { - uint16_t group; - - uint16_t syndrom; - - /* 0 : phase */ - uint8_t flags; - - uint8_t reserved1[3]; - - uint32_t timestamp_low; - - uint32_t timestamp_high; -}; - -/* asynchronous event notification groups */ -enum ena_admin_aenq_group { - ENA_ADMIN_LINK_CHANGE = 0, - - ENA_ADMIN_FATAL_ERROR = 1, - - ENA_ADMIN_WARNING = 2, - - ENA_ADMIN_NOTIFICATION = 3, - - ENA_ADMIN_KEEP_ALIVE = 4, - - ENA_ADMIN_AENQ_GROUPS_NUM = 5, -}; - -enum ena_admin_aenq_notification_syndrom { - ENA_ADMIN_SUSPEND = 0, - - ENA_ADMIN_RESUME = 1, - - ENA_ADMIN_UPDATE_HINTS = 2, -}; - -struct ena_admin_aenq_entry { - struct ena_admin_aenq_common_desc aenq_common_desc; - - /* command specific inline data */ - uint32_t inline_data_w4[12]; -}; - -struct ena_admin_aenq_link_change_desc { - struct ena_admin_aenq_common_desc aenq_common_desc; - - /* 0 : link_status */ - uint32_t flags; -}; - -struct ena_admin_aenq_keep_alive_desc { - struct ena_admin_aenq_common_desc aenq_common_desc; - - uint32_t rx_drops_low; - - uint32_t rx_drops_high; -}; - -struct ena_admin_ena_mmio_req_read_less_resp { - uint16_t req_id; - - uint16_t reg_off; - - /* value is valid when poll is cleared */ - uint32_t reg_val; -}; - -/* aq_common_desc */ -#define ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK GENMASK(11, 0) -#define ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK BIT(0) -#define ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT 1 -#define ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK BIT(1) -#define ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT 2 -#define ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK BIT(2) - -/* sq */ -#define ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT 5 -#define ENA_ADMIN_SQ_SQ_DIRECTION_MASK GENMASK(7, 5) - -/* acq_common_desc */ -#define ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK GENMASK(11, 0) -#define ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK BIT(0) - -/* aq_create_sq_cmd */ -#define ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT 5 -#define ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK GENMASK(7, 5) -#define ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK GENMASK(3, 0) -#define ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT 4 -#define ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK GENMASK(6, 4) -#define ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK BIT(0) - -/* aq_create_cq_cmd */ -#define ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_SHIFT 5 -#define ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK BIT(5) -#define ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK GENMASK(4, 0) - -/* get_set_feature_common_desc */ -#define ENA_ADMIN_GET_SET_FEATURE_COMMON_DESC_SELECT_MASK GENMASK(1, 0) - -/* get_feature_link_desc */ -#define ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK BIT(0) -#define ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_SHIFT 1 -#define ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_MASK BIT(1) - -/* feature_offload_desc */ -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK BIT(0) -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_SHIFT 1 -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK BIT(1) -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_SHIFT 2 -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK BIT(2) -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_SHIFT 3 -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK BIT(3) -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_SHIFT 4 -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK BIT(4) -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_SHIFT 5 -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK BIT(5) -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_SHIFT 6 -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK BIT(6) -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_SHIFT 7 -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK BIT(7) -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK BIT(0) -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_SHIFT 1 -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK BIT(1) -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_SHIFT 2 -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK BIT(2) -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_SHIFT 3 -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_MASK BIT(3) - -/* feature_rss_flow_hash_function */ -#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_FUNCS_MASK GENMASK(7, 0) -#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_SELECTED_FUNC_MASK GENMASK(7, 0) - -/* feature_rss_flow_hash_input */ -#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_SHIFT 1 -#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK BIT(1) -#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_SHIFT 2 -#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK BIT(2) -#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_SHIFT 1 -#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_MASK BIT(1) -#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_SHIFT 2 -#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_MASK BIT(2) - -/* host_info */ -#define ENA_ADMIN_HOST_INFO_MAJOR_MASK GENMASK(7, 0) -#define ENA_ADMIN_HOST_INFO_MINOR_SHIFT 8 -#define ENA_ADMIN_HOST_INFO_MINOR_MASK GENMASK(15, 8) -#define ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT 16 -#define ENA_ADMIN_HOST_INFO_SUB_MINOR_MASK GENMASK(23, 16) - -/* aenq_common_desc */ -#define ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK BIT(0) - -/* aenq_link_change_desc */ -#define ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK BIT(0) - -#if !defined(ENA_DEFS_LINUX_MAINLINE) -static inline uint16_t get_ena_admin_aq_common_desc_command_id(const struct ena_admin_aq_common_desc *p) -{ - return p->command_id & ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK; -} - -static inline void set_ena_admin_aq_common_desc_command_id(struct ena_admin_aq_common_desc *p, uint16_t val) -{ - p->command_id |= val & ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK; -} - -static inline uint8_t get_ena_admin_aq_common_desc_phase(const struct ena_admin_aq_common_desc *p) -{ - return p->flags & ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK; -} - -static inline void set_ena_admin_aq_common_desc_phase(struct ena_admin_aq_common_desc *p, uint8_t val) -{ - p->flags |= val & ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK; -} - -static inline uint8_t get_ena_admin_aq_common_desc_ctrl_data(const struct ena_admin_aq_common_desc *p) -{ - return (p->flags & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK) >> ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT; -} - -static inline void set_ena_admin_aq_common_desc_ctrl_data(struct ena_admin_aq_common_desc *p, uint8_t val) -{ - p->flags |= (val << ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT) & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK; -} - -static inline uint8_t get_ena_admin_aq_common_desc_ctrl_data_indirect(const struct ena_admin_aq_common_desc *p) -{ - return (p->flags & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK) >> ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT; -} - -static inline void set_ena_admin_aq_common_desc_ctrl_data_indirect(struct ena_admin_aq_common_desc *p, uint8_t val) -{ - p->flags |= (val << ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT) & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK; -} - -static inline uint8_t get_ena_admin_sq_sq_direction(const struct ena_admin_sq *p) -{ - return (p->sq_identity & ENA_ADMIN_SQ_SQ_DIRECTION_MASK) >> ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT; -} - -static inline void set_ena_admin_sq_sq_direction(struct ena_admin_sq *p, uint8_t val) -{ - p->sq_identity |= (val << ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT) & ENA_ADMIN_SQ_SQ_DIRECTION_MASK; -} - -static inline uint16_t get_ena_admin_acq_common_desc_command_id(const struct ena_admin_acq_common_desc *p) -{ - return p->command & ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK; -} - -static inline void set_ena_admin_acq_common_desc_command_id(struct ena_admin_acq_common_desc *p, uint16_t val) -{ - p->command |= val & ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK; -} - -static inline uint8_t get_ena_admin_acq_common_desc_phase(const struct ena_admin_acq_common_desc *p) -{ - return p->flags & ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK; -} - -static inline void set_ena_admin_acq_common_desc_phase(struct ena_admin_acq_common_desc *p, uint8_t val) -{ - p->flags |= val & ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK; -} - -static inline uint8_t get_ena_admin_aq_create_sq_cmd_sq_direction(const struct ena_admin_aq_create_sq_cmd *p) -{ - return (p->sq_identity & ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK) >> ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT; -} - -static inline void set_ena_admin_aq_create_sq_cmd_sq_direction(struct ena_admin_aq_create_sq_cmd *p, uint8_t val) -{ - p->sq_identity |= (val << ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT) & ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK; -} - -static inline uint8_t get_ena_admin_aq_create_sq_cmd_placement_policy(const struct ena_admin_aq_create_sq_cmd *p) -{ - return p->sq_caps_2 & ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK; -} - -static inline void set_ena_admin_aq_create_sq_cmd_placement_policy(struct ena_admin_aq_create_sq_cmd *p, uint8_t val) -{ - p->sq_caps_2 |= val & ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK; -} - -static inline uint8_t get_ena_admin_aq_create_sq_cmd_completion_policy(const struct ena_admin_aq_create_sq_cmd *p) -{ - return (p->sq_caps_2 & ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK) >> ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT; -} - -static inline void set_ena_admin_aq_create_sq_cmd_completion_policy(struct ena_admin_aq_create_sq_cmd *p, uint8_t val) -{ - p->sq_caps_2 |= (val << ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT) & ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK; -} - -static inline uint8_t get_ena_admin_aq_create_sq_cmd_is_physically_contiguous(const struct ena_admin_aq_create_sq_cmd *p) -{ - return p->sq_caps_3 & ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK; -} - -static inline void set_ena_admin_aq_create_sq_cmd_is_physically_contiguous(struct ena_admin_aq_create_sq_cmd *p, uint8_t val) -{ - p->sq_caps_3 |= val & ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK; -} - -static inline uint8_t get_ena_admin_aq_create_cq_cmd_interrupt_mode_enabled(const struct ena_admin_aq_create_cq_cmd *p) -{ - return (p->cq_caps_1 & ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK) >> ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_SHIFT; -} - -static inline void set_ena_admin_aq_create_cq_cmd_interrupt_mode_enabled(struct ena_admin_aq_create_cq_cmd *p, uint8_t val) -{ - p->cq_caps_1 |= (val << ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_SHIFT) & ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK; -} - -static inline uint8_t get_ena_admin_aq_create_cq_cmd_cq_entry_size_words(const struct ena_admin_aq_create_cq_cmd *p) -{ - return p->cq_caps_2 & ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK; -} - -static inline void set_ena_admin_aq_create_cq_cmd_cq_entry_size_words(struct ena_admin_aq_create_cq_cmd *p, uint8_t val) -{ - p->cq_caps_2 |= val & ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK; -} - -static inline uint8_t get_ena_admin_get_set_feature_common_desc_select(const struct ena_admin_get_set_feature_common_desc *p) -{ - return p->flags & ENA_ADMIN_GET_SET_FEATURE_COMMON_DESC_SELECT_MASK; -} - -static inline void set_ena_admin_get_set_feature_common_desc_select(struct ena_admin_get_set_feature_common_desc *p, uint8_t val) -{ - p->flags |= val & ENA_ADMIN_GET_SET_FEATURE_COMMON_DESC_SELECT_MASK; -} - -static inline uint32_t get_ena_admin_get_feature_link_desc_autoneg(const struct ena_admin_get_feature_link_desc *p) -{ - return p->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK; -} - -static inline void set_ena_admin_get_feature_link_desc_autoneg(struct ena_admin_get_feature_link_desc *p, uint32_t val) -{ - p->flags |= val & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK; -} - -static inline uint32_t get_ena_admin_get_feature_link_desc_duplex(const struct ena_admin_get_feature_link_desc *p) -{ - return (p->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_MASK) >> ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_SHIFT; -} - -static inline void set_ena_admin_get_feature_link_desc_duplex(struct ena_admin_get_feature_link_desc *p, uint32_t val) -{ - p->flags |= (val << ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_SHIFT) & ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_MASK; -} - -static inline uint32_t get_ena_admin_feature_offload_desc_TX_L3_csum_ipv4(const struct ena_admin_feature_offload_desc *p) -{ - return p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK; -} - -static inline void set_ena_admin_feature_offload_desc_TX_L3_csum_ipv4(struct ena_admin_feature_offload_desc *p, uint32_t val) -{ - p->tx |= val & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK; -} - -static inline uint32_t get_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_part(const struct ena_admin_feature_offload_desc *p) -{ - return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_SHIFT; -} - -static inline void set_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_part(struct ena_admin_feature_offload_desc *p, uint32_t val) -{ - p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK; -} - -static inline uint32_t get_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_full(const struct ena_admin_feature_offload_desc *p) -{ - return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_SHIFT; -} - -static inline void set_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_full(struct ena_admin_feature_offload_desc *p, uint32_t val) -{ - p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK; -} - -static inline uint32_t get_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_part(const struct ena_admin_feature_offload_desc *p) -{ - return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_SHIFT; -} - -static inline void set_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_part(struct ena_admin_feature_offload_desc *p, uint32_t val) -{ - p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK; -} - -static inline uint32_t get_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_full(const struct ena_admin_feature_offload_desc *p) -{ - return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_SHIFT; -} - -static inline void set_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_full(struct ena_admin_feature_offload_desc *p, uint32_t val) -{ - p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK; -} - -static inline uint32_t get_ena_admin_feature_offload_desc_tso_ipv4(const struct ena_admin_feature_offload_desc *p) -{ - return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_SHIFT; -} - -static inline void set_ena_admin_feature_offload_desc_tso_ipv4(struct ena_admin_feature_offload_desc *p, uint32_t val) -{ - p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK; -} - -static inline uint32_t get_ena_admin_feature_offload_desc_tso_ipv6(const struct ena_admin_feature_offload_desc *p) -{ - return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_SHIFT; -} - -static inline void set_ena_admin_feature_offload_desc_tso_ipv6(struct ena_admin_feature_offload_desc *p, uint32_t val) -{ - p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK; -} - -static inline uint32_t get_ena_admin_feature_offload_desc_tso_ecn(const struct ena_admin_feature_offload_desc *p) -{ - return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_SHIFT; -} - -static inline void set_ena_admin_feature_offload_desc_tso_ecn(struct ena_admin_feature_offload_desc *p, uint32_t val) -{ - p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK; -} - -static inline uint32_t get_ena_admin_feature_offload_desc_RX_L3_csum_ipv4(const struct ena_admin_feature_offload_desc *p) -{ - return p->rx_supported & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK; -} - -static inline void set_ena_admin_feature_offload_desc_RX_L3_csum_ipv4(struct ena_admin_feature_offload_desc *p, uint32_t val) -{ - p->rx_supported |= val & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK; -} - -static inline uint32_t get_ena_admin_feature_offload_desc_RX_L4_ipv4_csum(const struct ena_admin_feature_offload_desc *p) -{ - return (p->rx_supported & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_SHIFT; -} - -static inline void set_ena_admin_feature_offload_desc_RX_L4_ipv4_csum(struct ena_admin_feature_offload_desc *p, uint32_t val) -{ - p->rx_supported |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK; -} - -static inline uint32_t get_ena_admin_feature_offload_desc_RX_L4_ipv6_csum(const struct ena_admin_feature_offload_desc *p) -{ - return (p->rx_supported & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_SHIFT; -} - -static inline void set_ena_admin_feature_offload_desc_RX_L4_ipv6_csum(struct ena_admin_feature_offload_desc *p, uint32_t val) -{ - p->rx_supported |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK; -} - -static inline uint32_t get_ena_admin_feature_offload_desc_RX_hash(const struct ena_admin_feature_offload_desc *p) -{ - return (p->rx_supported & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_SHIFT; -} - -static inline void set_ena_admin_feature_offload_desc_RX_hash(struct ena_admin_feature_offload_desc *p, uint32_t val) -{ - p->rx_supported |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_MASK; -} - -static inline uint32_t get_ena_admin_feature_rss_flow_hash_function_funcs(const struct ena_admin_feature_rss_flow_hash_function *p) -{ - return p->supported_func & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_FUNCS_MASK; -} - -static inline void set_ena_admin_feature_rss_flow_hash_function_funcs(struct ena_admin_feature_rss_flow_hash_function *p, uint32_t val) -{ - p->supported_func |= val & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_FUNCS_MASK; -} - -static inline uint32_t get_ena_admin_feature_rss_flow_hash_function_selected_func(const struct ena_admin_feature_rss_flow_hash_function *p) -{ - return p->selected_func & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_SELECTED_FUNC_MASK; -} - -static inline void set_ena_admin_feature_rss_flow_hash_function_selected_func(struct ena_admin_feature_rss_flow_hash_function *p, uint32_t val) -{ - p->selected_func |= val & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_SELECTED_FUNC_MASK; -} - -static inline uint16_t get_ena_admin_feature_rss_flow_hash_input_L3_sort(const struct ena_admin_feature_rss_flow_hash_input *p) -{ - return (p->supported_input_sort & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK) >> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_SHIFT; -} - -static inline void set_ena_admin_feature_rss_flow_hash_input_L3_sort(struct ena_admin_feature_rss_flow_hash_input *p, uint16_t val) -{ - p->supported_input_sort |= (val << ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_SHIFT) & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK; -} - -static inline uint16_t get_ena_admin_feature_rss_flow_hash_input_L4_sort(const struct ena_admin_feature_rss_flow_hash_input *p) -{ - return (p->supported_input_sort & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK) >> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_SHIFT; -} - -static inline void set_ena_admin_feature_rss_flow_hash_input_L4_sort(struct ena_admin_feature_rss_flow_hash_input *p, uint16_t val) -{ - p->supported_input_sort |= (val << ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_SHIFT) & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK; -} - -static inline uint16_t get_ena_admin_feature_rss_flow_hash_input_enable_L3_sort(const struct ena_admin_feature_rss_flow_hash_input *p) -{ - return (p->enabled_input_sort & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_MASK) >> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_SHIFT; -} - -static inline void set_ena_admin_feature_rss_flow_hash_input_enable_L3_sort(struct ena_admin_feature_rss_flow_hash_input *p, uint16_t val) -{ - p->enabled_input_sort |= (val << ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_SHIFT) & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_MASK; -} - -static inline uint16_t get_ena_admin_feature_rss_flow_hash_input_enable_L4_sort(const struct ena_admin_feature_rss_flow_hash_input *p) -{ - return (p->enabled_input_sort & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_MASK) >> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_SHIFT; -} - -static inline void set_ena_admin_feature_rss_flow_hash_input_enable_L4_sort(struct ena_admin_feature_rss_flow_hash_input *p, uint16_t val) -{ - p->enabled_input_sort |= (val << ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_SHIFT) & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_MASK; -} - -static inline uint32_t get_ena_admin_host_info_major(const struct ena_admin_host_info *p) -{ - return p->driver_version & ENA_ADMIN_HOST_INFO_MAJOR_MASK; -} - -static inline void set_ena_admin_host_info_major(struct ena_admin_host_info *p, uint32_t val) -{ - p->driver_version |= val & ENA_ADMIN_HOST_INFO_MAJOR_MASK; -} - -static inline uint32_t get_ena_admin_host_info_minor(const struct ena_admin_host_info *p) -{ - return (p->driver_version & ENA_ADMIN_HOST_INFO_MINOR_MASK) >> ENA_ADMIN_HOST_INFO_MINOR_SHIFT; -} - -static inline void set_ena_admin_host_info_minor(struct ena_admin_host_info *p, uint32_t val) -{ - p->driver_version |= (val << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) & ENA_ADMIN_HOST_INFO_MINOR_MASK; -} - -static inline uint32_t get_ena_admin_host_info_sub_minor(const struct ena_admin_host_info *p) -{ - return (p->driver_version & ENA_ADMIN_HOST_INFO_SUB_MINOR_MASK) >> ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT; -} - -static inline void set_ena_admin_host_info_sub_minor(struct ena_admin_host_info *p, uint32_t val) -{ - p->driver_version |= (val << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT) & ENA_ADMIN_HOST_INFO_SUB_MINOR_MASK; -} - -static inline uint8_t get_ena_admin_aenq_common_desc_phase(const struct ena_admin_aenq_common_desc *p) -{ - return p->flags & ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK; -} - -static inline void set_ena_admin_aenq_common_desc_phase(struct ena_admin_aenq_common_desc *p, uint8_t val) -{ - p->flags |= val & ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK; -} - -static inline uint32_t get_ena_admin_aenq_link_change_desc_link_status(const struct ena_admin_aenq_link_change_desc *p) -{ - return p->flags & ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK; -} - -static inline void set_ena_admin_aenq_link_change_desc_link_status(struct ena_admin_aenq_link_change_desc *p, uint32_t val) -{ - p->flags |= val & ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK; -} - -#endif /* !defined(ENA_DEFS_LINUX_MAINLINE) */ -#endif /*_ENA_ADMIN_H_ */ diff --git a/sys/contrib/ena-com/ena_com.c b/sys/contrib/ena-com/ena_com.c index 3af3b14eca44..91c711eaa438 100644 --- a/sys/contrib/ena-com/ena_com.c +++ b/sys/contrib/ena-com/ena_com.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * - * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates. + * Copyright (c) 2015-2023 Amazon.com, Inc. or its affiliates. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -74,6 +74,12 @@ #define ENA_MAX_ADMIN_POLL_US 5000 +/* PHC definitions */ +#define ENA_PHC_DEFAULT_EXPIRE_TIMEOUT_USEC 20 +#define ENA_PHC_DEFAULT_BLOCK_TIMEOUT_USEC 1000 +#define ENA_PHC_TIMESTAMP_ERROR 0xFFFFFFFFFFFFFFFF +#define ENA_PHC_REQ_ID_OFFSET 0xDEAD + /*****************************************************************************/ /*****************************************************************************/ /*****************************************************************************/ @@ -396,7 +402,7 @@ static int ena_com_init_io_sq(struct ena_com_dev *ena_dev, ENA_COM_BOUNCE_BUFFER_CNTRL_CNT; io_sq->bounce_buf_ctrl.next_to_use = 0; - size = io_sq->bounce_buf_ctrl.buffer_size * + size = (size_t)io_sq->bounce_buf_ctrl.buffer_size * io_sq->bounce_buf_ctrl.buffers_num; ENA_MEM_ALLOC_NODE(ena_dev->dmadev, @@ -894,7 +900,7 @@ static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset) } if (unlikely(i == timeout)) { - ena_trc_err(ena_dev, "Reading reg failed for timeout. expected: req id[%hu] offset[%hu] actual: req id[%hu] offset[%hu]\n", + ena_trc_err(ena_dev, "Reading reg failed for timeout. expected: req id[%u] offset[%u] actual: req id[%u] offset[%u]\n", mmio_read->seq_num, offset, read_resp->req_id, @@ -1332,9 +1338,6 @@ static int ena_com_create_io_sq(struct ena_com_dev *ena_dev, (uintptr_t)cmd_completion.sq_doorbell_offset); if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { - io_sq->header_addr = (u8 __iomem *)((uintptr_t)ena_dev->mem_bar - + cmd_completion.llq_headers_offset); - io_sq->desc_addr.pbuf_dev_addr = (u8 __iomem *)((uintptr_t)ena_dev->mem_bar + cmd_completion.llq_descriptors_offset); @@ -1477,11 +1480,6 @@ int ena_com_create_io_cq(struct ena_com_dev *ena_dev, io_cq->unmask_reg = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + cmd_completion.cq_interrupt_unmask_register_offset); - if (cmd_completion.cq_head_db_register_offset) - io_cq->cq_head_db_reg = - (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + - cmd_completion.cq_head_db_register_offset); - if (cmd_completion.numa_node_register_offset) io_cq->numa_node_cfg_reg = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + @@ -1777,6 +1775,221 @@ void ena_com_set_admin_auto_polling_mode(struct ena_com_dev *ena_dev, ena_dev->admin_queue.auto_polling = polling; } +bool ena_com_phc_supported(struct ena_com_dev *ena_dev) +{ + return ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_PHC_CONFIG); +} + +int ena_com_phc_init(struct ena_com_dev *ena_dev) +{ + struct ena_com_phc_info *phc = &ena_dev->phc; + + memset(phc, 0x0, sizeof(*phc)); + + /* Allocate shared mem used PHC timestamp retrieved from device */ + ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, + sizeof(*phc->virt_addr), + phc->virt_addr, + phc->phys_addr, + phc->mem_handle); + if (unlikely(!phc->virt_addr)) + return ENA_COM_NO_MEM; + + ENA_SPINLOCK_INIT(phc->lock); + + phc->virt_addr->req_id = 0; + phc->virt_addr->timestamp = 0; + + return 0; +} + +int ena_com_phc_config(struct ena_com_dev *ena_dev) +{ + struct ena_com_phc_info *phc = &ena_dev->phc; + struct ena_admin_get_feat_resp get_feat_resp; + struct ena_admin_set_feat_resp set_feat_resp; + struct ena_admin_set_feat_cmd set_feat_cmd; + int ret = 0; + + /* Get device PHC default configuration */ + ret = ena_com_get_feature(ena_dev, &get_feat_resp, ENA_ADMIN_PHC_CONFIG, 0); + if (unlikely(ret)) { + ena_trc_err(ena_dev, "Failed to get PHC feature configuration, error: %d\n", ret); + return ret; + } + + /* Suporting only readless PHC retrieval */ + if (get_feat_resp.u.phc.type != ENA_ADMIN_PHC_TYPE_READLESS) { + ena_trc_err(ena_dev, "Unsupprted PHC type, error: %d\n", ENA_COM_UNSUPPORTED); + return ENA_COM_UNSUPPORTED; + } + + /* Update PHC doorbell offset according to device value, used to write req_id to PHC bar */ + phc->doorbell_offset = get_feat_resp.u.phc.doorbell_offset; + + /* Update PHC expire timeout according to device or default driver value */ + phc->expire_timeout_usec = (get_feat_resp.u.phc.expire_timeout_usec) ? + get_feat_resp.u.phc.expire_timeout_usec : + ENA_PHC_DEFAULT_EXPIRE_TIMEOUT_USEC; + + /* Update PHC block timeout according to device or default driver value */ + phc->block_timeout_usec = (get_feat_resp.u.phc.block_timeout_usec) ? + get_feat_resp.u.phc.block_timeout_usec : + ENA_PHC_DEFAULT_BLOCK_TIMEOUT_USEC; + + /* Sanity check - expire timeout must not be above skip timeout */ + if (phc->expire_timeout_usec > phc->block_timeout_usec) + phc->expire_timeout_usec = phc->block_timeout_usec; + + /* Prepare PHC feature command with PHC output address */ + memset(&set_feat_cmd, 0x0, sizeof(set_feat_cmd)); + set_feat_cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; + set_feat_cmd.feat_common.feature_id = ENA_ADMIN_PHC_CONFIG; + set_feat_cmd.u.phc.output_length = sizeof(*phc->virt_addr); + ret = ena_com_mem_addr_set(ena_dev, &set_feat_cmd.u.phc.output_address, phc->phys_addr); + if (unlikely(ret)) { + ena_trc_err(ena_dev, "Failed setting PHC output address, error: %d\n", ret); + return ret; + } + + /* Send PHC feature command to the device */ + ret = ena_com_execute_admin_command(&ena_dev->admin_queue, + (struct ena_admin_aq_entry *)&set_feat_cmd, + sizeof(set_feat_cmd), + (struct ena_admin_acq_entry *)&set_feat_resp, + sizeof(set_feat_resp)); + + if (unlikely(ret)) { + ena_trc_err(ena_dev, "Failed to enable PHC, error: %d\n", ret); + return ret; + } + + phc->active = true; + ena_trc_dbg(ena_dev, "PHC is active in the device\n"); + + return ret; +} + +void ena_com_phc_destroy(struct ena_com_dev *ena_dev) +{ + struct ena_com_phc_info *phc = &ena_dev->phc; + + phc->active = false; + + /* In case PHC is not supported by the device, silently exiting */ + if (!phc->virt_addr) + return; + + ENA_MEM_FREE_COHERENT(ena_dev->dmadev, + sizeof(*phc->virt_addr), + phc->virt_addr, + phc->phys_addr, + phc->mem_handle); + phc->virt_addr = NULL; + + ENA_SPINLOCK_DESTROY(phc->lock); +} + +int ena_com_phc_get(struct ena_com_dev *ena_dev, u64 *timestamp) +{ + volatile struct ena_admin_phc_resp *read_resp = ena_dev->phc.virt_addr; + struct ena_com_phc_info *phc = &ena_dev->phc; + ena_time_high_res_t initial_time = ENA_TIME_INIT_HIGH_RES(); + static ena_time_high_res_t start_time; + unsigned long flags = 0; + ena_time_high_res_t expire_time; + ena_time_high_res_t block_time; + int ret = ENA_COM_OK; + + if (!phc->active) { + ena_trc_err(ena_dev, "PHC feature is not active in the device\n"); + return ENA_COM_UNSUPPORTED; + } + + ENA_SPINLOCK_LOCK(phc->lock, flags); + + /* Check if PHC is in blocked state */ + if (unlikely(ENA_TIME_COMPARE_HIGH_RES(start_time, initial_time))) { + /* Check if blocking time expired */ + block_time = ENA_GET_SYSTEM_TIMEOUT_HIGH_RES(start_time, phc->block_timeout_usec); + if (!ENA_TIME_EXPIRE_HIGH_RES(block_time)) { + /* PHC is still in blocked state, skip PHC request */ + phc->stats.phc_skp++; + ret = ENA_COM_DEVICE_BUSY; + goto skip; + } + + /* PHC is in active state, update statistics according to req_id and timestamp */ + if ((READ_ONCE16(read_resp->req_id) != phc->req_id) || + (read_resp->timestamp == ENA_PHC_TIMESTAMP_ERROR)) { + /* Device didn't update req_id during blocking time or timestamp is invalid, + * this indicates on a device error + */ + phc->stats.phc_err++; + } else { + /* Device updated req_id during blocking time with valid timestamp */ + phc->stats.phc_exp++; + } + } + + /* Setting relative timeouts */ + start_time = ENA_GET_SYSTEM_TIME_HIGH_RES(); + block_time = ENA_GET_SYSTEM_TIMEOUT_HIGH_RES(start_time, phc->block_timeout_usec); + expire_time = ENA_GET_SYSTEM_TIMEOUT_HIGH_RES(start_time, phc->expire_timeout_usec); + + /* We expect the device to return this req_id once the new PHC timestamp is updated */ + phc->req_id++; + + /* Initialize PHC shared memory with different req_id value to be able to identify once the + * device changes it to req_id + */ + read_resp->req_id = phc->req_id + ENA_PHC_REQ_ID_OFFSET; + + /* Writing req_id to PHC bar */ + ENA_REG_WRITE32(ena_dev->bus, phc->req_id, ena_dev->reg_bar + phc->doorbell_offset); + + /* Stalling until the device updates req_id */ + while (1) { + if (unlikely(ENA_TIME_EXPIRE_HIGH_RES(expire_time))) { + /* Gave up waiting for updated req_id, PHC enters into blocked state until + * passing blocking time + */ + ret = ENA_COM_DEVICE_BUSY; + break; + } + + /* Check if req_id was updated by the device */ + if (READ_ONCE16(read_resp->req_id) != phc->req_id) { + /* req_id was not updated by the device, check again on next loop */ + continue; + } + + /* req_id was updated which indicates that PHC timestamp was updated too */ + *timestamp = read_resp->timestamp; + + /* PHC timestamp validty check */ + if (unlikely(*timestamp == ENA_PHC_TIMESTAMP_ERROR)) { + /* Retrieved invalid PHC timestamp, PHC enters into blocked state until + * passing blocking time + */ + ret = ENA_COM_DEVICE_BUSY; + break; + } + + /* Retrieved valid PHC timestamp */ + phc->stats.phc_cnt++; + + /* This indicates PHC state is active */ + start_time = initial_time; + break; + } + +skip: + ENA_SPINLOCK_UNLOCK(phc->lock, flags); + + return ret; +} + int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev) { struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; @@ -2011,6 +2224,55 @@ int ena_com_get_link_params(struct ena_com_dev *ena_dev, return ena_com_get_feature(ena_dev, resp, ENA_ADMIN_LINK_CONFIG, 0); } +static int ena_get_dev_stats(struct ena_com_dev *ena_dev, + struct ena_com_stats_ctx *ctx, + enum ena_admin_get_stats_type type) +{ + struct ena_admin_acq_get_stats_resp *get_resp = &ctx->get_resp; + struct ena_admin_aq_get_stats_cmd *get_cmd = &ctx->get_cmd; + struct ena_com_admin_queue *admin_queue; + int ret; + + admin_queue = &ena_dev->admin_queue; + + get_cmd->aq_common_descriptor.opcode = ENA_ADMIN_GET_STATS; + get_cmd->aq_common_descriptor.flags = 0; + get_cmd->type = type; + + ret = ena_com_execute_admin_command(admin_queue, + (struct ena_admin_aq_entry *)get_cmd, + sizeof(*get_cmd), + (struct ena_admin_acq_entry *)get_resp, + sizeof(*get_resp)); + + if (unlikely(ret)) + ena_trc_err(ena_dev, "Failed to get stats. error: %d\n", ret); + + return ret; +} + +static void ena_com_set_supported_customer_metrics(struct ena_com_dev *ena_dev) +{ + struct ena_customer_metrics *customer_metrics; + struct ena_com_stats_ctx ctx; + int ret; + + customer_metrics = &ena_dev->customer_metrics; + if (!ena_com_get_cap(ena_dev, ENA_ADMIN_CUSTOMER_METRICS)) { + customer_metrics->supported_metrics = ENA_ADMIN_CUSTOMER_METRICS_MIN_SUPPORT_MASK; + return; + } + + memset(&ctx, 0x0, sizeof(ctx)); + ctx.get_cmd.requested_metrics = ENA_ADMIN_CUSTOMER_METRICS_SUPPORT_MASK; + ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_CUSTOMER_METRICS); + if (likely(ret == 0)) + customer_metrics->supported_metrics = + ctx.get_resp.u.customer_metrics.reported_metrics; + else + ena_trc_err(ena_dev, "Failed to query customer metrics support. error: %d\n", ret); +} + int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev, struct ena_com_dev_get_features_ctx *get_feat_ctx) { @@ -2026,6 +2288,7 @@ int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev, sizeof(get_resp.u.dev_attr)); ena_dev->supported_features = get_resp.u.dev_attr.supported_features; + ena_dev->capabilities = get_resp.u.dev_attr.capabilities; if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { rc = ena_com_get_feature(ena_dev, &get_resp, @@ -2091,6 +2354,8 @@ int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev, else return rc; + ena_com_set_supported_customer_metrics(ena_dev); + return 0; } @@ -2262,39 +2527,17 @@ int ena_com_dev_reset(struct ena_com_dev *ena_dev, return 0; } -static int ena_get_dev_stats(struct ena_com_dev *ena_dev, - struct ena_com_stats_ctx *ctx, - enum ena_admin_get_stats_type type) -{ - struct ena_admin_aq_get_stats_cmd *get_cmd = &ctx->get_cmd; - struct ena_admin_acq_get_stats_resp *get_resp = &ctx->get_resp; - struct ena_com_admin_queue *admin_queue; - int ret; - - admin_queue = &ena_dev->admin_queue; - - get_cmd->aq_common_descriptor.opcode = ENA_ADMIN_GET_STATS; - get_cmd->aq_common_descriptor.flags = 0; - get_cmd->type = type; - - ret = ena_com_execute_admin_command(admin_queue, - (struct ena_admin_aq_entry *)get_cmd, - sizeof(*get_cmd), - (struct ena_admin_acq_entry *)get_resp, - sizeof(*get_resp)); - - if (unlikely(ret)) - ena_trc_err(ena_dev, "Failed to get stats. error: %d\n", ret); - - return ret; -} - int ena_com_get_eni_stats(struct ena_com_dev *ena_dev, struct ena_admin_eni_stats *stats) { struct ena_com_stats_ctx ctx; int ret; + if (!ena_com_get_cap(ena_dev, ENA_ADMIN_ENI_STATS)) { + ena_trc_err(ena_dev, "Capability %d isn't supported\n", ENA_ADMIN_ENI_STATS); + return ENA_COM_UNSUPPORTED; + } + memset(&ctx, 0x0, sizeof(ctx)); ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_ENI); if (likely(ret == 0)) @@ -2304,6 +2547,26 @@ int ena_com_get_eni_stats(struct ena_com_dev *ena_dev, return ret; } +int ena_com_get_ena_srd_info(struct ena_com_dev *ena_dev, + struct ena_admin_ena_srd_info *info) +{ + struct ena_com_stats_ctx ctx; + int ret; + + if (!ena_com_get_cap(ena_dev, ENA_ADMIN_ENA_SRD_INFO)) { + ena_trc_err(ena_dev, "Capability %d isn't supported\n", ENA_ADMIN_ENA_SRD_INFO); + return ENA_COM_UNSUPPORTED; + } + + memset(&ctx, 0x0, sizeof(ctx)); + ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_ENA_SRD); + if (likely(ret == 0)) + memcpy(info, &ctx.get_resp.u.ena_srd_info, + sizeof(ctx.get_resp.u.ena_srd_info)); + + return ret; +} + int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev, struct ena_admin_basic_stats *stats) { @@ -2364,6 +2627,48 @@ done: } #endif +int ena_com_get_customer_metrics(struct ena_com_dev *ena_dev, char *buffer, u32 len) +{ + struct ena_admin_aq_get_stats_cmd *get_cmd; + struct ena_com_stats_ctx ctx; + int ret; + + if (unlikely(len > ena_dev->customer_metrics.buffer_len)) { + ena_trc_err(ena_dev, "Invalid buffer size %u. The given buffer is too big.\n", len); + return ENA_COM_INVAL; + } + + if (!ena_com_get_cap(ena_dev, ENA_ADMIN_CUSTOMER_METRICS)) { + ena_trc_err(ena_dev, "Capability %d not supported.\n", ENA_ADMIN_CUSTOMER_METRICS); + return ENA_COM_UNSUPPORTED; + } + + if (!ena_dev->customer_metrics.supported_metrics) { + ena_trc_err(ena_dev, "No supported customer metrics.\n"); + return ENA_COM_UNSUPPORTED; + } + + get_cmd = &ctx.get_cmd; + memset(&ctx, 0x0, sizeof(ctx)); + ret = ena_com_mem_addr_set(ena_dev, + &get_cmd->u.control_buffer.address, + ena_dev->customer_metrics.buffer_dma_addr); + if (unlikely(ret)) { + ena_trc_err(ena_dev, "Memory address set failed.\n"); + return ret; + } + + get_cmd->u.control_buffer.length = ena_dev->customer_metrics.buffer_len; + get_cmd->requested_metrics = ena_dev->customer_metrics.supported_metrics; + ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_CUSTOMER_METRICS); + if (likely(ret == 0)) + memcpy(buffer, ena_dev->customer_metrics.buffer_virt_addr, len); + else + ena_trc_err(ena_dev, "Failed to get customer metrics. error: %d\n", ret); + + return ret; +} + int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, u32 mtu) { struct ena_com_admin_queue *admin_queue; @@ -2503,27 +2808,17 @@ int ena_com_fill_hash_function(struct ena_com_dev *ena_dev, return ENA_COM_UNSUPPORTED; } - switch (func) { - case ENA_ADMIN_TOEPLITZ: - if (key) { - if (key_len != sizeof(hash_key->key)) { - ena_trc_err(ena_dev, "key len (%hu) doesn't equal the supported size (%zu)\n", - key_len, sizeof(hash_key->key)); - return ENA_COM_INVAL; - } - memcpy(hash_key->key, key, key_len); - rss->hash_init_val = init_val; - hash_key->key_parts = key_len / sizeof(hash_key->key[0]); + if ((func == ENA_ADMIN_TOEPLITZ) && key) { + if (key_len != sizeof(hash_key->key)) { + ena_trc_err(ena_dev, "key len (%u) doesn't equal the supported size (%zu)\n", + key_len, sizeof(hash_key->key)); + return ENA_COM_INVAL; } - break; - case ENA_ADMIN_CRC32: - rss->hash_init_val = init_val; - break; - default: - ena_trc_err(ena_dev, "Invalid hash function (%d)\n", func); - return ENA_COM_INVAL; + memcpy(hash_key->key, key, key_len); + hash_key->key_parts = key_len / sizeof(hash_key->key[0]); } + rss->hash_init_val = init_val; old_func = rss->hash_func; rss->hash_func = func; rc = ena_com_set_hash_function(ena_dev); @@ -2917,6 +3212,22 @@ int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev, return 0; } +int ena_com_allocate_customer_metrics_buffer(struct ena_com_dev *ena_dev) +{ + struct ena_customer_metrics *customer_metrics = &ena_dev->customer_metrics; + + customer_metrics->buffer_len = ENA_CUSTOMER_METRICS_BUFFER_SIZE; + ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, + customer_metrics->buffer_len, + customer_metrics->buffer_virt_addr, + customer_metrics->buffer_dma_addr, + customer_metrics->buffer_dma_handle); + if (!customer_metrics->buffer_virt_addr) + return ENA_COM_NO_MEM; + + return 0; +} + void ena_com_delete_host_info(struct ena_com_dev *ena_dev) { struct ena_host_attribute *host_attr = &ena_dev->host_attr; @@ -2945,6 +3256,20 @@ void ena_com_delete_debug_area(struct ena_com_dev *ena_dev) } } +void ena_com_delete_customer_metrics_buffer(struct ena_com_dev *ena_dev) +{ + struct ena_customer_metrics *customer_metrics = &ena_dev->customer_metrics; + + if (customer_metrics->buffer_virt_addr) { + ENA_MEM_FREE_COHERENT(ena_dev->dmadev, + customer_metrics->buffer_len, + customer_metrics->buffer_virt_addr, + customer_metrics->buffer_dma_addr, + customer_metrics->buffer_dma_handle); + customer_metrics->buffer_virt_addr = NULL; + } +} + int ena_com_set_host_attributes(struct ena_com_dev *ena_dev) { struct ena_host_attribute *host_attr = &ena_dev->host_attr; diff --git a/sys/contrib/ena-com/ena_com.h b/sys/contrib/ena-com/ena_com.h index 7ff0e00c1f17..5b091c64572c 100644 --- a/sys/contrib/ena-com/ena_com.h +++ b/sys/contrib/ena-com/ena_com.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * - * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates. + * Copyright (c) 2015-2023 Amazon.com, Inc. or its affiliates. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -51,12 +51,14 @@ #define ADMIN_CQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_acq_entry)) #define ADMIN_AENQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_aenq_entry)) +#define ENA_CUSTOMER_METRICS_BUFFER_SIZE 512 + /*****************************************************************************/ /*****************************************************************************/ /* ENA adaptive interrupt moderation settings */ #define ENA_INTR_INITIAL_TX_INTERVAL_USECS ENA_INTR_INITIAL_TX_INTERVAL_USECS_PLAT -#define ENA_INTR_INITIAL_RX_INTERVAL_USECS 0 +#define ENA_INTR_INITIAL_RX_INTERVAL_USECS ENA_INTR_INITIAL_RX_INTERVAL_USECS_PLAT #define ENA_DEFAULT_INTR_DELAY_RESOLUTION 1 #define ENA_HASH_KEY_SIZE 40 @@ -120,8 +122,6 @@ struct ena_com_io_cq { /* Interrupt unmask register */ u32 __iomem *unmask_reg; - /* The completion queue head doorbell register */ - u32 __iomem *cq_head_db_reg; /* numa configuration register (for TPH) */ u32 __iomem *numa_node_cfg_reg; @@ -129,13 +129,13 @@ struct ena_com_io_cq { /* The value to write to the above register to unmask * the interrupt of this queue */ - u32 msix_vector; + u32 msix_vector ____cacheline_aligned; enum queue_direction direction; /* holds the number of cdesc of the current packet */ u16 cur_rx_pkt_cdesc_count; - /* save the firt cdesc idx of the current packet */ + /* save the first cdesc idx of the current packet */ u16 cur_rx_pkt_cdesc_start_idx; u16 q_depth; @@ -145,7 +145,6 @@ struct ena_com_io_cq { /* Device queue index */ u16 idx; u16 head; - u16 last_head_update; u8 phase; u8 cdesc_entry_size_in_bytes; @@ -170,7 +169,6 @@ struct ena_com_io_sq { void *bus; u32 __iomem *db_addr; - u8 __iomem *header_addr; enum queue_direction direction; enum ena_admin_placement_policy_type mem_queue_type; @@ -227,6 +225,13 @@ struct ena_com_stats_admin { u64 no_completion; }; +struct ena_com_stats_phc { + u64 phc_cnt; + u64 phc_exp; + u64 phc_skp; + u64 phc_err; +}; + struct ena_com_admin_queue { void *q_dmadev; void *bus; @@ -281,6 +286,46 @@ struct ena_com_mmio_read { ena_spinlock_t lock; }; +/* PTP hardware clock (PHC) MMIO read data info */ +struct ena_com_phc_info { + /* Internal PHC statistics */ + struct ena_com_stats_phc stats; + + /* PHC shared memory - virtual address */ + struct ena_admin_phc_resp *virt_addr; + + /* Spin lock to ensure a single outstanding PHC read */ + ena_spinlock_t lock; + + /* PHC doorbell address as an offset to PCIe MMIO REG BAR */ + u32 doorbell_offset; + + /* Shared memory read expire timeout (usec) + * Max time for valid PHC retrieval, passing this threshold will fail the get time request + * and block new PHC requests for block_timeout_usec in order to prevent floods on busy + * device + */ + u32 expire_timeout_usec; + + /* Shared memory read abort timeout (usec) + * PHC requests block period, blocking starts once PHC request expired in order to prevent + * floods on busy device, any PHC requests during block period will be skipped + */ + u32 block_timeout_usec; + + /* Request id sent to the device */ + u16 req_id; + + /* True if PHC is active in the device */ + bool active; + + /* PHC shared memory - memory handle */ + ena_mem_handle_t mem_handle; + + /* PHC shared memory - physical address */ + dma_addr_t phys_addr; +}; + struct ena_rss { /* Indirect table */ u16 *host_rss_ind_tbl; @@ -303,6 +348,17 @@ struct ena_rss { }; +struct ena_customer_metrics { + /* in correlation with ENA_ADMIN_CUSTOMER_METRICS_SUPPORT_MASK + * and ena_admin_customer_metrics_id + */ + uint64_t supported_metrics; + dma_addr_t buffer_dma_addr; + void *buffer_virt_addr; + ena_mem_handle_t buffer_dma_handle; + u32 buffer_len; +}; + struct ena_host_attribute { /* Debug area */ u8 *debug_area_virt_addr; @@ -333,10 +389,14 @@ struct ena_com_dev { u16 stats_func; /* Selected function for extended statistic dump */ u16 stats_queue; /* Selected queue for extended statistic dump */ + u32 ena_min_poll_delay_us; + struct ena_com_mmio_read mmio_read; + struct ena_com_phc_info phc; struct ena_rss rss; u32 supported_features; + u32 capabilities; u32 dma_addr_bits; struct ena_host_attribute host_attr; @@ -353,7 +413,7 @@ struct ena_com_dev { struct ena_com_llq_info llq_info; - u32 ena_min_poll_delay_us; + struct ena_customer_metrics customer_metrics; }; struct ena_com_dev_get_features_ctx { @@ -401,6 +461,40 @@ extern "C" { */ int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev); +/* ena_com_phc_init - Allocate and initialize PHC feature + * @ena_dev: ENA communication layer struct + * @note: This method assumes PHC is supported by the device + * @return - 0 on success, negative value on failure + */ +int ena_com_phc_init(struct ena_com_dev *ena_dev); + +/* ena_com_phc_supported - Return if PHC feature is supported by the device + * @ena_dev: ENA communication layer struct + * @note: This method must be called after getting supported features + * @return - supported or not + */ +bool ena_com_phc_supported(struct ena_com_dev *ena_dev); + +/* ena_com_phc_config - Configure PHC feature + * @ena_dev: ENA communication layer struct + * Configure PHC feature in driver and device + * @note: This method assumes PHC is supported by the device + * @return - 0 on success, negative value on failure + */ +int ena_com_phc_config(struct ena_com_dev *ena_dev); + +/* ena_com_phc_destroy - Destroy PHC feature + * @ena_dev: ENA communication layer struct + */ +void ena_com_phc_destroy(struct ena_com_dev *ena_dev); + +/* ena_com_phc_get - Retrieve PHC timestamp + * @ena_dev: ENA communication layer struct + * @timestamp: Retrieve PHC timestamp + * @return - 0 on success, negative value on failure + */ +int ena_com_phc_get(struct ena_com_dev *ena_dev, u64 *timestamp); + /* ena_com_set_mmio_read_mode - Enable/disable the indirect mmio reg read mechanism * @ena_dev: ENA communication layer struct * @readless_supported: readless mode (enable/disable) @@ -637,6 +731,24 @@ int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev, int ena_com_get_eni_stats(struct ena_com_dev *ena_dev, struct ena_admin_eni_stats *stats); +/* ena_com_get_ena_srd_info - Get ENA SRD network interface statistics + * @ena_dev: ENA communication layer struct + * @info: ena srd stats and flags + * + * @return: 0 on Success and negative value otherwise. + */ +int ena_com_get_ena_srd_info(struct ena_com_dev *ena_dev, + struct ena_admin_ena_srd_info *info); + +/* ena_com_get_customer_metrics - Get customer metrics for network interface + * @ena_dev: ENA communication layer struct + * @buffer: buffer for returned customer metrics + * @len: size of the buffer + * + * @return: 0 on Success and negative value otherwise. + */ +int ena_com_get_customer_metrics(struct ena_com_dev *ena_dev, char *buffer, u32 len); + /* ena_com_set_dev_mtu - Configure the device mtu. * @ena_dev: ENA communication layer struct * @mtu: mtu value @@ -847,6 +959,13 @@ int ena_com_allocate_host_info(struct ena_com_dev *ena_dev); int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev, u32 debug_area_size); +/* ena_com_allocate_customer_metrics_buffer - Allocate customer metrics resources. + * @ena_dev: ENA communication layer struct + * + * @return: 0 on Success and negative value otherwise. + */ +int ena_com_allocate_customer_metrics_buffer(struct ena_com_dev *ena_dev); + /* ena_com_delete_debug_area - Free the debug area resources. * @ena_dev: ENA communication layer struct * @@ -861,6 +980,13 @@ void ena_com_delete_debug_area(struct ena_com_dev *ena_dev); */ void ena_com_delete_host_info(struct ena_com_dev *ena_dev); +/* ena_com_delete_customer_metrics_buffer - Free the customer metrics resources. + * @ena_dev: ENA communication layer struct + * + * Free the allocated customer metrics area. + */ +void ena_com_delete_customer_metrics_buffer(struct ena_com_dev *ena_dev); + /* ena_com_set_host_attributes - Update the device with the host * attributes (debug area and host info) base address. * @ena_dev: ENA communication layer struct @@ -1005,18 +1131,55 @@ static inline void ena_com_disable_adaptive_moderation(struct ena_com_dev *ena_d ena_dev->adaptive_coalescing = false; } +/* ena_com_get_cap - query whether device supports a capability. + * @ena_dev: ENA communication layer struct + * @cap_id: enum value representing the capability + * + * @return - true if capability is supported or false otherwise + */ +static inline bool ena_com_get_cap(struct ena_com_dev *ena_dev, + enum ena_admin_aq_caps_id cap_id) +{ + return !!(ena_dev->capabilities & BIT(cap_id)); +} + +/* ena_com_get_customer_metric_support - query whether device supports a given customer metric. + * @ena_dev: ENA communication layer struct + * @metric_id: enum value representing the customer metric + * + * @return - true if customer metric is supported or false otherwise + */ +static inline bool ena_com_get_customer_metric_support(struct ena_com_dev *ena_dev, + enum ena_admin_customer_metrics_id metric_id) +{ + return !!(ena_dev->customer_metrics.supported_metrics & BIT64(metric_id)); +} + +/* ena_com_get_customer_metric_count - return the number of supported customer metrics. + * @ena_dev: ENA communication layer struct + * + * @return - the number of supported customer metrics + */ +static inline int ena_com_get_customer_metric_count(struct ena_com_dev *ena_dev) +{ + return ENA_BITS_PER_U64(ena_dev->customer_metrics.supported_metrics); +} + /* ena_com_update_intr_reg - Prepare interrupt register * @intr_reg: interrupt register to update. * @rx_delay_interval: Rx interval in usecs * @tx_delay_interval: Tx interval in usecs * @unmask: unmask enable/disable + * @no_moderation_update: 0 - Indicates that any of the TX/RX intervals was + * updated, 1 - otherwise * * Prepare interrupt update register with the supplied parameters. */ static inline void ena_com_update_intr_reg(struct ena_eth_io_intr_reg *intr_reg, u32 rx_delay_interval, u32 tx_delay_interval, - bool unmask) + bool unmask, + bool no_moderation_update) { intr_reg->intr_control = 0; intr_reg->intr_control |= rx_delay_interval & @@ -1028,6 +1191,10 @@ static inline void ena_com_update_intr_reg(struct ena_eth_io_intr_reg *intr_reg, if (unmask) intr_reg->intr_control |= ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK; + + intr_reg->intr_control |= + (((u32)no_moderation_update) << ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_SHIFT) & + ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_MASK; } static inline u8 *ena_com_get_next_bounce_buffer(struct ena_com_io_bounce_buffer_control *bounce_buf_ctrl) diff --git a/sys/contrib/ena-com/ena_common_defs.h b/sys/contrib/ena-com/ena_common_defs.h deleted file mode 100644 index f2d8189860ae..000000000000 --- a/sys/contrib/ena-com/ena_common_defs.h +++ /dev/null @@ -1,50 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of copyright holder nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _ENA_COMMON_H_ -#define _ENA_COMMON_H_ - -#define ENA_COMMON_SPEC_VERSION_MAJOR 0 /* */ -#define ENA_COMMON_SPEC_VERSION_MINOR 10 /* */ - -/* ENA operates with 48-bit memory addresses. ena_mem_addr_t */ -struct ena_common_mem_addr { - uint32_t mem_addr_low; - - uint16_t mem_addr_high; - - /* MBZ */ - uint16_t reserved16; -}; - -#endif /*_ENA_COMMON_H_ */ diff --git a/sys/contrib/ena-com/ena_defs/ena_admin_defs.h b/sys/contrib/ena-com/ena_defs/ena_admin_defs.h index 33010bd1a2e8..abd1ae5d57fb 100644 --- a/sys/contrib/ena-com/ena_defs/ena_admin_defs.h +++ b/sys/contrib/ena-com/ena_defs/ena_admin_defs.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * - * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates. + * Copyright (c) 2015-2023 Amazon.com, Inc. or its affiliates. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,6 +30,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + #ifndef _ENA_ADMIN_H_ #define _ENA_ADMIN_H_ @@ -38,6 +39,21 @@ #define ENA_ADMIN_RSS_KEY_PARTS 10 +#define ENA_ADMIN_CUSTOMER_METRICS_SUPPORT_MASK 0x3F +#define ENA_ADMIN_CUSTOMER_METRICS_MIN_SUPPORT_MASK 0x1F + + /* customer metrics - in correlation with + * ENA_ADMIN_CUSTOMER_METRICS_SUPPORT_MASK + */ +enum ena_admin_customer_metrics_id { + ENA_ADMIN_BW_IN_ALLOWANCE_EXCEEDED = 0, + ENA_ADMIN_BW_OUT_ALLOWANCE_EXCEEDED = 1, + ENA_ADMIN_PPS_ALLOWANCE_EXCEEDED = 2, + ENA_ADMIN_CONNTRACK_ALLOWANCE_EXCEEDED = 3, + ENA_ADMIN_LINKLOCAL_ALLOWANCE_EXCEEDED = 4, + ENA_ADMIN_CONNTRACK_ALLOWANCE_AVAILABLE = 5, +}; + enum ena_admin_aq_opcode { ENA_ADMIN_CREATE_SQ = 1, ENA_ADMIN_DESTROY_SQ = 2, @@ -78,9 +94,18 @@ enum ena_admin_aq_feature_id { ENA_ADMIN_AENQ_CONFIG = 26, ENA_ADMIN_LINK_CONFIG = 27, ENA_ADMIN_HOST_ATTR_CONFIG = 28, + ENA_ADMIN_PHC_CONFIG = 29, ENA_ADMIN_FEATURES_OPCODE_NUM = 32, }; +/* device capabilities */ +enum ena_admin_aq_caps_id { + ENA_ADMIN_ENI_STATS = 0, + /* ENA SRD customer metrics */ + ENA_ADMIN_ENA_SRD_INFO = 1, + ENA_ADMIN_CUSTOMER_METRICS = 2, +}; + enum ena_admin_placement_policy_type { /* descriptors and headers are in host memory */ ENA_ADMIN_PLACEMENT_POLICY_HOST = 1, @@ -127,6 +152,10 @@ enum ena_admin_get_stats_type { ENA_ADMIN_GET_STATS_TYPE_EXTENDED = 1, /* extra HW stats for specific network interface */ ENA_ADMIN_GET_STATS_TYPE_ENI = 2, + /* extra HW stats for ENA SRD */ + ENA_ADMIN_GET_STATS_TYPE_ENA_SRD = 3, + ENA_ADMIN_GET_STATS_TYPE_CUSTOMER_METRICS = 4, + }; enum ena_admin_get_stats_scope { @@ -134,6 +163,20 @@ enum ena_admin_get_stats_scope { ENA_ADMIN_ETH_TRAFFIC = 1, }; +enum ena_admin_get_phc_type { + ENA_ADMIN_PHC_TYPE_READLESS = 0, +}; + +/* ENA SRD configuration for ENI */ +enum ena_admin_ena_srd_flags { + /* Feature enabled */ + ENA_ADMIN_ENA_SRD_ENABLED = BIT(0), + /* UDP support enabled */ + ENA_ADMIN_ENA_SRD_UDP_ENABLED = BIT(1), + /* Bypass Rx UDP ordering */ + ENA_ADMIN_ENA_SRD_UDP_ORDERING_BYPASS_ENABLED = BIT(2), +}; + struct ena_admin_aq_common_desc { /* 11:0 : command_id * 15:12 : reserved12 @@ -391,6 +434,9 @@ struct ena_admin_aq_get_stats_cmd { * stats of other device */ uint16_t device_id; + + /* a bitmap representing the requested metric values */ + uint64_t requested_metrics; }; /* Basic Statistics Command. */ @@ -447,6 +493,40 @@ struct ena_admin_eni_stats { uint64_t linklocal_allowance_exceeded; }; +struct ena_admin_ena_srd_stats { + /* Number of packets transmitted over ENA SRD */ + uint64_t ena_srd_tx_pkts; + + /* Number of packets transmitted or could have been + * transmitted over ENA SRD + */ + uint64_t ena_srd_eligible_tx_pkts; + + /* Number of packets received over ENA SRD */ + uint64_t ena_srd_rx_pkts; + + /* Percentage of the ENA SRD resources that is in use */ + uint64_t ena_srd_resource_utilization; +}; + +/* ENA SRD Statistics Command */ +struct ena_admin_ena_srd_info { + /* ENA SRD configuration bitmap. See ena_admin_ena_srd_flags for + * details + */ + uint64_t flags; + + struct ena_admin_ena_srd_stats ena_srd_stats; +}; + +/* Customer Metrics Command. */ +struct ena_admin_customer_metrics { + /* A bitmap representing the reported customer metrics according to + * the order they are reported + */ + uint64_t reported_metrics; +}; + struct ena_admin_acq_get_stats_resp { struct ena_admin_acq_common_desc acq_common_desc; @@ -456,6 +536,10 @@ struct ena_admin_acq_get_stats_resp { struct ena_admin_basic_stats basic_stats; struct ena_admin_eni_stats eni_stats; + + struct ena_admin_ena_srd_info ena_srd_info; + + struct ena_admin_customer_metrics customer_metrics; } u; }; @@ -488,7 +572,10 @@ struct ena_admin_device_attr_feature_desc { */ uint32_t supported_features; - uint32_t reserved3; + /* bitmap of ena_admin_aq_caps_id, which represents device + * capabilities. + */ + uint32_t capabilities; /* Indicates how many bits are used physical address access. */ uint32_t phys_addr_width; @@ -990,6 +1077,43 @@ struct ena_admin_queue_ext_feature_desc { }; }; +struct ena_admin_feature_phc_desc { + /* PHC type as defined in enum ena_admin_get_phc_type, + * used only for GET command. + */ + uint8_t type; + + /* Reserved - MBZ */ + uint8_t reserved1[3]; + + /* PHC doorbell address as an offset to PCIe MMIO REG BAR, + * used only for GET command. + */ + uint32_t doorbell_offset; + + /* Max time for valid PHC retrieval, passing this threshold will + * fail the get-time request and block PHC requests for + * block_timeout_usec, used only for GET command. + */ + uint32_t expire_timeout_usec; + + /* PHC requests block period, blocking starts if PHC request expired + * in order to prevent floods on busy device, + * used only for GET command. + */ + uint32_t block_timeout_usec; + + /* Shared PHC physical address (ena_admin_phc_resp), + * used only for SET command. + */ + struct ena_common_mem_addr output_address; + + /* Shared PHC Size (ena_admin_phc_resp), + * used only for SET command. + */ + uint32_t output_length; +}; + struct ena_admin_get_feat_resp { struct ena_admin_acq_common_desc acq_common_desc; @@ -1020,6 +1144,8 @@ struct ena_admin_get_feat_resp { struct ena_admin_ena_hw_hints hw_hints; + struct ena_admin_feature_phc_desc phc; + struct ena_admin_get_extra_properties_strings_desc extra_properties_strings; struct ena_admin_get_extra_properties_flags_desc extra_properties_flags; @@ -1056,6 +1182,9 @@ struct ena_admin_set_feat_cmd { /* LLQ configuration */ struct ena_admin_feature_llq_desc llq; + + /* PHC configuration */ + struct ena_admin_feature_phc_desc phc; } u; }; @@ -1091,7 +1220,8 @@ enum ena_admin_aenq_group { ENA_ADMIN_WARNING = 2, ENA_ADMIN_NOTIFICATION = 3, ENA_ADMIN_KEEP_ALIVE = 4, - ENA_ADMIN_AENQ_GROUPS_NUM = 5, + ENA_ADMIN_REFRESH_CAPABILITIES = 5, + ENA_ADMIN_AENQ_GROUPS_NUM = 6, }; enum ena_admin_aenq_notification_syndrome { @@ -1133,6 +1263,16 @@ struct ena_admin_ena_mmio_req_read_less_resp { uint32_t reg_val; }; +struct ena_admin_phc_resp { + uint16_t req_id; + + uint8_t reserved1[6]; + + uint64_t timestamp; + + uint8_t reserved2[48]; +}; + /* aq_common_desc */ #define ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK GENMASK(11, 0) #define ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK BIT(0) diff --git a/sys/contrib/ena-com/ena_defs/ena_common_defs.h b/sys/contrib/ena-com/ena_defs/ena_common_defs.h index ee49ff6e2776..55173f39f47f 100644 --- a/sys/contrib/ena-com/ena_defs/ena_common_defs.h +++ b/sys/contrib/ena-com/ena_defs/ena_common_defs.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * - * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. + * Copyright (c) 2015-2023 Amazon.com, Inc. or its affiliates. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,6 +30,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + #ifndef _ENA_COMMON_H_ #define _ENA_COMMON_H_ diff --git a/sys/contrib/ena-com/ena_defs/ena_eth_io_defs.h b/sys/contrib/ena-com/ena_defs/ena_eth_io_defs.h index 817375a947c5..62e0762e54ce 100644 --- a/sys/contrib/ena-com/ena_defs/ena_eth_io_defs.h +++ b/sys/contrib/ena-com/ena_defs/ena_eth_io_defs.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * - * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. + * Copyright (c) 2015-2023 Amazon.com, Inc. or its affiliates. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,6 +30,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + #ifndef _ENA_ETH_IO_H_ #define _ENA_ETH_IO_H_ @@ -289,7 +290,8 @@ struct ena_eth_io_intr_reg { /* 14:0 : rx_intr_delay * 29:15 : tx_intr_delay * 30 : intr_unmask - * 31 : reserved + * 31 : no_moderation_update - 0 - moderation + * updated, 1 - moderation not updated */ uint32_t intr_control; }; @@ -409,6 +411,8 @@ struct ena_eth_io_numa_node_cfg_reg { #define ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK GENMASK(29, 15) #define ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT 30 #define ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK BIT(30) +#define ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_SHIFT 31 +#define ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_MASK BIT(31) /* numa_node_cfg_reg */ #define ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK GENMASK(7, 0) @@ -946,6 +950,16 @@ static inline void set_ena_eth_io_intr_reg_intr_unmask(struct ena_eth_io_intr_re p->intr_control |= (val << ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT) & ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK; } +static inline uint32_t get_ena_eth_io_intr_reg_no_moderation_update(const struct ena_eth_io_intr_reg *p) +{ + return (p->intr_control & ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_MASK) >> ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_SHIFT; +} + +static inline void set_ena_eth_io_intr_reg_no_moderation_update(struct ena_eth_io_intr_reg *p, uint32_t val) +{ + p->intr_control |= (val << ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_SHIFT) & ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_MASK; +} + static inline uint32_t get_ena_eth_io_numa_node_cfg_reg_numa(const struct ena_eth_io_numa_node_cfg_reg *p) { return p->numa_cfg & ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK; diff --git a/sys/contrib/ena-com/ena_defs/ena_regs_defs.h b/sys/contrib/ena-com/ena_defs/ena_regs_defs.h index bdd91ef2e026..f42a1074c12e 100644 --- a/sys/contrib/ena-com/ena_defs/ena_regs_defs.h +++ b/sys/contrib/ena-com/ena_defs/ena_regs_defs.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * - * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. + * Copyright (c) 2015-2023 Amazon.com, Inc. or its affiliates. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,6 +30,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + #ifndef _ENA_REGS_H_ #define _ENA_REGS_H_ @@ -49,6 +50,8 @@ enum ena_regs_reset_reason_types { ENA_REGS_RESET_USER_TRIGGER = 12, ENA_REGS_RESET_GENERIC = 13, ENA_REGS_RESET_MISS_INTERRUPT = 14, + ENA_REGS_RESET_SUSPECTED_POLL_STARVATION = 15, + ENA_REGS_RESET_RX_DESCRIPTOR_MALFORMED = 16, ENA_REGS_RESET_LAST, }; @@ -80,6 +83,11 @@ enum ena_regs_reset_reason_types { #define ENA_REGS_MMIO_RESP_HI_OFF 0x64 #define ENA_REGS_RSS_IND_ENTRY_UPDATE_OFF 0x68 +/* phc_registers offsets */ + +/* 100 base */ +#define ENA_REGS_PHC_DB_OFF 0x100 + /* version register */ #define ENA_REGS_VERSION_MINOR_VERSION_MASK 0xff #define ENA_REGS_VERSION_MAJOR_VERSION_SHIFT 8 @@ -156,4 +164,7 @@ enum ena_regs_reset_reason_types { #define ENA_REGS_RSS_IND_ENTRY_UPDATE_CQ_IDX_SHIFT 16 #define ENA_REGS_RSS_IND_ENTRY_UPDATE_CQ_IDX_MASK 0xffff0000 +/* phc_db_req_id register */ +#define ENA_REGS_PHC_DB_REQ_ID_MASK 0xffff + #endif /* _ENA_REGS_H_ */ diff --git a/sys/contrib/ena-com/ena_eth_com.c b/sys/contrib/ena-com/ena_eth_com.c index 2d66c34c0aa7..507c990c69af 100644 --- a/sys/contrib/ena-com/ena_eth_com.c +++ b/sys/contrib/ena-com/ena_eth_com.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * - * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates. + * Copyright (c) 2015-2023 Amazon.com, Inc. or its affiliates. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -260,30 +260,43 @@ static struct ena_eth_io_rx_cdesc_base * idx * io_cq->cdesc_entry_size_in_bytes); } -static u16 ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq, - u16 *first_cdesc_idx) +static int ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq, + u16 *first_cdesc_idx, + u16 *num_descs) { + u16 count = io_cq->cur_rx_pkt_cdesc_count, head_masked; struct ena_eth_io_rx_cdesc_base *cdesc; - u16 count = 0, head_masked; u32 last = 0; do { + u32 status; + cdesc = ena_com_get_next_rx_cdesc(io_cq); if (!cdesc) break; + status = READ_ONCE32(cdesc->status); ena_com_cq_inc_head(io_cq); + if (unlikely((status & ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK) >> + ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT && count != 0)) { + struct ena_com_dev *dev = ena_com_io_cq_to_ena_dev(io_cq); + + ena_trc_err(dev, + "First bit is on in descriptor #%d on q_id: %d, req_id: %u\n", + count, io_cq->qid, cdesc->req_id); + return ENA_COM_FAULT; + } count++; - last = (READ_ONCE32(cdesc->status) & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK) >> + last = (status & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT; } while (!last); if (last) { *first_cdesc_idx = io_cq->cur_rx_pkt_cdesc_start_idx; - count += io_cq->cur_rx_pkt_cdesc_count; head_masked = io_cq->head & (io_cq->q_depth - 1); + *num_descs = count; io_cq->cur_rx_pkt_cdesc_count = 0; io_cq->cur_rx_pkt_cdesc_start_idx = head_masked; @@ -291,11 +304,11 @@ static u16 ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq, "ENA q_id: %d packets were completed. first desc idx %u descs# %d\n", io_cq->qid, *first_cdesc_idx, count); } else { - io_cq->cur_rx_pkt_cdesc_count += count; - count = 0; + io_cq->cur_rx_pkt_cdesc_count = count; + *num_descs = 0; } - return count; + return ENA_COM_OK; } static int ena_com_create_meta(struct ena_com_io_sq *io_sq, @@ -354,9 +367,6 @@ static int ena_com_create_and_store_tx_meta_desc(struct ena_com_io_sq *io_sq, * compare it to the stored version, just create the meta */ if (io_sq->disable_meta_caching) { - if (unlikely(!ena_tx_ctx->meta_valid)) - return ENA_COM_INVAL; - *have_meta = true; return ena_com_create_meta(io_sq, ena_meta); } @@ -445,7 +455,7 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq, if (unlikely(io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV && !buffer_to_push)) { ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq), - "Push header wasn't provided on LLQ mode\n"); + "Push header wasn't provided in LLQ mode\n"); return ENA_COM_INVAL; } @@ -565,9 +575,6 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq, } rc = ena_com_close_bounce_buffer(io_sq); - if (rc) - ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq), - "Failed when closing bounce buffer\n"); *nb_hw_desc = io_sq->tail - start_tail; return rc; @@ -583,11 +590,15 @@ int ena_com_rx_pkt(struct ena_com_io_cq *io_cq, u16 cdesc_idx = 0; u16 nb_hw_desc; u16 i = 0; + int rc; ENA_WARN(io_cq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX, ena_com_io_cq_to_ena_dev(io_cq), "wrong Q type"); - nb_hw_desc = ena_com_cdesc_rx_pkt_get(io_cq, &cdesc_idx); + rc = ena_com_cdesc_rx_pkt_get(io_cq, &cdesc_idx, &nb_hw_desc); + if (unlikely(rc != ENA_COM_OK)) + return ENA_COM_FAULT; + if (nb_hw_desc == 0) { ena_rx_ctx->descs = nb_hw_desc; return 0; diff --git a/sys/contrib/ena-com/ena_eth_com.h b/sys/contrib/ena-com/ena_eth_com.h index 85675bb004b9..571d1d549786 100644 --- a/sys/contrib/ena-com/ena_eth_com.h +++ b/sys/contrib/ena-com/ena_eth_com.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * - * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. + * Copyright (c) 2015-2023 Amazon.com, Inc. or its affiliates. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -39,9 +39,6 @@ extern "C" { #endif #include "ena_com.h" -/* head update threshold in units of (queue size / ENA_COMP_HEAD_THRESH) */ -#define ENA_COMP_HEAD_THRESH 4 - struct ena_com_tx_ctx { struct ena_com_tx_meta ena_meta; struct ena_com_buf *ena_bufs; @@ -76,7 +73,7 @@ struct ena_com_rx_ctx { bool frag; u32 hash; u16 descs; - int max_bufs; + u16 max_bufs; u8 pkt_offset; }; @@ -199,28 +196,6 @@ static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq) return 0; } -static inline int ena_com_update_dev_comp_head(struct ena_com_io_cq *io_cq) -{ - u16 unreported_comp, head; - bool need_update; - - if (unlikely(io_cq->cq_head_db_reg)) { - head = io_cq->head; - unreported_comp = head - io_cq->last_head_update; - need_update = unreported_comp > (io_cq->q_depth / ENA_COMP_HEAD_THRESH); - - if (unlikely(need_update)) { - ena_trc_dbg(ena_com_io_cq_to_ena_dev(io_cq), - "Write completion queue doorbell for queue %d: head: %d\n", - io_cq->qid, head); - ENA_REG_WRITE32(io_cq->bus, head, io_cq->cq_head_db_reg); - io_cq->last_head_update = head; - } - } - - return 0; -} - static inline void ena_com_update_numa_node(struct ena_com_io_cq *io_cq, u8 numa_node) { diff --git a/sys/contrib/ena-com/ena_eth_io_defs.h b/sys/contrib/ena-com/ena_eth_io_defs.h deleted file mode 100644 index c16fed8e2dec..000000000000 --- a/sys/contrib/ena-com/ena_eth_io_defs.h +++ /dev/null @@ -1,960 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of copyright holder nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _ENA_ETH_IO_H_ -#define _ENA_ETH_IO_H_ - -enum ena_eth_io_l3_proto_index { - ENA_ETH_IO_L3_PROTO_UNKNOWN = 0, - - ENA_ETH_IO_L3_PROTO_IPV4 = 8, - - ENA_ETH_IO_L3_PROTO_IPV6 = 11, - - ENA_ETH_IO_L3_PROTO_FCOE = 21, - - ENA_ETH_IO_L3_PROTO_ROCE = 22, -}; - -enum ena_eth_io_l4_proto_index { - ENA_ETH_IO_L4_PROTO_UNKNOWN = 0, - - ENA_ETH_IO_L4_PROTO_TCP = 12, - - ENA_ETH_IO_L4_PROTO_UDP = 13, - - ENA_ETH_IO_L4_PROTO_ROUTEABLE_ROCE = 23, -}; - -struct ena_eth_io_tx_desc { - /* 15:0 : length - Buffer length in bytes, must - * include any packet trailers that the ENA supposed - * to update like End-to-End CRC, Authentication GMAC - * etc. This length must not include the - * 'Push_Buffer' length. This length must not include - * the 4-byte added in the end for 802.3 Ethernet FCS - * 21:16 : req_id_hi - Request ID[15:10] - * 22 : reserved22 - MBZ - * 23 : meta_desc - MBZ - * 24 : phase - * 25 : reserved1 - MBZ - * 26 : first - Indicates first descriptor in - * transaction - * 27 : last - Indicates last descriptor in - * transaction - * 28 : comp_req - Indicates whether completion - * should be posted, after packet is transmitted. - * Valid only for first descriptor - * 30:29 : reserved29 - MBZ - * 31 : reserved31 - MBZ - */ - uint32_t len_ctrl; - - /* 3:0 : l3_proto_idx - L3 protocol. This field - * required when l3_csum_en,l3_csum or tso_en are set. - * 4 : DF - IPv4 DF, must be 0 if packet is IPv4 and - * DF flags of the IPv4 header is 0. Otherwise must - * be set to 1 - * 6:5 : reserved5 - * 7 : tso_en - Enable TSO, For TCP only. - * 12:8 : l4_proto_idx - L4 protocol. This field need - * to be set when l4_csum_en or tso_en are set. - * 13 : l3_csum_en - enable IPv4 header checksum. - * 14 : l4_csum_en - enable TCP/UDP checksum. - * 15 : ethernet_fcs_dis - when set, the controller - * will not append the 802.3 Ethernet Frame Check - * Sequence to the packet - * 16 : reserved16 - * 17 : l4_csum_partial - L4 partial checksum. when - * set to 0, the ENA calculates the L4 checksum, - * where the Destination Address required for the - * TCP/UDP pseudo-header is taken from the actual - * packet L3 header. when set to 1, the ENA doesn't - * calculate the sum of the pseudo-header, instead, - * the checksum field of the L4 is used instead. When - * TSO enabled, the checksum of the pseudo-header - * must not include the tcp length field. L4 partial - * checksum should be used for IPv6 packet that - * contains Routing Headers. - * 20:18 : reserved18 - MBZ - * 21 : reserved21 - MBZ - * 31:22 : req_id_lo - Request ID[9:0] - */ - uint32_t meta_ctrl; - - uint32_t buff_addr_lo; - - /* address high and header size - * 15:0 : addr_hi - Buffer Pointer[47:32] - * 23:16 : reserved16_w2 - * 31:24 : header_length - Header length. For Low - * Latency Queues, this fields indicates the number - * of bytes written to the headers' memory. For - * normal queues, if packet is TCP or UDP, and longer - * than max_header_size, then this field should be - * set to the sum of L4 header offset and L4 header - * size(without options), otherwise, this field - * should be set to 0. For both modes, this field - * must not exceed the max_header_size. - * max_header_size value is reported by the Max - * Queues Feature descriptor - */ - uint32_t buff_addr_hi_hdr_sz; -}; - -struct ena_eth_io_tx_meta_desc { - /* 9:0 : req_id_lo - Request ID[9:0] - * 11:10 : reserved10 - MBZ - * 12 : reserved12 - MBZ - * 13 : reserved13 - MBZ - * 14 : ext_valid - if set, offset fields in Word2 - * are valid Also MSS High in Word 0 and bits [31:24] - * in Word 3 - * 15 : reserved15 - * 19:16 : mss_hi - * 20 : eth_meta_type - 0: Tx Metadata Descriptor, 1: - * Extended Metadata Descriptor - * 21 : meta_store - Store extended metadata in queue - * cache - * 22 : reserved22 - MBZ - * 23 : meta_desc - MBO - * 24 : phase - * 25 : reserved25 - MBZ - * 26 : first - Indicates first descriptor in - * transaction - * 27 : last - Indicates last descriptor in - * transaction - * 28 : comp_req - Indicates whether completion - * should be posted, after packet is transmitted. - * Valid only for first descriptor - * 30:29 : reserved29 - MBZ - * 31 : reserved31 - MBZ - */ - uint32_t len_ctrl; - - /* 5:0 : req_id_hi - * 31:6 : reserved6 - MBZ - */ - uint32_t word1; - - /* 7:0 : l3_hdr_len - * 15:8 : l3_hdr_off - * 21:16 : l4_hdr_len_in_words - counts the L4 header - * length in words. there is an explicit assumption - * that L4 header appears right after L3 header and - * L4 offset is based on l3_hdr_off+l3_hdr_len - * 31:22 : mss_lo - */ - uint32_t word2; - - uint32_t reserved; -}; - -struct ena_eth_io_tx_cdesc { - /* Request ID[15:0] */ - uint16_t req_id; - - uint8_t status; - - /* flags - * 0 : phase - * 7:1 : reserved1 - */ - uint8_t flags; - - uint16_t sub_qid; - - uint16_t sq_head_idx; -}; - -struct ena_eth_io_rx_desc { - /* In bytes. 0 means 64KB */ - uint16_t length; - - /* MBZ */ - uint8_t reserved2; - - /* 0 : phase - * 1 : reserved1 - MBZ - * 2 : first - Indicates first descriptor in - * transaction - * 3 : last - Indicates last descriptor in transaction - * 4 : comp_req - * 5 : reserved5 - MBO - * 7:6 : reserved6 - MBZ - */ - uint8_t ctrl; - - uint16_t req_id; - - /* MBZ */ - uint16_t reserved6; - - uint32_t buff_addr_lo; - - uint16_t buff_addr_hi; - - /* MBZ */ - uint16_t reserved16_w3; -}; - -/* 4-word format Note: all ethernet parsing information are valid only when - * last=1 - */ -struct ena_eth_io_rx_cdesc_base { - /* 4:0 : l3_proto_idx - * 6:5 : src_vlan_cnt - * 7 : reserved7 - MBZ - * 12:8 : l4_proto_idx - * 13 : l3_csum_err - when set, either the L3 - * checksum error detected, or, the controller didn't - * validate the checksum. This bit is valid only when - * l3_proto_idx indicates IPv4 packet - * 14 : l4_csum_err - when set, either the L4 - * checksum error detected, or, the controller didn't - * validate the checksum. This bit is valid only when - * l4_proto_idx indicates TCP/UDP packet, and, - * ipv4_frag is not set - * 15 : ipv4_frag - Indicates IPv4 fragmented packet - * 23:16 : reserved16 - * 24 : phase - * 25 : l3_csum2 - second checksum engine result - * 26 : first - Indicates first descriptor in - * transaction - * 27 : last - Indicates last descriptor in - * transaction - * 29:28 : reserved28 - * 30 : buffer - 0: Metadata descriptor. 1: Buffer - * Descriptor was used - * 31 : reserved31 - */ - uint32_t status; - - uint16_t length; - - uint16_t req_id; - - /* 32-bit hash result */ - uint32_t hash; - - uint16_t sub_qid; - - uint16_t reserved; -}; - -/* 8-word format */ -struct ena_eth_io_rx_cdesc_ext { - struct ena_eth_io_rx_cdesc_base base; - - uint32_t buff_addr_lo; - - uint16_t buff_addr_hi; - - uint16_t reserved16; - - uint32_t reserved_w6; - - uint32_t reserved_w7; -}; - -struct ena_eth_io_intr_reg { - /* 14:0 : rx_intr_delay - * 29:15 : tx_intr_delay - * 30 : intr_unmask - * 31 : reserved - */ - uint32_t intr_control; -}; - -struct ena_eth_io_numa_node_cfg_reg { - /* 7:0 : numa - * 30:8 : reserved - * 31 : enabled - */ - uint32_t numa_cfg; -}; - -/* tx_desc */ -#define ENA_ETH_IO_TX_DESC_LENGTH_MASK GENMASK(15, 0) -#define ENA_ETH_IO_TX_DESC_REQ_ID_HI_SHIFT 16 -#define ENA_ETH_IO_TX_DESC_REQ_ID_HI_MASK GENMASK(21, 16) -#define ENA_ETH_IO_TX_DESC_META_DESC_SHIFT 23 -#define ENA_ETH_IO_TX_DESC_META_DESC_MASK BIT(23) -#define ENA_ETH_IO_TX_DESC_PHASE_SHIFT 24 -#define ENA_ETH_IO_TX_DESC_PHASE_MASK BIT(24) -#define ENA_ETH_IO_TX_DESC_FIRST_SHIFT 26 -#define ENA_ETH_IO_TX_DESC_FIRST_MASK BIT(26) -#define ENA_ETH_IO_TX_DESC_LAST_SHIFT 27 -#define ENA_ETH_IO_TX_DESC_LAST_MASK BIT(27) -#define ENA_ETH_IO_TX_DESC_COMP_REQ_SHIFT 28 -#define ENA_ETH_IO_TX_DESC_COMP_REQ_MASK BIT(28) -#define ENA_ETH_IO_TX_DESC_L3_PROTO_IDX_MASK GENMASK(3, 0) -#define ENA_ETH_IO_TX_DESC_DF_SHIFT 4 -#define ENA_ETH_IO_TX_DESC_DF_MASK BIT(4) -#define ENA_ETH_IO_TX_DESC_TSO_EN_SHIFT 7 -#define ENA_ETH_IO_TX_DESC_TSO_EN_MASK BIT(7) -#define ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_SHIFT 8 -#define ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_MASK GENMASK(12, 8) -#define ENA_ETH_IO_TX_DESC_L3_CSUM_EN_SHIFT 13 -#define ENA_ETH_IO_TX_DESC_L3_CSUM_EN_MASK BIT(13) -#define ENA_ETH_IO_TX_DESC_L4_CSUM_EN_SHIFT 14 -#define ENA_ETH_IO_TX_DESC_L4_CSUM_EN_MASK BIT(14) -#define ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_SHIFT 15 -#define ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_MASK BIT(15) -#define ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_SHIFT 17 -#define ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_MASK BIT(17) -#define ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT 22 -#define ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK GENMASK(31, 22) -#define ENA_ETH_IO_TX_DESC_ADDR_HI_MASK GENMASK(15, 0) -#define ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT 24 -#define ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK GENMASK(31, 24) - -/* tx_meta_desc */ -#define ENA_ETH_IO_TX_META_DESC_REQ_ID_LO_MASK GENMASK(9, 0) -#define ENA_ETH_IO_TX_META_DESC_EXT_VALID_SHIFT 14 -#define ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK BIT(14) -#define ENA_ETH_IO_TX_META_DESC_MSS_HI_SHIFT 16 -#define ENA_ETH_IO_TX_META_DESC_MSS_HI_MASK GENMASK(19, 16) -#define ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_SHIFT 20 -#define ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK BIT(20) -#define ENA_ETH_IO_TX_META_DESC_META_STORE_SHIFT 21 -#define ENA_ETH_IO_TX_META_DESC_META_STORE_MASK BIT(21) -#define ENA_ETH_IO_TX_META_DESC_META_DESC_SHIFT 23 -#define ENA_ETH_IO_TX_META_DESC_META_DESC_MASK BIT(23) -#define ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT 24 -#define ENA_ETH_IO_TX_META_DESC_PHASE_MASK BIT(24) -#define ENA_ETH_IO_TX_META_DESC_FIRST_SHIFT 26 -#define ENA_ETH_IO_TX_META_DESC_FIRST_MASK BIT(26) -#define ENA_ETH_IO_TX_META_DESC_LAST_SHIFT 27 -#define ENA_ETH_IO_TX_META_DESC_LAST_MASK BIT(27) -#define ENA_ETH_IO_TX_META_DESC_COMP_REQ_SHIFT 28 -#define ENA_ETH_IO_TX_META_DESC_COMP_REQ_MASK BIT(28) -#define ENA_ETH_IO_TX_META_DESC_REQ_ID_HI_MASK GENMASK(5, 0) -#define ENA_ETH_IO_TX_META_DESC_L3_HDR_LEN_MASK GENMASK(7, 0) -#define ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT 8 -#define ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK GENMASK(15, 8) -#define ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT 16 -#define ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK GENMASK(21, 16) -#define ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT 22 -#define ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK GENMASK(31, 22) - -/* tx_cdesc */ -#define ENA_ETH_IO_TX_CDESC_PHASE_MASK BIT(0) - -/* rx_desc */ -#define ENA_ETH_IO_RX_DESC_PHASE_MASK BIT(0) -#define ENA_ETH_IO_RX_DESC_FIRST_SHIFT 2 -#define ENA_ETH_IO_RX_DESC_FIRST_MASK BIT(2) -#define ENA_ETH_IO_RX_DESC_LAST_SHIFT 3 -#define ENA_ETH_IO_RX_DESC_LAST_MASK BIT(3) -#define ENA_ETH_IO_RX_DESC_COMP_REQ_SHIFT 4 -#define ENA_ETH_IO_RX_DESC_COMP_REQ_MASK BIT(4) - -/* rx_cdesc_base */ -#define ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK GENMASK(4, 0) -#define ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_SHIFT 5 -#define ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_MASK GENMASK(6, 5) -#define ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT 8 -#define ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK GENMASK(12, 8) -#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT 13 -#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK BIT(13) -#define ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT 14 -#define ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK BIT(14) -#define ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT 15 -#define ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK BIT(15) -#define ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT 24 -#define ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK BIT(24) -#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_SHIFT 25 -#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_MASK BIT(25) -#define ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT 26 -#define ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK BIT(26) -#define ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT 27 -#define ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK BIT(27) -#define ENA_ETH_IO_RX_CDESC_BASE_BUFFER_SHIFT 30 -#define ENA_ETH_IO_RX_CDESC_BASE_BUFFER_MASK BIT(30) - -/* intr_reg */ -#define ENA_ETH_IO_INTR_REG_RX_INTR_DELAY_MASK GENMASK(14, 0) -#define ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT 15 -#define ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK GENMASK(29, 15) -#define ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT 30 -#define ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK BIT(30) - -/* numa_node_cfg_reg */ -#define ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK GENMASK(7, 0) -#define ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_SHIFT 31 -#define ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK BIT(31) - -#if !defined(ENA_DEFS_LINUX_MAINLINE) -static inline uint32_t get_ena_eth_io_tx_desc_length(const struct ena_eth_io_tx_desc *p) -{ - return p->len_ctrl & ENA_ETH_IO_TX_DESC_LENGTH_MASK; -} - -static inline void set_ena_eth_io_tx_desc_length(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->len_ctrl |= val & ENA_ETH_IO_TX_DESC_LENGTH_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_req_id_hi(const struct ena_eth_io_tx_desc *p) -{ - return (p->len_ctrl & ENA_ETH_IO_TX_DESC_REQ_ID_HI_MASK) >> ENA_ETH_IO_TX_DESC_REQ_ID_HI_SHIFT; -} - -static inline void set_ena_eth_io_tx_desc_req_id_hi(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_REQ_ID_HI_SHIFT) & ENA_ETH_IO_TX_DESC_REQ_ID_HI_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_meta_desc(const struct ena_eth_io_tx_desc *p) -{ - return (p->len_ctrl & ENA_ETH_IO_TX_DESC_META_DESC_MASK) >> ENA_ETH_IO_TX_DESC_META_DESC_SHIFT; -} - -static inline void set_ena_eth_io_tx_desc_meta_desc(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_META_DESC_SHIFT) & ENA_ETH_IO_TX_DESC_META_DESC_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_phase(const struct ena_eth_io_tx_desc *p) -{ - return (p->len_ctrl & ENA_ETH_IO_TX_DESC_PHASE_MASK) >> ENA_ETH_IO_TX_DESC_PHASE_SHIFT; -} - -static inline void set_ena_eth_io_tx_desc_phase(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_PHASE_SHIFT) & ENA_ETH_IO_TX_DESC_PHASE_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_first(const struct ena_eth_io_tx_desc *p) -{ - return (p->len_ctrl & ENA_ETH_IO_TX_DESC_FIRST_MASK) >> ENA_ETH_IO_TX_DESC_FIRST_SHIFT; -} - -static inline void set_ena_eth_io_tx_desc_first(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_FIRST_SHIFT) & ENA_ETH_IO_TX_DESC_FIRST_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_last(const struct ena_eth_io_tx_desc *p) -{ - return (p->len_ctrl & ENA_ETH_IO_TX_DESC_LAST_MASK) >> ENA_ETH_IO_TX_DESC_LAST_SHIFT; -} - -static inline void set_ena_eth_io_tx_desc_last(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_LAST_SHIFT) & ENA_ETH_IO_TX_DESC_LAST_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_comp_req(const struct ena_eth_io_tx_desc *p) -{ - return (p->len_ctrl & ENA_ETH_IO_TX_DESC_COMP_REQ_MASK) >> ENA_ETH_IO_TX_DESC_COMP_REQ_SHIFT; -} - -static inline void set_ena_eth_io_tx_desc_comp_req(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_COMP_REQ_SHIFT) & ENA_ETH_IO_TX_DESC_COMP_REQ_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_l3_proto_idx(const struct ena_eth_io_tx_desc *p) -{ - return p->meta_ctrl & ENA_ETH_IO_TX_DESC_L3_PROTO_IDX_MASK; -} - -static inline void set_ena_eth_io_tx_desc_l3_proto_idx(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->meta_ctrl |= val & ENA_ETH_IO_TX_DESC_L3_PROTO_IDX_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_DF(const struct ena_eth_io_tx_desc *p) -{ - return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_DF_MASK) >> ENA_ETH_IO_TX_DESC_DF_SHIFT; -} - -static inline void set_ena_eth_io_tx_desc_DF(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_DF_SHIFT) & ENA_ETH_IO_TX_DESC_DF_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_tso_en(const struct ena_eth_io_tx_desc *p) -{ - return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_TSO_EN_MASK) >> ENA_ETH_IO_TX_DESC_TSO_EN_SHIFT; -} - -static inline void set_ena_eth_io_tx_desc_tso_en(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_TSO_EN_SHIFT) & ENA_ETH_IO_TX_DESC_TSO_EN_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_l4_proto_idx(const struct ena_eth_io_tx_desc *p) -{ - return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_MASK) >> ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_SHIFT; -} - -static inline void set_ena_eth_io_tx_desc_l4_proto_idx(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_SHIFT) & ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_l3_csum_en(const struct ena_eth_io_tx_desc *p) -{ - return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L3_CSUM_EN_MASK) >> ENA_ETH_IO_TX_DESC_L3_CSUM_EN_SHIFT; -} - -static inline void set_ena_eth_io_tx_desc_l3_csum_en(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_L3_CSUM_EN_SHIFT) & ENA_ETH_IO_TX_DESC_L3_CSUM_EN_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_l4_csum_en(const struct ena_eth_io_tx_desc *p) -{ - return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L4_CSUM_EN_MASK) >> ENA_ETH_IO_TX_DESC_L4_CSUM_EN_SHIFT; -} - -static inline void set_ena_eth_io_tx_desc_l4_csum_en(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_L4_CSUM_EN_SHIFT) & ENA_ETH_IO_TX_DESC_L4_CSUM_EN_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_ethernet_fcs_dis(const struct ena_eth_io_tx_desc *p) -{ - return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_MASK) >> ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_SHIFT; -} - -static inline void set_ena_eth_io_tx_desc_ethernet_fcs_dis(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_SHIFT) & ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_l4_csum_partial(const struct ena_eth_io_tx_desc *p) -{ - return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_MASK) >> ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_SHIFT; -} - -static inline void set_ena_eth_io_tx_desc_l4_csum_partial(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_SHIFT) & ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_req_id_lo(const struct ena_eth_io_tx_desc *p) -{ - return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK) >> ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT; -} - -static inline void set_ena_eth_io_tx_desc_req_id_lo(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT) & ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_addr_hi(const struct ena_eth_io_tx_desc *p) -{ - return p->buff_addr_hi_hdr_sz & ENA_ETH_IO_TX_DESC_ADDR_HI_MASK; -} - -static inline void set_ena_eth_io_tx_desc_addr_hi(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->buff_addr_hi_hdr_sz |= val & ENA_ETH_IO_TX_DESC_ADDR_HI_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_desc_header_length(const struct ena_eth_io_tx_desc *p) -{ - return (p->buff_addr_hi_hdr_sz & ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK) >> ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT; -} - -static inline void set_ena_eth_io_tx_desc_header_length(struct ena_eth_io_tx_desc *p, uint32_t val) -{ - p->buff_addr_hi_hdr_sz |= (val << ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT) & ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_meta_desc_req_id_lo(const struct ena_eth_io_tx_meta_desc *p) -{ - return p->len_ctrl & ENA_ETH_IO_TX_META_DESC_REQ_ID_LO_MASK; -} - -static inline void set_ena_eth_io_tx_meta_desc_req_id_lo(struct ena_eth_io_tx_meta_desc *p, uint32_t val) -{ - p->len_ctrl |= val & ENA_ETH_IO_TX_META_DESC_REQ_ID_LO_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_meta_desc_ext_valid(const struct ena_eth_io_tx_meta_desc *p) -{ - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK) >> ENA_ETH_IO_TX_META_DESC_EXT_VALID_SHIFT; -} - -static inline void set_ena_eth_io_tx_meta_desc_ext_valid(struct ena_eth_io_tx_meta_desc *p, uint32_t val) -{ - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_EXT_VALID_SHIFT) & ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_meta_desc_mss_hi(const struct ena_eth_io_tx_meta_desc *p) -{ - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_MSS_HI_MASK) >> ENA_ETH_IO_TX_META_DESC_MSS_HI_SHIFT; -} - -static inline void set_ena_eth_io_tx_meta_desc_mss_hi(struct ena_eth_io_tx_meta_desc *p, uint32_t val) -{ - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_MSS_HI_SHIFT) & ENA_ETH_IO_TX_META_DESC_MSS_HI_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_meta_desc_eth_meta_type(const struct ena_eth_io_tx_meta_desc *p) -{ - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK) >> ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_SHIFT; -} - -static inline void set_ena_eth_io_tx_meta_desc_eth_meta_type(struct ena_eth_io_tx_meta_desc *p, uint32_t val) -{ - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_SHIFT) & ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_meta_desc_meta_store(const struct ena_eth_io_tx_meta_desc *p) -{ - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_META_STORE_MASK) >> ENA_ETH_IO_TX_META_DESC_META_STORE_SHIFT; -} - -static inline void set_ena_eth_io_tx_meta_desc_meta_store(struct ena_eth_io_tx_meta_desc *p, uint32_t val) -{ - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_META_STORE_SHIFT) & ENA_ETH_IO_TX_META_DESC_META_STORE_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_meta_desc_meta_desc(const struct ena_eth_io_tx_meta_desc *p) -{ - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_META_DESC_MASK) >> ENA_ETH_IO_TX_META_DESC_META_DESC_SHIFT; -} - -static inline void set_ena_eth_io_tx_meta_desc_meta_desc(struct ena_eth_io_tx_meta_desc *p, uint32_t val) -{ - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_META_DESC_SHIFT) & ENA_ETH_IO_TX_META_DESC_META_DESC_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_meta_desc_phase(const struct ena_eth_io_tx_meta_desc *p) -{ - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_PHASE_MASK) >> ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT; -} - -static inline void set_ena_eth_io_tx_meta_desc_phase(struct ena_eth_io_tx_meta_desc *p, uint32_t val) -{ - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT) & ENA_ETH_IO_TX_META_DESC_PHASE_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_meta_desc_first(const struct ena_eth_io_tx_meta_desc *p) -{ - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_FIRST_MASK) >> ENA_ETH_IO_TX_META_DESC_FIRST_SHIFT; -} - -static inline void set_ena_eth_io_tx_meta_desc_first(struct ena_eth_io_tx_meta_desc *p, uint32_t val) -{ - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_FIRST_SHIFT) & ENA_ETH_IO_TX_META_DESC_FIRST_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_meta_desc_last(const struct ena_eth_io_tx_meta_desc *p) -{ - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_LAST_MASK) >> ENA_ETH_IO_TX_META_DESC_LAST_SHIFT; -} - -static inline void set_ena_eth_io_tx_meta_desc_last(struct ena_eth_io_tx_meta_desc *p, uint32_t val) -{ - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_LAST_SHIFT) & ENA_ETH_IO_TX_META_DESC_LAST_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_meta_desc_comp_req(const struct ena_eth_io_tx_meta_desc *p) -{ - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_COMP_REQ_MASK) >> ENA_ETH_IO_TX_META_DESC_COMP_REQ_SHIFT; -} - -static inline void set_ena_eth_io_tx_meta_desc_comp_req(struct ena_eth_io_tx_meta_desc *p, uint32_t val) -{ - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_COMP_REQ_SHIFT) & ENA_ETH_IO_TX_META_DESC_COMP_REQ_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_meta_desc_req_id_hi(const struct ena_eth_io_tx_meta_desc *p) -{ - return p->word1 & ENA_ETH_IO_TX_META_DESC_REQ_ID_HI_MASK; -} - -static inline void set_ena_eth_io_tx_meta_desc_req_id_hi(struct ena_eth_io_tx_meta_desc *p, uint32_t val) -{ - p->word1 |= val & ENA_ETH_IO_TX_META_DESC_REQ_ID_HI_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_meta_desc_l3_hdr_len(const struct ena_eth_io_tx_meta_desc *p) -{ - return p->word2 & ENA_ETH_IO_TX_META_DESC_L3_HDR_LEN_MASK; -} - -static inline void set_ena_eth_io_tx_meta_desc_l3_hdr_len(struct ena_eth_io_tx_meta_desc *p, uint32_t val) -{ - p->word2 |= val & ENA_ETH_IO_TX_META_DESC_L3_HDR_LEN_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_meta_desc_l3_hdr_off(const struct ena_eth_io_tx_meta_desc *p) -{ - return (p->word2 & ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK) >> ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT; -} - -static inline void set_ena_eth_io_tx_meta_desc_l3_hdr_off(struct ena_eth_io_tx_meta_desc *p, uint32_t val) -{ - p->word2 |= (val << ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT) & ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_meta_desc_l4_hdr_len_in_words(const struct ena_eth_io_tx_meta_desc *p) -{ - return (p->word2 & ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK) >> ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT; -} - -static inline void set_ena_eth_io_tx_meta_desc_l4_hdr_len_in_words(struct ena_eth_io_tx_meta_desc *p, uint32_t val) -{ - p->word2 |= (val << ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT) & ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_meta_desc_mss_lo(const struct ena_eth_io_tx_meta_desc *p) -{ - return (p->word2 & ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK) >> ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT; -} - -static inline void set_ena_eth_io_tx_meta_desc_mss_lo(struct ena_eth_io_tx_meta_desc *p, uint32_t val) -{ - p->word2 |= (val << ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT) & ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK; -} - -static inline uint8_t get_ena_eth_io_tx_cdesc_phase(const struct ena_eth_io_tx_cdesc *p) -{ - return p->flags & ENA_ETH_IO_TX_CDESC_PHASE_MASK; -} - -static inline void set_ena_eth_io_tx_cdesc_phase(struct ena_eth_io_tx_cdesc *p, uint8_t val) -{ - p->flags |= val & ENA_ETH_IO_TX_CDESC_PHASE_MASK; -} - -static inline uint8_t get_ena_eth_io_rx_desc_phase(const struct ena_eth_io_rx_desc *p) -{ - return p->ctrl & ENA_ETH_IO_RX_DESC_PHASE_MASK; -} - -static inline void set_ena_eth_io_rx_desc_phase(struct ena_eth_io_rx_desc *p, uint8_t val) -{ - p->ctrl |= val & ENA_ETH_IO_RX_DESC_PHASE_MASK; -} - -static inline uint8_t get_ena_eth_io_rx_desc_first(const struct ena_eth_io_rx_desc *p) -{ - return (p->ctrl & ENA_ETH_IO_RX_DESC_FIRST_MASK) >> ENA_ETH_IO_RX_DESC_FIRST_SHIFT; -} - -static inline void set_ena_eth_io_rx_desc_first(struct ena_eth_io_rx_desc *p, uint8_t val) -{ - p->ctrl |= (val << ENA_ETH_IO_RX_DESC_FIRST_SHIFT) & ENA_ETH_IO_RX_DESC_FIRST_MASK; -} - -static inline uint8_t get_ena_eth_io_rx_desc_last(const struct ena_eth_io_rx_desc *p) -{ - return (p->ctrl & ENA_ETH_IO_RX_DESC_LAST_MASK) >> ENA_ETH_IO_RX_DESC_LAST_SHIFT; -} - -static inline void set_ena_eth_io_rx_desc_last(struct ena_eth_io_rx_desc *p, uint8_t val) -{ - p->ctrl |= (val << ENA_ETH_IO_RX_DESC_LAST_SHIFT) & ENA_ETH_IO_RX_DESC_LAST_MASK; -} - -static inline uint8_t get_ena_eth_io_rx_desc_comp_req(const struct ena_eth_io_rx_desc *p) -{ - return (p->ctrl & ENA_ETH_IO_RX_DESC_COMP_REQ_MASK) >> ENA_ETH_IO_RX_DESC_COMP_REQ_SHIFT; -} - -static inline void set_ena_eth_io_rx_desc_comp_req(struct ena_eth_io_rx_desc *p, uint8_t val) -{ - p->ctrl |= (val << ENA_ETH_IO_RX_DESC_COMP_REQ_SHIFT) & ENA_ETH_IO_RX_DESC_COMP_REQ_MASK; -} - -static inline uint32_t get_ena_eth_io_rx_cdesc_base_l3_proto_idx(const struct ena_eth_io_rx_cdesc_base *p) -{ - return p->status & ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK; -} - -static inline void set_ena_eth_io_rx_cdesc_base_l3_proto_idx(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) -{ - p->status |= val & ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK; -} - -static inline uint32_t get_ena_eth_io_rx_cdesc_base_src_vlan_cnt(const struct ena_eth_io_rx_cdesc_base *p) -{ - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_SHIFT; -} - -static inline void set_ena_eth_io_rx_cdesc_base_src_vlan_cnt(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) -{ - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_MASK; -} - -static inline uint32_t get_ena_eth_io_rx_cdesc_base_l4_proto_idx(const struct ena_eth_io_rx_cdesc_base *p) -{ - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT; -} - -static inline void set_ena_eth_io_rx_cdesc_base_l4_proto_idx(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) -{ - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK; -} - -static inline uint32_t get_ena_eth_io_rx_cdesc_base_l3_csum_err(const struct ena_eth_io_rx_cdesc_base *p) -{ - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT; -} - -static inline void set_ena_eth_io_rx_cdesc_base_l3_csum_err(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) -{ - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK; -} - -static inline uint32_t get_ena_eth_io_rx_cdesc_base_l4_csum_err(const struct ena_eth_io_rx_cdesc_base *p) -{ - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT; -} - -static inline void set_ena_eth_io_rx_cdesc_base_l4_csum_err(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) -{ - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK; -} - -static inline uint32_t get_ena_eth_io_rx_cdesc_base_ipv4_frag(const struct ena_eth_io_rx_cdesc_base *p) -{ - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT; -} - -static inline void set_ena_eth_io_rx_cdesc_base_ipv4_frag(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) -{ - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK; -} - -static inline uint32_t get_ena_eth_io_rx_cdesc_base_phase(const struct ena_eth_io_rx_cdesc_base *p) -{ - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT; -} - -static inline void set_ena_eth_io_rx_cdesc_base_phase(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) -{ - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK; -} - -static inline uint32_t get_ena_eth_io_rx_cdesc_base_l3_csum2(const struct ena_eth_io_rx_cdesc_base *p) -{ - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_SHIFT; -} - -static inline void set_ena_eth_io_rx_cdesc_base_l3_csum2(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) -{ - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_MASK; -} - -static inline uint32_t get_ena_eth_io_rx_cdesc_base_first(const struct ena_eth_io_rx_cdesc_base *p) -{ - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT; -} - -static inline void set_ena_eth_io_rx_cdesc_base_first(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) -{ - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK; -} - -static inline uint32_t get_ena_eth_io_rx_cdesc_base_last(const struct ena_eth_io_rx_cdesc_base *p) -{ - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT; -} - -static inline void set_ena_eth_io_rx_cdesc_base_last(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) -{ - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK; -} - -static inline uint32_t get_ena_eth_io_rx_cdesc_base_buffer(const struct ena_eth_io_rx_cdesc_base *p) -{ - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_BUFFER_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_BUFFER_SHIFT; -} - -static inline void set_ena_eth_io_rx_cdesc_base_buffer(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) -{ - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_BUFFER_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_BUFFER_MASK; -} - -static inline uint32_t get_ena_eth_io_intr_reg_rx_intr_delay(const struct ena_eth_io_intr_reg *p) -{ - return p->intr_control & ENA_ETH_IO_INTR_REG_RX_INTR_DELAY_MASK; -} - -static inline void set_ena_eth_io_intr_reg_rx_intr_delay(struct ena_eth_io_intr_reg *p, uint32_t val) -{ - p->intr_control |= val & ENA_ETH_IO_INTR_REG_RX_INTR_DELAY_MASK; -} - -static inline uint32_t get_ena_eth_io_intr_reg_tx_intr_delay(const struct ena_eth_io_intr_reg *p) -{ - return (p->intr_control & ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK) >> ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT; -} - -static inline void set_ena_eth_io_intr_reg_tx_intr_delay(struct ena_eth_io_intr_reg *p, uint32_t val) -{ - p->intr_control |= (val << ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT) & ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK; -} - -static inline uint32_t get_ena_eth_io_intr_reg_intr_unmask(const struct ena_eth_io_intr_reg *p) -{ - return (p->intr_control & ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK) >> ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT; -} - -static inline void set_ena_eth_io_intr_reg_intr_unmask(struct ena_eth_io_intr_reg *p, uint32_t val) -{ - p->intr_control |= (val << ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT) & ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK; -} - -static inline uint32_t get_ena_eth_io_numa_node_cfg_reg_numa(const struct ena_eth_io_numa_node_cfg_reg *p) -{ - return p->numa_cfg & ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK; -} - -static inline void set_ena_eth_io_numa_node_cfg_reg_numa(struct ena_eth_io_numa_node_cfg_reg *p, uint32_t val) -{ - p->numa_cfg |= val & ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK; -} - -static inline uint32_t get_ena_eth_io_numa_node_cfg_reg_enabled(const struct ena_eth_io_numa_node_cfg_reg *p) -{ - return (p->numa_cfg & ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK) >> ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_SHIFT; -} - -static inline void set_ena_eth_io_numa_node_cfg_reg_enabled(struct ena_eth_io_numa_node_cfg_reg *p, uint32_t val) -{ - p->numa_cfg |= (val << ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_SHIFT) & ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK; -} - -#endif /* !defined(ENA_DEFS_LINUX_MAINLINE) */ -#endif /*_ENA_ETH_IO_H_ */ diff --git a/sys/contrib/ena-com/ena_fbsd_log.h b/sys/contrib/ena-com/ena_fbsd_log.h deleted file mode 100644 index 311e2f0f280a..000000000000 --- a/sys/contrib/ena-com/ena_fbsd_log.h +++ /dev/null @@ -1,74 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-3-Clause - * - * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of copyright holder nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef ENA_FBSD_LOG_H -#define ENA_FBSD_LOG_H - -enum ena_log_t { - ENA_ERR = 0, - ENA_WARN, - ENA_INFO, - ENA_DBG, -}; - -extern int ena_log_level; - -#define ena_log(dev, level, fmt, args...) \ - do { \ - if (ENA_ ## level <= ena_log_level) \ - device_printf((dev), fmt, ##args); \ - } while (0) - -#define ena_log_raw(level, fmt, args...) \ - do { \ - if (ENA_ ## level <= ena_log_level) \ - printf(fmt, ##args); \ - } while (0) - -#define ena_log_unused(dev, level, fmt, args...) \ - do { \ - (void)(dev); \ - } while (0) - -#ifdef ENA_LOG_IO_ENABLE -#define ena_log_io(dev, level, fmt, args...) \ - ena_log((dev), level, fmt, ##args) -#else -#define ena_log_io(dev, level, fmt, args...) \ - ena_log_unused((dev), level, fmt, ##args) -#endif - -#define ena_log_nm(dev, level, fmt, args...) \ - ena_log((dev), level, "[nm] " fmt, ##args) - -#endif /* !(ENA_FBSD_LOG_H) */ diff --git a/sys/contrib/ena-com/ena_plat.h b/sys/contrib/ena-com/ena_plat.h index 9287532b8476..b5dc4e91a73c 100644 --- a/sys/contrib/ena-com/ena_plat.h +++ b/sys/contrib/ena-com/ena_plat.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * - * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates. + * Copyright (c) 2015-2023 Amazon.com, Inc. or its affiliates. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -92,7 +92,42 @@ __FBSDID("$FreeBSD$"); #include #include -#include "ena_fbsd_log.h" +enum ena_log_t { + ENA_ERR = 0, + ENA_WARN, + ENA_INFO, + ENA_DBG, +}; + +extern int ena_log_level; + +#define ena_log(dev, level, fmt, args...) \ + do { \ + if (ENA_ ## level <= ena_log_level) \ + device_printf((dev), fmt, ##args); \ + } while (0) + +#define ena_log_raw(level, fmt, args...) \ + do { \ + if (ENA_ ## level <= ena_log_level) \ + printf(fmt, ##args); \ + } while (0) + +#define ena_log_unused(dev, level, fmt, args...) \ + do { \ + (void)(dev); \ + } while (0) + +#ifdef ENA_LOG_IO_ENABLE +#define ena_log_io(dev, level, fmt, args...) \ + ena_log((dev), level, fmt, ##args) +#else +#define ena_log_io(dev, level, fmt, args...) \ + ena_log_unused((dev), level, fmt, ##args) +#endif + +#define ena_log_nm(dev, level, fmt, args...) \ + ena_log((dev), level, "[nm] " fmt, ##args) extern struct ena_bus_space ebs; @@ -152,7 +187,7 @@ static inline long PTR_ERR(const void *ptr) #define GENMASK(h, l) (((~0U) - (1U << (l)) + 1) & (~0U >> (32 - 1 - (h)))) #define GENMASK_ULL(h, l) (((~0ULL) << (l)) & (~0ULL >> (64 - 1 - (h)))) #define BIT(x) (1UL << (x)) - +#define BIT64(x) BIT(x) #define ENA_ABORT() BUG() #define BUG() panic("ENA BUG") @@ -170,6 +205,7 @@ static inline long PTR_ERR(const void *ptr) #define ENA_COM_PERMISSION EPERM #define ENA_COM_TIMER_EXPIRED ETIMEDOUT #define ENA_COM_EIO EIO +#define ENA_COM_DEVICE_BUSY EBUSY #define ENA_NODE_ANY (-1) @@ -179,6 +215,13 @@ static inline long PTR_ERR(const void *ptr) #define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \ ((long)cputick2usec(cpu_ticks()) + (timeout_us)) #define ENA_TIME_EXPIRE(timeout) ((timeout) < cputick2usec(cpu_ticks())) +#define ENA_TIME_EXPIRE_HIGH_RES ENA_TIME_EXPIRE +#define ENA_TIME_INIT_HIGH_RES() (0) +#define ENA_TIME_COMPARE_HIGH_RES(time1, time2) \ + ((time1 < time2) ? -1 : ((time1 > time2) ? 1 : 0)) +#define ENA_GET_SYSTEM_TIMEOUT_HIGH_RES(current_time, timeout_us) \ + ((long)cputick2usec(cpu_ticks()) + (timeout_us)) +#define ENA_GET_SYSTEM_TIME_HIGH_RES() ENA_GET_SYSTEM_TIMEOUT(0) #define ENA_MIGHT_SLEEP() #define min_t(type, _x, _y) ((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y)) @@ -275,6 +318,7 @@ typedef uint32_t ena_atomic32_t; #define ENA_PRIu64 PRIu64 typedef uint64_t ena_time_t; +typedef uint64_t ena_time_high_res_t; typedef struct ifnet ena_netdev; void ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, @@ -427,4 +471,6 @@ void ena_rss_key_fill(void *key, size_t size); #include "ena_defs/ena_includes.h" +#define ENA_BITS_PER_U64(bitmap) (bitcount64(bitmap)) + #endif /* ENA_PLAT_H_ */ diff --git a/sys/contrib/ena-com/ena_regs_defs.h b/sys/contrib/ena-com/ena_regs_defs.h deleted file mode 100644 index 1438fb8faf96..000000000000 --- a/sys/contrib/ena-com/ena_regs_defs.h +++ /dev/null @@ -1,137 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of copyright holder nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _ENA_REGS_H_ -#define _ENA_REGS_H_ - -/* ena_registers offsets */ -#define ENA_REGS_VERSION_OFF 0x0 -#define ENA_REGS_CONTROLLER_VERSION_OFF 0x4 -#define ENA_REGS_CAPS_OFF 0x8 -#define ENA_REGS_CAPS_EXT_OFF 0xc -#define ENA_REGS_AQ_BASE_LO_OFF 0x10 -#define ENA_REGS_AQ_BASE_HI_OFF 0x14 -#define ENA_REGS_AQ_CAPS_OFF 0x18 -#define ENA_REGS_ACQ_BASE_LO_OFF 0x20 -#define ENA_REGS_ACQ_BASE_HI_OFF 0x24 -#define ENA_REGS_ACQ_CAPS_OFF 0x28 -#define ENA_REGS_AQ_DB_OFF 0x2c -#define ENA_REGS_ACQ_TAIL_OFF 0x30 -#define ENA_REGS_AENQ_CAPS_OFF 0x34 -#define ENA_REGS_AENQ_BASE_LO_OFF 0x38 -#define ENA_REGS_AENQ_BASE_HI_OFF 0x3c -#define ENA_REGS_AENQ_HEAD_DB_OFF 0x40 -#define ENA_REGS_AENQ_TAIL_OFF 0x44 -#define ENA_REGS_INTR_MASK_OFF 0x4c -#define ENA_REGS_DEV_CTL_OFF 0x54 -#define ENA_REGS_DEV_STS_OFF 0x58 -#define ENA_REGS_MMIO_REG_READ_OFF 0x5c -#define ENA_REGS_MMIO_RESP_LO_OFF 0x60 -#define ENA_REGS_MMIO_RESP_HI_OFF 0x64 -#define ENA_REGS_RSS_IND_ENTRY_UPDATE_OFF 0x68 - -/* version register */ -#define ENA_REGS_VERSION_MINOR_VERSION_MASK 0xff -#define ENA_REGS_VERSION_MAJOR_VERSION_SHIFT 8 -#define ENA_REGS_VERSION_MAJOR_VERSION_MASK 0xff00 - -/* controller_version register */ -#define ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK 0xff -#define ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT 8 -#define ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK 0xff00 -#define ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT 16 -#define ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK 0xff0000 -#define ENA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT 24 -#define ENA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK 0xff000000 - -/* caps register */ -#define ENA_REGS_CAPS_CONTIGUOUS_QUEUE_REQUIRED_MASK 0x1 -#define ENA_REGS_CAPS_RESET_TIMEOUT_SHIFT 1 -#define ENA_REGS_CAPS_RESET_TIMEOUT_MASK 0x3e -#define ENA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT 8 -#define ENA_REGS_CAPS_DMA_ADDR_WIDTH_MASK 0xff00 -#define ENA_REGS_CAPS_ADMIN_CMD_TO_SHIFT 16 -#define ENA_REGS_CAPS_ADMIN_CMD_TO_MASK 0xf0000 - -/* aq_caps register */ -#define ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK 0xffff -#define ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT 16 -#define ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK 0xffff0000 - -/* acq_caps register */ -#define ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK 0xffff -#define ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT 16 -#define ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK 0xffff0000 - -/* aenq_caps register */ -#define ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK 0xffff -#define ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT 16 -#define ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK 0xffff0000 - -/* dev_ctl register */ -#define ENA_REGS_DEV_CTL_DEV_RESET_MASK 0x1 -#define ENA_REGS_DEV_CTL_AQ_RESTART_SHIFT 1 -#define ENA_REGS_DEV_CTL_AQ_RESTART_MASK 0x2 -#define ENA_REGS_DEV_CTL_QUIESCENT_SHIFT 2 -#define ENA_REGS_DEV_CTL_QUIESCENT_MASK 0x4 -#define ENA_REGS_DEV_CTL_IO_RESUME_SHIFT 3 -#define ENA_REGS_DEV_CTL_IO_RESUME_MASK 0x8 - -/* dev_sts register */ -#define ENA_REGS_DEV_STS_READY_MASK 0x1 -#define ENA_REGS_DEV_STS_AQ_RESTART_IN_PROGRESS_SHIFT 1 -#define ENA_REGS_DEV_STS_AQ_RESTART_IN_PROGRESS_MASK 0x2 -#define ENA_REGS_DEV_STS_AQ_RESTART_FINISHED_SHIFT 2 -#define ENA_REGS_DEV_STS_AQ_RESTART_FINISHED_MASK 0x4 -#define ENA_REGS_DEV_STS_RESET_IN_PROGRESS_SHIFT 3 -#define ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK 0x8 -#define ENA_REGS_DEV_STS_RESET_FINISHED_SHIFT 4 -#define ENA_REGS_DEV_STS_RESET_FINISHED_MASK 0x10 -#define ENA_REGS_DEV_STS_FATAL_ERROR_SHIFT 5 -#define ENA_REGS_DEV_STS_FATAL_ERROR_MASK 0x20 -#define ENA_REGS_DEV_STS_QUIESCENT_STATE_IN_PROGRESS_SHIFT 6 -#define ENA_REGS_DEV_STS_QUIESCENT_STATE_IN_PROGRESS_MASK 0x40 -#define ENA_REGS_DEV_STS_QUIESCENT_STATE_ACHIEVED_SHIFT 7 -#define ENA_REGS_DEV_STS_QUIESCENT_STATE_ACHIEVED_MASK 0x80 - -/* mmio_reg_read register */ -#define ENA_REGS_MMIO_REG_READ_REQ_ID_MASK 0xffff -#define ENA_REGS_MMIO_REG_READ_REG_OFF_SHIFT 16 -#define ENA_REGS_MMIO_REG_READ_REG_OFF_MASK 0xffff0000 - -/* rss_ind_entry_update register */ -#define ENA_REGS_RSS_IND_ENTRY_UPDATE_INDEX_MASK 0xffff -#define ENA_REGS_RSS_IND_ENTRY_UPDATE_CQ_IDX_SHIFT 16 -#define ENA_REGS_RSS_IND_ENTRY_UPDATE_CQ_IDX_MASK 0xffff0000 - -#endif /*_ENA_REGS_H_ */