Skip to content

Commit 7fd01f0

Browse files
yishaihyishaih
authored
Merge pull request #1050 from yishaih/mlx5_dr
mlx5: DR, Enhance the support in few areas
2 parents b1b81ca + bdbe132 commit 7fd01f0

18 files changed

+816
-111
lines changed

‎debian/ibverbs-providers.symbols

+1
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ libmlx5.so.1 ibverbs-providers #MINVER#
135135
mlx5dv_qp_cancel_posted_send_wrs@MLX5_1.20 36
136136
_mlx5dv_mkey_check@MLX5_1.20 36
137137
mlx5dv_dci_stream_id_reset@MLX5_1.21 37
138+
mlx5dv_dr_action_create_dest_ib_port@MLX5_1.21 37
138139
mlx5dv_dr_matcher_set_layout@MLX5_1.21 37
139140
mlx5dv_get_vfio_device_list@MLX5_1.21 37
140141
mlx5dv_vfio_get_events_fd@MLX5_1.21 37

‎providers/mlx5/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ rdma_shared_provider(mlx5 libmlx5.map
2929
dr_ste_v1.c
3030
dr_table.c
3131
dr_send.c
32+
dr_vports.c
3233
mlx5.c
3334
mlx5_vfio.c
3435
qp.c

‎providers/mlx5/dr_action.c

+41-4
Original file line numberDiff line numberDiff line change
@@ -744,7 +744,7 @@ int dr_actions_build_ste_arr(struct mlx5dv_dr_matcher *matcher,
744744
}
745745
if (rx_rule) {
746746
/* Loopback on WIRE vport is not supported */
747-
if (action->vport.num == WIRE_PORT)
747+
if (action->vport.caps->num == WIRE_PORT)
748748
goto out_invalid_arg;
749749

750750
attr.final_icm_addr = action->vport.caps->icm_address_rx;
@@ -2075,7 +2075,14 @@ struct mlx5dv_dr_action
20752075
return NULL;
20762076
}
20772077

2078-
vport_cap = dr_get_vport_cap(&dmn->info.caps, vport);
2078+
/* vport number is limited to 16 bit */
2079+
if (vport > WIRE_PORT) {
2080+
dr_dbg(dmn, "The vport number is out of range\n");
2081+
errno = EINVAL;
2082+
return NULL;
2083+
}
2084+
2085+
vport_cap = dr_vports_table_get_vport_cap(&dmn->info.caps, vport);
20792086
if (!vport_cap) {
20802087
dr_dbg(dmn, "Failed to get vport %d caps\n", vport);
20812088
return NULL;
@@ -2086,7 +2093,37 @@ struct mlx5dv_dr_action
20862093
return NULL;
20872094

20882095
action->vport.dmn = dmn;
2089-
action->vport.num = vport;
2096+
action->vport.caps = vport_cap;
2097+
2098+
return action;
2099+
}
2100+
2101+
struct mlx5dv_dr_action *
2102+
mlx5dv_dr_action_create_dest_ib_port(struct mlx5dv_dr_domain *dmn,
2103+
uint32_t ib_port)
2104+
{
2105+
struct dr_devx_vport_cap *vport_cap;
2106+
struct mlx5dv_dr_action *action;
2107+
2108+
if (!dmn->info.supp_sw_steering ||
2109+
dmn->type != MLX5DV_DR_DOMAIN_TYPE_FDB) {
2110+
dr_dbg(dmn, "Domain doesn't support ib_port actions\n");
2111+
errno = EOPNOTSUPP;
2112+
return NULL;
2113+
}
2114+
2115+
vport_cap = dr_vports_table_get_ib_port_cap(&dmn->info.caps, ib_port);
2116+
if (!vport_cap) {
2117+
dr_dbg(dmn, "Failed to get ib_port %d caps\n", ib_port);
2118+
errno = EINVAL;
2119+
return NULL;
2120+
}
2121+
2122+
action = dr_action_create_generic(DR_ACTION_TYP_VPORT);
2123+
if (!action)
2124+
return NULL;
2125+
2126+
action->vport.dmn = dmn;
20902127
action->vport.caps = vport_cap;
20912128

20922129
return action;
@@ -2115,7 +2152,7 @@ dr_action_convert_to_fte_dest(struct mlx5dv_dr_domain *dmn,
21152152

21162153
fte_attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
21172154
dest_info->type = MLX5_FLOW_DEST_TYPE_VPORT;
2118-
dest_info->vport_num = dest->vport.num;
2155+
dest_info->vport_num = dest->vport.caps->num;
21192156
break;
21202157
case DR_ACTION_TYP_QP:
21212158
fte_attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;

‎providers/mlx5/dr_dbg.c

+37-15
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ static int dr_dump_rule_action(FILE *f, const uint64_t rule_id,
142142
case DR_ACTION_TYP_VPORT:
143143
ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",0x%x\n",
144144
DR_DUMP_REC_TYPE_ACTION_VPORT, action_id, rule_id,
145-
action->vport.num);
145+
action->vport.caps->num);
146146
break;
147147
case DR_ACTION_TYP_TNL_L2_TO_L2:
148148
ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 "\n",
@@ -616,10 +616,39 @@ static int dr_dump_domain_info_flex_parser(FILE *f, const char *flex_parser_name
616616
return 0;
617617
}
618618

619+
static int dr_dump_vports_table(FILE *f, struct dr_vports_table *vports_tbl,
620+
const uint64_t domain_id)
621+
{
622+
struct dr_devx_vport_cap *vport_cap;
623+
int i, ret;
624+
625+
if (!vports_tbl)
626+
return 0;
627+
628+
for (i = 0; i < DR_VPORTS_BUCKETS; i++) {
629+
vport_cap = vports_tbl->buckets[i];
630+
while (vport_cap) {
631+
ret = fprintf(f, "%d,0x%" PRIx64 ",%d,0x%x,0x%" PRIx64 ",0x%" PRIx64 "\n",
632+
DR_DUMP_REC_TYPE_DOMAIN_INFO_VPORT,
633+
domain_id,
634+
vport_cap->num,
635+
vport_cap->vport_gvmi,
636+
vport_cap->icm_address_rx,
637+
vport_cap->icm_address_tx);
638+
if (ret < 0)
639+
return ret;
640+
641+
vport_cap = vport_cap->next;
642+
}
643+
}
644+
645+
return 0;
646+
}
647+
619648
static int dr_dump_domain_info_caps(FILE *f, struct dr_devx_caps *caps,
620649
const uint64_t domain_id)
621650
{
622-
int i, ret;
651+
int ret;
623652

624653
ret = fprintf(f, "%d,0x%" PRIx64 ",0x%x,0x%" PRIx64 ",0x%" PRIx64 ",0x%x,%d,%d\n",
625654
DR_DUMP_REC_TYPE_DOMAIN_INFO_CAPS,
@@ -628,22 +657,15 @@ static int dr_dump_domain_info_caps(FILE *f, struct dr_devx_caps *caps,
628657
caps->nic_rx_drop_address,
629658
caps->nic_tx_drop_address,
630659
caps->flex_protocols,
631-
caps->num_vports,
660+
caps->vports.num_ports,
632661
caps->eswitch_manager);
633662
if (ret < 0)
634663
return ret;
635664

636-
for (i = 0; i < caps->num_vports; i++) {
637-
ret = fprintf(f, "%d,0x%" PRIx64 ",%d,0x%x,0x%" PRIx64 ",0x%" PRIx64 "\n",
638-
DR_DUMP_REC_TYPE_DOMAIN_INFO_VPORT,
639-
domain_id,
640-
i,
641-
caps->vports_caps[i].gvmi,
642-
caps->vports_caps[i].icm_address_rx,
643-
caps->vports_caps[i].icm_address_tx);
644-
if (ret < 0)
645-
return ret;
646-
}
665+
ret = dr_dump_vports_table(f, caps->vports.vports, domain_id);
666+
if (ret < 0)
667+
return ret;
668+
647669
return 0;
648670
}
649671

@@ -655,7 +677,7 @@ static int dr_dump_domain_info_dev_attr(FILE *f, struct dr_domain_info *info,
655677
ret = fprintf(f, "%d,0x%" PRIx64 ",%u,%s\n",
656678
DR_DUMP_REC_TYPE_DOMAIN_INFO_DEV_ATTR,
657679
domain_id,
658-
info->caps.num_vports + 1,
680+
info->caps.vports.num_ports,
659681
info->attr.orig_attr.fw_ver);
660682
if (ret < 0)
661683
return ret;

‎providers/mlx5/dr_devx.c

+64-1
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,43 @@ int dr_devx_query_gvmi(struct ibv_context *ctx, bool other_vport,
111111
return 0;
112112
}
113113

114+
static int dr_devx_query_esw_func(struct ibv_context *ctx,
115+
uint16_t max_sfs,
116+
bool *host_pf_vhca_id_valid,
117+
uint16_t *host_pf_vhca_id)
118+
{
119+
uint32_t in[DEVX_ST_SZ_DW(query_esw_functions_in)] = {};
120+
size_t outsz;
121+
void *out;
122+
int err;
123+
124+
outsz = DEVX_ST_SZ_BYTES(query_esw_functions_out) +
125+
(max_sfs - 1) * DEVX_FLD_SZ_BYTES(query_esw_functions_out, host_sf_enable);
126+
out = calloc(1, outsz);
127+
if (!out) {
128+
errno = ENOMEM;
129+
return errno;
130+
}
131+
132+
DEVX_SET(query_esw_functions_in, in, opcode,
133+
MLX5_CMD_OP_QUERY_ESW_FUNCTIONS);
134+
135+
err = mlx5dv_devx_general_cmd(ctx, in, sizeof(in), out, outsz);
136+
if (err) {
137+
dr_dbg_ctx(ctx, "Query esw func failed %d\n", err);
138+
free(out);
139+
return err;
140+
}
141+
142+
*host_pf_vhca_id_valid = DEVX_GET(query_esw_functions_out, out,
143+
host_params_context.host_pf_vhca_id_valid);
144+
145+
*host_pf_vhca_id = DEVX_GET(query_esw_functions_out, out,
146+
host_params_context.host_pf_vhca_id);
147+
free(out);
148+
return 0;
149+
}
150+
114151
int dr_devx_query_esw_caps(struct ibv_context *ctx, struct dr_esw_caps *caps)
115152
{
116153
uint32_t out[DEVX_ST_SZ_DW(query_hca_cap_out)] = {};
@@ -156,7 +193,10 @@ int dr_devx_query_device(struct ibv_context *ctx, struct dr_devx_caps *caps)
156193
{
157194
uint32_t out[DEVX_ST_SZ_DW(query_hca_cap_out)] = {};
158195
uint32_t in[DEVX_ST_SZ_DW(query_hca_cap_in)] = {};
159-
bool roce;
196+
bool host_pf_vhca_id_valid;
197+
uint16_t host_pf_vhca_id;
198+
uint32_t max_sfs = 0;
199+
bool roce, sf_supp;
160200
int err;
161201

162202
DEVX_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
@@ -182,6 +222,7 @@ int dr_devx_query_device(struct ibv_context *ctx, struct dr_devx_caps *caps)
182222
caps->flex_parser_header_modify =
183223
DEVX_GET(query_hca_cap_out, out,
184224
capability.cmd_hca_cap.flex_parser_header_modify);
225+
sf_supp = DEVX_GET(query_hca_cap_out, out, capability.cmd_hca_cap.sf);
185226
caps->definer_format_sup =
186227
DEVX_GET64(query_hca_cap_out, out,
187228
capability.cmd_hca_cap.match_definer_format_supported);
@@ -299,6 +340,28 @@ int dr_devx_query_device(struct ibv_context *ctx, struct dr_devx_caps *caps)
299340
ft_field_bitmask_support_2_nic_receive.
300341
outer_l4_checksum_ok);
301342

343+
if (sf_supp && caps->eswitch_manager) {
344+
DEVX_SET(query_hca_cap_in, in, op_mod,
345+
MLX5_SET_HCA_CAP_OP_MOD_ESW | HCA_CAP_OPMOD_GET_CUR);
346+
347+
err = mlx5dv_devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out));
348+
if (err) {
349+
dr_dbg_ctx(ctx, "Query eswitch capabilities failed %d\n", err);
350+
return err;
351+
}
352+
max_sfs = 1 << DEVX_GET(query_hca_cap_out, out,
353+
capability.e_switch_cap.log_max_esw_sf);
354+
}
355+
356+
if (caps->eswitch_manager) {
357+
/* Check if ECPF */
358+
err = dr_devx_query_esw_func(ctx, max_sfs,
359+
&host_pf_vhca_id_valid,
360+
&host_pf_vhca_id);
361+
if (!err && host_pf_vhca_id_valid && host_pf_vhca_id != caps->gvmi)
362+
caps->is_ecpf = true;
363+
}
364+
302365
DEVX_SET(query_hca_cap_in, in, op_mod,
303366
MLX5_SET_HCA_CAP_OP_MOD_DEVICE_MEMORY |
304367
HCA_CAP_OPMOD_GET_CUR);

0 commit comments

Comments
 (0)