# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2020 Nvidia Corporation. All rights reserved. See COPYING file import unittest import errno from pyverbs.providers.mlx5.mlx5dv import Mlx5DVQPInitAttr, Mlx5QP, \ Mlx5DVDCInitAttr, Mlx5Context from tests.test_rdmacm import CMAsyncConnection from tests.mlx5_base import Mlx5PyverbsAPITestCase, Mlx5RDMACMBaseTest from pyverbs.pyverbs_error import PyverbsRDMAError from pyverbs.srq import SRQ, SrqInitAttr, SrqAttr import pyverbs.providers.mlx5.mlx5_enums as dve from tests.base_rdmacm import AsyncCMResources from pyverbs.qp import QPCap, QPInitAttrEx from pyverbs.cmid import ConnParam from tests.base import DCT_KEY from pyverbs.addr import AH import pyverbs.enums as e from pyverbs.cq import CQ import tests.utils as u class DcCMConnection(CMAsyncConnection): """ Implement RDMACM connection management for asynchronous CMIDs using DC as an external QP. """ def create_cm_res(self, ip_addr, passive, **kwargs): self.cm_res = DcCMResources(addr=ip_addr, passive=passive, **kwargs) if passive: self.cm_res.create_cmid() else: for conn_idx in range(self.num_conns): self.cm_res.create_cmid(conn_idx) def _ext_qp_server_traffic(self): recv_wr = u.get_recv_wr(self.cm_res) for _ in range(self.cm_res.num_msgs): u.post_recv(self.cm_res, recv_wr) self.syncer.wait() for _ in range(self.cm_res.num_msgs): u.poll_cq(self.cm_res.cq) def _ext_qp_client_traffic(self): self.cm_res.remote_dct_num = self.cm_res.remote_qpn _, send_wr = u.get_send_elements(self.cm_res, self.cm_res.passive) ah = AH(self.cm_res.cmid.pd, attr=self.cm_res.remote_ah) self.syncer.wait() for send_idx in range(self.cm_res.num_msgs): dci_idx = send_idx % len(self.cm_res.qps) u.post_send_ex(self.cm_res, send_wr, e.IBV_WR_SEND, ah=ah, qp_idx=dci_idx) u.poll_cq(self.cm_res.cq) def disconnect(self): if self.cm_res.reserved_qp_num and self.cm_res.passive: Mlx5Context.reserved_qpn_dealloc(self.cm_res.child_id.context, self.cm_res.reserved_qp_num) self.cm_res.reserved_qp_num = 0 super().disconnect() class DcCMResources(AsyncCMResources): """ DcCMResources class contains resources for RDMA CM asynchronous communication using DC as an external QP. """ def __init__(self, addr=None, passive=None, **kwargs): """ Init DcCMResources instance. :param addr: Local address to bind to. :param passive: Indicate if this CM is the passive CM. """ super().__init__(addr=addr, passive=passive, **kwargs) self.srq = None self.remote_dct_num = None self.reserved_qp_num = 0 def create_qp(self, conn_idx=0): """ Create an RDMACM QP. If self.with_ext_qp is set, then an external CQ and DC QP will be created. In case that CQ is already created, it is used for the newly created QP. """ try: if not self.passive: # Create the DCI QPs. cmid = self.cmids[conn_idx] self.create_cq(cmid) qp_init_attr = self.create_qp_init_attr(cmid, e.IBV_QP_EX_WITH_SEND) attr = Mlx5DVQPInitAttr(comp_mask=dve.MLX5DV_QP_INIT_ATTR_MASK_DC, dc_init_attr=Mlx5DVDCInitAttr()) self.qps[conn_idx] = Mlx5QP(cmid.context, qp_init_attr, attr) if self.passive and conn_idx == 0: # Create the DCT QP only for the first connection. cmid = self.child_id self.create_cq(cmid) self.create_srq(cmid) qp_init_attr = self.create_qp_init_attr(cmid) dc_attr = Mlx5DVDCInitAttr(dc_type=dve.MLX5DV_DCTYPE_DCT, dct_access_key=DCT_KEY) attr = Mlx5DVQPInitAttr(comp_mask=dve.MLX5DV_QP_INIT_ATTR_MASK_DC, dc_init_attr=dc_attr) self.qps[conn_idx] = Mlx5QP(cmid.context, qp_init_attr, attr) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create DC QP is not supported') raise ex def create_qp_cap(self): return QPCap(self.num_msgs, 0, 1, 0) def create_qp_init_attr(self, cmid, send_ops_flags=0): comp_mask = e.IBV_QP_INIT_ATTR_PD if send_ops_flags: comp_mask |= e.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS return QPInitAttrEx(cap=self.create_qp_cap(), pd=cmid.pd, scq=self.cq, rcq=self.cq, srq=self.srq, qp_type=e.IBV_QPT_DRIVER, send_ops_flags=send_ops_flags, comp_mask=comp_mask, sq_sig_all=1) def create_srq(self, cmid): srq_init_attr = SrqInitAttr(SrqAttr(max_wr=self.num_msgs)) try: self.srq = SRQ(cmid.pd, srq_init_attr) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create SRQ is not supported') raise ex def modify_ext_qp_to_rts(self, conn_idx=0): cmids = self.child_ids if self.passive else self.cmids if not self.passive or not conn_idx: qp = self.qps[conn_idx] attr, _ = cmids[conn_idx].init_qp_attr(e.IBV_QPS_INIT) qp.to_init(attr) attr, _ = cmids[conn_idx].init_qp_attr(e.IBV_QPS_RTR) qp.to_rtr(attr) if not self.passive: # The passive QP is DCT which should stay in RTR state. self.remote_ah = attr.ah_attr attr, _ = cmids[conn_idx].init_qp_attr(e.IBV_QPS_RTS) qp.to_rts(attr) def create_conn_param(self, qp_num=0, conn_idx=0): if conn_idx and self.passive: try: ctx = self.child_id.context self.reserved_qp_num = Mlx5Context.reserved_qpn_alloc(ctx) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Alloc reserved QP number is not supported') raise ex qp_num = self.reserved_qp_num else: qp_num = self.qps[conn_idx].qp_num return ConnParam(qp_num=qp_num) class Mlx5CMTestCase(Mlx5RDMACMBaseTest): """ Mlx5 RDMACM test class. """ def test_rdmacm_async_traffic_dc_external_qp(self): """ Connect multiple RDMACM connections using DC as an external QP for traffic. """ self.two_nodes_rdmacm_traffic(DcCMConnection, self.rdmacm_traffic, with_ext_qp=True, num_conns=2) class ReservedQPTest(Mlx5PyverbsAPITestCase): def test_reservered_qpn(self): """ Alloc reserved qpn multiple times and then dealloc the qpns. In addition, the test includes bad flows where a fake qpn gets deallocated, and a real qpn gets deallocated twice. """ try: # Alloc qp number multiple times. qpns = [] for i in range(1000): qpns.append(Mlx5Context.reserved_qpn_alloc(self.ctx)) for i in range(1000): Mlx5Context.reserved_qpn_dealloc(self.ctx, qpns[i]) # Dealloc qp number that was not allocated. qpn = Mlx5Context.reserved_qpn_alloc(self.ctx) with self.assertRaises(PyverbsRDMAError) as ex: fake_qpn = qpn - 1 Mlx5Context.reserved_qpn_dealloc(self.ctx, fake_qpn) self.assertEqual(ex.exception.error_code, errno.EINVAL) # Try to dealloc same qp number twice. Mlx5Context.reserved_qpn_dealloc(self.ctx, qpn) with self.assertRaises(PyverbsRDMAError) as ex: Mlx5Context.reserved_qpn_dealloc(self.ctx, qpn) self.assertEqual(ex.exception.error_code, errno.EINVAL) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Alloc reserved QP number is not supported') raise ex