/* * Copyright (c) 2015-2016 QLogic Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and /or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef __QELR_H__ #define __QELR_H__ #include #include #include #include #include #include #include #include #define writel(b, p) (*(uint32_t *)(p) = (b)) #define writeq(b, p) (*(uint64_t *)(p) = (b)) #include "qelr_abi.h" #include "qelr_hsi.h" #include "qelr_chain.h" struct qelr_buf { void *addr; size_t len; /* a 64 uint is used as s preparation * for double layer pbl. */ }; #define IS_IWARP(_dev) (_dev->node_type == IBV_NODE_RNIC) #define IS_ROCE(_dev) (_dev->node_type == IBV_NODE_CA) struct qelr_device { struct verbs_device ibv_dev; }; enum qelr_dpm_flags { QELR_DPM_FLAGS_ENHANCED = (1 << 0), QELR_DPM_FLAGS_LEGACY = (1 << 1), QELR_DPM_FLAGS_EDPM_MODE = (1 << 2), }; #define QELR_MAX_SRQ_ID 4096 struct qelr_devctx { struct verbs_context ibv_ctx; FILE *dbg_fp; void *db_addr; uint64_t db_pa; struct qedr_user_db_rec db_rec_addr_dummy; uint32_t db_size; enum qelr_dpm_flags dpm_flags; uint32_t kernel_page_size; uint16_t ldpm_limit_size; uint16_t edpm_limit_size; uint8_t edpm_trans_size; uint32_t max_send_wr; uint32_t max_recv_wr; uint32_t max_srq_wr; uint32_t sges_per_send_wr; uint32_t sges_per_recv_wr; uint32_t sges_per_srq_wr; struct qelr_srq **srq_table; int max_cqes; }; struct qelr_pd { struct ibv_pd ibv_pd; uint32_t pd_id; }; struct qelr_mr { struct verbs_mr vmr; }; union db_prod64 { struct rdma_pwm_val32_data data; uint64_t raw; }; struct qelr_cq { struct ibv_cq ibv_cq; /* must be first */ struct qelr_chain chain; void *db_addr; union db_prod64 db; /* Doorbell recovery entry address */ void *db_rec_map; struct qedr_user_db_rec *db_rec_addr; uint8_t chain_toggle; union rdma_cqe *latest_cqe; union rdma_cqe *toggle_cqe; uint8_t arm_flags; }; enum qelr_qp_state { QELR_QPS_RST, QELR_QPS_INIT, QELR_QPS_RTR, QELR_QPS_RTS, QELR_QPS_SQD, QELR_QPS_ERR, QELR_QPS_SQE }; union db_prod32 { struct rdma_pwm_val16_data data; uint32_t raw; }; struct qelr_qp_hwq_info { /* WQE */ struct qelr_chain chain; uint8_t max_sges; /* WQ */ uint16_t prod; uint16_t wqe_cons; uint16_t cons; uint16_t max_wr; /* DB */ void *db; /* Doorbell address */ void *edpm_db; union db_prod32 db_data; /* Doorbell data */ /* Doorbell recovery entry address */ void *db_rec_map; struct qedr_user_db_rec *db_rec_addr; void *iwarp_db2; union db_prod32 iwarp_db2_data; uint16_t icid; }; struct qelr_rdma_ext { __be64 remote_va; __be32 remote_key; __be32 dma_length; }; struct qelr_xrceth { __be32 xrc_srq; }; /* rdma extension, invalidate / immediate data + padding, inline data... */ #define QELR_MAX_DPM_PAYLOAD (sizeof(struct qelr_rdma_ext) + sizeof(uint64_t) +\ ROCE_REQ_MAX_INLINE_DATA_SIZE) struct qelr_dpm { uint8_t is_edpm; uint8_t is_ldpm; union { struct db_roce_dpm_data data; uint64_t raw; } msg; uint8_t payload[QELR_MAX_DPM_PAYLOAD]; uint32_t payload_size; uint32_t payload_offset; struct qelr_rdma_ext *rdma_ext; }; struct qelr_srq_hwq_info { uint32_t max_sges; uint32_t max_wr; struct qelr_chain chain; uint32_t wqe_prod; /* WQE prod index in HW ring */ uint32_t sge_prod; /* SGE prod index in HW ring */ uint32_t wr_prod_cnt; /* wr producer count */ uint32_t wr_cons_cnt; /* wr consumer count */ uint32_t num_elems; void *virt_prod_pair_addr; /* producer pair virtual address */ }; struct qelr_srq { struct verbs_srq verbs_srq; struct qelr_srq_hwq_info hw_srq; uint16_t srq_id; pthread_spinlock_t lock; bool is_xrc; }; enum qelr_qp_flags { QELR_QP_FLAG_SQ = 1 << 0, QELR_QP_FLAG_RQ = 1 << 1, }; struct qelr_qp { struct verbs_qp verbs_qp; struct ibv_qp *ibv_qp; pthread_spinlock_t q_lock; enum qelr_qp_state state; /* QP state */ uint8_t flags; struct qelr_qp_hwq_info sq; struct qelr_qp_hwq_info rq; struct { uint64_t wr_id; enum ibv_wc_opcode opcode; uint32_t bytes_len; uint8_t wqe_size; uint8_t signaled; } *wqe_wr_id; struct { uint64_t wr_id; uint8_t wqe_size; } *rqe_wr_id; uint8_t prev_wqe_size; uint32_t max_inline_data; uint32_t qp_id; int sq_sig_all; int atomic_supported; uint8_t edpm_disabled; uint8_t edpm_mode; struct qelr_srq *srq; }; static inline struct qelr_devctx *get_qelr_ctx(struct ibv_context *ibctx) { return container_of(ibctx, struct qelr_devctx, ibv_ctx.context); } static inline struct qelr_device *get_qelr_dev(struct ibv_device *ibdev) { return container_of(ibdev, struct qelr_device, ibv_dev.device); } static inline struct ibv_qp *get_ibv_qp(struct qelr_qp *qp) { return &qp->verbs_qp.qp; } static inline struct qelr_qp *get_qelr_qp(struct ibv_qp *ibqp) { struct verbs_qp *vqp = (struct verbs_qp *)ibqp; return container_of(vqp, struct qelr_qp, verbs_qp); } static inline struct qelr_pd *get_qelr_pd(struct ibv_pd *ibpd) { return container_of(ibpd, struct qelr_pd, ibv_pd); } static inline struct qelr_cq *get_qelr_cq(struct ibv_cq *ibcq) { return container_of(ibcq, struct qelr_cq, ibv_cq); } static inline struct qelr_srq *get_qelr_srq(struct ibv_srq *ibsrq) { struct verbs_srq *vsrq = (struct verbs_srq *)ibsrq; return container_of(vsrq, struct qelr_srq, verbs_srq); } static inline struct ibv_srq *get_ibv_srq(struct qelr_srq *srq) { return &srq->verbs_srq.srq; } #define SET_FIELD(value, name, flag) \ do { \ (value) &= ~(name ## _MASK << name ## _SHIFT); \ (value) |= ((flag) << (name ## _SHIFT)); \ } while (0) #define SET_FIELD2(value, name, flag) \ ((value) |= ((flag) << (name ## _SHIFT))) #define GET_FIELD(value, name) \ (((value) >> (name ## _SHIFT)) & name ## _MASK) #define ROCE_WQE_ELEM_SIZE sizeof(struct rdma_sq_sge) #define RDMA_WQE_BYTES (16) #define QELR_RESP_IMM (RDMA_CQE_RESPONDER_IMM_FLG_MASK << \ RDMA_CQE_RESPONDER_IMM_FLG_SHIFT) #define QELR_RESP_INV (RDMA_CQE_RESPONDER_INV_FLG_MASK << \ RDMA_CQE_RESPONDER_INV_FLG_SHIFT) #define QELR_RESP_RDMA (RDMA_CQE_RESPONDER_RDMA_FLG_MASK << \ RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT) #define QELR_RESP_RDMA_IMM (QELR_RESP_IMM | QELR_RESP_RDMA) #define TYPEPTR_ADDR_SET(type_ptr, field, vaddr) \ do { \ (type_ptr)->field.hi = htole32(U64_HI(vaddr)); \ (type_ptr)->field.lo = htole32(U64_LO(vaddr)); \ } while (0) #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ do { \ TYPEPTR_ADDR_SET(sge, addr, vaddr); \ (sge)->length = htole32(vlength); \ (sge)->flags = htole32(vflags); \ } while (0) #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ do { \ TYPEPTR_ADDR_SET(hdr, wr_id, vwr_id); \ (hdr)->num_sges = num_sge; \ } while (0) #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ do { \ TYPEPTR_ADDR_SET(sge, addr, vaddr); \ (sge)->length = htole32(vlength); \ (sge)->l_key = htole32(vlkey); \ } while (0) #define U64_HI(val) ((uint32_t)(((uint64_t)(uintptr_t)(val)) >> 32)) #define U64_LO(val) ((uint32_t)(((uint64_t)(uintptr_t)(val)) & 0xffffffff)) #define HILO_U64(hi, lo) ((uintptr_t)((((uint64_t)(hi)) << 32) + (lo))) #define QELR_MAX_RQ_WQE_SIZE (RDMA_MAX_SGE_PER_RQ_WQE) #define QELR_MAX_SQ_WQE_SIZE (ROCE_REQ_MAX_SINGLE_SQ_WQE_SIZE / \ ROCE_WQE_ELEM_SIZE) #endif /* __QELR_H__ */