Skip to content

Commit

Permalink
RDMA/hns: Add support for XRC on HIP09
Browse files Browse the repository at this point in the history 
The HIP09 supports XRC transport service, it greatly saves the number of
QPs required to connect all processes in a large cluster.

Link: https://lore.kernel.org/r/1614826558-35423-1-git-send-email-liweihang@huawei.com
Signed-off-by: Wenpeng Liang <liangwenpeng@huawei.com>
Signed-off-by: Weihang Li <liweihang@huawei.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
  • Loading branch information
@jgunthorpe
Wenpeng Liang authored and jgunthorpe committed Mar 11, 2021
1 parent c33d516 commit 3254887
Show file tree
Hide file tree
Showing 9 changed files with 258 additions and 70 deletions.
3 changes: 3 additions & 0 deletions drivers/infiniband/hw/hns/hns_roce_alloc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,9 @@ int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,

void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev)
{
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC)
hns_roce_cleanup_xrcd_table(hr_dev);

if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
hns_roce_cleanup_srq_table(hr_dev);
hns_roce_cleanup_qp_table(hr_dev);
Expand Down
25 changes: 25 additions & 0 deletions drivers/infiniband/hw/hns/hns_roce_device.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ enum {
SERV_TYPE_UC,
SERV_TYPE_RD,
SERV_TYPE_UD,
SERV_TYPE_XRC = 5,
};

enum hns_roce_qp_state {
Expand Down Expand Up @@ -168,6 +169,8 @@ enum hns_roce_event {
HNS_ROCE_EVENT_TYPE_DB_OVERFLOW = 0x12,
HNS_ROCE_EVENT_TYPE_MB = 0x13,
HNS_ROCE_EVENT_TYPE_FLR = 0x15,
HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION = 0x16,
HNS_ROCE_EVENT_TYPE_INVALID_XRCETH = 0x17,
};

#define HNS_ROCE_CAP_FLAGS_EX_SHIFT 12
Expand All @@ -179,6 +182,7 @@ enum {
HNS_ROCE_CAP_FLAG_RECORD_DB = BIT(3),
HNS_ROCE_CAP_FLAG_SQ_RECORD_DB = BIT(4),
HNS_ROCE_CAP_FLAG_SRQ = BIT(5),
HNS_ROCE_CAP_FLAG_XRC = BIT(6),
HNS_ROCE_CAP_FLAG_MW = BIT(7),
HNS_ROCE_CAP_FLAG_FRMR = BIT(8),
HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL = BIT(9),
Expand Down Expand Up @@ -244,6 +248,11 @@ struct hns_roce_pd {
unsigned long pdn;
};

struct hns_roce_xrcd {
struct ib_xrcd ibxrcd;
u32 xrcdn;
};

struct hns_roce_bitmap {
/* Bitmap Traversal last a bit which is 1 */
unsigned long last;
Expand Down Expand Up @@ -467,6 +476,7 @@ struct hns_roce_srq {
u32 rsv_sge;
int wqe_shift;
u32 cqn;
u32 xrcdn;
void __iomem *db_reg_l;

atomic_t refcount;
Expand Down Expand Up @@ -640,6 +650,8 @@ struct hns_roce_qp {
enum hns_roce_event event_type);
unsigned long qpn;

u32 xrcdn;

atomic_t refcount;
struct completion free;

Expand Down Expand Up @@ -766,6 +778,8 @@ struct hns_roce_caps {
int reserved_uars;
int num_pds;
int reserved_pds;
u32 num_xrcds;
u32 reserved_xrcds;
u32 mtt_entry_sz;
u32 cqe_sz;
u32 page_size_cap;
Expand Down Expand Up @@ -963,6 +977,7 @@ struct hns_roce_dev {

struct hns_roce_cmdq cmd;
struct hns_roce_bitmap pd_bitmap;
struct hns_roce_bitmap xrcd_bitmap;
struct hns_roce_uar_table uar_table;
struct hns_roce_mr_table mr_table;
struct hns_roce_cq_table cq_table;
Expand Down Expand Up @@ -1004,6 +1019,11 @@ static inline struct hns_roce_pd *to_hr_pd(struct ib_pd *ibpd)
return container_of(ibpd, struct hns_roce_pd, ibpd);
}

static inline struct hns_roce_xrcd *to_hr_xrcd(struct ib_xrcd *ibxrcd)
{
return container_of(ibxrcd, struct hns_roce_xrcd, ibxrcd);
}

static inline struct hns_roce_ah *to_hr_ah(struct ib_ah *ibah)
{
return container_of(ibah, struct hns_roce_ah, ibah);
Expand Down Expand Up @@ -1136,13 +1156,15 @@ int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev);
void hns_roce_init_cq_table(struct hns_roce_dev *hr_dev);
int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev);
int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev);
int hns_roce_init_xrcd_table(struct hns_roce_dev *hr_dev);

void hns_roce_cleanup_pd_table(struct hns_roce_dev *hr_dev);
void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev);
void hns_roce_cleanup_eq_table(struct hns_roce_dev *hr_dev);
void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev);
void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev);
void hns_roce_cleanup_srq_table(struct hns_roce_dev *hr_dev);
void hns_roce_cleanup_xrcd_table(struct hns_roce_dev *hr_dev);

int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj);
void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj,
Expand Down Expand Up @@ -1207,6 +1229,9 @@ int hns_roce_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr,
struct ib_udata *udata);
int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata);

int hns_roce_alloc_xrcd(struct ib_xrcd *ib_xrcd, struct ib_udata *udata);
int hns_roce_dealloc_xrcd(struct ib_xrcd *ib_xrcd, struct ib_udata *udata);

struct ib_qp *hns_roce_create_qp(struct ib_pd *ib_pd,
struct ib_qp_init_attr *init_attr,
struct ib_udata *udata);
Expand Down
Loading

0 comments on commit 3254887

Please sign in to comment.