From 29a335dd758af8da14808d691f4072ac5b7d7b19 Mon Sep 17 00:00:00 2001 From: Tian Xin Date: Tue, 24 Sep 2024 14:43:56 +0800 Subject: [PATCH 01/13] libxscale: Introduce xscale user space RDMA provider This patch adds a new userspace RDMA provider driver for the xscale family of RDMA devices. The provider implements the standard libibverbs interface, allowing applications to use RDMA operations via the verbs API. The initial implementation includes: - Basic CMake build infrastructure and provider registration. - Device probing and context allocation via the verbs framework. - Query operations: query_port and query_device_ex with firmware version formatting. - Import of necessary kernel ABI headers (xsc-abi.h) and integration with the kernel driver. This is the foundational patch; subsequent patches will add full functionality for PD, MR, CQ, QP, and advanced features. Signed-off-by: Tian Xin Signed-off-by: Wei Honggang Signed-off-by: Zhao Qianwei Signed-off-by: Li Qiang Signed-off-by: Yan Lei --- CMakeLists.txt | 1 + MAINTAINERS | 9 + kernel-headers/CMakeLists.txt | 2 + kernel-headers/rdma/ib_user_ioctl_verbs.h | 1 + kernel-headers/rdma/xsc-abi.h | 39 +++ libibverbs/verbs.h | 1 + providers/xscale/CMakeLists.txt | 8 + providers/xscale/libxsc.map | 6 + providers/xscale/verbs.c | 75 ++++++ providers/xscale/xsc-abi.h | 21 ++ providers/xscale/xscale.c | 293 ++++++++++++++++++++++ providers/xscale/xscale.h | 163 ++++++++++++ 12 files changed, 619 insertions(+) create mode 100644 kernel-headers/rdma/xsc-abi.h create mode 100644 providers/xscale/CMakeLists.txt create mode 100644 providers/xscale/libxsc.map create mode 100644 providers/xscale/verbs.c create mode 100644 providers/xscale/xsc-abi.h create mode 100644 providers/xscale/xscale.c create mode 100644 providers/xscale/xscale.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 41cd7b3d9..2d23ba42d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -823,6 +823,7 @@ add_subdirectory(providers/mthca) add_subdirectory(providers/ocrdma) add_subdirectory(providers/qedr) add_subdirectory(providers/vmw_pvrdma) +add_subdirectory(providers/xscale) endif() add_subdirectory(providers/hfi1verbs) diff --git a/MAINTAINERS b/MAINTAINERS index ed1dc0d6f..6a4517eff 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -196,3 +196,12 @@ PYVERBS M: Edward Srouji S: Supported F: pyverbs/ + +XSCALE USERSPACE PROVIDER (for xsc_ib.ko) +M: Wei Honggang +M: Zhao Qianwei +M: Li Qiang +M: Tian Xin +M: Yan Lei +S: Supported +F: providers/xscale/ diff --git a/kernel-headers/CMakeLists.txt b/kernel-headers/CMakeLists.txt index cd9c08cad..eff602986 100644 --- a/kernel-headers/CMakeLists.txt +++ b/kernel-headers/CMakeLists.txt @@ -27,6 +27,7 @@ publish_internal_headers(rdma rdma/rvt-abi.h rdma/siw-abi.h rdma/vmw_pvrdma-abi.h + rdma/xsc-abi.h ) publish_internal_headers(rdma/hfi @@ -82,6 +83,7 @@ rdma_kernel_provider_abi( rdma/rdma_user_rxe.h rdma/siw-abi.h rdma/vmw_pvrdma-abi.h + rdma/xsc-abi.h ) publish_headers(infiniband diff --git a/kernel-headers/rdma/ib_user_ioctl_verbs.h b/kernel-headers/rdma/ib_user_ioctl_verbs.h index 51030c27d..485d8ec54 100644 --- a/kernel-headers/rdma/ib_user_ioctl_verbs.h +++ b/kernel-headers/rdma/ib_user_ioctl_verbs.h @@ -257,6 +257,7 @@ enum rdma_driver_id { RDMA_DRIVER_ERDMA, RDMA_DRIVER_MANA, RDMA_DRIVER_IONIC, + RDMA_DRIVER_XSC, }; enum ib_uverbs_gid_type { diff --git a/kernel-headers/rdma/xsc-abi.h b/kernel-headers/rdma/xsc-abi.h new file mode 100644 index 000000000..f79940716 --- /dev/null +++ b/kernel-headers/rdma/xsc-abi.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_ABI_USER_H +#define XSC_ABI_USER_H + +#include +#include /* For ETH_ALEN. */ +#include + +/* Make sure that all structs defined in this file remain laid out so + * that they pack the same way on 32-bit and 64-bit architectures (to + * avoid incompatibility between 32-bit userspace and 64-bit kernels). + * In particular do not use pointer types -- pass pointers in __u64 + * instead. + */ + +struct xsc_ib_alloc_ucontext_resp { + __u32 qp_tab_size; + __u32 cache_line_size; + __u16 max_sq_desc_sz; + __u16 max_rq_desc_sz; + __u32 max_send_wr; + __u32 max_recv_wr; + __u16 num_ports; + __u16 device_id; + __aligned_u64 qpm_tx_db; + __aligned_u64 qpm_rx_db; + __aligned_u64 cqm_next_cid_reg; + __aligned_u64 cqm_armdb; + __u32 send_ds_num; + __u32 recv_ds_num; + __u32 resv; +}; + +#endif /* XSC_ABI_USER_H */ diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h index 60103d98a..6e86f388d 100644 --- a/libibverbs/verbs.h +++ b/libibverbs/verbs.h @@ -2344,6 +2344,7 @@ extern const struct verbs_device_ops verbs_provider_rxe; extern const struct verbs_device_ops verbs_provider_siw; extern const struct verbs_device_ops verbs_provider_vmw_pvrdma; extern const struct verbs_device_ops verbs_provider_ionic; +extern const struct verbs_device_ops verbs_provider_xscale; extern const struct verbs_device_ops verbs_provider_all; extern const struct verbs_device_ops verbs_provider_none; void ibv_static_providers(void *unused, ...); diff --git a/providers/xscale/CMakeLists.txt b/providers/xscale/CMakeLists.txt new file mode 100644 index 000000000..220944d52 --- /dev/null +++ b/providers/xscale/CMakeLists.txt @@ -0,0 +1,8 @@ +rdma_shared_provider(xscale libxsc.map + 1 1.24.${PACKAGE_VERSION} + xscale.c + verbs.c +) + +rdma_pkg_config("xscale" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}") + diff --git a/providers/xscale/libxsc.map b/providers/xscale/libxsc.map new file mode 100644 index 000000000..2af4bc736 --- /dev/null +++ b/providers/xscale/libxsc.map @@ -0,0 +1,6 @@ +/* Export symbols should be added below according to + Documentation/versioning.md document. */ +XSC_1.0 { + local: *; +}; + diff --git a/providers/xscale/verbs.c b/providers/xscale/verbs.c new file mode 100644 index 000000000..ab5d7b584 --- /dev/null +++ b/providers/xscale/verbs.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "xscale.h" +#include "xsc-abi.h" + +int xsc_query_port(struct ibv_context *context, uint8_t port, + struct ibv_port_attr *attr) +{ + struct ibv_query_port cmd; + + return ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd)); +} + +static void xsc_set_fw_version(struct ibv_device_attr *attr, + union xsc_ib_fw_ver *fw_ver) +{ + uint8_t ver_major = fw_ver->s.ver_major; + uint8_t ver_minor = fw_ver->s.ver_minor; + uint16_t ver_patch = fw_ver->s.ver_patch; + uint32_t ver_tweak = fw_ver->s.ver_tweak; + + if (ver_tweak == 0) { + snprintf(attr->fw_ver, sizeof(attr->fw_ver), "v%u.%u.%u", + ver_major, ver_minor, ver_patch); + } else { + snprintf(attr->fw_ver, sizeof(attr->fw_ver), "v%u.%u.%u+%u", + ver_major, ver_minor, ver_patch, ver_tweak); + } +} + +int xsc_query_device_ex(struct ibv_context *context, + const struct ibv_query_device_ex_input *input, + struct ibv_device_attr_ex *attr, size_t attr_size) +{ + struct ib_uverbs_ex_query_device_resp resp; + size_t resp_size = sizeof(resp); + union xsc_ib_fw_ver raw_fw_ver; + int err; + + raw_fw_ver.data = 0; + err = ibv_cmd_query_device_any(context, input, attr, attr_size, + &resp, &resp_size); + if (err) + return err; + + raw_fw_ver.data = resp.base.fw_ver; + xsc_set_fw_version(&attr->orig_attr, &raw_fw_ver); + + return 0; +} diff --git a/providers/xscale/xsc-abi.h b/providers/xscale/xsc-abi.h new file mode 100644 index 000000000..f7d48c6e6 --- /dev/null +++ b/providers/xscale/xsc-abi.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_ABI_H +#define XSC_ABI_H + +#include +#include +#include +#include + +#define XSC_UVERBS_MIN_ABI_VERSION 1 +#define XSC_UVERBS_MAX_ABI_VERSION 1 + +DECLARE_DRV_CMD(xsc_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT, + empty, xsc_ib_alloc_ucontext_resp); + +#endif /* XSC_ABI_H */ diff --git a/providers/xscale/xscale.c b/providers/xscale/xscale.c new file mode 100644 index 000000000..bd263a5d2 --- /dev/null +++ b/providers/xscale/xscale.c @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#define _GNU_SOURCE +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "xscale.h" +#include "xsc-abi.h" + +static const struct verbs_match_ent hca_table[] = { + VERBS_MODALIAS_MATCH("*xscale*", NULL), + {} +}; + +uint32_t xsc_debug_mask; +static void xsc_free_context(struct ibv_context *ibctx); + +static const struct verbs_context_ops xsc_ctx_common_ops = { + .query_port = xsc_query_port, + .query_device_ex = xsc_query_device_ex, + .free_context = xsc_free_context, +}; + +static void open_debug_file(struct xsc_context *ctx) +{ + char *env; + + env = getenv("XSC_DEBUG_FILE"); + if (!env) { + ctx->dbg_fp = stderr; + return; + } + + ctx->dbg_fp = fopen(env, "aw+"); + if (!ctx->dbg_fp) { + fprintf(stderr, "Failed opening debug file %s, using stderr\n", + env); + ctx->dbg_fp = stderr; + return; + } +} + +static void close_debug_file(struct xsc_context *ctx) +{ + if (ctx->dbg_fp && ctx->dbg_fp != stderr) + fclose(ctx->dbg_fp); +} + +static void set_debug_mask(void) +{ + char *env; + + env = getenv("XSC_DEBUG_MASK"); + if (env) + xsc_debug_mask = strtol(env, NULL, 0); +} + +static int xsc_cmd_get_context(struct xsc_context *context, + struct xsc_alloc_ucontext *req, size_t req_len, + struct xsc_alloc_ucontext_resp *resp, + size_t resp_len) +{ + struct verbs_context *verbs_ctx = &context->ibv_ctx; + + return ibv_cmd_get_context(verbs_ctx, &req->ibv_cmd, + req_len, NULL, &resp->ibv_resp, resp_len); +} + +static int xsc_mmap(struct xsc_device *xdev, struct xsc_context *context, + int cmd_fd, int size) +{ + uint64_t page_mask; + size_t mdb_size = 0; + + page_mask = (~(xdev->page_size - 1)); + xsc_dbg(context->dbg_fp, XSC_DBG_CTX, "page size:%d\n", size); + context->sqm_reg_va = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, + cmd_fd, context->qpm_tx_db & page_mask); + if (context->sqm_reg_va == MAP_FAILED) + return -1; + + xsc_dbg(context->dbg_fp, XSC_DBG_CTX, "qpm reg va:%p\n", context->sqm_reg_va); + + context->rqm_reg_va = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, + cmd_fd, context->qpm_rx_db & page_mask); + if (context->rqm_reg_va == MAP_FAILED) + goto free_sqm; + + xsc_dbg(context->dbg_fp, XSC_DBG_CTX, "qpm reg va:%p\n", context->rqm_reg_va); + + context->cqm_reg_va = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, + cmd_fd, context->cqm_next_cid_reg & page_mask); + if (context->cqm_reg_va == MAP_FAILED) + goto free_rqm; + + xsc_dbg(context->dbg_fp, XSC_DBG_CTX, "cqm ci va:%p\n", context->cqm_reg_va); + context->db_mmap_size = size; + + context->cqm_armdb_va = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, + cmd_fd, context->cqm_armdb & page_mask); + if (context->cqm_armdb_va == MAP_FAILED) + goto free_cqm; + + xsc_dbg(context->dbg_fp, XSC_DBG_CTX, "cqm armdb va:%p\n", context->cqm_armdb_va); + + if (context->multidb_num) { + mdb_size = context->multidb_num * sizeof(uint64_t); + mdb_size = (mdb_size + (xdev->page_size - 1)) & page_mask; + context->mdb_base = mmap(NULL, mdb_size, PROT_READ | PROT_WRITE, MAP_SHARED, cmd_fd, + context->tx_multidb_base & page_mask); + if (context->mdb_base == MAP_FAILED) + goto free_mdb; + xsc_dbg(context->dbg_fp, XSC_DBG_CTX, "multidb va base:%p, mdb_size:0x%lx\n", + context->mdb_base, mdb_size); + context->mdb_mmap_size = mdb_size; + } + + return 0; + +free_mdb: + munmap(context->mdb_base, mdb_size); +free_cqm: + munmap(context->cqm_reg_va, size); +free_rqm: + munmap(context->rqm_reg_va, size); +free_sqm: + munmap(context->sqm_reg_va, size); + + return -1; + +} +static void xsc_munmap(struct xsc_context *context) +{ + if (context->sqm_reg_va) + munmap(context->sqm_reg_va, context->db_mmap_size); + + if (context->rqm_reg_va) + munmap(context->rqm_reg_va, context->db_mmap_size); + + if (context->cqm_reg_va) + munmap(context->cqm_reg_va, context->db_mmap_size); + + if (context->cqm_armdb_va) + munmap(context->cqm_armdb_va, context->db_mmap_size); + + if (context->mdb_base) + munmap(context->mdb_base, context->mdb_mmap_size); +} + +static struct verbs_context *xsc_alloc_context(struct ibv_device *ibdev, + int cmd_fd, void *private_data) +{ + struct xsc_context *context; + struct xsc_alloc_ucontext req; + struct xsc_alloc_ucontext_resp resp; + int i; + int page_size; + struct xsc_device *xdev = to_xdev(ibdev); + struct verbs_context *v_ctx; + struct ibv_device_attr_ex device_attr; + + context = verbs_init_and_alloc_context(ibdev, cmd_fd, context, ibv_ctx, + RDMA_DRIVER_XSC); + if (!context) + return NULL; + + v_ctx = &context->ibv_ctx; + page_size = xdev->page_size; + + open_debug_file(context); + set_debug_mask(); + if (gethostname(context->hostname, sizeof(context->hostname))) + strncpy(context->hostname, "host_unknown", NAME_BUFFER_SIZE - 1); + + memset(&req, 0, sizeof(req)); + memset(&resp, 0, sizeof(resp)); + + if (xsc_cmd_get_context(context, &req, sizeof(req), &resp, + sizeof(resp))) + goto err_free; + + context->max_num_qps = resp.qp_tab_size; + context->max_sq_desc_sz = resp.max_sq_desc_sz; + context->max_rq_desc_sz = resp.max_rq_desc_sz; + context->max_send_wr = resp.max_send_wr; + context->num_ports = resp.num_ports; + context->max_recv_wr = resp.max_recv_wr; + context->qpm_tx_db = resp.qpm_tx_db; + context->qpm_rx_db = resp.qpm_rx_db; + context->cqm_next_cid_reg = resp.cqm_next_cid_reg; + context->cqm_armdb = resp.cqm_armdb; + context->send_ds_num = resp.send_ds_num; + context->send_ds_shift = xsc_ilog2(resp.send_ds_num); + context->recv_ds_num = resp.recv_ds_num; + context->recv_ds_shift = xsc_ilog2(resp.recv_ds_num); + + xsc_dbg(context->dbg_fp, XSC_DBG_CTX, + "max_num_qps:%u, max_sq_desc_sz:%u max_rq_desc_sz:%u\n", + context->max_num_qps, context->max_sq_desc_sz, + context->max_rq_desc_sz); + xsc_dbg(context->dbg_fp, XSC_DBG_CTX, + "max_send_wr:%u, num_ports:%u, max_recv_wr:%u\n", + context->max_send_wr, + context->num_ports, context->max_recv_wr); + xsc_dbg(context->dbg_fp, XSC_DBG_CTX, + "send_ds_num:%u shift:%u recv_ds_num:%u shift:%u\n", + context->send_ds_num, context->send_ds_shift, + context->recv_ds_num, context->recv_ds_shift); + + pthread_mutex_init(&context->qp_table_mutex, NULL); + for (i = 0; i < XSC_QP_TABLE_SIZE; ++i) + context->qp_table[i].refcnt = 0; + + context->page_size = page_size; + if (xsc_mmap(xdev, context, cmd_fd, page_size)) + goto err_free; + + verbs_set_ops(v_ctx, &xsc_ctx_common_ops); + + memset(&device_attr, 0, sizeof(device_attr)); + if (!xsc_query_device_ex(&v_ctx->context, NULL, &device_attr, + sizeof(struct ibv_device_attr_ex))) { + context->max_cqe = device_attr.orig_attr.max_cqe; + } + + return v_ctx; + +err_free: + verbs_uninit_context(&context->ibv_ctx); + close_debug_file(context); + free(context); + return NULL; +} + +static void xsc_free_context(struct ibv_context *ibctx) +{ + struct xsc_context *context = to_xctx(ibctx); + + xsc_dbg(context->dbg_fp, XSC_DBG_CTX, "\n"); + xsc_munmap(context); + + verbs_uninit_context(&context->ibv_ctx); + close_debug_file(context); + free(context); +} + +static void xsc_uninit_device(struct verbs_device *verbs_device) +{ + struct xsc_device *xdev = to_xdev(&verbs_device->device); + + free(xdev); +} + +static struct verbs_device *xsc_device_alloc(struct verbs_sysfs_dev *sysfs_dev) +{ + struct xsc_device *xdev; + + xdev = calloc(1, sizeof(*xdev)); + if (!xdev) + return NULL; + + xdev->page_size = sysconf(_SC_PAGESIZE); + + return &xdev->verbs_dev; +} + +static const struct verbs_device_ops xsc_dev_ops = { + .name = "xscale", + .match_min_abi_version = XSC_UVERBS_MIN_ABI_VERSION, + .match_max_abi_version = XSC_UVERBS_MAX_ABI_VERSION, + .match_table = hca_table, + .alloc_device = xsc_device_alloc, + .uninit_device = xsc_uninit_device, + .alloc_context = xsc_alloc_context, +}; + +PROVIDER_DRIVER(xscale, xsc_dev_ops); diff --git a/providers/xscale/xscale.h b/providers/xscale/xscale.h new file mode 100644 index 000000000..04c3390ef --- /dev/null +++ b/providers/xscale/xscale.h @@ -0,0 +1,163 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSCALE_H +#define XSCALE_H + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "xsc-abi.h" + +enum { + XSC_DBG_QP = 1 << 0, + XSC_DBG_CQ = 1 << 1, + XSC_DBG_QP_SEND = 1 << 2, + XSC_DBG_QP_SEND_ERR = 1 << 3, + XSC_DBG_CQ_CQE = 1 << 4, + XSC_DBG_CONTIG = 1 << 5, + XSC_DBG_DR = 1 << 6, + XSC_DBG_CTX = 1 << 7, + XSC_DBG_PD = 1 << 8, + XSC_DBG_MR = 1 << 9, +}; + +extern uint32_t xsc_debug_mask; + +#define xsc_dbg(fp, mask, fmt, args...) \ + do { \ + if (xsc_debug_mask & (mask)) { \ + char host[256]; \ + char timestr[32]; \ + struct tm now_tm; \ + time_t now_time; \ + time(&now_time); \ + localtime_r(&now_time, &now_tm); \ + strftime(timestr, sizeof(timestr), "%Y-%m-%d %X", \ + &now_tm); \ + gethostname(host, 256); \ + fprintf(fp, "[%s %s %s %d] " fmt, timestr, host, \ + __func__, __LINE__, ##args); \ + } \ + } while (0) + +#define xsc_err(fmt, args...) \ + do { \ + char host[256]; \ + char timestr[32]; \ + struct tm now_tm; \ + time_t now_time; \ + time(&now_time); \ + localtime_r(&now_time, &now_tm); \ + strftime(timestr, sizeof(timestr), "%Y-%m-%d %X", &now_tm); \ + gethostname(host, 256); \ + printf("[%s %s %s %d] " fmt, timestr, host, __func__, \ + __LINE__, ##args); \ + } while (0) + +enum { + XSC_QP_TABLE_SHIFT = 12, + XSC_QP_TABLE_MASK = (1 << XSC_QP_TABLE_SHIFT) - 1, + XSC_QP_TABLE_SIZE = 1 << (24 - XSC_QP_TABLE_SHIFT), +}; + +struct xsc_device { + struct verbs_device verbs_dev; + int page_size; +}; + +#define NAME_BUFFER_SIZE 64 + +struct xsc_context { + struct verbs_context ibv_ctx; + int max_num_qps; + struct { + struct xsc_qp **table; + int refcnt; + } qp_table[XSC_QP_TABLE_SIZE]; + pthread_mutex_t qp_table_mutex; + + int max_sq_desc_sz; + int max_rq_desc_sz; + int max_send_wr; + int max_recv_wr; + int num_ports; + char hostname[NAME_BUFFER_SIZE]; + uint32_t max_cqe; + void *sqm_reg_va; + void *rqm_reg_va; + void *cqm_reg_va; + void *cqm_armdb_va; + int db_mmap_size; + uint32_t page_size; + uint64_t qpm_tx_db; + uint64_t qpm_rx_db; + uint64_t cqm_next_cid_reg; + uint64_t cqm_armdb; + uint32_t send_ds_num; + uint32_t recv_ds_num; + uint32_t send_ds_shift; + uint32_t recv_ds_shift; + FILE *dbg_fp; + uint32_t multidb_num; + uint32_t tx_multidb_base; + void *mdb_base; + uint32_t tx_mdb_idx; + uint32_t mdb_mmap_size; +}; + +union xsc_ib_fw_ver { + uint64_t data; + struct { + uint8_t ver_major; + uint8_t ver_minor; + uint16_t ver_patch; + uint32_t ver_tweak; + } s; +}; + +static inline int xsc_ilog2(int n) +{ + int t; + + if (n <= 0) + return -1; + + t = 0; + while ((1 << t) < n) + ++t; + + return t; +} + +static inline struct xsc_device *to_xdev(struct ibv_device *ibdev) +{ + return container_of(ibdev, struct xsc_device, verbs_dev.device); +} + +static inline struct xsc_context *to_xctx(struct ibv_context *ibctx) +{ + return container_of(ibctx, struct xsc_context, ibv_ctx.context); +} + +int xsc_query_device(struct ibv_context *context, struct ibv_device_attr *attr); +int xsc_query_device_ex(struct ibv_context *context, + const struct ibv_query_device_ex_input *input, + struct ibv_device_attr_ex *attr, size_t attr_size); +int xsc_query_port(struct ibv_context *context, uint8_t port, + struct ibv_port_attr *attr); + +#endif /* XSC_H */ From 3362d59d6343600334d6f89abcefcbca2f5615bb Mon Sep 17 00:00:00 2001 From: Tian Xin Date: Wed, 8 Jul 2026 18:46:21 +0800 Subject: [PATCH 02/13] libxscale: Add hardware definitions and low-level APIs Define the hardware-specific structures, constants, and inline helper functions required to interact with the XScale RDMA engine. This patch introduces: - xsc_hsi.h: Host-software interface definitions, including message opcodes, completion queue entries (CQE), work queue elements (WQE), doorbell formats, and register layouts. - xsc_hw.h: Low-level hardware access routines for Andes and Diamond device families, such as setting data segments, ringing doorbells, updating CQ consumer indices, and translating error codes to verbs statuses. These definitions are used by all subsequent patches to construct and post work requests, poll completions, and manage hardware resources. Signed-off-by: Tian Xin Signed-off-by: Wei Honggang Signed-off-by: Zhao Qianwei Signed-off-by: Li Qiang Signed-off-by: Yan Lei --- providers/xscale/xsc_hsi.h | 326 ++++++++++++++++ providers/xscale/xsc_hw.h | 767 +++++++++++++++++++++++++++++++++++++ providers/xscale/xscale.c | 1 + providers/xscale/xscale.h | 21 + 4 files changed, 1115 insertions(+) create mode 100644 providers/xscale/xsc_hsi.h create mode 100644 providers/xscale/xsc_hw.h diff --git a/providers/xscale/xsc_hsi.h b/providers/xscale/xsc_hsi.h new file mode 100644 index 000000000..fb5da97d1 --- /dev/null +++ b/providers/xscale/xsc_hsi.h @@ -0,0 +1,326 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef __XSC_HSI_H__ +#define __XSC_HSI_H__ + +#include +#include +#include + +#define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16)) +#define lower_32_bits(n) ((uint32_t)(n)) + +#define DMA_LO_LE(x) __cpu_to_le32(lower_32_bits(x)) +#define DMA_HI_LE(x) __cpu_to_le32(upper_32_bits(x)) +#define DMA_REGPAIR_LE(x, val) do { \ + (x).hi = DMA_HI_LE((val)); \ + (x).lo = DMA_LO_LE((val)); \ + } while (0) + +#define WR_LE_16(x, val) do { (x) = __cpu_to_le16(val); } while (0) +#define WR_LE_32(x, val) do { (x) = __cpu_to_le32(val); } while (0) +#define WR_LE_64(x, val) do { (x) = __cpu_to_le64(val); } while (0) +#define WR_LE_R64(x, val) DMA_REGPAIR_LE(x, val) +#define WR_BE_32(x, val) do { (x) = __cpu_to_be32(val); } while (0) + +#define RD_LE_16(x) __le16_to_cpu(x) +#define RD_LE_32(x) __le32_to_cpu(x) +#define RD_BE_32(x) __be32_to_cpu(x) +#define RD_BE_64(x) __be64_to_cpu(x) + +#define WR_REG(addr, val) mmio_write32_le(addr, val) +#define RD_REG(addr) mmio_read32_le(addr) + +/* message opcode */ +enum { + XSC_MSG_OPCODE_SEND = 0, + XSC_MSG_OPCODE_RDMA_WRITE = 1, + XSC_MSG_OPCODE_RDMA_READ = 2, + XSC_MSG_OPCODE_MAD = 3, + XSC_MSG_OPCODE_RDMA_ACK = 4, + XSC_MSG_OPCODE_RDMA_ACK_READ = 5, + XSC_MSG_OPCODE_RDMA_CNP = 6, + XSC_MSG_OPCODE_RAW = 7, + XSC_MSG_OPCODE_VIRTIO_NET = 8, + XSC_MSG_OPCODE_VIRTIO_BLK = 9, + XSC_MSG_OPCODE_RAW_TPE = 10, + XSC_MSG_OPCODE_INIT_QP_REQ = 11, + XSC_MSG_OPCODE_INIT_QP_RSP = 12, + XSC_MSG_OPCODE_INIT_PATH_REQ = 13, + XSC_MSG_OPCODE_INIT_PATH_RSP = 14, + XSC_MSG_OPCODE_SO_WRITE = 22, + XSC_MSG_OPCODE_RDMA_ATOMIC_CMP_AND_SWAP = 26, + XSC_MSG_OPCODE_RDMA_ATOMIC_FETCH_AND_ADD = 27, + + XSC_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_CMP_AND_SWAP = 31, + XSC_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_FETCH_AND_ADD = 32, + XSC_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_CMP_AND_SWAP = 33, + XSC_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_FETCH_AND_ADD = 34, + + XSC_MSG_OPCODE_MAX, +}; + +enum { + XSC_MSG_OPCODE_SEND_DIAMOND_NEXT = 3, +}; + +enum { + XSC_REQ = 0, + XSC_RSP = 1, +}; + +enum { + XSC_WITHOUT_IMMDT = 0, + XSC_WITH_IMMDT = 1, +}; + +enum { + XSC_ANDES_ERR_CODE_NAK_RETRY = 0x40, + XSC_ANDES_ERR_CODE_NAK_OPCODE = 0x41, + XSC_ANDES_ERR_CODE_NAK_MR = 0x42, + XSC_ANDES_ERR_CODE_NAK_OPERATION = 0x43, + XSC_ANDES_ERR_CODE_NAK_RNR = 0x44, + XSC_ANDES_ERR_CODE_LOCAL_MR = 0x45, + XSC_ANDES_ERR_CODE_LOCAL_LEN = 0x46, + XSC_ANDES_ERR_CODE_LOCAL_OPCODE = 0x47, + XSC_ANDES_ERR_CODE_CQ_OVER_FLOW = 0x48, + XSC_ANDES_ERR_CODE_LOCAL_OPERATION_WQE = 0x49, + XSC_ANDES_ERR_CODE_STRG_ACC_GEN_CQE = 0x4b, + XSC_ANDES_ERR_CODE_STRG_ACC = 0x4c, + XSC_ANDES_ERR_CODE_CQE_ACC = 0x4d, + XSC_ANDES_ERR_CODE_FLUSH = 0x4e, + XSC_ANDES_ERR_CODE_MALF_WQE_HOST = 0x50, + XSC_ANDES_ERR_CODE_MALF_WQE_INFO = 0x51, + XSC_ANDES_ERR_CODE_MR_NON_NAK = 0x52, + XSC_ANDES_ERR_CODE_OPCODE_GEN_CQE = 0x61, + XSC_ANDES_ERR_CODE_MANY_READ = 0x62, + XSC_ANDES_ERR_CODE_LEN_GEN_CQE = 0x63, + XSC_ANDES_ERR_CODE_MR = 0x65, + XSC_ANDES_ERR_CODE_MR_GEN_CQE = 0x66, + XSC_ANDES_ERR_CODE_OPERATION = 0x67, + XSC_ANDES_ERR_CODE_MALF_WQE_INFO_GEN_NAK = 0x68, +}; + +enum { + XSC_DIAMOND_ERR_CODE_NAK_FLUSH = 0x1f, + XSC_DIAMOND_ERR_CODE_CEM = 0x28, + XSC_DIAMOND_ERR_CODE_QPM_CFG = 0x80, + XSC_DIAMOND_ERR_CODE_MET_CFG = 0x81, + XSC_DIAMOND_ERR_CODE_PPE_CFG = 0x82, + XSC_DIAMOND_ERR_CODE_PG_CFG = 0x83, + XSC_DIAMOND_ERR_CODE_SV_CFG = 0x84, + XSC_DIAMOND_ERR_CODE_ATOMIC_VA_REQ = 0x89, + XSC_DIAMOND_ERR_CODE_SND_WQE_LEN = 0x9f, + XSC_DIAMOND_ERR_CODE_NAK_SEQ_ERR = 0xa0, + XSC_DIAMOND_ERR_CODE_READ_DROP = 0xa1, + XSC_DIAMOND_ERR_CODE_RTO_REQ = 0xa2, + XSC_DIAMOND_ERR_CODE_NAK_RNR = 0xa3, + XSC_DIAMOND_ERR_CODE_NAK_INV_REQ = 0xa4, + XSC_DIAMOND_ERR_CODE_NAK_MR = 0xa5, + XSC_DIAMOND_ERR_CODE_NAK_REMOTE_OPER_ERR = 0xa6, + XSC_DIAMOND_ERR_CODE_LOCAL_MR_REQ = 0xa7, + XSC_DIAMOND_ERR_CODE_READ_RSP_LEN = 0xa8, + XSC_DIAMOND_ERR_CODE_READ_RSP_OPCODE = 0xaa, + XSC_DIAMOND_ERR_CODE_SND_WQE_FORMAT = 0xab, + XSC_DIAMOND_ERR_CODE_SND_WQE_DMA = 0xac, + XSC_DIAMOND_ERR_CODE_SND_WQE_STATUS = 0xad, + XSC_DIAMOND_ERR_CODE_SV_RTO_RSP = 0xae, + XSC_DIAMOND_ERR_CODE_RCV_WQE_DMA = 0xaf, + XSC_DIAMOND_ERR_CODE_DATA_DMA_RD_REQ = 0xb2, + XSC_DIAMOND_ERR_CODE_DATA_DMA_WR_REQ = 0xb3, + XSC_DIAMOND_ERR_CODE_DATA_DMA_WR_RSP_GEN_CQE = 0xb4, + XSC_DIAMOND_ERR_CODE_DATA_DMA_WR_RSP = 0xb5, + XSC_DIAMOND_ERR_CODE_DATA_DMA_RD_RSP = 0xb6, + XSC_DIAMOND_ERR_CODE_NO_CLASSIFIED = 0xb7, + XSC_DIAMOND_ERR_CODE_RCV_WQE_STATUS = 0xb9, + XSC_DIAMOND_ERR_CODE_QPM = 0xba, + XSC_DIAMOND_ERR_CODE_SV = 0xbb, + XSC_DIAMOND_ERR_CODE_MET = 0xbc, + XSC_DIAMOND_ERR_CODE_PPE = 0xbd, + XSC_DIAMOND_ERR_CODE_PG = 0xbe, + XSC_DIAMOND_ERR_CODE_REQ_MTT_DMA_RD = 0xbf, + XSC_DIAMOND_ERR_CODE_RCV_WQE_SIZE_RC = 0xc0, + XSC_DIAMOND_ERR_CODE_OPCODE_SEQ_GEN_CQE = 0xc1, + XSC_DIAMOND_ERR_CODE_OPCODE_SEQ = 0xc2, + XSC_DIAMOND_ERR_CODE_LEN_GEN_CQE = 0xc4, + XSC_DIAMOND_ERR_CODE_LEN = 0xc5, + XSC_DIAMOND_ERR_CODE_READ_RSP_ATOMIC_MSN_BDT = 0xc8, + XSC_DIAMOND_ERR_CODE_EMSN_GEN_CQE = 0xca, + XSC_DIAMOND_ERR_CODE_RQ_EMSN_GEN_CQE = 0xcb, + XSC_DIAMOND_ERR_CODE_READ_RSP_WQE_DMA_RD = 0xcc, + XSC_DIAMOND_ERR_CODE_RO_DMA_RD = 0xcd, + XSC_DIAMOND_ERR_CODE_RSP_MTT_DMA_RD = 0xce, + XSC_DIAMOND_ERR_CODE_ATOMIC_VA_REQ_BDT = 0xcf, + XSC_DIAMOND_ERR_CODE_ATOMIC_VA_RSP = 0xd0, + XSC_DIAMOND_ERR_CODE_RSP_MTT_DMA_RD_GEN_CQE = 0xd1, + XSC_DIAMOND_ERR_CODE_REMOTE_MR = 0xd4, + XSC_DIAMOND_ERR_CODE_REMOTE_MR_GEN_CQE = 0xd5, + XSC_DIAMOND_ERR_CODE_LOCAL_MR_RSP = 0xd6, + XSC_DIAMOND_ERR_CODE_ACC_TYPE_MR = 0xd7, + XSC_DIAMOND_ERR_CODE_ACC_TYPE_MR_GEN_CQE = 0xd8, + XSC_DIAMOND_ERR_CODE_LOCAL_MR_REQ_BDT = 0xd9, + XSC_DIAMOND_ERR_CODE_READ_RSP_LEN_BDT = 0xda, + XSC_DIAMOND_ERR_CODE_READ_RSP_OPCODE_BDT = 0xdb, + XSC_DIAMOND_ERR_CODE_REQ_MTT_DMA_RD_BDT = 0xdc, + XSC_DIAMOND_ERR_CODE_MSN_GEN_CQE = 0xdd, + XSC_DIAMOND_ERR_CODE_MSN = 0xde, + XSC_DIAMOND_ERR_CODE_FLUSH = 0xff, +}; + +/* TODO: sw cqe opcode*/ +enum { + XSC_OPCODE_RDMA_REQ_SEND = 0, + XSC_OPCODE_RDMA_REQ_SEND_IMMDT = 1, + XSC_OPCODE_RDMA_RSP_RECV = 2, + XSC_OPCODE_RDMA_RSP_RECV_IMMDT = 3, + XSC_OPCODE_RDMA_REQ_WRITE = 4, + XSC_OPCODE_RDMA_REQ_WRITE_IMMDT = 5, + XSC_OPCODE_RDMA_RSP_WRITE_IMMDT = 6, + XSC_OPCODE_RDMA_REQ_READ = 7, + XSC_OPCODE_RDMA_REQ_ERROR = 8, + XSC_OPCODE_RDMA_RSP_ERROR = 9, + XSC_OPCODE_RDMA_CQE_ERROR = 10, + XSC_OPCODE_RDMA_MAD_REQ_SEND = 11, + XSC_OPCODE_RDMA_MAD_RSP_RECV = 12, + XSC_OPCODE_RDMA_CQE_RAW_SNF = 13, + XSC_OPCODE_RDMA_REQ_ATOMIC_CMP_AND_SWAP = 14, + XSC_OPCODE_RDMA_REQ_ATOMIC_FETCH_AND_ADD = 15, + XSC_OPCODE_RDMA_REQ_ATOMIC_8B_MASK_CS = 16, + XSC_OPCODE_RDMA_REQ_ATOMIC_8B_MASK_FA = 17, + XSC_OPCODE_RDMA_REQ_ATOMIC_4B_MASK_CS = 18, + XSC_OPCODE_RDMA_REQ_ATOMIC_4B_MASK_FA = 19, +}; + +enum { + XSC_BASE_WQE_SHIFT = 4, +}; + +/* + * Descriptors that are allocated by SW and accessed by HW, 32-byte aligned + */ +/* this is to keep descriptor structures packed */ +struct regpair { + __le32 lo; + __le32 hi; +}; + +struct xsc_send_wqe_ctrl_seg { + uint8_t msg_opcode; + uint8_t data0; +#define XSC_SWQE_CTRL_SEG_WITH_IMMDT_MASK BIT(0) +#define XSC_SWQE_CTRL_SEG_CSUM_EN_MASK GENMASK(2, 1) +#define XSC_SWQE_CTRL_SEG_DS_DATA_NUM_MASK GENMASK(7, 3) + __le16 wqe_id; + __le32 msg_len; + __le32 opcode_data; + uint8_t data1; +#define XSC_SWQE_CTRL_SEG_SE_MASK BIT(0) +#define XSC_SWQE_CTRL_SEG_CE_MASK BIT(1) +#define XSC_SWQE_CTRL_SEG_IN_LINE_MASK BIT(2) +#define XSC_SWQE_CTRL_SEG_FENCE_MODE_MASK GENMASK(4, 3) +#define XSC_SWQE_CTRL_SEG_MASK_MASK GENMASK(6, 5) + uint8_t rsv[3]; +}; + + +struct xsc_wqe_data_seg { + union { + struct { + uint32_t data0; +#define XSC_WQE_DATA_SEG_LENGTH_MASK GENMASK(31, 1) + __le32 mkey; + __le64 va; + }; + struct { + uint8_t in_line_data[16]; + }; + }; +}; + +struct xsc_cqe { + uint8_t placeholder1; + __le32 data1; +#define XSC_CQE_QP_ID_MASK GENMASK(14, 0) +#define XSC_CQE_SE_MASK BIT(1) +#define XSC_CQE_HAS_PPH_MASK BIT(2) +#define XSC_CQE_TYPE_MASK BIT(3) +#define XSC_CQE_WITH_IMMDT_MASK BIT(4) +#define XSC_CQE_CSUM_ERR_MASK GENMASK(8, 5) + __le32 imm_data; + __le32 msg_len; + __le32 vni; + __le64 data2; +#define XSC_CQE_TS_MASK GENMASK_ULL(47, 0) + __le16 wqe_id; + uint8_t placeholder2; + uint8_t rsv1; + __le16 rsv2[2]; + __le16 data3; +#define XSC_CQE_OWNER_MASK BIT(15) +}; + +struct xsc_cqe64 { + struct xsc_cqe cqe; + uint8_t padding[32]; +}; + +/* Size of CQE */ +#define XSC_CQE_SIZE sizeof(struct xsc_cqe) +#define XSC_CQE_SIZE64 sizeof(struct xsc_cqe64) + +#define XSC_SEND_WQE_RING_DEPTH_MIN 16 +#define XSC_CQE_RING_DEPTH_MIN 2 + +/* + * Registers that are allocated by HW and accessed by SW in 4-byte granularity + */ +/* MMT table (32 bytes) */ +struct xsc_mmt_tbl { + struct regpair pa; + struct regpair va; + __le32 size; +#define XSC_MMT_TBL_PD_MASK 0x00FFFFFF +#define XSC_MMT_TBL_KEY_MASK 0xFF000000 + __le32 key_pd; +#define XSC_MMT_TBL_ACC_MASK 0x0000000F + __le32 acc; + uint8_t padding[4]; +}; + +/* QP Context (16 bytes) */ +struct xsc_qp_context { +#define XSC_QP_CONTEXT_STATE_MASK 0x00000007 +#define XSC_QP_CONTEXT_FUNC_MASK 0x00000018 +#define XSC_QP_CONTEXT_DSTID_MASK 0x000000E0 +#define XSC_QP_CONTEXT_PD_MASK 0xFFFFFF00 + __le32 pd_dstid_func_state; +#define XSC_QP_CONTEXT_DSTQP_MASK 0x00FFFFFF +#define XSC_QP_CONTEXT_RCQIDL_MASK 0xFF000000 + __le32 rcqidl_dstqp; +#define XSC_QP_CONTEXT_RCQIDH_MASK 0x0000FFFF +#define XSC_QP_CONTEXT_SCQIDL_MASK 0xFFFF0000 + __le32 scqidl_rcqidh; +#define XSC_QP_CONTEXT_SCQIDH_MASK 0x000000FF + __le32 scqidh; +}; + +static inline bool xsc_get_cqe_sw_own(struct xsc_cqe *cqe, + int cid, int ring_sz) ALWAYS_INLINE; + +static inline void xsc_set_cqe_sw_own(struct xsc_cqe *cqe, + int pid, int ring_sz) ALWAYS_INLINE; + +static inline bool xsc_get_cqe_sw_own(struct xsc_cqe *cqe, int cid, int ring_sz) +{ + return FIELD_GET(XSC_CQE_OWNER_MASK, cqe->data3) == ((cid >> ring_sz) & 1); +} + +static inline void xsc_set_cqe_sw_own(struct xsc_cqe *cqe, int pid, int ring_sz) +{ + cqe->data3 |= FIELD_PREP(XSC_CQE_OWNER_MASK, ((pid >> ring_sz) & 1)); +} +#endif /* __XSC_HSI_H__ */ diff --git a/providers/xscale/xsc_hw.h b/providers/xscale/xsc_hw.h new file mode 100644 index 000000000..9aea2dd2d --- /dev/null +++ b/providers/xscale/xsc_hw.h @@ -0,0 +1,767 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ +#ifndef _XSC_HW_H_ +#define _XSC_HW_H_ + +#include +#include +#include "xsc_hsi.h" + +struct andes_send_wqe_ctrl_seg { + uint8_t msg_opcode; + uint8_t data0; +#define XSC_ANDES_WQE_CTRL_SEG_WITH_IMMDT_MASK BIT(0) +#define XSC_ANDES_WQE_CTRL_SEG_CSUM_EN_MASK GENMASK(1, 2) +#define XSC_ANDES_WQE_CTRL_SEG_DS_DATA_NUM_MASK GENMASK(7, 3) + __le16 wqe_id; + __le32 msg_len; + __le32 opcode_data; + uint8_t data1; +#define XSC_ANDES_WQE_CTRL_SEG_SE_MASK BIT(0) +#define XSC_ANDES_WQE_CTRL_SEG_CE_MASK BIT(1) +#define XSC_ANDES_WQE_CTRL_SEG_IN_LINE_MASK BIT(2) +#define XSC_ANDES_WQE_CTRL_SEG_FENCE_MODE_MASK GENMASK(4, 3) +#define XSC_ANDES_WQE_CTRL_SEG_MASK_MASK GENMASK(6, 5) + uint8_t rsv[3]; +}; + +struct xsc_andes_cqe { + union { + uint8_t msg_opcode; + uint8_t data0; +#define XSC_ANDES_CQE_ERR_CODE_MASK GENMASK(6, 0) +#define XSC_ANDES_CQE_IS_ERR_MASK BIT(7) + }; + __le32 data1; +#define XSC_ANDES_CQE_QP_ID_MASK GENMASK(14, 0) +#define XSC_ANDES_CQE_SE_MASK BIT(1) +#define XSC_ANDES_CQE_HAS_PPH_MASK BIT(2) +#define XSC_ANDES_CQE_TYPE_MASK BIT(3) +#define XSC_ANDES_CQE_WITH_IMMDT_MASK BIT(4) +#define XSC_ANDES_CQE_CSUM_ERR_MASK GENMASK(8, 5) + __le32 imm_data; + __le32 msg_len; + __le32 vni; + __le64 data2; +#define XSC_ANDES_CQE_TS_MASK GENMASK_ULL(47, 0) + __le16 wqe_id; + __le16 rsv[3]; + __le16 data3; +#define XSC_ANDES_CQE_OWNER_MASK BIT(15) +}; + +struct xsc_andes_cq_doorbell { + uint32_t raw; +#define XSC_ANDES_CQ_DOORBELL_CQ_NEXT_CID_MASK GENMASK(15, 0) +#define XSC_ANDES_CQ_DOORBELL_CQ_ID_MASK GENMASK(30, 16) +#define XSC_ANDES_CQ_DOORBELL_ARM_MASK BIT(31) +}; + +struct xsc_andes_send_doorbell { + uint32_t raw; +#define XSC_ANDES_SEND_DOORBELL_NEXT_PID_MASK GENMASK(15, 0) +#define XSC_ANDES_SEND_DOORBELL_QP_ID_MASK GENMASK(30, 16) +}; + +struct xsc_andes_recv_doorbell { + uint32_t raw; +#define XSC_ANDES_RECV_DOORBELL_NEXT_PID_MASK GENMASK(12, 0) +#define XSC_ANDES_RECV_DOORBELL_QP_ID_MASK GENMASK(27, 13) +}; + +struct xsc_andes_data_seg { + uint32_t data0; +#define XSC_ANDES_DATA_SEG_LENGTH_MASK GENMASK(31, 1) + uint32_t key; + uint64_t addr; +}; + +struct diamond_send_wqe_ctrl_seg { + uint8_t msg_opcode; + uint8_t data0; +#define XSC_DIAMOND_WQE_CTRL_SEG_WITH_IMMDT_MASK BIT(0) +#define XSC_DIAMOND_WQE_CTRL_SEG_CSUM_EN_MASK GENMASK(1, 2) +#define XSC_DIAMOND_WQE_CTRL_SEG_DS_DATA_NUM_MASK GENMASK(7, 3) + __le16 rsv1; + __le32 msg_len; + __le32 opcode_data; + __le32 data1; +#define XSC_DIAMOND_WQE_CTRL_SEG_SE_MASK BIT(0) +#define XSC_DIAMOND_WQE_CTRL_SEG_CE_MASK BIT(1) +#define XSC_DIAMOND_WQE_CTRL_SEG_IN_LINE_MASK BIT(2) +#define XSC_DIAMOND_WQE_CTRL_SEG_FENCE_MODE_MASK GENMASK(4, 3) +#define XSC_DIAMOND_WQE_CTRL_SEG_MASK_MASK GENMASK(6, 5) +#define XSC_DIAMOND_WQE_CTRL_SEG_WQE_ID_MASK GENMASK(31, 12) +}; + +struct xsc_diamond_cqe { + uint8_t error_code; + __le32 data0; +#define XSC_DIAMOND_CQE_QP_ID_MASK GENMASK(14, 0) +#define XSC_DIAMOND_CQE_SE_MASK BIT(16) +#define XSC_DIAMOND_CQE_HAS_PPH_MASK BIT(17) +#define XSC_DIAMOND_CQE_TYPE_MASK BIT(18) +#define XSC_DIAMOND_CQE_WITH_IMMDT_MASK BIT(19) +#define XSC_DIAMOND_CQE_CSUM_ERR_MASK GENMASK(23, 20) + __le32 imm_data; + __le32 msg_len; + __le32 vni; + __le64 data1; +#define XSC_DIAMOND_CQE_TS_MASK GENMASK_ULL(47, 0) + __le32 data2; +#define XSC_DIAMOND_CQE_MSG_OPCODE_MASK GENMASK(7, 0) +#define XSC_DIAMOND_CQE_WQE_ID_MASK GENMASK(31, 12) + __le32 data3; +#define XSC_DIAMOND_CQE_OWNER_MASK BIT(31) +}; + +struct xsc_diamond_cq_doorbell { + uint64_t raw; +#define XSC_DIAMOND_CQ_DOORBELL_CQ_NEXT_CID_MASK GENMASK_ULL(22, 0) +#define XSC_DIAMOND_CQ_DOORBELL_CQ_ID_MASK GENMASK_ULL(38, 23) +#define XSC_DIAMOND_CQ_DOORBELL_CQ_STA_MASK GENMASK_ULL(40, 39) +}; + +struct xsc_diamond_recv_doorbell { + uint64_t raw; +#define XSC_DIAMOND_RECV_DOORBELL_NEXT_PID_MASK GENMASK_ULL(17, 0) +#define XSC_DIAMOND_RECV_DOORBELL_QP_ID_MASK GENMASK_ULL(33, 18) +}; + +struct xsc_diamond_send_doorbell { + uint64_t raw; +#define XSC_DIAMOND_SEND_DOORBELL_NEXT_PID_MASK GENMASK_ULL(20, 0) +#define XSC_DIAMOND_SEND_DOORBELL_QP_ID_MASK GENMASK_ULL(36, 21) +}; + +struct xsc_diamond_data_seg { + uint32_t length; + uint32_t key; + uint64_t addr; +}; + +struct xsc_diamond_atomic_seg { + uint64_t swap_add; + uint64_t compare; +}; + +struct xsc_diamond_next_cq_doorbell { + uint64_t raw; +#define XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_NEXT_CID_MASK GENMASK_ULL(22, 0) +#define XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_ID_MASK GENMASK_ULL(38, 23) +#define XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_STA_MASK GENMASK_ULL(40, 39) +}; + +struct xsc_diamond_next_send_doorbell { + uint64_t raw; +#define XSC_DIAMOND_NEXT_SEND_DOORBELL_NEXT_PID_MASK GENMASK_ULL(16, 0) +#define XSC_DIAMOND_NEXT_SEND_DOORBELL_QP_ID_MASK GENMASK_ULL(32, 17) +}; + +struct xsc_diamond_next_recv_doorbell { + uint64_t raw; +#define XSC_DIAMOND_NEXT_RECV_DOORBELL_NEXT_PID_MASK GENMASK_ULL(13, 0) +#define XSC_DIAMOND_NEXT_RECV_DOORBELL_QP_ID_MASK GENMASK_ULL(29, 14) +}; + +enum { + XSC_CQ_STAT_FIRED, + XSC_CQ_STAT_KEEP, + XSC_CQ_STAT_ARM_NEXT, + XSC_CQ_STAT_ARM_SOLICITED, +}; + +static inline uint8_t xsc_diamond_get_cqe_msg_opcode(void *cqe) +{ + return FIELD_GET(XSC_DIAMOND_CQE_MSG_OPCODE_MASK, + ((struct xsc_diamond_cqe *)cqe)->data2); +} + +static inline uint8_t xsc_andes_get_cqe_msg_opcode(void *cqe) +{ + return ((struct xsc_andes_cqe *)cqe)->msg_opcode; +} + +static inline uint8_t xsc_hw_get_cqe_msg_opcode(uint16_t device_id, void *cqe) +{ + switch (device_id) { + case XSC_MS_PF_DEV_ID: + case XSC_MS_VF_DEV_ID: + return xsc_andes_get_cqe_msg_opcode(cqe); + case XSC_MC_PF_DEV_ID_DIAMOND: + case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: + return xsc_diamond_get_cqe_msg_opcode(cqe); + default: + return xsc_andes_get_cqe_msg_opcode(cqe); + } +} + +static inline uint32_t xsc_hw_get_wqe_id(uint16_t device_id, void *cqe) +{ + switch (device_id) { + case XSC_MS_PF_DEV_ID: + case XSC_MS_VF_DEV_ID: + return RD_LE_16(((struct xsc_andes_cqe *)cqe)->wqe_id); + case XSC_MC_PF_DEV_ID_DIAMOND: + case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: + return FIELD_GET(XSC_DIAMOND_CQE_WQE_ID_MASK, + ((struct xsc_diamond_cqe *)cqe)->data2); + default: + return RD_LE_16(((struct xsc_andes_cqe *)cqe)->wqe_id); + } +} + +static inline void xsc_hw_set_wqe_id(uint16_t device_id, void *cseg, uint32_t wqe_id) +{ + uint16_t wqe_id16 = wqe_id; + + switch (device_id) { + case XSC_MS_PF_DEV_ID: + case XSC_MS_VF_DEV_ID: + WR_LE_16(((struct andes_send_wqe_ctrl_seg *)cseg)->wqe_id, wqe_id16); + break; + case XSC_MC_PF_DEV_ID_DIAMOND: + case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: + ((struct diamond_send_wqe_ctrl_seg *)cseg)->data1 |= + FIELD_PREP(XSC_DIAMOND_WQE_CTRL_SEG_WQE_ID_MASK, wqe_id); + break; + default: + WR_LE_16(((struct andes_send_wqe_ctrl_seg *)cseg)->wqe_id, wqe_id16); + break; + } +} + +static inline bool xsc_diamond_is_err_cqe(void *cqe) +{ + return !!((struct xsc_diamond_cqe *)cqe)->error_code; +} + +static inline bool xsc_andes_is_err_cqe(void *cqe) +{ + return FIELD_GET(XSC_ANDES_CQE_IS_ERR_MASK, + ((struct xsc_andes_cqe *)cqe)->data0); +} + +static inline bool xsc_hw_is_err_cqe(uint16_t device_id, void *cqe) +{ + switch (device_id) { + case XSC_MS_PF_DEV_ID: + case XSC_MS_VF_DEV_ID: + return xsc_andes_is_err_cqe(cqe); + case XSC_MC_PF_DEV_ID_DIAMOND: + case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: + return xsc_diamond_is_err_cqe(cqe); + default: + return xsc_andes_is_err_cqe(cqe); + } +} + +static inline uint8_t xsc_diamond_get_cqe_err_code(void *cqe) +{ + return ((struct xsc_diamond_cqe *)cqe)->error_code; +} + +static inline uint8_t xsc_andes_get_cqe_err_code(void *cqe) +{ + return FIELD_GET(XSC_ANDES_CQE_ERR_CODE_MASK, + ((struct xsc_andes_cqe *)cqe)->data0); +} + +static inline uint8_t xsc_hw_get_cqe_err_code(uint16_t device_id, void *cqe) +{ + switch (device_id) { + case XSC_MS_PF_DEV_ID: + case XSC_MS_VF_DEV_ID: + return xsc_andes_get_cqe_err_code(cqe); + case XSC_MC_PF_DEV_ID_DIAMOND: + case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: + return xsc_diamond_get_cqe_err_code(cqe); + default: + return xsc_andes_get_cqe_err_code(cqe); + } +} + +static inline enum ibv_wc_status xsc_andes_cqe_err_code(uint8_t error_code) +{ + switch (error_code) { + case XSC_ANDES_ERR_CODE_NAK_RETRY: + return IBV_WC_RETRY_EXC_ERR; + case XSC_ANDES_ERR_CODE_NAK_OPCODE: + return IBV_WC_REM_INV_REQ_ERR; + case XSC_ANDES_ERR_CODE_NAK_MR: + return IBV_WC_REM_ACCESS_ERR; + case XSC_ANDES_ERR_CODE_NAK_OPERATION: + return IBV_WC_REM_OP_ERR; + case XSC_ANDES_ERR_CODE_NAK_RNR: + return IBV_WC_RNR_RETRY_EXC_ERR; + case XSC_ANDES_ERR_CODE_LOCAL_MR: + return IBV_WC_LOC_PROT_ERR; + case XSC_ANDES_ERR_CODE_LOCAL_LEN: + return IBV_WC_LOC_LEN_ERR; + case XSC_ANDES_ERR_CODE_LEN_GEN_CQE: + return IBV_WC_LOC_LEN_ERR; + case XSC_ANDES_ERR_CODE_OPERATION: + return IBV_WC_LOC_ACCESS_ERR; + case XSC_ANDES_ERR_CODE_FLUSH: + return IBV_WC_WR_FLUSH_ERR; + case XSC_ANDES_ERR_CODE_MALF_WQE_HOST: + case XSC_ANDES_ERR_CODE_STRG_ACC_GEN_CQE: + case XSC_ANDES_ERR_CODE_STRG_ACC: + return IBV_WC_FATAL_ERR; + case XSC_ANDES_ERR_CODE_MR_GEN_CQE: + return IBV_WC_LOC_PROT_ERR; + case XSC_ANDES_ERR_CODE_LOCAL_OPERATION_WQE: + return IBV_WC_LOC_QP_OP_ERR; + case XSC_ANDES_ERR_CODE_OPCODE_GEN_CQE: + case XSC_ANDES_ERR_CODE_LOCAL_OPCODE: + default: + return IBV_WC_GENERAL_ERR; + } +} + +static inline enum ibv_wc_status xsc_diamond_cqe_err_code(uint8_t error_code) +{ + switch (error_code) { + case XSC_DIAMOND_ERR_CODE_READ_DROP: + case XSC_DIAMOND_ERR_CODE_NAK_SEQ_ERR: + case XSC_DIAMOND_ERR_CODE_RTO_REQ: + case XSC_DIAMOND_ERR_CODE_SV_RTO_RSP: + return IBV_WC_RETRY_EXC_ERR; + case XSC_DIAMOND_ERR_CODE_NAK_INV_REQ: + case XSC_DIAMOND_ERR_CODE_OPCODE_SEQ_GEN_CQE: + case XSC_DIAMOND_ERR_CODE_OPCODE_SEQ: + case XSC_DIAMOND_ERR_CODE_ATOMIC_VA_REQ: + case XSC_DIAMOND_ERR_CODE_ATOMIC_VA_REQ_BDT: + case XSC_DIAMOND_ERR_CODE_ACC_TYPE_MR: + case XSC_DIAMOND_ERR_CODE_ACC_TYPE_MR_GEN_CQE: + return IBV_WC_REM_INV_REQ_ERR; + case XSC_DIAMOND_ERR_CODE_NAK_MR: + return IBV_WC_REM_ACCESS_ERR; + case XSC_DIAMOND_ERR_CODE_NAK_REMOTE_OPER_ERR: + return IBV_WC_REM_OP_ERR; + case XSC_DIAMOND_ERR_CODE_LOCAL_MR_REQ: + case XSC_DIAMOND_ERR_CODE_LOCAL_MR_REQ_BDT: + case XSC_DIAMOND_ERR_CODE_LOCAL_MR_RSP: + case XSC_DIAMOND_ERR_CODE_ATOMIC_VA_RSP: + return IBV_WC_LOC_PROT_ERR; + case XSC_DIAMOND_ERR_CODE_READ_RSP_LEN: + case XSC_DIAMOND_ERR_CODE_READ_RSP_LEN_BDT: + case XSC_DIAMOND_ERR_CODE_SND_WQE_LEN: + case XSC_DIAMOND_ERR_CODE_LEN: + case XSC_DIAMOND_ERR_CODE_LEN_GEN_CQE: + case XSC_DIAMOND_ERR_CODE_RCV_WQE_SIZE_RC: + return IBV_WC_LOC_LEN_ERR; + case XSC_DIAMOND_ERR_CODE_READ_RSP_OPCODE: + case XSC_DIAMOND_ERR_CODE_READ_RSP_OPCODE_BDT: + return IBV_WC_BAD_RESP_ERR; + case XSC_DIAMOND_ERR_CODE_NAK_FLUSH: + case XSC_DIAMOND_ERR_CODE_FLUSH: + return IBV_WC_WR_FLUSH_ERR; + case XSC_DIAMOND_ERR_CODE_QPM_CFG: + case XSC_DIAMOND_ERR_CODE_MET_CFG: + case XSC_DIAMOND_ERR_CODE_PPE_CFG: + case XSC_DIAMOND_ERR_CODE_PG_CFG: + case XSC_DIAMOND_ERR_CODE_SV_CFG: + case XSC_DIAMOND_ERR_CODE_QPM: + case XSC_DIAMOND_ERR_CODE_SV: + case XSC_DIAMOND_ERR_CODE_MET: + case XSC_DIAMOND_ERR_CODE_PPE: + case XSC_DIAMOND_ERR_CODE_PG: + case XSC_DIAMOND_ERR_CODE_CEM: + case XSC_DIAMOND_ERR_CODE_RCV_WQE_DMA: + case XSC_DIAMOND_ERR_CODE_DATA_DMA_RD_REQ: + case XSC_DIAMOND_ERR_CODE_DATA_DMA_WR_REQ: + case XSC_DIAMOND_ERR_CODE_DATA_DMA_WR_RSP_GEN_CQE: + case XSC_DIAMOND_ERR_CODE_DATA_DMA_WR_RSP: + case XSC_DIAMOND_ERR_CODE_DATA_DMA_RD_RSP: + case XSC_DIAMOND_ERR_CODE_READ_RSP_WQE_DMA_RD: + case XSC_DIAMOND_ERR_CODE_RO_DMA_RD: + case XSC_DIAMOND_ERR_CODE_REQ_MTT_DMA_RD: + case XSC_DIAMOND_ERR_CODE_REQ_MTT_DMA_RD_BDT: + case XSC_DIAMOND_ERR_CODE_RSP_MTT_DMA_RD: + case XSC_DIAMOND_ERR_CODE_RSP_MTT_DMA_RD_GEN_CQE: + return IBV_WC_FATAL_ERR; + case XSC_DIAMOND_ERR_CODE_SND_WQE_FORMAT: + case XSC_DIAMOND_ERR_CODE_SND_WQE_DMA: + case XSC_DIAMOND_ERR_CODE_SND_WQE_STATUS: + case XSC_DIAMOND_ERR_CODE_RCV_WQE_STATUS: + return IBV_WC_LOC_QP_OP_ERR; + case XSC_DIAMOND_ERR_CODE_REMOTE_MR: + case XSC_DIAMOND_ERR_CODE_REMOTE_MR_GEN_CQE: + return IBV_WC_LOC_ACCESS_ERR; + case XSC_DIAMOND_ERR_CODE_NAK_RNR: + return IBV_WC_RNR_RETRY_EXC_ERR; + default: + return IBV_WC_GENERAL_ERR; + } +} + +static inline enum ibv_wc_status xsc_hw_cqe_err_status(uint16_t device_id, + void *cqe) +{ + switch (device_id) { + case XSC_MS_PF_DEV_ID: + case XSC_MS_VF_DEV_ID: + return xsc_andes_cqe_err_code(xsc_andes_get_cqe_err_code(cqe)); + case XSC_MC_PF_DEV_ID_DIAMOND: + case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: + return xsc_diamond_cqe_err_code(xsc_diamond_get_cqe_err_code(cqe)); + default: + return xsc_andes_cqe_err_code(xsc_andes_get_cqe_err_code(cqe)); + } +} + +static inline void xsc_diamond_set_data_seg(void *data_seg, + uint64_t addr, uint32_t key, + uint32_t length) +{ + struct xsc_diamond_data_seg *seg = data_seg; + + seg->length = length; + seg->key = key; + seg->addr = addr; +} + +static inline void xsc_diamond_set_atomic_seg(void *atomic_seg, int opcode, + uint64_t swap, uint64_t compare_add) +{ + struct xsc_diamond_atomic_seg *aseg = (struct xsc_diamond_atomic_seg *)atomic_seg; + + if (opcode == IBV_WR_ATOMIC_CMP_AND_SWP) { + aseg->swap_add = RD_BE_64(swap); + aseg->compare = RD_BE_64(compare_add); + } else { + aseg->swap_add = RD_BE_64(compare_add); + } +} + +static inline void xsc_andes_set_data_seg(void *data_seg, + uint64_t addr, uint32_t key, + uint32_t length) +{ + struct xsc_andes_data_seg *seg = data_seg; + + seg->data0 |= FIELD_PREP(XSC_ANDES_DATA_SEG_LENGTH_MASK, length); + seg->key = key; + seg->addr = addr; +} + +static inline void xsc_hw_set_data_seg(uint16_t device_id, void *data_seg, + uint64_t addr, uint32_t key, uint32_t length) +{ + switch (device_id) { + case XSC_MS_PF_DEV_ID: + case XSC_MS_VF_DEV_ID: + xsc_andes_set_data_seg(data_seg, addr, key, length); + break; + case XSC_MC_PF_DEV_ID_DIAMOND: + case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: + xsc_diamond_set_data_seg(data_seg, addr, key, length); + break; + default: + xsc_andes_set_data_seg(data_seg, addr, key, length); + } +} + +static inline void xsc_hw_set_atomic_seg(uint16_t device_id, void *atomic_seg, + int opcode, uint64_t swap, uint64_t compare_add) +{ + switch (device_id) { + case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: + xsc_diamond_set_atomic_seg(atomic_seg, opcode, swap, compare_add); + break; + default: + xsc_diamond_set_atomic_seg(atomic_seg, opcode, swap, compare_add); + break; + } +} + +static inline void xsc_diamond_set_cq_ci(void *db_addr, + uint32_t cqn, uint32_t next_cid) +{ + struct xsc_diamond_cq_doorbell db; + + db.raw = FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_ID_MASK, cqn) | + FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | + FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_STA_MASK, XSC_CQ_STAT_KEEP); + udma_to_device_barrier(); + mmio_write64_le(db_addr, db.raw); +} + +static inline void xsc_diamond_next_set_cq_ci(void *db_addr, + uint32_t cqn, uint32_t next_cid) +{ + struct xsc_diamond_next_cq_doorbell db; + + db.raw = FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_ID_MASK, cqn) | + FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | + FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_STA_MASK, XSC_CQ_STAT_KEEP); + udma_to_device_barrier(); + mmio_write64_le(db_addr, db.raw); +} + +static inline void xsc_andes_set_cq_ci(void *db_addr, + uint32_t cqn, uint32_t next_cid) +{ + struct xsc_andes_cq_doorbell db; + + db.raw = FIELD_PREP(XSC_ANDES_CQ_DOORBELL_CQ_ID_MASK, cqn) | + FIELD_PREP(XSC_ANDES_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | + FIELD_PREP(XSC_ANDES_CQ_DOORBELL_ARM_MASK, XSC_CQ_STAT_FIRED); + udma_to_device_barrier(); + mmio_write32_le(db_addr, db.raw); +} + + +static inline void xsc_hw_set_cq_ci(uint16_t device_id, void *db_addr, + uint32_t cqn, uint32_t next_cid) +{ + switch (device_id) { + case XSC_MS_PF_DEV_ID: + case XSC_MS_VF_DEV_ID: + xsc_andes_set_cq_ci(db_addr, cqn, next_cid); + break; + case XSC_MC_PF_DEV_ID_DIAMOND: + xsc_diamond_set_cq_ci(db_addr, cqn, next_cid); + break; + case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: + xsc_diamond_next_set_cq_ci(db_addr, cqn, next_cid); + break; + default: + xsc_andes_set_cq_ci(db_addr, cqn, next_cid); + } +} + +static inline void xsc_diamond_update_cq_db(void *db_addr, + uint32_t cqn, uint32_t next_cid, + uint8_t solicited) +{ + struct xsc_diamond_cq_doorbell db; + + db.raw = FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_ID_MASK, cqn) | + FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | + FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_STA_MASK, + solicited ? XSC_CQ_STAT_ARM_SOLICITED : XSC_CQ_STAT_ARM_NEXT); + udma_to_device_barrier(); + mmio_wc_start(); + mmio_write64_le(db_addr, db.raw); + mmio_flush_writes(); +} + +static inline void xsc_diamond_next_update_cq_db(void *db_addr, + uint32_t cqn, uint32_t next_cid, + uint8_t solicited) +{ + struct xsc_diamond_next_cq_doorbell db; + + db.raw = FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_ID_MASK, cqn) | + FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | + FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_STA_MASK, + solicited ? XSC_CQ_STAT_ARM_SOLICITED : XSC_CQ_STAT_ARM_NEXT); + udma_to_device_barrier(); + mmio_wc_start(); + mmio_write64_le(db_addr, db.raw); + mmio_flush_writes(); +} + +static inline void xsc_andes_update_cq_db(void *db_addr, + uint32_t cqn, uint32_t next_cid, + uint8_t solicited) +{ + struct xsc_andes_cq_doorbell db; + + db.raw = FIELD_PREP(XSC_ANDES_CQ_DOORBELL_CQ_ID_MASK, cqn) | + FIELD_PREP(XSC_ANDES_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | + FIELD_PREP(XSC_ANDES_CQ_DOORBELL_ARM_MASK, + solicited ? XSC_CQ_STAT_ARM_SOLICITED : XSC_CQ_STAT_ARM_NEXT); + udma_to_device_barrier(); + mmio_wc_start(); + mmio_write32_le(db_addr, db.raw); + mmio_flush_writes(); +} + +static inline void xsc_hw_update_cq_db(uint16_t device_id, void *db_addr, + uint32_t cqn, uint32_t next_cid, + uint8_t solicited) +{ + switch (device_id) { + case XSC_MS_PF_DEV_ID: + case XSC_MS_VF_DEV_ID: + xsc_andes_update_cq_db(db_addr, cqn, next_cid, solicited); + break; + case XSC_MC_PF_DEV_ID_DIAMOND: + xsc_diamond_update_cq_db(db_addr, cqn, next_cid, solicited); + break; + case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: + xsc_diamond_next_update_cq_db(db_addr, cqn, next_cid, solicited); + break; + default: + xsc_andes_update_cq_db(db_addr, cqn, next_cid, solicited); + } +} + +static inline void xsc_diamond_ring_rx_doorbell(void *db_addr, + uint32_t rqn, uint32_t next_pid) +{ + struct xsc_diamond_recv_doorbell db; + + db.raw = FIELD_PREP(XSC_DIAMOND_RECV_DOORBELL_QP_ID_MASK, rqn) | + FIELD_PREP(XSC_DIAMOND_RECV_DOORBELL_NEXT_PID_MASK, next_pid); + udma_to_device_barrier(); + mmio_write64_le(db_addr, db.raw); +} + + +static inline void xsc_diamond_next_ring_rx_doorbell(void *db_addr, + uint32_t rqn, uint32_t next_pid) +{ + struct xsc_diamond_next_recv_doorbell db; + + db.raw = FIELD_PREP(XSC_DIAMOND_NEXT_RECV_DOORBELL_QP_ID_MASK, rqn) | + FIELD_PREP(XSC_DIAMOND_NEXT_RECV_DOORBELL_NEXT_PID_MASK, next_pid); + udma_to_device_barrier(); + mmio_write64_le(db_addr, db.raw); +} + +static inline void xsc_andes_ring_rx_doorbell(void *db_addr, + uint32_t rqn, uint32_t next_pid) +{ + struct xsc_andes_recv_doorbell db; + + db.raw = FIELD_PREP(XSC_ANDES_RECV_DOORBELL_QP_ID_MASK, rqn) | + FIELD_PREP(XSC_ANDES_RECV_DOORBELL_NEXT_PID_MASK, next_pid); + udma_to_device_barrier(); + mmio_write32_le(db_addr, db.raw); +} + +static inline void xsc_hw_ring_rx_doorbell(uint16_t device_id, + void *db_addr, + uint32_t rqn, uint32_t next_pid) +{ + switch (device_id) { + case XSC_MS_PF_DEV_ID: + case XSC_MS_VF_DEV_ID: + xsc_andes_ring_rx_doorbell(db_addr, rqn, next_pid); + break; + case XSC_MC_PF_DEV_ID_DIAMOND: + xsc_diamond_ring_rx_doorbell(db_addr, rqn, next_pid); + break; + case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: + xsc_diamond_next_ring_rx_doorbell(db_addr, rqn, next_pid); + break; + default: + xsc_andes_ring_rx_doorbell(db_addr, rqn, next_pid); + } +} + +static inline void xsc_diamond_ring_tx_doorbell(void *db_addr, + uint32_t rqn, uint32_t next_pid) +{ + struct xsc_diamond_send_doorbell db; + + db.raw = FIELD_PREP(XSC_DIAMOND_SEND_DOORBELL_QP_ID_MASK, rqn) | + FIELD_PREP(XSC_DIAMOND_SEND_DOORBELL_NEXT_PID_MASK, next_pid); + + udma_to_device_barrier(); + mmio_write64_le(db_addr, db.raw); +} + + +static inline void xsc_diamond_next_ring_tx_doorbell(void *db_addr, + uint32_t rqn, uint32_t next_pid) +{ + struct xsc_diamond_next_send_doorbell db; + + db.raw = FIELD_PREP(XSC_DIAMOND_NEXT_SEND_DOORBELL_QP_ID_MASK, rqn) | + FIELD_PREP(XSC_DIAMOND_NEXT_SEND_DOORBELL_NEXT_PID_MASK, next_pid); + + udma_to_device_barrier(); + mmio_write64_le(db_addr, db.raw); +} + +static inline void xsc_andes_ring_tx_doorbell(void *db_addr, + uint32_t rqn, uint32_t next_pid) +{ + struct xsc_andes_send_doorbell db; + + db.raw = FIELD_PREP(XSC_ANDES_SEND_DOORBELL_QP_ID_MASK, rqn) | + FIELD_PREP(XSC_ANDES_SEND_DOORBELL_NEXT_PID_MASK, next_pid); + + udma_to_device_barrier(); + mmio_write32_le(db_addr, db.raw); +} + +static inline void xsc_hw_ring_tx_doorbell(uint16_t device_id, + void *db_addr, + uint32_t sqn, uint32_t next_pid) +{ + switch (device_id) { + case XSC_MS_PF_DEV_ID: + case XSC_MS_VF_DEV_ID: + xsc_andes_ring_tx_doorbell(db_addr, sqn, next_pid); + break; + case XSC_MC_PF_DEV_ID_DIAMOND: + xsc_diamond_ring_tx_doorbell(db_addr, sqn, next_pid); + break; + case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: + xsc_diamond_next_ring_tx_doorbell(db_addr, sqn, next_pid); + break; + default: + xsc_andes_ring_tx_doorbell(db_addr, sqn, next_pid); + } +} + +static inline int xsc_get_recv_ds_shift(struct xsc_context *ctx, int qp_type) +{ + uint16_t device_id = ctx->device_id; + + switch (device_id) { + case XSC_MC_PF_DEV_ID_DIAMOND: + case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: + return (qp_type == IBV_QPT_RAW_PACKET) ? 2 : ctx->recv_ds_shift; + default: + return ctx->recv_ds_shift; + } +} + +static inline int xsc_check_cqe_msg_opcode(struct xsc_context *ctx, int msg_opcode) +{ + uint16_t device_id = ctx->device_id; + + switch (device_id) { + case XSC_MC_PF_DEV_ID_DIAMOND: + case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: + return (msg_opcode > XSC_MSG_OPCODE_SEND_DIAMOND_NEXT && + msg_opcode != XSC_MSG_OPCODE_RDMA_ATOMIC_CMP_AND_SWAP && + msg_opcode != XSC_MSG_OPCODE_RDMA_ATOMIC_FETCH_AND_ADD && + msg_opcode != XSC_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_CMP_AND_SWAP && + msg_opcode != XSC_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_FETCH_AND_ADD && + msg_opcode != XSC_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_CMP_AND_SWAP && + msg_opcode != XSC_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_FETCH_AND_ADD); + default: + return (msg_opcode > XSC_MSG_OPCODE_RDMA_READ && + msg_opcode != XSC_MSG_OPCODE_RDMA_ATOMIC_CMP_AND_SWAP && + msg_opcode != XSC_MSG_OPCODE_RDMA_ATOMIC_FETCH_AND_ADD && + msg_opcode != XSC_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_CMP_AND_SWAP && + msg_opcode != XSC_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_FETCH_AND_ADD && + msg_opcode != XSC_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_CMP_AND_SWAP && + msg_opcode != XSC_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_FETCH_AND_ADD); + } +} + +static inline bool xsc_need_modify_qp_to_err(uint16_t device_id, void *cqe) +{ + switch (device_id) { + case XSC_MC_PF_DEV_ID_DIAMOND: + case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: + return xsc_diamond_get_cqe_err_code(cqe) == XSC_DIAMOND_ERR_CODE_NAK_FLUSH ? + false : true; + default: + return true; + } +} + +#endif /* _XSC_HW_H_ */ diff --git a/providers/xscale/xscale.c b/providers/xscale/xscale.c index bd263a5d2..d277a38e7 100644 --- a/providers/xscale/xscale.c +++ b/providers/xscale/xscale.c @@ -22,6 +22,7 @@ #include "xscale.h" #include "xsc-abi.h" +#include "xsc_hw.h" static const struct verbs_match_ent hca_table[] = { VERBS_MODALIAS_MATCH("*xscale*", NULL), diff --git a/providers/xscale/xscale.h b/providers/xscale/xscale.h index 04c3390ef..34b989afd 100644 --- a/providers/xscale/xscale.h +++ b/providers/xscale/xscale.h @@ -79,6 +79,26 @@ struct xsc_device { int page_size; }; +#define XSC_PCI_VENDOR_ID 0x1f67 +#define VEROCE_VENDOR_ID 0x1e93 +#define XSC_VEROCE_INIT_PROFILE 0x10 + +#define XSC_MC_PF_DEV_ID 0x1011 +#define XSC_MC_VF_DEV_ID 0x1012 +#define XSC_MC_PF_DEV_ID_DIAMOND 0x1021 +#define XSC_MC_PF_DEV_ID_DIAMOND_NEXT 0x1023 + +#define XSC_MF_HOST_PF_DEV_ID 0x1051 +#define XSC_MF_HOST_VF_DEV_ID 0x1052 +#define XSC_MF_SOC_PF_DEV_ID 0x1053 + +#define XSC_MS_PF_DEV_ID 0x1111 +#define XSC_MS_VF_DEV_ID 0x1112 + +#define XSC_MV_HOST_PF_DEV_ID 0x1151 +#define XSC_MV_HOST_VF_DEV_ID 0x1152 +#define XSC_MV_SOC_PF_DEV_ID 0x1153 + #define NAME_BUFFER_SIZE 64 struct xsc_context { @@ -112,6 +132,7 @@ struct xsc_context { uint32_t send_ds_shift; uint32_t recv_ds_shift; FILE *dbg_fp; + uint16_t device_id; uint32_t multidb_num; uint32_t tx_multidb_base; void *mdb_base; From b0e378715ad88bebfc9ec05d0ba5a5f74626dd9b Mon Sep 17 00:00:00 2001 From: Tian Xin Date: Tue, 24 Sep 2024 15:06:40 +0800 Subject: [PATCH 03/13] libxscale: Add protection domain (PD) and memory region (MR) support Implement the basic memory management verbs required for RDMA operations: - alloc_pd / dealloc_pd: Manage protection domains, with reference counting to prevent premature deallocation. - reg_mr / dereg_mr: Register and deregister memory regions, supporting standard access flags. The implementation uses the common ibv_cmd_* wrappers and stores local keys (lkey/rkey) for later use. These verbs are prerequisites for creating QPs and posting work requests. Signed-off-by: Tian Xin Signed-off-by: Wei Honggang Signed-off-by: Zhao Qianwei Signed-off-by: Li Qiang Signed-off-by: Yan Lei --- kernel-headers/rdma/xsc-abi.h | 4 ++ providers/xscale/verbs.c | 85 +++++++++++++++++++++++++++++++++++ providers/xscale/xsc-abi.h | 2 + providers/xscale/xscale.c | 4 ++ providers/xscale/xscale.h | 29 ++++++++++++ 5 files changed, 124 insertions(+) diff --git a/kernel-headers/rdma/xsc-abi.h b/kernel-headers/rdma/xsc-abi.h index f79940716..36563f165 100644 --- a/kernel-headers/rdma/xsc-abi.h +++ b/kernel-headers/rdma/xsc-abi.h @@ -36,4 +36,8 @@ struct xsc_ib_alloc_ucontext_resp { __u32 resv; }; +struct xsc_ib_alloc_pd_resp { + __u32 pdn; +}; + #endif /* XSC_ABI_USER_H */ diff --git a/providers/xscale/verbs.c b/providers/xscale/verbs.c index ab5d7b584..27af46d7f 100644 --- a/providers/xscale/verbs.c +++ b/providers/xscale/verbs.c @@ -36,6 +36,91 @@ int xsc_query_port(struct ibv_context *context, uint8_t port, return ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd)); } +struct ibv_pd *xsc_alloc_pd(struct ibv_context *context) +{ + struct ibv_alloc_pd cmd; + struct xsc_alloc_pd_resp resp; + struct xsc_pd *pd; + + pd = calloc(1, sizeof(*pd)); + if (!pd) + return NULL; + + if (ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof(cmd), + &resp.ibv_resp, sizeof(resp))) { + free(pd); + return NULL; + } + + atomic_init(&pd->refcount, 1); + pd->pdn = resp.pdn; + xsc_dbg(to_xctx(context)->dbg_fp, XSC_DBG_PD, "pd number:%u\n", pd->pdn); + + return &pd->ibv_pd; +} + +int xsc_free_pd(struct ibv_pd *pd) +{ + int ret; + struct xsc_pd *xpd = to_xpd(pd); + + if (atomic_load(&xpd->refcount) > 1) + return EBUSY; + + ret = ibv_cmd_dealloc_pd(pd); + if (ret) + return ret; + + xsc_dbg(to_xctx(pd->context)->dbg_fp, XSC_DBG_PD, "dealloc pd\n"); + free(xpd); + + return 0; +} + +struct ibv_mr *xsc_reg_mr(struct ibv_pd *pd, void *addr, size_t length, + uint64_t hca_va, int acc) +{ + struct xsc_mr *mr; + struct ibv_reg_mr cmd; + int ret; + enum ibv_access_flags access = (enum ibv_access_flags)acc; + struct ib_uverbs_reg_mr_resp resp; + + mr = calloc(1, sizeof(*mr)); + if (!mr) + return NULL; + + ret = ibv_cmd_reg_mr(pd, addr, length, hca_va, access, + &mr->vmr, &cmd, sizeof(cmd), &resp, + sizeof(resp)); + if (ret) { + free(mr); + return NULL; + } + mr->alloc_flags = acc; + + xsc_dbg(to_xctx(pd->context)->dbg_fp, XSC_DBG_MR, "lkey:%u, rkey:%u\n", + mr->vmr.ibv_mr.lkey, mr->vmr.ibv_mr.rkey); + + return &mr->vmr.ibv_mr; +} + +int xsc_dereg_mr(struct verbs_mr *vmr) +{ + int ret; + + if (vmr->mr_type == IBV_MR_TYPE_NULL_MR) + goto free; + + ret = ibv_cmd_dereg_mr(vmr); + if (ret) + return ret; + +free: + free(vmr); + return 0; +} + static void xsc_set_fw_version(struct ibv_device_attr *attr, union xsc_ib_fw_ver *fw_ver) { diff --git a/providers/xscale/xsc-abi.h b/providers/xscale/xsc-abi.h index f7d48c6e6..b6d03979d 100644 --- a/providers/xscale/xsc-abi.h +++ b/providers/xscale/xsc-abi.h @@ -17,5 +17,7 @@ DECLARE_DRV_CMD(xsc_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT, empty, xsc_ib_alloc_ucontext_resp); +DECLARE_DRV_CMD(xsc_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, + empty, xsc_ib_alloc_pd_resp); #endif /* XSC_ABI_H */ diff --git a/providers/xscale/xscale.c b/providers/xscale/xscale.c index d277a38e7..db4a76504 100644 --- a/providers/xscale/xscale.c +++ b/providers/xscale/xscale.c @@ -34,6 +34,10 @@ static void xsc_free_context(struct ibv_context *ibctx); static const struct verbs_context_ops xsc_ctx_common_ops = { .query_port = xsc_query_port, + .alloc_pd = xsc_alloc_pd, + .dealloc_pd = xsc_free_pd, + .reg_mr = xsc_reg_mr, + .dereg_mr = xsc_dereg_mr, .query_device_ex = xsc_query_device_ex, .free_context = xsc_free_context, }; diff --git a/providers/xscale/xscale.h b/providers/xscale/xscale.h index 34b989afd..fed24b79c 100644 --- a/providers/xscale/xscale.h +++ b/providers/xscale/xscale.h @@ -140,6 +140,17 @@ struct xsc_context { uint32_t mdb_mmap_size; }; +struct xsc_pd { + struct ibv_pd ibv_pd; + uint32_t pdn; + atomic_int refcount; +}; + +struct xsc_mr { + struct verbs_mr vmr; + uint32_t alloc_flags; +}; + union xsc_ib_fw_ver { uint64_t data; struct { @@ -174,6 +185,17 @@ static inline struct xsc_context *to_xctx(struct ibv_context *ibctx) return container_of(ibctx, struct xsc_context, ibv_ctx.context); } +/* to_xpd always returns the real xsc_pd object ie the protection domain. */ +static inline struct xsc_pd *to_xpd(struct ibv_pd *ibpd) +{ + return container_of(ibpd, struct xsc_pd, ibv_pd); +} + +static inline struct xsc_mr *to_xmr(struct ibv_mr *ibmr) +{ + return container_of(ibmr, struct xsc_mr, vmr.ibv_mr); +} + int xsc_query_device(struct ibv_context *context, struct ibv_device_attr *attr); int xsc_query_device_ex(struct ibv_context *context, const struct ibv_query_device_ex_input *input, @@ -181,4 +203,11 @@ int xsc_query_device_ex(struct ibv_context *context, int xsc_query_port(struct ibv_context *context, uint8_t port, struct ibv_port_attr *attr); +struct ibv_pd *xsc_alloc_pd(struct ibv_context *context); +int xsc_free_pd(struct ibv_pd *pd); + +struct ibv_mr *xsc_reg_mr(struct ibv_pd *pd, void *addr, size_t length, + uint64_t hca_va, int access); +int xsc_dereg_mr(struct verbs_mr *mr); + #endif /* XSC_H */ From 9f5efabdff64bd3bf7b6a066b75cec4683a3af04 Mon Sep 17 00:00:00 2001 From: Tian Xin Date: Tue, 24 Sep 2024 15:08:07 +0800 Subject: [PATCH 04/13] libxscale: Add completion queue (CQ) support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch adds complete CQ management: - create_cq / create_cq_ex: Allocate CQ buffers using either huge pages, contiguous memory, or regular anonymous memory (controlled by environment variables). Support for GPU‑backed buffers via dmabuf is also provided. - resize_cq: Resize an existing CQ. - destroy_cq: Free CQ resources and clean up associated QP error lists. This lays the groundwork for polling completions and event handling. Signed-off-by: Tian Xin Signed-off-by: Wei Honggang Signed-off-by: Zhao Qianwei Signed-off-by: Li Qiang Signed-off-by: Yan Lei --- kernel-headers/rdma/xsc-abi.h | 41 ++- providers/xscale/CMakeLists.txt | 2 + providers/xscale/bitmap.h | 85 +++++ providers/xscale/buf.c | 594 ++++++++++++++++++++++++++++++++ providers/xscale/cq.c | 90 +++++ providers/xscale/verbs.c | 313 +++++++++++++++++ providers/xscale/xsc-abi.h | 7 + providers/xscale/xscale.c | 9 +- providers/xscale/xscale.h | 192 +++++++++++ providers/xscale/xscdv.h | 63 ++++ 10 files changed, 1388 insertions(+), 8 deletions(-) create mode 100644 providers/xscale/bitmap.h create mode 100644 providers/xscale/buf.c create mode 100644 providers/xscale/cq.c create mode 100644 providers/xscale/xscdv.h diff --git a/kernel-headers/rdma/xsc-abi.h b/kernel-headers/rdma/xsc-abi.h index 36563f165..ebb519365 100644 --- a/kernel-headers/rdma/xsc-abi.h +++ b/kernel-headers/rdma/xsc-abi.h @@ -11,12 +11,9 @@ #include /* For ETH_ALEN. */ #include -/* Make sure that all structs defined in this file remain laid out so - * that they pack the same way on 32-bit and 64-bit architectures (to - * avoid incompatibility between 32-bit userspace and 64-bit kernels). - * In particular do not use pointer types -- pass pointers in __u64 - * instead. - */ +enum { + XSC_HW_FEATURE_FLAG_SUPPORT_CQE64 = 1 << 0, +}; struct xsc_ib_alloc_ucontext_resp { __u32 qp_tab_size; @@ -40,4 +37,36 @@ struct xsc_ib_alloc_pd_resp { __u32 pdn; }; +struct xsc_ib_create_cq { + __aligned_u64 buf_addr; + __aligned_u64 db_addr; + __u32 cqe_size; + + int dmabuf_fd; + size_t dmabuf_sz; +}; + +struct xsc_ib_create_cq_resp { + __u32 cqn; + __u32 reserved; +}; + +struct xsc_ib_resize_cq { + __aligned_u64 buf_addr; + __u16 cqe_size; + __u16 reserved0; + __u32 reserved1; +}; + +enum xsc_ib_mmap_cmd { + XSC_IB_MMAP_REGULAR_PAGE = 0, + XSC_IB_MMAP_GET_CONTIGUOUS_PAGES = 1, + XSC_IB_MMAP_WC_PAGE = 2, + XSC_IB_MMAP_NC_PAGE = 3, + XSC_IB_MMAP_CORE_CLOCK = 5, + XSC_IB_MMAP_ALLOC_WC = 6, + XSC_IB_MMAP_CLOCK_INFO = 7, + XSC_IB_MMAP_DEVICE_MEM = 8, +}; + #endif /* XSC_ABI_USER_H */ diff --git a/providers/xscale/CMakeLists.txt b/providers/xscale/CMakeLists.txt index 220944d52..05710d500 100644 --- a/providers/xscale/CMakeLists.txt +++ b/providers/xscale/CMakeLists.txt @@ -2,6 +2,8 @@ rdma_shared_provider(xscale libxsc.map 1 1.24.${PACKAGE_VERSION} xscale.c verbs.c + cq.c + buf.c ) rdma_pkg_config("xscale" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}") diff --git a/providers/xscale/bitmap.h b/providers/xscale/bitmap.h new file mode 100644 index 000000000..d59b1ab32 --- /dev/null +++ b/providers/xscale/bitmap.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef BITMAP_H +#define BITMAP_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "xscale.h" + +/* Only ia64 requires this */ +#ifdef __ia64__ +#define XSC_SHM_ADDR ((void *)0x8000000000000000UL) +#define XSC_SHMAT_FLAGS (SHM_RND) +#else +#define XSC_SHM_ADDR NULL +#define XSC_SHMAT_FLAGS 0 +#endif + +#define BITS_PER_LONG (8 * sizeof(long)) + +#ifndef HPAGE_SIZE +#define HPAGE_SIZE (2UL * 1024 * 1024) +#endif + +#define XSC_SHM_LENGTH HPAGE_SIZE +#define XSC_Q_CHUNK_SIZE 32768 +#define XSC_SHM_NUM_REGION 64 + +static inline unsigned long xsc_ffz(uint32_t word) +{ + return __builtin_ffs(~word) - 1; +} + +static inline uint32_t xsc_find_first_zero_bit(const unsigned long *addr, + uint32_t size) +{ + const unsigned long *p = addr; + uint32_t result = 0; + unsigned long tmp; + + while (size & ~(BITS_PER_LONG - 1)) { + tmp = *(p++); + if (~tmp) + goto found; + result += BITS_PER_LONG; + size -= BITS_PER_LONG; + } + if (!size) + return result; + + tmp = (*p) | (~0UL << size); + if (tmp == (uint32_t)~0UL) /* Are any bits zero? */ + return result + size; /* Nope. */ +found: + return result + xsc_ffz(tmp); +} + +static inline void xsc_set_bit(unsigned int nr, unsigned long *addr) +{ + addr[(nr / BITS_PER_LONG)] |= (1 << (nr % BITS_PER_LONG)); +} + +static inline void xsc_clear_bit(unsigned int nr, unsigned long *addr) +{ + addr[(nr / BITS_PER_LONG)] &= ~(1 << (nr % BITS_PER_LONG)); +} + +static inline int xsc_test_bit(unsigned int nr, const unsigned long *addr) +{ + return !!(addr[(nr / BITS_PER_LONG)] & (1 << (nr % BITS_PER_LONG))); +} + +#endif diff --git a/providers/xscale/buf.c b/providers/xscale/buf.c new file mode 100644 index 000000000..9d280eca0 --- /dev/null +++ b/providers/xscale/buf.c @@ -0,0 +1,594 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include "xscale.h" +#include "bitmap.h" + +static int xsc_bitmap_init(struct xsc_bitmap *bitmap, uint32_t num, + uint32_t mask) +{ + bitmap->last = 0; + bitmap->top = 0; + bitmap->max = num; + bitmap->avail = num; + bitmap->mask = mask; + bitmap->avail = bitmap->max; + bitmap->table = calloc(BITS_TO_LONGS(bitmap->max), sizeof(*bitmap->table)); + if (!bitmap->table) + return -ENOMEM; + + return 0; +} + +static void bitmap_free_range(struct xsc_bitmap *bitmap, uint32_t obj, + int cnt) +{ + int i; + + obj &= bitmap->max - 1; + + for (i = 0; i < cnt; i++) + xsc_clear_bit(obj + i, bitmap->table); + bitmap->last = min(bitmap->last, obj); + bitmap->top = (bitmap->top + bitmap->max) & bitmap->mask; + bitmap->avail += cnt; +} + +static int bitmap_empty(struct xsc_bitmap *bitmap) +{ + return (bitmap->avail == bitmap->max) ? 1 : 0; +} + +static int bitmap_avail(struct xsc_bitmap *bitmap) +{ + return bitmap->avail; +} + +static void xsc_bitmap_cleanup(struct xsc_bitmap *bitmap) +{ + if (bitmap->table) + free(bitmap->table); +} + +static void free_huge_mem(struct xsc_hugetlb_mem *hmem) +{ + xsc_bitmap_cleanup(&hmem->bitmap); + if (shmdt(hmem->shmaddr) == -1) + xsc_dbg(stderr, XSC_DBG_CONTIG, "%s\n", strerror(errno)); + shmctl(hmem->shmid, IPC_RMID, NULL); + free(hmem); +} + +static int xsc_bitmap_alloc(struct xsc_bitmap *bitmap) +{ + uint32_t obj; + int ret; + + obj = xsc_find_first_zero_bit(bitmap->table, bitmap->max); + if (obj < bitmap->max) { + xsc_set_bit(obj, bitmap->table); + bitmap->last = (obj + 1); + if (bitmap->last == bitmap->max) + bitmap->last = 0; + obj |= bitmap->top; + ret = obj; + } else + ret = -1; + + if (ret != -1) + --bitmap->avail; + + return ret; +} + +static uint32_t find_aligned_range(unsigned long *bitmap, + uint32_t start, uint32_t nbits, + int len, int alignment) +{ + uint32_t end, i; + +again: + start = align(start, alignment); + + while ((start < nbits) && xsc_test_bit(start, bitmap)) + start += alignment; + + if (start >= nbits) + return -1; + + end = start + len; + if (end > nbits) + return -1; + + for (i = start + 1; i < end; i++) { + if (xsc_test_bit(i, bitmap)) { + start = i + 1; + goto again; + } + } + + return start; +} + +static int bitmap_alloc_range(struct xsc_bitmap *bitmap, int cnt, + int align) +{ + uint32_t obj; + int ret, i; + + if (cnt == 1 && align == 1) + return xsc_bitmap_alloc(bitmap); + + if (cnt > bitmap->max) + return -1; + + obj = find_aligned_range(bitmap->table, bitmap->last, + bitmap->max, cnt, align); + if (obj >= bitmap->max) { + bitmap->top = (bitmap->top + bitmap->max) & bitmap->mask; + obj = find_aligned_range(bitmap->table, 0, bitmap->max, + cnt, align); + } + + if (obj < bitmap->max) { + for (i = 0; i < cnt; i++) + xsc_set_bit(obj + i, bitmap->table); + if (obj == bitmap->last) { + bitmap->last = (obj + cnt); + if (bitmap->last >= bitmap->max) + bitmap->last = 0; + } + obj |= bitmap->top; + ret = obj; + } else + ret = -1; + + if (ret != -1) + bitmap->avail -= cnt; + + return obj; +} + +static struct xsc_hugetlb_mem *alloc_huge_mem(size_t size) +{ + struct xsc_hugetlb_mem *hmem; + size_t shm_len; + + hmem = malloc(sizeof(*hmem)); + if (!hmem) + return NULL; + + shm_len = align(size, XSC_SHM_LENGTH); + hmem->shmid = shmget(IPC_PRIVATE, shm_len, SHM_HUGETLB | SHM_R | SHM_W); + if (hmem->shmid == -1) { + xsc_dbg(stderr, XSC_DBG_CONTIG, "%s\n", strerror(errno)); + goto out_free; + } + + hmem->shmaddr = shmat(hmem->shmid, XSC_SHM_ADDR, XSC_SHMAT_FLAGS); + if (hmem->shmaddr == (void *)-1) { + xsc_dbg(stderr, XSC_DBG_CONTIG, "%s\n", strerror(errno)); + goto out_rmid; + } + + if (xsc_bitmap_init(&hmem->bitmap, shm_len / XSC_Q_CHUNK_SIZE, + shm_len / XSC_Q_CHUNK_SIZE - 1)) { + xsc_dbg(stderr, XSC_DBG_CONTIG, "%s\n", strerror(errno)); + goto out_shmdt; + } + + /* + * Marked to be destroyed when process detaches from shmget segment + */ + shmctl(hmem->shmid, IPC_RMID, NULL); + + return hmem; + +out_shmdt: + if (shmdt(hmem->shmaddr) == -1) + xsc_dbg(stderr, XSC_DBG_CONTIG, "%s\n", strerror(errno)); + +out_rmid: + shmctl(hmem->shmid, IPC_RMID, NULL); + +out_free: + free(hmem); + return NULL; +} + +static int alloc_huge_buf(struct xsc_context *xctx, struct xsc_buf *buf, + size_t size, int page_size) +{ + int found = 0; + int nchunk; + struct xsc_hugetlb_mem *hmem; + int ret; + + buf->length = align(size, XSC_Q_CHUNK_SIZE); + nchunk = buf->length / XSC_Q_CHUNK_SIZE; + + if (!nchunk) + return 0; + + xsc_spin_lock(&xctx->hugetlb_lock); + list_for_each(&xctx->hugetlb_list, hmem, entry) { + if (bitmap_avail(&hmem->bitmap)) { + buf->base = bitmap_alloc_range(&hmem->bitmap, nchunk, 1); + if (buf->base != -1) { + buf->hmem = hmem; + found = 1; + break; + } + } + } + xsc_spin_unlock(&xctx->hugetlb_lock); + + if (!found) { + hmem = alloc_huge_mem(buf->length); + if (!hmem) + return -1; + + buf->base = bitmap_alloc_range(&hmem->bitmap, nchunk, 1); + if (buf->base == -1) { + free_huge_mem(hmem); + /* TBD: remove after proven stability */ + fprintf(stderr, "BUG: huge allocation\n"); + return -1; + } + + buf->hmem = hmem; + + xsc_spin_lock(&xctx->hugetlb_lock); + if (bitmap_avail(&hmem->bitmap)) + list_add(&xctx->hugetlb_list, &hmem->entry); + else + list_add_tail(&xctx->hugetlb_list, &hmem->entry); + xsc_spin_unlock(&xctx->hugetlb_lock); + } + + buf->buf = hmem->shmaddr + buf->base * XSC_Q_CHUNK_SIZE; + + ret = ibv_dontfork_range(buf->buf, buf->length); + if (ret) + goto out_fork; + + buf->type = XSC_ALLOC_TYPE_HUGE; + + return 0; + +out_fork: + xsc_spin_lock(&xctx->hugetlb_lock); + bitmap_free_range(&hmem->bitmap, buf->base, nchunk); + if (bitmap_empty(&hmem->bitmap)) { + list_del(&hmem->entry); + xsc_spin_unlock(&xctx->hugetlb_lock); + free_huge_mem(hmem); + } else + xsc_spin_unlock(&xctx->hugetlb_lock); + + return -1; +} + +static void free_huge_buf(struct xsc_context *ctx, struct xsc_buf *buf) +{ + int nchunk; + + nchunk = buf->length / XSC_Q_CHUNK_SIZE; + if (!nchunk) + return; + + xsc_spin_lock(&ctx->hugetlb_lock); + bitmap_free_range(&buf->hmem->bitmap, buf->base, nchunk); + if (bitmap_empty(&buf->hmem->bitmap)) { + list_del(&buf->hmem->entry); + xsc_spin_unlock(&ctx->hugetlb_lock); + free_huge_mem(buf->hmem); + } else + xsc_spin_unlock(&ctx->hugetlb_lock); +} + +static void xsc_free_buf_extern(struct xsc_context *ctx, struct xsc_buf *buf) +{ + ibv_dofork_range(buf->buf, buf->length); + ctx->extern_alloc.free(buf->buf, ctx->extern_alloc.data); +} + +static int xsc_alloc_buf_extern(struct xsc_context *ctx, struct xsc_buf *buf, + size_t size) +{ + void *addr; + + addr = ctx->extern_alloc.alloc(size, ctx->extern_alloc.data); + if (addr || size == 0) { + if (ibv_dontfork_range(addr, size)) { + xsc_err("External mode dontfork_range failed\n"); + ctx->extern_alloc.free(addr, + ctx->extern_alloc.data); + return -1; + } + buf->buf = addr; + buf->length = size; + buf->type = XSC_ALLOC_TYPE_EXTERNAL; + return 0; + } + + xsc_err("External alloc failed\n"); + return -1; +} + +int xsc_alloc_prefered_buf(struct xsc_context *xctx, + struct xsc_buf *buf, + size_t size, int page_size, + enum xsc_alloc_type type, + const char *component) +{ + int ret; + + /* + * Fallback mechanism priority: + * huge pages + * contig pages + * default + */ + if (type == XSC_ALLOC_TYPE_HUGE || + type == XSC_ALLOC_TYPE_PREFER_HUGE || + type == XSC_ALLOC_TYPE_ALL) { + ret = alloc_huge_buf(xctx, buf, size, page_size); + if (!ret) + return 0; + + if (type == XSC_ALLOC_TYPE_HUGE) + return -1; + + xsc_dbg(xctx->dbg_fp, XSC_DBG_CONTIG, "Huge mode allocation failed, fallback to %s mode\n", + XSC_ALLOC_TYPE_ALL ? "contig" : "default"); + } + + if (type == XSC_ALLOC_TYPE_CONTIG || + type == XSC_ALLOC_TYPE_PREFER_CONTIG || + type == XSC_ALLOC_TYPE_ALL) { + ret = xsc_alloc_buf_contig(xctx, buf, size, page_size, component); + if (!ret) + return 0; + + if (type == XSC_ALLOC_TYPE_CONTIG) + return -1; + xsc_dbg(xctx->dbg_fp, XSC_DBG_CONTIG, "Contig allocation failed, fallback to default mode\n"); + } + + if (type == XSC_ALLOC_TYPE_EXTERNAL) + return xsc_alloc_buf_extern(xctx, buf, size); + + return xsc_alloc_buf(buf, size, page_size); + +} + +int xsc_free_actual_buf(struct xsc_context *ctx, struct xsc_buf *buf) +{ + int err = 0; + + switch (buf->type) { + case XSC_ALLOC_TYPE_ANON: + xsc_free_buf(buf); + break; + + case XSC_ALLOC_TYPE_HUGE: + free_huge_buf(ctx, buf); + break; + + case XSC_ALLOC_TYPE_CONTIG: + xsc_free_buf_contig(ctx, buf); + break; + + case XSC_ALLOC_TYPE_EXTERNAL: + xsc_free_buf_extern(ctx, buf); + break; + + default: + fprintf(stderr, "Bad allocation type\n"); + } + + return err; +} + +/* This function computes log2(v) rounded up. + * We don't want to have a dependency to libm which exposes ceil & log2 APIs. + * Code was written based on public domain code: + * URL: http://graphics.stanford.edu/~seander/bithacks.html#IntegerLog. + */ +static uint32_t xsc_get_block_order(uint32_t v) +{ + static const uint32_t bits_arr[] = {0x2, 0xC, 0xF0, 0xFF00, 0xFFFF0000}; + static const uint32_t shift_arr[] = {1, 2, 4, 8, 16}; + int i; + uint32_t input_val = v; + register uint32_t r = 0;/* result of log2(v) will go here */ + + for (i = 4; i >= 0; i--) { + if (v & bits_arr[i]) { + v >>= shift_arr[i]; + r |= shift_arr[i]; + } + } + /* Rounding up if required */ + r += !!(input_val & ((1 << r) - 1)); + + return r; +} + +static bool xsc_is_extern_alloc(struct xsc_context *context) +{ + return context->extern_alloc.alloc && context->extern_alloc.free; +} + +void xsc_get_alloc_type(struct xsc_context *context, + const char *component, + enum xsc_alloc_type *alloc_type, + enum xsc_alloc_type default_type) +{ + char *env_value; + char name[128]; + + if (xsc_is_extern_alloc(context)) { + *alloc_type = XSC_ALLOC_TYPE_EXTERNAL; + return; + } + + snprintf(name, sizeof(name), "%s_ALLOC_TYPE", component); + + *alloc_type = default_type; + + env_value = getenv(name); + if (env_value) { + if (!strcasecmp(env_value, "ANON")) + *alloc_type = XSC_ALLOC_TYPE_ANON; + else if (!strcasecmp(env_value, "HUGE")) + *alloc_type = XSC_ALLOC_TYPE_HUGE; + else if (!strcasecmp(env_value, "CONTIG")) + *alloc_type = XSC_ALLOC_TYPE_CONTIG; + else if (!strcasecmp(env_value, "PREFER_CONTIG")) + *alloc_type = XSC_ALLOC_TYPE_PREFER_CONTIG; + else if (!strcasecmp(env_value, "PREFER_HUGE")) + *alloc_type = XSC_ALLOC_TYPE_PREFER_HUGE; + else if (!strcasecmp(env_value, "ALL")) + *alloc_type = XSC_ALLOC_TYPE_ALL; + } +} + +static void xsc_alloc_get_env_info(int *max_block_log, + int *min_block_log, + const char *component) + +{ + char *env; + int value; + char name[128]; + + /* First set defaults */ + *max_block_log = XSC_MAX_LOG2_CONTIG_BLOCK_SIZE; + *min_block_log = XSC_MIN_LOG2_CONTIG_BLOCK_SIZE; + + snprintf(name, sizeof(name), "%s_MAX_LOG2_CONTIG_BSIZE", component); + env = getenv(name); + if (env) { + value = atoi(env); + if (value <= XSC_MAX_LOG2_CONTIG_BLOCK_SIZE && + value >= XSC_MIN_LOG2_CONTIG_BLOCK_SIZE) + *max_block_log = value; + else + fprintf(stderr, "Invalid value %d for %s\n", + value, name); + } + sprintf(name, "%s_MIN_LOG2_CONTIG_BSIZE", component); + env = getenv(name); + if (env) { + value = atoi(env); + if (value >= XSC_MIN_LOG2_CONTIG_BLOCK_SIZE && + value <= *max_block_log) + *min_block_log = value; + else + fprintf(stderr, "Invalid value %d for %s\n", + value, name); + } +} + +int xsc_alloc_buf_contig(struct xsc_context *xctx, + struct xsc_buf *buf, size_t size, + int page_size, + const char *component) +{ + void *addr = MAP_FAILED; + int block_size_exp; + int max_block_log; + int min_block_log; + struct ibv_context *context = &xctx->ibv_ctx.context; + off_t offset; + + xsc_alloc_get_env_info(&max_block_log, + &min_block_log, + component); + + block_size_exp = xsc_get_block_order(size); + + if (block_size_exp > max_block_log) + block_size_exp = max_block_log; + + do { + offset = 0; + set_command(XSC_IB_MMAP_GET_CONTIGUOUS_PAGES, &offset); + set_order(block_size_exp, &offset); + addr = mmap(NULL, size, PROT_WRITE | PROT_READ, MAP_SHARED, + context->cmd_fd, page_size * offset); + if (addr != MAP_FAILED) + break; + + /* + * The kernel returns EINVAL if not supported + */ + if (errno == EINVAL) + return -1; + + block_size_exp -= 1; + } while (block_size_exp >= min_block_log); + xsc_dbg(xctx->dbg_fp, XSC_DBG_CONTIG, "block order %d, addr %p\n", block_size_exp, addr); + + if (addr == MAP_FAILED) + return -1; + + if (ibv_dontfork_range(addr, size)) { + munmap(addr, size); + return -1; + } + + buf->buf = addr; + buf->length = size; + buf->type = XSC_ALLOC_TYPE_CONTIG; + + return 0; +} + +void xsc_free_buf_contig(struct xsc_context *xctx, struct xsc_buf *buf) +{ + ibv_dofork_range(buf->buf, buf->length); + munmap(buf->buf, buf->length); +} + +int xsc_alloc_buf(struct xsc_buf *buf, size_t size, int page_size) +{ + int ret; + int al_size; + + al_size = align(size, page_size); + ret = posix_memalign(&buf->buf, page_size, al_size); + if (ret) + return ret; + + ret = ibv_dontfork_range(buf->buf, al_size); + if (ret) + free(buf->buf); + + if (!ret) { + buf->length = al_size; + buf->type = XSC_ALLOC_TYPE_ANON; + } + + return ret; +} + +void xsc_free_buf(struct xsc_buf *buf) +{ + ibv_dofork_range(buf->buf, buf->length); + free(buf->buf); +} diff --git a/providers/xscale/cq.c b/providers/xscale/cq.c new file mode 100644 index 000000000..0ffe2bcfe --- /dev/null +++ b/providers/xscale/cq.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "xscale.h" +#include "xsc_hsi.h" + +void xsc_cq_fill_pfns(struct xsc_cq *cq, const struct ibv_cq_init_attr_ex *cq_attr) +{ +} + +static int xsc_use_huge(const char *key) +{ + char *e; + + e = getenv(key); + if (e && !strcmp(e, "y")) + return 1; + + return 0; +} + +int xsc_alloc_cq_buf(struct xsc_context *xctx, struct xsc_cq *cq, + struct xsc_buf *buf, int nent, int cqe_sz, + struct xscdv_devx_umem_in *umem_in) +{ + struct xsc_device *xdev = to_xdev(xctx->ibv_ctx.context.device); + int ret; + enum xsc_alloc_type type; + enum xsc_alloc_type default_type = XSC_ALLOC_TYPE_ANON; + + if (umem_in) { + if (umem_in->comp_mask & XSCDV_UMEM_MASK_DMABUF) { + if (umem_in->dmabuf_fd == -1) { + xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, + "dmabuf_fd is invalid :%u\n", + umem_in->dmabuf_fd); + return -1; + } + } + + buf->type = XSC_ALLOC_TYPE_GPU; + buf->buf = umem_in->addr; + buf->length = umem_in->size; + + xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, + "cq_buf(%p),cq_buf_size=0x%lx is given by gpu memory,infact need size=0x%x.\n", + buf->buf, buf->length, (nent * cqe_sz)); + } else { + if (xsc_use_huge("HUGE_CQ")) + default_type = XSC_ALLOC_TYPE_HUGE; + + xsc_get_alloc_type(xctx, XSC_CQ_PREFIX, &type, default_type); + + ret = xsc_alloc_prefered_buf(xctx, buf, + align(nent * cqe_sz, xdev->page_size), + xdev->page_size, + type, + XSC_CQ_PREFIX); + + if (ret) + return -1; + + memset(buf->buf, 0, nent * cqe_sz); + } + + return 0; +} + +int xsc_free_cq_buf(struct xsc_context *ctx, struct xsc_buf *buf) +{ + if (buf->type != XSC_ALLOC_TYPE_GPU) + return xsc_free_actual_buf(ctx, buf); + else + return 0; +} diff --git a/providers/xscale/verbs.c b/providers/xscale/verbs.c index 27af46d7f..e4c20dee9 100644 --- a/providers/xscale/verbs.c +++ b/providers/xscale/verbs.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -27,6 +28,9 @@ #include "xscale.h" #include "xsc-abi.h" +#include "xsc_hw.h" + +int xsc_single_threaded; int xsc_query_port(struct ibv_context *context, uint8_t port, struct ibv_port_attr *attr) @@ -121,6 +125,315 @@ int xsc_dereg_mr(struct verbs_mr *vmr) return 0; } +static int xsc_round_up_power_of_two(long long sz) +{ + long long ret; + + for (ret = 1; ret < sz; ret <<= 1) + ; /* nothing */ + + if (ret > INT_MAX) { + fprintf(stderr, "%s: roundup overflow\n", __func__); + return -ENOMEM; + } + + return (int)ret; +} + +static int align_queue_size(long long req) +{ + return xsc_round_up_power_of_two(req); +} + +enum { + CREATE_CQ_SUPPORTED_WC_FLAGS = IBV_WC_STANDARD_FLAGS | + IBV_WC_EX_WITH_COMPLETION_TIMESTAMP +}; + +enum { + CREATE_CQ_SUPPORTED_COMP_MASK = IBV_CQ_INIT_ATTR_MASK_FLAGS +}; + +enum { + CREATE_CQ_SUPPORTED_FLAGS = + IBV_CREATE_CQ_ATTR_SINGLE_THREADED | + IBV_CREATE_CQ_ATTR_IGNORE_OVERRUN +}; + +static int xsc_cqe_depth_check(void) +{ + char *e; + + e = getenv("XSC_CQE_DEPTH_CHECK"); + if (e && !strcmp(e, "n")) + return 0; + + return 1; +} + +static void init_cqe(struct xsc_context *xctx, struct xsc_buf *buf, int ncqe, int cqe_sz) +{ + int i; + + if (buf->type != XSC_ALLOC_TYPE_GPU) { + for (i = 0; i < ncqe; i++) { + struct xsc_cqe *cqe = (struct xsc_cqe *)(buf->buf + i * cqe_sz); + + cqe->data3 |= FIELD_PREP(XSC_CQE_OWNER_MASK, 1); + } + } else { + xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, "cq buff is given by gpu,do nothing in host.\n"); + } +} + +static struct ibv_cq_ex *create_cq(struct ibv_context *context, + const struct ibv_cq_init_attr_ex *cq_attr, + int cq_alloc_flags, + struct xscdv_cq_init_attr *xcq_attr, + struct xscdv_devx_umem_in *umem_in) +{ + struct xsc_create_cq cmd = {}; + struct xsc_create_cq_resp resp = {}; + struct xsc_create_cq_ex cmd_ex = {}; + struct xsc_create_cq_ex_resp resp_ex = {}; + struct xsc_ib_create_cq *cmd_drv; + struct xsc_ib_create_cq_resp *resp_drv; + struct xsc_cq *cq; + int cqe_sz; + int ret; + int ncqe; + struct xsc_context *xctx = to_xctx(context); + bool use_ex = false; + char *env; + + if (!cq_attr->cqe) { + xsc_err("CQE invalid\n"); + errno = EINVAL; + return NULL; + } + + xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, "CQE number:%u\n", cq_attr->cqe); + + if (cq_attr->comp_mask & ~CREATE_CQ_SUPPORTED_COMP_MASK) { + xsc_err("Unsupported comp_mask for create cq\n"); + errno = EINVAL; + return NULL; + } + + if (cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS && + cq_attr->flags & ~CREATE_CQ_SUPPORTED_FLAGS) { + xsc_err("Unsupported creation flags requested for create cq\n"); + errno = EINVAL; + return NULL; + } + + if (cq_attr->wc_flags & ~CREATE_CQ_SUPPORTED_WC_FLAGS) { + xsc_err("unsupported wc flags:0x%lx\n", cq_attr->wc_flags); + errno = ENOTSUP; + return NULL; + } + + cq = calloc(1, sizeof(*cq)); + if (!cq) { + xsc_err("Alloc CQ failed\n"); + errno = ENOMEM; + return NULL; + } + + if (cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS) { + if (cq_attr->flags & IBV_CREATE_CQ_ATTR_SINGLE_THREADED) + cq->flags |= XSC_CQ_FLAGS_SINGLE_THREADED; + if (cq_attr->flags & IBV_CREATE_CQ_ATTR_IGNORE_OVERRUN) + use_ex = true; + } + + xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, "use_ex:%u\n", use_ex); + + cmd_drv = use_ex ? &cmd_ex.drv_payload : &cmd.drv_payload; + resp_drv = use_ex ? &resp_ex.drv_payload : &resp.drv_payload; + + cq->cons_index = 0; + + if (xsc_spinlock_init(&cq->lock, !xsc_single_threaded)) + goto err; + + ncqe = align_queue_size(cq_attr->cqe); + if (ncqe < XSC_CQE_RING_DEPTH_MIN) { + xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, "CQE ring size %u is not enough, set it as %u\n", + ncqe, XSC_CQE_RING_DEPTH_MIN); + ncqe = XSC_CQE_RING_DEPTH_MIN; + } + + if (ncqe > xctx->max_cqe) { + if (xsc_cqe_depth_check()) { + xsc_err("CQE ring size %u exceeds CQE ring depth %u, abort!\n", + ncqe, xctx->max_cqe); + errno = EINVAL; + goto err_spl; + } else { + xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, "CQE ring size %u exceeds the MAX ring szie, set it as %u\n", + ncqe, xctx->max_cqe); + ncqe = xctx->max_cqe; + } + } + + cqe_sz = (xctx->hw_feature_flag & XSC_HW_FEATURE_FLAG_SUPPORT_CQE64) ? + XSC_CQE_SIZE64 : XSC_CQE_SIZE; + xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, "CQE number:%u, size:%u\n", ncqe, cqe_sz); + + if (xsc_alloc_cq_buf(to_xctx(context), cq, &cq->buf_a, ncqe, cqe_sz, umem_in)) { + xsc_err("Alloc cq buffer failed.\n"); + errno = ENOMEM; + goto err_spl; + } + + cq->arm_sn = 0; + cq->cqe_sz = cqe_sz; + cq->flags = cq_alloc_flags; + + cmd_drv->buf_addr = (uintptr_t) cq->buf_a.buf; + cmd_drv->db_addr = (uintptr_t) cq->dbrec; + cmd_drv->cqe_size = cqe_sz; + + xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, "buf_addr:%p\n", cq->buf_a.buf); + + if (cq_alloc_flags & XSC_CQ_FLAGS_EXTENDED) + xsc_cq_fill_pfns(cq, cq_attr); + + if (cq->buf_a.type == XSC_ALLOC_TYPE_GPU) { + cq->flags |= XSC_CQ_FLAGS_OWNED_BY_GPU; + cmd_drv->dmabuf_fd = umem_in->dmabuf_fd; + cmd_drv->dmabuf_sz = umem_in->size; + xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, "cq_buf %p, buffer size:0x%lx, dmabuf_fd:%d\n", + umem_in->addr, umem_in->size, umem_in->dmabuf_fd); + } else { + cmd_drv->dmabuf_fd = -1; + cmd_drv->dmabuf_sz = 0; + } + + if (use_ex) { + struct ibv_cq_init_attr_ex cq_attr_ex = *cq_attr; + + cq_attr_ex.cqe = ncqe; + ret = ibv_cmd_create_cq_ex(context, &cq_attr_ex, NULL, + &cq->verbs_cq, + &cmd_ex.ibv_cmd, sizeof(cmd_ex), + &resp_ex.ibv_resp, sizeof(resp_ex), + 0); + } else { + ret = ibv_cmd_create_cq(context, ncqe, cq_attr->channel, + cq_attr->comp_vector, + ibv_cq_ex_to_cq(&cq->verbs_cq.cq_ex), + &cmd.ibv_cmd, sizeof(cmd), + &resp.ibv_resp, sizeof(resp)); + } + + if (ret) { + xsc_err("ibv_cmd_create_cq failed,ret %d\n", ret); + goto err_buf; + } + + cq->active_buf = &cq->buf_a; + cq->resize_buf = NULL; + cq->cqn = resp_drv->cqn; + cq->stall_enable = to_xctx(context)->stall_enable; + cq->stall_adaptive_enable = to_xctx(context)->stall_adaptive_enable; + cq->stall_cycles = to_xctx(context)->stall_cycles; + + cq->db = xctx->cqm_reg_va + + (xctx->cqm_next_cid_reg & (xctx->page_size - 1)); + cq->armdb = xctx->cqm_armdb_va + + (xctx->cqm_armdb & (xctx->page_size - 1)); + cq->cqe_cnt = ncqe; + cq->log2_cq_ring_sz = xsc_ilog2(ncqe); + + init_cqe(xctx, &cq->buf_a, ncqe, cqe_sz); + + env = getenv("XSC_DISABLE_FLUSH_ERROR"); + cq->disable_flush_error_cqe = env ? true : false; + xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, "cqe count:%u cqn:%u cq_buf:%p cq_buf_sz:0x%lx dmabuf_fd:%d\n", + cq->cqe_cnt, cq->cqn, cq->buf_a.buf, cq->buf_a.length, + umem_in ? umem_in->dmabuf_fd : -1); + list_head_init(&cq->err_state_qp_list); + return &cq->verbs_cq.cq_ex; + + +err_buf: + xsc_free_cq_buf(to_xctx(context), &cq->buf_a); + +err_spl: + xsc_spinlock_destroy(&cq->lock); + +err: + free(cq); + + return NULL; +} + +struct ibv_cq *xsc_create_cq(struct ibv_context *context, int cqe, + struct ibv_comp_channel *channel, + int comp_vector) +{ + struct ibv_cq_ex *cq; + struct ibv_cq_init_attr_ex cq_attr = {.cqe = cqe, .channel = channel, + .comp_vector = comp_vector, + .wc_flags = IBV_WC_STANDARD_FLAGS}; + + if (cqe <= 0) { + errno = EINVAL; + return NULL; + } + + cq = create_cq(context, &cq_attr, 0, NULL, NULL); + return cq ? ibv_cq_ex_to_cq(cq) : NULL; +} + +struct ibv_cq_ex *xsc_create_cq_ex(struct ibv_context *context, + struct ibv_cq_init_attr_ex *cq_attr) +{ + return create_cq(context, cq_attr, XSC_CQ_FLAGS_EXTENDED, NULL, NULL); +} + +int xsc_resize_cq(struct ibv_cq *ibcq, int cqe) +{ + struct xsc_cq *cq = to_xcq(ibcq); + + if (cqe < 0) { + errno = EINVAL; + return errno; + } + + xsc_spin_lock(&cq->lock); + cq->active_cqes = cq->verbs_cq.cq_ex.cqe; + /* currently we don't change cqe size */ + cq->resize_cqe_sz = cq->cqe_sz; + cq->resize_cqes = cq->verbs_cq.cq_ex.cqe; + xsc_spin_unlock(&cq->lock); + cq->resize_buf = NULL; + return 0; +} + +int xsc_destroy_cq(struct ibv_cq *cq) +{ + int ret; + struct xsc_err_state_qp_node *tmp, *err_qp_node; + + xsc_dbg(to_xctx(cq->context)->dbg_fp, XSC_DBG_CQ, "\n"); + ret = ibv_cmd_destroy_cq(cq); + if (ret) + return ret; + + list_for_each_safe(&to_xcq(cq)->err_state_qp_list, err_qp_node, tmp, entry) { + list_del(&err_qp_node->entry); + free(err_qp_node); + } + + xsc_free_cq_buf(to_xctx(cq->context), to_xcq(cq)->active_buf); + free(to_xcq(cq)); + + return 0; +} + static void xsc_set_fw_version(struct ibv_device_attr *attr, union xsc_ib_fw_ver *fw_ver) { diff --git a/providers/xscale/xsc-abi.h b/providers/xscale/xsc-abi.h index b6d03979d..8e0baff37 100644 --- a/providers/xscale/xsc-abi.h +++ b/providers/xscale/xsc-abi.h @@ -11,6 +11,7 @@ #include #include #include +#include "xscdv.h" #define XSC_UVERBS_MIN_ABI_VERSION 1 #define XSC_UVERBS_MAX_ABI_VERSION 1 @@ -19,5 +20,11 @@ DECLARE_DRV_CMD(xsc_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT, empty, xsc_ib_alloc_ucontext_resp); DECLARE_DRV_CMD(xsc_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, empty, xsc_ib_alloc_pd_resp); +DECLARE_DRV_CMD(xsc_create_cq, IB_USER_VERBS_CMD_CREATE_CQ, + xsc_ib_create_cq, xsc_ib_create_cq_resp); +DECLARE_DRV_CMD(xsc_create_cq_ex, IB_USER_VERBS_EX_CMD_CREATE_CQ, + xsc_ib_create_cq, xsc_ib_create_cq_resp); +DECLARE_DRV_CMD(xsc_resize_cq, IB_USER_VERBS_CMD_RESIZE_CQ, + xsc_ib_resize_cq, empty); #endif /* XSC_ABI_H */ diff --git a/providers/xscale/xscale.c b/providers/xscale/xscale.c index db4a76504..fa699bd25 100644 --- a/providers/xscale/xscale.c +++ b/providers/xscale/xscale.c @@ -34,12 +34,17 @@ static void xsc_free_context(struct ibv_context *ibctx); static const struct verbs_context_ops xsc_ctx_common_ops = { .query_port = xsc_query_port, + .query_device_ex = xsc_query_device_ex, + .free_context = xsc_free_context, + .alloc_pd = xsc_alloc_pd, .dealloc_pd = xsc_free_pd, .reg_mr = xsc_reg_mr, .dereg_mr = xsc_dereg_mr, - .query_device_ex = xsc_query_device_ex, - .free_context = xsc_free_context, + + .create_cq = xsc_create_cq, + .resize_cq = xsc_resize_cq, + .destroy_cq = xsc_destroy_cq, }; static void open_debug_file(struct xsc_context *ctx) diff --git a/providers/xscale/xscale.h b/providers/xscale/xscale.h index fed24b79c..57ac474cf 100644 --- a/providers/xscale/xscale.h +++ b/providers/xscale/xscale.h @@ -21,6 +21,19 @@ #include #include "xsc-abi.h" +#include "xscdv.h" + +enum { + XSC_IB_MMAP_CMD_SHIFT = 8, + XSC_IB_MMAP_CMD_MASK = 0xff, +}; + +#define XSC_CQ_PREFIX "XSC_CQ" +#define XSC_QP_PREFIX "XSC_QP" +#define XSC_MR_PREFIX "XSC_MR" +#define XSC_RWQ_PREFIX "XSC_RWQ" +#define XSC_MAX_LOG2_CONTIG_BLOCK_SIZE 23 +#define XSC_MIN_LOG2_CONTIG_BLOCK_SIZE 12 enum { XSC_DBG_QP = 1 << 0, @@ -74,11 +87,32 @@ enum { XSC_QP_TABLE_SIZE = 1 << (24 - XSC_QP_TABLE_SHIFT), }; +enum xsc_alloc_type { + XSC_ALLOC_TYPE_ANON, + XSC_ALLOC_TYPE_HUGE, + XSC_ALLOC_TYPE_CONTIG, + XSC_ALLOC_TYPE_PREFER_HUGE, + XSC_ALLOC_TYPE_PREFER_CONTIG, + XSC_ALLOC_TYPE_EXTERNAL, + XSC_ALLOC_TYPE_GPU, + XSC_ALLOC_TYPE_ALL +}; + +struct xsc_resource { + uint32_t rsn; +}; + struct xsc_device { struct verbs_device verbs_dev; int page_size; }; +struct xsc_spinlock { + pthread_spinlock_t lock; + int in_use; + int need_lock; +}; + #define XSC_PCI_VENDOR_ID 0x1f67 #define VEROCE_VENDOR_ID 0x1e93 #define XSC_VEROCE_INIT_PROFILE 0x10 @@ -115,7 +149,13 @@ struct xsc_context { int max_send_wr; int max_recv_wr; int num_ports; + int stall_enable; + int stall_adaptive_enable; + int stall_cycles; char hostname[NAME_BUFFER_SIZE]; + struct xsc_spinlock hugetlb_lock; + struct list_head hugetlb_list; + struct xscdv_ctx_allocators extern_alloc; uint32_t max_cqe; void *sqm_reg_va; void *rqm_reg_va; @@ -137,15 +177,92 @@ struct xsc_context { uint32_t tx_multidb_base; void *mdb_base; uint32_t tx_mdb_idx; + uint32_t hw_feature_flag; uint32_t mdb_mmap_size; }; +struct xsc_bitmap { + uint32_t last; + uint32_t top; + uint32_t max; + uint32_t avail; + uint32_t mask; + unsigned long *table; +}; + +struct xsc_hugetlb_mem { + int shmid; + void *shmaddr; + struct xsc_bitmap bitmap; + struct list_node entry; +}; + +struct xsc_buf { + void *buf; + size_t length; + int base; + struct xsc_hugetlb_mem *hmem; + enum xsc_alloc_type type; +}; + struct xsc_pd { struct ibv_pd ibv_pd; uint32_t pdn; atomic_int refcount; }; +enum { + XSC_CQ_FLAGS_RX_CSUM_VALID = 1 << 0, + XSC_CQ_FLAGS_EMPTY_DURING_POLL = 1 << 1, + XSC_CQ_FLAGS_FOUND_CQES = 1 << 2, + XSC_CQ_FLAGS_EXTENDED = 1 << 3, + XSC_CQ_FLAGS_SINGLE_THREADED = 1 << 4, + XSC_CQ_FLAGS_DV_OWNED = 1 << 5, + XSC_CQ_FLAGS_TM_SYNC_REQ = 1 << 6, + XSC_CQ_FLAGS_OWNED_BY_GPU = 1 << 7, +}; + +struct xsc_err_state_qp_node { + struct list_node entry; + uint32_t qp_id; + int is_sq; +}; + +struct xsc_cq { + /* ibv_cq should always be subset of ibv_cq_ex */ + struct verbs_cq verbs_cq; + struct xsc_buf buf_a; + struct xsc_buf buf_b; + struct xsc_buf *active_buf; + struct xsc_buf *resize_buf; + int resize_cqes; + int active_cqes; + struct xsc_spinlock lock; + uint32_t cqn; + uint32_t cons_index; + __le32 *dbrec; + __le32 *db; + __le32 *armdb; + uint32_t cqe_cnt; + int log2_cq_ring_sz; + int arm_sn; + int cqe_sz; + int resize_cqe_sz; + int stall_next_poll; + int stall_enable; + uint64_t stall_last_count; + int stall_adaptive_enable; + int stall_cycles; + struct xsc_resource *cur_rsc; + struct xsc_cqe *cqe; + uint32_t flags; + int umr_opcode; + bool disable_flush_error_cqe; + struct list_head err_state_qp_list; +}; + +struct xsc_qp; + struct xsc_mr { struct verbs_mr vmr; uint32_t alloc_flags; @@ -191,11 +308,37 @@ static inline struct xsc_pd *to_xpd(struct ibv_pd *ibpd) return container_of(ibpd, struct xsc_pd, ibv_pd); } +static inline struct xsc_cq *to_xcq(struct ibv_cq *ibcq) +{ + return container_of((struct ibv_cq_ex *)ibcq, struct xsc_cq, + verbs_cq.cq_ex); +} + static inline struct xsc_mr *to_xmr(struct ibv_mr *ibmr) { return container_of(ibmr, struct xsc_mr, vmr.ibv_mr); } +static inline struct xsc_qp *rsc_to_xqp(struct xsc_resource *rsc) +{ + return (struct xsc_qp *)rsc; +} + +int xsc_alloc_buf(struct xsc_buf *buf, size_t size, int page_size); +void xsc_free_buf(struct xsc_buf *buf); +int xsc_alloc_buf_contig(struct xsc_context *xctx, struct xsc_buf *buf, + size_t size, int page_size, const char *component); +void xsc_free_buf_contig(struct xsc_context *xctx, struct xsc_buf *buf); +int xsc_alloc_prefered_buf(struct xsc_context *xctx, + struct xsc_buf *buf, + size_t size, int page_size, + enum xsc_alloc_type alloc_type, + const char *component); +int xsc_free_actual_buf(struct xsc_context *ctx, struct xsc_buf *buf); +void xsc_get_alloc_type(struct xsc_context *context, + const char *component, + enum xsc_alloc_type *alloc_type, + enum xsc_alloc_type default_alloc_type); int xsc_query_device(struct ibv_context *context, struct ibv_device_attr *attr); int xsc_query_device_ex(struct ibv_context *context, const struct ibv_query_device_ex_input *input, @@ -209,5 +352,54 @@ int xsc_free_pd(struct ibv_pd *pd); struct ibv_mr *xsc_reg_mr(struct ibv_pd *pd, void *addr, size_t length, uint64_t hca_va, int access); int xsc_dereg_mr(struct verbs_mr *mr); +struct ibv_cq *xsc_create_cq(struct ibv_context *context, int cqe, + struct ibv_comp_channel *channel, int comp_vector); +struct ibv_cq_ex *xsc_create_cq_ex(struct ibv_context *context, + struct ibv_cq_init_attr_ex *cq_attr); +void xsc_cq_fill_pfns(struct xsc_cq *cq, + const struct ibv_cq_init_attr_ex *cq_attr); +int xsc_alloc_cq_buf(struct xsc_context *xctx, struct xsc_cq *cq, + struct xsc_buf *buf, int nent, int cqe_sz, + struct xscdv_devx_umem_in *umem_in); +int xsc_free_cq_buf(struct xsc_context *ctx, struct xsc_buf *buf); +int xsc_resize_cq(struct ibv_cq *cq, int cqe); +int xsc_destroy_cq(struct ibv_cq *cq); + +static inline int xsc_spin_lock(struct xsc_spinlock *lock) +{ + return pthread_spin_lock(&lock->lock); +} + +static inline int xsc_spin_unlock(struct xsc_spinlock *lock) +{ + return pthread_spin_unlock(&lock->lock); +} + +static inline int xsc_spinlock_init(struct xsc_spinlock *lock, int need_lock) +{ + lock->in_use = 0; + lock->need_lock = need_lock; + return pthread_spin_init(&lock->lock, PTHREAD_PROCESS_PRIVATE); +} + +static inline int xsc_spinlock_destroy(struct xsc_spinlock *lock) +{ + return pthread_spin_destroy(&lock->lock); +} + +static inline void set_command(int command, off_t *offset) +{ + *offset |= (command << XSC_IB_MMAP_CMD_SHIFT); +} + +static inline void set_arg(int arg, off_t *offset) +{ + *offset |= arg; +} + +static inline void set_order(int order, off_t *offset) +{ + set_arg(order, offset); +} #endif /* XSC_H */ diff --git a/providers/xscale/xscdv.h b/providers/xscale/xscdv.h new file mode 100644 index 000000000..a723712b8 --- /dev/null +++ b/providers/xscale/xscdv.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef _XSCDV_H_ +#define _XSCDV_H_ + +#include +#include /* For the __be64 type */ +#include +#include +#if defined(__SSE3__) +#include +#include +#include +#endif /* defined(__SSE3__) */ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +enum xscdv_cq_init_attr_mask { + XSCDV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE = 1 << 0, + XSCDV_CQ_INIT_ATTR_MASK_FLAGS = 1 << 1, + XSCDV_CQ_INIT_ATTR_MASK_CQE_SIZE = 1 << 2, +}; + +struct xscdv_cq_init_attr { + uint64_t comp_mask; /* Use enum xscdv_cq_init_attr_mask */ + uint8_t cqe_comp_res_format; /* Use enum xscdv_cqe_comp_res_format */ + uint32_t flags; + uint16_t cqe_size; /* when XSCDV_CQ_INIT_ATTR_MASK_CQE_SIZE set */ +}; + +enum xscdv_devx_umem_in_mask { + XSCDV_UMEM_MASK_DMABUF = 1 << 0, +}; + +struct xscdv_devx_umem_in { + void *addr; + size_t size; + uint32_t access; + uint64_t pgsz_bitmap; + uint64_t comp_mask; + int dmabuf_fd; +}; + +struct xscdv_ctx_allocators { + void *(*alloc)(size_t size, void *priv_data); + void (*free)(void *ptr, void *priv_data); + void *data; +}; + +#ifdef __cplusplus +} +#endif + +#endif /* _XSCDV_H_ */ From 029c51c6a0d4370439107db0d0f25bbaa5447b09 Mon Sep 17 00:00:00 2001 From: Tian Xin Date: Tue, 7 Jul 2026 14:30:00 +0800 Subject: [PATCH 05/13] libxscale: Add queue pair (QP) management verbs Implement the core QP lifecycle operations: - create_qp / create_qp_ex: Allocate QP buffers (SQ and RQ), calculate WQE sizes based on device capabilities, and initialize QP indices. Supports raw packet QPs and RC QPs. - modify_qp: Transition QP states (RTR, RTS, etc.) and handle error state transitions, including flushing pending work requests. - query_qp: Retrieve current QP attributes. - destroy_qp: Clean up QP resources, remove from the QP table, and flush any remaining CQEs. A global QP table is maintained to map QP numbers to internal structures for fast lookups during completion processing. Error state handling ensures that CQs are notified when a QP enters the error state. Signed-off-by: Tian Xin Signed-off-by: Wei Honggang Signed-off-by: Zhao Qianwei Signed-off-by: Li Qiang Signed-off-by: Yan Lei --- kernel-headers/rdma/xsc-abi.h | 38 +- kernel-headers/rdma/xsc_user_ioctl_cmds.h | 164 +++++ kernel-headers/rdma/xsc_user_ioctl_verbs.h | 29 + providers/xscale/CMakeLists.txt | 9 +- providers/xscale/cq.c | 99 ++- providers/xscale/qp.c | 133 ++++ providers/xscale/verbs.c | 810 +++++++++++++++++++++ providers/xscale/xsc-abi.h | 13 + providers/xscale/xsc_api.h | 35 + providers/xscale/xscale.c | 16 +- providers/xscale/xscale.h | 181 ++++- providers/xscale/xscdv.h | 33 + 12 files changed, 1543 insertions(+), 17 deletions(-) create mode 100644 kernel-headers/rdma/xsc_user_ioctl_cmds.h create mode 100644 kernel-headers/rdma/xsc_user_ioctl_verbs.h create mode 100644 providers/xscale/qp.c create mode 100644 providers/xscale/xsc_api.h diff --git a/kernel-headers/rdma/xsc-abi.h b/kernel-headers/rdma/xsc-abi.h index ebb519365..ada746bfa 100644 --- a/kernel-headers/rdma/xsc-abi.h +++ b/kernel-headers/rdma/xsc-abi.h @@ -20,7 +20,7 @@ struct xsc_ib_alloc_ucontext_resp { __u32 cache_line_size; __u16 max_sq_desc_sz; __u16 max_rq_desc_sz; - __u32 max_send_wr; + __u32 max_send_wqebb; __u32 max_recv_wr; __u16 num_ports; __u16 device_id; @@ -58,6 +58,42 @@ struct xsc_ib_resize_cq { __u32 reserved1; }; +struct xsc_ib_create_qp { + __aligned_u64 buf_addr; + __aligned_u64 db_addr; + __u32 sq_wqe_count; + __u32 rq_wqe_count; + __u32 rq_wqe_shift; + __u32 flags; + + int dmabuf_fd; + size_t dmabuf_sz; +}; + +struct xsc_ib_create_qp_resp { + __u32 bfreg_index; + __u32 resv; +}; + +struct xsc_ib_burst_info { + __u32 max_burst_sz; + __u16 typical_pkt_sz; + __u16 reserved; +}; + +struct xsc_ib_modify_qp { + __u32 comp_mask; + struct xsc_ib_burst_info burst_info; + __u32 profile; +}; + +struct xsc_ib_modify_qp_resp { + __u32 response_length; + __u32 dctn; + __u32 profile; + __u32 reserved; +}; + enum xsc_ib_mmap_cmd { XSC_IB_MMAP_REGULAR_PAGE = 0, XSC_IB_MMAP_GET_CONTIGUOUS_PAGES = 1, diff --git a/kernel-headers/rdma/xsc_user_ioctl_cmds.h b/kernel-headers/rdma/xsc_user_ioctl_cmds.h new file mode 100644 index 000000000..fe0e3c8f6 --- /dev/null +++ b/kernel-headers/rdma/xsc_user_ioctl_cmds.h @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_USER_IOCTL_CMDS_H +#define XSC_USER_IOCTL_CMDS_H + +#include +#include + +enum xsc_ib_create_flow_action_attrs { + /* This attribute belong to the driver namespace */ + XSC_IB_ATTR_CREATE_FLOW_ACTION_FLAGS = (1U << UVERBS_ID_NS_SHIFT), +}; + +enum xsc_ib_alloc_dm_attrs { + XSC_IB_ATTR_ALLOC_DM_RESP_START_OFFSET = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX, +}; + +enum xsc_ib_devx_methods { + XSC_IB_METHOD_DEVX_OTHER = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_METHOD_DEVX_QUERY_UAR, + XSC_IB_METHOD_DEVX_QUERY_EQN, +}; + +enum xsc_ib_devx_other_attrs { + XSC_IB_ATTR_DEVX_OTHER_CMD_IN = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_ATTR_DEVX_OTHER_CMD_OUT, +}; + +enum xsc_ib_devx_obj_create_attrs { + XSC_IB_ATTR_DEVX_OBJ_CREATE_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN, + XSC_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, +}; + +enum xsc_ib_devx_query_uar_attrs { + XSC_IB_ATTR_DEVX_QUERY_UAR_USER_IDX = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX, +}; + +enum xsc_ib_devx_obj_destroy_attrs { + XSC_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE = (1U << UVERBS_ID_NS_SHIFT), +}; + +enum xsc_ib_devx_obj_modify_attrs { + XSC_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN, + XSC_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT, +}; + +enum xsc_ib_devx_obj_query_attrs { + XSC_IB_ATTR_DEVX_OBJ_QUERY_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN, + XSC_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT, +}; + +enum xsc_ib_devx_query_eqn_attrs { + XSC_IB_ATTR_DEVX_QUERY_EQN_USER_VEC = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN, +}; + +enum xsc_ib_devx_obj_methods { + XSC_IB_METHOD_DEVX_OBJ_CREATE = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_METHOD_DEVX_OBJ_DESTROY, + XSC_IB_METHOD_DEVX_OBJ_MODIFY, + XSC_IB_METHOD_DEVX_OBJ_QUERY, +}; + +enum xsc_ib_devx_umem_reg_attrs { + XSC_IB_ATTR_DEVX_UMEM_REG_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_ATTR_DEVX_UMEM_REG_ADDR, + XSC_IB_ATTR_DEVX_UMEM_REG_LEN, + XSC_IB_ATTR_DEVX_UMEM_REG_ACCESS, + XSC_IB_ATTR_DEVX_UMEM_REG_OUT_ID, +}; + +enum xsc_ib_devx_umem_dereg_attrs { + XSC_IB_ATTR_DEVX_UMEM_DEREG_HANDLE = (1U << UVERBS_ID_NS_SHIFT), +}; + +enum xsc_ib_devx_umem_methods { + XSC_IB_METHOD_DEVX_UMEM_REG = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_METHOD_DEVX_UMEM_DEREG, +}; + +enum xsc_ib_objects { + XSC_IB_OBJECT_DEVX = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_OBJECT_DEVX_OBJ, + XSC_IB_OBJECT_DEVX_UMEM, + XSC_IB_OBJECT_FLOW_MATCHER, +}; + +enum xsc_ib_flow_matcher_create_attrs { + XSC_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_ATTR_FLOW_MATCHER_MATCH_MASK, + XSC_IB_ATTR_FLOW_MATCHER_FLOW_TYPE, + XSC_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA, + XSC_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, +}; + +enum xsc_ib_flow_matcher_destroy_attrs { + XSC_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE = (1U << UVERBS_ID_NS_SHIFT), +}; + +enum xsc_ib_flow_matcher_methods { + XSC_IB_METHOD_FLOW_MATCHER_CREATE = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_METHOD_FLOW_MATCHER_DESTROY, +}; + +#define XSC_IB_DW_MATCH_PARAM 0x80 + +struct xsc_ib_match_params { + __u32 match_params[XSC_IB_DW_MATCH_PARAM]; +}; + +enum xsc_ib_flow_type { + XSC_IB_FLOW_TYPE_NORMAL, + XSC_IB_FLOW_TYPE_SNIFFER, + XSC_IB_FLOW_TYPE_ALL_DEFAULT, + XSC_IB_FLOW_TYPE_MC_DEFAULT, +}; + +enum xsc_ib_create_flow_attrs { + XSC_IB_ATTR_CREATE_FLOW_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_ATTR_CREATE_FLOW_MATCH_VALUE, + XSC_IB_ATTR_CREATE_FLOW_DEST_QP, + XSC_IB_ATTR_CREATE_FLOW_DEST_DEVX, + XSC_IB_ATTR_CREATE_FLOW_MATCHER, + XSC_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, + XSC_IB_ATTR_CREATE_FLOW_TAG, +}; + +enum xsc_ib_destoy_flow_attrs { + XSC_IB_ATTR_DESTROY_FLOW_HANDLE = (1U << UVERBS_ID_NS_SHIFT), +}; + +enum xsc_ib_flow_methods { + XSC_IB_METHOD_CREATE_FLOW = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_METHOD_DESTROY_FLOW, +}; + +enum xsc_ib_flow_action_methods { + XSC_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT, +}; + +enum xsc_ib_create_flow_action_create_modify_header_attrs { + XSC_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, + XSC_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE, +}; + +enum xsc_ib_create_flow_action_create_packet_reformat_attrs { + XSC_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE, + XSC_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE, + XSC_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF, +}; + +#endif diff --git a/kernel-headers/rdma/xsc_user_ioctl_verbs.h b/kernel-headers/rdma/xsc_user_ioctl_verbs.h new file mode 100644 index 000000000..fee13cc73 --- /dev/null +++ b/kernel-headers/rdma/xsc_user_ioctl_verbs.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_USER_IOCTL_VERBS_H +#define XSC_USER_IOCTL_VERBS_H + +#include + +enum xsc_ib_uapi_flow_action_flags { + XSC_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA = 1 << 0, +}; + +enum xsc_ib_uapi_flow_table_type { + XSC_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX = 0x0, + XSC_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX = 0x1, +}; + +enum xsc_ib_uapi_flow_action_packet_reformat_type { + XSC_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 = 0x0, + XSC_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL = 0x1, + XSC_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 = 0x2, + XSC_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL = 0x3, +}; + +#endif + diff --git a/providers/xscale/CMakeLists.txt b/providers/xscale/CMakeLists.txt index 05710d500..046eaa04f 100644 --- a/providers/xscale/CMakeLists.txt +++ b/providers/xscale/CMakeLists.txt @@ -3,8 +3,15 @@ rdma_shared_provider(xscale libxsc.map xscale.c verbs.c cq.c + qp.c buf.c ) -rdma_pkg_config("xscale" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}") +publish_headers(infiniband + ../../kernel-headers/rdma/xsc_user_ioctl_verbs.h + ../../kernel-headers/rdma/xsc_user_ioctl_cmds.h + xsc_api.h + xscdv.h +) +rdma_pkg_config("xscale" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}") diff --git a/providers/xscale/cq.c b/providers/xscale/cq.c index 0ffe2bcfe..4d864d419 100644 --- a/providers/xscale/cq.c +++ b/providers/xscale/cq.c @@ -17,13 +17,108 @@ #include #include "xscale.h" -#include "xsc_hsi.h" +#include "xsc_hw.h" + +static void *get_cqe(struct xsc_cq *cq, int n) +{ + return cq->active_buf->buf + n * cq->cqe_sz; +} + +static void *get_sw_cqe(struct xsc_cq *cq, int n) +{ + int cid = n & (cq->verbs_cq.cq_ex.cqe - 1); + struct xsc_cqe *cqe = get_cqe(cq, cid); + + if (likely(xsc_get_cqe_sw_own(cqe, n, cq->log2_cq_ring_sz))) + return cqe; + else + return NULL; +} + +static void update_cons_index(struct xsc_cq *cq) +{ + struct xsc_context *ctx = to_xctx(ibv_cq_ex_to_cq(&cq->verbs_cq.cq_ex)->context); + + xsc_hw_set_cq_ci(ctx->device_id, cq->db, cq->cqn, cq->cons_index); +} void xsc_cq_fill_pfns(struct xsc_cq *cq, const struct ibv_cq_init_attr_ex *cq_attr) { } -static int xsc_use_huge(const char *key) +static int is_equal_rsn(struct xsc_cqe *cqe, uint32_t rsn) +{ + uint32_t qp_id = FIELD_GET(XSC_CQE_QP_ID_MASK, cqe->data1); + + qp_id = RD_LE_16(qp_id); + return rsn == qp_id; +} + +static inline int free_res_cqe(struct xsc_cqe *cqe, uint32_t rsn) +{ + if (is_equal_rsn(cqe, rsn)) + return 1; + + return 0; +} + +void __xsc_cq_clean(struct xsc_cq *cq, uint32_t rsn) +{ + uint32_t prod_index; + int nfreed = 0; + struct xsc_cqe *cqe, *dest; + uint8_t owner_bit; + + if (!cq || cq->flags & XSC_CQ_FLAGS_DV_OWNED) + return; + xsc_dbg(to_xctx(cq->verbs_cq.cq_ex.context)->dbg_fp, XSC_DBG_CQ, "\n"); + + /* + * First we need to find the current producer index, so we + * know where to start cleaning from. It doesn't matter if HW + * adds new entries after this loop -- the QP we're worried + * about is already in RESET, so the new entries won't come + * from our QP and therefore don't need to be checked. + */ + for (prod_index = cq->cons_index; get_sw_cqe(cq, prod_index); ++prod_index) + if (prod_index == cq->cons_index + cq->verbs_cq.cq_ex.cqe) + break; + + /* + * Now sweep backwards through the CQ, removing CQ entries + * that match our QP by copying older entries on top of them. + */ + while ((int) --prod_index - (int) cq->cons_index >= 0) { + cqe = get_cqe(cq, prod_index & (cq->verbs_cq.cq_ex.cqe - 1)); + if (free_res_cqe(cqe, rsn)) { + ++nfreed; + } else if (nfreed) { + dest = get_cqe(cq, (prod_index + nfreed) & (cq->verbs_cq.cq_ex.cqe - 1)); + owner_bit = FIELD_GET(XSC_CQE_OWNER_MASK, dest->data3); + memcpy(dest, cqe, cq->cqe_sz); + dest->data3 |= FIELD_PREP(XSC_CQE_OWNER_MASK, owner_bit); + } + } + + if (nfreed) { + cq->cons_index += nfreed; + /* + * Make sure update of buffer contents is done before + * updating consumer index. + */ + udma_to_device_barrier(); + update_cons_index(cq); + } +} + +void xsc_cq_clean(struct xsc_cq *cq, uint32_t qpn) +{ + xsc_spin_lock(&cq->lock); + __xsc_cq_clean(cq, qpn); + xsc_spin_unlock(&cq->lock); +} + +int xsc_use_huge(const char *key) { char *e; diff --git a/providers/xscale/qp.c b/providers/xscale/qp.c new file mode 100644 index 000000000..a8e9ffbfa --- /dev/null +++ b/providers/xscale/qp.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include "xscale.h" +#include "xsc_hw.h" + +int xsc_qp_fill_wr_pfns(struct xsc_context *ctx, + struct xsc_qp *xqp, + const struct ibv_qp_init_attr_ex *attr) +{ + return 0; +} + +void xsc_init_qp_indices(struct xsc_qp *qp) +{ + qp->sq.head = 0; + qp->sq.tail = 0; + qp->rq.head = 0; + qp->rq.tail = 0; + qp->sq.cur_post = 0; + qp->has_trig_cq_evt = false; +} + +struct xsc_qp *xsc_find_qp(struct xsc_context *ctx, uint32_t qpn) +{ + int tind = qpn >> XSC_QP_TABLE_SHIFT; + + if (ctx->qp_table[tind].refcnt) + return ctx->qp_table[tind].table[qpn & XSC_QP_TABLE_MASK]; + else + return NULL; +} + +int xsc_store_qp(struct xsc_context *ctx, uint32_t qpn, struct xsc_qp *qp) +{ + int tind = qpn >> XSC_QP_TABLE_SHIFT; + + if (!ctx->qp_table[tind].refcnt) { + ctx->qp_table[tind].table = calloc(XSC_QP_TABLE_MASK + 1, + sizeof(struct xsc_qp *)); + if (!ctx->qp_table[tind].table) + return -1; + } + + ++ctx->qp_table[tind].refcnt; + ctx->qp_table[tind].table[qpn & XSC_QP_TABLE_MASK] = qp; + return 0; +} + +void xsc_clear_qp(struct xsc_context *ctx, uint32_t qpn) +{ + int tind = qpn >> XSC_QP_TABLE_SHIFT; + + if (!--ctx->qp_table[tind].refcnt) + free(ctx->qp_table[tind].table); + else + ctx->qp_table[tind].table[qpn & XSC_QP_TABLE_MASK] = NULL; +} + +int xsc_err_state_qp(struct ibv_qp *qp, enum ibv_qp_state cur_state, + enum ibv_qp_state state) +{ + struct xsc_err_state_qp_node *tmp, *err_rq_node, *err_sq_node; + struct xsc_qp *xqp = to_xqp(qp); + struct ibv_qp_attr attr = {0}; + int attr_mask = 0; + struct ibv_qp_init_attr init_attr = {0}; + + xsc_dbg(to_xctx(qp->context)->dbg_fp, XSC_DBG_QP, + "modify qp: qpid %d, cur_qp_state %d, qp_state %d\n", + xqp->rsc.rsn, cur_state, state); + if (cur_state == IBV_QPS_ERR && state != IBV_QPS_ERR) { + if (qp->recv_cq) { + list_for_each_safe(&to_xcq(qp->recv_cq)->err_state_qp_list, + err_rq_node, tmp, entry) { + if (err_rq_node->qp_id == xqp->rsc.rsn) { + list_del(&err_rq_node->entry); + free(err_rq_node); + } + } + } + + if (qp->send_cq) { + list_for_each_safe(&to_xcq(qp->send_cq)->err_state_qp_list, + err_sq_node, tmp, entry) { + if (err_sq_node->qp_id == xqp->rsc.rsn) { + list_del(&err_sq_node->entry); + free(err_sq_node); + } + } + } + return 0; + } + + if (cur_state != IBV_QPS_ERR && state == IBV_QPS_ERR && !xqp->has_trig_cq_evt) { + if (qp->recv_cq) { + err_rq_node = calloc(1, sizeof(*err_rq_node)); + if (!err_rq_node) + return ENOMEM; + err_rq_node->qp_id = xqp->rsc.rsn; + err_rq_node->is_sq = false; + list_add_tail(&to_xcq(qp->recv_cq)->err_state_qp_list, &err_rq_node->entry); + } + + if (qp->send_cq) { + err_sq_node = calloc(1, sizeof(*err_sq_node)); + if (!err_sq_node) + return ENOMEM; + err_sq_node->qp_id = xqp->rsc.rsn; + err_sq_node->is_sq = true; + list_add_tail(&to_xcq(qp->send_cq)->err_state_qp_list, &err_sq_node->entry); + + if (qp->send_cq->channel) { + attr_mask |= XSC_QP_FLUSH; + xsc_query_qp(qp, &attr, attr_mask, &init_attr); + xqp->has_trig_cq_evt = true; + } + } + } + return 0; +} diff --git a/providers/xscale/verbs.c b/providers/xscale/verbs.c index e4c20dee9..454e77b04 100644 --- a/providers/xscale/verbs.c +++ b/providers/xscale/verbs.c @@ -29,6 +29,7 @@ #include "xscale.h" #include "xsc-abi.h" #include "xsc_hw.h" +#include "xsc_api.h" int xsc_single_threaded; @@ -434,6 +435,815 @@ int xsc_destroy_cq(struct ibv_cq *cq) return 0; } +static int xsc_calc_sq_size(struct xsc_context *ctx, + struct ibv_qp_init_attr_ex *attr, + struct xsc_qp *qp) +{ + int wqe_size; + int wq_size; + int wq_size_min = 0; + int max_inline_cap; + int max_sge = (ctx->device_id == XSC_MC_PF_DEV_ID_DIAMOND || + ctx->device_id == XSC_MC_PF_DEV_ID_DIAMOND_NEXT) ? 1 : 4; + + if (!attr->cap.max_send_wr) + return 0; + + if (attr->cap.max_send_sge > max_sge) { + xsc_err("max_send_sge:%d exceeded\n", attr->cap.max_send_sge); + return -EINVAL; + } + + wqe_size = 1 << (XSC_BASE_WQE_SHIFT + ctx->send_ds_shift); + + wq_size = xsc_round_up_power_of_two(attr->cap.max_send_wr); + + if (attr->qp_type != IBV_QPT_RAW_PACKET) + wq_size_min = XSC_SEND_WQE_RING_DEPTH_MIN; + if (wq_size < wq_size_min) { + xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, "WQE size %u is not enough, set it as %u\n", + wq_size, wq_size_min); + wq_size = wq_size_min; + } + + if (wq_size > ctx->max_send_wqebb) { + if (ctx->device_id == XSC_MC_PF_DEV_ID_DIAMOND || + ctx->device_id == XSC_MC_PF_DEV_ID_DIAMOND_NEXT) { + xsc_err("WQE size %u exceeds WQE ring depth\n", wq_size); + return -EINVAL; + } + xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, + "WQE size %u exceeds WQE ring depth, set it as %u\n", + wq_size, ctx->max_send_wqebb); + wq_size = ctx->max_send_wqebb; + } + + qp->sq.wqe_cnt = wq_size; + qp->sq.ds_cnt = wq_size << ctx->send_ds_shift; + qp->sq.seg_cnt = 1 << ctx->send_ds_shift; + qp->sq.wqe_shift = XSC_BASE_WQE_SHIFT + ctx->send_ds_shift; + qp->sq.max_gs = attr->cap.max_send_sge; + qp->sq.max_post = qp->sq.wqe_cnt; + + if (ctx->device_id == XSC_MC_PF_DEV_ID_DIAMOND || + ctx->device_id == XSC_MC_PF_DEV_ID_DIAMOND_NEXT) + max_inline_cap = 64; + else + max_inline_cap = (qp->sq.seg_cnt - 2) * sizeof(struct xsc_wqe_data_seg); + if (attr->cap.max_inline_data > max_inline_cap) + return -EINVAL; + qp->max_inline_data = attr->cap.max_inline_data; + + xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, "Send WQE count:%u, max post:%u wqe shift:%u\n", + qp->sq.wqe_cnt, qp->sq.max_post, qp->sq.wqe_shift); + + return wqe_size * qp->sq.wqe_cnt; +} + +enum { + DV_CREATE_WQ_SUPPORTED_COMP_MASK = XSCDV_WQ_INIT_ATTR_MASK_STRIDING_RQ +}; + +static int xsc_calc_rq_size(struct xsc_context *ctx, + struct ibv_qp_init_attr_ex *attr, + struct xsc_qp *qp) +{ + int wqe_size; + int wq_size; + int wq_size_min = 0; + int recv_ds_shift = 0; + + if (!attr->cap.max_recv_wr) + return 0; + + recv_ds_shift = xsc_get_recv_ds_shift(ctx, attr->qp_type); + wqe_size = 1 << (XSC_BASE_WQE_SHIFT + recv_ds_shift); + + wq_size = xsc_round_up_power_of_two(attr->cap.max_recv_wr); + /* due to hardware limit, rdma rq depth should be one send wqe ds num at least*/ + if (attr->qp_type != IBV_QPT_RAW_PACKET) + wq_size_min = ctx->send_ds_num; + if (wq_size < wq_size_min) { + xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, "WQE size %u is not enough, set it as %u\n", + wq_size, wq_size_min); + wq_size = wq_size_min; + } + + if (wq_size > ctx->max_recv_wr) { + if (ctx->device_id == XSC_MC_PF_DEV_ID_DIAMOND || + ctx->device_id == XSC_MC_PF_DEV_ID_DIAMOND_NEXT) { + xsc_err("WQE size %u exceeds WQE ring depth\n", wq_size); + return -EINVAL; + } + xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, + "WQE size %u exceeds WQE ring depth, set it as %u\n", + wq_size, ctx->max_recv_wr); + wq_size = ctx->max_recv_wr; + } + + qp->rq.wqe_cnt = wq_size; + qp->rq.ds_cnt = qp->rq.wqe_cnt << recv_ds_shift; + qp->rq.seg_cnt = 1 << recv_ds_shift; + qp->rq.wqe_shift = XSC_BASE_WQE_SHIFT + recv_ds_shift; + qp->rq.max_post = qp->rq.wqe_cnt; + qp->rq.max_gs = attr->cap.max_recv_sge; + + xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, "Recv WQE count:%u, max post:%u wqe shift:%u\n", + qp->rq.wqe_cnt, qp->rq.max_post, qp->rq.wqe_shift); + return wqe_size * qp->rq.wqe_cnt; +} + +static int xsc_calc_wq_size(struct xsc_context *ctx, + struct ibv_qp_init_attr_ex *attr, + struct xsc_qp *qp) +{ + int ret; + int result; + + ret = xsc_calc_sq_size(ctx, attr, qp); + if (ret < 0) + return ret; + + result = ret; + + ret = xsc_calc_rq_size(ctx, attr, qp); + if (ret < 0) + return ret; + + result += ret; + + qp->sq.offset = ret; + qp->rq.offset = 0; + + return result; +} + +static const char *qptype2key(enum ibv_qp_type type) +{ + switch (type) { + case IBV_QPT_RC: return "HUGE_RC"; + case IBV_QPT_UC: return "HUGE_UC"; + case IBV_QPT_UD: return "HUGE_UD"; + case IBV_QPT_RAW_PACKET: return "HUGE_RAW_ETH"; + default: return "HUGE_NA"; + } +} + +static int xsc_alloc_qp_buf(struct ibv_context *context, + struct ibv_qp_init_attr_ex *attr, + struct xsc_qp *qp, + int size, struct xscdv_devx_umem_in *umem_in) +{ + int err; + enum xsc_alloc_type alloc_type; + enum xsc_alloc_type default_alloc_type = XSC_ALLOC_TYPE_ANON; + const char *qp_huge_key; + + if (qp->sq.wqe_cnt) { + qp->sq.wrid = malloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wrid)); + if (!qp->sq.wrid) { + errno = ENOMEM; + err = -1; + return err; + } + + qp->sq.wr_data = malloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wr_data)); + if (!qp->sq.wr_data) { + errno = ENOMEM; + err = -1; + goto ex_wrid; + } + + qp->sq.wqe_head = malloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wqe_head)); + if (!qp->sq.wqe_head) { + errno = ENOMEM; + err = -1; + goto ex_wrid; + } + + qp->sq.need_flush = malloc(qp->sq.wqe_cnt * sizeof(*qp->sq.need_flush)); + if (!qp->sq.need_flush) { + errno = ENOMEM; + err = -1; + goto ex_wrid; + } + memset(qp->sq.need_flush, 0, qp->sq.wqe_cnt); + + qp->sq.wr_opcode = malloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wr_opcode)); + if (!qp->sq.wr_opcode) { + errno = ENOMEM; + err = -1; + goto ex_wrid; + } + } + + if (qp->rq.wqe_cnt) { + qp->rq.wrid = malloc(qp->rq.wqe_cnt * sizeof(uint64_t)); + if (!qp->rq.wrid) { + errno = ENOMEM; + err = -1; + goto ex_wrid; + } + } + + if (umem_in) { + if (umem_in->comp_mask & XSCDV_UMEM_MASK_DMABUF) { + if (umem_in->dmabuf_fd == -1) { + xsc_dbg(to_xctx(context)->dbg_fp, XSC_DBG_QP, + "dmabuf_fd is invalid :%u\n", umem_in->dmabuf_fd); + err = -ENOMEM; + goto ex_wrid; + } + } + + qp->buf.type = XSC_ALLOC_TYPE_GPU; + qp->buf.buf = umem_in->addr; + qp->buf.length = umem_in->size; + + xsc_dbg(to_xctx(context)->dbg_fp, XSC_DBG_QP, + "qp_buf(%p),qp_buf_size=0x%lx is given by gpu memory, infact need size=0x%x.\n", + qp->buf.buf, qp->buf.length, size); + } else { + /* compatibility support */ + qp_huge_key = qptype2key(qp->ibv_qp->qp_type); + if (xsc_use_huge(qp_huge_key)) + default_alloc_type = XSC_ALLOC_TYPE_HUGE; + + xsc_get_alloc_type(to_xctx(context), XSC_QP_PREFIX, &alloc_type, + default_alloc_type); + + err = xsc_alloc_prefered_buf(to_xctx(context), &qp->buf, + align(qp->buf_size, + to_xdev(context->device)->page_size), + to_xdev(context->device)->page_size, + alloc_type, + XSC_QP_PREFIX); + + if (err) { + err = -ENOMEM; + goto ex_wrid; + } + + memset(qp->buf.buf, 0, qp->buf_size); + + if (attr->qp_type == IBV_QPT_RAW_PACKET || + qp->flags & XSC_QP_FLAGS_USE_UNDERLAY) { + size_t aligned_sq_buf_size = align(qp->sq_buf_size, + to_xdev(context->device)->page_size); + /* For Raw Packet QP, allocate a separate buffer for the SQ */ + err = xsc_alloc_prefered_buf(to_xctx(context), &qp->sq_buf, + aligned_sq_buf_size, + to_xdev(context->device)->page_size, + alloc_type, + XSC_QP_PREFIX); + if (err) { + err = -ENOMEM; + goto rq_buf; + } + + memset(qp->sq_buf.buf, 0, aligned_sq_buf_size); + } + } + + return 0; +rq_buf: + xsc_free_actual_buf(to_xctx(context), &qp->buf); +ex_wrid: + if (qp->rq.wrid) + free(qp->rq.wrid); + + if (qp->sq.wqe_head) + free(qp->sq.wqe_head); + + if (qp->sq.wr_data) + free(qp->sq.wr_data); + if (qp->sq.wrid) + free(qp->sq.wrid); + + if (qp->sq.need_flush) + free(qp->sq.need_flush); + + if (qp->sq.wr_opcode) + free(qp->sq.wr_opcode); + + return err; +} + +static void xsc_free_qp_buf(struct xsc_context *ctx, struct xsc_qp *qp) +{ + if (qp->buf.type != XSC_ALLOC_TYPE_GPU) { + xsc_free_actual_buf(ctx, &qp->buf); + + if (qp->sq_buf.buf) + xsc_free_actual_buf(ctx, &qp->sq_buf); + } + + if (qp->rq.wrid) + free(qp->rq.wrid); + + if (qp->sq.wqe_head) + free(qp->sq.wqe_head); + + if (qp->sq.wrid) + free(qp->sq.wrid); + + if (qp->sq.wr_data) + free(qp->sq.wr_data); + + if (qp->sq.need_flush) + free(qp->sq.need_flush); + + if (qp->sq.wr_opcode) + free(qp->sq.wr_opcode); +} + +enum { + XSC_CREATE_QP_SUP_COMP_MASK = (IBV_QP_INIT_ATTR_PD | + IBV_QP_INIT_ATTR_CREATE_FLAGS | + IBV_QP_INIT_ATTR_SEND_OPS_FLAGS | + IBV_QP_INIT_ATTR_MAX_TSO_HEADER), +}; + +enum { + XSC_DV_CREATE_QP_SUP_COMP_MASK = XSCDV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS | + XSCDV_QP_INIT_ATTR_MASK_DC +}; + +enum { + XSC_CREATE_QP_EX2_COMP_MASK = (IBV_QP_INIT_ATTR_CREATE_FLAGS | + IBV_QP_INIT_ATTR_MAX_TSO_HEADER | + IBV_QP_INIT_ATTR_IND_TABLE | + IBV_QP_INIT_ATTR_RX_HASH), +}; + +enum { + XSCDV_QP_CREATE_SUP_FLAGS = + (XSCDV_QP_CREATE_TUNNEL_OFFLOADS | + XSCDV_QP_CREATE_TIR_ALLOW_SELF_LOOPBACK_UC | + XSCDV_QP_CREATE_TIR_ALLOW_SELF_LOOPBACK_MC | + XSCDV_QP_CREATE_DISABLE_SCATTER_TO_CQE | + XSCDV_QP_CREATE_ALLOW_SCATTER_TO_CQE), +}; + +static int xsc_cmd_create_qp_ex(struct ibv_context *context, + struct ibv_qp_init_attr_ex *attr, + struct xsc_create_qp *cmd, + struct xsc_qp *qp, + struct xsc_create_qp_resp *resp, + struct xsc_create_qp_ex_resp *resp_ex) +{ + struct xsc_create_qp_ex cmd_ex; + int ret; + + if (attr->comp_mask & XSC_CREATE_QP_EX2_COMP_MASK) { + memset(&cmd_ex, 0, sizeof(cmd_ex)); + *ibv_create_qp_ex_to_reg(&cmd_ex.ibv_cmd) = cmd->ibv_cmd.core_payload; + cmd_ex.drv_payload = cmd->drv_payload; + + ret = ibv_cmd_create_qp_ex2(context, &qp->verbs_qp, + attr, &cmd_ex.ibv_cmd, + sizeof(cmd_ex), &resp_ex->ibv_resp, + sizeof(*resp_ex), NULL); + } else { + ret = ibv_cmd_create_qp_ex(context, &qp->verbs_qp, attr, + &cmd->ibv_cmd, sizeof(*cmd), + &resp->ibv_resp, sizeof(*resp)); + } + + return ret; +} + +static struct ibv_qp *create_qp(struct ibv_context *context, + struct ibv_qp_init_attr_ex *attr, + struct xscdv_qp_init_attr *xqp_attr, + struct xscdv_devx_umem_in *umem_in) +{ + struct xsc_create_qp cmd; + struct xsc_create_qp_resp resp; + struct xsc_create_qp_ex_resp resp_ex; + struct xsc_qp *qp; + int ret; + struct xsc_context *ctx = to_xctx(context); + struct ibv_qp *ibqp; + struct xsc_parent_domain *xparent_domain; + struct xsc_device *xdev = to_xdev(context->device); + + xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, "comp_mask=0x%x.\n", attr->comp_mask); + + if (attr->comp_mask & ~XSC_CREATE_QP_SUP_COMP_MASK) { + xsc_err("Not supported comp_mask:0x%x\n", attr->comp_mask); + return NULL; + } + + /*check qp_type*/ + if ((attr->qp_type != IBV_QPT_RC) && (attr->qp_type != IBV_QPT_RAW_PACKET)) { + xsc_err("Not supported qp_type:0x%x\n", attr->qp_type); + return NULL; + } + + qp = calloc(1, sizeof(*qp)); + if (!qp) { + xsc_err("QP calloc failed\n"); + return NULL; + } + + ibqp = &qp->verbs_qp.qp; + qp->ibv_qp = ibqp; + + memset(&cmd, 0, sizeof(cmd)); + memset(&resp, 0, sizeof(resp)); + memset(&resp_ex, 0, sizeof(resp_ex)); + + if (attr->comp_mask & IBV_QP_INIT_ATTR_SEND_OPS_FLAGS) { + ret = xsc_qp_fill_wr_pfns(ctx, qp, attr); + if (ret) { + errno = ret; + xsc_err("Fill wr pfns failed\n"); + goto err; + } + } + + ret = xsc_calc_wq_size(ctx, attr, qp); + if (ret < 0) { + xsc_err("Calculate WQ size failed\n"); + errno = EINVAL; + goto err; + } + + qp->buf_size = ret; + qp->sq_buf_size = 0; + + if (xsc_alloc_qp_buf(context, attr, qp, ret, umem_in)) { + xsc_err("Alloc QP buffer failed\n"); + errno = ENOMEM; + goto err; + } + + qp->sq_start = qp->buf.buf + qp->sq.offset; + qp->rq_start = qp->buf.buf + qp->rq.offset; + qp->sq.qend = qp->buf.buf + qp->sq.offset + + (qp->sq.wqe_cnt << qp->sq.wqe_shift); + + xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, "sq start:%p, sq qend:%p, buffer size:%u\n", + qp->sq_start, qp->sq.qend, qp->buf_size); + + xsc_init_qp_indices(qp); + + if (xsc_spinlock_init_pd(&qp->sq.lock, attr->pd) || + xsc_spinlock_init_pd(&qp->rq.lock, attr->pd)) + goto err_free_qp_buf; + + cmd.buf_addr = (uintptr_t) qp->buf.buf; + cmd.db_addr = (uintptr_t) qp->db; + cmd.sq_wqe_count = qp->sq.ds_cnt; + cmd.rq_wqe_count = qp->rq.ds_cnt; + cmd.rq_wqe_shift = qp->rq.wqe_shift; + + if (qp->buf.type == XSC_ALLOC_TYPE_GPU) { + qp->flags |= XSC_QP_FLAG_OWNED_BY_GPU; + cmd.dmabuf_fd = umem_in->dmabuf_fd; + cmd.dmabuf_sz = umem_in->size; + xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, "wq_buf %p, buffer size:0x%lx, dmabuf_fd:%d\n", + umem_in->addr, umem_in->size, umem_in->dmabuf_fd); + } else { + cmd.dmabuf_fd = -1; + cmd.dmabuf_sz = 0; + } + + if (attr->qp_type == IBV_QPT_RAW_PACKET) { + if (attr->comp_mask & IBV_QP_INIT_ATTR_CREATE_FLAGS) { + if (attr->create_flags & XSC_QP_CREATE_RAWPACKET_TSO) { + cmd.flags |= XSC_QP_FLAG_RAWPACKET_TSO;/*revert to command flags*/ + xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, + "revert create_flags(0x%x) to cmd_flags(0x%x)\n", + attr->create_flags, cmd.flags); + } + + if (attr->create_flags & XSC_QP_CREATE_RAWPACKET_TX) { + cmd.flags |= XSC_QP_FLAG_RAWPACKET_TX;/*revert to command flags*/ + xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, + "revert create_flags(0x%x) to cmd_flags(0x%x)\n", + attr->create_flags, cmd.flags); + } + + if (attr->create_flags & XSC_QP_CREATE_RAWPACKET_SNIFFER) { + cmd.flags |= XSC_QP_FLAG_RAWPACKET_SNIFFER; + qp->flags |= XSC_QP_FLAG_RAWPACKET_SNIFFER; + xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, + "revert create_flags(0x%x) to cmd_flags(0x%x)\n", + attr->create_flags, cmd.flags); + } + + attr->comp_mask &= ~IBV_QP_INIT_ATTR_CREATE_FLAGS; + } + + if (attr->comp_mask & IBV_QP_INIT_ATTR_MAX_TSO_HEADER) + cmd.flags |= XSC_QP_FLAG_RAWPACKET_TSO; + + } else if (attr->qp_type == IBV_QPT_RC) { + if (attr->comp_mask & IBV_QP_INIT_ATTR_CREATE_FLAGS) { + if (attr->create_flags & XSC_QP_CREATE_WORC) { + cmd.flags |= XSC_QP_FLAG_WORC; + xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, + "revert create_flags(0x%x) to cmd_flags(0x%x)\n", + attr->create_flags, cmd.flags); + } + attr->comp_mask &= ~IBV_QP_INIT_ATTR_CREATE_FLAGS; + } + } + + pthread_mutex_lock(&ctx->qp_table_mutex); + + xparent_domain = to_xparent_domain(attr->pd); + + ret = xsc_cmd_create_qp_ex(context, attr, &cmd, qp, &resp, &resp_ex); + if (ret) { + xsc_err("ibv_cmd_create_qp_ex failed,ret %d\n", ret); + errno = ret; + goto err_free_uidx; + } + + if (qp->sq.wqe_cnt || qp->rq.wqe_cnt) { + ret = xsc_store_qp(ctx, ibqp->qp_num, qp); + if (ret) { + xsc_err("xsc_store_qp failed,ret %d\n", ret); + errno = EINVAL; + goto err_destroy; + } + } + + pthread_mutex_unlock(&ctx->qp_table_mutex); + + qp->rq.max_post = qp->rq.wqe_cnt; + + if (attr->sq_sig_all) + qp->sq_signal_bits = 1; + else + qp->sq_signal_bits = 0; + + if (ctx->atomic_cap) + qp->atomics_enabled = 1; + + qp->get_ece = (ctx->rdma_proto_mode == RDMA_PROTO_VEROCE) ? VEROCE_PROFILE_P3 : 0; + qp->profile = XSC_VEROCE_INIT_PROFILE; + + attr->cap.max_send_wr = qp->sq.max_post; + attr->cap.max_recv_wr = qp->rq.max_post; + attr->cap.max_recv_sge = qp->rq.max_gs; + + qp->rsc.type = XSC_RSC_TYPE_QP; + qp->rsc.rsn = ibqp->qp_num; + + if (xparent_domain) + atomic_fetch_add(&xparent_domain->xpd.refcount, 1); + + qp->rqn = ibqp->qp_num; + qp->sqn = ibqp->qp_num; + + xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, "qp rqn:%u, sqn:%u\n", qp->rqn, qp->sqn); + if (ctx->multidb_num && ctx->device_id != XSC_MC_PF_DEV_ID_DIAMOND) { + pthread_mutex_lock(&context->mutex); + qp->sq.db = ctx->mdb_base + ctx->tx_multidb_base + + (ctx->tx_mdb_idx & (ctx->multidb_num - 1)) * sizeof(uint32_t); + ctx->tx_mdb_idx++; + pthread_mutex_unlock(&context->mutex); + } else { + qp->sq.db = ctx->sqm_reg_va + (ctx->qpm_tx_db & (xdev->page_size - 1)); + } + qp->rq.db = ctx->rqm_reg_va + (ctx->qpm_rx_db & (xdev->page_size - 1)); + + if (attr->comp_mask & IBV_QP_INIT_ATTR_SEND_OPS_FLAGS) + qp->verbs_qp.comp_mask |= VERBS_QP_EX; + + return ibqp; + +err_destroy: + ibv_cmd_destroy_qp(ibqp); + +err_free_uidx: + pthread_mutex_unlock(&to_xctx(context)->qp_table_mutex); + +err_free_qp_buf: + xsc_free_qp_buf(ctx, qp); + +err: + free(qp); + + return NULL; +} + +struct ibv_qp *xsc_create_qp(struct ibv_pd *pd, + struct ibv_qp_init_attr *attr) +{ + struct ibv_qp *qp; + struct ibv_qp_init_attr_ex attrx; + + memset(&attrx, 0, sizeof(attrx)); + memcpy(&attrx, attr, sizeof(*attr)); + attrx.comp_mask = IBV_QP_INIT_ATTR_PD; + attrx.pd = pd; + qp = create_qp(pd->context, &attrx, NULL, NULL); + if (qp) + memcpy(attr, &attrx, sizeof(*attr)); + + return qp; +} + +struct ibv_qp *xsc_create_qp_ex(struct ibv_context *context, + struct ibv_qp_init_attr_ex *attr) +{ + return create_qp(context, attr, NULL, NULL); +} + +static void xsc_lock_cqs(struct ibv_qp *qp) +{ + struct xsc_cq *send_cq = to_xcq(qp->send_cq); + struct xsc_cq *recv_cq = to_xcq(qp->recv_cq); + + if (send_cq && recv_cq) { + if (send_cq == recv_cq) { + xsc_spin_lock(&send_cq->lock); + } else if (send_cq->cqn < recv_cq->cqn) { + xsc_spin_lock(&send_cq->lock); + xsc_spin_lock(&recv_cq->lock); + } else { + xsc_spin_lock(&recv_cq->lock); + xsc_spin_lock(&send_cq->lock); + } + } else if (send_cq) { + xsc_spin_lock(&send_cq->lock); + } else if (recv_cq) { + xsc_spin_lock(&recv_cq->lock); + } +} + +static void xsc_unlock_cqs(struct ibv_qp *qp) +{ + struct xsc_cq *send_cq = to_xcq(qp->send_cq); + struct xsc_cq *recv_cq = to_xcq(qp->recv_cq); + + if (send_cq && recv_cq) { + if (send_cq == recv_cq) { + xsc_spin_unlock(&send_cq->lock); + } else if (send_cq->cqn < recv_cq->cqn) { + xsc_spin_unlock(&recv_cq->lock); + xsc_spin_unlock(&send_cq->lock); + } else { + xsc_spin_unlock(&send_cq->lock); + xsc_spin_unlock(&recv_cq->lock); + } + } else if (send_cq) { + xsc_spin_unlock(&send_cq->lock); + } else if (recv_cq) { + xsc_spin_unlock(&recv_cq->lock); + } +} + +int xsc_destroy_qp(struct ibv_qp *ibqp) +{ + struct xsc_qp *qp = to_xqp(ibqp); + struct xsc_context *ctx = to_xctx(ibqp->context); + int ret; + struct xsc_parent_domain *xparent_domain = to_xparent_domain(ibqp->pd); + struct xsc_err_state_qp_node *tmp, *err_rq_node, *err_sq_node; + + xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, "\n"); + + pthread_mutex_lock(&ctx->qp_table_mutex); + + ret = ibv_cmd_destroy_qp(ibqp); + if (ret) { + pthread_mutex_unlock(&ctx->qp_table_mutex); + return ret; + } + + xsc_lock_cqs(ibqp); + + list_for_each_safe(&to_xcq(ibqp->recv_cq)->err_state_qp_list, err_rq_node, tmp, entry) { + if (err_rq_node->qp_id == qp->rsc.rsn) { + list_del(&err_rq_node->entry); + free(err_rq_node); + } + } + + list_for_each_safe(&to_xcq(ibqp->send_cq)->err_state_qp_list, err_sq_node, tmp, entry) { + if (err_sq_node->qp_id == qp->rsc.rsn) { + list_del(&err_sq_node->entry); + free(err_sq_node); + } + } + + if (!(qp->flags & XSC_QP_FLAG_OWNED_BY_GPU)) { + __xsc_cq_clean(to_xcq(ibqp->recv_cq), qp->rsc.rsn); + if (ibqp->send_cq != ibqp->recv_cq) + __xsc_cq_clean(to_xcq(ibqp->send_cq), qp->rsc.rsn); + } + + if (qp->sq.wqe_cnt || qp->rq.wqe_cnt) + xsc_clear_qp(ctx, ibqp->qp_num); + + xsc_unlock_cqs(ibqp); + pthread_mutex_unlock(&ctx->qp_table_mutex); + + xsc_free_qp_buf(ctx, qp); + + if (xparent_domain) + atomic_fetch_sub(&xparent_domain->xpd.refcount, 1); + + free(qp); + + return 0; +} + +int xsc_query_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr, + int attr_mask, struct ibv_qp_init_attr *init_attr) +{ + struct ibv_query_qp cmd; + struct xsc_qp *qp = to_xqp(ibqp); + int ret; + + xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP, "\n"); + + if (qp->rss_qp) + return EINVAL; + + ret = ibv_cmd_query_qp(ibqp, attr, attr_mask, init_attr, &cmd, sizeof(cmd)); + if (ret) + return ret; + + init_attr->cap.max_send_wr = qp->sq.max_post; + init_attr->cap.max_send_sge = qp->sq.max_gs; + init_attr->cap.max_inline_data = qp->max_inline_data; + + attr->cap = init_attr->cap; + if (qp->err_occurred) { + qp->err_occurred = 0; + qp->ibv_qp->state = IBV_QPS_ERR; + attr->qp_state = IBV_QPS_ERR; + } + + return 0; +} + +enum { + XSC_MODIFY_QP_EX_ATTR_MASK = IBV_QP_RATE_LIMIT, +}; + +int xsc_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + int attr_mask) +{ + struct ibv_modify_qp cmd = {}; + struct xsc_modify_qp cmd_ex = {}; + struct xsc_modify_qp_ex_resp resp = {}; + struct xsc_qp *xqp = to_xqp(qp); + struct xsc_context *ctx = to_xctx(qp->context); + int ret; + + xsc_dbg(to_xctx(qp->context)->dbg_fp, XSC_DBG_QP, "\n"); + if (!xqp->profile && ctx->rdma_proto_mode == RDMA_PROTO_VEROCE && + attr_mask & IBV_QP_STATE && attr->qp_state == IBV_QPS_RTR) { + cmd_ex.profile = xqp->profile; + ret = ibv_cmd_modify_qp_ex(qp, attr, attr_mask, &cmd_ex.ibv_cmd, + sizeof(cmd_ex), &resp.ibv_resp, + sizeof(resp)); + } else { + ret = ibv_cmd_modify_qp(qp, attr, attr_mask, + &cmd, sizeof(cmd)); + } + + + if (!ret && (attr_mask & IBV_QP_STATE) && + attr->qp_state == IBV_QPS_RESET) { + if (xqp->flags & XSC_QP_FLAG_OWNED_BY_GPU) { + xsc_dbg(to_xctx(qp->context)->dbg_fp, XSC_DBG_QP, + "qp buf is given by GPU, do nothing in host\n"); + } else { + if (qp->recv_cq) + xsc_cq_clean(to_xcq(qp->recv_cq), xqp->rsc.rsn); + + if (qp->send_cq != qp->recv_cq && qp->send_cq) + xsc_cq_clean(to_xcq(qp->send_cq), + to_xqp(qp)->rsc.rsn); + } + + xsc_init_qp_indices(xqp); + } + + if (!ret && (attr_mask & IBV_QP_STATE)) + qp->state = attr->qp_state; + + /*workaround: generate flush err cqe if qp status turns to ERR*/ + if (!ret && (attr_mask & IBV_QP_STATE)) { + xsc_lock_cqs(qp); + ret = xsc_err_state_qp(qp, attr->cur_qp_state, attr->qp_state); + xsc_unlock_cqs(qp); + } + + if (xqp->profile != XSC_VEROCE_INIT_PROFILE) + xqp->get_ece = xqp->profile; + return ret; +} + static void xsc_set_fw_version(struct ibv_device_attr *attr, union xsc_ib_fw_ver *fw_ver) { diff --git a/providers/xscale/xsc-abi.h b/providers/xscale/xsc-abi.h index 8e0baff37..3ac5e0960 100644 --- a/providers/xscale/xsc-abi.h +++ b/providers/xscale/xsc-abi.h @@ -26,5 +26,18 @@ DECLARE_DRV_CMD(xsc_create_cq_ex, IB_USER_VERBS_EX_CMD_CREATE_CQ, xsc_ib_create_cq, xsc_ib_create_cq_resp); DECLARE_DRV_CMD(xsc_resize_cq, IB_USER_VERBS_CMD_RESIZE_CQ, xsc_ib_resize_cq, empty); +DECLARE_DRV_CMD(xsc_create_qp_ex, IB_USER_VERBS_EX_CMD_CREATE_QP, + xsc_ib_create_qp, xsc_ib_create_qp_resp); +DECLARE_DRV_CMD(xsc_create_qp, IB_USER_VERBS_CMD_CREATE_QP, + xsc_ib_create_qp, xsc_ib_create_qp_resp); +DECLARE_DRV_CMD(xsc_modify_qp_ex, IB_USER_VERBS_EX_CMD_MODIFY_QP, + empty, xsc_ib_modify_qp_resp); + +struct xsc_modify_qp { + struct ibv_modify_qp_ex ibv_cmd; + __u32 comp_mask; + struct xsc_ib_burst_info burst_info; + __u32 profile; +}; #endif /* XSC_ABI_H */ diff --git a/providers/xscale/xsc_api.h b/providers/xscale/xsc_api.h new file mode 100644 index 000000000..c2f568eea --- /dev/null +++ b/providers/xscale/xsc_api.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_API_H +#define XSC_API_H + +#include + +#define xscdv_flow_action_flags xsc_ib_uapi_flow_action_flags +#define XSCDV_FLOW_ACTION_FLAGS_REQUIRE_METADATA \ + XSC_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA +#define xscdv_flow_table_type xsc_ib_uapi_flow_table_type +#define XSCDV_FLOW_TABLE_TYPE_NIC_RX XSC_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX +#define XSCDV_FLOW_TABLE_TYPE_NIC_TX XSC_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX +#define xscdv_flow_action_packet_reformat_type xsc_ib_uapi_flow_action_packet_reformat_type +#define XSCDV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 \ + XSC_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 +#define XSCDV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL \ + XSC_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL +#define XSCDV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 \ + XSC_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 +#define XSCDV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL \ + XSC_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL + +enum xsc_qp_create_flags { + XSC_QP_CREATE_RAWPACKET_TSO = 1 << 0, + XSC_QP_CREATE_RAWPACKET_SNIFFER = 1 << 2, + XSC_QP_CREATE_RAWPACKET_TX = 1 << 3, + XSC_QP_CREATE_WORC = 1 << 4, +}; + +#endif diff --git a/providers/xscale/xscale.c b/providers/xscale/xscale.c index fa699bd25..c4a1306e4 100644 --- a/providers/xscale/xscale.c +++ b/providers/xscale/xscale.c @@ -45,6 +45,12 @@ static const struct verbs_context_ops xsc_ctx_common_ops = { .create_cq = xsc_create_cq, .resize_cq = xsc_resize_cq, .destroy_cq = xsc_destroy_cq, + + .create_qp = xsc_create_qp, + .query_qp = xsc_query_qp, + .modify_qp = xsc_modify_qp, + .destroy_qp = xsc_destroy_qp, + .create_qp_ex = xsc_create_qp_ex, }; static void open_debug_file(struct xsc_context *ctx) @@ -208,9 +214,9 @@ static struct verbs_context *xsc_alloc_context(struct ibv_device *ibdev, context->max_num_qps = resp.qp_tab_size; context->max_sq_desc_sz = resp.max_sq_desc_sz; context->max_rq_desc_sz = resp.max_rq_desc_sz; - context->max_send_wr = resp.max_send_wr; - context->num_ports = resp.num_ports; - context->max_recv_wr = resp.max_recv_wr; + context->max_send_wqebb = resp.max_send_wqebb; + context->num_ports = resp.num_ports; + context->max_recv_wr = resp.max_recv_wr; context->qpm_tx_db = resp.qpm_tx_db; context->qpm_rx_db = resp.qpm_rx_db; context->cqm_next_cid_reg = resp.cqm_next_cid_reg; @@ -225,8 +231,8 @@ static struct verbs_context *xsc_alloc_context(struct ibv_device *ibdev, context->max_num_qps, context->max_sq_desc_sz, context->max_rq_desc_sz); xsc_dbg(context->dbg_fp, XSC_DBG_CTX, - "max_send_wr:%u, num_ports:%u, max_recv_wr:%u\n", - context->max_send_wr, + "max_send_wqebb:%u, num_ports:%u, max_recv_wr:%u\n", + context->max_send_wqebb, context->num_ports, context->max_recv_wr); xsc_dbg(context->dbg_fp, XSC_DBG_CTX, "send_ds_num:%u shift:%u recv_ds_num:%u shift:%u\n", diff --git a/providers/xscale/xscale.h b/providers/xscale/xscale.h index 57ac474cf..6e937b8ce 100644 --- a/providers/xscale/xscale.h +++ b/providers/xscale/xscale.h @@ -23,11 +23,23 @@ #include "xsc-abi.h" #include "xscdv.h" +enum xsc_qp_ex_attr_mask { + XSC_QP_FLUSH = 1 << 21, +}; + enum { XSC_IB_MMAP_CMD_SHIFT = 8, XSC_IB_MMAP_CMD_MASK = 0xff, }; +enum { + XSC_QP_FLAG_RAWPACKET_TSO = 1 << 9, + XSC_QP_FLAG_RAWPACKET_TX = 1 << 10, + XSC_QP_FLAG_RAWPACKET_SNIFFER = 1 << 11, + XSC_QP_FLAG_WORC = 1 << 12, + XSC_QP_FLAG_OWNED_BY_GPU = 1 << 13, +}; + #define XSC_CQ_PREFIX "XSC_CQ" #define XSC_QP_PREFIX "XSC_QP" #define XSC_MR_PREFIX "XSC_MR" @@ -98,8 +110,17 @@ enum xsc_alloc_type { XSC_ALLOC_TYPE_ALL }; +enum xsc_rsc_type { + XSC_RSC_TYPE_QP, + XSC_RSC_TYPE_XSRQ, + XSC_RSC_TYPE_SRQ, + XSC_RSC_TYPE_RWQ, + XSC_RSC_TYPE_INVAL, +}; + struct xsc_resource { - uint32_t rsn; + enum xsc_rsc_type type; + uint32_t rsn; }; struct xsc_device { @@ -133,6 +154,18 @@ struct xsc_spinlock { #define XSC_MV_HOST_VF_DEV_ID 0x1152 #define XSC_MV_SOC_PF_DEV_ID 0x1153 +enum { + RDMA_PROTO_ROCEV2, + RDMA_PROTO_VEROCE, +}; + +enum veroce_profile { + VEROCE_PROFILE_P0, + VEROCE_PROFILE_P1, + VEROCE_PROFILE_P2, + VEROCE_PROFILE_P3 +}; + #define NAME_BUFFER_SIZE 64 struct xsc_context { @@ -146,7 +179,7 @@ struct xsc_context { int max_sq_desc_sz; int max_rq_desc_sz; - int max_send_wr; + int max_send_wqebb; int max_recv_wr; int num_ports; int stall_enable; @@ -155,6 +188,7 @@ struct xsc_context { char hostname[NAME_BUFFER_SIZE]; struct xsc_spinlock hugetlb_lock; struct list_head hugetlb_list; + enum ibv_atomic_cap atomic_cap; struct xscdv_ctx_allocators extern_alloc; uint32_t max_cqe; void *sqm_reg_va; @@ -177,6 +211,7 @@ struct xsc_context { uint32_t tx_multidb_base; void *mdb_base; uint32_t tx_mdb_idx; + uint32_t rdma_proto_mode; uint32_t hw_feature_flag; uint32_t mdb_mmap_size; }; @@ -206,9 +241,14 @@ struct xsc_buf { }; struct xsc_pd { - struct ibv_pd ibv_pd; - uint32_t pdn; - atomic_int refcount; + struct ibv_pd ibv_pd; + uint32_t pdn; + atomic_int refcount; + struct xsc_pd *xprotection_domain; +}; + +struct xsc_parent_domain { + struct xsc_pd xpd; }; enum { @@ -261,13 +301,85 @@ struct xsc_cq { struct list_head err_state_qp_list; }; -struct xsc_qp; - struct xsc_mr { struct verbs_mr vmr; uint32_t alloc_flags; }; +struct xsc_wq { + uint64_t *wrid; + uint32_t *wqe_head; + struct xsc_spinlock lock; + uint32_t wqe_cnt; + uint32_t max_post; + uint32_t head; + uint32_t tail; + uint32_t cur_post; + int max_gs; + int wqe_shift; + int offset; + void *qend; + uint32_t *wr_data; + __le32 *db; + unsigned int ds_cnt; + unsigned int seg_cnt; + unsigned int *wr_opcode; + unsigned int *need_flush; +}; + +enum xsc_qp_flags { + XSC_QP_FLAGS_USE_UNDERLAY = 0x01, +}; + +struct xsc_qp { + struct xsc_resource rsc; /* This struct must be first */ + struct verbs_qp verbs_qp; + struct ibv_qp *ibv_qp; + struct xsc_buf buf; + void *sq_start; + void *rq_start; + int max_inline_data; + int buf_size; + /* For Raw Packet QP, use different buffers for the SQ and RQ */ + struct xsc_buf sq_buf; + int sq_buf_size; + + int err; + /* Number of WR entries posted in the current wr session */ + int nreq; + uint32_t cur_post_rb; + void *cur_ctrl; + void *cur_data; + int cur_ds_num; + uint32_t cur_data_len; + uint64_t cur_remote_addr; + uint32_t cur_remote_key; + + uint8_t fm_cache; + uint8_t sq_signal_bits; + struct xsc_wq sq; + + __le32 *db; + struct xsc_wq rq; + int wq_sig; + uint32_t qp_cap_cache; + int atomics_enabled; + uint32_t max_tso; + uint16_t max_tso_header; + int rss_qp; + uint32_t flags; /* Use enum xsc_qp_flags */ + enum xscdv_dc_type dc_type; + uint32_t tirn; + uint32_t tisn; + uint32_t rqn; + uint32_t sqn; + unsigned int err_occurred; + uint32_t profile; + uint32_t get_ece; + bool set_ece; + bool has_trig_cq_evt; +}; + union xsc_ib_fw_ver { uint64_t data; struct { @@ -278,6 +390,8 @@ union xsc_ib_fw_ver { } s; }; +extern int xsc_single_threaded; + static inline int xsc_ilog2(int n) { int t; @@ -305,7 +419,31 @@ static inline struct xsc_context *to_xctx(struct ibv_context *ibctx) /* to_xpd always returns the real xsc_pd object ie the protection domain. */ static inline struct xsc_pd *to_xpd(struct ibv_pd *ibpd) { - return container_of(ibpd, struct xsc_pd, ibv_pd); + struct xsc_pd *xpd = container_of(ibpd, struct xsc_pd, ibv_pd); + + if (xpd->xprotection_domain) + return xpd->xprotection_domain; + + return xpd; +} + +static inline struct xsc_parent_domain *to_xparent_domain(struct ibv_pd *ibpd) +{ + struct xsc_parent_domain *xparent_domain = + ibpd ? container_of(ibpd, struct xsc_parent_domain, xpd.ibv_pd) : NULL; + + if (xparent_domain && xparent_domain->xpd.xprotection_domain) + return xparent_domain; + + /* Otherwise ibpd isn't a parent_domain */ + return NULL; +} + +static inline struct xsc_qp *to_xqp(struct ibv_qp *ibqp) +{ + struct verbs_qp *vqp = (struct verbs_qp *)ibqp; + + return container_of(vqp, struct xsc_qp, verbs_qp); } static inline struct xsc_cq *to_xcq(struct ibv_cq *ibcq) @@ -335,6 +473,7 @@ int xsc_alloc_prefered_buf(struct xsc_context *xctx, enum xsc_alloc_type alloc_type, const char *component); int xsc_free_actual_buf(struct xsc_context *ctx, struct xsc_buf *buf); +int xsc_use_huge(const char *key); void xsc_get_alloc_type(struct xsc_context *context, const char *component, enum xsc_alloc_type *alloc_type, @@ -364,6 +503,25 @@ int xsc_alloc_cq_buf(struct xsc_context *xctx, struct xsc_cq *cq, int xsc_free_cq_buf(struct xsc_context *ctx, struct xsc_buf *buf); int xsc_resize_cq(struct ibv_cq *cq, int cqe); int xsc_destroy_cq(struct ibv_cq *cq); +void __xsc_cq_clean(struct xsc_cq *cq, uint32_t qpn); +void xsc_cq_clean(struct xsc_cq *cq, uint32_t qpn); + +struct ibv_qp *xsc_create_qp_ex(struct ibv_context *context, + struct ibv_qp_init_attr_ex *attr); +struct ibv_qp *xsc_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr); +int xsc_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask, + struct ibv_qp_init_attr *init_attr); +int xsc_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask); +int xsc_destroy_qp(struct ibv_qp *qp); +void xsc_init_qp_indices(struct xsc_qp *qp); +struct xsc_qp *xsc_find_qp(struct xsc_context *ctx, uint32_t qpn); +int xsc_store_qp(struct xsc_context *ctx, uint32_t qpn, struct xsc_qp *qp); +void xsc_clear_qp(struct xsc_context *ctx, uint32_t qpn); +int xsc_err_state_qp(struct ibv_qp *qp, enum ibv_qp_state cur_state, + enum ibv_qp_state state); +int xsc_qp_fill_wr_pfns(struct xsc_context *ctx, + struct xsc_qp *xqp, + const struct ibv_qp_init_attr_ex *attr); static inline int xsc_spin_lock(struct xsc_spinlock *lock) { @@ -382,6 +540,13 @@ static inline int xsc_spinlock_init(struct xsc_spinlock *lock, int need_lock) return pthread_spin_init(&lock->lock, PTHREAD_PROCESS_PRIVATE); } +static inline int xsc_spinlock_init_pd(struct xsc_spinlock *lock, struct ibv_pd *pd) +{ + int thread_safe = xsc_single_threaded; + + return xsc_spinlock_init(lock, !thread_safe); +} + static inline int xsc_spinlock_destroy(struct xsc_spinlock *lock) { return pthread_spin_destroy(&lock->lock); diff --git a/providers/xscale/xscdv.h b/providers/xscale/xscdv.h index a723712b8..8acb2d341 100644 --- a/providers/xscale/xscdv.h +++ b/providers/xscale/xscdv.h @@ -37,6 +37,39 @@ struct xscdv_cq_init_attr { uint16_t cqe_size; /* when XSCDV_CQ_INIT_ATTR_MASK_CQE_SIZE set */ }; +enum xscdv_wq_init_attr_mask { + XSCDV_WQ_INIT_ATTR_MASK_STRIDING_RQ = 1 << 0, +}; + +enum xscdv_qp_create_flags { + XSCDV_QP_CREATE_TUNNEL_OFFLOADS = 1 << 0, + XSCDV_QP_CREATE_TIR_ALLOW_SELF_LOOPBACK_UC = 1 << 1, + XSCDV_QP_CREATE_TIR_ALLOW_SELF_LOOPBACK_MC = 1 << 2, + XSCDV_QP_CREATE_DISABLE_SCATTER_TO_CQE = 1 << 3, + XSCDV_QP_CREATE_ALLOW_SCATTER_TO_CQE = 1 << 4, +}; + +enum xscdv_qp_init_attr_mask { + XSCDV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS = 1 << 0, + XSCDV_QP_INIT_ATTR_MASK_DC = 1 << 1, +}; + +enum xscdv_dc_type { + XSCDV_DCTYPE_DCT = 1, + XSCDV_DCTYPE_DCI, +}; + +struct xscdv_dc_init_attr { + enum xscdv_dc_type dc_type; + uint64_t dct_access_key; +}; + +struct xscdv_qp_init_attr { + uint64_t comp_mask; /* Use enum xscdv_qp_init_attr_mask */ + uint32_t create_flags; /* Use enum xsc_qp_create_flags */ + struct xscdv_dc_init_attr dc_init_attr; +}; + enum xscdv_devx_umem_in_mask { XSCDV_UMEM_MASK_DMABUF = 1 << 0, }; From 0235f5bc204f4336eb400389cecf4e987767d232 Mon Sep 17 00:00:00 2001 From: Tian Xin Date: Tue, 7 Jul 2026 14:31:00 +0800 Subject: [PATCH 06/13] libxscale: Implement post_send and post_recv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the ability to post work requests to the hardware: - post_send: Construct send WQEs (including RDMA write/read, send, atomic operations) and ring the send doorbell. Inline data and scatter‑gather lists are supported. - post_recv: Build receive WQEs and ring the receive doorbell. The implementation carefully manages WQE rings, tracks work request IDs and completion flags, and handles overflow conditions. Doorbell ringing is done via MMIO writes with proper barriers. Signed-off-by: Tian Xin Signed-off-by: Wei Honggang Signed-off-by: Zhao Qianwei Signed-off-by: Li Qiang Signed-off-by: Yan Lei --- providers/xscale/qp.c | 517 +++++++++++++++++++++++++++++++++++++- providers/xscale/xscale.c | 100 ++++++++ providers/xscale/xscale.h | 36 ++- providers/xscale/xscdv.h | 5 + 4 files changed, 643 insertions(+), 15 deletions(-) diff --git a/providers/xscale/qp.c b/providers/xscale/qp.c index a8e9ffbfa..cf6ef8f49 100644 --- a/providers/xscale/qp.c +++ b/providers/xscale/qp.c @@ -16,11 +16,14 @@ #include "xscale.h" #include "xsc_hw.h" -int xsc_qp_fill_wr_pfns(struct xsc_context *ctx, - struct xsc_qp *xqp, - const struct ibv_qp_init_attr_ex *attr) +static inline void *get_recv_wqe(struct xsc_qp *qp, int n) { - return 0; + return qp->rq_start + (n << qp->rq.wqe_shift); +} + +static inline void *get_seg_wqe(void *first, int n) +{ + return first + (n << XSC_BASE_WQE_SHIFT); } void xsc_init_qp_indices(struct xsc_qp *qp) @@ -33,6 +36,512 @@ void xsc_init_qp_indices(struct xsc_qp *qp) qp->has_trig_cq_evt = false; } +static inline int xsc_wq_overflow(struct xsc_wq *wq, int nreq, struct xsc_cq *cq) +{ + uint32_t cur; + + cur = wq->head - wq->tail; + if (cur + nreq < wq->max_post) + return 0; + + xsc_spin_lock(&cq->lock); + cur = wq->head - wq->tail; + xsc_spin_unlock(&cq->lock); + + return cur + nreq >= wq->max_post; +} + +static inline void set_data_seg_with_value(struct xsc_qp *qp, struct xsc_wqe_data_seg *data_seg, + uint64_t addr, uint32_t key, uint32_t length) +{ + struct xsc_context *ctx = to_xctx(qp->ibv_qp->context); + + xsc_hw_set_data_seg(ctx->device_id, data_seg, addr, key, length); +} + +static inline void set_local_data_seg_from_sge(struct xsc_qp *qp, struct xsc_wqe_data_seg *data_seg, + const struct ibv_sge *sg) +{ + struct xsc_context *ctx = to_xctx(qp->ibv_qp->context); + + xsc_hw_set_data_seg(ctx->device_id, data_seg, sg->addr, sg->lkey, sg->length); +} + +static inline void set_atomic_seg_with_value(struct xsc_qp *qp, + struct xsc_wqe_atomic_seg *atomic_seg, + int opcode, uint64_t swap, uint64_t compare_add) +{ + struct xsc_context *ctx = to_xctx(qp->ibv_qp->context); + + xsc_hw_set_atomic_seg(ctx->device_id, atomic_seg, opcode, swap, compare_add); +} + +static void *get_addr_from_wr(const void *list, int idx) +{ + const struct ibv_send_wr *wr = list; + + return (void *)wr->sg_list[idx].addr; +} + +static int get_len_from_wr(const void *list, int idx) +{ + const struct ibv_send_wr *wr = list; + + return wr->sg_list[idx].length; +} + +static int _set_wqe_inline(void *data_seg, size_t num_buf, const void *list, + void *(*get_addr)(const void *, int), + int (*get_len)(const void *, int)) +{ + int i; + int offset = 0; + int len = 0; + void *addr; + void *data_seg_base = data_seg; + int seg_index = 0; + const int ds_len = sizeof(struct xsc_wqe_data_seg); + + for (i = 0; i < num_buf; i++) { + addr = get_addr(list, i); + len = get_len(list, i); + if (likely(len)) { + if (offset) { + int copy_len = min_t(int, len, ds_len - offset); + + memcpy(data_seg + offset, addr, copy_len); + addr += copy_len; + len -= copy_len; + offset += copy_len; + if (offset == ds_len) + offset = 0; + else + continue; + } + + while (len >= ds_len) { + data_seg = get_seg_wqe(data_seg_base, seg_index); + seg_index++; + memcpy(data_seg, addr, ds_len); + addr += ds_len; + len -= ds_len; + } + + if (len > 0) { + data_seg = get_seg_wqe(data_seg_base, seg_index); + seg_index++; + memcpy(data_seg, addr, len); + offset += len; + } + } + } + return seg_index; +} + +static int set_wqe_inline_from_wr(struct xsc_qp *qp, struct ibv_send_wr *wr, + struct xsc_send_wqe_ctrl_seg *ctrl) +{ + void *data_seg; + unsigned int seg_index; + int msg_len = ctrl->msg_len; + int filled_ds_num; + + if (wr->opcode == IBV_WR_SEND || wr->opcode == IBV_WR_SEND_WITH_IMM) + seg_index = 1; + else + seg_index = 2; + data_seg = get_seg_wqe(ctrl, seg_index); + + if (unlikely(msg_len > qp->max_inline_data)) + return ENOMEM; + + filled_ds_num = _set_wqe_inline(data_seg, wr->num_sge, wr, + get_addr_from_wr, + get_len_from_wr); + ctrl->data0 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_DS_DATA_NUM_MASK, + seg_index - 1 + filled_ds_num); + + return 0; +} + +static inline void _zero_send_ds(int idx, struct xsc_qp *qp, int keep_ctrl) +{ + void *seg; + uint64_t *p; + int i; + + seg = (void *)xsc_get_send_wqe(qp, idx); + for (i = keep_ctrl; i < qp->sq.seg_cnt; i++) { + p = get_seg_wqe(seg, i); + p[0] = p[1] = 0; + } +} + +static inline void clear_send_wqe(int idx, struct xsc_qp *qp) +{ + _zero_send_ds(idx, qp, 0); +} + +static inline void clear_send_wqe_except_ctrl(int idx, struct xsc_qp *qp) +{ + _zero_send_ds(idx, qp, 1); +} + +static void clear_recv_wqe(int idx, struct xsc_qp *qp) +{ + void *seg; + uint64_t *p; + int i; + + seg = (void *)get_recv_wqe(qp, idx); + for (i = 0; i < qp->rq.seg_cnt; i++) { + p = get_seg_wqe(seg, i); + p[0] = p[1] = 0; + } +} + +#ifdef XSC_DEBUG +static void dump_wqe(int type, int idx, struct xsc_qp *qp) +{ + /* type0 send type1 recv */ + uint32_t *p; + int i; + void *seg; + + if (type == 0) { + seg = (void *)xsc_get_send_wqe(qp, idx); + xsc_dbg(to_xctx(qp->ibv_qp->context)->dbg_fp, XSC_DBG_QP, + "dump send wqe at %p\n", seg); + for (i = 0; i < qp->sq.seg_cnt; i++) { + p = get_seg_wqe(seg, i); + xsc_dbg(to_xctx(qp->ibv_qp->context)->dbg_fp, XSC_DBG_QP, + "0x%08x 0x%08x 0x%08x 0x%08x\n", p[0], p[1], p[2], p[3]); + } + } else if (type == 1) { + seg = (void *)get_recv_wqe(qp, idx); + xsc_dbg(to_xctx(qp->ibv_qp->context)->dbg_fp, XSC_DBG_QP, + "dump recv wqe at %p\n", seg); + for (i = 0; i < qp->rq.seg_cnt; i++) { + p = get_seg_wqe(seg, i); + xsc_dbg(to_xctx(qp->ibv_qp->context)->dbg_fp, XSC_DBG_QP, + "0x%08x 0x%08x 0x%08x 0x%08x\n", p[0], p[1], p[2], p[3]); + } + } else { + xsc_dbg(to_xctx(qp->ibv_qp->context)->dbg_fp, XSC_DBG_QP, + "unknown type %d\n", type); + } +} +#else +static inline void dump_wqe(int type, int idx, struct xsc_qp *qp) {}; +#endif + +static inline void xsc_post_send_db(struct xsc_qp *qp, int nreq) +{ + struct xsc_context *ctx = to_xctx(qp->ibv_qp->context); + uint32_t next_pid; + + if (unlikely(!nreq)) + return; + + qp->sq.head += nreq; + next_pid = qp->sq.head << (qp->sq.wqe_shift - XSC_BASE_WQE_SHIFT); + xsc_dbg(to_xctx(qp->ibv_qp->context)->dbg_fp, XSC_DBG_QP_SEND, "nreq:%d\n", nreq); + xsc_hw_ring_tx_doorbell(ctx->device_id, qp->sq.db, qp->sqn, next_pid); +} + +static inline int _xsc_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, + struct ibv_send_wr **bad_wr) +{ + struct xsc_qp *qp = to_xqp(ibqp); + void *seg; + struct xsc_send_wqe_ctrl_seg *ctrl; + struct xsc_wqe_data_seg *data_seg; + struct xsc_wqe_atomic_seg *atomic_seg; + struct xsc_context *ctx = to_xctx(ibqp->context); + int nreq; + int err = 0; + int i; + unsigned int idx; + unsigned int seg_index = 1; + unsigned int msg_len = 0; + + if (unlikely(ibqp->state < IBV_QPS_RTS)) { + xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, + "qp state is %u, should not post send\n", ibqp->state); + err = EINVAL; + *bad_wr = wr; + return err; + } + + xsc_spin_lock(&qp->sq.lock); + + for (nreq = 0; wr; ++nreq, wr = wr->next) { + seg_index = 1; + msg_len = 0; + if (unlikely(wr->opcode < 0 || + wr->opcode >= sizeof(ctx->wr2msg) / sizeof(ctx->wr2msg[0]))) { + xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, + "bad opcode %d\n", wr->opcode); + err = EINVAL; + *bad_wr = wr; + goto out; + } + + if (unlikely(xsc_wq_overflow(&qp->sq, nreq, + to_xcq(qp->ibv_qp->send_cq)))) { + xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, + "send work queue overflow\n"); + err = ENOMEM; + *bad_wr = wr; + goto out; + } + + if (unlikely(wr->num_sge > qp->sq.max_gs)) { + xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, + "max gs exceeded %d (max = %d)\n", + wr->num_sge, qp->sq.max_gs); + err = ENOMEM; + *bad_wr = wr; + goto out; + } + + if (unlikely(wr->opcode == IBV_WR_RDMA_READ && wr->num_sge > 1)) { + xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, + "rdma read, max gs exceeded %d (max = 1)\n", + wr->num_sge); + err = ENOMEM; + *bad_wr = wr; + goto out; + } + + idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1); + clear_send_wqe(idx, qp); + ctrl = seg = xsc_get_send_wqe(qp, idx); + ctrl->data0 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_DS_DATA_NUM_MASK, 0) | + FIELD_PREP(XSC_SWQE_CTRL_SEG_WITH_IMMDT_MASK, 0); + xsc_hw_set_wqe_id(to_xctx(ibqp->context)->device_id, ctrl, + qp->sq.cur_post << (qp->sq.wqe_shift - XSC_BASE_WQE_SHIFT)); + ctrl->data1 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_SE_MASK, + wr->send_flags & IBV_SEND_SOLICITED ? 1 : 0) | + FIELD_PREP(XSC_SWQE_CTRL_SEG_CE_MASK, + qp->sq_signal_bits ? + 1 : (wr->send_flags & IBV_SEND_SIGNALED ? 1 : 0)) | + FIELD_PREP(XSC_SWQE_CTRL_SEG_IN_LINE_MASK, + wr->send_flags & IBV_SEND_INLINE ? 1 : 0) | + FIELD_PREP(XSC_SWQE_CTRL_SEG_FENCE_MODE_MASK, + wr->send_flags & IBV_SEND_FENCE ? 1 : 0) | + FIELD_PREP(XSC_SWQE_CTRL_SEG_MASK_MASK, 0); + for (i = 0; i < wr->num_sge; ++i) { + if (likely(wr->sg_list[i].length)) + msg_len += wr->sg_list[i].length; + } + ctrl->msg_len = msg_len; + + if (unlikely(wr->opcode == IBV_WR_RDMA_READ && msg_len == 0)) { + xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, + "rdma read, msg len should not be 0\n"); + /* workaround, return success for posting zero-length read */ + err = 0; + goto out; + } + + switch (ibqp->qp_type) { + case IBV_QPT_RC: + switch (wr->opcode) { + case IBV_WR_SEND_WITH_INV: + case IBV_WR_SEND: + break; + case IBV_WR_SEND_WITH_IMM: + ctrl->data0 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_WITH_IMMDT_MASK, 1); + WR_LE_32(ctrl->opcode_data, RD_BE_32(wr->imm_data)); + break; + case IBV_WR_RDMA_WRITE_WITH_IMM: + ctrl->data0 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_WITH_IMMDT_MASK, 1); + WR_LE_32(ctrl->opcode_data, RD_BE_32(wr->imm_data)); + SWITCH_FALLTHROUGH; + case IBV_WR_RDMA_READ: + case IBV_WR_RDMA_WRITE: + if (ctrl->msg_len == 0) + break; + data_seg = get_seg_wqe(ctrl, seg_index); + set_data_seg_with_value(qp, + data_seg, + wr->wr.rdma.remote_addr, + wr->wr.rdma.rkey, + msg_len); + seg_index++; + break; + case IBV_WR_ATOMIC_CMP_AND_SWP: + case IBV_WR_ATOMIC_FETCH_AND_ADD: + if (unlikely(!qp->atomics_enabled)) { + xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, + "atomic operations are not supported\n"); + err = EOPNOTSUPP; + *bad_wr = wr; + goto out; + } + + ctrl->msg_len = 16; + data_seg = get_seg_wqe(ctrl, seg_index); + set_data_seg_with_value(qp, + data_seg, + wr->wr.atomic.remote_addr, + wr->wr.atomic.rkey, 8); + seg_index++; + break; + default: + printf("debug: opcode:%u NOT supported\n", wr->opcode); + err = EPERM; + *bad_wr = wr; + goto out; + } + break; + default: + xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, + "qp type:%u NOT supported\n", ibqp->qp_type); + err = EPERM; + *bad_wr = wr; + goto out; + } + + if ((wr->opcode == IBV_WR_ATOMIC_CMP_AND_SWP || + wr->opcode == IBV_WR_ATOMIC_FETCH_AND_ADD) && qp->atomics_enabled) { + ctrl->data1 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_IN_LINE_MASK, 1); + data_seg = get_seg_wqe(ctrl, seg_index); + set_data_seg_with_value(qp, + data_seg, + wr->sg_list[0].addr, + wr->sg_list[0].lkey, + 8); + seg_index++; + atomic_seg = get_seg_wqe(ctrl, seg_index); + set_atomic_seg_with_value(qp, + atomic_seg, + wr->opcode, + wr->wr.atomic.swap, + wr->wr.atomic.compare_add); + seg_index++; + } else { + if (wr->send_flags & IBV_SEND_INLINE && wr->num_sge) { + err = set_wqe_inline_from_wr(qp, wr, ctrl); + if (unlikely(err)) { + *bad_wr = wr; + xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, + "inline layout failed, err %d\n", err); + goto out; + } + } else { + for (i = 0; i < wr->num_sge; ++i, ++seg_index) { + if (likely(wr->sg_list[i].length)) { + data_seg = get_seg_wqe(ctrl, seg_index); + set_local_data_seg_from_sge(qp, data_seg, + &wr->sg_list[i]); + } + } + } + } + + ctrl->msg_opcode = ctx->wr2msg[wr->opcode]; + if (ctrl->msg_len == 0) { + ctrl->data0 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_DS_DATA_NUM_MASK, 0); + clear_send_wqe_except_ctrl(idx, qp); + } + ctrl->data0 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_DS_DATA_NUM_MASK, seg_index - 1); + qp->sq.wrid[idx] = wr->wr_id; + qp->sq.wqe_head[idx] = qp->sq.head + nreq; + qp->sq.cur_post += 1; + if (FIELD_GET(XSC_SWQE_CTRL_SEG_CE_MASK, ctrl->data1)) + qp->sq.need_flush[idx] = 1; + qp->sq.wr_opcode[idx] = wr->opcode; + + if (unlikely(xsc_debug_mask & XSC_DBG_QP_SEND)) + dump_wqe(0, idx, qp); + } + +out: + xsc_post_send_db(qp, nreq); + xsc_spin_unlock(&qp->sq.lock); + + return err; +} + +int xsc_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, + struct ibv_send_wr **bad_wr) +{ + return _xsc_post_send(ibqp, wr, bad_wr); +} + +int xsc_qp_fill_wr_pfns(struct xsc_context *ctx, + struct xsc_qp *xqp, + const struct ibv_qp_init_attr_ex *attr) +{ + return 0; +} + +int xsc_post_recv_dump_wqe = 1; +int xsc_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, + struct ibv_recv_wr **bad_wr) +{ + struct xsc_qp *qp = to_xqp(ibqp); + struct xsc_wqe_data_seg *recv_head; + struct xsc_wqe_data_seg *data_seg; + int err = 0; + uint32_t next_pid = 0; + int nreq; + uint16_t idx; + int i; + + xsc_spin_lock(&qp->rq.lock); + + idx = qp->rq.head & (qp->rq.wqe_cnt - 1); + + clear_recv_wqe(idx, qp); + for (nreq = 0; wr; ++nreq, wr = wr->next) { + if (unlikely(xsc_wq_overflow(&qp->rq, nreq, + to_xcq(qp->ibv_qp->recv_cq)))) { + err = ENOMEM; + *bad_wr = wr; + goto out; + } + + if (unlikely(wr->num_sge > qp->rq.max_gs)) { + printf("max gs exceeded %d (max = %d)\n", + wr->num_sge, qp->rq.max_gs); + err = EINVAL; + *bad_wr = wr; + goto out; + } + + recv_head = get_recv_wqe(qp, idx); + + for (i = 0; i < wr->num_sge; ++i) { + if (unlikely(!wr->sg_list[i].length)) + continue; + data_seg = get_seg_wqe(recv_head, i); + set_local_data_seg_from_sge(qp, data_seg, &wr->sg_list[i]); + } + + qp->rq.wrid[idx] = wr->wr_id; + + if (xsc_post_recv_dump_wqe || (xsc_debug_mask & XSC_DBG_QP_RECV)) + dump_wqe(1, idx, qp); + idx = (idx + 1) & (qp->rq.wqe_cnt - 1); + } + +out: + if (likely(nreq)) { + struct xsc_context *ctx = to_xctx(ibqp->context); + + qp->rq.head += nreq; + next_pid = qp->rq.head << (qp->rq.wqe_shift - XSC_BASE_WQE_SHIFT); + xsc_hw_ring_rx_doorbell(ctx->device_id, qp->rq.db, qp->rqn, next_pid); + } + + xsc_spin_unlock(&qp->rq.lock); + + return err; +} + struct xsc_qp *xsc_find_qp(struct xsc_context *ctx, uint32_t qpn) { int tind = qpn >> XSC_QP_TABLE_SHIFT; diff --git a/providers/xscale/xscale.c b/providers/xscale/xscale.c index c4a1306e4..ba1fac4ce 100644 --- a/providers/xscale/xscale.c +++ b/providers/xscale/xscale.c @@ -51,6 +51,8 @@ static const struct verbs_context_ops xsc_ctx_common_ops = { .modify_qp = xsc_modify_qp, .destroy_qp = xsc_destroy_qp, .create_qp_ex = xsc_create_qp_ex, + .post_send = xsc_post_send, + .post_recv = xsc_post_recv, }; static void open_debug_file(struct xsc_context *ctx) @@ -179,6 +181,102 @@ static void xsc_munmap(struct xsc_context *context) munmap(context->mdb_base, context->mdb_mmap_size); } +static void xsc_build_opcode_map(struct xsc_context *ctx) +{ + uint16_t device_id = ctx->device_id; + uint32_t msg_opcode_send = (device_id == XSC_MC_PF_DEV_ID_DIAMOND || + device_id == XSC_MC_PF_DEV_ID_DIAMOND_NEXT) ? + XSC_MSG_OPCODE_SEND_DIAMOND_NEXT : XSC_MSG_OPCODE_SEND; + uint32_t xsc_ib_opcode[] = { + [IBV_WR_SEND] = msg_opcode_send, + [IBV_WR_SEND_WITH_IMM] = msg_opcode_send, + [IBV_WR_RDMA_WRITE] = XSC_MSG_OPCODE_RDMA_WRITE, + [IBV_WR_RDMA_WRITE_WITH_IMM] = XSC_MSG_OPCODE_RDMA_WRITE, + [IBV_WR_RDMA_READ] = XSC_MSG_OPCODE_RDMA_READ, + [IBV_WR_SEND_WITH_INV] = msg_opcode_send, + [IBV_WR_ATOMIC_CMP_AND_SWP] = XSC_MSG_OPCODE_RDMA_ATOMIC_CMP_AND_SWAP, + [IBV_WR_ATOMIC_FETCH_AND_ADD] = XSC_MSG_OPCODE_RDMA_ATOMIC_FETCH_AND_ADD, + }; + + uint32_t xsc_msg_opcode[][2][2] = { + [XSC_MSG_OPCODE_RDMA_WRITE][XSC_REQ][XSC_WITHOUT_IMMDT] = XSC_OPCODE_RDMA_REQ_WRITE, + [XSC_MSG_OPCODE_RDMA_WRITE][XSC_REQ][XSC_WITH_IMMDT] = + XSC_OPCODE_RDMA_REQ_WRITE_IMMDT, + [XSC_MSG_OPCODE_RDMA_WRITE][XSC_RSP][XSC_WITHOUT_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_WRITE][XSC_RSP][XSC_WITH_IMMDT] = + XSC_OPCODE_RDMA_RSP_WRITE_IMMDT, + [XSC_MSG_OPCODE_RDMA_READ][XSC_REQ][XSC_WITHOUT_IMMDT] = + XSC_OPCODE_RDMA_REQ_READ, + [XSC_MSG_OPCODE_RDMA_READ][XSC_REQ][XSC_WITH_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_READ][XSC_RSP][XSC_WITHOUT_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_READ][XSC_RSP][XSC_WITH_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_ATOMIC_CMP_AND_SWAP][XSC_REQ][XSC_WITHOUT_IMMDT] = + XSC_OPCODE_RDMA_REQ_ATOMIC_CMP_AND_SWAP, + [XSC_MSG_OPCODE_RDMA_ATOMIC_CMP_AND_SWAP][XSC_REQ][XSC_WITH_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_ATOMIC_CMP_AND_SWAP][XSC_RSP][XSC_WITHOUT_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_ATOMIC_CMP_AND_SWAP][XSC_RSP][XSC_WITH_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_ATOMIC_FETCH_AND_ADD][XSC_REQ][XSC_WITHOUT_IMMDT] = + XSC_OPCODE_RDMA_REQ_ATOMIC_FETCH_AND_ADD, + [XSC_MSG_OPCODE_RDMA_ATOMIC_FETCH_AND_ADD][XSC_REQ][XSC_WITH_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_ATOMIC_FETCH_AND_ADD][XSC_RSP][XSC_WITHOUT_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_ATOMIC_FETCH_AND_ADD][XSC_RSP][XSC_WITH_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + + [XSC_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_CMP_AND_SWAP][XSC_REQ][XSC_WITHOUT_IMMDT] = + XSC_OPCODE_RDMA_REQ_ATOMIC_8B_MASK_CS, + [XSC_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_CMP_AND_SWAP][XSC_REQ][XSC_WITH_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_CMP_AND_SWAP][XSC_RSP][XSC_WITHOUT_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_CMP_AND_SWAP][XSC_RSP][XSC_WITH_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + + [XSC_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_FETCH_AND_ADD][XSC_REQ][XSC_WITHOUT_IMMDT] = + XSC_OPCODE_RDMA_REQ_ATOMIC_8B_MASK_FA, + [XSC_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_FETCH_AND_ADD][XSC_REQ][XSC_WITH_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_FETCH_AND_ADD][XSC_RSP][XSC_WITHOUT_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_FETCH_AND_ADD][XSC_RSP][XSC_WITH_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + + [XSC_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_CMP_AND_SWAP][XSC_REQ][XSC_WITHOUT_IMMDT] = + XSC_OPCODE_RDMA_REQ_ATOMIC_4B_MASK_CS, + [XSC_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_CMP_AND_SWAP][XSC_REQ][XSC_WITH_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_CMP_AND_SWAP][XSC_RSP][XSC_WITHOUT_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_CMP_AND_SWAP][XSC_RSP][XSC_WITH_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + + [XSC_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_FETCH_AND_ADD][XSC_REQ][XSC_WITHOUT_IMMDT] = + XSC_OPCODE_RDMA_REQ_ATOMIC_4B_MASK_FA, + [XSC_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_FETCH_AND_ADD][XSC_REQ][XSC_WITH_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_FETCH_AND_ADD][XSC_RSP][XSC_WITHOUT_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_FETCH_AND_ADD][XSC_RSP][XSC_WITH_IMMDT] = + XSC_OPCODE_RDMA_CQE_ERROR, + }; + + memcpy(ctx->wr2msg, xsc_ib_opcode, sizeof(xsc_ib_opcode)); + + xsc_msg_opcode[msg_opcode_send][XSC_REQ][XSC_WITHOUT_IMMDT] = XSC_OPCODE_RDMA_REQ_SEND, + xsc_msg_opcode[msg_opcode_send][XSC_REQ][XSC_WITH_IMMDT] = XSC_OPCODE_RDMA_REQ_SEND_IMMDT, + xsc_msg_opcode[msg_opcode_send][XSC_RSP][XSC_WITHOUT_IMMDT] = XSC_OPCODE_RDMA_RSP_RECV, + xsc_msg_opcode[msg_opcode_send][XSC_RSP][XSC_WITH_IMMDT] = XSC_OPCODE_RDMA_RSP_RECV_IMMDT, + memcpy(ctx->msg2cqe, xsc_msg_opcode, sizeof(xsc_msg_opcode)); +} + static struct verbs_context *xsc_alloc_context(struct ibv_device *ibdev, int cmd_fd, void *private_data) { @@ -239,6 +337,8 @@ static struct verbs_context *xsc_alloc_context(struct ibv_device *ibdev, context->send_ds_num, context->send_ds_shift, context->recv_ds_num, context->recv_ds_shift); + xsc_build_opcode_map(context); + pthread_mutex_init(&context->qp_table_mutex, NULL); for (i = 0; i < XSC_QP_TABLE_SIZE; ++i) context->qp_table[i].refcnt = 0; diff --git a/providers/xscale/xscale.h b/providers/xscale/xscale.h index 6e937b8ce..b228d8102 100644 --- a/providers/xscale/xscale.h +++ b/providers/xscale/xscale.h @@ -22,6 +22,7 @@ #include "xsc-abi.h" #include "xscdv.h" +#include "xsc_hsi.h" enum xsc_qp_ex_attr_mask { XSC_QP_FLUSH = 1 << 21, @@ -48,16 +49,17 @@ enum { #define XSC_MIN_LOG2_CONTIG_BLOCK_SIZE 12 enum { - XSC_DBG_QP = 1 << 0, - XSC_DBG_CQ = 1 << 1, - XSC_DBG_QP_SEND = 1 << 2, - XSC_DBG_QP_SEND_ERR = 1 << 3, - XSC_DBG_CQ_CQE = 1 << 4, - XSC_DBG_CONTIG = 1 << 5, - XSC_DBG_DR = 1 << 6, - XSC_DBG_CTX = 1 << 7, - XSC_DBG_PD = 1 << 8, - XSC_DBG_MR = 1 << 9, + XSC_DBG_QP = 1 << 0, + XSC_DBG_CQ = 1 << 1, + XSC_DBG_QP_SEND = 1 << 2, + XSC_DBG_QP_SEND_ERR = 1 << 3, + XSC_DBG_CQ_CQE = 1 << 4, + XSC_DBG_CONTIG = 1 << 5, + XSC_DBG_DR = 1 << 6, + XSC_DBG_CTX = 1 << 7, + XSC_DBG_PD = 1 << 8, + XSC_DBG_MR = 1 << 9, + XSC_DBG_QP_RECV = 1 << 10, }; extern uint32_t xsc_debug_mask; @@ -166,7 +168,8 @@ enum veroce_profile { VEROCE_PROFILE_P3 }; -#define NAME_BUFFER_SIZE 64 +#define NAME_BUFFER_SIZE 64 +#define MAX_OPCODE IBV_WR_ATOMIC_WRITE struct xsc_context { struct verbs_context ibv_ctx; @@ -212,6 +215,8 @@ struct xsc_context { void *mdb_base; uint32_t tx_mdb_idx; uint32_t rdma_proto_mode; + uint32_t wr2msg[MAX_OPCODE + 1]; + uint32_t msg2cqe[XSC_MSG_OPCODE_MAX][2][2]; uint32_t hw_feature_flag; uint32_t mdb_mmap_size; }; @@ -513,6 +518,10 @@ int xsc_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask, struct ibv_qp_init_attr *init_attr); int xsc_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask); int xsc_destroy_qp(struct ibv_qp *qp); +int xsc_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, + struct ibv_send_wr **bad_wr); +int xsc_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, + struct ibv_recv_wr **bad_wr); void xsc_init_qp_indices(struct xsc_qp *qp); struct xsc_qp *xsc_find_qp(struct xsc_context *ctx, uint32_t qpn); int xsc_store_qp(struct xsc_context *ctx, uint32_t qpn, struct xsc_qp *qp); @@ -567,4 +576,9 @@ static inline void set_order(int order, off_t *offset) set_arg(order, offset); } +static inline void *xsc_get_send_wqe(struct xsc_qp *qp, int n) +{ + return qp->sq_start + (n << qp->sq.wqe_shift); +} + #endif /* XSC_H */ diff --git a/providers/xscale/xscdv.h b/providers/xscale/xscdv.h index 8acb2d341..1d57bfd83 100644 --- a/providers/xscale/xscdv.h +++ b/providers/xscale/xscdv.h @@ -83,6 +83,11 @@ struct xscdv_devx_umem_in { int dmabuf_fd; }; +struct xsc_wqe_atomic_seg { + __be64 swap_add; + __be64 compare; +}; + struct xscdv_ctx_allocators { void *(*alloc)(size_t size, void *priv_data); void (*free)(void *ptr, void *priv_data); From 071771e1d13374ec58c187c0c1da7b1b2800ace7 Mon Sep 17 00:00:00 2001 From: Tian Xin Date: Tue, 7 Jul 2026 14:32:00 +0800 Subject: [PATCH 07/13] libxscale: Add CQ polling and completion event handling Implement the completion retrieval path: - poll_cq: Scan the CQ ring for new CQEs, parse them (both good and error completions), and fill ibv_wc structures. - arm_cq / cq_event: Arm the CQ for notification and handle event callbacks. Error CQEs are translated to appropriate verbs status codes, and QPs are moved to error state when fatal errors occur. Flush error CQEs are generated for QPs that have been moved to error state to ensure all pending WRs are completed. Signed-off-by: Tian Xin Signed-off-by: Wei Honggang Signed-off-by: Zhao Qianwei Signed-off-by: Li Qiang Signed-off-by: Yan Lei --- providers/xscale/cq.c | 586 +++++++++++++++++++++++++++++++++++++- providers/xscale/xscale.c | 3 + providers/xscale/xscale.h | 3 + 3 files changed, 590 insertions(+), 2 deletions(-) diff --git a/providers/xscale/cq.c b/providers/xscale/cq.c index 4d864d419..5fac7be4e 100644 --- a/providers/xscale/cq.c +++ b/providers/xscale/cq.c @@ -19,6 +19,108 @@ #include "xscale.h" #include "xsc_hw.h" +enum { + CQ_OK = 0, + CQ_EMPTY = -1, + CQ_POLL_ERR = -2 +}; + +enum { + XSC_CQE_APP_TAG_MATCHING = 1, +}; + +enum { + XSC_CQE_APP_OP_TM_CONSUMED = 0x1, + XSC_CQE_APP_OP_TM_EXPECTED = 0x2, + XSC_CQE_APP_OP_TM_UNEXPECTED = 0x3, + XSC_CQE_APP_OP_TM_NO_TAG = 0x4, + XSC_CQE_APP_OP_TM_APPEND = 0x5, + XSC_CQE_APP_OP_TM_REMOVE = 0x6, + XSC_CQE_APP_OP_TM_NOOP = 0x7, + XSC_CQE_APP_OP_TM_CONSUMED_SW_RDNV = 0x9, + XSC_CQE_APP_OP_TM_CONSUMED_MSG = 0xA, + XSC_CQE_APP_OP_TM_CONSUMED_MSG_SW_RDNV = 0xB, + XSC_CQE_APP_OP_TM_MSG_COMPLETION_CANCELED = 0xC, +}; + +static const uint32_t xsc_cqe_opcode[] = { + [XSC_OPCODE_RDMA_REQ_SEND] = IBV_WC_SEND, + [XSC_OPCODE_RDMA_REQ_SEND_IMMDT] = IBV_WC_SEND, + [XSC_OPCODE_RDMA_RSP_RECV] = IBV_WC_RECV, + [XSC_OPCODE_RDMA_RSP_RECV_IMMDT] = IBV_WC_RECV, + [XSC_OPCODE_RDMA_REQ_WRITE] = IBV_WC_RDMA_WRITE, + [XSC_OPCODE_RDMA_REQ_WRITE_IMMDT] = IBV_WC_RDMA_WRITE, + [XSC_OPCODE_RDMA_RSP_WRITE_IMMDT] = IBV_WC_RECV_RDMA_WITH_IMM, + [XSC_OPCODE_RDMA_REQ_READ] = IBV_WC_RDMA_READ, + [XSC_OPCODE_RDMA_CQE_RAW_SNF] = IBV_WC_RECV, + [XSC_OPCODE_RDMA_REQ_ATOMIC_CMP_AND_SWAP] = IBV_WC_COMP_SWAP, + [XSC_OPCODE_RDMA_REQ_ATOMIC_FETCH_AND_ADD] = IBV_WC_FETCH_ADD, +}; + +static void xsc_stall_poll_cq(void) +{ + int i; + + for (i = 0; i < 60; i++) + __asm__ volatile ("nop"); +} + +static inline int get_qp_ctx(struct xsc_context *xctx, + struct xsc_resource **cur_rsc, + uint32_t qpn) + ALWAYS_INLINE; +static inline int get_qp_ctx(struct xsc_context *xctx, + struct xsc_resource **cur_rsc, + uint32_t qpn) +{ + if (!*cur_rsc || (qpn != (*cur_rsc)->rsn)) { + /* + * We do not have to take the QP table lock here, + * because CQs will be locked while QPs are removed + * from the table. + */ + *cur_rsc = (struct xsc_resource *)xsc_find_qp(xctx, qpn); + if (unlikely(!*cur_rsc)) + return CQ_POLL_ERR; + } + + return CQ_OK; +} + +static inline uint8_t xsc_get_cqe_opcode(struct xsc_context *ctx, + struct xsc_resource **cur_rsc, + struct xsc_cqe *cqe) ALWAYS_INLINE; +static inline uint8_t xsc_get_cqe_opcode(struct xsc_context *ctx, + struct xsc_resource **cur_rsc, + struct xsc_cqe *cqe) +{ + uint8_t msg_opcode = xsc_hw_get_cqe_msg_opcode(ctx->device_id, cqe); + struct xsc_qp *qp; + uint8_t type; + uint8_t with_immdt; + int err; + + type = FIELD_GET(XSC_CQE_TYPE_MASK, cqe->data1); + with_immdt = FIELD_GET(XSC_CQE_WITH_IMMDT_MASK, cqe->data1); + if (xsc_hw_is_err_cqe(ctx->device_id, cqe)) + return type ? XSC_OPCODE_RDMA_RSP_ERROR : XSC_OPCODE_RDMA_REQ_ERROR; + + err = get_qp_ctx(ctx, cur_rsc, RD_LE_16(FIELD_GET(XSC_CQE_QP_ID_MASK, cqe->data1))); + if (unlikely(err)) + goto msg_opcode_err_check; + qp = rsc_to_xqp(*cur_rsc); + if (qp->flags & XSC_QP_FLAG_RAWPACKET_SNIFFER) + return XSC_OPCODE_RDMA_CQE_RAW_SNF; + +msg_opcode_err_check: + if (xsc_check_cqe_msg_opcode(ctx, msg_opcode)) { + printf("rdma cqe msg code should be send/write/read/atomic\n"); + return XSC_OPCODE_RDMA_CQE_ERROR; + } + + return ctx->msg2cqe[msg_opcode][type][with_immdt]; +} + static void *get_cqe(struct xsc_cq *cq, int n) { return cq->active_buf->buf + n * cq->cqe_sz; @@ -42,15 +144,495 @@ static void update_cons_index(struct xsc_cq *cq) xsc_hw_set_cq_ci(ctx->device_id, cq->db, cq->cqn, cq->cons_index); } +static inline void handle_good_req(struct ibv_wc *wc, struct xsc_cqe *cqe, + struct xsc_qp *qp, struct xsc_wq *wq, + uint8_t opcode) +{ + int idx; + struct xsc_send_wqe_ctrl_seg *ctrl; + + wc->opcode = xsc_cqe_opcode[opcode]; + wc->status = IBV_WC_SUCCESS; + idx = xsc_hw_get_wqe_id(to_xctx(qp->ibv_qp->context)->device_id, cqe); + idx >>= (qp->sq.wqe_shift - XSC_BASE_WQE_SHIFT); + idx &= (wq->wqe_cnt - 1); + wc->wr_id = wq->wrid[idx]; + wq->tail = wq->wqe_head[idx] + 1; + if (opcode == XSC_OPCODE_RDMA_REQ_READ) { + ctrl = xsc_get_send_wqe(qp, idx); + wc->byte_len = ctrl->msg_len; + } + wq->need_flush[idx] = 0; + + xsc_dbg(to_xctx(qp->ibv_qp->context)->dbg_fp, XSC_DBG_CQ_CQE, + "wqeid:%u, wq tail:%u\n", idx, wq->tail); +} + +static inline void handle_good_responder( + struct ibv_wc *wc, struct xsc_cqe *cqe, struct xsc_wq *wq, uint8_t opcode) +{ + uint16_t idx; + struct xsc_qp *qp = container_of(wq, struct xsc_qp, rq); + + wc->byte_len = RD_LE_32(cqe->msg_len); + wc->opcode = xsc_cqe_opcode[opcode]; + wc->status = IBV_WC_SUCCESS; + + idx = wq->tail & (wq->wqe_cnt - 1); + wc->wr_id = wq->wrid[idx]; + ++wq->tail; + + xsc_dbg(to_xctx(qp->ibv_qp->context)->dbg_fp, XSC_DBG_CQ_CQE, + "recv cqe idx:%u, len:%u\n", idx, wc->byte_len); +} + +static void dump_cqe(void *buf) +{ + __le32 *p = buf; + int i; + + for (i = 0; i < 8; i += 4) + printf("0x%08x 0x%08x 0x%08x 0x%08x\n", p[i], p[i+1], p[i+2], p[i+3]); +} + +static inline void handle_bad_req( + struct xsc_context *xctx, + struct ibv_wc *wc, struct xsc_cqe *cqe, struct xsc_qp *qp, struct xsc_wq *wq) +{ + int idx; + + wc->status = xsc_hw_cqe_err_status(xctx->device_id, cqe); + wc->vendor_err = xsc_hw_get_cqe_err_code(xctx->device_id, cqe); + idx = xsc_hw_get_wqe_id(xctx->device_id, cqe); + idx >>= (qp->sq.wqe_shift - XSC_BASE_WQE_SHIFT); + idx &= (wq->wqe_cnt - 1); + wq->tail = wq->wqe_head[idx] + 1; + wc->wr_id = wq->wrid[idx]; + wq->need_flush[idx] = 0; + if (wc->status != IBV_WC_WR_FLUSH_ERR) { + printf("%s: got completion with error:\n", xctx->hostname); + dump_cqe(cqe); + } + if (xsc_need_modify_qp_to_err(xctx->device_id, cqe)) { + qp->err_occurred = 1; + xsc_err_state_qp(qp->ibv_qp, qp->ibv_qp->state, IBV_QPS_ERR); + } +} + +static inline bool xsc_need_report_rsp_err_cqe(struct xsc_context *ctx, struct xsc_cqe *cqe) +{ + uint8_t msg_opcode; + bool with_immdt; + + switch (ctx->device_id) { + case XSC_MC_PF_DEV_ID_DIAMOND: + case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: + msg_opcode = xsc_diamond_get_cqe_msg_opcode(cqe); + with_immdt = FIELD_GET(XSC_CQE_WITH_IMMDT_MASK, cqe->data1); + xsc_dbg(ctx->dbg_fp, XSC_DBG_CQ_CQE, "msg_opcode:0x%x with_immdt:%u\n", + msg_opcode, with_immdt); + if (msg_opcode == XSC_MSG_OPCODE_SEND_DIAMOND_NEXT || + (msg_opcode == XSC_MSG_OPCODE_RDMA_WRITE && with_immdt)) + return true; + else + return false; + break; + default: + return true; + } +} + +static inline void handle_bad_responder( + struct xsc_context *xctx, struct ibv_wc *wc, struct xsc_cqe *cqe, + struct xsc_qp *qp, struct xsc_wq *wq, uint8_t *cqe_valid) +{ + int idx; + + if (xsc_need_report_rsp_err_cqe(xctx, cqe)) { + idx = wq->tail & (wq->wqe_cnt - 1); + wc->wr_id = wq->wrid[idx]; + wc->status = xsc_hw_cqe_err_status(xctx->device_id, cqe); + wc->vendor_err = xsc_hw_get_cqe_err_code(xctx->device_id, cqe); + + ++wq->tail; + if (wc->status != IBV_WC_WR_FLUSH_ERR) { + printf("%s: got completion with error:\n", xctx->hostname); + dump_cqe(cqe); + } + } else { + *cqe_valid = 0; + } + + if (xsc_need_modify_qp_to_err(xctx->device_id, cqe)) { + qp->err_occurred = 1; + xsc_err_state_qp(qp->ibv_qp, qp->ibv_qp->state, IBV_QPS_ERR); + } +} + +static inline int xsc_parse_cqe(struct xsc_cq *cq, + struct xsc_cqe *cqe, + struct xsc_resource **cur_rsc, + struct ibv_wc *wc, uint8_t *cqe_valid) + ALWAYS_INLINE; +static inline int xsc_parse_cqe(struct xsc_cq *cq, + struct xsc_cqe *cqe, + struct xsc_resource **cur_rsc, + struct ibv_wc *wc, uint8_t *cqe_valid) +{ + struct xsc_wq *wq; + uint32_t qp_id; + uint8_t opcode; + int err = 0; + struct xsc_qp *xqp = NULL; + struct xsc_context *xctx; + + *cqe_valid = 1; + memset(wc, 0, sizeof(*wc)); + wc->wc_flags = 0; + + xctx = to_xctx(ibv_cq_ex_to_cq(&cq->verbs_cq.cq_ex)->context); + qp_id = RD_LE_16(FIELD_GET(XSC_CQE_QP_ID_MASK, cqe->data1)); + wc->qp_num = qp_id; + opcode = xsc_get_cqe_opcode(xctx, cur_rsc, cqe); + + xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ_CQE, "opcode:0x%x qp_num:%u\n", opcode, qp_id); + switch (opcode) { + case XSC_OPCODE_RDMA_REQ_SEND_IMMDT: + case XSC_OPCODE_RDMA_REQ_WRITE_IMMDT: + wc->wc_flags |= IBV_WC_WITH_IMM; + SWITCH_FALLTHROUGH; + case XSC_OPCODE_RDMA_REQ_SEND: + case XSC_OPCODE_RDMA_REQ_WRITE: + case XSC_OPCODE_RDMA_REQ_READ: + case XSC_OPCODE_RDMA_REQ_ATOMIC_CMP_AND_SWAP: + case XSC_OPCODE_RDMA_REQ_ATOMIC_FETCH_AND_ADD: + case XSC_OPCODE_RDMA_REQ_ATOMIC_8B_MASK_CS: + case XSC_OPCODE_RDMA_REQ_ATOMIC_8B_MASK_FA: + case XSC_OPCODE_RDMA_REQ_ATOMIC_4B_MASK_CS: + case XSC_OPCODE_RDMA_REQ_ATOMIC_4B_MASK_FA: + err = get_qp_ctx(xctx, cur_rsc, qp_id); + if (unlikely(err)) + return CQ_EMPTY; + xqp = rsc_to_xqp(*cur_rsc); + wq = &xqp->sq; + handle_good_req(wc, cqe, xqp, wq, opcode); + break; + case XSC_OPCODE_RDMA_RSP_RECV_IMMDT: + case XSC_OPCODE_RDMA_RSP_WRITE_IMMDT: + wc->wc_flags |= IBV_WC_WITH_IMM; + WR_BE_32(wc->imm_data, RD_LE_32(cqe->imm_data)); + SWITCH_FALLTHROUGH; + case XSC_OPCODE_RDMA_CQE_RAW_SNF: + case XSC_OPCODE_RDMA_RSP_RECV: + err = get_qp_ctx(xctx, cur_rsc, qp_id); + if (unlikely(err)) + return CQ_EMPTY; + xqp = rsc_to_xqp(*cur_rsc); + wq = &xqp->rq; + handle_good_responder(wc, cqe, wq, opcode); + break; + case XSC_OPCODE_RDMA_REQ_ERROR: + err = get_qp_ctx(xctx, cur_rsc, qp_id); + if (unlikely(err)) + return CQ_POLL_ERR; + xqp = rsc_to_xqp(*cur_rsc); + wq = &xqp->sq; + handle_bad_req(xctx, wc, cqe, xqp, wq); + break; + case XSC_OPCODE_RDMA_RSP_ERROR: + err = get_qp_ctx(xctx, cur_rsc, qp_id); + if (unlikely(err)) + return CQ_POLL_ERR; + xqp = rsc_to_xqp(*cur_rsc); + wq = &xqp->rq; + handle_bad_responder(xctx, wc, cqe, xqp, wq, cqe_valid); + break; + case XSC_OPCODE_RDMA_CQE_ERROR: + printf("%s: got completion with cqe format error:\n", xctx->hostname); + dump_cqe(cqe); + SWITCH_FALLTHROUGH; + default: + return CQ_POLL_ERR; + } + return CQ_OK; +} + +static inline int xsc_parse_cqe_lazy(struct xsc_cq *cq, struct xsc_cqe *cqe) ALWAYS_INLINE; +static inline int xsc_parse_cqe_lazy(struct xsc_cq *cq, struct xsc_cqe *cqe) +{ + struct xsc_resource *cur_rsc = NULL; + struct xsc_qp *xqp = NULL; + struct xsc_context *xctx; + struct xsc_wq *wq; + uint32_t qp_id; + uint8_t opcode; + int err = 0; + int idx; + + cq->cqe = cqe; + xctx = to_xctx(ibv_cq_ex_to_cq(&cq->verbs_cq.cq_ex)->context); + qp_id = RD_LE_16(FIELD_GET(XSC_CQE_QP_ID_MASK, cqe->data1)); + opcode = xsc_get_cqe_opcode(xctx, &cur_rsc, cqe); + + xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ_CQE, "opcode:0x%x qp_num:%u\n", opcode, qp_id); + switch (opcode) { + case XSC_OPCODE_RDMA_REQ_SEND_IMMDT: + case XSC_OPCODE_RDMA_REQ_WRITE_IMMDT: + case XSC_OPCODE_RDMA_REQ_SEND: + case XSC_OPCODE_RDMA_REQ_WRITE: + case XSC_OPCODE_RDMA_REQ_READ: + cq->verbs_cq.cq_ex.status = IBV_WC_SUCCESS; + err = get_qp_ctx(xctx, &cur_rsc, qp_id); + if (unlikely(err)) + return CQ_EMPTY; + xqp = rsc_to_xqp(cur_rsc); + wq = &xqp->sq; + idx = xsc_hw_get_wqe_id(xctx->device_id, cqe); + idx >>= (wq->wqe_shift - XSC_BASE_WQE_SHIFT); + idx &= (wq->wqe_cnt - 1); + cq->verbs_cq.cq_ex.wr_id = wq->wrid[idx]; + wq->tail = wq->wqe_head[idx] + 1; + wq->need_flush[idx] = 0; + break; + case XSC_OPCODE_RDMA_RSP_RECV_IMMDT: + case XSC_OPCODE_RDMA_RSP_WRITE_IMMDT: + case XSC_OPCODE_RDMA_RSP_RECV: + cq->verbs_cq.cq_ex.status = IBV_WC_SUCCESS; + err = get_qp_ctx(xctx, &cur_rsc, qp_id); + if (unlikely(err)) + return CQ_EMPTY; + xqp = rsc_to_xqp(cur_rsc); + wq = &xqp->rq; + idx = wq->tail & (wq->wqe_cnt - 1); + cq->verbs_cq.cq_ex.wr_id = wq->wrid[idx]; + ++wq->tail; + break; + case XSC_OPCODE_RDMA_REQ_ERROR: + cq->verbs_cq.cq_ex.status = xsc_hw_cqe_err_status(xctx->device_id, cqe); + err = get_qp_ctx(xctx, &cur_rsc, qp_id); + if (unlikely(err)) + return CQ_POLL_ERR; + xqp = rsc_to_xqp(cur_rsc); + wq = &xqp->sq; + idx = xsc_hw_get_wqe_id(xctx->device_id, cqe); + idx >>= (wq->wqe_shift - XSC_BASE_WQE_SHIFT); + idx &= (wq->wqe_cnt - 1); + wq->tail = wq->wqe_head[idx] + 1; + cq->verbs_cq.cq_ex.wr_id = wq->wrid[idx]; + wq->need_flush[idx] = 0; + if (cq->verbs_cq.cq_ex.status != IBV_WC_WR_FLUSH_ERR) { + printf("%s: got completion with error:\n", xctx->hostname); + dump_cqe(cqe); + } + xqp->ibv_qp->state = IBV_QPS_ERR; + break; + case XSC_OPCODE_RDMA_RSP_ERROR: + cq->verbs_cq.cq_ex.status = xsc_hw_cqe_err_status(xctx->device_id, cqe); + err = get_qp_ctx(xctx, &cur_rsc, qp_id); + if (unlikely(err)) + return CQ_POLL_ERR; + xqp = rsc_to_xqp(cur_rsc); + wq = &xqp->rq; + + ++wq->tail; + if (cq->verbs_cq.cq_ex.status != IBV_WC_WR_FLUSH_ERR) { + printf("%s: got completion with error:\n", xctx->hostname); + dump_cqe(cqe); + } + xqp->ibv_qp->state = IBV_QPS_ERR; + break; + case XSC_OPCODE_RDMA_CQE_ERROR: + printf("%s: got completion with cqe format error:\n", xctx->hostname); + dump_cqe(cqe); + SWITCH_FALLTHROUGH; + default: + return CQ_POLL_ERR; + } + return CQ_OK; +} + +static inline int xsc_poll_one(struct xsc_cq *cq, + struct xsc_resource **cur_rsc, + struct ibv_wc *wc, + int lazy, uint8_t *cqe_valid) + ALWAYS_INLINE; +static inline int xsc_poll_one(struct xsc_cq *cq, + struct xsc_resource **cur_rsc, + struct ibv_wc *wc, + int lazy, uint8_t *cqe_valid) +{ + struct xsc_cqe *cqe = get_sw_cqe(cq, cq->cons_index); + int err = 0; + + if (!cqe) + return CQ_EMPTY; + + ++cq->cons_index; + + /* + * Make sure we read CQ entry contents after we've checked the + * ownership bit. + */ + udma_from_device_barrier(); + if (!lazy) + err = xsc_parse_cqe(cq, cqe, cur_rsc, wc, cqe_valid); + else + err = xsc_parse_cqe_lazy(cq, cqe); + + return err; +} + +static inline void gen_flush_err_cqe(struct xsc_err_state_qp_node *err_node, + uint32_t qp_id, struct xsc_wq *wq, uint32_t idx, + struct ibv_wc *wc) +{ + memset(wc, 0, sizeof(*wc)); + if (err_node->is_sq) { + switch (wq->wr_opcode[idx]) { + case IBV_WR_SEND: + case IBV_WR_SEND_WITH_IMM: + case IBV_WR_SEND_WITH_INV: + wc->opcode = IBV_WC_SEND; + break; + case IBV_WR_RDMA_WRITE: + case IBV_WR_RDMA_WRITE_WITH_IMM: + wc->opcode = IBV_WC_RDMA_WRITE; + break; + case IBV_WR_RDMA_READ: + wc->opcode = IBV_WC_RDMA_READ; + } + } else { + wc->opcode = IBV_WC_RECV; + } + + wc->qp_num = qp_id; + wc->status = IBV_WC_WR_FLUSH_ERR; + wc->vendor_err = XSC_ANDES_ERR_CODE_FLUSH; + wc->wr_id = wq->wrid[idx]; + wq->tail++; + if (err_node->is_sq) + wq->need_flush[idx] = 0; +} + +static inline int xsc_generate_flush_err_cqe(struct ibv_cq *ibcq, + int ne, int *npolled, struct ibv_wc *wc) +{ + uint32_t qp_id = 0; + int sw_npolled = 0; + int ret = 0; + uint32_t idx = 0; + struct xsc_err_state_qp_node *err_qp_node, *tmp; + struct xsc_resource *res = NULL; + struct xsc_context *xctx = to_xctx(ibcq->context); + struct xsc_cq *cq = to_xcq(ibcq); + struct xsc_wq *wq; + + list_for_each_safe(&cq->err_state_qp_list, err_qp_node, tmp, entry) { + if (!err_qp_node) + break; + + sw_npolled = 0; + qp_id = err_qp_node->qp_id; + ret = get_qp_ctx(xctx, &res, qp_id); + if (unlikely(ret)) + continue; + wq = err_qp_node->is_sq ? &(rsc_to_xqp(res)->sq) : &(rsc_to_xqp(res)->rq); + xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ_CQE, "is_sq %d, head %d, tail %d, ne %d, npolled %d, qp_id %d\n", + err_qp_node->is_sq, wq->head, wq->tail, ne, *npolled, qp_id); + + while (wq->head != wq->tail) { + idx = wq->tail & (wq->wqe_cnt - 1); + if (err_qp_node->is_sq && !wq->need_flush[idx]) { + wq->tail++; + continue; + } else { + gen_flush_err_cqe(err_qp_node, err_qp_node->qp_id, wq, + idx, wc + *npolled + sw_npolled); + ++sw_npolled; + if (sw_npolled == (ne - *npolled)) + break; + } + } + + if (wq->head == wq->tail) { + list_del(&err_qp_node->entry); + free(err_qp_node); + } + *npolled += sw_npolled; + if (*npolled == ne) + return 0; + } + + return 0; +} + +static inline int poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) ALWAYS_INLINE; +static inline int poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) +{ + struct xsc_cq *cq = to_xcq(ibcq); + struct xsc_resource *rsc = NULL; + int npolled = 0; + int err = CQ_OK; + uint32_t next_cid = cq->cons_index; + uint8_t cqe_valid = 1; + + if (cq->stall_enable && cq->stall_next_poll) { + cq->stall_next_poll = 0; + xsc_stall_poll_cq(); + } + + xsc_spin_lock(&cq->lock); + for (npolled = 0; npolled < ne; npolled += cqe_valid) { + err = xsc_poll_one(cq, &rsc, wc + npolled, 0, &cqe_valid); + if (err != CQ_OK) + break; + } + + if (err == CQ_EMPTY) { + if (npolled < ne && !(list_empty(&cq->err_state_qp_list))) + xsc_generate_flush_err_cqe(ibcq, ne, &npolled, wc); + } + + udma_to_device_barrier(); + if (next_cid != cq->cons_index) + update_cons_index(cq); + xsc_spin_unlock(&cq->lock); + + if (cq->stall_enable && err == CQ_EMPTY) + cq->stall_next_poll = 1; + + return err == CQ_POLL_ERR ? err : npolled; +} + +int xsc_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) +{ + return poll_cq(ibcq, ne, wc); +} + void xsc_cq_fill_pfns(struct xsc_cq *cq, const struct ibv_cq_init_attr_ex *cq_attr) { } +int xsc_arm_cq(struct ibv_cq *ibvcq, int solicited) +{ + struct xsc_cq *cq = to_xcq(ibvcq); + struct xsc_context *ctx = to_xctx(ibvcq->context); + + xsc_hw_update_cq_db(ctx->device_id, cq->armdb, cq->cqn, cq->cons_index, solicited); + + return 0; +} + +void xsc_cq_event(struct ibv_cq *cq) +{ + to_xcq(cq)->arm_sn++; +} + static int is_equal_rsn(struct xsc_cqe *cqe, uint32_t rsn) { - uint32_t qp_id = FIELD_GET(XSC_CQE_QP_ID_MASK, cqe->data1); + uint32_t qp_id; - qp_id = RD_LE_16(qp_id); + qp_id = RD_LE_16(FIELD_GET(XSC_CQE_QP_ID_MASK, cqe->data1)); return rsn == qp_id; } diff --git a/providers/xscale/xscale.c b/providers/xscale/xscale.c index ba1fac4ce..2231ccf0a 100644 --- a/providers/xscale/xscale.c +++ b/providers/xscale/xscale.c @@ -43,6 +43,9 @@ static const struct verbs_context_ops xsc_ctx_common_ops = { .dereg_mr = xsc_dereg_mr, .create_cq = xsc_create_cq, + .poll_cq = xsc_poll_cq, + .req_notify_cq = xsc_arm_cq, + .cq_event = xsc_cq_event, .resize_cq = xsc_resize_cq, .destroy_cq = xsc_destroy_cq, diff --git a/providers/xscale/xscale.h b/providers/xscale/xscale.h index b228d8102..2c6fed573 100644 --- a/providers/xscale/xscale.h +++ b/providers/xscale/xscale.h @@ -508,6 +508,9 @@ int xsc_alloc_cq_buf(struct xsc_context *xctx, struct xsc_cq *cq, int xsc_free_cq_buf(struct xsc_context *ctx, struct xsc_buf *buf); int xsc_resize_cq(struct ibv_cq *cq, int cqe); int xsc_destroy_cq(struct ibv_cq *cq); +int xsc_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc); +int xsc_arm_cq(struct ibv_cq *cq, int solicited); +void xsc_cq_event(struct ibv_cq *cq); void __xsc_cq_clean(struct xsc_cq *cq, uint32_t qpn); void xsc_cq_clean(struct xsc_cq *cq, uint32_t qpn); From b3e329546618fea58ce175e39bb720022999deec Mon Sep 17 00:00:00 2001 From: Tian Xin Date: Wed, 8 Jul 2026 11:06:53 +0800 Subject: [PATCH 08/13] =?UTF-8?q?libxscale:=20Support=20registration=20of?= =?UTF-8?q?=20dmabuf=E2=80=91based=20memory=20regions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the MR registration to handle dmabuf file descriptors, allowing external (e.g., GPU) memory to be used as RDMA buffers. The new function reg_dmabuf_mr wraps the common ibv_cmd_reg_dmabuf_mr and stores the access flags for later use. Signed-off-by: Tian Xin Signed-off-by: Wei Honggang Signed-off-by: Zhao Qianwei Signed-off-by: Li Qiang Signed-off-by: Yan Lei --- providers/xscale/verbs.c | 21 +++++++++++++++++++++ providers/xscale/xscale.c | 1 + providers/xscale/xscale.h | 2 ++ 3 files changed, 24 insertions(+) diff --git a/providers/xscale/verbs.c b/providers/xscale/verbs.c index 454e77b04..5d02c25c4 100644 --- a/providers/xscale/verbs.c +++ b/providers/xscale/verbs.c @@ -110,6 +110,27 @@ struct ibv_mr *xsc_reg_mr(struct ibv_pd *pd, void *addr, size_t length, return &mr->vmr.ibv_mr; } +struct ibv_mr *xsc_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset, size_t length, + uint64_t iova, int fd, int acc) +{ + struct xsc_mr *mr; + int ret; + + mr = calloc(1, sizeof(*mr)); + if (!mr) + return NULL; + + ret = ibv_cmd_reg_dmabuf_mr(pd, offset, length, iova, fd, acc, + &mr->vmr, NULL); + if (ret) { + free(mr); + return NULL; + } + mr->alloc_flags = acc; + + return &mr->vmr.ibv_mr; +} + int xsc_dereg_mr(struct verbs_mr *vmr) { int ret; diff --git a/providers/xscale/xscale.c b/providers/xscale/xscale.c index 2231ccf0a..036f363d6 100644 --- a/providers/xscale/xscale.c +++ b/providers/xscale/xscale.c @@ -40,6 +40,7 @@ static const struct verbs_context_ops xsc_ctx_common_ops = { .alloc_pd = xsc_alloc_pd, .dealloc_pd = xsc_free_pd, .reg_mr = xsc_reg_mr, + .reg_dmabuf_mr = xsc_reg_dmabuf_mr, .dereg_mr = xsc_dereg_mr, .create_cq = xsc_create_cq, diff --git a/providers/xscale/xscale.h b/providers/xscale/xscale.h index 2c6fed573..37e4a720c 100644 --- a/providers/xscale/xscale.h +++ b/providers/xscale/xscale.h @@ -495,6 +495,8 @@ int xsc_free_pd(struct ibv_pd *pd); struct ibv_mr *xsc_reg_mr(struct ibv_pd *pd, void *addr, size_t length, uint64_t hca_va, int access); +struct ibv_mr *xsc_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset, size_t length, + uint64_t iova, int fd, int acc); int xsc_dereg_mr(struct verbs_mr *mr); struct ibv_cq *xsc_create_cq(struct ibv_context *context, int cqe, struct ibv_comp_channel *channel, int comp_vector); From c6b587a54b236999fb02d70a1cb6566a2034e5dc Mon Sep 17 00:00:00 2001 From: Tian Xin Date: Wed, 8 Jul 2026 12:06:53 +0800 Subject: [PATCH 09/13] libxscale: Add extended work request API (ibv_wr_*) Implement the ibv_wr_api (start, set SGE/inline data, complete). The following operations are supported for RC QPs: - wr_send / wr_send_imm - wr_rdma_write / wr_rdma_write_imm / wr_rdma_read - wr_atomic_cmp_swp / wr_atomic_fetch_add - wr_set_sge / wr_set_sge_list / wr_set_inline_data / wr_set_inline_data_list Signed-off-by: Tian Xin Signed-off-by: Wei Honggang Signed-off-by: Zhao Qianwei Signed-off-by: Li Qiang Signed-off-by: Yan Lei --- providers/xscale/qp.c | 380 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 380 insertions(+) diff --git a/providers/xscale/qp.c b/providers/xscale/qp.c index cf6ef8f49..73296bc2d 100644 --- a/providers/xscale/qp.c +++ b/providers/xscale/qp.c @@ -90,6 +90,18 @@ static int get_len_from_wr(const void *list, int idx) return wr->sg_list[idx].length; } +static void *get_addr_from_buf_list(const void *list, int idx) +{ + const struct ibv_data_buf *buf_list = list; + return buf_list[idx].addr; +} + +static int get_len_from_wr_list(const void *list, int idx) +{ + const struct ibv_data_buf *buf_list = list; + return buf_list[idx].length; +} + static int _set_wqe_inline(void *data_seg, size_t num_buf, const void *list, void *(*get_addr)(const void *, int), int (*get_len)(const void *, int)) @@ -164,6 +176,15 @@ static int set_wqe_inline_from_wr(struct xsc_qp *qp, struct ibv_send_wr *wr, return 0; } +static int set_wqe_inline_from_buf_list(void *data_seg, + size_t num_buf, + const struct ibv_data_buf *buf_list) +{ + return _set_wqe_inline(data_seg, num_buf, buf_list, + get_addr_from_buf_list, + get_len_from_wr_list); +} + static inline void _zero_send_ds(int idx, struct xsc_qp *qp, int keep_ctrl) { void *seg; @@ -471,10 +492,369 @@ int xsc_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, return _xsc_post_send(ibqp, wr, bad_wr); } +static inline void xsc_wr_start(struct ibv_qp_ex *ibqp) +{ + struct xsc_qp *qp = to_xqp((struct ibv_qp *)ibqp); + + xsc_spin_lock(&qp->sq.lock); + + qp->cur_post_rb = qp->sq.cur_post; + qp->err = 0; + qp->nreq = 0; +} + +static inline int xsc_wr_complete(struct ibv_qp_ex *ibqp) +{ + struct xsc_qp *qp = to_xqp((struct ibv_qp *)ibqp); + int err = qp->err; + + if (unlikely(err)) { + qp->sq.cur_post = qp->cur_post_rb; + goto out; + } + + xsc_post_send_db(qp, qp->nreq); +out: + xsc_spin_unlock(&qp->sq.lock); + return err; +} + +static inline void xsc_wr_abort(struct ibv_qp_ex *ibqp) +{ + struct xsc_qp *qp = to_xqp((struct ibv_qp *)ibqp); + + qp->sq.cur_post = qp->cur_post_rb; + + xsc_spin_unlock(&qp->sq.lock); +} + +#define RDMA_REMOTE_DATA_SEG_IDX 1 +#define RDMA_ATOMIC_LOCAL_DATA_SEG 2 +#define RDMA_ATOMIC_INLINE_DATE_SEG 3 +static const int local_ds_base_idx[] = { + [IBV_WR_RDMA_WRITE] = 2, + [IBV_WR_RDMA_WRITE_WITH_IMM] = 2, + [IBV_WR_SEND] = 1, + [IBV_WR_SEND_WITH_IMM] = 1, + [IBV_WR_RDMA_READ] = 2, + [IBV_WR_ATOMIC_CMP_AND_SWP] = 2, + [IBV_WR_ATOMIC_FETCH_AND_ADD] = 2 +}; + +static inline void _common_wqe_init(struct ibv_qp_ex *ibqp, + enum ibv_wr_opcode ib_op) +{ + struct xsc_qp *qp = to_xqp((struct ibv_qp *)ibqp); + struct xsc_send_wqe_ctrl_seg *ctrl; + uint32_t idx; + struct xsc_context *ctx = to_xctx(ibqp->qp_base.context); + + if (unlikely(xsc_wq_overflow(&qp->sq, qp->nreq, + to_xcq(qp->ibv_qp->send_cq)))) { + xsc_dbg(to_xctx(ibqp->qp_base.context)->dbg_fp, XSC_DBG_QP_SEND, + "send work queue overflow\n"); + if (!qp->err) + qp->err = ENOMEM; + + return; + } + + idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1); + clear_send_wqe(idx, qp); + ctrl = xsc_get_send_wqe(qp, idx); + qp->cur_ctrl = ctrl; + qp->cur_ds_num = 0; + qp->cur_data_len = 0; + qp->cur_data = get_seg_wqe(ctrl, local_ds_base_idx[ib_op]); + qp->cur_remote_addr = 0; + qp->cur_remote_key = 0; + ctrl->msg_opcode = ctx->wr2msg[ib_op]; + ctrl->data1 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_CE_MASK, + qp->sq_signal_bits ? + 1 : (ibqp->wr_flags & IBV_SEND_SIGNALED ? 1 : 0)) | + FIELD_PREP(XSC_SWQE_CTRL_SEG_SE_MASK, + ibqp->wr_flags & IBV_SEND_SOLICITED ? 1 : 0) | + FIELD_PREP(XSC_SWQE_CTRL_SEG_IN_LINE_MASK, + ibqp->wr_flags & IBV_SEND_INLINE ? 1 : 0); + qp->sq.wrid[idx] = ibqp->wr_id; + qp->sq.wqe_head[idx] = qp->sq.head + qp->nreq; + qp->sq.wr_opcode[idx] = ib_op; + xsc_hw_set_wqe_id(ctx->device_id, ctrl, + qp->sq.cur_post << (qp->sq.wqe_shift - XSC_BASE_WQE_SHIFT)); +} + +static inline void _common_wqe_finalize(struct ibv_qp_ex *ibqp) +{ + struct xsc_qp *qp = to_xqp((struct ibv_qp *)ibqp); + struct xsc_send_wqe_ctrl_seg *ctrl = qp->cur_ctrl; + struct xsc_wqe_data_seg *remote_seg; + uint32_t idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1); + + ctrl->data0 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_DS_DATA_NUM_MASK, qp->cur_ds_num); + ctrl->msg_len = qp->cur_data_len; + if (ctrl->msg_opcode == XSC_MSG_OPCODE_RDMA_WRITE || + ctrl->msg_opcode == XSC_MSG_OPCODE_RDMA_READ) { + remote_seg = get_seg_wqe(qp->cur_ctrl, RDMA_REMOTE_DATA_SEG_IDX); + set_data_seg_with_value(qp, remote_seg, + qp->cur_remote_addr, + qp->cur_remote_key, + ctrl->msg_len); + } else if (ctrl->msg_opcode == XSC_MSG_OPCODE_RDMA_ATOMIC_CMP_AND_SWAP || + ctrl->msg_opcode == XSC_MSG_OPCODE_RDMA_ATOMIC_FETCH_AND_ADD) { + remote_seg = get_seg_wqe(qp->cur_ctrl, RDMA_REMOTE_DATA_SEG_IDX); + set_data_seg_with_value(qp, remote_seg, + qp->cur_remote_addr, + qp->cur_remote_key, + 8); + ctrl->msg_len = 16; + } + + dump_wqe(0, idx, qp); + qp->sq.cur_post++; + qp->nreq++; + if (FIELD_GET(XSC_SWQE_CTRL_SEG_CE_MASK, ctrl->data1)) + qp->sq.need_flush[idx] = 1; +} + +static inline void xsc_wr_send(struct ibv_qp_ex *ibqp) +{ + _common_wqe_init(ibqp, IBV_WR_SEND); +} + +static inline void xsc_wr_send_imm(struct ibv_qp_ex *ibqp, __be32 imm_data) +{ + struct xsc_qp *qp = to_xqp((struct ibv_qp *)ibqp); + struct xsc_send_wqe_ctrl_seg *ctrl; + + _common_wqe_init(ibqp, IBV_WR_SEND_WITH_IMM); + ctrl = qp->cur_ctrl; + ctrl->data0 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_WITH_IMMDT_MASK, 1); + WR_LE_32(ctrl->opcode_data, RD_BE_32(imm_data)); +} + +static inline void _xsc_wr_rdma(struct ibv_qp_ex *ibqp, + uint32_t rkey, + uint64_t remote_addr, + enum ibv_wr_opcode ib_op) +{ + struct xsc_qp *qp = to_xqp((struct ibv_qp *)ibqp); + + _common_wqe_init(ibqp, ib_op); + qp->cur_remote_addr = remote_addr; + qp->cur_remote_key = rkey; + qp->cur_ds_num++; +} + +static inline void xsc_wr_rdma_write(struct ibv_qp_ex *ibqp, uint32_t rkey, + uint64_t remote_addr) +{ + _xsc_wr_rdma(ibqp, rkey, remote_addr, IBV_WR_RDMA_WRITE); +} + +static inline void xsc_wr_rdma_write_imm(struct ibv_qp_ex *ibqp, uint32_t rkey, + uint64_t remote_addr, __be32 imm_data) +{ + struct xsc_qp *qp = to_xqp((struct ibv_qp *)ibqp); + struct xsc_send_wqe_ctrl_seg *ctrl; + + _xsc_wr_rdma(ibqp, rkey, remote_addr, IBV_WR_RDMA_WRITE_WITH_IMM); + ctrl = qp->cur_ctrl; + ctrl->data0 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_WITH_IMMDT_MASK, 1); + WR_LE_32(ctrl->opcode_data, RD_BE_32(imm_data)); +} + +static inline void xsc_wr_rdma_read(struct ibv_qp_ex *ibqp, uint32_t rkey, + uint64_t remote_addr) +{ + _xsc_wr_rdma(ibqp, rkey, remote_addr, IBV_WR_RDMA_READ); +} + +static inline void _xsc_wr_atomic(struct ibv_qp_ex *ibqp, + uint32_t rkey, + uint64_t remote_addr, + uint64_t compare_add, + uint64_t swap, + enum ibv_wr_opcode ib_op) +{ + struct xsc_qp *qp = to_xqp((struct ibv_qp *)ibqp); + struct xsc_send_wqe_ctrl_seg *ctrl; + struct xsc_wqe_atomic_seg *atomic_seg; + + _common_wqe_init(ibqp, ib_op); + qp->cur_remote_addr = remote_addr; + qp->cur_remote_key = rkey; + qp->cur_ds_num++; + + atomic_seg = get_seg_wqe(qp->cur_ctrl, RDMA_ATOMIC_INLINE_DATE_SEG); + set_atomic_seg_with_value(qp, atomic_seg, ib_op, swap, compare_add); + qp->cur_ds_num++; + + ctrl = (struct xsc_send_wqe_ctrl_seg *)qp->cur_ctrl; + ctrl->data1 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_IN_LINE_MASK, 1); +} + +static inline void xsc_wr_atomic_cmp_swp(struct ibv_qp_ex *ibqp, uint32_t rkey, + uint64_t remote_addr, uint64_t compare, + uint64_t swap) +{ + _xsc_wr_atomic(ibqp, rkey, remote_addr, compare, swap, + IBV_WR_ATOMIC_CMP_AND_SWP); +} + +static inline void xsc_wr_atomic_fetch_add(struct ibv_qp_ex *ibqp, uint32_t rkey, + uint64_t remote_addr, uint64_t add) +{ + _xsc_wr_atomic(ibqp, rkey, remote_addr, add, 0, + IBV_WR_ATOMIC_FETCH_AND_ADD); +} + +static inline void xsc_wr_set_sge(struct ibv_qp_ex *ibqp, uint32_t lkey, uint64_t addr, + uint32_t length) +{ + struct xsc_qp *qp = to_xqp((struct ibv_qp *)ibqp); + struct xsc_wqe_data_seg *data_seg = qp->cur_data; + + if (unlikely(!length)) + return; + + set_data_seg_with_value(qp, data_seg, addr, lkey, length); + qp->cur_ds_num++; + qp->cur_data_len = length; + _common_wqe_finalize(ibqp); +} + +static inline void xsc_wr_set_sge_list(struct ibv_qp_ex *ibqp, size_t num_sge, + const struct ibv_sge *sg_list) +{ + struct xsc_qp *qp = to_xqp((struct ibv_qp *)ibqp); + struct xsc_wqe_data_seg *data_seg = qp->cur_data; + int i; + + if (unlikely(num_sge > qp->sq.max_gs)) { + xsc_dbg(to_xctx(ibqp->qp_base.context)->dbg_fp, XSC_DBG_QP_SEND, + "rdma read, max gs exceeded %lu (max = 1)\n", + num_sge); + if (!qp->err) + qp->err = ENOMEM; + return; + } + + for (i = 0; i < num_sge; i++) { + if (unlikely(!sg_list[i].length)) + continue; + set_local_data_seg_from_sge(qp, data_seg, &sg_list[i]); + data_seg++; + qp->cur_ds_num++; + qp->cur_data_len += sg_list[i].length; + } + _common_wqe_finalize(ibqp); +} + +static inline void xsc_wr_set_inline_data(struct ibv_qp_ex *ibqp, void *addr, + size_t length) +{ + struct xsc_qp *qp = to_xqp((struct ibv_qp *)ibqp); + struct xsc_wqe_data_seg *data_seg = qp->cur_data; + size_t num_buf = 1; + struct ibv_data_buf data_buf = {.addr = addr, .length = length}; + int num_filled_ds = 0; + + if (unlikely(length > qp->max_inline_data)) { + if (!qp->err) + qp->err = ENOMEM; + return; + } + + num_filled_ds = set_wqe_inline_from_buf_list(data_seg, num_buf, &data_buf); + + qp->cur_ds_num += num_filled_ds; + qp->cur_data_len = length; + _common_wqe_finalize(ibqp); +} + +static inline void xsc_wr_set_inline_data_list(struct ibv_qp_ex *ibqp, + size_t num_buf, + const struct ibv_data_buf *buf_list) +{ + struct xsc_qp *qp = to_xqp((struct ibv_qp *)ibqp); + struct xsc_wqe_data_seg *data_seg = qp->cur_data; + int num_filled_ds = 0; + int i; + size_t total_len = 0; + + for (i = 0; i < num_buf; i++) + total_len += buf_list[i].length; + if (unlikely(total_len > qp->max_inline_data)) { + if (!qp->err) + qp->err = ENOMEM; + return; + } + + num_filled_ds = set_wqe_inline_from_buf_list(data_seg, num_buf, buf_list); + + qp->cur_ds_num += num_filled_ds; + qp->cur_data_len = total_len; + _common_wqe_finalize(ibqp); +} + +enum { + XSC_DIAMOND_SUPPORTED_SEND_OPS_FLAGS_RC = + IBV_QP_EX_WITH_SEND | + IBV_QP_EX_WITH_SEND_WITH_IMM | + IBV_QP_EX_WITH_RDMA_WRITE | + IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM | + IBV_QP_EX_WITH_RDMA_READ | + IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP | + IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD, + XSC_DEFAULT_SUPPORTED_SEND_OPS_FLAGS_RC = + IBV_QP_EX_WITH_SEND | + IBV_QP_EX_WITH_SEND_WITH_IMM | + IBV_QP_EX_WITH_RDMA_WRITE | + IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM | + IBV_QP_EX_WITH_RDMA_READ, +}; + +static void fill_wr_pfns_rc(struct ibv_qp_ex *ibqp) +{ + ibqp->wr_send = xsc_wr_send; + ibqp->wr_send_imm = xsc_wr_send_imm; + ibqp->wr_rdma_write = xsc_wr_rdma_write; + ibqp->wr_rdma_write_imm = xsc_wr_rdma_write_imm; + ibqp->wr_rdma_read = xsc_wr_rdma_read; + ibqp->wr_atomic_cmp_swp = xsc_wr_atomic_cmp_swp; + ibqp->wr_atomic_fetch_add = xsc_wr_atomic_fetch_add; + + ibqp->wr_set_sge = xsc_wr_set_sge; + ibqp->wr_set_sge_list = xsc_wr_set_sge_list; + ibqp->wr_set_inline_data = xsc_wr_set_inline_data; + ibqp->wr_set_inline_data_list = xsc_wr_set_inline_data_list; +} + int xsc_qp_fill_wr_pfns(struct xsc_context *ctx, struct xsc_qp *xqp, const struct ibv_qp_init_attr_ex *attr) { + struct ibv_qp_ex *ibqp = &xqp->verbs_qp.qp_ex; + uint64_t ops = attr->send_ops_flags; + uint64_t xsc_flag; + + ibqp->wr_start = xsc_wr_start; + ibqp->wr_complete = xsc_wr_complete; + ibqp->wr_abort = xsc_wr_abort; + if (ctx->device_id == XSC_MC_PF_DEV_ID_DIAMOND) + xsc_flag = XSC_DIAMOND_SUPPORTED_SEND_OPS_FLAGS_RC; + else + xsc_flag = XSC_DEFAULT_SUPPORTED_SEND_OPS_FLAGS_RC; + + switch (attr->qp_type) { + case IBV_QPT_RC: + if (ops & ~xsc_flag) + return EOPNOTSUPP; + fill_wr_pfns_rc(ibqp); + break; + default: + return EOPNOTSUPP; + } return 0; } From 20b85fb0f75db4e635e56571a5e9538216a6f278 Mon Sep 17 00:00:00 2001 From: Tian Xin Date: Thu, 9 Jul 2026 11:06:53 +0800 Subject: [PATCH 10/13] libxscale: Implement extended CQ (cq_ex) polling functions Add the cq_ex interface for efficient completion polling: - start_poll: Begin polling a CQ, lock it, and retrieve the first CQE. - next_poll: Fetch subsequent CQEs without releasing the lock. - end_poll: Release the CQ lock and update the consumer index. Signed-off-by: Tian Xin Signed-off-by: Wei Honggang Signed-off-by: Zhao Qianwei Signed-off-by: Li Qiang Signed-off-by: Yan Lei --- providers/xscale/cq.c | 123 ++++++++++++++++++++++++++++++++++++++ providers/xscale/xscale.c | 1 + 2 files changed, 124 insertions(+) diff --git a/providers/xscale/cq.c b/providers/xscale/cq.c index 5fac7be4e..e58d1abe8 100644 --- a/providers/xscale/cq.c +++ b/providers/xscale/cq.c @@ -609,8 +609,131 @@ int xsc_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) return poll_cq(ibcq, ne, wc); } +static inline int xsc_start_poll(struct ibv_cq_ex *ibcq, + struct ibv_poll_cq_attr *attr) + ALWAYS_INLINE; +static inline int xsc_start_poll(struct ibv_cq_ex *ibcq, + struct ibv_poll_cq_attr *attr) +{ + struct xsc_cq *cq = to_xcq(ibv_cq_ex_to_cq(ibcq)); + int err; + uint8_t cqe_valid = 1; + + xsc_spin_lock(&cq->lock); + err = xsc_poll_one(cq, NULL, NULL, 1, &cqe_valid); + if (err == CQ_EMPTY) + xsc_spin_unlock(&cq->lock); + + return (err == CQ_EMPTY) ? ENOENT : err; +} + +static inline void xsc_end_poll(struct ibv_cq_ex *ibcq) + ALWAYS_INLINE; +static inline void xsc_end_poll(struct ibv_cq_ex *ibcq) +{ + struct xsc_cq *cq = to_xcq(ibv_cq_ex_to_cq(ibcq)); + + udma_to_device_barrier(); + update_cons_index(cq); + xsc_spin_unlock(&cq->lock); +} + +static inline int xsc_next_poll(struct ibv_cq_ex *ibcq) + ALWAYS_INLINE; +static inline int xsc_next_poll(struct ibv_cq_ex *ibcq) +{ + struct xsc_cq *cq = to_xcq(ibv_cq_ex_to_cq(ibcq)); + int err; + uint8_t cqe_valid = 1; + + err = xsc_poll_one(cq, NULL, NULL, 1, &cqe_valid); + + return (err == CQ_EMPTY) ? ENOENT : err; +} + +static inline enum ibv_wc_opcode xsc_wc_read_opcode(struct ibv_cq_ex *ibcq) +{ + struct xsc_cqe *cqe = to_xcq(ibv_cq_ex_to_cq(ibcq))->cqe; + struct xsc_context *xctx = to_xctx(ibv_cq_ex_to_cq(ibcq)->context); + uint8_t opcode = xsc_hw_get_cqe_msg_opcode(xctx->device_id, cqe); + + return xsc_cqe_opcode[opcode]; +} + +static inline uint32_t xsc_wc_read_qp_num(struct ibv_cq_ex *ibcq) +{ + struct xsc_cqe *cqe = to_xcq(ibv_cq_ex_to_cq(ibcq))->cqe; + + return le32toh(FIELD_GET(XSC_CQE_QP_ID_MASK, cqe->data1)); +} + +static inline unsigned int xsc_wc_read_flags(struct ibv_cq_ex *ibcq) +{ + struct xsc_cqe *cqe = to_xcq(ibv_cq_ex_to_cq(ibcq))->cqe; + struct xsc_context *xctx = to_xctx(ibv_cq_ex_to_cq(ibcq)->context); + uint8_t opcode = xsc_hw_get_cqe_msg_opcode(xctx->device_id, cqe); + + switch (opcode) { + case XSC_OPCODE_RDMA_REQ_SEND_IMMDT: + case XSC_OPCODE_RDMA_REQ_WRITE_IMMDT: + case XSC_OPCODE_RDMA_RSP_RECV_IMMDT: + case XSC_OPCODE_RDMA_RSP_WRITE_IMMDT: + return IBV_WC_WITH_IMM; + default: + return 0; + } +} + +static inline uint32_t xsc_wc_read_byte_len(struct ibv_cq_ex *ibcq) +{ + struct xsc_cqe *cqe = to_xcq(ibv_cq_ex_to_cq(ibcq))->cqe; + + return le32toh(cqe->msg_len); +} + +static inline uint32_t xsc_wc_read_vendor_err(struct ibv_cq_ex *ibcq) +{ + struct xsc_cqe *cqe = to_xcq(ibv_cq_ex_to_cq(ibcq))->cqe; + struct xsc_context *xctx = to_xctx(ibv_cq_ex_to_cq(ibcq)->context); + + return xsc_hw_get_cqe_err_code(xctx->device_id, cqe); +} + +static inline __be32 xsc_wc_read_imm_data(struct ibv_cq_ex *ibcq) +{ + struct xsc_cqe *cqe = to_xcq(ibv_cq_ex_to_cq(ibcq))->cqe; + __be32 imm_data; + + WR_BE_32(imm_data, RD_LE_32(cqe->imm_data)); + + return imm_data; +} + +static inline uint64_t xsc_wc_read_completion_ts(struct ibv_cq_ex *ibcq) +{ + struct xsc_cqe *cqe = to_xcq(ibv_cq_ex_to_cq(ibcq))->cqe; + + return le64toh(FIELD_GET(XSC_CQE_TS_MASK, cqe->data2)); +} + void xsc_cq_fill_pfns(struct xsc_cq *cq, const struct ibv_cq_init_attr_ex *cq_attr) { + + cq->verbs_cq.cq_ex.start_poll = xsc_start_poll; + cq->verbs_cq.cq_ex.next_poll = xsc_next_poll; + cq->verbs_cq.cq_ex.end_poll = xsc_end_poll; + + cq->verbs_cq.cq_ex.read_opcode = xsc_wc_read_opcode; + cq->verbs_cq.cq_ex.read_vendor_err = xsc_wc_read_vendor_err; + cq->verbs_cq.cq_ex.read_wc_flags = xsc_wc_read_flags; + if (cq_attr->wc_flags & IBV_WC_EX_WITH_BYTE_LEN) + cq->verbs_cq.cq_ex.read_byte_len = xsc_wc_read_byte_len; + if (cq_attr->wc_flags & IBV_WC_EX_WITH_IMM) + cq->verbs_cq.cq_ex.read_imm_data = xsc_wc_read_imm_data; + if (cq_attr->wc_flags & IBV_WC_EX_WITH_QP_NUM) + cq->verbs_cq.cq_ex.read_qp_num = xsc_wc_read_qp_num; + if (cq_attr->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP) + cq->verbs_cq.cq_ex.read_completion_ts = xsc_wc_read_completion_ts; } int xsc_arm_cq(struct ibv_cq *ibvcq, int solicited) diff --git a/providers/xscale/xscale.c b/providers/xscale/xscale.c index 036f363d6..57b8d61ae 100644 --- a/providers/xscale/xscale.c +++ b/providers/xscale/xscale.c @@ -49,6 +49,7 @@ static const struct verbs_context_ops xsc_ctx_common_ops = { .cq_event = xsc_cq_event, .resize_cq = xsc_resize_cq, .destroy_cq = xsc_destroy_cq, + .create_cq_ex = xsc_create_cq_ex, .create_qp = xsc_create_qp, .query_qp = xsc_query_qp, From c5caf99459283b6767ec52e72b8ba610f36687b9 Mon Sep 17 00:00:00 2001 From: Tian Xin Date: Thu, 9 Jul 2026 12:30:33 +0800 Subject: [PATCH 11/13] libxscale: Add Direct Verbs (DV) extensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce a set of xscale‑specific direct verbs that provide advanced functionality not covered by the standard verbs API: - devx_create_qp / devx_destroy_qp: Create/destroy QPs with user‑supplied memory (e.g., GPU buffers) via devx_umem_in. - devx_create_cq / devx_destroy_cq: Similar for CQs. - devx_alloc_uar / devx_free_uar: Allocate/free User Access Regions (UAR) for direct doorbell ringing. - devx_alloc_sq_uar: Allocate dedicated SQ UAR for multi‑QP performance. - devx_modify_qp: Wrapper for modify_qp with DV context. - devx_get_device_id: Retrieve the hardware device ID. - devx_exp_post_send: Extended post_send that supports masked atomic operations (8‑byte and 4‑byte masked compare‑and‑swap and fetch‑and‑add). Also add ECE (Extended Connection Establishment) support via set_ece and query_ece, enabling negotiation of protocol profiles (e.g., Veroce). These extensions enable high‑performance and custom use cases for applications using the xscale provider. Signed-off-by: Tian Xin Signed-off-by: Wei Honggang Signed-off-by: Zhao Qianwei Signed-off-by: Li Qiang Signed-off-by: Yan Lei --- providers/xscale/qp.c | 250 +++++++++++++++++++++++++ providers/xscale/verbs.c | 145 +++++++++++++++ providers/xscale/xscale.c | 3 + providers/xscale/xscale.h | 8 + providers/xscale/xscdv.h | 375 +++++++++++++++++++++++++++++++++++++- 5 files changed, 774 insertions(+), 7 deletions(-) diff --git a/providers/xscale/qp.c b/providers/xscale/qp.c index 73296bc2d..032c1bb20 100644 --- a/providers/xscale/qp.c +++ b/providers/xscale/qp.c @@ -1020,3 +1020,253 @@ int xsc_err_state_qp(struct ibv_qp *qp, enum ibv_qp_state cur_state, } return 0; } + +int xsc_post_send_mask_atomic(struct ibv_qp *ibqp, + struct xscdv_exp_send_wr *wr, + struct xscdv_exp_send_wr **bad_wr) +{ + struct xsc_qp *qp = to_xqp(ibqp); + void *seg; + struct xsc_send_wqe_ctrl_seg *ctrl; + struct xsc_wqe_data_seg *data_seg; + struct xscdv_wqe_atomic_32_masked_cs_seg *mask_32_cs; + struct xscdv_wqe_atomic_32_masked_fa_seg *mask_32_fa; + struct xscdv_wqe_atomic_64_masked_cs_seg *mask_64_cs_seg0; + struct xscdv_wqe_atomic_64_masked_cs_seg *mask_64_cs_seg1; + struct xscdv_wqe_atomic_64_masked_fa_seg *mask_64_fa; + struct xsc_context *ctx = to_xctx(ibqp->context); + int nreq; + int err = 0; + int i; + unsigned int idx; + unsigned int seg_index = 1; + unsigned int msg_len = 0; + uint64_t cmp_val; + uint64_t swap_val; + uint64_t cmp_mask; + uint64_t swap_mask; + uint64_t add_val; + uint64_t field_boundary; + + if (unlikely(ibqp->state < IBV_QPS_RTS)) { + xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, + "qp state is %u, should not post send\n", ibqp->state); + err = EINVAL; + *bad_wr = wr; + return err; + } + + xsc_spin_lock(&qp->sq.lock); + + for (nreq = 0; wr; ++nreq, wr = wr->next) { + seg_index = 1; + msg_len = 0; + if (unlikely(wr->opcode < 0 || + wr->opcode > XSCDV_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_FETCH_AND_ADD)) { + xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, + "bad opcode %d\n", wr->opcode); + err = EINVAL; + *bad_wr = wr; + goto out; + } + + if (unlikely(xsc_wq_overflow(&qp->sq, nreq, + to_xcq(qp->ibv_qp->send_cq)))) { + xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, + "send work queue overflow\n"); + err = ENOMEM; + *bad_wr = wr; + goto out; + } + + if (unlikely(wr->num_sge > qp->sq.max_gs)) { + xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, + "max gs exceeded %d (max = %d)\n", + wr->num_sge, qp->sq.max_gs); + err = ENOMEM; + *bad_wr = wr; + goto out; + } + + idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1); + clear_send_wqe(idx, qp); + ctrl = seg = xsc_get_send_wqe(qp, idx); + ctrl->data0 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_DS_DATA_NUM_MASK, 0) | + FIELD_PREP(XSC_SWQE_CTRL_SEG_WITH_IMMDT_MASK, 0); + xsc_hw_set_wqe_id(ctx->device_id, ctrl, + qp->sq.cur_post << (qp->sq.wqe_shift - XSC_BASE_WQE_SHIFT)); + ctrl->data1 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_SE_MASK, + wr->send_flags & IBV_SEND_SOLICITED ? 1 : 0) | + FIELD_PREP(XSC_SWQE_CTRL_SEG_CE_MASK, + qp->sq_signal_bits ? + 1 : (wr->send_flags & IBV_SEND_SIGNALED ? 1 : 0)) | + FIELD_PREP(XSC_SWQE_CTRL_SEG_IN_LINE_MASK, + wr->send_flags & IBV_SEND_INLINE ? 1 : 0) | + FIELD_PREP(XSC_SWQE_CTRL_SEG_FENCE_MODE_MASK, + wr->send_flags & IBV_SEND_FENCE ? 1 : 0) | + FIELD_PREP(XSC_SWQE_CTRL_SEG_MASK_MASK, 0); + for (i = 0; i < wr->num_sge; ++i) { + if (likely(wr->sg_list[i].length)) + msg_len += wr->sg_list[i].length; + } + ctrl->msg_len = msg_len; + + if (unlikely(!qp->atomics_enabled)) { + xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, + "atomic operations are not supported\n"); + err = EOPNOTSUPP; + *bad_wr = wr; + goto out; + } + + switch (ibqp->qp_type) { + case IBV_QPT_RC: + switch (wr->opcode) { + case XSCDV_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_CMP_AND_SWAP: + ctrl->msg_len = 32; + ctrl->data1 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_IN_LINE_MASK, 1); + data_seg = get_seg_wqe(ctrl, seg_index); + data_seg->mkey = wr->ext_op.masked_atomics.rkey; + data_seg->va = wr->ext_op.masked_atomics.remote_addr; + data_seg->data0 |= FIELD_PREP(XSC_WQE_DATA_SEG_LENGTH_MASK, 8); + seg_index++; + + data_seg = get_seg_wqe(ctrl, seg_index); + set_data_seg_with_value(qp, + data_seg, + wr->sg_list[0].addr, + wr->sg_list[0].lkey, + 8); + seg_index++; + + mask_64_cs_seg0 = get_seg_wqe(ctrl, seg_index); + swap_val = wr->ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.swap_val; + cmp_val = wr->ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.compare_val; + mask_64_cs_seg0->swap_add = RD_BE_64(swap_val); + mask_64_cs_seg0->compare = RD_BE_64(cmp_val); + seg_index++; + + mask_64_cs_seg1 = get_seg_wqe(ctrl, seg_index); + swap_mask = wr->ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.swap_mask; + cmp_mask = wr->ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.compare_mask; + mask_64_cs_seg1->swap_add = RD_BE_64(swap_mask); + mask_64_cs_seg1->compare = RD_BE_64(cmp_mask); + seg_index++; + + break; + case XSCDV_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_FETCH_AND_ADD: + ctrl->msg_len = 16; + ctrl->data1 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_IN_LINE_MASK, 1); + data_seg = get_seg_wqe(ctrl, seg_index); + data_seg->mkey = wr->ext_op.masked_atomics.rkey; + data_seg->va = wr->ext_op.masked_atomics.remote_addr; + data_seg->data0 |= FIELD_PREP(XSC_WQE_DATA_SEG_LENGTH_MASK, 8); + seg_index++; + + data_seg = get_seg_wqe(ctrl, seg_index); + set_data_seg_with_value(qp, + data_seg, + wr->sg_list[0].addr, + wr->sg_list[0].lkey, + 8); + seg_index++; + + mask_64_fa = get_seg_wqe(ctrl, seg_index); + add_val = wr->ext_op.masked_atomics.wr_data.inline_data.op.fetch_add.add_val; + field_boundary = wr->ext_op.masked_atomics.wr_data.inline_data.op.fetch_add.field_boundary; + mask_64_fa->add_data = RD_BE_64(add_val); + mask_64_fa->field_boundary = RD_BE_64(field_boundary); + seg_index++; + break; + case XSCDV_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_CMP_AND_SWAP: + ctrl->msg_len = 16; + ctrl->data1 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_IN_LINE_MASK, 1); + data_seg = get_seg_wqe(ctrl, seg_index); + data_seg->mkey = wr->ext_op.masked_atomics.rkey; + data_seg->va = wr->ext_op.masked_atomics.remote_addr; + data_seg->data0 |= FIELD_PREP(XSC_WQE_DATA_SEG_LENGTH_MASK, 4); + seg_index++; + + data_seg = get_seg_wqe(ctrl, seg_index); + set_data_seg_with_value(qp, + data_seg, + wr->sg_list[0].addr, + wr->sg_list[0].lkey, + 4); + seg_index++; + + mask_32_cs = get_seg_wqe(ctrl, seg_index); + swap_val = wr->ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.swap_val; + cmp_val = wr->ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.compare_val; + swap_mask = wr->ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.swap_mask; + cmp_mask = wr->ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.compare_mask; + mask_32_cs->swap_data = RD_BE_32((uint32_t)swap_val); + mask_32_cs->compare_data = RD_BE_32((uint32_t)cmp_val); + mask_32_cs->swap_mask = RD_BE_32((uint32_t)swap_mask); + mask_32_cs->compare_mask = RD_BE_32((uint32_t)cmp_mask); + seg_index++; + + break; + case XSCDV_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_FETCH_AND_ADD: + ctrl->msg_len = 8; + ctrl->data1 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_IN_LINE_MASK, 1); + data_seg = get_seg_wqe(ctrl, seg_index); + data_seg->mkey = wr->ext_op.masked_atomics.rkey; + data_seg->va = wr->ext_op.masked_atomics.remote_addr; + data_seg->data0 |= FIELD_PREP(XSC_WQE_DATA_SEG_LENGTH_MASK, 4); + seg_index++; + + data_seg = get_seg_wqe(ctrl, seg_index); + set_data_seg_with_value(qp, + data_seg, + wr->sg_list[0].addr, + wr->sg_list[0].lkey, + 4); + seg_index++; + + mask_32_fa = get_seg_wqe(ctrl, seg_index); + add_val = wr->ext_op.masked_atomics.wr_data.inline_data.op.fetch_add.add_val; + field_boundary = wr->ext_op.masked_atomics.wr_data.inline_data.op.fetch_add.field_boundary; + mask_32_fa->add_data = RD_BE_32((uint32_t)add_val); + mask_32_fa->field_boundary = RD_BE_32((uint32_t)field_boundary); + mask_32_fa->reserved = 0; + seg_index++; + break; + default: + printf("debug: opcode:%u NOT supported\n", wr->opcode); + err = EPERM; + *bad_wr = wr; + goto out; + } + break; + default: + xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, + "qp type:%u NOT supported\n", ibqp->qp_type); + err = EPERM; + *bad_wr = wr; + goto out; + } + + ctrl->msg_opcode = wr->opcode; + if (ctrl->msg_len == 0) { + ctrl->data0 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_DS_DATA_NUM_MASK, 0); + clear_send_wqe_except_ctrl(idx, qp); + } + ctrl->data0 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_DS_DATA_NUM_MASK, seg_index - 1); + qp->sq.wrid[idx] = wr->wr_id; + qp->sq.wqe_head[idx] = qp->sq.head + nreq; + qp->sq.cur_post += 1; + if (FIELD_GET(XSC_SWQE_CTRL_SEG_CE_MASK, ctrl->data1)) + qp->sq.need_flush[idx] = 1; + qp->sq.wr_opcode[idx] = wr->opcode; + + if (unlikely(xsc_debug_mask & XSC_DBG_QP_SEND)) + dump_wqe(0, idx, qp); + } + +out: + xsc_post_send_db(qp, nreq); + xsc_spin_unlock(&qp->sq.lock); + + return err; +} diff --git a/providers/xscale/verbs.c b/providers/xscale/verbs.c index 5d02c25c4..97f063236 100644 --- a/providers/xscale/verbs.c +++ b/providers/xscale/verbs.c @@ -1302,3 +1302,148 @@ int xsc_query_device_ex(struct ibv_context *context, return 0; } + +struct ibv_qp *xscdv_devx_create_qp(struct ibv_context *context, + struct ibv_qp_init_attr_ex *attr, + struct xscdv_devx_umem_in *umem_in) +{ + return create_qp(context, attr, NULL, umem_in); +} + +int xscdv_devx_destroy_qp(struct ibv_qp *ibqp) +{ + return xsc_destroy_qp(ibqp); +} + +struct ibv_cq *xscdv_devx_create_cq(struct ibv_context *context, + const struct ibv_cq_init_attr_ex *cq_attr, + struct xscdv_devx_umem_in *umem_in) +{ + struct ibv_cq_ex *xcq; + + xcq = create_cq(context, cq_attr, XSC_CQ_FLAGS_EXTENDED, NULL, umem_in); + + return xcq ? ibv_cq_ex_to_cq(xcq) : NULL; +} + +int xscdv_devx_destroy_cq(struct ibv_cq *cq) +{ + return xsc_destroy_cq(cq); +} + +struct xscdv_devx_uar *xscdv_devx_alloc_uar(struct ibv_context *context, uint32_t flags) +{ + struct xscdv_devx_uar *uar = NULL; + struct xsc_context *xsc_ctx = to_xctx(context); + + uar = calloc(1, sizeof(*uar)); + + if (!uar) { + xsc_err("alloc uar failed!\n"); + return NULL; + } + + uar->cq_db = xsc_ctx->cqm_reg_va + + (xsc_ctx->cqm_next_cid_reg & (xsc_ctx->page_size - 1)); + uar->cq_armdb = xsc_ctx->cqm_armdb_va + + (xsc_ctx->cqm_armdb & (xsc_ctx->page_size - 1)); + uar->sq_db = xsc_ctx->sqm_reg_va + + (xsc_ctx->qpm_tx_db & (xsc_ctx->page_size - 1)); + uar->rq_db = xsc_ctx->rqm_reg_va + + (xsc_ctx->qpm_rx_db & (xsc_ctx->page_size - 1)); + + return uar; +} + +void xscdv_devx_free_uar(struct xscdv_devx_uar *dv_devx_uar) +{ + if (dv_devx_uar) + free(dv_devx_uar); +} + +struct xscdv_devx_sq_uar *xscdv_devx_alloc_sq_uar(struct ibv_context *context, uint32_t qpn) +{ + struct xscdv_devx_sq_uar *uar = NULL; + struct xsc_context *xsc_ctx = to_xctx(context); + + uar = calloc(1, sizeof(*uar)); + + if (!uar) { + xsc_err("alloc uar failed!\n"); + return NULL; + } + + if (xsc_ctx->multidb_num) { + uar->sq_db = xsc_ctx->mdb_base + + (xsc_ctx->tx_multidb_base & (xsc_ctx->page_size - 1)) + + (qpn & (xsc_ctx->multidb_num - 1)) * XSC_MULTI_SQ_DB_STEP; + uar->dedicated = 1; + } else { + uar->sq_db = xsc_ctx->sqm_reg_va + + (xsc_ctx->qpm_tx_db & (xsc_ctx->page_size - 1)); + } + + return uar; +} + +void xscdv_devx_free_sq_uar(struct xscdv_devx_sq_uar *uar) +{ + if (uar) + free(uar); +} + +int xscdv_devx_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + int attr_mask) +{ + return xsc_modify_qp(qp, attr, attr_mask); +} + +uint32_t xscdv_devx_get_device_id(struct ibv_qp *qp) +{ + struct xsc_context *ctx = to_xctx(qp->context); + + return ctx->device_id; +} + +int xsc_set_ece(struct ibv_qp *qp, struct ibv_ece *ece) +{ + struct xsc_context *ctx = to_xctx(qp->context); + struct xsc_qp *xqp = to_xqp(qp); + uint32_t vid = ece->vendor_id & 0xfffff; + uint32_t l_profile = ctx->rdma_proto_mode == RDMA_PROTO_VEROCE ? VEROCE_PROFILE_P3 : 0; + uint32_t profile = min(l_profile, (ece->options >> 24) & 0xF); + + if (ece->comp_mask) { + errno = EINVAL; + return errno; + } + + if (vid != VEROCE_VENDOR_ID) { + errno = EINVAL; + return errno; + } + + xqp->profile = profile; + xqp->set_ece = 1; + return 0; +} + +int xsc_query_ece(struct ibv_qp *qp, struct ibv_ece *ece) +{ + struct xsc_qp *xqp = to_xqp(qp); + uint32_t profile = xqp->get_ece << 24; + + if (!xqp->set_ece) + xqp->profile = 0; + ece->vendor_id = VEROCE_VENDOR_ID; + ece->options = profile; + ece->comp_mask = 0; + return 0; +} + +int xscdv_devx_exp_post_send(struct ibv_qp *ibqp, + struct xscdv_exp_send_wr *wr, + struct xscdv_exp_send_wr **bad_wr) +{ + return xsc_post_send_mask_atomic(ibqp, wr, bad_wr); +} diff --git a/providers/xscale/xscale.c b/providers/xscale/xscale.c index 57b8d61ae..9c97af6ca 100644 --- a/providers/xscale/xscale.c +++ b/providers/xscale/xscale.c @@ -58,6 +58,9 @@ static const struct verbs_context_ops xsc_ctx_common_ops = { .create_qp_ex = xsc_create_qp_ex, .post_send = xsc_post_send, .post_recv = xsc_post_recv, + + .query_ece = xsc_query_ece, + .set_ece = xsc_set_ece, }; static void open_debug_file(struct xsc_context *ctx) diff --git a/providers/xscale/xscale.h b/providers/xscale/xscale.h index 37e4a720c..e04a718b0 100644 --- a/providers/xscale/xscale.h +++ b/providers/xscale/xscale.h @@ -156,6 +156,8 @@ struct xsc_spinlock { #define XSC_MV_HOST_VF_DEV_ID 0x1152 #define XSC_MV_SOC_PF_DEV_ID 0x1153 +#define XSC_MULTI_SQ_DB_STEP 16 + enum { RDMA_PROTO_ROCEV2, RDMA_PROTO_VEROCE, @@ -586,4 +588,10 @@ static inline void *xsc_get_send_wqe(struct xsc_qp *qp, int n) return qp->sq_start + (n << qp->sq.wqe_shift); } +int xsc_set_ece(struct ibv_qp *qp, struct ibv_ece *ece); +int xsc_query_ece(struct ibv_qp *qp, struct ibv_ece *ece); +int xsc_post_send_mask_atomic(struct ibv_qp *ibqp, + struct xscdv_exp_send_wr *wr, + struct xscdv_exp_send_wr **bad_wr); + #endif /* XSC_H */ diff --git a/providers/xscale/xscdv.h b/providers/xscale/xscdv.h index 1d57bfd83..bcfe6ee18 100644 --- a/providers/xscale/xscdv.h +++ b/providers/xscale/xscdv.h @@ -24,6 +24,30 @@ extern "C" { #endif +#define XSC_BITS_PER_LONG (8 * sizeof(long)) +#define XSC_BITS_PER_LONG_LONG (8 * sizeof(long long)) +#define XSC_BITS_TO_LONGS(nr) (((nr) + XSC_BITS_PER_LONG - 1) / XSC_BITS_PER_LONG) + +#define XSC_GENMASK(h, l) \ + (((~0UL) - (1UL << (l)) + 1) & (~0UL >> (XSC_BITS_PER_LONG - 1 - (h)))) +#define XSC_GENMASK_ULL(h, l) \ + (((~0ULL) << (l)) & (~0ULL >> (XSC_BITS_PER_LONG_LONG - 1 - (h)))) + +#define XSC_BIT(nr) (1UL << (nr)) +#define XSC_BIT_ULL(nr) (1ULL << (nr)) + +#define __xsc_bf_shf(x) (__builtin_ffsll(x) - 1) + +#define XSC_FIELD_PREP(_mask, _val) \ + ({ \ + ((typeof(_mask))(_val) << __xsc_bf_shf(_mask)) & (_mask); \ + }) + +#define XSC_FIELD_GET(_mask, _reg) \ + ({ \ + (typeof(_mask))(((_reg) & (_mask)) >> __xsc_bf_shf(_mask)); \ + }) + enum xscdv_cq_init_attr_mask { XSCDV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE = 1 << 0, XSCDV_CQ_INIT_ATTR_MASK_FLAGS = 1 << 1, @@ -70,6 +94,149 @@ struct xscdv_qp_init_attr { struct xscdv_dc_init_attr dc_init_attr; }; +struct xsc_wqe_atomic_seg { + __be64 swap_add; + __be64 compare; +}; + +struct xscdv_ctx_allocators { + void *(*alloc)(size_t size, void *priv_data); + void (*free)(void *ptr, void *priv_data); + void *data; +}; + +enum xscdv_msg_type { + XSCDV_MSG_OPCODE_SEND = 0, + XSCDV_MSG_OPCODE_RDMA_WRITE = 1, + XSCDV_MSG_OPCODE_RDMA_READ = 2, + XSCDV_MSG_OPCODE_MAD = 3, + XSCDV_MSG_OPCODE_RDMA_ACK = 4, + XSCDV_MSG_OPCODE_RDMA_ACK_READ = 5, + XSCDV_MSG_OPCODE_RDMA_CNP = 6, + XSCDV_MSG_OPCODE_RAW = 7, + XSCDV_MSG_OPCODE_VIRTIO_NET = 8, + XSCDV_MSG_OPCODE_VIRTIO_BLK = 9, + XSCDV_MSG_OPCODE_RAW_TPE = 10, + XSCDV_MSG_OPCODE_INIT_QP_REQ = 11, + XSCDV_MSG_OPCODE_INIT_QP_RSP = 12, + XSCDV_MSG_OPCODE_INIT_PATH_REQ = 13, + XSCDV_MSG_OPCODE_INIT_PATH_RSP = 14, + XSCDV_MSG_OPCODE_SO_WRITE = 22, + + XSCDV_MSG_OPCODE_RDMA_ATOMIC_CMP_AND_SWAP = 26, + XSCDV_MSG_OPCODE_RDMA_ATOMIC_FETCH_AND_ADD = 27, + + XSCDV_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_CMP_AND_SWAP = 31, + XSCDV_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_FETCH_AND_ADD = 32, + XSCDV_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_CMP_AND_SWAP = 33, + XSCDV_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_FETCH_AND_ADD = 34, +}; + +enum { + XSCDV_MSG_OPCODE_SEND_DIAMOND_NEXT = 3, +}; + +struct xscdv_wqe_atomic_64_masked_cs_seg { + __be64 swap_add; + __be64 compare; +}; + +struct xscdv_wqe_atomic_64_masked_fa_seg { + __be64 add_data; + __be64 field_boundary; +}; + +struct xscdv_wqe_atomic_32_masked_cs_seg { + __be32 swap_data; + __be32 compare_data; + __be32 swap_mask; + __be32 compare_mask; +}; + +struct xscdv_wqe_atomic_32_masked_fa_seg { + __be32 add_data; + __be32 field_boundary; + __be64 reserved; +}; + +struct xscdv_cqe { + uint8_t placeholder1; + __le32 data1; +#define XSCDV_CQE_QP_ID_MASK XSC_GENMASK(14, 0) +#define XSCDV_CQE_SE_MASK XSC_BIT(1) +#define XSCDV_CQE_HAS_PPH_MASK XSC_BIT(2) +#define XSCDV_CQE_TYPE_MASK XSC_BIT(3) +#define XSCDV_CQE_WITH_IMMDT_MASK XSC_BIT(4) +#define XSCDV_CQE_CSUM_ERR_MASK XSC_GENMASK(8, 5) + __le32 imm_data; + __le32 msg_len; + __le32 vni; + __le64 data2; +#define XSCDV_CQE_TS_MASK XSC_GENMASK_ULL(47, 0) + __le16 wqe_id; + uint8_t placeholder2; + uint8_t rsv1; + __le16 rsv2[2]; + __le16 data3; +#define XSCDV_CQE_OWNER_MASK XSC_BIT(15) +}; + +struct xscdv_cqe64 { + struct xscdv_cqe cqe; + uint8_t padding[32]; +}; + + +struct xscdv_wqe_ctrl_seg { + uint8_t msg_opcode; + uint8_t data0; +#define XSCDV_SWQE_CTRL_SEG_WITH_IMMDT_MASK XSC_BIT(0) +#define XSCDV_SWQE_CTRL_SEG_CSUM_EN_MASK XSC_GENMASK(2, 1) +#define XSCDV_SWQE_CTRL_SEG_DS_DATA_NUM_MASK XSC_GENMASK(7, 3) + __le16 wqe_id; + __le32 msg_len; + __le32 opcode_data; + uint8_t data1; +#define XSCDV_SWQE_CTRL_SEG_SE_MASK XSC_BIT(0) +#define XSCDV_SWQE_CTRL_SEG_CE_MASK XSC_BIT(1) +#define XSCDV_SWQE_CTRL_SEG_IN_LINE_MASK XSC_BIT(2) +#define XSCDV_SWQE_CTRL_SEG_FENCE_MODE_MASK XSC_GENMASK(4, 3) +#define XSCDV_SWQE_CTRL_SEG_MASK_MASK XSC_GENMASK(6, 5) + uint8_t rsv[3]; +}; + +struct xscdv_diamond_wqe_ctrl_seg { + uint8_t msg_opcode; + uint8_t data0; +#define XSCDV_DIAMOND_SWQE_CTRL_SEG_WITH_IMMDT_MASK XSC_BIT(0) +#define XSCDV_DIAMOND_SWQE_CTRL_SEG_CSUM_EN_MASK XSC_GENMASK(2, 1) +#define XSCDV_DIAMOND_SWQE_CTRL_SEG_DS_DATA_NUM_MASK XSC_GENMASK(7, 3) + __le16 rsv1; + __le32 msg_len; + __le32 opcode_data; + __le32 data1; +#define XSCDV_DIAMOND_SWQE_CTRL_SEG_SE_MASK XSC_BIT(0) +#define XSCDV_DIAMOND_SWQE_CTRL_SEG_CE_MASK XSC_BIT(1) +#define XSCDV_DIAMOND_SWQE_CTRL_SEG_IN_LINE_MASK XSC_BIT(2) +#define XSCDV_DIAMOND_SWQE_CTRL_SEG_FENCE_MODE_MASK XSC_GENMASK(4, 3) +#define XSCDV_DIAMOND_SWQE_CTRL_SEG_MASK_MASK XSC_GENMASK(6, 5) +#define XSCDV_DIAMOND_SWQE_CTRL_SEG_WQE_ID_MASK XSC_GENMASK(31, 12) +}; + +struct xscdv_wqe_data_seg { + union { + struct { + uint32_t data0; +#define XSCDV_WQE_DATA_SEG_LENGTH_MASK XSC_GENMASK(31, 1) + __le32 mkey; + __le64 va; + }; + struct { + uint8_t in_line_data[16]; + }; + }; +}; + enum xscdv_devx_umem_in_mask { XSCDV_UMEM_MASK_DMABUF = 1 << 0, }; @@ -83,17 +250,211 @@ struct xscdv_devx_umem_in { int dmabuf_fd; }; -struct xsc_wqe_atomic_seg { - __be64 swap_add; - __be64 compare; +struct xscdv_devx_uar { + __le32 *cq_db; + __le32 *cq_armdb; + __le32 *sq_db; + __le32 *rq_db; }; -struct xscdv_ctx_allocators { - void *(*alloc)(size_t size, void *priv_data); - void (*free)(void *ptr, void *priv_data); - void *data; +struct xscdv_devx_sq_uar { + void *sq_db; + uint32_t dedicated; +}; + +struct xscdv_andes_cq_doorbell { + uint32_t raw; +#define XSCDV_ANDES_CQ_DOORBELL_CQ_NEXT_CID_MASK XSC_GENMASK(15, 0) +#define XSCDV_ANDES_CQ_DOORBELL_CQ_ID_MASK XSC_GENMASK(30, 16) +#define XSCDV_ANDES_CQ_DOORBELL_ARM_MASK XSC_BIT(31) +}; + +struct xscdv_andes_send_doorbell { + uint32_t raw; +#define XSCDV_ANDES_SEND_DOORBELL_NEXT_PID_MASK XSC_GENMASK(15, 0) +#define XSCDV_ANDES_SEND_DOORBELL_QP_ID_MASK XSC_GENMASK(30, 16) }; +union xscdv_andes_recv_doorbell { + uint32_t raw; +#define XSCDV_ANDES_RECV_DOORBELL_NEXT_PID_MASK XSC_GENMASK(12, 0) +#define XSCDV_ANDES_RECV_DOORBELL_QP_ID_MASK XSC_GENMASK(27, 13) +}; + +struct xscdv_diamond_cqe { + uint32_t flags_qp_id_err_code; +#define XSCDV_DIAMOND_CQE_ERR_CODE_MASK XSC_GENMASK(7, 0) +#define XSCDV_DIAMOND_CQE_QP_ID_MASK XSC_GENMASK(22, 8) +#define XSCDV_DIAMOND_CQE_FLAGS_MASK XSC_GENMASK(31, 23) + uint32_t imm_data; + uint32_t msg_len; + uint32_t vni; + uint32_t ts_l; + uint32_t ts_h; + uint32_t wqe_id_rsv_opcode; +#define XSCDV_DIAMOND_CQE_OPCODE_MASK XSC_GENMASK(7, 0) +#define XSCDV_DIAMOND_CQE_WQE_ID_MASK XSC_GENMASK(31, 12) + uint32_t owner_rsv; +#define XSCDV_DIAMOND_CQE_OWNER_MASK XSC_BIT(31) +}; + +struct xscdv_diamond_cqe64 { + struct xscdv_diamond_cqe cqe; + uint8_t padding[32]; +}; + +struct xscdv_diamond_data_seg { + uint32_t length; + uint32_t key; + uint64_t addr; +}; + +struct xscdv_diamond_next_cq_doorbell { + uint64_t raw; +#define XSCDV_DIAMOND_NEXT_CQ_DOORBELL_NEXT_CID_MASK XSC_GENMASK(22, 0) +#define XSCDV_DIAMOND_NEXT_CQ_DOORBELL_NEXT_CQ_ID_MASK XSC_GENMASK(38, 23) +#define XSCDV_DIAMOND_NEXT_CQ_DOORBELL_CQ_STA_MASK XSC_GENMASK(40, 39) +}; + +struct xscdv_diamond_next_send_doorbell { + uint64_t raw; +#define XSCDV_DIAMOND_NEXT_SEND_DOORBELL_NEXT_PID_MASK XSC_GENMASK(20, 0) +#define XSCDV_DIAMOND_NEXT_SEND_DOORBELL_QP_ID_MASK XSC_GENMASK(36, 21) +}; + +struct xscdv_diamond_next_recv_doorbell { + struct { + uint64_t next_pid:18; + uint64_t qp_id:16; + }; + uint64_t raw; +#define XSCDV_DIAMOND_NEXT_RECV_DOORBELL_NEXT_PID_MASK XSC_GENMASK(17, 0) +#define XSCDV_DIAMOND_NEXT_RECV_DOORBELL_QP_ID_MASK XSC_GENMASK(33, 18) +}; + +enum { + XSCDV_BASE_WQE_SHIF = 4, + XSCDV_CQE_SHIFT = 5, + XSCDV_RCV_WQE_SHIFT = 6, + XSCDV_SND_WQE_SHIFT = 7, +}; + +struct xscdv_exp_cmp_swap { + uint64_t compare_mask; + uint64_t compare_val; + uint64_t swap_val; + uint64_t swap_mask; +}; + +struct xscdv_exp_fetch_add { + uint64_t add_val; + uint64_t field_boundary; +}; + +struct xscdv_exp_send_wr { + uint64_t wr_id; + struct xscdv_exp_send_wr *next; + struct ibv_sge *sg_list; + int num_sge; + enum xscdv_msg_type opcode; + unsigned int send_flags; + /* When opcode is *_WITH_IMM: Immediate data in network byte order. + * When opcode is *_INV: Stores the rkey to invalidate + */ + union { + __be32 imm_data; + uint32_t invalidate_rkey; + }; + union { + struct { + uint64_t remote_addr; + uint32_t rkey; + } rdma; + struct { + uint64_t remote_addr; + uint64_t compare_add; + uint64_t swap; + uint32_t rkey; + } atomic; + struct { + struct ibv_ah *ah; + uint32_t remote_qpn; + uint32_t remote_qkey; + } ud; + } wr; + union { + struct { + uint32_t remote_srqn; + } xrc; + } qp_type; + union { + struct { + struct ibv_mw *mw; + uint32_t rkey; + struct ibv_mw_bind_info bind_info; + } bind_mw; + struct { + void *hdr; + uint16_t hdr_sz; + uint16_t mss; + } tso; + }; + + union { + struct { + uint32_t log_arg_sz; + uint64_t remote_addr; + uint32_t rkey; + union { + struct { + union { + struct xscdv_exp_cmp_swap cmp_swap; + struct xscdv_exp_fetch_add fetch_add; + } op; + } inline_data; + } wr_data; + } masked_atomics; + } ext_op; +}; + + +#define XSCDV_MS_PF_DEV_ID 0x1111 +#define XSCDV_MS_VF_DEV_ID 0x1112 +#define XSCDV_MC_PF_DEV_ID_DIAMOND 0x1021 +#define XSCDV_MC_PF_DEV_ID_DIAMOND_NEXT 0x1023 + +static inline +uint8_t xscdv_get_cqe_owner(struct xscdv_cqe *cqe) +{ + return XSC_FIELD_GET(XSCDV_CQE_OWNER_MASK, cqe->data3); +} + +static inline +void xscdv_set_cqe_owner(struct xscdv_cqe *cqe, uint8_t val) +{ + cqe->data3 |= XSC_FIELD_PREP(XSCDV_CQE_OWNER_MASK, val & 0x1); +} + +struct ibv_cq *xscdv_devx_create_cq(struct ibv_context *context, + const struct ibv_cq_init_attr_ex *cq_attr, + struct xscdv_devx_umem_in *umem_in); +struct ibv_qp *xscdv_devx_create_qp(struct ibv_context *context, + struct ibv_qp_init_attr_ex *attr, + struct xscdv_devx_umem_in *umem_in); + +int xscdv_devx_destroy_qp(struct ibv_qp *ibqp); +int xscdv_devx_destroy_cq(struct ibv_cq *cq); +struct xscdv_devx_uar *xscdv_devx_alloc_uar(struct ibv_context *context, uint32_t flags); +void xscdv_devx_free_uar(struct xscdv_devx_uar *dv_devx_uar); +int xscdv_devx_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask); +uint32_t xscdv_devx_get_device_id(struct ibv_qp *qp); +struct xscdv_devx_sq_uar *xscdv_devx_alloc_sq_uar(struct ibv_context *context, uint32_t qpn); +void xscdv_devx_free_sq_uar(struct xscdv_devx_sq_uar *uar); +int xscdv_devx_exp_post_send(struct ibv_qp *ibqp, + struct xscdv_exp_send_wr *wr, + struct xscdv_exp_send_wr **bad_wr); + + #ifdef __cplusplus } #endif From fe85e5ffe952c742e265c80ac8260b464735c650 Mon Sep 17 00:00:00 2001 From: Tian Xin Date: Thu, 16 Jul 2026 20:05:36 +0800 Subject: [PATCH 12/13] libxscale: Add xscale provider to packaging files This patch integrates the xscale RDMA provider into the rdma-core build and packaging system for both Debian and RPM-based distributions. Signed-off-by: Tian Xin Signed-off-by: Wei Honggang Signed-off-by: Zhao Qianwei Signed-off-by: Li Qiang Signed-off-by: Yan Lei --- README.md | 1 + debian/control | 1 + debian/copyright | 4 ++++ debian/ibverbs-providers.install | 1 + debian/libibverbs-dev.install | 2 ++ redhat/rdma-core.spec | 4 ++++ 6 files changed, 13 insertions(+) diff --git a/README.md b/README.md index 2a0c2737d..95629a448 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ is included: - rdma_rxe.ko - siw.ko - vmw_pvrdma.ko + - xsc_ib.ko Additional service daemons are provided for: - srp_daemon (ib_srp.ko) diff --git a/debian/control b/debian/control index b7a731abe..c2058fcfe 100644 --- a/debian/control +++ b/debian/control @@ -104,6 +104,7 @@ Description: User space provider drivers for libibverbs - rxe: A software implementation of the RoCE protocol - siw: A software implementation of the iWarp protocol - vmw_pvrdma: VMware paravirtual RDMA device + - xscale: Yunsilicon RDMA device Package: ibverbs-utils Architecture: linux-any diff --git a/debian/copyright b/debian/copyright index e32de0f17..b3ca41e0c 100644 --- a/debian/copyright +++ b/debian/copyright @@ -233,6 +233,10 @@ Files: providers/vmw_pvrdma/* Copyright: 2012-2016 VMware, Inc. License: BSD-2-clause or GPL-2 +Files: providers/xscale/* +Copyright: 2021-2026, Yunsilicon Technology Co., Ltd. +License: GPL-2 + Files: rdma-ndd/* Copyright: 2004-2016, Intel Corporation. License: BSD-MIT or GPL-2 diff --git a/debian/ibverbs-providers.install b/debian/ibverbs-providers.install index 38d870e9e..a8028fccb 100644 --- a/debian/ibverbs-providers.install +++ b/debian/ibverbs-providers.install @@ -6,3 +6,4 @@ usr/lib/*/libionic.so.* usr/lib/*/libmana.so.* usr/lib/*/libmlx4.so.* usr/lib/*/libmlx5.so.* +usr/lib/*/libxscale.so.* diff --git a/debian/libibverbs-dev.install b/debian/libibverbs-dev.install index 141ad7975..99f0fb23e 100644 --- a/debian/libibverbs-dev.install +++ b/debian/libibverbs-dev.install @@ -29,6 +29,8 @@ usr/lib/*/libmlx4.a usr/lib/*/libmlx4.so usr/lib/*/libmlx5.a usr/lib/*/libmlx5.so +usr/lib/*/libxscale.a +usr/lib/*/libxscale.so usr/lib/*/pkgconfig/libefa.pc usr/lib/*/pkgconfig/libhns.pc usr/lib/*/pkgconfig/libibverbs.pc diff --git a/redhat/rdma-core.spec b/redhat/rdma-core.spec index 8cdf46054..174b3dbff 100644 --- a/redhat/rdma-core.spec +++ b/redhat/rdma-core.spec @@ -178,6 +178,8 @@ Provides: libocrdma = %{version}-%{release} Obsoletes: libocrdma < %{version}-%{release} Provides: librxe = %{version}-%{release} Obsoletes: librxe < %{version}-%{release} +Provides: libxscale = %{version}-%{release} +Obsoletes: libxscale < %{version}-%{release} %description -n libibverbs libibverbs is a library that allows userspace processes to use RDMA @@ -205,6 +207,7 @@ Device-specific plug-in ibverbs userspace drivers are included: - librxe: A software implementation of the RoCE protocol - libsiw: A software implementation of the iWarp protocol - libvmw_pvrdma: VMware paravirtual RDMA device +- libxscale: Yunsilicon RDMA device %package -n libibverbs-utils Summary: Examples for the libibverbs library @@ -604,6 +607,7 @@ fi %{_libdir}/libmana.so.* %{_libdir}/libmlx5.so.* %{_libdir}/libmlx4.so.* +%{_libdir}/libxscale.so.* %config(noreplace) %{_sysconfdir}/libibverbs.d/*.driver %doc %{_docdir}/%{name}/libibverbs.md %{_prefix}/lib/sysusers.d/rdma.conf From ef7f8db46f509d326c645ca41d7693474f2669ae Mon Sep 17 00:00:00 2001 From: Tian Xin Date: Sat, 18 Jul 2026 22:10:56 +0800 Subject: [PATCH 13/13] libxscale: fix pipeline issues fix pipeline issues Signed-off-by: Tian Xin --- CMakeLists.txt | 4 +- debian/libibverbs-dev.install | 4 + kernel-headers/CMakeLists.txt | 2 + kernel-headers/rdma/xsc-abi.h | 1 + kernel-headers/rdma/xsc_user_ioctl_verbs.h | 8 ++ providers/xscale/CMakeLists.txt | 3 +- providers/xscale/cq.c | 28 ++--- providers/xscale/libxsc.map | 2 +- providers/xscale/qp.c | 74 ++++++------ providers/xscale/verbs.c | 12 +- providers/xscale/xsc_api.h | 8 ++ providers/xscale/xsc_hsi.h | 26 ++--- providers/xscale/xsc_hw.h | 124 ++++++++++----------- providers/xscale/xscale.c | 2 +- suse/rdma-core.spec | 17 +++ 15 files changed, 178 insertions(+), 137 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2d23ba42d..1d0108fd6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -823,7 +823,9 @@ add_subdirectory(providers/mthca) add_subdirectory(providers/ocrdma) add_subdirectory(providers/qedr) add_subdirectory(providers/vmw_pvrdma) -add_subdirectory(providers/xscale) +if(NOT CMAKE_SIZEOF_VOID_P EQUAL 4) + add_subdirectory(providers/xscale) +endif() endif() add_subdirectory(providers/hfi1verbs) diff --git a/debian/libibverbs-dev.install b/debian/libibverbs-dev.install index 99f0fb23e..e85961c5d 100644 --- a/debian/libibverbs-dev.install +++ b/debian/libibverbs-dev.install @@ -8,6 +8,9 @@ usr/include/infiniband/mlx4dv.h usr/include/infiniband/mlx5_api.h usr/include/infiniband/mlx5_user_ioctl_verbs.h usr/include/infiniband/mlx5dv.h +usr/include/infiniband/xsc_api.h +usr/include/infiniband/xsc_user_ioctl_verbs.h +usr/include/infiniband/xscdv.h usr/include/infiniband/opcode.h usr/include/infiniband/sa-kern-abi.h usr/include/infiniband/sa.h @@ -38,6 +41,7 @@ usr/lib/*/pkgconfig/libionic.pc usr/lib/*/pkgconfig/libmana.pc usr/lib/*/pkgconfig/libmlx4.pc usr/lib/*/pkgconfig/libmlx5.pc +usr/lib/*/pkgconfig/libxscale.pc usr/share/man/man3/efadv_*.3 usr/share/man/man3/hnsdv_*.3 usr/share/man/man3/ibv_* diff --git a/kernel-headers/CMakeLists.txt b/kernel-headers/CMakeLists.txt index eff602986..32997ca6b 100644 --- a/kernel-headers/CMakeLists.txt +++ b/kernel-headers/CMakeLists.txt @@ -28,6 +28,8 @@ publish_internal_headers(rdma rdma/siw-abi.h rdma/vmw_pvrdma-abi.h rdma/xsc-abi.h + rdma/xsc_user_ioctl_cmds.h + rdma/xsc_user_ioctl_verbs.h ) publish_internal_headers(rdma/hfi diff --git a/kernel-headers/rdma/xsc-abi.h b/kernel-headers/rdma/xsc-abi.h index ada746bfa..e32992838 100644 --- a/kernel-headers/rdma/xsc-abi.h +++ b/kernel-headers/rdma/xsc-abi.h @@ -10,6 +10,7 @@ #include #include /* For ETH_ALEN. */ #include +#include enum { XSC_HW_FEATURE_FLAG_SUPPORT_CQE64 = 1 << 0, diff --git a/kernel-headers/rdma/xsc_user_ioctl_verbs.h b/kernel-headers/rdma/xsc_user_ioctl_verbs.h index fee13cc73..a7a0295fb 100644 --- a/kernel-headers/rdma/xsc_user_ioctl_verbs.h +++ b/kernel-headers/rdma/xsc_user_ioctl_verbs.h @@ -9,6 +9,10 @@ #include +#ifdef __cplusplus +extern "C" { +#endif + enum xsc_ib_uapi_flow_action_flags { XSC_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA = 1 << 0, }; @@ -25,5 +29,9 @@ enum xsc_ib_uapi_flow_action_packet_reformat_type { XSC_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL = 0x3, }; +#ifdef __cplusplus +} +#endif + #endif diff --git a/providers/xscale/CMakeLists.txt b/providers/xscale/CMakeLists.txt index 046eaa04f..0436689f9 100644 --- a/providers/xscale/CMakeLists.txt +++ b/providers/xscale/CMakeLists.txt @@ -1,5 +1,5 @@ rdma_shared_provider(xscale libxsc.map - 1 1.24.${PACKAGE_VERSION} + 1 1.0.${PACKAGE_VERSION} xscale.c verbs.c cq.c @@ -9,7 +9,6 @@ rdma_shared_provider(xscale libxsc.map publish_headers(infiniband ../../kernel-headers/rdma/xsc_user_ioctl_verbs.h - ../../kernel-headers/rdma/xsc_user_ioctl_cmds.h xsc_api.h xscdv.h ) diff --git a/providers/xscale/cq.c b/providers/xscale/cq.c index e58d1abe8..656b92fae 100644 --- a/providers/xscale/cq.c +++ b/providers/xscale/cq.c @@ -100,12 +100,12 @@ static inline uint8_t xsc_get_cqe_opcode(struct xsc_context *ctx, uint8_t with_immdt; int err; - type = FIELD_GET(XSC_CQE_TYPE_MASK, cqe->data1); - with_immdt = FIELD_GET(XSC_CQE_WITH_IMMDT_MASK, cqe->data1); + type = FIELD_GET(XSC_CQE_TYPE_MASK, le32toh(cqe->data1)); + with_immdt = FIELD_GET(XSC_CQE_WITH_IMMDT_MASK, le32toh(cqe->data1)); if (xsc_hw_is_err_cqe(ctx->device_id, cqe)) return type ? XSC_OPCODE_RDMA_RSP_ERROR : XSC_OPCODE_RDMA_REQ_ERROR; - err = get_qp_ctx(ctx, cur_rsc, RD_LE_16(FIELD_GET(XSC_CQE_QP_ID_MASK, cqe->data1))); + err = get_qp_ctx(ctx, cur_rsc, FIELD_GET(XSC_CQE_QP_ID_MASK, le32toh(cqe->data1))); if (unlikely(err)) goto msg_opcode_err_check; qp = rsc_to_xqp(*cur_rsc); @@ -160,7 +160,7 @@ static inline void handle_good_req(struct ibv_wc *wc, struct xsc_cqe *cqe, wq->tail = wq->wqe_head[idx] + 1; if (opcode == XSC_OPCODE_RDMA_REQ_READ) { ctrl = xsc_get_send_wqe(qp, idx); - wc->byte_len = ctrl->msg_len; + wc->byte_len = le32toh(ctrl->msg_len); } wq->need_flush[idx] = 0; @@ -174,7 +174,7 @@ static inline void handle_good_responder( uint16_t idx; struct xsc_qp *qp = container_of(wq, struct xsc_qp, rq); - wc->byte_len = RD_LE_32(cqe->msg_len); + wc->byte_len = le32toh(cqe->msg_len); wc->opcode = xsc_cqe_opcode[opcode]; wc->status = IBV_WC_SUCCESS; @@ -228,7 +228,7 @@ static inline bool xsc_need_report_rsp_err_cqe(struct xsc_context *ctx, struct x case XSC_MC_PF_DEV_ID_DIAMOND: case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: msg_opcode = xsc_diamond_get_cqe_msg_opcode(cqe); - with_immdt = FIELD_GET(XSC_CQE_WITH_IMMDT_MASK, cqe->data1); + with_immdt = FIELD_GET(XSC_CQE_WITH_IMMDT_MASK, le32toh(cqe->data1)); xsc_dbg(ctx->dbg_fp, XSC_DBG_CQ_CQE, "msg_opcode:0x%x with_immdt:%u\n", msg_opcode, with_immdt); if (msg_opcode == XSC_MSG_OPCODE_SEND_DIAMOND_NEXT || @@ -291,7 +291,7 @@ static inline int xsc_parse_cqe(struct xsc_cq *cq, wc->wc_flags = 0; xctx = to_xctx(ibv_cq_ex_to_cq(&cq->verbs_cq.cq_ex)->context); - qp_id = RD_LE_16(FIELD_GET(XSC_CQE_QP_ID_MASK, cqe->data1)); + qp_id = FIELD_GET(XSC_CQE_QP_ID_MASK, le32toh(cqe->data1)); wc->qp_num = qp_id; opcode = xsc_get_cqe_opcode(xctx, cur_rsc, cqe); @@ -371,7 +371,7 @@ static inline int xsc_parse_cqe_lazy(struct xsc_cq *cq, struct xsc_cqe *cqe) cq->cqe = cqe; xctx = to_xctx(ibv_cq_ex_to_cq(&cq->verbs_cq.cq_ex)->context); - qp_id = RD_LE_16(FIELD_GET(XSC_CQE_QP_ID_MASK, cqe->data1)); + qp_id = FIELD_GET(XSC_CQE_QP_ID_MASK, le32toh(cqe->data1)); opcode = xsc_get_cqe_opcode(xctx, &cur_rsc, cqe); xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ_CQE, "opcode:0x%x qp_num:%u\n", opcode, qp_id); @@ -664,7 +664,7 @@ static inline uint32_t xsc_wc_read_qp_num(struct ibv_cq_ex *ibcq) { struct xsc_cqe *cqe = to_xcq(ibv_cq_ex_to_cq(ibcq))->cqe; - return le32toh(FIELD_GET(XSC_CQE_QP_ID_MASK, cqe->data1)); + return FIELD_GET(XSC_CQE_QP_ID_MASK, le32toh(cqe->data1)); } static inline unsigned int xsc_wc_read_flags(struct ibv_cq_ex *ibcq) @@ -713,7 +713,7 @@ static inline uint64_t xsc_wc_read_completion_ts(struct ibv_cq_ex *ibcq) { struct xsc_cqe *cqe = to_xcq(ibv_cq_ex_to_cq(ibcq))->cqe; - return le64toh(FIELD_GET(XSC_CQE_TS_MASK, cqe->data2)); + return FIELD_GET(XSC_CQE_TS_MASK, le64toh(cqe->data2)); } void xsc_cq_fill_pfns(struct xsc_cq *cq, const struct ibv_cq_init_attr_ex *cq_attr) @@ -755,7 +755,7 @@ static int is_equal_rsn(struct xsc_cqe *cqe, uint32_t rsn) { uint32_t qp_id; - qp_id = RD_LE_16(FIELD_GET(XSC_CQE_QP_ID_MASK, cqe->data1)); + qp_id = FIELD_GET(XSC_CQE_QP_ID_MASK, le32toh(cqe->data1)); return rsn == qp_id; } @@ -799,9 +799,9 @@ void __xsc_cq_clean(struct xsc_cq *cq, uint32_t rsn) ++nfreed; } else if (nfreed) { dest = get_cqe(cq, (prod_index + nfreed) & (cq->verbs_cq.cq_ex.cqe - 1)); - owner_bit = FIELD_GET(XSC_CQE_OWNER_MASK, dest->data3); + owner_bit = FIELD_GET(XSC_CQE_OWNER_MASK, le16toh(dest->data3)); memcpy(dest, cqe, cq->cqe_sz); - dest->data3 |= FIELD_PREP(XSC_CQE_OWNER_MASK, owner_bit); + dest->data3 |= htole16(FIELD_PREP(XSC_CQE_OWNER_MASK, owner_bit)); } } @@ -858,7 +858,7 @@ int xsc_alloc_cq_buf(struct xsc_context *xctx, struct xsc_cq *cq, buf->length = umem_in->size; xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, - "cq_buf(%p),cq_buf_size=0x%lx is given by gpu memory,infact need size=0x%x.\n", + "cq_buf(%p),cq_buf_size=0x%zx is given by gpu memory,infact need size=0x%x.\n", buf->buf, buf->length, (nent * cqe_sz)); } else { if (xsc_use_huge("HUGE_CQ")) diff --git a/providers/xscale/libxsc.map b/providers/xscale/libxsc.map index 2af4bc736..3b694ea0b 100644 --- a/providers/xscale/libxsc.map +++ b/providers/xscale/libxsc.map @@ -1,6 +1,6 @@ /* Export symbols should be added below according to Documentation/versioning.md document. */ -XSC_1.0 { +XSCALE_1.0 { local: *; }; diff --git a/providers/xscale/qp.c b/providers/xscale/qp.c index 032c1bb20..63801f981 100644 --- a/providers/xscale/qp.c +++ b/providers/xscale/qp.c @@ -155,7 +155,7 @@ static int set_wqe_inline_from_wr(struct xsc_qp *qp, struct ibv_send_wr *wr, { void *data_seg; unsigned int seg_index; - int msg_len = ctrl->msg_len; + int msg_len = le32toh(ctrl->msg_len); int filled_ds_num; if (wr->opcode == IBV_WR_SEND || wr->opcode == IBV_WR_SEND_WITH_IMM) @@ -356,7 +356,7 @@ static inline int _xsc_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, if (likely(wr->sg_list[i].length)) msg_len += wr->sg_list[i].length; } - ctrl->msg_len = msg_len; + ctrl->msg_len = htole32(msg_len); if (unlikely(wr->opcode == IBV_WR_RDMA_READ && msg_len == 0)) { xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, @@ -402,7 +402,7 @@ static inline int _xsc_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, goto out; } - ctrl->msg_len = 16; + ctrl->msg_len = htole32(16); data_seg = get_seg_wqe(ctrl, seg_index); set_data_seg_with_value(qp, data_seg, @@ -591,14 +591,14 @@ static inline void _common_wqe_finalize(struct ibv_qp_ex *ibqp) uint32_t idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1); ctrl->data0 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_DS_DATA_NUM_MASK, qp->cur_ds_num); - ctrl->msg_len = qp->cur_data_len; + ctrl->msg_len = htole32(qp->cur_data_len); if (ctrl->msg_opcode == XSC_MSG_OPCODE_RDMA_WRITE || ctrl->msg_opcode == XSC_MSG_OPCODE_RDMA_READ) { remote_seg = get_seg_wqe(qp->cur_ctrl, RDMA_REMOTE_DATA_SEG_IDX); set_data_seg_with_value(qp, remote_seg, qp->cur_remote_addr, qp->cur_remote_key, - ctrl->msg_len); + qp->cur_data_len); } else if (ctrl->msg_opcode == XSC_MSG_OPCODE_RDMA_ATOMIC_CMP_AND_SWAP || ctrl->msg_opcode == XSC_MSG_OPCODE_RDMA_ATOMIC_FETCH_AND_ADD) { remote_seg = get_seg_wqe(qp->cur_ctrl, RDMA_REMOTE_DATA_SEG_IDX); @@ -606,7 +606,7 @@ static inline void _common_wqe_finalize(struct ibv_qp_ex *ibqp) qp->cur_remote_addr, qp->cur_remote_key, 8); - ctrl->msg_len = 16; + ctrl->msg_len = htole32(16); } dump_wqe(0, idx, qp); @@ -732,7 +732,7 @@ static inline void xsc_wr_set_sge_list(struct ibv_qp_ex *ibqp, size_t num_sge, if (unlikely(num_sge > qp->sq.max_gs)) { xsc_dbg(to_xctx(ibqp->qp_base.context)->dbg_fp, XSC_DBG_QP_SEND, - "rdma read, max gs exceeded %lu (max = 1)\n", + "rdma read, max gs exceeded %zu (max = 1)\n", num_sge); if (!qp->err) qp->err = ENOMEM; @@ -858,7 +858,7 @@ int xsc_qp_fill_wr_pfns(struct xsc_context *ctx, return 0; } -int xsc_post_recv_dump_wqe = 1; +static int xsc_post_recv_dump_wqe = 1; int xsc_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr) { @@ -1109,7 +1109,7 @@ int xsc_post_send_mask_atomic(struct ibv_qp *ibqp, if (likely(wr->sg_list[i].length)) msg_len += wr->sg_list[i].length; } - ctrl->msg_len = msg_len; + ctrl->msg_len = htole32(msg_len); if (unlikely(!qp->atomics_enabled)) { xsc_dbg(to_xctx(ibqp->context)->dbg_fp, XSC_DBG_QP_SEND, @@ -1123,12 +1123,12 @@ int xsc_post_send_mask_atomic(struct ibv_qp *ibqp, case IBV_QPT_RC: switch (wr->opcode) { case XSCDV_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_CMP_AND_SWAP: - ctrl->msg_len = 32; + ctrl->msg_len = htole32(32); ctrl->data1 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_IN_LINE_MASK, 1); data_seg = get_seg_wqe(ctrl, seg_index); - data_seg->mkey = wr->ext_op.masked_atomics.rkey; - data_seg->va = wr->ext_op.masked_atomics.remote_addr; - data_seg->data0 |= FIELD_PREP(XSC_WQE_DATA_SEG_LENGTH_MASK, 8); + data_seg->mkey = htole32(wr->ext_op.masked_atomics.rkey); + data_seg->va = htole64(wr->ext_op.masked_atomics.remote_addr); + data_seg->data0 |= htole32(FIELD_PREP(XSC_WQE_DATA_SEG_LENGTH_MASK, 8)); seg_index++; data_seg = get_seg_wqe(ctrl, seg_index); @@ -1142,25 +1142,25 @@ int xsc_post_send_mask_atomic(struct ibv_qp *ibqp, mask_64_cs_seg0 = get_seg_wqe(ctrl, seg_index); swap_val = wr->ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.swap_val; cmp_val = wr->ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.compare_val; - mask_64_cs_seg0->swap_add = RD_BE_64(swap_val); - mask_64_cs_seg0->compare = RD_BE_64(cmp_val); + mask_64_cs_seg0->swap_add = htobe64(swap_val); + mask_64_cs_seg0->compare = htobe64(cmp_val); seg_index++; mask_64_cs_seg1 = get_seg_wqe(ctrl, seg_index); swap_mask = wr->ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.swap_mask; cmp_mask = wr->ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.compare_mask; - mask_64_cs_seg1->swap_add = RD_BE_64(swap_mask); - mask_64_cs_seg1->compare = RD_BE_64(cmp_mask); + mask_64_cs_seg1->swap_add = htobe64(swap_mask); + mask_64_cs_seg1->compare = htobe64(cmp_mask); seg_index++; break; case XSCDV_MSG_OPCODE_RDMA_ATOMIC_8B_MSK_FETCH_AND_ADD: - ctrl->msg_len = 16; + ctrl->msg_len = htole32(16); ctrl->data1 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_IN_LINE_MASK, 1); data_seg = get_seg_wqe(ctrl, seg_index); - data_seg->mkey = wr->ext_op.masked_atomics.rkey; - data_seg->va = wr->ext_op.masked_atomics.remote_addr; - data_seg->data0 |= FIELD_PREP(XSC_WQE_DATA_SEG_LENGTH_MASK, 8); + data_seg->mkey = htole32(wr->ext_op.masked_atomics.rkey); + data_seg->va = htole64(wr->ext_op.masked_atomics.remote_addr); + data_seg->data0 |= htole32(FIELD_PREP(XSC_WQE_DATA_SEG_LENGTH_MASK, 8)); seg_index++; data_seg = get_seg_wqe(ctrl, seg_index); @@ -1174,17 +1174,17 @@ int xsc_post_send_mask_atomic(struct ibv_qp *ibqp, mask_64_fa = get_seg_wqe(ctrl, seg_index); add_val = wr->ext_op.masked_atomics.wr_data.inline_data.op.fetch_add.add_val; field_boundary = wr->ext_op.masked_atomics.wr_data.inline_data.op.fetch_add.field_boundary; - mask_64_fa->add_data = RD_BE_64(add_val); - mask_64_fa->field_boundary = RD_BE_64(field_boundary); + mask_64_fa->add_data = htobe64(add_val); + mask_64_fa->field_boundary = htobe64(field_boundary); seg_index++; break; case XSCDV_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_CMP_AND_SWAP: - ctrl->msg_len = 16; + ctrl->msg_len = htole32(16); ctrl->data1 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_IN_LINE_MASK, 1); data_seg = get_seg_wqe(ctrl, seg_index); - data_seg->mkey = wr->ext_op.masked_atomics.rkey; - data_seg->va = wr->ext_op.masked_atomics.remote_addr; - data_seg->data0 |= FIELD_PREP(XSC_WQE_DATA_SEG_LENGTH_MASK, 4); + data_seg->mkey = htole32(wr->ext_op.masked_atomics.rkey); + data_seg->va = htole64(wr->ext_op.masked_atomics.remote_addr); + data_seg->data0 |= htole32(FIELD_PREP(XSC_WQE_DATA_SEG_LENGTH_MASK, 4)); seg_index++; data_seg = get_seg_wqe(ctrl, seg_index); @@ -1200,20 +1200,20 @@ int xsc_post_send_mask_atomic(struct ibv_qp *ibqp, cmp_val = wr->ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.compare_val; swap_mask = wr->ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.swap_mask; cmp_mask = wr->ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.compare_mask; - mask_32_cs->swap_data = RD_BE_32((uint32_t)swap_val); - mask_32_cs->compare_data = RD_BE_32((uint32_t)cmp_val); - mask_32_cs->swap_mask = RD_BE_32((uint32_t)swap_mask); - mask_32_cs->compare_mask = RD_BE_32((uint32_t)cmp_mask); + mask_32_cs->swap_data = htobe32((uint32_t)swap_val); + mask_32_cs->compare_data = htobe32((uint32_t)cmp_val); + mask_32_cs->swap_mask = htobe32((uint32_t)swap_mask); + mask_32_cs->compare_mask = htobe32((uint32_t)cmp_mask); seg_index++; break; case XSCDV_MSG_OPCODE_RDMA_ATOMIC_4B_MSK_FETCH_AND_ADD: - ctrl->msg_len = 8; + ctrl->msg_len = htole32(8); ctrl->data1 |= FIELD_PREP(XSC_SWQE_CTRL_SEG_IN_LINE_MASK, 1); data_seg = get_seg_wqe(ctrl, seg_index); - data_seg->mkey = wr->ext_op.masked_atomics.rkey; - data_seg->va = wr->ext_op.masked_atomics.remote_addr; - data_seg->data0 |= FIELD_PREP(XSC_WQE_DATA_SEG_LENGTH_MASK, 4); + data_seg->mkey = htole32(wr->ext_op.masked_atomics.rkey); + data_seg->va = htole64(wr->ext_op.masked_atomics.remote_addr); + data_seg->data0 |= htole32(FIELD_PREP(XSC_WQE_DATA_SEG_LENGTH_MASK, 4)); seg_index++; data_seg = get_seg_wqe(ctrl, seg_index); @@ -1227,8 +1227,8 @@ int xsc_post_send_mask_atomic(struct ibv_qp *ibqp, mask_32_fa = get_seg_wqe(ctrl, seg_index); add_val = wr->ext_op.masked_atomics.wr_data.inline_data.op.fetch_add.add_val; field_boundary = wr->ext_op.masked_atomics.wr_data.inline_data.op.fetch_add.field_boundary; - mask_32_fa->add_data = RD_BE_32((uint32_t)add_val); - mask_32_fa->field_boundary = RD_BE_32((uint32_t)field_boundary); + mask_32_fa->add_data = htobe32((uint32_t)add_val); + mask_32_fa->field_boundary = htobe32((uint32_t)field_boundary); mask_32_fa->reserved = 0; seg_index++; break; diff --git a/providers/xscale/verbs.c b/providers/xscale/verbs.c index 97f063236..c7af7a27b 100644 --- a/providers/xscale/verbs.c +++ b/providers/xscale/verbs.c @@ -201,7 +201,7 @@ static void init_cqe(struct xsc_context *xctx, struct xsc_buf *buf, int ncqe, in for (i = 0; i < ncqe; i++) { struct xsc_cqe *cqe = (struct xsc_cqe *)(buf->buf + i * cqe_sz); - cqe->data3 |= FIELD_PREP(XSC_CQE_OWNER_MASK, 1); + cqe->data3 |= htole16(FIELD_PREP(XSC_CQE_OWNER_MASK, 1)); } } else { xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, "cq buff is given by gpu,do nothing in host.\n"); @@ -250,7 +250,7 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *context, } if (cq_attr->wc_flags & ~CREATE_CQ_SUPPORTED_WC_FLAGS) { - xsc_err("unsupported wc flags:0x%lx\n", cq_attr->wc_flags); + xsc_err("unsupported wc flags:0x%" PRIx64 "\n", cq_attr->wc_flags); errno = ENOTSUP; return NULL; } @@ -326,7 +326,7 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *context, cq->flags |= XSC_CQ_FLAGS_OWNED_BY_GPU; cmd_drv->dmabuf_fd = umem_in->dmabuf_fd; cmd_drv->dmabuf_sz = umem_in->size; - xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, "cq_buf %p, buffer size:0x%lx, dmabuf_fd:%d\n", + xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, "cq_buf %p, buffer size:0x%zx, dmabuf_fd:%d\n", umem_in->addr, umem_in->size, umem_in->dmabuf_fd); } else { cmd_drv->dmabuf_fd = -1; @@ -373,7 +373,7 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *context, env = getenv("XSC_DISABLE_FLUSH_ERROR"); cq->disable_flush_error_cqe = env ? true : false; - xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, "cqe count:%u cqn:%u cq_buf:%p cq_buf_sz:0x%lx dmabuf_fd:%d\n", + xsc_dbg(xctx->dbg_fp, XSC_DBG_CQ, "cqe count:%u cqn:%u cq_buf:%p cq_buf_sz:0x%zx dmabuf_fd:%d\n", cq->cqe_cnt, cq->cqn, cq->buf_a.buf, cq->buf_a.length, umem_in ? umem_in->dmabuf_fd : -1); list_head_init(&cq->err_state_qp_list); @@ -682,7 +682,7 @@ static int xsc_alloc_qp_buf(struct ibv_context *context, qp->buf.length = umem_in->size; xsc_dbg(to_xctx(context)->dbg_fp, XSC_DBG_QP, - "qp_buf(%p),qp_buf_size=0x%lx is given by gpu memory, infact need size=0x%x.\n", + "qp_buf(%p),qp_buf_size=0x%zx is given by gpu memory, infact need size=0x%x.\n", qp->buf.buf, qp->buf.length, size); } else { /* compatibility support */ @@ -924,7 +924,7 @@ static struct ibv_qp *create_qp(struct ibv_context *context, qp->flags |= XSC_QP_FLAG_OWNED_BY_GPU; cmd.dmabuf_fd = umem_in->dmabuf_fd; cmd.dmabuf_sz = umem_in->size; - xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, "wq_buf %p, buffer size:0x%lx, dmabuf_fd:%d\n", + xsc_dbg(ctx->dbg_fp, XSC_DBG_QP, "wq_buf %p, buffer size:0x%zx, dmabuf_fd:%d\n", umem_in->addr, umem_in->size, umem_in->dmabuf_fd); } else { cmd.dmabuf_fd = -1; diff --git a/providers/xscale/xsc_api.h b/providers/xscale/xsc_api.h index c2f568eea..b5b489195 100644 --- a/providers/xscale/xsc_api.h +++ b/providers/xscale/xsc_api.h @@ -9,6 +9,10 @@ #include +#ifdef __cplusplus +extern "C" { +#endif + #define xscdv_flow_action_flags xsc_ib_uapi_flow_action_flags #define XSCDV_FLOW_ACTION_FLAGS_REQUIRE_METADATA \ XSC_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA @@ -32,4 +36,8 @@ enum xsc_qp_create_flags { XSC_QP_CREATE_WORC = 1 << 4, }; +#ifdef __cplusplus +} +#endif + #endif diff --git a/providers/xscale/xsc_hsi.h b/providers/xscale/xsc_hsi.h index fb5da97d1..d6879dcc6 100644 --- a/providers/xscale/xsc_hsi.h +++ b/providers/xscale/xsc_hsi.h @@ -14,23 +14,23 @@ #define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16)) #define lower_32_bits(n) ((uint32_t)(n)) -#define DMA_LO_LE(x) __cpu_to_le32(lower_32_bits(x)) -#define DMA_HI_LE(x) __cpu_to_le32(upper_32_bits(x)) +#define DMA_LO_LE(x) htole32(lower_32_bits(x)) +#define DMA_HI_LE(x) htole32(upper_32_bits(x)) #define DMA_REGPAIR_LE(x, val) do { \ (x).hi = DMA_HI_LE((val)); \ (x).lo = DMA_LO_LE((val)); \ } while (0) -#define WR_LE_16(x, val) do { (x) = __cpu_to_le16(val); } while (0) -#define WR_LE_32(x, val) do { (x) = __cpu_to_le32(val); } while (0) -#define WR_LE_64(x, val) do { (x) = __cpu_to_le64(val); } while (0) +#define WR_LE_16(x, val) do { (x) = htole16(val); } while (0) +#define WR_LE_32(x, val) do { (x) = htole32(val); } while (0) +#define WR_LE_64(x, val) do { (x) = htole64(val); } while (0) #define WR_LE_R64(x, val) DMA_REGPAIR_LE(x, val) -#define WR_BE_32(x, val) do { (x) = __cpu_to_be32(val); } while (0) +#define WR_BE_32(x, val) do { (x) = htobe32(val); } while (0) -#define RD_LE_16(x) __le16_to_cpu(x) -#define RD_LE_32(x) __le32_to_cpu(x) -#define RD_BE_32(x) __be32_to_cpu(x) -#define RD_BE_64(x) __be64_to_cpu(x) +#define RD_LE_16(x) le16toh(x) +#define RD_LE_32(x) le32toh(x) +#define RD_BE_32(x) be32toh(x) +#define RD_BE_64(x) be64toh(x) #define WR_REG(addr, val) mmio_write32_le(addr, val) #define RD_REG(addr) mmio_read32_le(addr) @@ -230,7 +230,7 @@ struct xsc_send_wqe_ctrl_seg { struct xsc_wqe_data_seg { union { struct { - uint32_t data0; + __le32 data0; #define XSC_WQE_DATA_SEG_LENGTH_MASK GENMASK(31, 1) __le32 mkey; __le64 va; @@ -316,11 +316,11 @@ static inline void xsc_set_cqe_sw_own(struct xsc_cqe *cqe, static inline bool xsc_get_cqe_sw_own(struct xsc_cqe *cqe, int cid, int ring_sz) { - return FIELD_GET(XSC_CQE_OWNER_MASK, cqe->data3) == ((cid >> ring_sz) & 1); + return FIELD_GET(XSC_CQE_OWNER_MASK, le16toh(cqe->data3)) == ((cid >> ring_sz) & 1); } static inline void xsc_set_cqe_sw_own(struct xsc_cqe *cqe, int pid, int ring_sz) { - cqe->data3 |= FIELD_PREP(XSC_CQE_OWNER_MASK, ((pid >> ring_sz) & 1)); + cqe->data3 |= htole16(FIELD_PREP(XSC_CQE_OWNER_MASK, ((pid >> ring_sz) & 1))); } #endif /* __XSC_HSI_H__ */ diff --git a/providers/xscale/xsc_hw.h b/providers/xscale/xsc_hw.h index 9aea2dd2d..312fbe0e8 100644 --- a/providers/xscale/xsc_hw.h +++ b/providers/xscale/xsc_hw.h @@ -54,29 +54,29 @@ struct xsc_andes_cqe { }; struct xsc_andes_cq_doorbell { - uint32_t raw; + __le32 raw; #define XSC_ANDES_CQ_DOORBELL_CQ_NEXT_CID_MASK GENMASK(15, 0) #define XSC_ANDES_CQ_DOORBELL_CQ_ID_MASK GENMASK(30, 16) #define XSC_ANDES_CQ_DOORBELL_ARM_MASK BIT(31) }; struct xsc_andes_send_doorbell { - uint32_t raw; + __le32 raw; #define XSC_ANDES_SEND_DOORBELL_NEXT_PID_MASK GENMASK(15, 0) #define XSC_ANDES_SEND_DOORBELL_QP_ID_MASK GENMASK(30, 16) }; struct xsc_andes_recv_doorbell { - uint32_t raw; + __le32 raw; #define XSC_ANDES_RECV_DOORBELL_NEXT_PID_MASK GENMASK(12, 0) #define XSC_ANDES_RECV_DOORBELL_QP_ID_MASK GENMASK(27, 13) }; struct xsc_andes_data_seg { - uint32_t data0; + __le32 data0; #define XSC_ANDES_DATA_SEG_LENGTH_MASK GENMASK(31, 1) - uint32_t key; - uint64_t addr; + __le32 key; + __le64 addr; }; struct diamond_send_wqe_ctrl_seg { @@ -119,50 +119,50 @@ struct xsc_diamond_cqe { }; struct xsc_diamond_cq_doorbell { - uint64_t raw; + __le64 raw; #define XSC_DIAMOND_CQ_DOORBELL_CQ_NEXT_CID_MASK GENMASK_ULL(22, 0) #define XSC_DIAMOND_CQ_DOORBELL_CQ_ID_MASK GENMASK_ULL(38, 23) #define XSC_DIAMOND_CQ_DOORBELL_CQ_STA_MASK GENMASK_ULL(40, 39) }; struct xsc_diamond_recv_doorbell { - uint64_t raw; + __le64 raw; #define XSC_DIAMOND_RECV_DOORBELL_NEXT_PID_MASK GENMASK_ULL(17, 0) #define XSC_DIAMOND_RECV_DOORBELL_QP_ID_MASK GENMASK_ULL(33, 18) }; struct xsc_diamond_send_doorbell { - uint64_t raw; + __le64 raw; #define XSC_DIAMOND_SEND_DOORBELL_NEXT_PID_MASK GENMASK_ULL(20, 0) #define XSC_DIAMOND_SEND_DOORBELL_QP_ID_MASK GENMASK_ULL(36, 21) }; struct xsc_diamond_data_seg { - uint32_t length; - uint32_t key; - uint64_t addr; + __le32 length; + __le32 key; + __le64 addr; }; struct xsc_diamond_atomic_seg { - uint64_t swap_add; - uint64_t compare; + __le64 swap_add; + __le64 compare; }; struct xsc_diamond_next_cq_doorbell { - uint64_t raw; + __le64 raw; #define XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_NEXT_CID_MASK GENMASK_ULL(22, 0) #define XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_ID_MASK GENMASK_ULL(38, 23) #define XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_STA_MASK GENMASK_ULL(40, 39) }; struct xsc_diamond_next_send_doorbell { - uint64_t raw; + __le64 raw; #define XSC_DIAMOND_NEXT_SEND_DOORBELL_NEXT_PID_MASK GENMASK_ULL(16, 0) #define XSC_DIAMOND_NEXT_SEND_DOORBELL_QP_ID_MASK GENMASK_ULL(32, 17) }; struct xsc_diamond_next_recv_doorbell { - uint64_t raw; + __le64 raw; #define XSC_DIAMOND_NEXT_RECV_DOORBELL_NEXT_PID_MASK GENMASK_ULL(13, 0) #define XSC_DIAMOND_NEXT_RECV_DOORBELL_QP_ID_MASK GENMASK_ULL(29, 14) }; @@ -177,7 +177,7 @@ enum { static inline uint8_t xsc_diamond_get_cqe_msg_opcode(void *cqe) { return FIELD_GET(XSC_DIAMOND_CQE_MSG_OPCODE_MASK, - ((struct xsc_diamond_cqe *)cqe)->data2); + le32toh(((struct xsc_diamond_cqe *)cqe)->data2)); } static inline uint8_t xsc_andes_get_cqe_msg_opcode(void *cqe) @@ -208,7 +208,7 @@ static inline uint32_t xsc_hw_get_wqe_id(uint16_t device_id, void *cqe) case XSC_MC_PF_DEV_ID_DIAMOND: case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: return FIELD_GET(XSC_DIAMOND_CQE_WQE_ID_MASK, - ((struct xsc_diamond_cqe *)cqe)->data2); + le32toh(((struct xsc_diamond_cqe *)cqe)->data2)); default: return RD_LE_16(((struct xsc_andes_cqe *)cqe)->wqe_id); } @@ -226,7 +226,7 @@ static inline void xsc_hw_set_wqe_id(uint16_t device_id, void *cseg, uint32_t wq case XSC_MC_PF_DEV_ID_DIAMOND: case XSC_MC_PF_DEV_ID_DIAMOND_NEXT: ((struct diamond_send_wqe_ctrl_seg *)cseg)->data1 |= - FIELD_PREP(XSC_DIAMOND_WQE_CTRL_SEG_WQE_ID_MASK, wqe_id); + htole32(FIELD_PREP(XSC_DIAMOND_WQE_CTRL_SEG_WQE_ID_MASK, wqe_id)); break; default: WR_LE_16(((struct andes_send_wqe_ctrl_seg *)cseg)->wqe_id, wqe_id16); @@ -420,9 +420,9 @@ static inline void xsc_diamond_set_data_seg(void *data_seg, { struct xsc_diamond_data_seg *seg = data_seg; - seg->length = length; - seg->key = key; - seg->addr = addr; + seg->length = htole32(length); + seg->key = htole32(key); + seg->addr = htole64(addr); } static inline void xsc_diamond_set_atomic_seg(void *atomic_seg, int opcode, @@ -431,10 +431,10 @@ static inline void xsc_diamond_set_atomic_seg(void *atomic_seg, int opcode, struct xsc_diamond_atomic_seg *aseg = (struct xsc_diamond_atomic_seg *)atomic_seg; if (opcode == IBV_WR_ATOMIC_CMP_AND_SWP) { - aseg->swap_add = RD_BE_64(swap); - aseg->compare = RD_BE_64(compare_add); + aseg->swap_add = htole64(swap); + aseg->compare = htole64(compare_add); } else { - aseg->swap_add = RD_BE_64(compare_add); + aseg->swap_add = htole64(compare_add); } } @@ -444,9 +444,9 @@ static inline void xsc_andes_set_data_seg(void *data_seg, { struct xsc_andes_data_seg *seg = data_seg; - seg->data0 |= FIELD_PREP(XSC_ANDES_DATA_SEG_LENGTH_MASK, length); - seg->key = key; - seg->addr = addr; + seg->data0 |= htole32(FIELD_PREP(XSC_ANDES_DATA_SEG_LENGTH_MASK, length)); + seg->key = htole32(key); + seg->addr = htole64(addr); } static inline void xsc_hw_set_data_seg(uint16_t device_id, void *data_seg, @@ -484,9 +484,9 @@ static inline void xsc_diamond_set_cq_ci(void *db_addr, { struct xsc_diamond_cq_doorbell db; - db.raw = FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_ID_MASK, cqn) | - FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | - FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_STA_MASK, XSC_CQ_STAT_KEEP); + db.raw = htole64(FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_ID_MASK, cqn) | + FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | + FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_STA_MASK, XSC_CQ_STAT_KEEP)); udma_to_device_barrier(); mmio_write64_le(db_addr, db.raw); } @@ -496,9 +496,9 @@ static inline void xsc_diamond_next_set_cq_ci(void *db_addr, { struct xsc_diamond_next_cq_doorbell db; - db.raw = FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_ID_MASK, cqn) | - FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | - FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_STA_MASK, XSC_CQ_STAT_KEEP); + db.raw = htole64(FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_ID_MASK, cqn) | + FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | + FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_STA_MASK, XSC_CQ_STAT_KEEP)); udma_to_device_barrier(); mmio_write64_le(db_addr, db.raw); } @@ -508,9 +508,9 @@ static inline void xsc_andes_set_cq_ci(void *db_addr, { struct xsc_andes_cq_doorbell db; - db.raw = FIELD_PREP(XSC_ANDES_CQ_DOORBELL_CQ_ID_MASK, cqn) | - FIELD_PREP(XSC_ANDES_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | - FIELD_PREP(XSC_ANDES_CQ_DOORBELL_ARM_MASK, XSC_CQ_STAT_FIRED); + db.raw = htole32(FIELD_PREP(XSC_ANDES_CQ_DOORBELL_CQ_ID_MASK, cqn) | + FIELD_PREP(XSC_ANDES_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | + FIELD_PREP(XSC_ANDES_CQ_DOORBELL_ARM_MASK, XSC_CQ_STAT_FIRED)); udma_to_device_barrier(); mmio_write32_le(db_addr, db.raw); } @@ -541,10 +541,10 @@ static inline void xsc_diamond_update_cq_db(void *db_addr, { struct xsc_diamond_cq_doorbell db; - db.raw = FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_ID_MASK, cqn) | - FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | - FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_STA_MASK, - solicited ? XSC_CQ_STAT_ARM_SOLICITED : XSC_CQ_STAT_ARM_NEXT); + db.raw = htole64(FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_ID_MASK, cqn) | + FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | + FIELD_PREP(XSC_DIAMOND_CQ_DOORBELL_CQ_STA_MASK, + solicited ? XSC_CQ_STAT_ARM_SOLICITED : XSC_CQ_STAT_ARM_NEXT)); udma_to_device_barrier(); mmio_wc_start(); mmio_write64_le(db_addr, db.raw); @@ -557,10 +557,10 @@ static inline void xsc_diamond_next_update_cq_db(void *db_addr, { struct xsc_diamond_next_cq_doorbell db; - db.raw = FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_ID_MASK, cqn) | - FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | - FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_STA_MASK, - solicited ? XSC_CQ_STAT_ARM_SOLICITED : XSC_CQ_STAT_ARM_NEXT); + db.raw = htole64(FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_ID_MASK, cqn) | + FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | + FIELD_PREP(XSC_DIAMOND_NEXT_CQ_DOORBELL_CQ_STA_MASK, + solicited ? XSC_CQ_STAT_ARM_SOLICITED : XSC_CQ_STAT_ARM_NEXT)); udma_to_device_barrier(); mmio_wc_start(); mmio_write64_le(db_addr, db.raw); @@ -573,10 +573,10 @@ static inline void xsc_andes_update_cq_db(void *db_addr, { struct xsc_andes_cq_doorbell db; - db.raw = FIELD_PREP(XSC_ANDES_CQ_DOORBELL_CQ_ID_MASK, cqn) | - FIELD_PREP(XSC_ANDES_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | - FIELD_PREP(XSC_ANDES_CQ_DOORBELL_ARM_MASK, - solicited ? XSC_CQ_STAT_ARM_SOLICITED : XSC_CQ_STAT_ARM_NEXT); + db.raw = htole32(FIELD_PREP(XSC_ANDES_CQ_DOORBELL_CQ_ID_MASK, cqn) | + FIELD_PREP(XSC_ANDES_CQ_DOORBELL_CQ_NEXT_CID_MASK, next_cid) | + FIELD_PREP(XSC_ANDES_CQ_DOORBELL_ARM_MASK, + solicited ? XSC_CQ_STAT_ARM_SOLICITED : XSC_CQ_STAT_ARM_NEXT)); udma_to_device_barrier(); mmio_wc_start(); mmio_write32_le(db_addr, db.raw); @@ -608,8 +608,8 @@ static inline void xsc_diamond_ring_rx_doorbell(void *db_addr, { struct xsc_diamond_recv_doorbell db; - db.raw = FIELD_PREP(XSC_DIAMOND_RECV_DOORBELL_QP_ID_MASK, rqn) | - FIELD_PREP(XSC_DIAMOND_RECV_DOORBELL_NEXT_PID_MASK, next_pid); + db.raw = htole64(FIELD_PREP(XSC_DIAMOND_RECV_DOORBELL_QP_ID_MASK, rqn) | + FIELD_PREP(XSC_DIAMOND_RECV_DOORBELL_NEXT_PID_MASK, next_pid)); udma_to_device_barrier(); mmio_write64_le(db_addr, db.raw); } @@ -620,8 +620,8 @@ static inline void xsc_diamond_next_ring_rx_doorbell(void *db_addr, { struct xsc_diamond_next_recv_doorbell db; - db.raw = FIELD_PREP(XSC_DIAMOND_NEXT_RECV_DOORBELL_QP_ID_MASK, rqn) | - FIELD_PREP(XSC_DIAMOND_NEXT_RECV_DOORBELL_NEXT_PID_MASK, next_pid); + db.raw = htole64(FIELD_PREP(XSC_DIAMOND_NEXT_RECV_DOORBELL_QP_ID_MASK, rqn) | + FIELD_PREP(XSC_DIAMOND_NEXT_RECV_DOORBELL_NEXT_PID_MASK, next_pid)); udma_to_device_barrier(); mmio_write64_le(db_addr, db.raw); } @@ -631,8 +631,8 @@ static inline void xsc_andes_ring_rx_doorbell(void *db_addr, { struct xsc_andes_recv_doorbell db; - db.raw = FIELD_PREP(XSC_ANDES_RECV_DOORBELL_QP_ID_MASK, rqn) | - FIELD_PREP(XSC_ANDES_RECV_DOORBELL_NEXT_PID_MASK, next_pid); + db.raw = htole32(FIELD_PREP(XSC_ANDES_RECV_DOORBELL_QP_ID_MASK, rqn) | + FIELD_PREP(XSC_ANDES_RECV_DOORBELL_NEXT_PID_MASK, next_pid)); udma_to_device_barrier(); mmio_write32_le(db_addr, db.raw); } @@ -662,8 +662,8 @@ static inline void xsc_diamond_ring_tx_doorbell(void *db_addr, { struct xsc_diamond_send_doorbell db; - db.raw = FIELD_PREP(XSC_DIAMOND_SEND_DOORBELL_QP_ID_MASK, rqn) | - FIELD_PREP(XSC_DIAMOND_SEND_DOORBELL_NEXT_PID_MASK, next_pid); + db.raw = htole64(FIELD_PREP(XSC_DIAMOND_SEND_DOORBELL_QP_ID_MASK, rqn) | + FIELD_PREP(XSC_DIAMOND_SEND_DOORBELL_NEXT_PID_MASK, next_pid)); udma_to_device_barrier(); mmio_write64_le(db_addr, db.raw); @@ -675,8 +675,8 @@ static inline void xsc_diamond_next_ring_tx_doorbell(void *db_addr, { struct xsc_diamond_next_send_doorbell db; - db.raw = FIELD_PREP(XSC_DIAMOND_NEXT_SEND_DOORBELL_QP_ID_MASK, rqn) | - FIELD_PREP(XSC_DIAMOND_NEXT_SEND_DOORBELL_NEXT_PID_MASK, next_pid); + db.raw = htole64(FIELD_PREP(XSC_DIAMOND_NEXT_SEND_DOORBELL_QP_ID_MASK, rqn) | + FIELD_PREP(XSC_DIAMOND_NEXT_SEND_DOORBELL_NEXT_PID_MASK, next_pid)); udma_to_device_barrier(); mmio_write64_le(db_addr, db.raw); @@ -687,8 +687,8 @@ static inline void xsc_andes_ring_tx_doorbell(void *db_addr, { struct xsc_andes_send_doorbell db; - db.raw = FIELD_PREP(XSC_ANDES_SEND_DOORBELL_QP_ID_MASK, rqn) | - FIELD_PREP(XSC_ANDES_SEND_DOORBELL_NEXT_PID_MASK, next_pid); + db.raw = htole32(FIELD_PREP(XSC_ANDES_SEND_DOORBELL_QP_ID_MASK, rqn) | + FIELD_PREP(XSC_ANDES_SEND_DOORBELL_NEXT_PID_MASK, next_pid)); udma_to_device_barrier(); mmio_write32_le(db_addr, db.raw); diff --git a/providers/xscale/xscale.c b/providers/xscale/xscale.c index 9c97af6ca..a4087d8fe 100644 --- a/providers/xscale/xscale.c +++ b/providers/xscale/xscale.c @@ -152,7 +152,7 @@ static int xsc_mmap(struct xsc_device *xdev, struct xsc_context *context, context->tx_multidb_base & page_mask); if (context->mdb_base == MAP_FAILED) goto free_mdb; - xsc_dbg(context->dbg_fp, XSC_DBG_CTX, "multidb va base:%p, mdb_size:0x%lx\n", + xsc_dbg(context->dbg_fp, XSC_DBG_CTX, "multidb va base:%p, mdb_size:0x%zx\n", context->mdb_base, mdb_size); context->mdb_mmap_size = mdb_size; } diff --git a/suse/rdma-core.spec b/suse/rdma-core.spec index e82ba39e6..1941bfc04 100644 --- a/suse/rdma-core.spec +++ b/suse/rdma-core.spec @@ -45,6 +45,7 @@ Group: Productivity/Networking/Other %define mlx5_so_major 1 %define ibnetdisc_major 5 %define mad_major 5 +%define xscale_so_major 1 %define efa_lname libefa%{efa_so_major} %define hns_lname libhns%{hns_so_major} @@ -55,6 +56,7 @@ Group: Productivity/Networking/Other %define mana_lname libmana%{mana_so_major} %define mlx4_lname libmlx4-%{mlx4_so_major} %define mlx5_lname libmlx5-%{mlx5_so_major} +%define xscale_lname libxscale-%{xscale_so_major} %ifnarch s390 %arm %define dma_coherent 1 @@ -169,6 +171,7 @@ Requires: %{ionic_lname} = %{version}-%{release} Requires: %{mana_lname} = %{version}-%{release} Requires: %{mlx4_lname} = %{version}-%{release} Requires: %{mlx5_lname} = %{version}-%{release} +Requires: %{xscale_lname} = %{version}-%{release} %endif Requires: rsocket = %{version}-%{release} @@ -215,6 +218,7 @@ Obsoletes: libmlx5-rdmav2 < %{version}-%{release} Obsoletes: libmthca-rdmav2 < %{version}-%{release} Obsoletes: libocrdma-rdmav2 < %{version}-%{release} Obsoletes: librxe-rdmav2 < %{version}-%{release} +Obsoletes: libxscale-rdmav2 < %{version}-%{release} %if 0%{?dma_coherent} Requires: %{efa_lname} = %{version}-%{release} Requires: %{hns_lname} = %{version}-%{release} @@ -222,6 +226,7 @@ Requires: %{ionic_lname} = %{version}-%{release} Requires: %{mana_lname} = %{version}-%{release} Requires: %{mlx4_lname} = %{version}-%{release} Requires: %{mlx5_lname} = %{version}-%{release} +Requires: %{xscale_lname} = %{version}-%{release} %endif # Recommended packages for rxe Recommends: iproute2 @@ -251,6 +256,7 @@ Device-specific plug-in ibverbs userspace drivers are included: - librxe: A software implementation of the RoCE protocol - libsiw: A software implementation of the iWarp protocol - libvmw_pvrdma: VMware paravirtual RDMA device +- libxscale: Yunsilicon RDMA Device %package -n %verbs_lname Summary: Ibverbs runtime library @@ -302,6 +308,13 @@ Group: System/Libraries %description -n %mlx5_lname This package contains the mlx5 runtime library. +%package -n %xscale_lname +Summary: Yunsilicon xscale runtime library +Group: System/Libraries + +%description -n %xscale_lname +This package contains the Yunsilicon xscale runtime library. + %package -n libibnetdisc%{ibnetdisc_major} Summary: Infiniband Net Discovery runtime library Group: System/Libraries @@ -536,6 +549,7 @@ rm -rf %{buildroot}/%{_sbindir}/srp_daemon.sh %ldconfig_scriptlets -n %mana_lname %ldconfig_scriptlets -n %mlx4_lname %ldconfig_scriptlets -n %mlx5_lname +%ldconfig_scriptlets -n %xscale_lname %ldconfig_scriptlets -n %umad_lname %ldconfig_scriptlets -n %rdmacm_lname %ldconfig_scriptlets -n libibnetdisc%{ibnetdisc_major} @@ -735,6 +749,9 @@ done %files -n %mlx5_lname %{_libdir}/libmlx5*.so.* + +%files -n %xscale_lname +%{_libdir}/libxscale*.so.* %endif %files -n libibverbs-utils