svcrdma: Add FRMR get/put services
Add services for the allocating, freeing, and unmapping Fast Reg MR. These services will be used by the transport connection setup, send and receive routines. Signed-off-by: Tom Tucker <tom@opengridcomputing.com>
This commit is contained in:
parent
0d3ebb9ae9
commit
64be8608c1
@ -214,6 +214,9 @@ extern struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *);
|
|||||||
extern void svc_rdma_put_context(struct svc_rdma_op_ctxt *, int);
|
extern void svc_rdma_put_context(struct svc_rdma_op_ctxt *, int);
|
||||||
extern struct svc_rdma_req_map *svc_rdma_get_req_map(void);
|
extern struct svc_rdma_req_map *svc_rdma_get_req_map(void);
|
||||||
extern void svc_rdma_put_req_map(struct svc_rdma_req_map *);
|
extern void svc_rdma_put_req_map(struct svc_rdma_req_map *);
|
||||||
|
extern struct svc_rdma_fastreg_mr *svc_rdma_get_frmr(struct svcxprt_rdma *);
|
||||||
|
extern void svc_rdma_put_frmr(struct svcxprt_rdma *,
|
||||||
|
struct svc_rdma_fastreg_mr *);
|
||||||
extern void svc_sq_reap(struct svcxprt_rdma *);
|
extern void svc_sq_reap(struct svcxprt_rdma *);
|
||||||
extern void svc_rq_reap(struct svcxprt_rdma *);
|
extern void svc_rq_reap(struct svcxprt_rdma *);
|
||||||
extern struct svc_xprt_class svc_rdma_class;
|
extern struct svc_xprt_class svc_rdma_class;
|
||||||
|
@ -100,6 +100,7 @@ struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *xprt)
|
|||||||
ctxt->xprt = xprt;
|
ctxt->xprt = xprt;
|
||||||
INIT_LIST_HEAD(&ctxt->dto_q);
|
INIT_LIST_HEAD(&ctxt->dto_q);
|
||||||
ctxt->count = 0;
|
ctxt->count = 0;
|
||||||
|
ctxt->frmr = NULL;
|
||||||
atomic_inc(&xprt->sc_ctxt_used);
|
atomic_inc(&xprt->sc_ctxt_used);
|
||||||
return ctxt;
|
return ctxt;
|
||||||
}
|
}
|
||||||
@ -109,11 +110,19 @@ static void svc_rdma_unmap_dma(struct svc_rdma_op_ctxt *ctxt)
|
|||||||
struct svcxprt_rdma *xprt = ctxt->xprt;
|
struct svcxprt_rdma *xprt = ctxt->xprt;
|
||||||
int i;
|
int i;
|
||||||
for (i = 0; i < ctxt->count && ctxt->sge[i].length; i++) {
|
for (i = 0; i < ctxt->count && ctxt->sge[i].length; i++) {
|
||||||
atomic_dec(&xprt->sc_dma_used);
|
/*
|
||||||
ib_dma_unmap_single(xprt->sc_cm_id->device,
|
* Unmap the DMA addr in the SGE if the lkey matches
|
||||||
ctxt->sge[i].addr,
|
* the sc_dma_lkey, otherwise, ignore it since it is
|
||||||
ctxt->sge[i].length,
|
* an FRMR lkey and will be unmapped later when the
|
||||||
ctxt->direction);
|
* last WR that uses it completes.
|
||||||
|
*/
|
||||||
|
if (ctxt->sge[i].lkey == xprt->sc_dma_lkey) {
|
||||||
|
atomic_dec(&xprt->sc_dma_used);
|
||||||
|
ib_dma_unmap_single(xprt->sc_cm_id->device,
|
||||||
|
ctxt->sge[i].addr,
|
||||||
|
ctxt->sge[i].length,
|
||||||
|
ctxt->direction);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -150,6 +159,7 @@ struct svc_rdma_req_map *svc_rdma_get_req_map(void)
|
|||||||
schedule_timeout_uninterruptible(msecs_to_jiffies(500));
|
schedule_timeout_uninterruptible(msecs_to_jiffies(500));
|
||||||
}
|
}
|
||||||
map->count = 0;
|
map->count = 0;
|
||||||
|
map->frmr = NULL;
|
||||||
return map;
|
return map;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -425,10 +435,12 @@ static struct svcxprt_rdma *rdma_create_xprt(struct svc_serv *serv,
|
|||||||
INIT_LIST_HEAD(&cma_xprt->sc_dto_q);
|
INIT_LIST_HEAD(&cma_xprt->sc_dto_q);
|
||||||
INIT_LIST_HEAD(&cma_xprt->sc_rq_dto_q);
|
INIT_LIST_HEAD(&cma_xprt->sc_rq_dto_q);
|
||||||
INIT_LIST_HEAD(&cma_xprt->sc_read_complete_q);
|
INIT_LIST_HEAD(&cma_xprt->sc_read_complete_q);
|
||||||
|
INIT_LIST_HEAD(&cma_xprt->sc_frmr_q);
|
||||||
init_waitqueue_head(&cma_xprt->sc_send_wait);
|
init_waitqueue_head(&cma_xprt->sc_send_wait);
|
||||||
|
|
||||||
spin_lock_init(&cma_xprt->sc_lock);
|
spin_lock_init(&cma_xprt->sc_lock);
|
||||||
spin_lock_init(&cma_xprt->sc_rq_dto_lock);
|
spin_lock_init(&cma_xprt->sc_rq_dto_lock);
|
||||||
|
spin_lock_init(&cma_xprt->sc_frmr_q_lock);
|
||||||
|
|
||||||
cma_xprt->sc_ord = svcrdma_ord;
|
cma_xprt->sc_ord = svcrdma_ord;
|
||||||
|
|
||||||
@ -686,6 +698,97 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,
|
|||||||
return ERR_PTR(ret);
|
return ERR_PTR(ret);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static struct svc_rdma_fastreg_mr *rdma_alloc_frmr(struct svcxprt_rdma *xprt)
|
||||||
|
{
|
||||||
|
struct ib_mr *mr;
|
||||||
|
struct ib_fast_reg_page_list *pl;
|
||||||
|
struct svc_rdma_fastreg_mr *frmr;
|
||||||
|
|
||||||
|
frmr = kmalloc(sizeof(*frmr), GFP_KERNEL);
|
||||||
|
if (!frmr)
|
||||||
|
goto err;
|
||||||
|
|
||||||
|
mr = ib_alloc_fast_reg_mr(xprt->sc_pd, RPCSVC_MAXPAGES);
|
||||||
|
if (!mr)
|
||||||
|
goto err_free_frmr;
|
||||||
|
|
||||||
|
pl = ib_alloc_fast_reg_page_list(xprt->sc_cm_id->device,
|
||||||
|
RPCSVC_MAXPAGES);
|
||||||
|
if (!pl)
|
||||||
|
goto err_free_mr;
|
||||||
|
|
||||||
|
frmr->mr = mr;
|
||||||
|
frmr->page_list = pl;
|
||||||
|
INIT_LIST_HEAD(&frmr->frmr_list);
|
||||||
|
return frmr;
|
||||||
|
|
||||||
|
err_free_mr:
|
||||||
|
ib_dereg_mr(mr);
|
||||||
|
err_free_frmr:
|
||||||
|
kfree(frmr);
|
||||||
|
err:
|
||||||
|
return ERR_PTR(-ENOMEM);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void rdma_dealloc_frmr_q(struct svcxprt_rdma *xprt)
|
||||||
|
{
|
||||||
|
struct svc_rdma_fastreg_mr *frmr;
|
||||||
|
|
||||||
|
while (!list_empty(&xprt->sc_frmr_q)) {
|
||||||
|
frmr = list_entry(xprt->sc_frmr_q.next,
|
||||||
|
struct svc_rdma_fastreg_mr, frmr_list);
|
||||||
|
list_del_init(&frmr->frmr_list);
|
||||||
|
ib_dereg_mr(frmr->mr);
|
||||||
|
ib_free_fast_reg_page_list(frmr->page_list);
|
||||||
|
kfree(frmr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct svc_rdma_fastreg_mr *svc_rdma_get_frmr(struct svcxprt_rdma *rdma)
|
||||||
|
{
|
||||||
|
struct svc_rdma_fastreg_mr *frmr = NULL;
|
||||||
|
|
||||||
|
spin_lock_bh(&rdma->sc_frmr_q_lock);
|
||||||
|
if (!list_empty(&rdma->sc_frmr_q)) {
|
||||||
|
frmr = list_entry(rdma->sc_frmr_q.next,
|
||||||
|
struct svc_rdma_fastreg_mr, frmr_list);
|
||||||
|
list_del_init(&frmr->frmr_list);
|
||||||
|
frmr->map_len = 0;
|
||||||
|
frmr->page_list_len = 0;
|
||||||
|
}
|
||||||
|
spin_unlock_bh(&rdma->sc_frmr_q_lock);
|
||||||
|
if (frmr)
|
||||||
|
return frmr;
|
||||||
|
|
||||||
|
return rdma_alloc_frmr(rdma);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void frmr_unmap_dma(struct svcxprt_rdma *xprt,
|
||||||
|
struct svc_rdma_fastreg_mr *frmr)
|
||||||
|
{
|
||||||
|
int page_no;
|
||||||
|
for (page_no = 0; page_no < frmr->page_list_len; page_no++) {
|
||||||
|
dma_addr_t addr = frmr->page_list->page_list[page_no];
|
||||||
|
if (ib_dma_mapping_error(frmr->mr->device, addr))
|
||||||
|
continue;
|
||||||
|
atomic_dec(&xprt->sc_dma_used);
|
||||||
|
ib_dma_unmap_single(frmr->mr->device, addr, PAGE_SIZE,
|
||||||
|
frmr->direction);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void svc_rdma_put_frmr(struct svcxprt_rdma *rdma,
|
||||||
|
struct svc_rdma_fastreg_mr *frmr)
|
||||||
|
{
|
||||||
|
if (frmr) {
|
||||||
|
frmr_unmap_dma(rdma, frmr);
|
||||||
|
spin_lock_bh(&rdma->sc_frmr_q_lock);
|
||||||
|
BUG_ON(!list_empty(&frmr->frmr_list));
|
||||||
|
list_add(&frmr->frmr_list, &rdma->sc_frmr_q);
|
||||||
|
spin_unlock_bh(&rdma->sc_frmr_q_lock);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* This is the xpo_recvfrom function for listening endpoints. Its
|
* This is the xpo_recvfrom function for listening endpoints. Its
|
||||||
* purpose is to accept incoming connections. The CMA callback handler
|
* purpose is to accept incoming connections. The CMA callback handler
|
||||||
@ -961,6 +1064,9 @@ static void __svc_rdma_free(struct work_struct *work)
|
|||||||
WARN_ON(atomic_read(&rdma->sc_ctxt_used) != 0);
|
WARN_ON(atomic_read(&rdma->sc_ctxt_used) != 0);
|
||||||
WARN_ON(atomic_read(&rdma->sc_dma_used) != 0);
|
WARN_ON(atomic_read(&rdma->sc_dma_used) != 0);
|
||||||
|
|
||||||
|
/* De-allocate fastreg mr */
|
||||||
|
rdma_dealloc_frmr_q(rdma);
|
||||||
|
|
||||||
/* Destroy the QP if present (not a listener) */
|
/* Destroy the QP if present (not a listener) */
|
||||||
if (rdma->sc_qp && !IS_ERR(rdma->sc_qp))
|
if (rdma->sc_qp && !IS_ERR(rdma->sc_qp))
|
||||||
ib_destroy_qp(rdma->sc_qp);
|
ib_destroy_qp(rdma->sc_qp);
|
||||||
|
Loading…
Reference in New Issue
Block a user