RDMA/nes: Improve use of PBLs
Two level 256 byte PBLs was not implemented so the driver could report out of memory when in fact there were PBLs still available. This solution prefers to use 4KB PBLs over two level 256B PBLs until the number of 4KB PBLs falls below a threshold. At this point the 4KB PBL structure is converted to use 256B PBLs which prevents the driver from running out of 4KB PBLs too quickly. Signed-off-by: Don Wood <donald.e.wood@intel.com> Signed-off-by: Roland Dreier <rolandd@cisco.com>
This commit is contained in:
parent
2869975cfb
commit
0145f341a9
1 changed files with 134 additions and 57 deletions
|
@ -1887,21 +1887,75 @@ static int nes_destroy_cq(struct ib_cq *ib_cq)
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* root_256
|
||||||
|
*/
|
||||||
|
static u32 root_256(struct nes_device *nesdev,
|
||||||
|
struct nes_root_vpbl *root_vpbl,
|
||||||
|
struct nes_root_vpbl *new_root,
|
||||||
|
u16 pbl_count_4k,
|
||||||
|
u16 pbl_count_256)
|
||||||
|
{
|
||||||
|
u64 leaf_pbl;
|
||||||
|
int i, j, k;
|
||||||
|
|
||||||
|
if (pbl_count_4k == 1) {
|
||||||
|
new_root->pbl_vbase = pci_alloc_consistent(nesdev->pcidev,
|
||||||
|
512, &new_root->pbl_pbase);
|
||||||
|
|
||||||
|
if (new_root->pbl_vbase == NULL)
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
leaf_pbl = (u64)root_vpbl->pbl_pbase;
|
||||||
|
for (i = 0; i < 16; i++) {
|
||||||
|
new_root->pbl_vbase[i].pa_low =
|
||||||
|
cpu_to_le32((u32)leaf_pbl);
|
||||||
|
new_root->pbl_vbase[i].pa_high =
|
||||||
|
cpu_to_le32((u32)((((u64)leaf_pbl) >> 32)));
|
||||||
|
leaf_pbl += 256;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for (i = 3; i >= 0; i--) {
|
||||||
|
j = i * 16;
|
||||||
|
root_vpbl->pbl_vbase[j] = root_vpbl->pbl_vbase[i];
|
||||||
|
leaf_pbl = le32_to_cpu(root_vpbl->pbl_vbase[j].pa_low) +
|
||||||
|
(((u64)le32_to_cpu(root_vpbl->pbl_vbase[j].pa_high))
|
||||||
|
<< 32);
|
||||||
|
for (k = 1; k < 16; k++) {
|
||||||
|
leaf_pbl += 256;
|
||||||
|
root_vpbl->pbl_vbase[j + k].pa_low =
|
||||||
|
cpu_to_le32((u32)leaf_pbl);
|
||||||
|
root_vpbl->pbl_vbase[j + k].pa_high =
|
||||||
|
cpu_to_le32((u32)((((u64)leaf_pbl) >> 32)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* nes_reg_mr
|
* nes_reg_mr
|
||||||
*/
|
*/
|
||||||
static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
|
static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
|
||||||
u32 stag, u64 region_length, struct nes_root_vpbl *root_vpbl,
|
u32 stag, u64 region_length, struct nes_root_vpbl *root_vpbl,
|
||||||
dma_addr_t single_buffer, u16 pbl_count, u16 residual_page_count,
|
dma_addr_t single_buffer, u16 pbl_count_4k,
|
||||||
int acc, u64 *iova_start)
|
u16 residual_page_count_4k, int acc, u64 *iova_start,
|
||||||
|
u16 *actual_pbl_cnt, u8 *used_4k_pbls)
|
||||||
{
|
{
|
||||||
struct nes_hw_cqp_wqe *cqp_wqe;
|
struct nes_hw_cqp_wqe *cqp_wqe;
|
||||||
struct nes_cqp_request *cqp_request;
|
struct nes_cqp_request *cqp_request;
|
||||||
unsigned long flags;
|
unsigned long flags;
|
||||||
int ret;
|
int ret;
|
||||||
struct nes_adapter *nesadapter = nesdev->nesadapter;
|
struct nes_adapter *nesadapter = nesdev->nesadapter;
|
||||||
/* int count; */
|
uint pg_cnt = 0;
|
||||||
|
u16 pbl_count_256;
|
||||||
|
u16 pbl_count = 0;
|
||||||
|
u8 use_256_pbls = 0;
|
||||||
|
u8 use_4k_pbls = 0;
|
||||||
|
u16 use_two_level = (pbl_count_4k > 1) ? 1 : 0;
|
||||||
|
struct nes_root_vpbl new_root = {0, 0, 0};
|
||||||
u32 opcode = 0;
|
u32 opcode = 0;
|
||||||
u16 major_code;
|
u16 major_code;
|
||||||
|
|
||||||
|
@ -1914,41 +1968,70 @@ static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
|
||||||
cqp_request->waiting = 1;
|
cqp_request->waiting = 1;
|
||||||
cqp_wqe = &cqp_request->cqp_wqe;
|
cqp_wqe = &cqp_request->cqp_wqe;
|
||||||
|
|
||||||
spin_lock_irqsave(&nesadapter->pbl_lock, flags);
|
if (pbl_count_4k) {
|
||||||
/* track PBL resources */
|
spin_lock_irqsave(&nesadapter->pbl_lock, flags);
|
||||||
if (pbl_count != 0) {
|
|
||||||
if (pbl_count > 1) {
|
pg_cnt = ((pbl_count_4k - 1) * 512) + residual_page_count_4k;
|
||||||
/* Two level PBL */
|
pbl_count_256 = (pg_cnt + 31) / 32;
|
||||||
if ((pbl_count+1) > nesadapter->free_4kpbl) {
|
if (pg_cnt <= 32) {
|
||||||
nes_debug(NES_DBG_MR, "Out of 4KB Pbls for two level request.\n");
|
if (pbl_count_256 <= nesadapter->free_256pbl)
|
||||||
spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
use_256_pbls = 1;
|
||||||
nes_free_cqp_request(nesdev, cqp_request);
|
else if (pbl_count_4k <= nesadapter->free_4kpbl)
|
||||||
return -ENOMEM;
|
use_4k_pbls = 1;
|
||||||
} else {
|
} else if (pg_cnt <= 2048) {
|
||||||
nesadapter->free_4kpbl -= pbl_count+1;
|
if (((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) &&
|
||||||
}
|
(nesadapter->free_4kpbl > (nesadapter->max_4kpbl >> 1))) {
|
||||||
} else if (residual_page_count > 32) {
|
use_4k_pbls = 1;
|
||||||
if (pbl_count > nesadapter->free_4kpbl) {
|
} else if ((pbl_count_256 + 1) <= nesadapter->free_256pbl) {
|
||||||
nes_debug(NES_DBG_MR, "Out of 4KB Pbls.\n");
|
use_256_pbls = 1;
|
||||||
spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
use_two_level = 1;
|
||||||
nes_free_cqp_request(nesdev, cqp_request);
|
} else if ((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) {
|
||||||
return -ENOMEM;
|
use_4k_pbls = 1;
|
||||||
} else {
|
|
||||||
nesadapter->free_4kpbl -= pbl_count;
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if (pbl_count > nesadapter->free_256pbl) {
|
if ((pbl_count_4k + 1) <= nesadapter->free_4kpbl)
|
||||||
nes_debug(NES_DBG_MR, "Out of 256B Pbls.\n");
|
use_4k_pbls = 1;
|
||||||
spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
|
||||||
nes_free_cqp_request(nesdev, cqp_request);
|
|
||||||
return -ENOMEM;
|
|
||||||
} else {
|
|
||||||
nesadapter->free_256pbl -= pbl_count;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (use_256_pbls) {
|
||||||
|
pbl_count = pbl_count_256;
|
||||||
|
nesadapter->free_256pbl -= pbl_count + use_two_level;
|
||||||
|
} else if (use_4k_pbls) {
|
||||||
|
pbl_count = pbl_count_4k;
|
||||||
|
nesadapter->free_4kpbl -= pbl_count + use_two_level;
|
||||||
|
} else {
|
||||||
|
spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
||||||
|
nes_debug(NES_DBG_MR, "Out of Pbls\n");
|
||||||
|
nes_free_cqp_request(nesdev, cqp_request);
|
||||||
|
return -ENOMEM;
|
||||||
|
}
|
||||||
|
|
||||||
|
spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
if (use_256_pbls && use_two_level) {
|
||||||
|
if (root_256(nesdev, root_vpbl, &new_root, pbl_count_4k, pbl_count_256) == 1) {
|
||||||
|
if (new_root.pbl_pbase != 0)
|
||||||
|
root_vpbl = &new_root;
|
||||||
|
} else {
|
||||||
|
spin_lock_irqsave(&nesadapter->pbl_lock, flags);
|
||||||
|
nesadapter->free_256pbl += pbl_count_256 + use_two_level;
|
||||||
|
use_256_pbls = 0;
|
||||||
|
|
||||||
|
if (pbl_count_4k == 1)
|
||||||
|
use_two_level = 0;
|
||||||
|
pbl_count = pbl_count_4k;
|
||||||
|
|
||||||
|
if ((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) {
|
||||||
|
nesadapter->free_4kpbl -= pbl_count + use_two_level;
|
||||||
|
use_4k_pbls = 1;
|
||||||
|
}
|
||||||
|
spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
||||||
|
|
||||||
|
if (use_4k_pbls == 0)
|
||||||
|
return -ENOMEM;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
opcode = NES_CQP_REGISTER_STAG | NES_CQP_STAG_RIGHTS_LOCAL_READ |
|
opcode = NES_CQP_REGISTER_STAG | NES_CQP_STAG_RIGHTS_LOCAL_READ |
|
||||||
NES_CQP_STAG_VA_TO | NES_CQP_STAG_MR;
|
NES_CQP_STAG_VA_TO | NES_CQP_STAG_MR;
|
||||||
|
@ -1977,10 +2060,9 @@ static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
|
||||||
} else {
|
} else {
|
||||||
set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, root_vpbl->pbl_pbase);
|
set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, root_vpbl->pbl_pbase);
|
||||||
set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, pbl_count);
|
set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, pbl_count);
|
||||||
set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX,
|
set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, (pg_cnt * 8));
|
||||||
(((pbl_count - 1) * 4096) + (residual_page_count*8)));
|
|
||||||
|
|
||||||
if ((pbl_count > 1) || (residual_page_count > 32))
|
if (use_4k_pbls)
|
||||||
cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE);
|
cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE);
|
||||||
}
|
}
|
||||||
barrier();
|
barrier();
|
||||||
|
@ -1996,23 +2078,26 @@ static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
|
||||||
stag, ret, cqp_request->major_code, cqp_request->minor_code);
|
stag, ret, cqp_request->major_code, cqp_request->minor_code);
|
||||||
major_code = cqp_request->major_code;
|
major_code = cqp_request->major_code;
|
||||||
nes_put_cqp_request(nesdev, cqp_request);
|
nes_put_cqp_request(nesdev, cqp_request);
|
||||||
|
|
||||||
if ((!ret || major_code) && pbl_count != 0) {
|
if ((!ret || major_code) && pbl_count != 0) {
|
||||||
spin_lock_irqsave(&nesadapter->pbl_lock, flags);
|
spin_lock_irqsave(&nesadapter->pbl_lock, flags);
|
||||||
if (pbl_count > 1)
|
if (use_256_pbls)
|
||||||
nesadapter->free_4kpbl += pbl_count+1;
|
nesadapter->free_256pbl += pbl_count + use_two_level;
|
||||||
else if (residual_page_count > 32)
|
else if (use_4k_pbls)
|
||||||
nesadapter->free_4kpbl += pbl_count;
|
nesadapter->free_4kpbl += pbl_count + use_two_level;
|
||||||
else
|
|
||||||
nesadapter->free_256pbl += pbl_count;
|
|
||||||
spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
||||||
}
|
}
|
||||||
|
if (new_root.pbl_pbase)
|
||||||
|
pci_free_consistent(nesdev->pcidev, 512, new_root.pbl_vbase,
|
||||||
|
new_root.pbl_pbase);
|
||||||
|
|
||||||
if (!ret)
|
if (!ret)
|
||||||
return -ETIME;
|
return -ETIME;
|
||||||
else if (major_code)
|
else if (major_code)
|
||||||
return -EIO;
|
return -EIO;
|
||||||
else
|
|
||||||
return 0;
|
|
||||||
|
|
||||||
|
*actual_pbl_cnt = pbl_count + use_two_level;
|
||||||
|
*used_4k_pbls = use_4k_pbls;
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2177,18 +2262,14 @@ static struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd,
|
||||||
pbl_count = root_pbl_index;
|
pbl_count = root_pbl_index;
|
||||||
}
|
}
|
||||||
ret = nes_reg_mr(nesdev, nespd, stag, region_length, &root_vpbl,
|
ret = nes_reg_mr(nesdev, nespd, stag, region_length, &root_vpbl,
|
||||||
buffer_list[0].addr, pbl_count, (u16)cur_pbl_index, acc, iova_start);
|
buffer_list[0].addr, pbl_count, (u16)cur_pbl_index, acc, iova_start,
|
||||||
|
&nesmr->pbls_used, &nesmr->pbl_4k);
|
||||||
|
|
||||||
if (ret == 0) {
|
if (ret == 0) {
|
||||||
nesmr->ibmr.rkey = stag;
|
nesmr->ibmr.rkey = stag;
|
||||||
nesmr->ibmr.lkey = stag;
|
nesmr->ibmr.lkey = stag;
|
||||||
nesmr->mode = IWNES_MEMREG_TYPE_MEM;
|
nesmr->mode = IWNES_MEMREG_TYPE_MEM;
|
||||||
ibmr = &nesmr->ibmr;
|
ibmr = &nesmr->ibmr;
|
||||||
nesmr->pbl_4k = ((pbl_count > 1) || (cur_pbl_index > 32)) ? 1 : 0;
|
|
||||||
nesmr->pbls_used = pbl_count;
|
|
||||||
if (pbl_count > 1) {
|
|
||||||
nesmr->pbls_used++;
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
kfree(nesmr);
|
kfree(nesmr);
|
||||||
ibmr = ERR_PTR(-ENOMEM);
|
ibmr = ERR_PTR(-ENOMEM);
|
||||||
|
@ -2466,8 +2547,9 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
|
||||||
stag, (unsigned int)iova_start,
|
stag, (unsigned int)iova_start,
|
||||||
(unsigned int)region_length, stag_index,
|
(unsigned int)region_length, stag_index,
|
||||||
(unsigned long long)region->length, pbl_count);
|
(unsigned long long)region->length, pbl_count);
|
||||||
ret = nes_reg_mr( nesdev, nespd, stag, region->length, &root_vpbl,
|
ret = nes_reg_mr(nesdev, nespd, stag, region->length, &root_vpbl,
|
||||||
first_dma_addr, pbl_count, (u16)cur_pbl_index, acc, &iova_start);
|
first_dma_addr, pbl_count, (u16)cur_pbl_index, acc,
|
||||||
|
&iova_start, &nesmr->pbls_used, &nesmr->pbl_4k);
|
||||||
|
|
||||||
nes_debug(NES_DBG_MR, "ret=%d\n", ret);
|
nes_debug(NES_DBG_MR, "ret=%d\n", ret);
|
||||||
|
|
||||||
|
@ -2476,11 +2558,6 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
|
||||||
nesmr->ibmr.lkey = stag;
|
nesmr->ibmr.lkey = stag;
|
||||||
nesmr->mode = IWNES_MEMREG_TYPE_MEM;
|
nesmr->mode = IWNES_MEMREG_TYPE_MEM;
|
||||||
ibmr = &nesmr->ibmr;
|
ibmr = &nesmr->ibmr;
|
||||||
nesmr->pbl_4k = ((pbl_count > 1) || (cur_pbl_index > 32)) ? 1 : 0;
|
|
||||||
nesmr->pbls_used = pbl_count;
|
|
||||||
if (pbl_count > 1) {
|
|
||||||
nesmr->pbls_used++;
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
ib_umem_release(region);
|
ib_umem_release(region);
|
||||||
kfree(nesmr);
|
kfree(nesmr);
|
||||||
|
|
Reference in a new issue