Commit 31b60e5c authored by James Smart's avatar James Smart Committed by Greg Kroah-Hartman

scsi: lpfc: Mitigate high memory pre-allocation by SCSI-MQ

[ Upstream commit 77ffd346 ]

When SCSI-MQ is enabled, the SCSI-MQ layers will do pre-allocation of MQ
resources based on shost values set by the driver. In newer cases of the
driver, which attempts to set nr_hw_queues to the cpu count, the
multipliers become excessive, with a single shost having SCSI-MQ
pre-allocation reaching into the multiple GBytes range.  NPIV, which
creates additional shosts, only multiply this overhead. On lower-memory
systems, this can exhaust system memory very quickly, resulting in a system
crash or failures in the driver or elsewhere due to low memory conditions.

After testing several scenarios, the situation can be mitigated by limiting
the value set in shost->nr_hw_queues to 4. Although the shost values were
changed, the driver still had per-cpu hardware queues of its own that
allowed parallelization per-cpu.  Testing revealed that even with the
smallish number for nr_hw_queues for SCSI-MQ, performance levels remained
near maximum with the within-driver affiinitization.

A module parameter was created to allow the value set for the nr_hw_queues
to be tunable.
Signed-off-by: default avatarDick Kennedy <>
Signed-off-by: James Smart's avatarJames Smart <>
Reviewed-by: default avatarMing Lei <>
Reviewed-by: default avatarEwan D. Milne <>
Signed-off-by: Martin K. Petersen's avatarMartin K. Petersen <>
Signed-off-by: default avatarSasha Levin <>
parent aa4bc3a3
......@@ -822,6 +822,7 @@ struct lpfc_hba {
uint32_t cfg_cq_poll_threshold;
uint32_t cfg_cq_max_proc_limit;
uint32_t cfg_fcp_cpu_map;
uint32_t cfg_fcp_mq_threshold;
uint32_t cfg_hdw_queue;
uint32_t cfg_irq_chann;
uint32_t cfg_suppress_rsp;
......@@ -5640,6 +5640,19 @@ LPFC_ATTR_RW(nvme_oas, 0, 0, 1,
LPFC_ATTR_RW(nvme_embed_cmd, 1, 0, 2,
"Embed NVME Command in WQE");
* lpfc_fcp_mq_threshold: Set the maximum number of Hardware Queues
* the driver will advertise it supports to the SCSI layer.
* 0 = Set nr_hw_queues by the number of CPUs or HW queues.
* 1,128 = Manually specify the maximum nr_hw_queue value to be set,
* Value range is [0,128]. Default value is 8.
"Set the number of SCSI Queues advertised");
* lpfc_hdw_queue: Set the number of Hardware Queues the driver
* will advertise it supports to the NVME and SCSI layers. This also
......@@ -5961,6 +5974,7 @@ struct device_attribute *lpfc_hba_attrs[] = {
......@@ -7042,6 +7056,7 @@ lpfc_get_cfgparam(struct lpfc_hba *phba)
/* Initialize first burst. Target vs Initiator are different. */
lpfc_nvme_enable_fb_init(phba, lpfc_nvme_enable_fb);
lpfc_nvmet_fb_size_init(phba, lpfc_nvmet_fb_size);
lpfc_fcp_mq_threshold_init(phba, lpfc_fcp_mq_threshold);
lpfc_hdw_queue_init(phba, lpfc_hdw_queue);
lpfc_irq_chann_init(phba, lpfc_irq_chann);
lpfc_enable_bbcr_init(phba, lpfc_enable_bbcr);
......@@ -4308,10 +4308,12 @@ lpfc_create_port(struct lpfc_hba *phba, int instance, struct device *dev)
shost->max_cmd_len = 16;
if (phba->sli_rev == LPFC_SLI_REV4) {
if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ)
shost->nr_hw_queues = phba->cfg_hdw_queue;
shost->nr_hw_queues = phba->sli4_hba.num_present_cpu;
if (!phba->cfg_fcp_mq_threshold ||
phba->cfg_fcp_mq_threshold > phba->cfg_hdw_queue)
phba->cfg_fcp_mq_threshold = phba->cfg_hdw_queue;
shost->nr_hw_queues = min_t(int, 2 * num_possible_nodes(),
shost->dma_boundary =
......@@ -44,6 +44,11 @@
#define LPFC_HBA_HDWQ_MAX 128
/* FCP MQ queue count limiting */
/* Common buffer size to accomidate SCSI and NVME IO buffers */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment