From a9294d86743298c87fd9c39d9ddebf4b04d5da10 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sat, 27 Feb 2021 10:59:42 -0600 Subject: scsi: target: core: Move t_task_cdb initialization Prepare to split target_submit_cmd_map_sgls() so the initialization and submission part can be called at different times. If the init part fails we can reference the t_task_cdb early in some of the logging and tracing code. Move it to transport_init_se_cmd() so we don't hit NULL pointer crashes. Link: https://lore.kernel.org/r/20210227170006.5077-2-michael.christie@oracle.com Tested-by: Laurence Oberman Reviewed-by: Christoph Hellwig Reviewed-by: Himanshu Madhani Signed-off-by: Mike Christie Signed-off-by: Martin K. Petersen --- drivers/target/target_core_transport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/target/target_core_transport.c') diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index 5ecb9f18a53d..2dfbf9cb9581 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -1391,6 +1391,7 @@ void transport_init_se_cmd( INIT_WORK(&cmd->work, NULL); kref_init(&cmd->cmd_kref); + cmd->t_task_cdb = &cmd->__t_task_cdb[0]; cmd->se_tfo = tfo; cmd->se_sess = se_sess; cmd->data_length = data_length; @@ -1432,7 +1433,6 @@ target_cmd_init_cdb(struct se_cmd *cmd, unsigned char *cdb) { sense_reason_t ret; - cmd->t_task_cdb = &cmd->__t_task_cdb[0]; /* * Ensure that the received CDB is less than the max (252 + 8) bytes * for VARIABLE_LENGTH_CMD -- cgit v1.2.3 From cb222a013dca1872deeb49fe8c7176f8aa656d5f Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sat, 27 Feb 2021 10:59:43 -0600 Subject: scsi: target: core: Drop kref_get_unless_zero() in target_get_sess_cmd() The kref_get_unless_zero() use in target_get_sess_cmd() was added in: commit 1b4c59b7a1d0 ("target: fix potential race window in target_sess_cmd_list_waiting()")' but it does not seem to do anything. The original patch might have thought we could have added the cmd to the sess_wait_list and then target_wait_for_sess_cmds could do a put before target_get_sess_cmd did its get. That wouldn't happen because we do the get first then grab the sess lock and put it on the list. It is also not needed now, because the sess_cmd_list does not exist anymore and we instead wait on the session cmd_count. The other problem with the commit is that several target_submit_cmd_map_sgls()/target_submit_cmd() callers do not handle the error case properly if it were to ever happen. These drivers think they have their normal refcount on the cmd and in many cases do a transport_generic_free_cmd() plus target_put_sess_cmd() so they would have fired off the refcount WARN/BUGs. This patch just changes the kref_get_unless_zero() to kref_get(). Link: https://lore.kernel.org/r/20210227170006.5077-3-michael.christie@oracle.com Tested-by: Laurence Oberman Reviewed-by: Christoph Hellwig Reviewed-by: Himanshu Madhani Signed-off-by: Mike Christie Signed-off-by: Martin K. Petersen --- drivers/target/target_core_transport.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'drivers/target/target_core_transport.c') diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index 2dfbf9cb9581..b8d7edadce46 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -2778,9 +2778,7 @@ int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref) * invocations before se_cmd descriptor release. */ if (ack_kref) { - if (!kref_get_unless_zero(&se_cmd->cmd_kref)) - return -EINVAL; - + kref_get(&se_cmd->cmd_kref); se_cmd->se_cmd_flags |= SCF_ACK_KREF; } -- cgit v1.2.3 From a78b713618c02752310b2be7da465a34fb660ed9 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sat, 27 Feb 2021 10:59:44 -0600 Subject: scsi: target: core: Rename transport_init_se_cmd() Rename transport_init_se_cmd() to __target_init_cmd() to reflect that it is more of an internal function that drivers should normally not use and because we are going to add a new init function in the next patches. Link: https://lore.kernel.org/r/20210227170006.5077-4-michael.christie@oracle.com Reviewed-by: Christoph Hellwig Reviewed-by: Himanshu Madhani Signed-off-by: Mike Christie Signed-off-by: Martin K. Petersen --- drivers/target/target_core_transport.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'drivers/target/target_core_transport.c') diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index b8d7edadce46..44ebabad3b99 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -1304,7 +1304,7 @@ target_check_max_data_sg_nents(struct se_cmd *cmd, struct se_device *dev, * Compare the data buffer size from the CDB with the data buffer limit from the transport * header. Set @cmd->residual_count and SCF_OVERFLOW_BIT or SCF_UNDERFLOW_BIT if necessary. * - * Note: target drivers set @cmd->data_length by calling transport_init_se_cmd(). + * Note: target drivers set @cmd->data_length by calling __target_init_cmd(). * * Return: TCM_NO_SENSE */ @@ -1371,7 +1371,7 @@ target_cmd_size_check(struct se_cmd *cmd, unsigned int size) * * Preserves the value of @cmd->tag. */ -void transport_init_se_cmd( +void __target_init_cmd( struct se_cmd *cmd, const struct target_core_fabric_ops *tfo, struct se_session *se_sess, @@ -1405,7 +1405,7 @@ void transport_init_se_cmd( cmd->state_active = false; } -EXPORT_SYMBOL(transport_init_se_cmd); +EXPORT_SYMBOL(__target_init_cmd); static sense_reason_t transport_check_alloc_task_attr(struct se_cmd *cmd) @@ -1625,9 +1625,9 @@ int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess * exceptions are handled by sending exception status via * target_core_fabric_ops->queue_status() callback */ - transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, - data_length, data_dir, task_attr, sense, - unpacked_lun); + __target_init_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, + data_length, data_dir, task_attr, sense, + unpacked_lun); if (flags & TARGET_SCF_UNKNOWN_SIZE) se_cmd->unknown_data_length = 1; @@ -1799,8 +1799,8 @@ int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess, se_tpg = se_sess->se_tpg; BUG_ON(!se_tpg); - transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, - 0, DMA_NONE, TCM_SIMPLE_TAG, sense, unpacked_lun); + __target_init_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, + 0, DMA_NONE, TCM_SIMPLE_TAG, sense, unpacked_lun); /* * FIXME: Currently expect caller to handle se_cmd->se_tmr_req * allocation failure. -- cgit v1.2.3 From 750a1d93f90583a270eb62f36e6d32ebbb6af779 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sat, 27 Feb 2021 10:59:45 -0600 Subject: scsi: target: core: Break up target_submit_cmd_map_sgls() This breaks up target_submit_cmd_map_sgls() into 3 helpers: - target_init_cmd(): Do the basic general setup and get a refcount to the session to make sure the caller can execute the cmd. - target_submit_prep(): Do the mapping, cdb processing and get a ref to the LUN. - target_submit(): Pass the cmd to LIO core for execution. The above functions must be used by drivers that either: 1. Rely on LIO for session shutdown synchronization by calling target_stop_session(). 2. Need to map sgls. When the next patches are applied then simple drivers that do not need the extra functionality above can use target_submit_cmd() and not worry about failures being returned and how to handle them, since many drivers were getting this wrong and would have hit refcount bugs. Also, by breaking target_submit_cmd_map_sgls() up into these 3 helper functions, we can allow the later patches to do the init/prep from interrupt context and then do the submission from a workqueue. Link: https://lore.kernel.org/r/20210227170006.5077-5-michael.christie@oracle.com Cc: Bart Van Assche Cc: Juergen Gross Cc: Hannes Reinecke Cc: Nilesh Javali Cc: Michael Cyr Cc: Chris Boot Cc: Felipe Balbi Cc: "Michael S. Tsirkin" Cc: Stefan Hajnoczi Tested-by: Laurence Oberman Reviewed-by: Christoph Hellwig Reviewed-by: Himanshu Madhani Signed-off-by: Mike Christie Signed-off-by: Martin K. Petersen --- drivers/target/target_core_transport.c | 201 +++++++++++++++++++++++---------- 1 file changed, 140 insertions(+), 61 deletions(-) (limited to 'drivers/target/target_core_transport.c') diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index 44ebabad3b99..000819112bc7 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -1573,46 +1573,31 @@ transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl, } /** - * target_submit_cmd_map_sgls - lookup unpacked lun and submit uninitialized - * se_cmd + use pre-allocated SGL memory. - * - * @se_cmd: command descriptor to submit + * target_init_cmd - initialize se_cmd + * @se_cmd: command descriptor to init * @se_sess: associated se_sess for endpoint - * @cdb: pointer to SCSI CDB * @sense: pointer to SCSI sense buffer * @unpacked_lun: unpacked LUN to reference for struct se_lun * @data_length: fabric expected data transfer length * @task_attr: SAM task attribute * @data_dir: DMA data direction * @flags: flags for command submission from target_sc_flags_tables - * @sgl: struct scatterlist memory for unidirectional mapping - * @sgl_count: scatterlist count for unidirectional mapping - * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping - * @sgl_bidi_count: scatterlist count for bidirectional READ mapping - * @sgl_prot: struct scatterlist memory protection information - * @sgl_prot_count: scatterlist count for protection information * * Task tags are supported if the caller has set @se_cmd->tag. * - * Returns non zero to signal active I/O shutdown failure. All other - * setup exceptions will be returned as a SCSI CHECK_CONDITION response, - * but still return zero here. + * Returns: + * - less than zero to signal active I/O shutdown failure. + * - zero on success. * - * This may only be called from process context, and also currently - * assumes internal allocation of fabric payload buffer by target-core. + * If the fabric driver calls target_stop_session, then it must check the + * return code and handle failures. This will never fail for other drivers, + * and the return code can be ignored. */ -int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess, - unsigned char *cdb, unsigned char *sense, u64 unpacked_lun, - u32 data_length, int task_attr, int data_dir, int flags, - struct scatterlist *sgl, u32 sgl_count, - struct scatterlist *sgl_bidi, u32 sgl_bidi_count, - struct scatterlist *sgl_prot, u32 sgl_prot_count) +int target_init_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, + unsigned char *sense, u64 unpacked_lun, + u32 data_length, int task_attr, int data_dir, int flags) { struct se_portal_group *se_tpg; - sense_reason_t rc; - int ret; - - might_sleep(); se_tpg = se_sess->se_tpg; BUG_ON(!se_tpg); @@ -1620,53 +1605,69 @@ int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess if (flags & TARGET_SCF_USE_CPUID) se_cmd->se_cmd_flags |= SCF_USE_CPUID; + /* + * Signal bidirectional data payloads to target-core + */ + if (flags & TARGET_SCF_BIDI_OP) + se_cmd->se_cmd_flags |= SCF_BIDI; + + if (flags & TARGET_SCF_UNKNOWN_SIZE) + se_cmd->unknown_data_length = 1; /* * Initialize se_cmd for target operation. From this point * exceptions are handled by sending exception status via * target_core_fabric_ops->queue_status() callback */ - __target_init_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, - data_length, data_dir, task_attr, sense, - unpacked_lun); + __target_init_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, data_length, + data_dir, task_attr, sense, unpacked_lun); - if (flags & TARGET_SCF_UNKNOWN_SIZE) - se_cmd->unknown_data_length = 1; /* * Obtain struct se_cmd->cmd_kref reference. A second kref_get here is * necessary for fabrics using TARGET_SCF_ACK_KREF that expect a second * kref_put() to happen during fabric packet acknowledgement. */ - ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); - if (ret) - return ret; - /* - * Signal bidirectional data payloads to target-core - */ - if (flags & TARGET_SCF_BIDI_OP) - se_cmd->se_cmd_flags |= SCF_BIDI; + return target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); +} +EXPORT_SYMBOL_GPL(target_init_cmd); + +/** + * target_submit_prep - prepare cmd for submission + * @se_cmd: command descriptor to prep + * @cdb: pointer to SCSI CDB + * @sgl: struct scatterlist memory for unidirectional mapping + * @sgl_count: scatterlist count for unidirectional mapping + * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping + * @sgl_bidi_count: scatterlist count for bidirectional READ mapping + * @sgl_prot: struct scatterlist memory protection information + * @sgl_prot_count: scatterlist count for protection information + * + * Returns: + * - less than zero to signal failure. + * - zero on success. + * If failure is returned, lio will the callers queue_status to complete + * the cmd. + */ +int target_submit_prep(struct se_cmd *se_cmd, unsigned char *cdb, + struct scatterlist *sgl, u32 sgl_count, + struct scatterlist *sgl_bidi, u32 sgl_bidi_count, + struct scatterlist *sgl_prot, u32 sgl_prot_count) +{ + sense_reason_t rc; rc = target_cmd_init_cdb(se_cmd, cdb); - if (rc) { - transport_send_check_condition_and_sense(se_cmd, rc, 0); - target_put_sess_cmd(se_cmd); - return 0; - } + if (rc) + goto send_cc_direct; /* * Locate se_lun pointer and attach it to struct se_cmd */ rc = transport_lookup_cmd_lun(se_cmd); - if (rc) { - transport_send_check_condition_and_sense(se_cmd, rc, 0); - target_put_sess_cmd(se_cmd); - return 0; - } + if (rc) + goto send_cc_direct; rc = target_cmd_parse_cdb(se_cmd); - if (rc != 0) { - transport_generic_request_failure(se_cmd, rc); - return 0; - } + if (rc != 0) + goto generic_fail; /* * Save pointers for SGLs containing protection information, @@ -1686,6 +1687,41 @@ int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess if (sgl_count != 0) { BUG_ON(!sgl); + rc = transport_generic_map_mem_to_cmd(se_cmd, sgl, sgl_count, + sgl_bidi, sgl_bidi_count); + if (rc != 0) + goto generic_fail; + } + + return 0; + +send_cc_direct: + transport_send_check_condition_and_sense(se_cmd, rc, 0); + target_put_sess_cmd(se_cmd); + return -EIO; + +generic_fail: + transport_generic_request_failure(se_cmd, rc); + return -EIO; +} +EXPORT_SYMBOL_GPL(target_submit_prep); + +/** + * target_submit - perform final initialization and submit cmd to LIO core + * @se_cmd: command descriptor to submit + * + * target_submit_prep must have been called on the cmd, and this must be + * called from process context. + */ +void target_submit(struct se_cmd *se_cmd) +{ + struct scatterlist *sgl = se_cmd->t_data_sg; + unsigned char *buf = NULL; + + might_sleep(); + + if (se_cmd->t_data_nents != 0) { + BUG_ON(!sgl); /* * A work-around for tcm_loop as some userspace code via * scsi-generic do not memset their associated read buffers, @@ -1696,8 +1732,6 @@ int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess */ if (!(se_cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) && se_cmd->data_direction == DMA_FROM_DEVICE) { - unsigned char *buf = NULL; - if (sgl) buf = kmap(sg_page(sgl)) + sgl->offset; @@ -1707,12 +1741,6 @@ int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess } } - rc = transport_generic_map_mem_to_cmd(se_cmd, sgl, sgl_count, - sgl_bidi, sgl_bidi_count); - if (rc != 0) { - transport_generic_request_failure(se_cmd, rc); - return 0; - } } /* @@ -1722,6 +1750,57 @@ int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess core_alua_check_nonop_delay(se_cmd); transport_handle_cdb_direct(se_cmd); +} +EXPORT_SYMBOL_GPL(target_submit); + +/** + * target_submit_cmd_map_sgls - lookup unpacked lun and submit uninitialized + * se_cmd + use pre-allocated SGL memory. + * + * @se_cmd: command descriptor to submit + * @se_sess: associated se_sess for endpoint + * @cdb: pointer to SCSI CDB + * @sense: pointer to SCSI sense buffer + * @unpacked_lun: unpacked LUN to reference for struct se_lun + * @data_length: fabric expected data transfer length + * @task_attr: SAM task attribute + * @data_dir: DMA data direction + * @flags: flags for command submission from target_sc_flags_tables + * @sgl: struct scatterlist memory for unidirectional mapping + * @sgl_count: scatterlist count for unidirectional mapping + * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping + * @sgl_bidi_count: scatterlist count for bidirectional READ mapping + * @sgl_prot: struct scatterlist memory protection information + * @sgl_prot_count: scatterlist count for protection information + * + * Task tags are supported if the caller has set @se_cmd->tag. + * + * Returns non zero to signal active I/O shutdown failure. All other + * setup exceptions will be returned as a SCSI CHECK_CONDITION response, + * but still return zero here. + * + * This may only be called from process context, and also currently + * assumes internal allocation of fabric payload buffer by target-core. + */ +int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess, + unsigned char *cdb, unsigned char *sense, u64 unpacked_lun, + u32 data_length, int task_attr, int data_dir, int flags, + struct scatterlist *sgl, u32 sgl_count, + struct scatterlist *sgl_bidi, u32 sgl_bidi_count, + struct scatterlist *sgl_prot, u32 sgl_prot_count) +{ + int rc; + + rc = target_init_cmd(se_cmd, se_sess, sense, unpacked_lun, + data_length, task_attr, data_dir, flags); + if (rc < 0) + return rc; + + if (target_submit_prep(se_cmd, cdb, sgl, sgl_count, sgl_bidi, + sgl_bidi_count, sgl_prot, sgl_prot_count)) + return 0; + + target_submit(se_cmd); return 0; } EXPORT_SYMBOL(target_submit_cmd_map_sgls); -- cgit v1.2.3 From 0fa50a8b1244e7fc7363712e2c14a27db740cdcb Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sat, 27 Feb 2021 10:59:55 -0600 Subject: scsi: target: core: Remove target_submit_cmd_map_sgls() Convert target_submit_cmd() to do its own calls and then remove target_submit_cmd_map_sgls() since no one uses it. Link: https://lore.kernel.org/r/20210227170006.5077-15-michael.christie@oracle.com Tested-by: Laurence Oberman Reviewed-by: Bodo Stroesser Signed-off-by: Mike Christie Signed-off-by: Martin K. Petersen --- drivers/target/target_core_transport.c | 76 ++++++++-------------------------- 1 file changed, 17 insertions(+), 59 deletions(-) (limited to 'drivers/target/target_core_transport.c') diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index 000819112bc7..560daf9bb039 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -1754,8 +1754,7 @@ void target_submit(struct se_cmd *se_cmd) EXPORT_SYMBOL_GPL(target_submit); /** - * target_submit_cmd_map_sgls - lookup unpacked lun and submit uninitialized - * se_cmd + use pre-allocated SGL memory. + * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd * * @se_cmd: command descriptor to submit * @se_sess: associated se_sess for endpoint @@ -1766,76 +1765,35 @@ EXPORT_SYMBOL_GPL(target_submit); * @task_attr: SAM task attribute * @data_dir: DMA data direction * @flags: flags for command submission from target_sc_flags_tables - * @sgl: struct scatterlist memory for unidirectional mapping - * @sgl_count: scatterlist count for unidirectional mapping - * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping - * @sgl_bidi_count: scatterlist count for bidirectional READ mapping - * @sgl_prot: struct scatterlist memory protection information - * @sgl_prot_count: scatterlist count for protection information * * Task tags are supported if the caller has set @se_cmd->tag. * - * Returns non zero to signal active I/O shutdown failure. All other - * setup exceptions will be returned as a SCSI CHECK_CONDITION response, - * but still return zero here. - * * This may only be called from process context, and also currently * assumes internal allocation of fabric payload buffer by target-core. + * + * It also assumes interal target core SGL memory allocation. + * + * This function must only be used by drivers that do their own + * sync during shutdown and does not use target_stop_session. If there + * is a failure this function will call into the fabric driver's + * queue_status with a CHECK_CONDITION. */ -int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess, +void target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, unsigned char *cdb, unsigned char *sense, u64 unpacked_lun, - u32 data_length, int task_attr, int data_dir, int flags, - struct scatterlist *sgl, u32 sgl_count, - struct scatterlist *sgl_bidi, u32 sgl_bidi_count, - struct scatterlist *sgl_prot, u32 sgl_prot_count) + u32 data_length, int task_attr, int data_dir, int flags) { int rc; - rc = target_init_cmd(se_cmd, se_sess, sense, unpacked_lun, - data_length, task_attr, data_dir, flags); - if (rc < 0) - return rc; + rc = target_init_cmd(se_cmd, se_sess, sense, unpacked_lun, data_length, + task_attr, data_dir, flags); + WARN(rc, "Invalid target_submit_cmd use. Driver must not use target_stop_session or call target_init_cmd directly.\n"); + if (rc) + return; - if (target_submit_prep(se_cmd, cdb, sgl, sgl_count, sgl_bidi, - sgl_bidi_count, sgl_prot, sgl_prot_count)) - return 0; + if (target_submit_prep(se_cmd, cdb, NULL, 0, NULL, 0, NULL, 0)) + return; target_submit(se_cmd); - return 0; -} -EXPORT_SYMBOL(target_submit_cmd_map_sgls); - -/** - * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd - * - * @se_cmd: command descriptor to submit - * @se_sess: associated se_sess for endpoint - * @cdb: pointer to SCSI CDB - * @sense: pointer to SCSI sense buffer - * @unpacked_lun: unpacked LUN to reference for struct se_lun - * @data_length: fabric expected data transfer length - * @task_attr: SAM task attribute - * @data_dir: DMA data direction - * @flags: flags for command submission from target_sc_flags_tables - * - * Task tags are supported if the caller has set @se_cmd->tag. - * - * Returns non zero to signal active I/O shutdown failure. All other - * setup exceptions will be returned as a SCSI CHECK_CONDITION response, - * but still return zero here. - * - * This may only be called from process context, and also currently - * assumes internal allocation of fabric payload buffer by target-core. - * - * It also assumes interal target core SGL memory allocation. - */ -int target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, - unsigned char *cdb, unsigned char *sense, u64 unpacked_lun, - u32 data_length, int task_attr, int data_dir, int flags) -{ - return target_submit_cmd_map_sgls(se_cmd, se_sess, cdb, sense, - unpacked_lun, data_length, task_attr, data_dir, - flags, NULL, 0, NULL, 0, NULL, 0); } EXPORT_SYMBOL(target_submit_cmd); -- cgit v1.2.3 From 08694199477da412baf1852c6d1bf5fedbd40c7e Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sat, 27 Feb 2021 10:59:56 -0600 Subject: scsi: target: core: Add gfp_t arg to target_cmd_init_cdb() tcm_loop could be used like a normal block device, so we can't use GFP_KERNEL and should use GFP_NOIO. This adds a gfp_t arg to target_cmd_init_cdb() and converts the users. For every driver but loop GFP_KERNEL is kept. This will also be useful in subsequent patches where loop needs to do target_submit_prep() from interrupt context to get a ref to the se_device, and so it will need to use GFP_ATOMIC. Link: https://lore.kernel.org/r/20210227170006.5077-16-michael.christie@oracle.com Tested-by: Laurence Oberman Reviewed-by: Christoph Hellwig Signed-off-by: Mike Christie Signed-off-by: Martin K. Petersen --- drivers/target/target_core_transport.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'drivers/target/target_core_transport.c') diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index 560daf9bb039..bd3d125a3978 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -1429,7 +1429,7 @@ transport_check_alloc_task_attr(struct se_cmd *cmd) } sense_reason_t -target_cmd_init_cdb(struct se_cmd *cmd, unsigned char *cdb) +target_cmd_init_cdb(struct se_cmd *cmd, unsigned char *cdb, gfp_t gfp) { sense_reason_t ret; @@ -1450,8 +1450,7 @@ target_cmd_init_cdb(struct se_cmd *cmd, unsigned char *cdb) * setup the pointer from __t_task_cdb to t_task_cdb. */ if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) { - cmd->t_task_cdb = kzalloc(scsi_command_size(cdb), - GFP_KERNEL); + cmd->t_task_cdb = kzalloc(scsi_command_size(cdb), gfp); if (!cmd->t_task_cdb) { pr_err("Unable to allocate cmd->t_task_cdb" " %u > sizeof(cmd->__t_task_cdb): %lu ops\n", @@ -1640,6 +1639,7 @@ EXPORT_SYMBOL_GPL(target_init_cmd); * @sgl_bidi_count: scatterlist count for bidirectional READ mapping * @sgl_prot: struct scatterlist memory protection information * @sgl_prot_count: scatterlist count for protection information + * @gfp: gfp allocation type * * Returns: * - less than zero to signal failure. @@ -1650,11 +1650,12 @@ EXPORT_SYMBOL_GPL(target_init_cmd); int target_submit_prep(struct se_cmd *se_cmd, unsigned char *cdb, struct scatterlist *sgl, u32 sgl_count, struct scatterlist *sgl_bidi, u32 sgl_bidi_count, - struct scatterlist *sgl_prot, u32 sgl_prot_count) + struct scatterlist *sgl_prot, u32 sgl_prot_count, + gfp_t gfp) { sense_reason_t rc; - rc = target_cmd_init_cdb(se_cmd, cdb); + rc = target_cmd_init_cdb(se_cmd, cdb, gfp); if (rc) goto send_cc_direct; @@ -1790,7 +1791,8 @@ void target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, if (rc) return; - if (target_submit_prep(se_cmd, cdb, NULL, 0, NULL, 0, NULL, 0)) + if (target_submit_prep(se_cmd, cdb, NULL, 0, NULL, 0, NULL, 0, + GFP_KERNEL)) return; target_submit(se_cmd); -- cgit v1.2.3 From eb44ce8c8c7d3b45f9204c7f34577960c00d5919 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sat, 27 Feb 2021 10:59:57 -0600 Subject: scsi: target: core: Add workqueue based cmd submission loop and vhost/scsi do their target cmd submission from driver workqueues. This allows them to avoid an issue where the backend may block waiting for resources like tags/requests, mem/locks, etc and that ends up blocking their entire submission path and for the case of vhost-scsi both the submission and completion path. This patch adds a helper drivers can use to submit from a LIO workqueue. This code will then be extended in the next patches to fix the plugging of backend devices. We are only converting vhost/loop initially, but the workqueue based submission will work for other drivers and have similar benefits where the main target loops will not end up blocking one some backend resource. Link: https://lore.kernel.org/r/20210227170006.5077-17-michael.christie@oracle.com Tested-by: Laurence Oberman Reviewed-by: Bodo Stroesser Signed-off-by: Mike Christie Signed-off-by: Martin K. Petersen --- drivers/target/target_core_transport.c | 42 +++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) (limited to 'drivers/target/target_core_transport.c') diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index bd3d125a3978..eea7c27dc4cd 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -41,6 +41,7 @@ #include static struct workqueue_struct *target_completion_wq; +static struct workqueue_struct *target_submission_wq; static struct kmem_cache *se_sess_cache; struct kmem_cache *se_ua_cache; struct kmem_cache *t10_pr_reg_cache; @@ -129,8 +130,15 @@ int init_se_kmem_caches(void) if (!target_completion_wq) goto out_free_lba_map_mem_cache; + target_submission_wq = alloc_workqueue("target_submission", + WQ_MEM_RECLAIM, 0); + if (!target_submission_wq) + goto out_free_completion_wq; + return 0; +out_free_completion_wq: + destroy_workqueue(target_completion_wq); out_free_lba_map_mem_cache: kmem_cache_destroy(t10_alua_lba_map_mem_cache); out_free_lba_map_cache: @@ -153,6 +161,7 @@ out: void release_se_kmem_caches(void) { + destroy_workqueue(target_submission_wq); destroy_workqueue(target_completion_wq); kmem_cache_destroy(se_sess_cache); kmem_cache_destroy(se_ua_cache); @@ -1382,7 +1391,6 @@ void __target_init_cmd( { INIT_LIST_HEAD(&cmd->se_delayed_node); INIT_LIST_HEAD(&cmd->se_qf_node); - INIT_LIST_HEAD(&cmd->se_cmd_list); INIT_LIST_HEAD(&cmd->state_list); init_completion(&cmd->t_transport_stop_comp); cmd->free_compl = NULL; @@ -1799,6 +1807,38 @@ void target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, } EXPORT_SYMBOL(target_submit_cmd); +void target_queued_submit_work(struct work_struct *work) +{ + struct se_cmd_queue *sq = container_of(work, struct se_cmd_queue, work); + struct se_cmd *se_cmd, *next_cmd; + struct llist_node *cmd_list; + + cmd_list = llist_del_all(&sq->cmd_list); + if (!cmd_list) + /* Previous call took what we were queued to submit */ + return; + + cmd_list = llist_reverse_order(cmd_list); + llist_for_each_entry_safe(se_cmd, next_cmd, cmd_list, se_cmd_list) + target_submit(se_cmd); +} + +/** + * target_queue_submission - queue the cmd to run on the LIO workqueue + * @se_cmd: command descriptor to submit + */ +void target_queue_submission(struct se_cmd *se_cmd) +{ + struct se_device *se_dev = se_cmd->se_dev; + int cpu = se_cmd->cpuid; + struct se_cmd_queue *sq; + + sq = &se_dev->queues[cpu].sq; + llist_add(&se_cmd->se_cmd_list, &sq->cmd_list); + queue_work_on(cpu, target_submission_wq, &sq->work); +} +EXPORT_SYMBOL_GPL(target_queue_submission); + static void target_complete_tmr_failure(struct work_struct *work) { struct se_cmd *se_cmd = container_of(work, struct se_cmd, work); -- cgit v1.2.3 From 302990ac3b1b1a2b7b66f59a5c88038a51fbe18e Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sat, 27 Feb 2021 11:00:02 -0600 Subject: scsi: target: core: Fix backend plugging target_core_iblock is plugging and unplugging on every command and this is causing perf issues for drivers that prefer batched cmds. With recent patches we can now take multiple cmds from a fabric driver queue and then pass them down the backend drivers in a batch. This patch adds this support by adding 2 callouts to the backend for plugging and unplugging the device. Subsequent commits will add support for iblock and tcmu device plugging. Link: https://lore.kernel.org/r/20210227170006.5077-22-michael.christie@oracle.com Reviewed-by: Bodo Stroesser Signed-off-by: Mike Christie Signed-off-by: Martin K. Petersen --- drivers/target/target_core_transport.c | 43 +++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) (limited to 'drivers/target/target_core_transport.c') diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index eea7c27dc4cd..1245c288d3bf 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -1807,10 +1807,42 @@ void target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, } EXPORT_SYMBOL(target_submit_cmd); + +static struct se_dev_plug *target_plug_device(struct se_device *se_dev) +{ + struct se_dev_plug *se_plug; + + if (!se_dev->transport->plug_device) + return NULL; + + se_plug = se_dev->transport->plug_device(se_dev); + if (!se_plug) + return NULL; + + se_plug->se_dev = se_dev; + /* + * We have a ref to the lun at this point, but the cmds could + * complete before we unplug, so grab a ref to the se_device so we + * can call back into the backend. + */ + config_group_get(&se_dev->dev_group); + return se_plug; +} + +static void target_unplug_device(struct se_dev_plug *se_plug) +{ + struct se_device *se_dev = se_plug->se_dev; + + se_dev->transport->unplug_device(se_plug); + config_group_put(&se_dev->dev_group); +} + void target_queued_submit_work(struct work_struct *work) { struct se_cmd_queue *sq = container_of(work, struct se_cmd_queue, work); struct se_cmd *se_cmd, *next_cmd; + struct se_dev_plug *se_plug = NULL; + struct se_device *se_dev = NULL; struct llist_node *cmd_list; cmd_list = llist_del_all(&sq->cmd_list); @@ -1819,8 +1851,17 @@ void target_queued_submit_work(struct work_struct *work) return; cmd_list = llist_reverse_order(cmd_list); - llist_for_each_entry_safe(se_cmd, next_cmd, cmd_list, se_cmd_list) + llist_for_each_entry_safe(se_cmd, next_cmd, cmd_list, se_cmd_list) { + if (!se_dev) { + se_dev = se_cmd->se_dev; + se_plug = target_plug_device(se_dev); + } + target_submit(se_cmd); + } + + if (se_plug) + target_unplug_device(se_plug); } /** -- cgit v1.2.3 From 39ae3edda325e9cf9e978c9788affe88231f3b34 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sat, 27 Feb 2021 11:00:06 -0600 Subject: scsi: target: core: Make completion affinity configurable It may not always be best to complete the IO on same CPU as it was submitted on. This commit allows userspace to configure it. This has been useful for vhost-scsi where we have a single thread for submissions and completions. If we force the completion on the submission CPU we may be adding conflicts with what the user has setup in the lower levels with settings like the block layer rq_affinity or the driver's IRQ or softirq (the network's rps_cpus value) settings. We may also want to set it up where the vhost thread runs on CPU N and does its submissions/completions there, and then have LIO do its completion booking on CPU M, but can't configure the lower levels due to issues like using dm-multipath with lots of paths (the path selector can throw commands all over the system because it's only taking into account latency/throughput at its level). The new setting is in: /sys/kernel/config/target/$fabric/$target/param/cmd_completion_affinity Writing: -1 -> Gives the current default behavior of completing on the submission CPU. -2 -> Completes the cmd on the CPU the lower layers sent it to us from. > 0 -> Completes on the CPU userspace has specified. Link: https://lore.kernel.org/r/20210227170006.5077-26-michael.christie@oracle.com Reviewed-by: Himanshu Madhani Signed-off-by: Mike Christie Signed-off-by: Martin K. Petersen --- drivers/target/target_core_transport.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'drivers/target/target_core_transport.c') diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index 1245c288d3bf..a75591c929c0 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -857,7 +857,8 @@ static bool target_cmd_interrupted(struct se_cmd *cmd) /* May be called from interrupt context so must not sleep. */ void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) { - int success; + struct se_wwn *wwn = cmd->se_sess->se_tpg->se_tpg_wwn; + int success, cpu; unsigned long flags; if (target_cmd_interrupted(cmd)) @@ -884,7 +885,13 @@ void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) INIT_WORK(&cmd->work, success ? target_complete_ok_work : target_complete_failure_work); - queue_work_on(cmd->cpuid, target_completion_wq, &cmd->work); + + if (wwn->cmd_compl_affinity == SE_COMPL_AFFINITY_CPUID) + cpu = cmd->cpuid; + else + cpu = wwn->cmd_compl_affinity; + + queue_work_on(cpu, target_completion_wq, &cmd->work); } EXPORT_SYMBOL(target_complete_cmd); -- cgit v1.2.3 From b1ebd3b0e4664c4aa8362bd8abb61861dff61849 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Thu, 18 Mar 2021 17:58:58 -0500 Subject: scsi: target: Fix htmldocs warning in target_submit_prep() Fix warning: drivers/target/target_core_transport.c:1661: WARNING: Block quote ends without a blank line; unexpected unindent. Link: https://lore.kernel.org/r/20210318225858.11863-1-michael.christie@oracle.com Signed-off-by: Mike Christie Signed-off-by: Martin K. Petersen --- drivers/target/target_core_transport.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/target/target_core_transport.c') diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index a75591c929c0..8fbfe75c5744 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -1659,6 +1659,7 @@ EXPORT_SYMBOL_GPL(target_init_cmd); * Returns: * - less than zero to signal failure. * - zero on success. + * * If failure is returned, lio will the callers queue_status to complete * the cmd. */ -- cgit v1.2.3