From 0b524abc2dd13c95a4427f91406c5a69ccca2ccb Mon Sep 17 00:00:00 2001 From: Julian Wiedmann Date: Wed, 28 Oct 2020 19:30:48 +0100 Subject: scsi: zfcp: Lift Input Queue tasklet from qdio Shift the IRQ tasklet processing from the qdio layer into zfcp. This will allow for a good amount of cleanups in qdio, and provides future opportunity to improve the IRQ processing inside zfcp. We continue to use the qdio layer's internal tasklet/timer mechanism (ie. scan_threshold etc) to check for Request Queue completions. Initially we planned to check for such completions after inspecting the Response Queue - this should typically work, but there's a theoretical race where the device only presents the Request Queue completions _after_ all Response Queue processing has finished. If the Request Queue is then also _completely_ full, we could send no further IOs and thus get no interrupt that would trigger an inspection of the Request Queue. So for now stick to the old model, where we can trust that such a race would be recovered by qdio's internal timer. Code-flow wise, this establishes two levels of control: 1. The qdio layer will only deliver IRQs to the device driver if the QDIO_IRQ_DISABLED flag is cleared. zfcp manages this through qdio_start_irq() / qdio_stop_irq(). The initial state is DISABLED, and zfcp_qdio_open() schedules zfcp's IRQ tasklet once during startup to explicitly enable IRQ delivery. 2. The zfcp tasklet is initialized with tasklet_disable(), and only gets enabled once we open the qdio device. When closing the qdio device, we must disable the tasklet _before_ disabling IRQ delivery (otherwise a concurrently running tasklet could re-enable IRQ delivery after we disabled it). A final tasklet_kill() during teardown ensures that no lingering tasklet_schedule() is still accessing the tasklet structure. Link: https://lore.kernel.org/r/94a765211c48b74a7b91c5e60b158de01db98d43.1603908167.git.bblock@linux.ibm.com Reviewed-by: Benjamin Block Signed-off-by: Julian Wiedmann Signed-off-by: Benjamin Block Signed-off-by: Martin K. Petersen --- drivers/s390/scsi/zfcp_qdio.c | 39 +++++++++++++++++++++++++++++++++++++++ drivers/s390/scsi/zfcp_qdio.h | 2 ++ 2 files changed, 41 insertions(+) (limited to 'drivers/s390/scsi') diff --git a/drivers/s390/scsi/zfcp_qdio.c b/drivers/s390/scsi/zfcp_qdio.c index a8a514074084..9fc045ddf66d 100644 --- a/drivers/s390/scsi/zfcp_qdio.c +++ b/drivers/s390/scsi/zfcp_qdio.c @@ -131,6 +131,33 @@ static void zfcp_qdio_int_resp(struct ccw_device *cdev, unsigned int qdio_err, zfcp_erp_adapter_reopen(qdio->adapter, 0, "qdires2"); } +static void zfcp_qdio_irq_tasklet(struct tasklet_struct *tasklet) +{ + struct zfcp_qdio *qdio = from_tasklet(qdio, tasklet, irq_tasklet); + struct ccw_device *cdev = qdio->adapter->ccw_device; + unsigned int start, error; + int completed; + + /* Check the Response Queue, and kick off the Request Queue tasklet: */ + completed = qdio_get_next_buffers(cdev, 0, &start, &error); + if (completed < 0) + return; + if (completed > 0) + zfcp_qdio_int_resp(cdev, error, 0, start, completed, + (unsigned long) qdio); + + if (qdio_start_irq(cdev)) + /* More work pending: */ + tasklet_schedule(&qdio->irq_tasklet); +} + +static void zfcp_qdio_poll(struct ccw_device *cdev, unsigned long data) +{ + struct zfcp_qdio *qdio = (struct zfcp_qdio *) data; + + tasklet_schedule(&qdio->irq_tasklet); +} + static struct qdio_buffer_element * zfcp_qdio_sbal_chain(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req) { @@ -332,6 +359,8 @@ void zfcp_qdio_close(struct zfcp_qdio *qdio) wake_up(&qdio->req_q_wq); + tasklet_disable(&qdio->irq_tasklet); + qdio_stop_irq(adapter->ccw_device); qdio_shutdown(adapter->ccw_device, QDIO_FLAG_CLEANUP_USING_CLEAR); /* cleanup used outbound sbals */ @@ -387,6 +416,7 @@ int zfcp_qdio_open(struct zfcp_qdio *qdio) init_data.no_output_qs = 1; init_data.input_handler = zfcp_qdio_int_resp; init_data.output_handler = zfcp_qdio_int_req; + init_data.irq_poll = zfcp_qdio_poll; init_data.int_parm = (unsigned long) qdio; init_data.input_sbal_addr_array = input_sbals; init_data.output_sbal_addr_array = output_sbals; @@ -433,6 +463,11 @@ int zfcp_qdio_open(struct zfcp_qdio *qdio) atomic_set(&qdio->req_q_free, QDIO_MAX_BUFFERS_PER_Q); atomic_or(ZFCP_STATUS_ADAPTER_QDIOUP, &qdio->adapter->status); + /* Enable processing for QDIO interrupts: */ + tasklet_enable(&qdio->irq_tasklet); + /* This results in a qdio_start_irq(): */ + tasklet_schedule(&qdio->irq_tasklet); + zfcp_qdio_shost_update(adapter, qdio); return 0; @@ -450,6 +485,8 @@ void zfcp_qdio_destroy(struct zfcp_qdio *qdio) if (!qdio) return; + tasklet_kill(&qdio->irq_tasklet); + if (qdio->adapter->ccw_device) qdio_free(qdio->adapter->ccw_device); @@ -475,6 +512,8 @@ int zfcp_qdio_setup(struct zfcp_adapter *adapter) spin_lock_init(&qdio->req_q_lock); spin_lock_init(&qdio->stat_lock); + tasklet_setup(&qdio->irq_tasklet, zfcp_qdio_irq_tasklet); + tasklet_disable(&qdio->irq_tasklet); adapter->qdio = qdio; return 0; diff --git a/drivers/s390/scsi/zfcp_qdio.h b/drivers/s390/scsi/zfcp_qdio.h index 6b43d6b254be..9c1f310db155 100644 --- a/drivers/s390/scsi/zfcp_qdio.h +++ b/drivers/s390/scsi/zfcp_qdio.h @@ -10,6 +10,7 @@ #ifndef ZFCP_QDIO_H #define ZFCP_QDIO_H +#include #include #define ZFCP_QDIO_SBALE_LEN PAGE_SIZE @@ -44,6 +45,7 @@ struct zfcp_qdio { u64 req_q_util; atomic_t req_q_full; wait_queue_head_t req_q_wq; + struct tasklet_struct irq_tasklet; struct zfcp_adapter *adapter; u16 max_sbale_per_sbal; u16 max_sbale_per_req; -- cgit v1.2.3 From 84e7b4169f949f8185a5adf0f0bfb893030d0fda Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Wed, 28 Oct 2020 19:30:49 +0100 Subject: scsi: zfcp: Remove orphaned function declarations drivers/s390/scsi/zfcp_ext.h: zfcp_sg_free_table - only declaration left after commit 58f3ead54752 ("scsi: zfcp: move SG table helper from aux to fc and make them static") drivers/s390/scsi/zfcp_ext.h: zfcp_sg_setup_table - only declaration left after commit 58f3ead54752 ("scsi: zfcp: move SG table helper from aux to fc and make them static") Link: https://lore.kernel.org/r/6854ae03c5c65805f746774eeb0f2869fcd6c397.1603908167.git.bblock@linux.ibm.com Reviewed-by: Steffen Maier Reviewed-by: Benjamin Block Signed-off-by: Vasily Gorbik Signed-off-by: Benjamin Block Signed-off-by: Martin K. Petersen --- drivers/s390/scsi/zfcp_ext.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'drivers/s390/scsi') diff --git a/drivers/s390/scsi/zfcp_ext.h b/drivers/s390/scsi/zfcp_ext.h index 3ef5d74331c3..8b50f580584c 100644 --- a/drivers/s390/scsi/zfcp_ext.h +++ b/drivers/s390/scsi/zfcp_ext.h @@ -20,8 +20,6 @@ extern struct zfcp_port *zfcp_get_port_by_wwpn(struct zfcp_adapter *, u64); extern struct zfcp_adapter *zfcp_adapter_enqueue(struct ccw_device *); extern struct zfcp_port *zfcp_port_enqueue(struct zfcp_adapter *, u64, u32, u32); -extern void zfcp_sg_free_table(struct scatterlist *, int); -extern int zfcp_sg_setup_table(struct scatterlist *, int); extern void zfcp_adapter_release(struct kref *); extern void zfcp_adapter_unregister(struct zfcp_adapter *); -- cgit v1.2.3 From efd321768d2e0e85083b83aefb15c949d4c8930f Mon Sep 17 00:00:00 2001 From: Julian Wiedmann Date: Wed, 28 Oct 2020 19:30:50 +0100 Subject: scsi: zfcp: Clarify & assert the stat_lock locking in zfcp_qdio_send() Explain why the plain spin_lock() suffices in current code, even when the stat_lock is also used by zfcp_qdio_int_req() in tasklet context. We could also promote the spin_lock() to a spin_lock_irqsave(), but that would just obfuscate the locking even further. Link: https://lore.kernel.org/r/b023b1472630f4bf9fce580577d7bb49de89ccbf.1603908167.git.bblock@linux.ibm.com Reviewed-by: Benjamin Block Reviewed-by: Steffen Maier Signed-off-by: Julian Wiedmann Signed-off-by: Benjamin Block Signed-off-by: Martin K. Petersen --- drivers/s390/scsi/zfcp_qdio.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'drivers/s390/scsi') diff --git a/drivers/s390/scsi/zfcp_qdio.c b/drivers/s390/scsi/zfcp_qdio.c index 9fc045ddf66d..23ab16d65f2a 100644 --- a/drivers/s390/scsi/zfcp_qdio.c +++ b/drivers/s390/scsi/zfcp_qdio.c @@ -10,6 +10,7 @@ #define KMSG_COMPONENT "zfcp" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#include #include #include #include "zfcp_ext.h" @@ -283,6 +284,13 @@ int zfcp_qdio_send(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req) int retval; u8 sbal_number = q_req->sbal_number; + /* + * This should actually be a spin_lock_bh(stat_lock), to protect against + * zfcp_qdio_int_req() in tasklet context. + * But we can't do so (and are safe), as we always get called with IRQs + * disabled by spin_lock_irq[save](req_q_lock). + */ + lockdep_assert_irqs_disabled(); spin_lock(&qdio->stat_lock); zfcp_qdio_account(qdio); spin_unlock(&qdio->stat_lock); -- cgit v1.2.3 From a6c37abe6988eb33a5f301e252ee41ed22b8df8d Mon Sep 17 00:00:00 2001 From: Julian Wiedmann Date: Wed, 28 Oct 2020 19:30:51 +0100 Subject: scsi: zfcp: Process Version Change events Handle notifications for a concurrent change of the FCP Channel firmware. Update the relevant user-visible fields to provide accurate data. Link: https://lore.kernel.org/r/d2c7bc57c6cf1b65eabbf7a5d0e3927b9f65647f.1603908167.git.bblock@linux.ibm.com Reviewed-by: Steffen Maier Reviewed-by: Benjamin Block Signed-off-by: Julian Wiedmann Signed-off-by: Benjamin Block Signed-off-by: Martin K. Petersen --- drivers/s390/scsi/zfcp_fsf.c | 16 ++++++++++++++++ drivers/s390/scsi/zfcp_fsf.h | 10 ++++++++++ 2 files changed, 26 insertions(+) (limited to 'drivers/s390/scsi') diff --git a/drivers/s390/scsi/zfcp_fsf.c b/drivers/s390/scsi/zfcp_fsf.c index 6cb963a06777..afa95e04eceb 100644 --- a/drivers/s390/scsi/zfcp_fsf.c +++ b/drivers/s390/scsi/zfcp_fsf.c @@ -242,6 +242,19 @@ static void zfcp_fsf_status_read_link_down(struct zfcp_fsf_req *req) } } +static void +zfcp_fsf_status_read_version_change(struct zfcp_adapter *adapter, + struct fsf_status_read_buffer *sr_buf) +{ + if (sr_buf->status_subtype == FSF_STATUS_READ_SUB_LIC_CHANGE) { + u32 version = sr_buf->payload.version_change.current_version; + + WRITE_ONCE(adapter->fsf_lic_version, version); + snprintf(fc_host_firmware_version(adapter->scsi_host), + FC_VERSION_STRING_SIZE, "%#08x", version); + } +} + static void zfcp_fsf_status_read_handler(struct zfcp_fsf_req *req) { struct zfcp_adapter *adapter = req->adapter; @@ -300,6 +313,9 @@ static void zfcp_fsf_status_read_handler(struct zfcp_fsf_req *req) case FSF_STATUS_READ_FEATURE_UPDATE_ALERT: adapter->adapter_features = sr_buf->payload.word[0]; break; + case FSF_STATUS_READ_VERSION_CHANGE: + zfcp_fsf_status_read_version_change(adapter, sr_buf); + break; } mempool_free(virt_to_page(sr_buf), adapter->pool.sr_data); diff --git a/drivers/s390/scsi/zfcp_fsf.h b/drivers/s390/scsi/zfcp_fsf.h index 09d73d0061ef..26ad7a0c5ce3 100644 --- a/drivers/s390/scsi/zfcp_fsf.h +++ b/drivers/s390/scsi/zfcp_fsf.h @@ -134,6 +134,7 @@ #define FSF_STATUS_READ_LINK_UP 0x00000006 #define FSF_STATUS_READ_NOTIFICATION_LOST 0x00000009 #define FSF_STATUS_READ_FEATURE_UPDATE_ALERT 0x0000000C +#define FSF_STATUS_READ_VERSION_CHANGE 0x0000000D /* status subtypes for link down */ #define FSF_STATUS_READ_SUB_NO_PHYSICAL_LINK 0x00000000 @@ -143,6 +144,9 @@ /* status subtypes for unsolicited status notification lost */ #define FSF_STATUS_READ_SUB_INCOMING_ELS 0x00000001 +/* status subtypes for version change */ +#define FSF_STATUS_READ_SUB_LIC_CHANGE 0x00000001 + /* topologie that is detected by the adapter */ #define FSF_TOPO_P2P 0x00000001 #define FSF_TOPO_FABRIC 0x00000002 @@ -226,6 +230,11 @@ struct fsf_link_down_info { u8 vendor_specific_code; } __attribute__ ((packed)); +struct fsf_version_change { + u32 current_version; + u32 previous_version; +} __packed; + struct fsf_status_read_buffer { u32 status_type; u32 status_subtype; @@ -242,6 +251,7 @@ struct fsf_status_read_buffer { u32 word[FSF_STATUS_READ_PAYLOAD_SIZE/sizeof(u32)]; struct fsf_link_down_info link_down_info; struct fsf_bit_error_payload bit_error; + struct fsf_version_change version_change; } payload; } __attribute__ ((packed)); -- cgit v1.2.3 From d90196317484b69bb46b7144c6e0e1a4f581200d Mon Sep 17 00:00:00 2001 From: Julian Wiedmann Date: Wed, 28 Oct 2020 19:30:52 +0100 Subject: scsi: zfcp: Handle event-lost notification for Version Change events As recovery for a lost Version Change event, trigger an Exchange Config Data cmd to retrieve the current FW version. Doing so requires process context (as eg. zfcp_qdio_sbal_get() might need to sleep), so defer from tasklet context into a work item. Link: https://lore.kernel.org/r/297c7be2944c3714863fcd22d531d910312d29f0.1603908167.git.bblock@linux.ibm.com Suggested-by: Steffen Maier Reviewed-by: Steffen Maier Reviewed-by: Benjamin Block Signed-off-by: Julian Wiedmann Signed-off-by: Benjamin Block Signed-off-by: Martin K. Petersen --- drivers/s390/scsi/zfcp_aux.c | 11 +++++++++++ drivers/s390/scsi/zfcp_def.h | 1 + drivers/s390/scsi/zfcp_fsf.c | 3 +++ drivers/s390/scsi/zfcp_fsf.h | 1 + 4 files changed, 16 insertions(+) (limited to 'drivers/s390/scsi') diff --git a/drivers/s390/scsi/zfcp_aux.c b/drivers/s390/scsi/zfcp_aux.c index 18b713a616de..768873dd55b8 100644 --- a/drivers/s390/scsi/zfcp_aux.c +++ b/drivers/s390/scsi/zfcp_aux.c @@ -292,6 +292,14 @@ static void _zfcp_status_read_scheduler(struct work_struct *work) stat_work)); } +static void zfcp_version_change_lost_work(struct work_struct *work) +{ + struct zfcp_adapter *adapter = container_of(work, struct zfcp_adapter, + version_change_lost_work); + + zfcp_fsf_exchange_config_data_sync(adapter->qdio, NULL); +} + static void zfcp_print_sl(struct seq_file *m, struct service_level *sl) { struct zfcp_adapter *adapter = @@ -353,6 +361,8 @@ struct zfcp_adapter *zfcp_adapter_enqueue(struct ccw_device *ccw_device) INIT_WORK(&adapter->stat_work, _zfcp_status_read_scheduler); INIT_DELAYED_WORK(&adapter->scan_work, zfcp_fc_scan_ports); INIT_WORK(&adapter->ns_up_work, zfcp_fc_sym_name_update); + INIT_WORK(&adapter->version_change_lost_work, + zfcp_version_change_lost_work); adapter->next_port_scan = jiffies; @@ -429,6 +439,7 @@ void zfcp_adapter_unregister(struct zfcp_adapter *adapter) cancel_delayed_work_sync(&adapter->scan_work); cancel_work_sync(&adapter->stat_work); cancel_work_sync(&adapter->ns_up_work); + cancel_work_sync(&adapter->version_change_lost_work); zfcp_destroy_adapter_work_queue(adapter); zfcp_fc_wka_ports_force_offline(adapter->gs); diff --git a/drivers/s390/scsi/zfcp_def.h b/drivers/s390/scsi/zfcp_def.h index da8a5ceb615c..f656d74a5f94 100644 --- a/drivers/s390/scsi/zfcp_def.h +++ b/drivers/s390/scsi/zfcp_def.h @@ -201,6 +201,7 @@ struct zfcp_adapter { struct zfcp_fc_events events; unsigned long next_port_scan; struct zfcp_diag_adapter *diagnostics; + struct work_struct version_change_lost_work; }; struct zfcp_port { diff --git a/drivers/s390/scsi/zfcp_fsf.c b/drivers/s390/scsi/zfcp_fsf.c index afa95e04eceb..7593a9667b3e 100644 --- a/drivers/s390/scsi/zfcp_fsf.c +++ b/drivers/s390/scsi/zfcp_fsf.c @@ -309,6 +309,9 @@ static void zfcp_fsf_status_read_handler(struct zfcp_fsf_req *req) case FSF_STATUS_READ_NOTIFICATION_LOST: if (sr_buf->status_subtype & FSF_STATUS_READ_SUB_INCOMING_ELS) zfcp_fc_conditional_port_scan(adapter); + if (sr_buf->status_subtype & FSF_STATUS_READ_SUB_VERSION_CHANGE) + queue_work(adapter->work_queue, + &adapter->version_change_lost_work); break; case FSF_STATUS_READ_FEATURE_UPDATE_ALERT: adapter->adapter_features = sr_buf->payload.word[0]; diff --git a/drivers/s390/scsi/zfcp_fsf.h b/drivers/s390/scsi/zfcp_fsf.h index 26ad7a0c5ce3..5e6b601af980 100644 --- a/drivers/s390/scsi/zfcp_fsf.h +++ b/drivers/s390/scsi/zfcp_fsf.h @@ -143,6 +143,7 @@ /* status subtypes for unsolicited status notification lost */ #define FSF_STATUS_READ_SUB_INCOMING_ELS 0x00000001 +#define FSF_STATUS_READ_SUB_VERSION_CHANGE 0x00000100 /* status subtypes for version change */ #define FSF_STATUS_READ_SUB_LIC_CHANGE 0x00000001 -- cgit v1.2.3