diff options
Diffstat (limited to 'drivers/crypto/hisilicon')
-rw-r--r-- | drivers/crypto/hisilicon/hpre/hpre.h | 1 | ||||
-rw-r--r-- | drivers/crypto/hisilicon/hpre/hpre_crypto.c | 59 | ||||
-rw-r--r-- | drivers/crypto/hisilicon/hpre/hpre_main.c | 33 | ||||
-rw-r--r-- | drivers/crypto/hisilicon/qm.c | 237 | ||||
-rw-r--r-- | drivers/crypto/hisilicon/qm.h | 31 | ||||
-rw-r--r-- | drivers/crypto/hisilicon/sec2/sec_crypto.c | 51 | ||||
-rw-r--r-- | drivers/crypto/hisilicon/sec2/sec_main.c | 39 | ||||
-rw-r--r-- | drivers/crypto/hisilicon/zip/zip.h | 15 | ||||
-rw-r--r-- | drivers/crypto/hisilicon/zip/zip_crypto.c | 140 | ||||
-rw-r--r-- | drivers/crypto/hisilicon/zip/zip_main.c | 195 |
10 files changed, 493 insertions, 308 deletions
diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index ed730d173e95..f69252b24671 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -56,7 +56,6 @@ struct hpre_dfx { * Just relevant for PF. */ struct hpre_debug { - struct dentry *debug_root; struct hpre_dfx dfx[HPRE_DFX_FILE_NUM]; struct hpre_debugfs_file files[HPRE_DEBUGFS_FILE_NUM]; }; diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 7b5cb27d473d..a87f9904087a 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -98,9 +98,6 @@ struct hpre_asym_request { struct timespec64 req_time; }; -static DEFINE_MUTEX(hpre_alg_lock); -static unsigned int hpre_active_devs; - static int hpre_alloc_req_id(struct hpre_ctx *ctx) { unsigned long flags; @@ -191,8 +188,7 @@ static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req, hpre_req->dst = NULL; dma_dir = DMA_FROM_DEVICE; } - *tmp = dma_map_single(dev, sg_virt(data), - len, dma_dir); + *tmp = dma_map_single(dev, sg_virt(data), len, dma_dir); if (unlikely(dma_mapping_error(dev, *tmp))) { dev_err(dev, "dma map data err!\n"); return -ENOMEM; @@ -242,8 +238,8 @@ static int hpre_hw_data_init(struct hpre_asym_request *hpre_req, ((is_dh && !is_src) || !is_dh)) ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp); else - ret = hpre_prepare_dma_buf(hpre_req, data, len, - is_src, &tmp); + ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp); + if (unlikely(ret)) return ret; @@ -270,11 +266,9 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, if (src) { if (req->src) - dma_free_coherent(dev, ctx->key_sz, - req->src, tmp); + dma_free_coherent(dev, ctx->key_sz, req->src, tmp); else - dma_unmap_single(dev, tmp, - ctx->key_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE); } tmp = le64_to_cpu(sqe->out); @@ -477,7 +471,7 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) h_req->areq.dh = kreq; msg = &h_req->req; memset(msg, 0, sizeof(*msg)); - msg->key = cpu_to_le64((u64)ctx->dh.dma_xa_p); + msg->key = cpu_to_le64(ctx->dh.dma_xa_p); } msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT); @@ -534,6 +528,8 @@ static int hpre_dh_compute_value(struct kpp_request *req) ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1); if (unlikely(ret)) goto clear_all; + } else { + msg->in = cpu_to_le64(ctx->dh.dma_g); } ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1); @@ -743,7 +739,7 @@ static int hpre_rsa_enc(struct akcipher_request *req) return ret; msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT); - msg->key = cpu_to_le64((u64)ctx->rsa.dma_pubkey); + msg->key = cpu_to_le64(ctx->rsa.dma_pubkey); ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0); if (unlikely(ret)) @@ -791,11 +787,11 @@ static int hpre_rsa_dec(struct akcipher_request *req) return ret; if (ctx->crt_g2_mode) { - msg->key = cpu_to_le64((u64)ctx->rsa.dma_crt_prikey); + msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey); msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_NC_CRT); } else { - msg->key = cpu_to_le64((u64)ctx->rsa.dma_prikey); + msg->key = cpu_to_le64(ctx->rsa.dma_prikey); msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_NC_NCRT); } @@ -1160,36 +1156,25 @@ static struct kpp_alg dh = { int hpre_algs_register(void) { - int ret = 0; - - mutex_lock(&hpre_alg_lock); - if (++hpre_active_devs == 1) { - rsa.base.cra_flags = 0; - ret = crypto_register_akcipher(&rsa); - if (ret) - goto unlock; + int ret; + + rsa.base.cra_flags = 0; + ret = crypto_register_akcipher(&rsa); + if (ret) + return ret; #ifdef CONFIG_CRYPTO_DH - ret = crypto_register_kpp(&dh); - if (ret) { - crypto_unregister_akcipher(&rsa); - goto unlock; - } + ret = crypto_register_kpp(&dh); + if (ret) + crypto_unregister_akcipher(&rsa); #endif - } -unlock: - mutex_unlock(&hpre_alg_lock); return ret; } void hpre_algs_unregister(void) { - mutex_lock(&hpre_alg_lock); - if (--hpre_active_devs == 0) { - crypto_unregister_akcipher(&rsa); + crypto_unregister_akcipher(&rsa); #ifdef CONFIG_CRYPTO_DH - crypto_unregister_kpp(&dh); + crypto_unregister_kpp(&dh); #endif - } - mutex_unlock(&hpre_alg_lock); } diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index b135c74fb619..a33394d91bbf 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -90,7 +90,6 @@ #define HPRE_SQE_MASK_OFFSET 8 #define HPRE_SQE_MASK_LEN 24 -static struct hisi_qm_list hpre_devices; static const char hpre_name[] = "hisi_hpre"; static struct dentry *hpre_debugfs_root; static const struct pci_device_id hpre_dev_ids[] = { @@ -106,6 +105,11 @@ struct hpre_hw_error { const char *msg; }; +static struct hisi_qm_list hpre_devices = { + .register_to_crypto = hpre_algs_register, + .unregister_from_crypto = hpre_algs_unregister, +}; + static const char * const hpre_debug_file_name[] = { [HPRE_CURRENT_QM] = "current_qm", [HPRE_CLEAR_ENABLE] = "rdclr_en", @@ -186,7 +190,7 @@ static const struct kernel_param_ops hpre_pf_q_num_ops = { static u32 pf_q_num = HPRE_PF_DEF_Q_NUM; module_param_cb(pf_q_num, &hpre_pf_q_num_ops, &pf_q_num, 0444); -MODULE_PARM_DESC(pf_q_num, "Number of queues in PF of CS(1-1024)"); +MODULE_PARM_DESC(pf_q_num, "Number of queues in PF of CS(2-1024)"); static const struct kernel_param_ops vfs_num_ops = { .set = vfs_num_set, @@ -864,9 +868,7 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) dev_warn(&pdev->dev, "init debugfs fail!\n"); - hisi_qm_add_to_list(qm, &hpre_devices); - - ret = hpre_algs_register(); + ret = hisi_qm_alg_register(qm, &hpre_devices); if (ret < 0) { pci_err(pdev, "fail to register algs to crypto!\n"); goto err_with_qm_start; @@ -875,18 +877,17 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (qm->fun_type == QM_HW_PF && vfs_num) { ret = hisi_qm_sriov_enable(pdev, vfs_num); if (ret < 0) - goto err_with_crypto_register; + goto err_with_alg_register; } return 0; -err_with_crypto_register: - hpre_algs_unregister(); +err_with_alg_register: + hisi_qm_alg_unregister(qm, &hpre_devices); err_with_qm_start: - hisi_qm_del_from_list(qm, &hpre_devices); hpre_debugfs_exit(qm); - hisi_qm_stop(qm); + hisi_qm_stop(qm, QM_NORMAL); err_with_err_init: hisi_qm_dev_err_uninit(qm); @@ -899,14 +900,13 @@ err_with_qm_init: static void hpre_remove(struct pci_dev *pdev) { - struct hpre *hpre = pci_get_drvdata(pdev); - struct hisi_qm *qm = &hpre->qm; + struct hisi_qm *qm = pci_get_drvdata(pdev); int ret; - hpre_algs_unregister(); - hisi_qm_del_from_list(qm, &hpre_devices); + hisi_qm_wait_task_finish(qm, &hpre_devices); + hisi_qm_alg_unregister(qm, &hpre_devices); if (qm->fun_type == QM_HW_PF && qm->vfs_num) { - ret = hisi_qm_sriov_disable(pdev); + ret = hisi_qm_sriov_disable(pdev, qm->is_frozen); if (ret) { pci_err(pdev, "Disable SRIOV fail!\n"); return; @@ -918,7 +918,7 @@ static void hpre_remove(struct pci_dev *pdev) } hpre_debugfs_exit(qm); - hisi_qm_stop(qm); + hisi_qm_stop(qm, QM_NORMAL); hisi_qm_dev_err_uninit(qm); hisi_qm_uninit(qm); } @@ -939,6 +939,7 @@ static struct pci_driver hpre_pci_driver = { .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ? hisi_qm_sriov_configure : NULL, .err_handler = &hpre_err_handler, + .shutdown = hisi_qm_dev_shutdown, }; static void hpre_register_debugfs(void) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 6527c53b073f..530f23116d7c 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -180,7 +180,10 @@ #define QM_DBG_TMP_BUF_LEN 22 #define QM_PCI_COMMAND_INVALID ~0 +#define WAIT_PERIOD 20 +#define REMOVE_WAIT_DELAY 10 #define QM_SQE_ADDR_MASK GENMASK(7, 0) +#define QM_EQ_DEPTH (1024 * 2) #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \ (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \ @@ -652,7 +655,7 @@ static void qm_work_process(struct work_struct *work) qp = qm_to_hisi_qp(qm, eqe); qm_poll_qp(qp, qm); - if (qm->status.eq_head == QM_Q_DEPTH - 1) { + if (qm->status.eq_head == QM_EQ_DEPTH - 1) { qm->status.eqc_phase = !qm->status.eqc_phase; eqe = qm->eqe; qm->status.eq_head = 0; @@ -661,7 +664,7 @@ static void qm_work_process(struct work_struct *work) qm->status.eq_head++; } - if (eqe_num == QM_Q_DEPTH / 2 - 1) { + if (eqe_num == QM_EQ_DEPTH / 2 - 1) { eqe_num = 0; qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); } @@ -754,7 +757,7 @@ static void qm_init_qp_status(struct hisi_qp *qp) qp_status->sq_tail = 0; qp_status->cq_head = 0; qp_status->cqc_phase = true; - atomic_set(&qp_status->flags, 0); + atomic_set(&qp_status->used, 0); } static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, @@ -1046,17 +1049,7 @@ static int qm_regs_show(struct seq_file *s, void *unused) return 0; } -static int qm_regs_open(struct inode *inode, struct file *file) -{ - return single_open(file, qm_regs_show, inode->i_private); -} - -static const struct file_operations qm_regs_fops = { - .owner = THIS_MODULE, - .open = qm_regs_open, - .read = seq_read, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(qm_regs); static ssize_t qm_cmd_read(struct file *filp, char __user *buffer, size_t count, loff_t *pos) @@ -1370,7 +1363,13 @@ static int qm_eq_aeq_dump(struct hisi_qm *qm, const char *s, return -EINVAL; ret = kstrtou32(s, 0, &xeqe_id); - if (ret || xeqe_id >= QM_Q_DEPTH) { + if (ret) + return -EINVAL; + + if (!strcmp(name, "EQE") && xeqe_id >= QM_EQ_DEPTH) { + dev_err(dev, "Please input eqe num (0-%d)", QM_EQ_DEPTH - 1); + return -EINVAL; + } else if (!strcmp(name, "AEQE") && xeqe_id >= QM_Q_DEPTH) { dev_err(dev, "Please input aeqe num (0-%d)", QM_Q_DEPTH - 1); return -EINVAL; } @@ -1420,17 +1419,18 @@ static int qm_dbg_help(struct hisi_qm *qm, char *s) static int qm_cmd_write_dump(struct hisi_qm *qm, const char *cmd_buf) { struct device *dev = &qm->pdev->dev; - char *presult, *s; + char *presult, *s, *s_tmp; int ret; s = kstrdup(cmd_buf, GFP_KERNEL); if (!s) return -ENOMEM; + s_tmp = s; presult = strsep(&s, " "); if (!presult) { - kfree(s); - return -EINVAL; + ret = -EINVAL; + goto err_buffer_free; } if (!strcmp(presult, "sqc")) @@ -1459,7 +1459,8 @@ static int qm_cmd_write_dump(struct hisi_qm *qm, const char *cmd_buf) if (ret) dev_info(dev, "Please echo help\n"); - kfree(s); +err_buffer_free: + kfree(s_tmp); return ret; } @@ -1644,7 +1645,7 @@ static void *qm_get_avail_sqe(struct hisi_qp *qp) struct hisi_qp_status *qp_status = &qp->qp_status; u16 sq_tail = qp_status->sq_tail; - if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH)) + if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH - 1)) return NULL; return qp->sqe + sq_tail * qp->qm->sqe_size; @@ -1981,7 +1982,7 @@ int hisi_qp_send(struct hisi_qp *qp, const void *msg) if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP || atomic_read(&qp->qm->status.flags) == QM_STOP || qp->is_resetting)) { - dev_info(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); + dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); return -EAGAIN; } @@ -2215,6 +2216,82 @@ static int qm_alloc_uacce(struct hisi_qm *qm) } /** + * qm_frozen() - Try to froze QM to cut continuous queue request. If + * there is user on the QM, return failure without doing anything. + * @qm: The qm needed to be fronzen. + * + * This function frozes QM, then we can do SRIOV disabling. + */ +static int qm_frozen(struct hisi_qm *qm) +{ + down_write(&qm->qps_lock); + + if (qm->is_frozen) { + up_write(&qm->qps_lock); + return 0; + } + + if (!qm->qp_in_used) { + qm->qp_in_used = qm->qp_num; + qm->is_frozen = true; + up_write(&qm->qps_lock); + return 0; + } + + up_write(&qm->qps_lock); + + return -EBUSY; +} + +static int qm_try_frozen_vfs(struct pci_dev *pdev, + struct hisi_qm_list *qm_list) +{ + struct hisi_qm *qm, *vf_qm; + struct pci_dev *dev; + int ret = 0; + + if (!qm_list || !pdev) + return -EINVAL; + + /* Try to frozen all the VFs as disable SRIOV */ + mutex_lock(&qm_list->lock); + list_for_each_entry(qm, &qm_list->list, list) { + dev = qm->pdev; + if (dev == pdev) + continue; + if (pci_physfn(dev) == pdev) { + vf_qm = pci_get_drvdata(dev); + ret = qm_frozen(vf_qm); + if (ret) + goto frozen_fail; + } + } + +frozen_fail: + mutex_unlock(&qm_list->lock); + + return ret; +} + +/** + * hisi_qm_wait_task_finish() - Wait until the task is finished + * when removing the driver. + * @qm: The qm needed to wait for the task to finish. + * @qm_list: The list of all available devices. + */ +void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list) +{ + while (qm_frozen(qm) || + ((qm->fun_type == QM_HW_PF) && + qm_try_frozen_vfs(qm->pdev, qm_list))) { + msleep(WAIT_PERIOD); + } + + udelay(REMOVE_WAIT_DELAY); +} +EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish); + +/** * hisi_qm_get_free_qp_num() - Get free number of qp in qm. * @qm: The qm which want to get free qp. * @@ -2282,7 +2359,7 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) } while (0) idr_init(&qm->qp_idr); - qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_Q_DEPTH) + + qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) + QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) + QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); @@ -2292,7 +2369,7 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) if (!qm->qdma.va) return -ENOMEM; - QM_INIT_BUF(qm, eqe, QM_Q_DEPTH); + QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH); QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH); QM_INIT_BUF(qm, sqc, qm->qp_num); QM_INIT_BUF(qm, cqc, qm->qp_num); @@ -2338,6 +2415,7 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) mutex_init(&qm->mailbox_lock); init_rwsem(&qm->qps_lock); qm->qp_in_used = 0; + qm->is_frozen = false; } /** @@ -2462,7 +2540,7 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) eqc->base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma)); if (qm->ver == QM_HW_V1) eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE); - eqc->dw6 = cpu_to_le32((QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); + eqc->dw6 = cpu_to_le32((QM_EQ_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0); dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE); kfree(eqc); @@ -2633,18 +2711,20 @@ static void qm_clear_queues(struct hisi_qm *qm) /** * hisi_qm_stop() - Stop a qm. * @qm: The qm which will be stopped. + * @r: The reason to stop qm. * * This function stops qm and its qps, then qm can not accept request. * Related resources are not released at this state, we can use hisi_qm_start * to let qm start again. */ -int hisi_qm_stop(struct hisi_qm *qm) +int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) { struct device *dev = &qm->pdev->dev; int ret = 0; down_write(&qm->qps_lock); + qm->status.stop_reason = r; if (!qm_avail_state(qm, QM_STOP)) { ret = -EPERM; goto err_unlock; @@ -3081,11 +3161,12 @@ EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable); /** * hisi_qm_sriov_disable - disable virtual functions - * @pdev: the PCI device + * @pdev: the PCI device. + * @is_frozen: true when all the VFs are frozen. * - * Return failure if there are VFs assigned already. + * Return failure if there are VFs assigned already or VF is in used. */ -int hisi_qm_sriov_disable(struct pci_dev *pdev) +int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) { struct hisi_qm *qm = pci_get_drvdata(pdev); @@ -3094,7 +3175,12 @@ int hisi_qm_sriov_disable(struct pci_dev *pdev) return -EPERM; } - /* remove in hpre_pci_driver will be called to free VF resources */ + /* While VF is in used, SRIOV cannot be disabled. */ + if (!is_frozen && qm_try_frozen_vfs(pdev, qm->qm_list)) { + pci_err(pdev, "Task is using its VF!\n"); + return -EBUSY; + } + pci_disable_sriov(pdev); return qm_clear_vft_config(qm); } @@ -3110,7 +3196,7 @@ EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable); int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs) { if (num_vfs == 0) - return hisi_qm_sriov_disable(pdev); + return hisi_qm_sriov_disable(pdev, 0); else return hisi_qm_sriov_enable(pdev, num_vfs); } @@ -3290,10 +3376,10 @@ static int qm_set_msi(struct hisi_qm *qm, bool set) return 0; } -static int qm_vf_reset_prepare(struct hisi_qm *qm) +static int qm_vf_reset_prepare(struct hisi_qm *qm, + enum qm_stop_reason stop_reason) { struct hisi_qm_list *qm_list = qm->qm_list; - int stop_reason = qm->status.stop_reason; struct pci_dev *pdev = qm->pdev; struct pci_dev *virtfn; struct hisi_qm *vf_qm; @@ -3306,8 +3392,10 @@ static int qm_vf_reset_prepare(struct hisi_qm *qm) continue; if (pci_physfn(virtfn) == pdev) { - vf_qm->status.stop_reason = stop_reason; - ret = hisi_qm_stop(vf_qm); + /* save VFs PCIE BAR configuration */ + pci_save_state(virtfn); + + ret = hisi_qm_stop(vf_qm, stop_reason); if (ret) goto stop_fail; } @@ -3346,15 +3434,14 @@ static int qm_controller_reset_prepare(struct hisi_qm *qm) } if (qm->vfs_num) { - ret = qm_vf_reset_prepare(qm); + ret = qm_vf_reset_prepare(qm, QM_SOFT_RESET); if (ret) { pci_err(pdev, "Fails to stop VFs!\n"); return ret; } } - qm->status.stop_reason = QM_SOFT_RESET; - ret = hisi_qm_stop(qm); + ret = hisi_qm_stop(qm, QM_SOFT_RESET); if (ret) { pci_err(pdev, "Fails to stop QM!\n"); return ret; @@ -3471,6 +3558,9 @@ static int qm_vf_reset_done(struct hisi_qm *qm) continue; if (pci_physfn(virtfn) == pdev) { + /* enable VFs PCIE BAR configuration */ + pci_restore_state(virtfn); + ret = qm_restart(vf_qm); if (ret) goto restart_fail; @@ -3695,7 +3785,7 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) } if (qm->vfs_num) { - ret = qm_vf_reset_prepare(qm); + ret = qm_vf_reset_prepare(qm, QM_FLR); if (ret) { pci_err(pdev, "Failed to prepare reset, ret = %d.\n", ret); @@ -3703,7 +3793,7 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) } } - ret = hisi_qm_stop(qm); + ret = hisi_qm_stop(qm, QM_FLR); if (ret) { pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret); return; @@ -3821,6 +3911,23 @@ err_aeq_irq: return ret; } +/** + * hisi_qm_dev_shutdown() - Shutdown device. + * @pdev: The device will be shutdown. + * + * This function will stop qm when OS shutdown or rebooting. + */ +void hisi_qm_dev_shutdown(struct pci_dev *pdev) +{ + struct hisi_qm *qm = pci_get_drvdata(pdev); + int ret; + + ret = hisi_qm_stop(qm, QM_NORMAL); + if (ret) + dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n"); +} +EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown); + static void hisi_qm_controller_reset(struct work_struct *rst_work) { struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work); @@ -3834,6 +3941,58 @@ static void hisi_qm_controller_reset(struct work_struct *rst_work) } /** + * hisi_qm_alg_register() - Register alg to crypto and add qm to qm_list. + * @qm: The qm needs add. + * @qm_list: The qm list. + * + * This function adds qm to qm list, and will register algorithm to + * crypto when the qm list is empty. + */ +int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list) +{ + int flag = 0; + int ret = 0; + + mutex_lock(&qm_list->lock); + if (list_empty(&qm_list->list)) + flag = 1; + list_add_tail(&qm->list, &qm_list->list); + mutex_unlock(&qm_list->lock); + + if (flag) { + ret = qm_list->register_to_crypto(); + if (ret) { + mutex_lock(&qm_list->lock); + list_del(&qm->list); + mutex_unlock(&qm_list->lock); + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(hisi_qm_alg_register); + +/** + * hisi_qm_alg_unregister() - Unregister alg from crypto and delete qm from + * qm list. + * @qm: The qm needs delete. + * @qm_list: The qm list. + * + * This function deletes qm from qm list, and will unregister algorithm + * from crypto when the qm list is empty. + */ +void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) +{ + mutex_lock(&qm_list->lock); + list_del(&qm->list); + mutex_unlock(&qm_list->lock); + + if (list_empty(&qm_list->list)) + qm_list->unregister_from_crypto(); +} +EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister); + +/** * hisi_qm_init() - Initialize configures about qm. * @qm: The qm needing init. * diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index 6c1d3c7d64ee..0420f4ce7197 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -79,7 +79,7 @@ #define QM_BASE_CE QM_ECC_1BIT #define QM_Q_DEPTH 1024 - +#define QM_MIN_QNUM 2 #define HISI_ACC_SGL_SGE_NR_MAX 255 /* page number for queue file region */ @@ -193,6 +193,8 @@ struct hisi_qm_err_ini { struct hisi_qm_list { struct mutex lock; struct list_head list; + int (*register_to_crypto)(void); + void (*unregister_from_crypto)(void); }; struct hisi_qm { @@ -243,6 +245,7 @@ struct hisi_qm { const char *algs; bool use_sva; + bool is_frozen; resource_size_t phys_base; resource_size_t phys_size; struct uacce_device *uacce; @@ -306,7 +309,7 @@ static inline int q_num_set(const char *val, const struct kernel_param *kp, } ret = kstrtou32(val, 10, &n); - if (ret || !n || n > q_num) + if (ret || n < QM_MIN_QNUM || n > q_num) return -EINVAL; return param_set_int(val, kp); @@ -336,26 +339,10 @@ static inline void hisi_qm_init_list(struct hisi_qm_list *qm_list) mutex_init(&qm_list->lock); } -static inline void hisi_qm_add_to_list(struct hisi_qm *qm, - struct hisi_qm_list *qm_list) -{ - mutex_lock(&qm_list->lock); - list_add_tail(&qm->list, &qm_list->list); - mutex_unlock(&qm_list->lock); -} - -static inline void hisi_qm_del_from_list(struct hisi_qm *qm, - struct hisi_qm_list *qm_list) -{ - mutex_lock(&qm_list->lock); - list_del(&qm->list); - mutex_unlock(&qm_list->lock); -} - int hisi_qm_init(struct hisi_qm *qm); void hisi_qm_uninit(struct hisi_qm *qm); int hisi_qm_start(struct hisi_qm *qm); -int hisi_qm_stop(struct hisi_qm *qm); +int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r); struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type); int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg); int hisi_qm_stop_qp(struct hisi_qp *qp); @@ -367,7 +354,7 @@ int hisi_qm_debug_init(struct hisi_qm *qm); enum qm_hw_ver hisi_qm_get_hw_version(struct pci_dev *pdev); void hisi_qm_debug_regs_clear(struct hisi_qm *qm); int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs); -int hisi_qm_sriov_disable(struct pci_dev *pdev); +int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen); int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs); void hisi_qm_dev_err_init(struct hisi_qm *qm); void hisi_qm_dev_err_uninit(struct hisi_qm *qm); @@ -390,4 +377,8 @@ void hisi_acc_free_sgl_pool(struct device *dev, int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, u8 alg_type, int node, struct hisi_qp **qps); void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num); +void hisi_qm_dev_shutdown(struct pci_dev *pdev); +void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list); +int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list); +void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list); #endif diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 497969ae8b23..bb493423668c 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -66,8 +66,6 @@ #define SEC_SQE_AEAD_FLAG 3 #define SEC_SQE_DONE 0x1 -static atomic_t sec_active_devs; - /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */ static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req) { @@ -342,11 +340,14 @@ static int sec_alg_resource_alloc(struct sec_ctx *ctx, ret = sec_alloc_pbuf_resource(dev, res); if (ret) { dev_err(dev, "fail to alloc pbuf dma resource!\n"); - goto alloc_fail; + goto alloc_pbuf_fail; } } return 0; +alloc_pbuf_fail: + if (ctx->alg_type == SEC_AEAD) + sec_free_mac_resource(dev, qp_ctx->res); alloc_fail: sec_free_civ_resource(dev, res); @@ -457,8 +458,10 @@ static int sec_ctx_base_init(struct sec_ctx *ctx) ctx->fake_req_limit = QM_Q_DEPTH >> 1; ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx), GFP_KERNEL); - if (!ctx->qp_ctx) - return -ENOMEM; + if (!ctx->qp_ctx) { + ret = -ENOMEM; + goto err_destroy_qps; + } for (i = 0; i < sec->ctx_q_num; i++) { ret = sec_create_qp_ctx(&sec->qm, ctx, i, 0); @@ -467,12 +470,15 @@ static int sec_ctx_base_init(struct sec_ctx *ctx) } return 0; + err_sec_release_qp_ctx: for (i = i - 1; i >= 0; i--) sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); - sec_destroy_qps(ctx->qps, sec->ctx_q_num); kfree(ctx->qp_ctx); +err_destroy_qps: + sec_destroy_qps(ctx->qps, sec->ctx_q_num); + return ret; } @@ -1633,33 +1639,24 @@ static struct aead_alg sec_aeads[] = { int sec_register_to_crypto(void) { - int ret = 0; + int ret; /* To avoid repeat register */ - if (atomic_add_return(1, &sec_active_devs) == 1) { - ret = crypto_register_skciphers(sec_skciphers, - ARRAY_SIZE(sec_skciphers)); - if (ret) - return ret; - - ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); - if (ret) - goto reg_aead_fail; - } - - return ret; - -reg_aead_fail: - crypto_unregister_skciphers(sec_skciphers, ARRAY_SIZE(sec_skciphers)); + ret = crypto_register_skciphers(sec_skciphers, + ARRAY_SIZE(sec_skciphers)); + if (ret) + return ret; + ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); + if (ret) + crypto_unregister_skciphers(sec_skciphers, + ARRAY_SIZE(sec_skciphers)); return ret; } void sec_unregister_from_crypto(void) { - if (atomic_sub_return(1, &sec_active_devs) == 0) { - crypto_unregister_skciphers(sec_skciphers, - ARRAY_SIZE(sec_skciphers)); - crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); - } + crypto_unregister_skciphers(sec_skciphers, + ARRAY_SIZE(sec_skciphers)); + crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); } diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 2297425486cb..548896394c4b 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -99,7 +99,11 @@ struct sec_dfx_item { static const char sec_name[] = "hisi_sec2"; static struct dentry *sec_debugfs_root; -static struct hisi_qm_list sec_devices; + +static struct hisi_qm_list sec_devices = { + .register_to_crypto = sec_register_to_crypto, + .unregister_from_crypto = sec_unregister_from_crypto, +}; static const struct sec_hw_error sec_hw_errors[] = { {.int_msk = BIT(0), .msg = "sec_axi_rresp_err_rint"}, @@ -165,7 +169,7 @@ static const struct kernel_param_ops sec_pf_q_num_ops = { static u32 pf_q_num = SEC_PF_DEF_Q_NUM; module_param_cb(pf_q_num, &sec_pf_q_num_ops, &pf_q_num, 0444); -MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 0-4096, v2 0-1024)"); +MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)"); static int sec_ctx_q_num_set(const char *val, const struct kernel_param *kp) { @@ -879,29 +883,26 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) pci_warn(pdev, "Failed to init debugfs!\n"); - hisi_qm_add_to_list(qm, &sec_devices); - - ret = sec_register_to_crypto(); + ret = hisi_qm_alg_register(qm, &sec_devices); if (ret < 0) { pr_err("Failed to register driver to crypto.\n"); - goto err_remove_from_list; + goto err_qm_stop; } if (qm->fun_type == QM_HW_PF && vfs_num) { ret = hisi_qm_sriov_enable(pdev, vfs_num); if (ret < 0) - goto err_crypto_unregister; + goto err_alg_unregister; } return 0; -err_crypto_unregister: - sec_unregister_from_crypto(); +err_alg_unregister: + hisi_qm_alg_unregister(qm, &sec_devices); -err_remove_from_list: - hisi_qm_del_from_list(qm, &sec_devices); +err_qm_stop: sec_debugfs_exit(qm); - hisi_qm_stop(qm); + hisi_qm_stop(qm, QM_NORMAL); err_probe_uninit: sec_probe_uninit(qm); @@ -914,19 +915,16 @@ err_qm_uninit: static void sec_remove(struct pci_dev *pdev) { - struct sec_dev *sec = pci_get_drvdata(pdev); - struct hisi_qm *qm = &sec->qm; - - sec_unregister_from_crypto(); - - hisi_qm_del_from_list(qm, &sec_devices); + struct hisi_qm *qm = pci_get_drvdata(pdev); + hisi_qm_wait_task_finish(qm, &sec_devices); + hisi_qm_alg_unregister(qm, &sec_devices); if (qm->fun_type == QM_HW_PF && qm->vfs_num) - hisi_qm_sriov_disable(pdev); + hisi_qm_sriov_disable(pdev, qm->is_frozen); sec_debugfs_exit(qm); - (void)hisi_qm_stop(qm); + (void)hisi_qm_stop(qm, QM_NORMAL); if (qm->fun_type == QM_HW_PF) sec_debug_regs_clear(qm); @@ -950,6 +948,7 @@ static struct pci_driver sec_pci_driver = { .remove = sec_remove, .err_handler = &sec_err_handler, .sriov_configure = hisi_qm_sriov_configure, + .shutdown = hisi_qm_dev_shutdown, }; static void sec_register_debugfs(void) diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index 4484be13812b..92397f993e23 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -9,20 +9,6 @@ #include <linux/list.h> #include "../qm.h" -/* hisi_zip_sqe dw3 */ -#define HZIP_BD_STATUS_M GENMASK(7, 0) -/* hisi_zip_sqe dw7 */ -#define HZIP_IN_SGE_DATA_OFFSET_M GENMASK(23, 0) -/* hisi_zip_sqe dw8 */ -#define HZIP_OUT_SGE_DATA_OFFSET_M GENMASK(23, 0) -/* hisi_zip_sqe dw9 */ -#define HZIP_REQ_TYPE_M GENMASK(7, 0) -#define HZIP_ALG_TYPE_ZLIB 0x02 -#define HZIP_ALG_TYPE_GZIP 0x03 -#define HZIP_BUF_TYPE_M GENMASK(11, 8) -#define HZIP_PBUFFER 0x0 -#define HZIP_SGL 0x1 - enum hisi_zip_error_type { /* negative compression */ HZIP_NC_ERR = 0x0d, @@ -39,7 +25,6 @@ struct hisi_zip_ctrl; struct hisi_zip { struct hisi_qm qm; - struct list_head list; struct hisi_zip_ctrl *ctrl; struct hisi_zip_dfx dfx; }; diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index 01fd6a78111d..08b4660b014c 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -6,6 +6,20 @@ #include <linux/scatterlist.h> #include "zip.h" +/* hisi_zip_sqe dw3 */ +#define HZIP_BD_STATUS_M GENMASK(7, 0) +/* hisi_zip_sqe dw7 */ +#define HZIP_IN_SGE_DATA_OFFSET_M GENMASK(23, 0) +/* hisi_zip_sqe dw8 */ +#define HZIP_OUT_SGE_DATA_OFFSET_M GENMASK(23, 0) +/* hisi_zip_sqe dw9 */ +#define HZIP_REQ_TYPE_M GENMASK(7, 0) +#define HZIP_ALG_TYPE_ZLIB 0x02 +#define HZIP_ALG_TYPE_GZIP 0x03 +#define HZIP_BUF_TYPE_M GENMASK(11, 8) +#define HZIP_PBUFFER 0x0 +#define HZIP_SGL 0x1 + #define HZIP_ZLIB_HEAD_SIZE 2 #define HZIP_GZIP_HEAD_SIZE 10 @@ -16,22 +30,29 @@ #define GZIP_HEAD_FLG_SHIFT 3 #define GZIP_HEAD_FEXTRA_SHIFT 10 -#define GZIP_HEAD_FEXTRA_XLEN 2 +#define GZIP_HEAD_FEXTRA_XLEN 2UL #define GZIP_HEAD_FHCRC_SIZE 2 -#define HZIP_CTX_Q_NUM 2 #define HZIP_GZIP_HEAD_BUF 256 #define HZIP_ALG_PRIORITY 300 #define HZIP_SGL_SGE_NR 10 static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c}; -static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = {0x1f, 0x8b, 0x08, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x03}; +static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = { + 0x1f, 0x8b, 0x08, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x03 +}; + enum hisi_zip_alg_type { HZIP_ALG_TYPE_COMP = 0, HZIP_ALG_TYPE_DECOMP = 1, }; +enum { + HZIP_QPC_COMP, + HZIP_QPC_DECOMP, + HZIP_CTX_Q_NUM +}; + #define COMP_NAME_TO_TYPE(alg_name) \ (!strcmp((alg_name), "zlib-deflate") ? HZIP_ALG_TYPE_ZLIB : \ !strcmp((alg_name), "gzip") ? HZIP_ALG_TYPE_GZIP : 0) \ @@ -46,13 +67,13 @@ enum hisi_zip_alg_type { struct hisi_zip_req { struct acomp_req *req; - int sskip; - int dskip; + u32 sskip; + u32 dskip; struct hisi_acc_hw_sgl *hw_src; struct hisi_acc_hw_sgl *hw_dst; dma_addr_t dma_src; dma_addr_t dma_dst; - int req_id; + u16 req_id; }; struct hisi_zip_req_q { @@ -71,8 +92,6 @@ struct hisi_zip_qp_ctx { }; struct hisi_zip_ctx { -#define QPC_COMP 0 -#define QPC_DECOMP 1 struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM]; }; @@ -116,7 +135,7 @@ static void hisi_zip_config_tag(struct hisi_zip_sqe *sqe, u32 tag) static void hisi_zip_fill_sqe(struct hisi_zip_sqe *sqe, u8 req_type, dma_addr_t s_addr, dma_addr_t d_addr, u32 slen, - u32 dlen, int sskip, int dskip) + u32 dlen, u32 sskip, u32 dskip) { memset(sqe, 0, sizeof(struct hisi_zip_sqe)); @@ -143,7 +162,7 @@ static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *ctx, ret = hisi_qm_start_qp(qp, 0); if (ret < 0) { - dev_err(dev, "start qp failed!\n"); + dev_err(dev, "failed to start qp (%d)!\n", ret); return ret; } @@ -166,7 +185,7 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node); if (ret) { - pr_err("Can not create zip qps!\n"); + pr_err("failed to create zip qps (%d)!\n", ret); return -ENODEV; } @@ -264,11 +283,11 @@ static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) return 0; err_free_loop1: - kfree(ctx->qp_ctx[QPC_DECOMP].req_q.req_bitmap); + kfree(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap); err_free_loop0: - kfree(ctx->qp_ctx[QPC_COMP].req_q.q); + kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.q); err_free_bitmap: - kfree(ctx->qp_ctx[QPC_COMP].req_q.req_bitmap); + kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap); return ret; } @@ -303,8 +322,8 @@ static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx) return 0; err_free_sgl_pool0: - hisi_acc_free_sgl_pool(&ctx->qp_ctx[QPC_COMP].qp->qm->pdev->dev, - ctx->qp_ctx[QPC_COMP].sgl_pool); + hisi_acc_free_sgl_pool(&ctx->qp_ctx[HZIP_QPC_COMP].qp->qm->pdev->dev, + ctx->qp_ctx[HZIP_QPC_COMP].sgl_pool); return -ENOMEM; } @@ -342,7 +361,6 @@ static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) atomic64_inc(&dfx->recv_cnt); status = sqe->dw3 & HZIP_BD_STATUS_M; - if (status != 0 && status != HZIP_NC_ERR) { dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n", (qp->alg_type == 0) ? "" : "de", qp->qp_id, status, @@ -377,19 +395,28 @@ static int hisi_zip_acomp_init(struct crypto_acomp *tfm) { const char *alg_name = crypto_tfm_alg_name(&tfm->base); struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); + struct device *dev; int ret; ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name), tfm->base.node); - if (ret) + if (ret) { + pr_err("failed to init ctx (%d)!\n", ret); return ret; + } + + dev = &ctx->qp_ctx[0].qp->qm->pdev->dev; ret = hisi_zip_create_req_q(ctx); - if (ret) + if (ret) { + dev_err(dev, "failed to create request queue (%d)!\n", ret); goto err_ctx_exit; + } ret = hisi_zip_create_sgl_pool(ctx); - if (ret) + if (ret) { + dev_err(dev, "failed to create sgl pool (%d)!\n", ret); goto err_release_req_q; + } hisi_zip_set_acomp_cb(ctx, hisi_zip_acomp_cb); @@ -419,13 +446,15 @@ static int add_comp_head(struct scatterlist *dst, u8 req_type) int ret; ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size); - if (ret != head_size) + if (ret != head_size) { + pr_err("the head size of buffer is wrong (%d)!\n", ret); return -ENOMEM; + } return head_size; } -static size_t get_gzip_head_size(struct scatterlist *sgl) +static size_t __maybe_unused get_gzip_head_size(struct scatterlist *sgl) { char buf[HZIP_GZIP_HEAD_BUF]; @@ -434,13 +463,20 @@ static size_t get_gzip_head_size(struct scatterlist *sgl) return __get_gzip_head_size(buf); } -static size_t get_comp_head_size(struct scatterlist *src, u8 req_type) +static int get_comp_head_size(struct acomp_req *acomp_req, u8 req_type) { + if (!acomp_req->src || !acomp_req->slen) + return -EINVAL; + + if ((req_type == HZIP_ALG_TYPE_GZIP) && + (acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT)) + return -EINVAL; + switch (req_type) { case HZIP_ALG_TYPE_ZLIB: return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB); case HZIP_ALG_TYPE_GZIP: - return get_gzip_head_size(src); + return TO_HEAD_SIZE(HZIP_ALG_TYPE_GZIP); default: pr_err("request type does not support!\n"); return -EINVAL; @@ -462,7 +498,7 @@ static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req, if (req_id >= req_q->size) { write_unlock(&req_q->req_lock); dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n"); - return ERR_PTR(-EBUSY); + return ERR_PTR(-EAGAIN); } set_bit(req_id, req_q->req_bitmap); @@ -492,8 +528,7 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; struct hisi_zip_sqe zip_sqe; - dma_addr_t input; - dma_addr_t output; + dma_addr_t input, output; int ret; if (!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen) @@ -501,8 +536,11 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, req->req_id << 1, &input); - if (IS_ERR(req->hw_src)) + if (IS_ERR(req->hw_src)) { + dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n", + PTR_ERR(req->hw_src)); return PTR_ERR(req->hw_src); + } req->dma_src = input; req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool, @@ -510,6 +548,8 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, &output); if (IS_ERR(req->hw_dst)) { ret = PTR_ERR(req->hw_dst); + dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", + ret); goto err_unmap_input; } req->dma_dst = output; @@ -524,6 +564,8 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, ret = hisi_qp_send(qp, &zip_sqe); if (ret < 0) { atomic64_inc(&dfx->send_busy_cnt); + ret = -EAGAIN; + dev_dbg_ratelimited(dev, "failed to send request!\n"); goto err_unmap_output; } @@ -539,23 +581,29 @@ err_unmap_input: static int hisi_zip_acompress(struct acomp_req *acomp_req) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); - struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_COMP]; + struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP]; + struct device *dev = &qp_ctx->qp->qm->pdev->dev; struct hisi_zip_req *req; int head_size; int ret; /* let's output compression head now */ head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type); - if (head_size < 0) - return -ENOMEM; + if (head_size < 0) { + dev_err_ratelimited(dev, "failed to add comp head (%d)!\n", + head_size); + return head_size; + } - req = hisi_zip_create_req(acomp_req, qp_ctx, (size_t)head_size, true); + req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, true); if (IS_ERR(req)) return PTR_ERR(req); ret = hisi_zip_do_work(req, qp_ctx); - if (ret != -EINPROGRESS) + if (ret != -EINPROGRESS) { + dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret); hisi_zip_remove_req(qp_ctx, req); + } return ret; } @@ -563,20 +611,28 @@ static int hisi_zip_acompress(struct acomp_req *acomp_req) static int hisi_zip_adecompress(struct acomp_req *acomp_req) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); - struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_DECOMP]; + struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP]; + struct device *dev = &qp_ctx->qp->qm->pdev->dev; struct hisi_zip_req *req; - size_t head_size; - int ret; + int head_size, ret; - head_size = get_comp_head_size(acomp_req->src, qp_ctx->qp->req_type); + head_size = get_comp_head_size(acomp_req, qp_ctx->qp->req_type); + if (head_size < 0) { + dev_err_ratelimited(dev, "failed to get comp head size (%d)!\n", + head_size); + return head_size; + } req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false); if (IS_ERR(req)) return PTR_ERR(req); ret = hisi_zip_do_work(req, qp_ctx); - if (ret != -EINPROGRESS) + if (ret != -EINPROGRESS) { + dev_info_ratelimited(dev, "failed to do decompress (%d)!\n", + ret); hisi_zip_remove_req(qp_ctx, req); + } return ret; } @@ -611,17 +667,17 @@ static struct acomp_alg hisi_zip_acomp_gzip = { int hisi_zip_register_to_crypto(void) { - int ret = 0; + int ret; ret = crypto_register_acomp(&hisi_zip_acomp_zlib); if (ret) { - pr_err("Zlib acomp algorithm registration failed\n"); + pr_err("failed to register to zlib (%d)!\n", ret); return ret; } ret = crypto_register_acomp(&hisi_zip_acomp_gzip); if (ret) { - pr_err("Gzip acomp algorithm registration failed\n"); + pr_err("failed to register to gzip (%d)!\n", ret); crypto_unregister_acomp(&hisi_zip_acomp_zlib); } diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index e2845b2c963d..4bd2c811abba 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -17,7 +17,6 @@ #define PCI_DEVICE_ID_ZIP_PF 0xa250 #define PCI_DEVICE_ID_ZIP_VF 0xa251 -#define HZIP_VF_NUM 63 #define HZIP_QUEUE_NUM_V1 4096 #define HZIP_QUEUE_NUM_V2 1024 @@ -30,18 +29,18 @@ #define DECOMP3_ENABLE BIT(5) #define DECOMP4_ENABLE BIT(6) #define DECOMP5_ENABLE BIT(7) -#define ALL_COMP_DECOMP_EN (COMP0_ENABLE | COMP1_ENABLE | \ +#define HZIP_ALL_COMP_DECOMP_EN (COMP0_ENABLE | COMP1_ENABLE | \ DECOMP0_ENABLE | DECOMP1_ENABLE | \ DECOMP2_ENABLE | DECOMP3_ENABLE | \ DECOMP4_ENABLE | DECOMP5_ENABLE) -#define DECOMP_CHECK_ENABLE BIT(16) +#define HZIP_DECOMP_CHECK_ENABLE BIT(16) #define HZIP_FSM_MAX_CNT 0x301008 #define HZIP_PORT_ARCA_CHE_0 0x301040 #define HZIP_PORT_ARCA_CHE_1 0x301044 #define HZIP_PORT_AWCA_CHE_0 0x301060 #define HZIP_PORT_AWCA_CHE_1 0x301064 -#define CACHE_ALL_EN 0xffffffff +#define HZIP_CACHE_ALL_EN 0xffffffff #define HZIP_BD_RUSER_32_63 0x301110 #define HZIP_SGL_RUSER_32_63 0x30111c @@ -83,7 +82,7 @@ #define HZIP_PF_DEF_Q_BASE 0 #define HZIP_SOFT_CTRL_CNT_CLR_CE 0x301000 -#define SOFT_CTRL_CNT_CLR_CE_BIT BIT(0) +#define HZIP_SOFT_CTRL_CNT_CLR_CE_BIT BIT(0) #define HZIP_SOFT_CTRL_ZIP_CONTROL 0x30100C #define HZIP_AXI_SHUTDOWN_ENABLE BIT(14) #define HZIP_WR_PORT BIT(11) @@ -92,9 +91,13 @@ #define HZIP_SQE_MASK_OFFSET 64 #define HZIP_SQE_MASK_LEN 48 +#define HZIP_CNT_CLR_CE_EN BIT(0) +#define HZIP_RO_CNT_CLR_CE_EN BIT(2) +#define HZIP_RD_CNT_CLR_CE_EN (HZIP_CNT_CLR_CE_EN | \ + HZIP_RO_CNT_CLR_CE_EN) + static const char hisi_zip_name[] = "hisi_zip"; static struct dentry *hzip_debugfs_root; -static struct hisi_qm_list zip_devices; struct hisi_zip_hw_error { u32 int_msk; @@ -106,6 +109,11 @@ struct zip_dfx_item { u32 offset; }; +static struct hisi_qm_list zip_devices = { + .register_to_crypto = hisi_zip_register_to_crypto, + .unregister_from_crypto = hisi_zip_unregister_from_crypto, +}; + static struct zip_dfx_item zip_dfx_files[] = { {"send_cnt", offsetof(struct hisi_zip_dfx, send_cnt)}, {"recv_cnt", offsetof(struct hisi_zip_dfx, recv_cnt)}, @@ -153,7 +161,6 @@ struct ctrl_debug_file { */ struct hisi_zip_ctrl { struct hisi_zip *hisi_zip; - struct dentry *debug_root; struct ctrl_debug_file files[HZIP_DEBUG_FILE_NUM]; }; @@ -216,7 +223,7 @@ static const struct kernel_param_ops pf_q_num_ops = { static u32 pf_q_num = HZIP_PF_DEF_Q_NUM; module_param_cb(pf_q_num, &pf_q_num_ops, &pf_q_num, 0444); -MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 1-4096, v2 1-1024)"); +MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)"); static const struct kernel_param_ops vfs_num_ops = { .set = vfs_num_set, @@ -256,15 +263,16 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) /* qm cache */ writel(AXI_M_CFG, base + QM_AXI_M_CFG); writel(AXI_M_CFG_ENABLE, base + QM_AXI_M_CFG_ENABLE); + /* disable FLR triggered by BME(bus master enable) */ writel(PEH_AXUSER_CFG, base + QM_PEH_AXUSER_CFG); writel(PEH_AXUSER_CFG_ENABLE, base + QM_PEH_AXUSER_CFG_ENABLE); /* cache */ - writel(CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_0); - writel(CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_1); - writel(CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_0); - writel(CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_1); + writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_0); + writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_1); + writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_0); + writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_1); /* user domain configurations */ writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63); @@ -280,10 +288,10 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) } /* let's open all compression/decompression cores */ - writel(DECOMP_CHECK_ENABLE | ALL_COMP_DECOMP_EN, + writel(HZIP_DECOMP_CHECK_ENABLE | HZIP_ALL_COMP_DECOMP_EN, base + HZIP_CLOCK_GATE_CTRL); - /* enable sqc writeback */ + /* enable sqc,cqc writeback */ writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE | CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) | FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL); @@ -309,7 +317,7 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) writel(0x1, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); writel(0x0, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB); writel(HZIP_CORE_INT_RAS_NFE_ENABLE, - qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); + qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); /* enable ZIP hw error interrupts */ writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG); @@ -356,7 +364,7 @@ static int current_qm_write(struct ctrl_debug_file *file, u32 val) if (val > qm->vfs_num) return -EINVAL; - /* Calculate curr_qm_qp_num and store */ + /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */ if (val == 0) { qm->debug.curr_qm_qp_num = qm->qp_num; } else { @@ -387,7 +395,7 @@ static u32 clear_enable_read(struct ctrl_debug_file *file) struct hisi_qm *qm = file_to_qm(file); return readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) & - SOFT_CTRL_CNT_CLR_CE_BIT; + HZIP_SOFT_CTRL_CNT_CLR_CE_BIT; } static int clear_enable_write(struct ctrl_debug_file *file, u32 val) @@ -399,14 +407,14 @@ static int clear_enable_write(struct ctrl_debug_file *file, u32 val) return -EINVAL; tmp = (readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) & - ~SOFT_CTRL_CNT_CLR_CE_BIT) | val; + ~HZIP_SOFT_CTRL_CNT_CLR_CE_BIT) | val; writel(tmp, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); return 0; } -static ssize_t ctrl_debug_read(struct file *filp, char __user *buf, - size_t count, loff_t *pos) +static ssize_t hisi_zip_ctrl_debug_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) { struct ctrl_debug_file *file = filp->private_data; char tbuf[HZIP_BUF_SIZE]; @@ -426,12 +434,13 @@ static ssize_t ctrl_debug_read(struct file *filp, char __user *buf, return -EINVAL; } spin_unlock_irq(&file->lock); - ret = sprintf(tbuf, "%u\n", val); + ret = scnprintf(tbuf, sizeof(tbuf), "%u\n", val); return simple_read_from_buffer(buf, count, pos, tbuf, ret); } -static ssize_t ctrl_debug_write(struct file *filp, const char __user *buf, - size_t count, loff_t *pos) +static ssize_t hisi_zip_ctrl_debug_write(struct file *filp, + const char __user *buf, + size_t count, loff_t *pos) { struct ctrl_debug_file *file = filp->private_data; char tbuf[HZIP_BUF_SIZE]; @@ -480,11 +489,10 @@ err_input: static const struct file_operations ctrl_debug_fops = { .owner = THIS_MODULE, .open = simple_open, - .read = ctrl_debug_read, - .write = ctrl_debug_write, + .read = hisi_zip_ctrl_debug_read, + .write = hisi_zip_ctrl_debug_write, }; - static int zip_debugfs_atomic64_set(void *data, u64 val) { if (val) @@ -505,10 +513,8 @@ static int zip_debugfs_atomic64_get(void *data, u64 *val) DEFINE_DEBUGFS_ATTRIBUTE(zip_atomic64_ops, zip_debugfs_atomic64_get, zip_debugfs_atomic64_set, "%llu\n"); -static int hisi_zip_core_debug_init(struct hisi_zip_ctrl *ctrl) +static int hisi_zip_core_debug_init(struct hisi_qm *qm) { - struct hisi_zip *hisi_zip = ctrl->hisi_zip; - struct hisi_qm *qm = &hisi_zip->qm; struct device *dev = &qm->pdev->dev; struct debugfs_regset32 *regset; struct dentry *tmp_d; @@ -517,9 +523,10 @@ static int hisi_zip_core_debug_init(struct hisi_zip_ctrl *ctrl) for (i = 0; i < HZIP_CORE_NUM; i++) { if (i < HZIP_COMP_CORE_NUM) - sprintf(buf, "comp_core%d", i); + scnprintf(buf, sizeof(buf), "comp_core%d", i); else - sprintf(buf, "decomp_core%d", i - HZIP_COMP_CORE_NUM); + scnprintf(buf, sizeof(buf), "decomp_core%d", + i - HZIP_COMP_CORE_NUM); regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); if (!regset) @@ -529,7 +536,7 @@ static int hisi_zip_core_debug_init(struct hisi_zip_ctrl *ctrl) regset->nregs = ARRAY_SIZE(hzip_dfx_regs); regset->base = qm->io_base + core_offsets[i]; - tmp_d = debugfs_create_dir(buf, ctrl->debug_root); + tmp_d = debugfs_create_dir(buf, qm->debug.debug_root); debugfs_create_regset32("regs", 0444, tmp_d, regset); } @@ -548,33 +555,32 @@ static void hisi_zip_dfx_debug_init(struct hisi_qm *qm) for (i = 0; i < ARRAY_SIZE(zip_dfx_files); i++) { data = (atomic64_t *)((uintptr_t)dfx + zip_dfx_files[i].offset); debugfs_create_file(zip_dfx_files[i].name, - 0644, - tmp_dir, - data, - &zip_atomic64_ops); + 0644, tmp_dir, data, + &zip_atomic64_ops); } } -static int hisi_zip_ctrl_debug_init(struct hisi_zip_ctrl *ctrl) +static int hisi_zip_ctrl_debug_init(struct hisi_qm *qm) { + struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm); int i; for (i = HZIP_CURRENT_QM; i < HZIP_DEBUG_FILE_NUM; i++) { - spin_lock_init(&ctrl->files[i].lock); - ctrl->files[i].ctrl = ctrl; - ctrl->files[i].index = i; + spin_lock_init(&zip->ctrl->files[i].lock); + zip->ctrl->files[i].ctrl = zip->ctrl; + zip->ctrl->files[i].index = i; debugfs_create_file(ctrl_debug_file_name[i], 0600, - ctrl->debug_root, ctrl->files + i, + qm->debug.debug_root, + zip->ctrl->files + i, &ctrl_debug_fops); } - return hisi_zip_core_debug_init(ctrl); + return hisi_zip_core_debug_init(qm); } -static int hisi_zip_debugfs_init(struct hisi_zip *hisi_zip) +static int hisi_zip_debugfs_init(struct hisi_qm *qm) { - struct hisi_qm *qm = &hisi_zip->qm; struct device *dev = &qm->pdev->dev; struct dentry *dev_d; int ret; @@ -589,8 +595,7 @@ static int hisi_zip_debugfs_init(struct hisi_zip *hisi_zip) goto failed_to_create; if (qm->fun_type == QM_HW_PF) { - hisi_zip->ctrl->debug_root = dev_d; - ret = hisi_zip_ctrl_debug_init(hisi_zip->ctrl); + ret = hisi_zip_ctrl_debug_init(qm); if (ret) goto failed_to_create; } @@ -604,25 +609,36 @@ failed_to_create: return ret; } -static void hisi_zip_debug_regs_clear(struct hisi_zip *hisi_zip) +/* hisi_zip_debug_regs_clear() - clear the zip debug regs */ +static void hisi_zip_debug_regs_clear(struct hisi_qm *qm) { - struct hisi_qm *qm = &hisi_zip->qm; + int i, j; + /* clear current_qm */ writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); + + /* enable register read_clear bit */ + writel(HZIP_RD_CNT_CLR_CE_EN, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); + for (i = 0; i < ARRAY_SIZE(core_offsets); i++) + for (j = 0; j < ARRAY_SIZE(hzip_dfx_regs); j++) + readl(qm->io_base + core_offsets[i] + + hzip_dfx_regs[j].offset); + + /* disable register read_clear bit */ writel(0x0, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); hisi_qm_debug_regs_clear(qm); } -static void hisi_zip_debugfs_exit(struct hisi_zip *hisi_zip) +static void hisi_zip_debugfs_exit(struct hisi_qm *qm) { - struct hisi_qm *qm = &hisi_zip->qm; - debugfs_remove_recursive(qm->debug.debug_root); - if (qm->fun_type == QM_HW_PF) - hisi_zip_debug_regs_clear(hisi_zip); + if (qm->fun_type == QM_HW_PF) { + hisi_zip_debug_regs_clear(qm); + qm->debug.curr_qm_qp_num = 0; + } } static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts) @@ -634,7 +650,7 @@ static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts) while (err->msg) { if (err->int_msk & err_sts) { dev_err(dev, "%s [error status=0x%x] found\n", - err->msg, err->int_msk); + err->msg, err->int_msk); if (err->int_msk & HZIP_CORE_INT_STATUS_M_ECC) { err_val = readl(qm->io_base + @@ -642,9 +658,6 @@ static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts) dev_err(dev, "hisi-zip multi ecc sram num=0x%x\n", ((err_val >> HZIP_SRAM_ECC_ERR_NUM_SHIFT) & 0xFF)); - dev_err(dev, "hisi-zip multi ecc sram addr=0x%x\n", - (err_val >> - HZIP_SRAM_ECC_ERR_ADDR_SHIFT)); } } err++; @@ -729,7 +742,7 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) hisi_zip_set_user_domain_and_cache(qm); hisi_qm_dev_err_init(qm); - hisi_zip_debug_regs_clear(hisi_zip); + hisi_zip_debug_regs_clear(qm); return 0; } @@ -747,6 +760,7 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) if (qm->fun_type == QM_HW_PF) { qm->qp_base = HZIP_PF_DEF_Q_BASE; qm->qp_num = pf_q_num; + qm->debug.curr_qm_qp_num = pf_q_num; qm->qm_list = &zip_devices; } else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) { /* @@ -803,32 +817,44 @@ static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = hisi_qm_start(qm); if (ret) - goto err_qm_uninit; + goto err_dev_err_uninit; - ret = hisi_zip_debugfs_init(hisi_zip); + ret = hisi_zip_debugfs_init(qm); if (ret) - dev_err(&pdev->dev, "Failed to init debugfs (%d)!\n", ret); + pci_err(pdev, "failed to init debugfs (%d)!\n", ret); - hisi_qm_add_to_list(qm, &zip_devices); + ret = hisi_qm_alg_register(qm, &zip_devices); + if (ret < 0) { + pci_err(pdev, "failed to register driver to crypto!\n"); + goto err_qm_stop; + } if (qm->uacce) { ret = uacce_register(qm->uacce); - if (ret) - goto err_qm_uninit; + if (ret) { + pci_err(pdev, "failed to register uacce (%d)!\n", ret); + goto err_qm_alg_unregister; + } } if (qm->fun_type == QM_HW_PF && vfs_num > 0) { ret = hisi_qm_sriov_enable(pdev, vfs_num); if (ret < 0) - goto err_remove_from_list; + goto err_qm_alg_unregister; } return 0; -err_remove_from_list: - hisi_qm_del_from_list(qm, &zip_devices); - hisi_zip_debugfs_exit(hisi_zip); - hisi_qm_stop(qm); +err_qm_alg_unregister: + hisi_qm_alg_unregister(qm, &zip_devices); + +err_qm_stop: + hisi_zip_debugfs_exit(qm); + hisi_qm_stop(qm, QM_NORMAL); + +err_dev_err_uninit: + hisi_qm_dev_err_uninit(qm); + err_qm_uninit: hisi_qm_uninit(qm); @@ -837,18 +863,18 @@ err_qm_uninit: static void hisi_zip_remove(struct pci_dev *pdev) { - struct hisi_zip *hisi_zip = pci_get_drvdata(pdev); - struct hisi_qm *qm = &hisi_zip->qm; + struct hisi_qm *qm = pci_get_drvdata(pdev); - if (qm->fun_type == QM_HW_PF && qm->vfs_num) - hisi_qm_sriov_disable(pdev); + hisi_qm_wait_task_finish(qm, &zip_devices); + hisi_qm_alg_unregister(qm, &zip_devices); - hisi_zip_debugfs_exit(hisi_zip); - hisi_qm_stop(qm); + if (qm->fun_type == QM_HW_PF && qm->vfs_num) + hisi_qm_sriov_disable(pdev, qm->is_frozen); + hisi_zip_debugfs_exit(qm); + hisi_qm_stop(qm, QM_NORMAL); hisi_qm_dev_err_uninit(qm); hisi_qm_uninit(qm); - hisi_qm_del_from_list(qm, &zip_devices); } static const struct pci_error_handlers hisi_zip_err_handler = { @@ -866,6 +892,7 @@ static struct pci_driver hisi_zip_pci_driver = { .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ? hisi_qm_sriov_configure : NULL, .err_handler = &hisi_zip_err_handler, + .shutdown = hisi_qm_dev_shutdown, }; static void hisi_zip_register_debugfs(void) @@ -890,29 +917,15 @@ static int __init hisi_zip_init(void) ret = pci_register_driver(&hisi_zip_pci_driver); if (ret < 0) { + hisi_zip_unregister_debugfs(); pr_err("Failed to register pci driver.\n"); - goto err_pci; } - ret = hisi_zip_register_to_crypto(); - if (ret < 0) { - pr_err("Failed to register driver to crypto.\n"); - goto err_crypto; - } - - return 0; - -err_crypto: - pci_unregister_driver(&hisi_zip_pci_driver); -err_pci: - hisi_zip_unregister_debugfs(); - return ret; } static void __exit hisi_zip_exit(void) { - hisi_zip_unregister_from_crypto(); pci_unregister_driver(&hisi_zip_pci_driver); hisi_zip_unregister_debugfs(); } |