From patchwork Mon Jan 1 12:06:37 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Gilad Ben-Yossef X-Patchwork-Id: 123041 Delivered-To: patch@linaro.org Received: by 10.140.22.227 with SMTP id 90csp7796235qgn; Mon, 1 Jan 2018 04:09:07 -0800 (PST) X-Google-Smtp-Source: ACJfBotwJnk+L1s01qoeGzqD7u4nXD4pHVzDSRocxWlnRf+jUQh8qHhLfK1lfUCgHI2D+sfT/Y/t X-Received: by 10.99.111.203 with SMTP id k194mr33541830pgc.21.1514808547262; Mon, 01 Jan 2018 04:09:07 -0800 (PST) ARC-Seal: i=1; a=rsa-sha256; t=1514808547; cv=none; d=google.com; s=arc-20160816; b=SmvbHzsaGyDKYqUeOMB5pcRaHjya1dom3wlAlffERimibMnW2TuwodjEgdFbdmW8A3 sZms8xuHeAQ31a3q8AcW+aJbxha6WA6PGDKC0voSKMqtvMa0ncJWZScrN4EDLbKoeBdo QP66MJTg2mG3uJQQwWB3++DhGqACc/ag6SOI6MYqmQNJLbfdMiqnc3mxvjFdnZnMhz+o QtZPhUPHAVhDBwAo/0vJZ2CVCPw3hJUAkr/P73wCgZ4oq3PDg8GezNMiI3ZqyuAS/D7U +di9F2Q0UMfr+COTQJpFLqdOKXZQUNw4OVZvYCOCKJjw4YGGXJawcW+GERvIEFg4/l0f Px6Q== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=list-id:precedence:sender:references:in-reply-to:message-id:date :subject:cc:to:from:arc-authentication-results; bh=G3DjZ/j5hAS2Xi70U6mhqf1VgyFK8Jt5Z6Ma0EEAHuQ=; b=MF0lZE/W5kh00T3ygZdbNUqLD+rgu++d7Mhwsg1/zxY+f4WrHZuVrBucxT8iq7WaiP +f5Kiea+uiwZXbco94IPNON7B33A7LcLuw3GY5PmCBTwG61Oog9dGwJpL95M2C0PrVeo Yh9E6ywMHmksDEAY5fDazxHJUmml2WSWh3fOcQgKMENLvq2EdrT/5DmDONXMpSVSrJGJ G8HOEASmv680gVQ+RekPbrAp0Lsoy763WNHbaTv75KGAi3ElAyeZ0J53xbZDtLMCcO96 Nf5PG2tvt4MJ6unYrsMnomBQa+OxUoyshj/KmJZ2DsdnO5vvADfIGvSNhcXK0SBekHsy kdfA== ARC-Authentication-Results: i=1; mx.google.com; spf=pass (google.com: best guess record for domain of linux-crypto-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) smtp.mailfrom=linux-crypto-owner@vger.kernel.org Return-Path: Received: from vger.kernel.org (vger.kernel.org. [209.132.180.67]) by mx.google.com with ESMTP id f6si10951230pgr.235.2018.01.01.04.09.07; Mon, 01 Jan 2018 04:09:07 -0800 (PST) Received-SPF: pass (google.com: best guess record for domain of linux-crypto-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) client-ip=209.132.180.67; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of linux-crypto-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) smtp.mailfrom=linux-crypto-owner@vger.kernel.org Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1751109AbeAAMJF (ORCPT + 1 other); Mon, 1 Jan 2018 07:09:05 -0500 Received: from foss.arm.com ([217.140.101.70]:59126 "EHLO foss.arm.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1751088AbeAAMJE (ORCPT ); Mon, 1 Jan 2018 07:09:04 -0500 Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.72.51.249]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id E85EB80D; Mon, 1 Jan 2018 04:09:03 -0800 (PST) Received: from sugar.kfn.arm.com (unknown [10.45.48.158]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id E32793F41F; Mon, 1 Jan 2018 04:09:00 -0800 (PST) From: Gilad Ben-Yossef To: Greg Kroah-Hartman Cc: Ofir Drang , linux-kernel@vger.kernel.org, linux-crypto@vger.kernel.org, driverdev-devel@linuxdriverproject.org, devel@driverdev.osuosl.org Subject: [PATCH 10/26] staging: ccree: add backlog processing Date: Mon, 1 Jan 2018 12:06:37 +0000 Message-Id: <1514808421-21993-11-git-send-email-gilad@benyossef.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1514808421-21993-1-git-send-email-gilad@benyossef.com> References: <1514808421-21993-1-git-send-email-gilad@benyossef.com> Sender: linux-crypto-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-crypto@vger.kernel.org Crypto API tfm providers are required to provide a backlog service, if so indicated, that queues up requests in the case of the provider being busy and processing them later. The ccree driver did not provide this facility. Add it now. Signed-off-by: Gilad Ben-Yossef --- drivers/staging/ccree/ssi_aead.c | 26 +++--- drivers/staging/ccree/ssi_cipher.c | 13 ++- drivers/staging/ccree/ssi_driver.h | 2 +- drivers/staging/ccree/ssi_hash.c | 28 +++---- drivers/staging/ccree/ssi_request_mgr.c | 136 ++++++++++++++++++++++++++++++-- 5 files changed, 163 insertions(+), 42 deletions(-) -- 2.7.4 diff --git a/drivers/staging/ccree/ssi_aead.c b/drivers/staging/ccree/ssi_aead.c index 939b1b1..9190632 100644 --- a/drivers/staging/ccree/ssi_aead.c +++ b/drivers/staging/ccree/ssi_aead.c @@ -211,19 +211,21 @@ static int cc_aead_init(struct crypto_aead *tfm) return -ENOMEM; } -static void cc_aead_complete(struct device *dev, void *cc_req) +static void cc_aead_complete(struct device *dev, void *cc_req, int err) { struct aead_request *areq = (struct aead_request *)cc_req; struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req); struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); - int err = 0; cc_unmap_aead_request(dev, areq); /* Restore ordinary iv pointer */ areq->iv = areq_ctx->backup_iv; + if (err) + goto done; + if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr, ctx->authsize) != 0) { @@ -258,7 +260,7 @@ static void cc_aead_complete(struct device *dev, void *cc_req) CCM_BLOCK_IV_OFFSET, CCM_BLOCK_IV_SIZE); } } - +done: aead_request_complete(areq, err); } @@ -2041,7 +2043,7 @@ static int cc_proc_aead(struct aead_request *req, rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base); - if (rc != -EINPROGRESS) { + if (rc != -EINPROGRESS && rc != -EBUSY) { dev_err(dev, "send_request() failed (rc=%d)\n", rc); cc_unmap_aead_request(dev, req); } @@ -2063,7 +2065,7 @@ static int cc_aead_encrypt(struct aead_request *req) areq_ctx->plaintext_authenticate_only = false; rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); - if (rc != -EINPROGRESS) + if (rc != -EINPROGRESS && rc != -EBUSY) req->iv = areq_ctx->backup_iv; return rc; @@ -2092,7 +2094,7 @@ static int cc_rfc4309_ccm_encrypt(struct aead_request *req) cc_proc_rfc4309_ccm(req); rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); - if (rc != -EINPROGRESS) + if (rc != -EINPROGRESS && rc != -EBUSY) req->iv = areq_ctx->backup_iv; out: return rc; @@ -2111,7 +2113,7 @@ static int cc_aead_decrypt(struct aead_request *req) areq_ctx->plaintext_authenticate_only = false; rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); - if (rc != -EINPROGRESS) + if (rc != -EINPROGRESS && rc != -EBUSY) req->iv = areq_ctx->backup_iv; return rc; @@ -2138,7 +2140,7 @@ static int cc_rfc4309_ccm_decrypt(struct aead_request *req) cc_proc_rfc4309_ccm(req); rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); - if (rc != -EINPROGRESS) + if (rc != -EINPROGRESS && rc != -EBUSY) req->iv = areq_ctx->backup_iv; out: @@ -2257,7 +2259,7 @@ static int cc_rfc4106_gcm_encrypt(struct aead_request *req) areq_ctx->is_gcm4543 = true; rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); - if (rc != -EINPROGRESS) + if (rc != -EINPROGRESS && rc != -EBUSY) req->iv = areq_ctx->backup_iv; out: return rc; @@ -2281,7 +2283,7 @@ static int cc_rfc4543_gcm_encrypt(struct aead_request *req) areq_ctx->is_gcm4543 = true; rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); - if (rc != -EINPROGRESS) + if (rc != -EINPROGRESS && rc != -EBUSY) req->iv = areq_ctx->backup_iv; return rc; @@ -2312,7 +2314,7 @@ static int cc_rfc4106_gcm_decrypt(struct aead_request *req) areq_ctx->is_gcm4543 = true; rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); - if (rc != -EINPROGRESS) + if (rc != -EINPROGRESS && rc != -EBUSY) req->iv = areq_ctx->backup_iv; out: return rc; @@ -2336,7 +2338,7 @@ static int cc_rfc4543_gcm_decrypt(struct aead_request *req) areq_ctx->is_gcm4543 = true; rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); - if (rc != -EINPROGRESS) + if (rc != -EINPROGRESS && rc != -EBUSY) req->iv = areq_ctx->backup_iv; return rc; diff --git a/drivers/staging/ccree/ssi_cipher.c b/drivers/staging/ccree/ssi_cipher.c index 6098d21..527596b 100644 --- a/drivers/staging/ccree/ssi_cipher.c +++ b/drivers/staging/ccree/ssi_cipher.c @@ -52,7 +52,7 @@ struct cc_cipher_ctx { struct crypto_shash *shash_tfm; }; -static void cc_cipher_complete(struct device *dev, void *cc_req); +static void cc_cipher_complete(struct device *dev, void *cc_req, int err); static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size) { @@ -590,7 +590,7 @@ static void cc_setup_cipher_data(struct crypto_tfm *tfm, } } -static void cc_cipher_complete(struct device *dev, void *cc_req) +static void cc_cipher_complete(struct device *dev, void *cc_req, int err) { struct ablkcipher_request *areq = (struct ablkcipher_request *)cc_req; struct scatterlist *dst = areq->dst; @@ -598,7 +598,6 @@ static void cc_cipher_complete(struct device *dev, void *cc_req) struct blkcipher_req_ctx *req_ctx = ablkcipher_request_ctx(areq); struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); - int completion_error = 0; struct ablkcipher_request *req = (struct ablkcipher_request *)areq; cc_unmap_blkcipher_request(dev, req_ctx, ivsize, src, dst); @@ -614,13 +613,13 @@ static void cc_cipher_complete(struct device *dev, void *cc_req) if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { memcpy(req->info, req_ctx->backup_info, ivsize); kfree(req_ctx->backup_info); - } else { + } else if (!err) { scatterwalk_map_and_copy(req->info, req->dst, (req->nbytes - ivsize), ivsize, 0); } - ablkcipher_request_complete(areq, completion_error); + ablkcipher_request_complete(areq, err); } static int cc_cipher_process(struct ablkcipher_request *req, @@ -719,7 +718,7 @@ static int cc_cipher_process(struct ablkcipher_request *req, rc = cc_send_request(ctx_p->drvdata, &cc_req, desc, seq_len, &req->base); - if (rc != -EINPROGRESS) { + if (rc != -EINPROGRESS && rc != -EBUSY) { /* Failed to send the request or request completed * synchronously */ @@ -730,7 +729,7 @@ static int cc_cipher_process(struct ablkcipher_request *req, if (cts_restore_flag) ctx_p->cipher_mode = DRV_CIPHER_CBC_CTS; - if (rc != -EINPROGRESS) { + if (rc != -EINPROGRESS && rc != -EBUSY) { kfree(req_ctx->backup_info); kfree(req_ctx->iv); } diff --git a/drivers/staging/ccree/ssi_driver.h b/drivers/staging/ccree/ssi_driver.h index 4041c43..f57043e 100644 --- a/drivers/staging/ccree/ssi_driver.h +++ b/drivers/staging/ccree/ssi_driver.h @@ -81,7 +81,7 @@ extern bool cc_dump_bytes; #define CC_MAX_IVGEN_DMA_ADDRESSES 3 struct cc_crypto_req { - void (*user_cb)(struct device *dev, void *req); + void (*user_cb)(struct device *dev, void *req, int err); void *user_arg; dma_addr_t ivgen_dma_addr[CC_MAX_IVGEN_DMA_ADDRESSES]; /* For the first 'ivgen_dma_addr_len' addresses of this array, diff --git a/drivers/staging/ccree/ssi_hash.c b/drivers/staging/ccree/ssi_hash.c index 076162c..cc11344 100644 --- a/drivers/staging/ccree/ssi_hash.c +++ b/drivers/staging/ccree/ssi_hash.c @@ -342,7 +342,7 @@ static void cc_unmap_result(struct device *dev, struct ahash_req_ctx *state, state->digest_result_dma_addr = 0; } -static void cc_update_complete(struct device *dev, void *cc_req) +static void cc_update_complete(struct device *dev, void *cc_req, int err) { struct ahash_request *req = (struct ahash_request *)cc_req; struct ahash_req_ctx *state = ahash_request_ctx(req); @@ -350,10 +350,10 @@ static void cc_update_complete(struct device *dev, void *cc_req) dev_dbg(dev, "req=%pK\n", req); cc_unmap_hash_request(dev, state, req->src, false); - req->base.complete(&req->base, 0); + req->base.complete(&req->base, err); } -static void cc_digest_complete(struct device *dev, void *cc_req) +static void cc_digest_complete(struct device *dev, void *cc_req, int err) { struct ahash_request *req = (struct ahash_request *)cc_req; struct ahash_req_ctx *state = ahash_request_ctx(req); @@ -366,10 +366,10 @@ static void cc_digest_complete(struct device *dev, void *cc_req) cc_unmap_hash_request(dev, state, req->src, false); cc_unmap_result(dev, state, digestsize, req->result); cc_unmap_req(dev, state, ctx); - req->base.complete(&req->base, 0); + req->base.complete(&req->base, err); } -static void cc_hash_complete(struct device *dev, void *cc_req) +static void cc_hash_complete(struct device *dev, void *cc_req, int err) { struct ahash_request *req = (struct ahash_request *)cc_req; struct ahash_req_ctx *state = ahash_request_ctx(req); @@ -382,7 +382,7 @@ static void cc_hash_complete(struct device *dev, void *cc_req) cc_unmap_hash_request(dev, state, req->src, false); cc_unmap_result(dev, state, digestsize, req->result); cc_unmap_req(dev, state, ctx); - req->base.complete(&req->base, 0); + req->base.complete(&req->base, err); } static int cc_hash_digest(struct ahash_request *req) @@ -533,7 +533,7 @@ static int cc_hash_digest(struct ahash_request *req) idx++; rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); - if (rc != -EINPROGRESS) { + if (rc != -EINPROGRESS && rc != -EBUSY) { dev_err(dev, "send_request() failed (rc=%d)\n", rc); cc_unmap_hash_request(dev, state, src, true); cc_unmap_result(dev, state, digestsize, result); @@ -621,7 +621,7 @@ static int cc_hash_update(struct ahash_request *req) idx++; rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); - if (rc != -EINPROGRESS) { + if (rc != -EINPROGRESS && rc != -EBUSY) { dev_err(dev, "send_request() failed (rc=%d)\n", rc); cc_unmap_hash_request(dev, state, src, true); } @@ -742,7 +742,7 @@ static int cc_hash_finup(struct ahash_request *req) idx++; rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); - if (rc != -EINPROGRESS) { + if (rc != -EINPROGRESS && rc != -EBUSY) { dev_err(dev, "send_request() failed (rc=%d)\n", rc); cc_unmap_hash_request(dev, state, src, true); cc_unmap_result(dev, state, digestsize, result); @@ -874,7 +874,7 @@ static int cc_hash_final(struct ahash_request *req) idx++; rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); - if (rc != -EINPROGRESS) { + if (rc != -EINPROGRESS && rc != -EBUSY) { dev_err(dev, "send_request() failed (rc=%d)\n", rc); cc_unmap_hash_request(dev, state, src, true); cc_unmap_result(dev, state, digestsize, result); @@ -1356,7 +1356,7 @@ static int cc_mac_update(struct ahash_request *req) cc_req.user_arg = (void *)req; rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); - if (rc != -EINPROGRESS) { + if (rc != -EINPROGRESS && rc != -EBUSY) { dev_err(dev, "send_request() failed (rc=%d)\n", rc); cc_unmap_hash_request(dev, state, req->src, true); } @@ -1469,7 +1469,7 @@ static int cc_mac_final(struct ahash_request *req) idx++; rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); - if (rc != -EINPROGRESS) { + if (rc != -EINPROGRESS && rc != -EBUSY) { dev_err(dev, "send_request() failed (rc=%d)\n", rc); cc_unmap_hash_request(dev, state, req->src, true); cc_unmap_result(dev, state, digestsize, req->result); @@ -1542,7 +1542,7 @@ static int cc_mac_finup(struct ahash_request *req) idx++; rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); - if (rc != -EINPROGRESS) { + if (rc != -EINPROGRESS && rc != -EBUSY) { dev_err(dev, "send_request() failed (rc=%d)\n", rc); cc_unmap_hash_request(dev, state, req->src, true); cc_unmap_result(dev, state, digestsize, req->result); @@ -1616,7 +1616,7 @@ static int cc_mac_digest(struct ahash_request *req) idx++; rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); - if (rc != -EINPROGRESS) { + if (rc != -EINPROGRESS && rc != -EBUSY) { dev_err(dev, "send_request() failed (rc=%d)\n", rc); cc_unmap_hash_request(dev, state, req->src, true); cc_unmap_result(dev, state, digestsize, req->result); diff --git a/drivers/staging/ccree/ssi_request_mgr.c b/drivers/staging/ccree/ssi_request_mgr.c index f52cd72..530ba7e 100644 --- a/drivers/staging/ccree/ssi_request_mgr.c +++ b/drivers/staging/ccree/ssi_request_mgr.c @@ -14,6 +14,8 @@ #include "ssi_pm.h" #define CC_MAX_POLL_ITER 10 +/* The highest descriptor count in used */ +#define CC_MAX_DESC_SEQ_LEN 23 struct cc_req_mgr_handle { /* Request manager resources */ @@ -33,6 +35,11 @@ struct cc_req_mgr_handle { u8 *dummy_comp_buff; dma_addr_t dummy_comp_buff_dma; + /* backlog queue */ + struct list_head backlog; + unsigned int bl_len; + spinlock_t bl_lock; /* protect backlog queue */ + #ifdef COMP_IN_WQ struct workqueue_struct *workq; struct delayed_work compwork; @@ -44,6 +51,14 @@ struct cc_req_mgr_handle { #endif }; +struct cc_bl_item { + struct cc_crypto_req creq; + struct cc_hw_desc desc[CC_MAX_DESC_SEQ_LEN]; + unsigned int len; + struct list_head list; + bool notif; +}; + static void comp_handler(unsigned long devarg); #ifdef COMP_IN_WQ static void comp_work_handler(struct work_struct *work); @@ -93,6 +108,9 @@ int cc_req_mgr_init(struct cc_drvdata *drvdata) drvdata->request_mgr_handle = req_mgr_h; spin_lock_init(&req_mgr_h->hw_lock); + spin_lock_init(&req_mgr_h->bl_lock); + INIT_LIST_HEAD(&req_mgr_h->backlog); + #ifdef COMP_IN_WQ dev_dbg(dev, "Initializing completion workqueue\n"); req_mgr_h->workq = create_singlethread_workqueue("arm_cc7x_wq"); @@ -177,7 +195,8 @@ static void enqueue_seq(struct cc_drvdata *drvdata, struct cc_hw_desc seq[], * \param dev * \param dx_compl_h The completion event to signal */ -static void request_mgr_complete(struct device *dev, void *dx_compl_h) +static void request_mgr_complete(struct device *dev, void *dx_compl_h, + int dummy) { struct completion *this_compl = dx_compl_h; @@ -322,6 +341,84 @@ static int cc_do_send_request(struct cc_drvdata *drvdata, return -EINPROGRESS; } +static void cc_enqueue_backlog(struct cc_drvdata *drvdata, + struct cc_bl_item *bli) +{ + struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; + + spin_lock_bh(&mgr->bl_lock); + list_add_tail(&bli->list, &mgr->backlog); + ++mgr->bl_len; + spin_unlock_bh(&mgr->bl_lock); + tasklet_schedule(&mgr->comptask); +} + +static void cc_proc_backlog(struct cc_drvdata *drvdata) +{ + struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; + struct cc_bl_item *bli; + struct cc_crypto_req *creq; + struct crypto_async_request *req; + bool ivgen; + unsigned int total_len; + struct device *dev = drvdata_to_dev(drvdata); + int rc; + + spin_lock(&mgr->bl_lock); + + while (mgr->bl_len) { + bli = list_first_entry(&mgr->backlog, struct cc_bl_item, list); + spin_unlock(&mgr->bl_lock); + + creq = &bli->creq; + req = (struct crypto_async_request *)creq->user_arg; + + /* + * Notify the request we're moving out of the backlog + * but only if we haven't done so already. + */ + if (!bli->notif) { + req->complete(req, -EINPROGRESS); + bli->notif = true; + } + + ivgen = !!creq->ivgen_dma_addr_len; + total_len = bli->len + (ivgen ? CC_IVPOOL_SEQ_LEN : 0); + + spin_lock(&mgr->hw_lock); + + rc = cc_queues_status(drvdata, mgr, total_len); + if (rc) { + /* + * There is still not room in the FIFO for + * this request. Bail out. We'll return here + * on the next completion irq. + */ + spin_unlock(&mgr->hw_lock); + return; + } + + rc = cc_do_send_request(drvdata, &bli->creq, bli->desc, + bli->len, false, ivgen); + + spin_unlock(&mgr->hw_lock); + + if (rc != -EINPROGRESS) { +#if defined(CONFIG_PM) + cc_pm_put_suspend(dev); +#endif + creq->user_cb(dev, req, rc); + } + + /* Remove ourselves from the backlog list */ + spin_lock(&mgr->bl_lock); + list_del(&bli->list); + --mgr->bl_len; + } + + spin_unlock(&mgr->bl_lock); +} + int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req, struct cc_hw_desc *desc, unsigned int len, struct crypto_async_request *req) @@ -331,6 +428,9 @@ int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req, bool ivgen = !!cc_req->ivgen_dma_addr_len; unsigned int total_len = len + (ivgen ? CC_IVPOOL_SEQ_LEN : 0); struct device *dev = drvdata_to_dev(drvdata); + bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG; + gfp_t flags = cc_gfp_flags(req); + struct cc_bl_item *bli; #if defined(CONFIG_PM) rc = cc_pm_get(dev); @@ -342,17 +442,35 @@ int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req, spin_lock_bh(&mgr->hw_lock); rc = cc_queues_status(drvdata, mgr, total_len); - if (!rc) - rc = cc_do_send_request(drvdata, cc_req, desc, len, false, - ivgen); +#ifdef CC_DEBUG_FORCE_BACKLOG + if (backlog_ok) + rc = -ENOSPC; +#endif /* CC_DEBUG_FORCE_BACKLOG */ - spin_unlock_bh(&mgr->hw_lock); + if (rc == -ENOSPC && backlog_ok) { + spin_unlock_bh(&mgr->hw_lock); + bli = kmalloc(sizeof(*bli), flags); + if (!bli) { #if defined(CONFIG_PM) - if (rc != -EINPROGRESS) - cc_pm_put_suspend(dev); + cc_pm_put_suspend(dev); #endif + return -ENOMEM; + } + + memcpy(&bli->creq, cc_req, sizeof(*cc_req)); + memcpy(&bli->desc, desc, len * sizeof(*desc)); + bli->len = len; + bli->notif = false; + cc_enqueue_backlog(drvdata, bli); + return -EBUSY; + } + if (!rc) + rc = cc_do_send_request(drvdata, cc_req, desc, len, false, + ivgen); + + spin_unlock_bh(&mgr->hw_lock); return rc; } @@ -501,7 +619,7 @@ static void proc_completions(struct cc_drvdata *drvdata) cc_req = &request_mgr_handle->req_queue[*tail]; if (cc_req->user_cb) - cc_req->user_cb(dev, cc_req->user_arg); + cc_req->user_cb(dev, cc_req->user_arg, 0); *tail = (*tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1); dev_dbg(dev, "Dequeue request tail=%u\n", *tail); dev_dbg(dev, "Request completed. axi_completed=%d\n", @@ -566,6 +684,8 @@ static void comp_handler(unsigned long devarg) */ cc_iowrite(drvdata, CC_REG(HOST_IMR), cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~irq); + + cc_proc_backlog(drvdata); } /*