@@ -84,6 +84,9 @@ static int crypto_acomp_init_tfm(struct crypto_tfm *tfm)
acomp->compress = alg->compress;
acomp->decompress = alg->decompress;
+ acomp->get_batch_size = alg->get_batch_size;
+ acomp->batch_compress = alg->batch_compress;
+ acomp->batch_decompress = alg->batch_decompress;
acomp->dst_free = alg->dst_free;
acomp->reqsize = alg->reqsize;
@@ -43,6 +43,10 @@ struct acomp_req {
*
* @compress: Function performs a compress operation
* @decompress: Function performs a de-compress operation
+ * @get_batch_size: Maximum batch-size for batching compress/decompress
+ * operations.
+ * @batch_compress: Function performs a batch compress operation
+ * @batch_decompress: Function performs a batch decompress operation
* @dst_free: Frees destination buffer if allocated inside the
* algorithm
* @reqsize: Context size for (de)compression requests
@@ -51,6 +55,21 @@ struct acomp_req {
struct crypto_acomp {
int (*compress)(struct acomp_req *req);
int (*decompress)(struct acomp_req *req);
+ unsigned int (*get_batch_size)(void);
+ bool (*batch_compress)(struct acomp_req *reqs[],
+ struct crypto_wait *wait,
+ struct page *pages[],
+ u8 *dsts[],
+ unsigned int dlens[],
+ int errors[],
+ int nr_pages);
+ bool (*batch_decompress)(struct acomp_req *reqs[],
+ struct crypto_wait *wait,
+ u8 *srcs[],
+ struct page *pages[],
+ unsigned int slens[],
+ int errors[],
+ int nr_pages);
void (*dst_free)(struct scatterlist *dst);
unsigned int reqsize;
struct crypto_tfm base;
@@ -142,6 +161,13 @@ static inline bool acomp_is_async(struct crypto_acomp *tfm)
CRYPTO_ALG_ASYNC;
}
+static inline bool acomp_has_async_batching(struct crypto_acomp *tfm)
+{
+ return (acomp_is_async(tfm) &&
+ (crypto_comp_alg_common(tfm)->base.cra_flags & CRYPTO_ALG_TYPE_ACOMPRESS) &&
+ tfm->get_batch_size && tfm->batch_compress && tfm->batch_decompress);
+}
+
static inline struct crypto_acomp *crypto_acomp_reqtfm(struct acomp_req *req)
{
return __crypto_acomp_tfm(req->base.tfm);
@@ -306,4 +332,89 @@ static inline int crypto_acomp_decompress(struct acomp_req *req)
return crypto_acomp_reqtfm(req)->decompress(req);
}
+/**
+ * crypto_acomp_batch_size() -- Get the algorithm's batch size
+ *
+ * Function returns the algorithm's batch size for batching operations
+ *
+ * @tfm: ACOMPRESS tfm handle allocated with crypto_alloc_acomp()
+ *
+ * Return: crypto_acomp's batch size.
+ */
+static inline unsigned int crypto_acomp_batch_size(struct crypto_acomp *tfm)
+{
+ if (acomp_has_async_batching(tfm))
+ return tfm->get_batch_size();
+
+ return 1;
+}
+
+/**
+ * crypto_acomp_batch_compress() -- Invoke asynchronous compress of
+ * a batch of requests
+ *
+ * Function invokes the asynchronous batch compress operation
+ *
+ * @reqs: @nr_pages asynchronous compress requests.
+ * @wait: crypto_wait for acomp batch compress with synchronous/asynchronous
+ * request chaining. If NULL, the driver must provide a way to process
+ * request completions asynchronously.
+ * @pages: Pages to be compressed.
+ * @dsts: Pre-allocated destination buffers to store results of compression.
+ * @dlens: Will contain the compressed lengths.
+ * @errors: zero on successful compression of the corresponding
+ * req, or error code in case of error.
+ * @nr_pages: The number of pages to be compressed.
+ *
+ * Returns true if all compress requests complete successfully,
+ * false otherwise.
+ */
+static inline bool crypto_acomp_batch_compress(struct acomp_req *reqs[],
+ struct crypto_wait *wait,
+ struct page *pages[],
+ u8 *dsts[],
+ unsigned int dlens[],
+ int errors[],
+ int nr_pages)
+{
+ struct crypto_acomp *tfm = crypto_acomp_reqtfm(reqs[0]);
+
+ return tfm->batch_compress(reqs, wait, pages, dsts,
+ dlens, errors, nr_pages);
+}
+
+/**
+ * crypto_acomp_batch_decompress() -- Invoke asynchronous decompress of
+ * a batch of requests
+ *
+ * Function invokes the asynchronous batch decompress operation
+ *
+ * @reqs: @nr_pages asynchronous decompress requests.
+ * @wait: crypto_wait for acomp batch decompress with synchronous/asynchronous
+ * request chaining. If NULL, the driver must provide a way to process
+ * request completions asynchronously.
+ * @srcs: The src buffers to be decompressed.
+ * @pages: The pages to store the decompressed buffers.
+ * @slens: Compressed lengths of @srcs.
+ * @errors: zero on successful compression of the corresponding
+ * req, or error code in case of error.
+ * @nr_pages: The number of pages to be decompressed.
+ *
+ * Returns true if all decompress requests complete successfully,
+ * false otherwise.
+ */
+static inline bool crypto_acomp_batch_decompress(struct acomp_req *reqs[],
+ struct crypto_wait *wait,
+ u8 *srcs[],
+ struct page *pages[],
+ unsigned int slens[],
+ int errors[],
+ int nr_pages)
+{
+ struct crypto_acomp *tfm = crypto_acomp_reqtfm(reqs[0]);
+
+ return tfm->batch_decompress(reqs, wait, srcs, pages,
+ slens, errors, nr_pages);
+}
+
#endif
@@ -17,6 +17,10 @@
*
* @compress: Function performs a compress operation
* @decompress: Function performs a de-compress operation
+ * @get_batch_size: Maximum batch-size for batching compress/decompress
+ * operations.
+ * @batch_compress: Function performs a batch compress operation
+ * @batch_decompress: Function performs a batch decompress operation
* @dst_free: Frees destination buffer if allocated inside the algorithm
* @init: Initialize the cryptographic transformation object.
* This function is used to initialize the cryptographic
@@ -37,6 +41,21 @@
struct acomp_alg {
int (*compress)(struct acomp_req *req);
int (*decompress)(struct acomp_req *req);
+ unsigned int (*get_batch_size)(void);
+ bool (*batch_compress)(struct acomp_req *reqs[],
+ struct crypto_wait *wait,
+ struct page *pages[],
+ u8 *dsts[],
+ unsigned int dlens[],
+ int errors[],
+ int nr_pages);
+ bool (*batch_decompress)(struct acomp_req *reqs[],
+ struct crypto_wait *wait,
+ u8 *srcs[],
+ struct page *pages[],
+ unsigned int slens[],
+ int errors[],
+ int nr_pages);
void (*dst_free)(struct scatterlist *dst);
int (*init)(struct crypto_acomp *tfm);
void (*exit)(struct crypto_acomp *tfm);
This commit adds get_batch_size(), batch_compress() and batch_decompress() interfaces to: struct acomp_alg struct crypto_acomp A crypto_acomp compression algorithm that supports batching of compressions and decompressions must provide implementations for these API. A new helper function acomp_has_async_batching() can be invoked to query if a crypto_acomp has registered these batching interfaces. A higher level module like zswap can call acomp_has_async_batching() to detect if the compressor supports batching, and if so, it can call the new crypto_acomp_batch_size() to detect the maximum batch-size supported by the compressor. Based on this, zswap can use the minimum of any zswap-specific upper limits for batch-size and the compressor's max batch-size, to allocate batching resources. This allows the iaa_crypto Intel IAA driver to register implementations for the get_batch_size(), batch_compress() and batch_decompress() acomp_alg interfaces, that can subsequently be invoked from the kernel zswap/zram modules to compress/decompress pages in parallel in the IAA hardware accelerator to improve swapout/swapin performance through these newly added corresponding crypto_acomp API: crypto_acomp_batch_size() crypto_acomp_batch_compress() crypto_acomp_batch_decompress() Signed-off-by: Kanchana P Sridhar <kanchana.p.sridhar@intel.com> --- crypto/acompress.c | 3 + include/crypto/acompress.h | 111 ++++++++++++++++++++++++++++ include/crypto/internal/acompress.h | 19 +++++ 3 files changed, 133 insertions(+)