@@ -1467,77 +1467,129 @@ static void shrink_worker(struct work_struct *w)
* main API
**********************************/
-static ssize_t zswap_store_page(struct page *page,
- struct obj_cgroup *objcg,
- struct zswap_pool *pool)
+static bool zswap_compress_folio(struct folio *folio,
+ struct zswap_entry *entries[],
+ struct zswap_pool *pool)
{
- swp_entry_t page_swpentry = page_swap_entry(page);
- struct zswap_entry *entry, *old;
+ long index, nr_pages = folio_nr_pages(folio);
- /* allocate entry */
- entry = zswap_entry_cache_alloc(GFP_KERNEL, page_to_nid(page));
- if (!entry) {
- zswap_reject_kmemcache_fail++;
- return -EINVAL;
+ for (index = 0; index < nr_pages; ++index) {
+ struct page *page = folio_page(folio, index);
+
+ if (!zswap_compress(page, entries[index], pool))
+ return false;
}
- if (!zswap_compress(page, entry, pool))
- goto compress_failed;
+ return true;
+}
- old = xa_store(swap_zswap_tree(page_swpentry),
- swp_offset(page_swpentry),
- entry, GFP_KERNEL);
- if (xa_is_err(old)) {
- int err = xa_err(old);
+/*
+ * Store all pages in a folio.
+ *
+ * The error handling from all failure points is consolidated to the
+ * "store_folio_failed" label, based on the initialization of the zswap entries'
+ * handles to ERR_PTR(-EINVAL) at allocation time, and the fact that the
+ * entry's handle is subsequently modified only upon a successful zpool_malloc()
+ * after the page is compressed.
+ */
+static ssize_t zswap_store_folio(struct folio *folio,
+ struct obj_cgroup *objcg,
+ struct zswap_pool *pool)
+{
+ long index, nr_pages = folio_nr_pages(folio);
+ struct zswap_entry **entries = NULL;
+ int node_id = folio_nid(folio);
+ size_t compressed_bytes = 0;
- WARN_ONCE(err != -ENOMEM, "unexpected xarray error: %d\n", err);
- zswap_reject_alloc_fail++;
- goto store_failed;
+ entries = kmalloc(nr_pages * sizeof(*entries), GFP_KERNEL);
+ if (!entries)
+ return -ENOMEM;
+
+ /* allocate entries */
+ for (index = 0; index < nr_pages; ++index) {
+ entries[index] = zswap_entry_cache_alloc(GFP_KERNEL, node_id);
+
+ if (!entries[index]) {
+ zswap_reject_kmemcache_fail++;
+ nr_pages = index;
+ goto store_folio_failed;
+ }
+
+ entries[index]->handle = (unsigned long)ERR_PTR(-EINVAL);
}
- /*
- * We may have had an existing entry that became stale when
- * the folio was redirtied and now the new version is being
- * swapped out. Get rid of the old.
- */
- if (old)
- zswap_entry_free(old);
+ if (!zswap_compress_folio(folio, entries, pool))
+ goto store_folio_failed;
- /*
- * The entry is successfully compressed and stored in the tree, there is
- * no further possibility of failure. Grab refs to the pool and objcg.
- * These refs will be dropped by zswap_entry_free() when the entry is
- * removed from the tree.
- */
- zswap_pool_get(pool);
- if (objcg)
- obj_cgroup_get(objcg);
+ for (index = 0; index < nr_pages; ++index) {
+ swp_entry_t page_swpentry = page_swap_entry(folio_page(folio, index));
+ struct zswap_entry *old, *entry = entries[index];
+
+ old = xa_store(swap_zswap_tree(page_swpentry),
+ swp_offset(page_swpentry),
+ entry, GFP_KERNEL);
+ if (xa_is_err(old)) {
+ int err = xa_err(old);
+
+ WARN_ONCE(err != -ENOMEM, "unexpected xarray error: %d\n", err);
+ zswap_reject_alloc_fail++;
+ goto store_folio_failed;
+ }
- /*
- * We finish initializing the entry while it's already in xarray.
- * This is safe because:
- *
- * 1. Concurrent stores and invalidations are excluded by folio lock.
- *
- * 2. Writeback is excluded by the entry not being on the LRU yet.
- * The publishing order matters to prevent writeback from seeing
- * an incoherent entry.
- */
- entry->pool = pool;
- entry->swpentry = page_swpentry;
- entry->objcg = objcg;
- entry->referenced = true;
- if (entry->length) {
- INIT_LIST_HEAD(&entry->lru);
- zswap_lru_add(&zswap_list_lru, entry);
+ /*
+ * We may have had an existing entry that became stale when
+ * the folio was redirtied and now the new version is being
+ * swapped out. Get rid of the old.
+ */
+ if (old)
+ zswap_entry_free(old);
+
+ /*
+ * The entry is successfully compressed and stored in the tree, there is
+ * no further possibility of failure. Grab refs to the pool and objcg.
+ * These refs will be dropped by zswap_entry_free() when the entry is
+ * removed from the tree.
+ */
+ zswap_pool_get(pool);
+ if (objcg)
+ obj_cgroup_get(objcg);
+
+ /*
+ * We finish initializing the entry while it's already in xarray.
+ * This is safe because:
+ *
+ * 1. Concurrent stores and invalidations are excluded by folio lock.
+ *
+ * 2. Writeback is excluded by the entry not being on the LRU yet.
+ * The publishing order matters to prevent writeback from seeing
+ * an incoherent entry.
+ */
+ entry->pool = pool;
+ entry->swpentry = page_swpentry;
+ entry->objcg = objcg;
+ entry->referenced = true;
+ if (entry->length) {
+ INIT_LIST_HEAD(&entry->lru);
+ zswap_lru_add(&zswap_list_lru, entry);
+ }
+
+ compressed_bytes += entry->length;
}
- return entry->length;
+ kfree(entries);
+
+ return compressed_bytes;
+
+store_folio_failed:
+ for (index = 0; index < nr_pages; ++index) {
+ if (!IS_ERR_VALUE(entries[index]->handle))
+ zpool_free(pool->zpool, entries[index]->handle);
+
+ zswap_entry_cache_free(entries[index]);
+ }
+
+ kfree(entries);
-store_failed:
- zpool_free(pool->zpool, entry->handle);
-compress_failed:
- zswap_entry_cache_free(entry);
return -EINVAL;
}
@@ -1549,8 +1601,8 @@ bool zswap_store(struct folio *folio)
struct mem_cgroup *memcg = NULL;
struct zswap_pool *pool;
size_t compressed_bytes = 0;
+ ssize_t bytes;
bool ret = false;
- long index;
VM_WARN_ON_ONCE(!folio_test_locked(folio));
VM_WARN_ON_ONCE(!folio_test_swapcache(folio));
@@ -1584,15 +1636,11 @@ bool zswap_store(struct folio *folio)
mem_cgroup_put(memcg);
}
- for (index = 0; index < nr_pages; ++index) {
- struct page *page = folio_page(folio, index);
- ssize_t bytes;
+ bytes = zswap_store_folio(folio, objcg, pool);
+ if (bytes < 0)
+ goto put_pool;
- bytes = zswap_store_page(page, objcg, pool);
- if (bytes < 0)
- goto put_pool;
- compressed_bytes += bytes;
- }
+ compressed_bytes = bytes;
if (objcg) {
obj_cgroup_charge_zswap(objcg, compressed_bytes);
@@ -1622,6 +1670,7 @@ bool zswap_store(struct folio *folio)
pgoff_t offset = swp_offset(swp);
struct zswap_entry *entry;
struct xarray *tree;
+ long index;
for (index = 0; index < nr_pages; ++index) {
tree = swap_zswap_tree(swp_entry(type, offset + index));
This patch introduces zswap_store_folio() that implements all the computes done earlier in zswap_store_page() for a single-page, for all the pages in a folio. This allows us to move the loop over the folio's pages from zswap_store() to zswap_store_folio(). A distinct zswap_compress_folio() is also added, that simply calls zswap_compress() for each page in the folio it is called with. zswap_store_folio() starts by allocating all zswap entries required to store the folio. Next, it calls zswap_compress_folio() and finally, adds the entries to the xarray and LRU. The error handling and cleanup required for all failure scenarios that can occur while storing a folio in zswap is now consolidated to a "store_folio_failed" label in zswap_store_folio(). These changes facilitate developing support for compress batching in zswap_store_folio(). Signed-off-by: Kanchana P Sridhar <kanchana.p.sridhar@intel.com> --- mm/zswap.c | 183 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 116 insertions(+), 67 deletions(-)