@@ -15,6 +15,9 @@
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/iversion.h>
+#include <linux/xarray.h>
+#include <linux/fscache.h>
+#include <linux/netfs.h>
#include "internal.h"
#include "iostat.h"
@@ -373,66 +376,123 @@ void __nfs_fscache_invalidate_page(struct page *page, struct inode *inode)
NFSIOS_FSCACHE_PAGES_UNCACHED);
}
-/*
- * Handle completion of a page being read from the cache.
- * - Called in process (keventd) context.
- */
-static void nfs_readpage_from_fscache_complete(struct page *page,
- void *context,
- int error)
+static void nfs_issue_op(struct netfs_read_subrequest *subreq)
{
- dfprintk(FSCACHE,
- "NFS: readpage_from_fscache_complete (0x%p/0x%p/%d)\n",
- page, context, error);
-
- /* if the read completes with an error, we just unlock the page and let
- * the VM reissue the readpage */
- if (!error) {
- SetPageUptodate(page);
- unlock_page(page);
- } else {
- error = nfs_readpage_async(context, page->mapping->host, page);
- if (error)
- unlock_page(page);
+ struct inode *inode = subreq->rreq->inode;
+ struct nfs_readdesc *desc = subreq->rreq->netfs_priv;
+ struct page *page;
+ pgoff_t start = (subreq->start + subreq->transferred) >> PAGE_SHIFT;
+ pgoff_t last = ((subreq->start + subreq->len -
+ subreq->transferred - 1) >> PAGE_SHIFT);
+ XA_STATE(xas, &subreq->rreq->mapping->i_pages, start);
+
+ dfprintk(FSCACHE, "NFS: %s(fsc:%p s:%lu l:%lu) subreq->start: %lld "
+ "subreq->len: %ld subreq->transferred: %ld\n",
+ __func__, nfs_i_fscache(inode), start, last, subreq->start,
+ subreq->len, subreq->transferred);
+
+ nfs_add_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_FAIL,
+ last - start + 1);
+ nfs_pageio_init_read(&desc->pgio, inode, false,
+ &nfs_async_read_completion_ops);
+
+ desc->pgio.pg_fsc = subreq; /* used in completion */
+
+ rcu_read_lock();
+ xas_for_each(&xas, page, last) {
+ subreq->error = readpage_async_filler(desc, page);
+ if (subreq->error < 0)
+ break;
+ }
+ rcu_read_unlock();
+ nfs_pageio_complete_read(&desc->pgio, inode);
+}
+
+static bool nfs_clamp_length(struct netfs_read_subrequest *subreq)
+{
+ struct inode *inode = subreq->rreq->mapping->host;
+ unsigned int rsize = NFS_SB(inode->i_sb)->rsize;
+
+ if (subreq->len > rsize) {
+ dfprintk(FSCACHE,
+ "NFS: %s(fsc:%p slen:%lu rsize: %u)\n",
+ __func__, nfs_i_fscache(inode), subreq->len, rsize);
+ subreq->len = rsize;
}
+
+ return true;
+}
+
+static void nfs_cleanup(struct address_space *mapping, void *netfs_priv)
+{
+ ; /* fscache assumes if netfs_priv is given we have cleanup */
+}
+
+atomic_t nfs_fscache_debug_id;
+static void nfs_init_rreq(struct netfs_read_request *rreq, struct file *file)
+{
+ struct nfs_inode *nfsi = NFS_I(rreq->inode);
+
+ if (nfsi->fscache && test_bit(NFS_INO_FSCACHE, &nfsi->flags))
+ rreq->cookie_debug_id = atomic_inc_return(&nfs_fscache_debug_id);
+}
+
+static bool nfs_is_cache_enabled(struct inode *inode)
+{
+ struct nfs_inode *nfsi = NFS_I(inode);
+
+ return nfsi->fscache && test_bit(NFS_INO_FSCACHE, &nfsi->flags);
+}
+
+static int nfs_begin_cache_operation(struct netfs_read_request *rreq)
+{
+ struct fscache_cookie *cookie = NFS_I(rreq->inode)->fscache;
+
+ return fscache_begin_read_operation(rreq, cookie);
}
+static struct netfs_read_request_ops nfs_fscache_req_ops = {
+ .init_rreq = nfs_init_rreq,
+ .is_cache_enabled = nfs_is_cache_enabled,
+ .begin_cache_operation = nfs_begin_cache_operation,
+ .issue_op = nfs_issue_op,
+ .clamp_length = nfs_clamp_length,
+ .cleanup = nfs_cleanup
+};
+
/*
* Retrieve a page from fscache
*/
-int __nfs_readpage_from_fscache(struct nfs_open_context *ctx,
- struct inode *inode, struct page *page)
+int __nfs_readpage_from_fscache(struct file *filp,
+ struct page *page,
+ struct nfs_readdesc *desc)
{
int ret;
+ struct inode *inode = file_inode(filp);
dfprintk(FSCACHE,
"NFS: readpage_from_fscache(fsc:%p/p:%p(i:%lx f:%lx)/0x%p)\n",
nfs_i_fscache(inode), page, page->index, page->flags, inode);
- ret = fscache_read_or_alloc_page(nfs_i_fscache(inode),
- page,
- nfs_readpage_from_fscache_complete,
- ctx,
- GFP_KERNEL);
+ ret = netfs_readpage(filp, page, &nfs_fscache_req_ops, desc);
switch (ret) {
- case 0: /* read BIO submitted (page in fscache) */
- dfprintk(FSCACHE,
- "NFS: readpage_from_fscache: BIO submitted\n");
+ case 0: /* read submitted */
+ dfprintk(FSCACHE, "NFS: readpage_from_fscache: submitted\n");
nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_OK);
return ret;
case -ENOBUFS: /* inode not in cache */
case -ENODATA: /* page not in cache */
nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_FAIL);
- dfprintk(FSCACHE,
- "NFS: readpage_from_fscache %d\n", ret);
+ dfprintk(FSCACHE, "NFS: readpage_from_fscache %d\n", ret);
return 1;
default:
dfprintk(FSCACHE, "NFS: readpage_from_fscache %d\n", ret);
nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_FAIL);
}
+
return ret;
}
@@ -487,30 +547,19 @@ int __nfs_readpages_from_fscache(struct nfs_open_context *ctx,
}
/*
- * Store a newly fetched page in fscache
- * - PG_fscache must be set on the page
+ * Store a newly fetched data in fscache
*/
-void __nfs_readpage_to_fscache(struct inode *inode, struct page *page, int sync)
+void __nfs_read_completion_to_fscache(struct nfs_pgio_header *hdr,
+ unsigned long bytes)
{
- int ret;
+ struct netfs_read_subrequest *subreq = hdr->fsc;
- dfprintk(FSCACHE,
- "NFS: readpage_to_fscache(fsc:%p/p:%p(i:%lx f:%lx)/%d)\n",
- nfs_i_fscache(inode), page, page->index, page->flags, sync);
-
- ret = fscache_write_page(nfs_i_fscache(inode), page,
- inode->i_size, GFP_KERNEL);
- dfprintk(FSCACHE,
- "NFS: readpage_to_fscache: p:%p(i:%lu f:%lx) ret %d\n",
- page, page->index, page->flags, ret);
-
- if (ret != 0) {
- fscache_uncache_page(nfs_i_fscache(inode), page);
- nfs_inc_fscache_stats(inode,
- NFSIOS_FSCACHE_PAGES_WRITTEN_FAIL);
- nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_UNCACHED);
- } else {
- nfs_inc_fscache_stats(inode,
- NFSIOS_FSCACHE_PAGES_WRITTEN_OK);
+ if (subreq) {
+ dfprintk(FSCACHE,
+ "NFS: read_completion_to_fscache(fsc:%p err:%d bytes:%lu subreq->len:%lu\n",
+ NFS_I(hdr->inode)->fscache, hdr->error, bytes, subreq->len);
+ __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
+ netfs_subreq_terminated(subreq, hdr->error ?: bytes);
+ hdr->fsc = NULL;
}
}
@@ -95,9 +95,9 @@ extern void nfs_fscache_open_file(struct inode *, struct file *);
extern void __nfs_fscache_invalidate_page(struct page *, struct inode *);
extern int nfs_fscache_release_page(struct page *, gfp_t);
-
-extern int __nfs_readpage_from_fscache(struct nfs_open_context *,
- struct inode *, struct page *);
+extern int __nfs_readpage_from_fscache(struct file *filp,
+ struct page *page,
+ struct nfs_readdesc *desc);
extern int __nfs_readpages_from_fscache(struct nfs_open_context *,
struct inode *, struct address_space *,
struct list_head *, unsigned *);
@@ -127,12 +127,12 @@ static inline void nfs_fscache_invalidate_page(struct page *page,
/*
* Retrieve a page from an inode data storage object.
*/
-static inline int nfs_readpage_from_fscache(struct nfs_open_context *ctx,
- struct inode *inode,
- struct page *page)
+static inline int nfs_readpage_from_fscache(struct file *filp,
+ struct page *page,
+ struct nfs_readdesc *desc)
{
- if (NFS_I(inode)->fscache)
- return __nfs_readpage_from_fscache(ctx, inode, page);
+ if (NFS_I(file_inode(filp))->fscache)
+ return __nfs_readpage_from_fscache(filp, page, desc);
return -ENOBUFS;
}
@@ -152,15 +152,14 @@ static inline int nfs_readpages_from_fscache(struct nfs_open_context *ctx,
}
/*
- * Store a page newly fetched from the server in an inode data storage object
+ * Store pages newly fetched from the server in an inode data storage object
* in the cache.
*/
-static inline void nfs_readpage_to_fscache(struct inode *inode,
- struct page *page,
- int sync)
+static inline void nfs_read_completion_to_fscache(struct nfs_pgio_header *hdr,
+ unsigned long bytes)
{
- if (PageFsCache(page))
- __nfs_readpage_to_fscache(inode, page, sync);
+ if (NFS_I(hdr->inode)->fscache)
+ __nfs_read_completion_to_fscache(hdr, bytes);
}
/*
@@ -212,9 +211,9 @@ static inline void nfs_fscache_invalidate_page(struct page *page,
static inline void nfs_fscache_wait_on_page_write(struct nfs_inode *nfsi,
struct page *page) {}
-static inline int nfs_readpage_from_fscache(struct nfs_open_context *ctx,
- struct inode *inode,
- struct page *page)
+static inline int nfs_readpage_from_fscache(struct file *filp,
+ struct page *page,
+ struct nfs_readdesc *desc)
{
return -ENOBUFS;
}
@@ -226,9 +225,8 @@ static inline int nfs_readpages_from_fscache(struct nfs_open_context *ctx,
{
return -ENOBUFS;
}
-static inline void nfs_readpage_to_fscache(struct inode *inode,
- struct page *page, int sync) {}
-
+static inline void nfs_read_completion_to_fscache(struct nfs_pgio_header *hdr,
+ unsigned long bytes) {}
static inline void nfs_fscache_invalidate(struct inode *inode) {}
static inline void nfs_fscache_wait_on_invalidate(struct inode *inode) {}
@@ -68,6 +68,7 @@ void nfs_pgheader_init(struct nfs_pageio_descriptor *desc,
hdr->good_bytes = mirror->pg_count;
hdr->io_completion = desc->pg_io_completion;
hdr->dreq = desc->pg_dreq;
+ hdr->fsc = desc->pg_fsc;
hdr->release = release;
hdr->completion_ops = desc->pg_completion_ops;
if (hdr->completion_ops->init_hdr)
@@ -849,6 +850,7 @@ void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
desc->pg_lseg = NULL;
desc->pg_io_completion = NULL;
desc->pg_dreq = NULL;
+ desc->pg_fsc = NULL;
desc->pg_bsize = bsize;
desc->pg_mirror_count = 1;
@@ -124,10 +124,11 @@ static void nfs_readpage_release(struct nfs_page *req, int error)
struct address_space *mapping = page_file_mapping(page);
if (PageUptodate(page))
- nfs_readpage_to_fscache(inode, page, 0);
+ ; /* FIXME: review fscache page error handling */
else if (!PageError(page) && !PagePrivate(page))
generic_error_remove_page(mapping, page);
- unlock_page(page);
+ if (!nfs_i_fscache(inode))
+ unlock_page(page);
}
nfs_release_request(req);
}
@@ -181,6 +182,8 @@ static void nfs_read_completion(struct nfs_pgio_header *hdr)
nfs_list_remove_request(req);
nfs_readpage_release(req, error);
}
+ /* FIXME: NFS_IOHDR_ERROR and NFS_IOHDR_EOF handled per-page */
+ nfs_read_completion_to_fscache(hdr, bytes);
out:
hdr->release(hdr);
}
@@ -359,7 +362,7 @@ int nfs_readpage(struct file *filp, struct page *page)
desc.ctx = get_nfs_open_context(nfs_file_open_context(filp));
if (!IS_SYNC(inode)) {
- ret = nfs_readpage_from_fscache(desc.ctx, inode, page);
+ ret = nfs_readpage_from_fscache(filp, page, &desc);
if (ret == 0)
goto out;
}
@@ -101,6 +101,7 @@ struct nfs_pageio_descriptor {
struct pnfs_layout_segment *pg_lseg;
struct nfs_io_completion *pg_io_completion;
struct nfs_direct_req *pg_dreq;
+ void *pg_fsc;
unsigned int pg_bsize; /* default bsize for mirrors */
u32 pg_mirror_count;
@@ -1607,6 +1607,7 @@ struct nfs_pgio_header {
const struct nfs_rw_ops *rw_ops;
struct nfs_io_completion *io_completion;
struct nfs_direct_req *dreq;
+ void *fsc;
int pnfs_error;
int error; /* merge with pnfs_error */