| Message ID | 20240326225048.785801-1-almasrymina@google.com |
|---|---|
| Headers | show |
| Series | Device Memory TCP | expand |
On Tue, Mar 26, 2024 at 3:51 PM Mina Almasry <almasrymina@google.com> wrote: > > Convert netmem to be a union of struct page and struct netmem. Overload > the LSB of struct netmem* to indicate that it's a net_iov, otherwise > it's a page. > > Currently these entries in struct page are rented by the page_pool and > used exclusively by the net stack: > > struct { > unsigned long pp_magic; > struct page_pool *pp; > unsigned long _pp_mapping_pad; > unsigned long dma_addr; > atomic_long_t pp_ref_count; > }; > > Mirror these (and only these) entries into struct net_iov and implement > netmem helpers that can access these common fields regardless of > whether the underlying type is page or net_iov. > > Implement checks for net_iov in netmem helpers which delegate to mm > APIs, to ensure net_iov are never passed to the mm stack. > > Signed-off-by: Mina Almasry <almasrymina@google.com> > > --- > > v7: > - Remove static_branch_unlikely from netmem_to_net_iov(). We're getting > better results from the fast path in bench_page_pool_simple tests > without the static_branch_unlikely, and the addition of > static_branch_unlikely doesn't improve performance of devmem TCP. > > Additionally only check netmem_to_net_iov() if > CONFIG_DMA_SHARED_BUFFER is enabled, otherwise dmabuf net_iovs cannot > exist anyway. > > net-next base: 8 cycle fast path. > with static_branch_unlikely: 10 cycle fast path. > without static_branch_unlikely: 9 cycle fast path. > CONFIG_DMA_SHARED_BUFFER disabled: 8 cycle fast path as baseline. > > Performance of devmem TCP is at 95% line rate is regardless of > static_branch_unlikely or not. > > v6: > - Rebased on top of the merged netmem_ref type. > - Rebased on top of the merged skb_pp_frag_ref() changes. > > v5: > - Use netmem instead of page* with LSB set. > - Use pp_ref_count for refcounting net_iov. > - Removed many of the custom checks for netmem. > > v1: > - Disable fragmentation support for iov properly. > - fix napi_pp_put_page() path (Yunsheng). > - Use pp_frag_count for devmem refcounting. > > To: linux-mm@kvack.org It looks like this tag to add linux-mm did not work as intended. CCing linux-mm manually. > Cc: Matthew Wilcox <willy@infradead.org> > > --- > include/net/netmem.h | 143 ++++++++++++++++++++++++++++++-- > include/net/page_pool/helpers.h | 25 +++--- > include/net/page_pool/types.h | 1 + > net/core/page_pool.c | 26 +++--- > net/core/skbuff.c | 23 +++-- > 5 files changed, 171 insertions(+), 47 deletions(-) > > diff --git a/include/net/netmem.h b/include/net/netmem.h > index 21f53b29e5fe..74eeaa34883e 100644 > --- a/include/net/netmem.h > +++ b/include/net/netmem.h > @@ -9,14 +9,51 @@ > #define _NET_NETMEM_H > > #include <net/devmem.h> > +#include <net/net_debug.h> > > /* net_iov */ > > +DECLARE_STATIC_KEY_FALSE(page_pool_mem_providers); > + > +/* We overload the LSB of the struct page pointer to indicate whether it's > + * a page or net_iov. > + */ > +#define NET_IOV 0x01UL > + > struct net_iov { > + unsigned long __unused_padding; > + unsigned long pp_magic; > + struct page_pool *pp; > struct dmabuf_genpool_chunk_owner *owner; > unsigned long dma_addr; > + atomic_long_t pp_ref_count; > }; > > +/* These fields in struct page are used by the page_pool and net stack: > + * > + * struct { > + * unsigned long pp_magic; > + * struct page_pool *pp; > + * unsigned long _pp_mapping_pad; > + * unsigned long dma_addr; > + * atomic_long_t pp_ref_count; > + * }; > + * > + * We mirror the page_pool fields here so the page_pool can access these fields > + * without worrying whether the underlying fields belong to a page or net_iov. > + * > + * The non-net stack fields of struct page are private to the mm stack and must > + * never be mirrored to net_iov. > + */ > +#define NET_IOV_ASSERT_OFFSET(pg, iov) \ > + static_assert(offsetof(struct page, pg) == \ > + offsetof(struct net_iov, iov)) > +NET_IOV_ASSERT_OFFSET(pp_magic, pp_magic); > +NET_IOV_ASSERT_OFFSET(pp, pp); > +NET_IOV_ASSERT_OFFSET(dma_addr, dma_addr); > +NET_IOV_ASSERT_OFFSET(pp_ref_count, pp_ref_count); > +#undef NET_IOV_ASSERT_OFFSET > + > static inline struct dmabuf_genpool_chunk_owner * > net_iov_owner(const struct net_iov *niov) > { > @@ -50,7 +87,7 @@ static inline dma_addr_t net_iov_dma_addr(const struct net_iov *niov) > ((dma_addr_t)net_iov_idx(niov) << PAGE_SHIFT); > } > > -static inline struct netdev_dmabuf_binding * > +static inline struct net_devmem_dmabuf_binding * > net_iov_binding(const struct net_iov *niov) > { > return net_iov_owner(niov)->binding; > @@ -69,20 +106,26 @@ net_iov_binding(const struct net_iov *niov) > */ > typedef unsigned long __bitwise netmem_ref; > > +static inline bool netmem_is_net_iov(const netmem_ref netmem) > +{ > +#if defined(CONFIG_PAGE_POOL) && defined(CONFIG_DMA_SHARED_BUFFER) > + return (__force unsigned long)netmem & NET_IOV; > +#else > + return false; > +#endif > +} > + > /* This conversion fails (returns NULL) if the netmem_ref is not struct page > * backed. > - * > - * Currently struct page is the only possible netmem, and this helper never > - * fails. > */ > static inline struct page *netmem_to_page(netmem_ref netmem) > { > + if (WARN_ON_ONCE(netmem_is_net_iov(netmem))) > + return NULL; > + > return (__force struct page *)netmem; > } > > -/* Converting from page to netmem is always safe, because a page can always be > - * a netmem. > - */ > static inline netmem_ref page_to_netmem(struct page *page) > { > return (__force netmem_ref)page; > @@ -90,17 +133,103 @@ static inline netmem_ref page_to_netmem(struct page *page) > > static inline int netmem_ref_count(netmem_ref netmem) > { > + /* The non-pp refcount of net_iov is always 1. On net_iov, we only > + * support pp refcounting which uses the pp_ref_count field. > + */ > + if (netmem_is_net_iov(netmem)) > + return 1; > + > return page_ref_count(netmem_to_page(netmem)); > } > > static inline unsigned long netmem_to_pfn(netmem_ref netmem) > { > + if (netmem_is_net_iov(netmem)) > + return 0; > + > return page_to_pfn(netmem_to_page(netmem)); > } > > +static inline struct net_iov *__netmem_clear_lsb(netmem_ref netmem) > +{ > + return (struct net_iov *)((__force unsigned long)netmem & ~NET_IOV); > +} > + > +static inline unsigned long netmem_get_pp_magic(netmem_ref netmem) > +{ > + return __netmem_clear_lsb(netmem)->pp_magic; > +} > + > +static inline void netmem_or_pp_magic(netmem_ref netmem, unsigned long pp_magic) > +{ > + __netmem_clear_lsb(netmem)->pp_magic |= pp_magic; > +} > + > +static inline void netmem_clear_pp_magic(netmem_ref netmem) > +{ > + __netmem_clear_lsb(netmem)->pp_magic = 0; > +} > + > +static inline struct page_pool *netmem_get_pp(netmem_ref netmem) > +{ > + return __netmem_clear_lsb(netmem)->pp; > +} > + > +static inline void netmem_set_pp(netmem_ref netmem, struct page_pool *pool) > +{ > + __netmem_clear_lsb(netmem)->pp = pool; > +} > + > +static inline unsigned long netmem_get_dma_addr(netmem_ref netmem) > +{ > + return __netmem_clear_lsb(netmem)->dma_addr; > +} > + > +static inline void netmem_set_dma_addr(netmem_ref netmem, > + unsigned long dma_addr) > +{ > + __netmem_clear_lsb(netmem)->dma_addr = dma_addr; > +} > + > +static inline atomic_long_t *netmem_get_pp_ref_count_ref(netmem_ref netmem) > +{ > + return &__netmem_clear_lsb(netmem)->pp_ref_count; > +} > + > +static inline bool netmem_is_pref_nid(netmem_ref netmem, int pref_nid) > +{ > + /* Assume net_iov are on the preferred node without actually > + * checking... > + * > + * This check is only used to check for recycling memory in the page > + * pool's fast paths. Currently the only implementation of net_iov > + * is dmabuf device memory. It's a deliberate decision by the user to > + * bind a certain dmabuf to a certain netdev, and the netdev rx queue > + * would not be able to reallocate memory from another dmabuf that > + * exists on the preferred node, so, this check doesn't make much sense > + * in this case. Assume all net_iovs can be recycled for now. > + */ > + if (netmem_is_net_iov(netmem)) > + return true; > + > + return page_to_nid(netmem_to_page(netmem)) == pref_nid; > +} > + > static inline netmem_ref netmem_compound_head(netmem_ref netmem) > { > + /* niov are never compounded */ > + if (netmem_is_net_iov(netmem)) > + return netmem; > + > return page_to_netmem(compound_head(netmem_to_page(netmem))); > } > > +static inline void *netmem_address(netmem_ref netmem) > +{ > + if (netmem_is_net_iov(netmem)) > + return NULL; > + > + return page_address(netmem_to_page(netmem)); > +} > + > #endif /* _NET_NETMEM_H */ > diff --git a/include/net/page_pool/helpers.h b/include/net/page_pool/helpers.h > index 61814f91a458..c6a55eddefae 100644 > --- a/include/net/page_pool/helpers.h > +++ b/include/net/page_pool/helpers.h > @@ -215,7 +215,7 @@ inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool) > > static inline void page_pool_fragment_netmem(netmem_ref netmem, long nr) > { > - atomic_long_set(&netmem_to_page(netmem)->pp_ref_count, nr); > + atomic_long_set(netmem_get_pp_ref_count_ref(netmem), nr); > } > > /** > @@ -243,7 +243,7 @@ static inline void page_pool_fragment_page(struct page *page, long nr) > > static inline long page_pool_unref_netmem(netmem_ref netmem, long nr) > { > - struct page *page = netmem_to_page(netmem); > + atomic_long_t *pp_ref_count = netmem_get_pp_ref_count_ref(netmem); > long ret; > > /* If nr == pp_ref_count then we have cleared all remaining > @@ -260,19 +260,19 @@ static inline long page_pool_unref_netmem(netmem_ref netmem, long nr) > * initially, and only overwrite it when the page is partitioned into > * more than one piece. > */ > - if (atomic_long_read(&page->pp_ref_count) == nr) { > + if (atomic_long_read(pp_ref_count) == nr) { > /* As we have ensured nr is always one for constant case using > * the BUILD_BUG_ON(), only need to handle the non-constant case > * here for pp_ref_count draining, which is a rare case. > */ > BUILD_BUG_ON(__builtin_constant_p(nr) && nr != 1); > if (!__builtin_constant_p(nr)) > - atomic_long_set(&page->pp_ref_count, 1); > + atomic_long_set(pp_ref_count, 1); > > return 0; > } > > - ret = atomic_long_sub_return(nr, &page->pp_ref_count); > + ret = atomic_long_sub_return(nr, pp_ref_count); > WARN_ON(ret < 0); > > /* We are the last user here too, reset pp_ref_count back to 1 to > @@ -281,7 +281,7 @@ static inline long page_pool_unref_netmem(netmem_ref netmem, long nr) > * page_pool_unref_page() currently. > */ > if (unlikely(!ret)) > - atomic_long_set(&page->pp_ref_count, 1); > + atomic_long_set(pp_ref_count, 1); > > return ret; > } > @@ -400,9 +400,7 @@ static inline void page_pool_free_va(struct page_pool *pool, void *va, > > static inline dma_addr_t page_pool_get_dma_addr_netmem(netmem_ref netmem) > { > - struct page *page = netmem_to_page(netmem); > - > - dma_addr_t ret = page->dma_addr; > + dma_addr_t ret = netmem_get_dma_addr(netmem); > > if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA) > ret <<= PAGE_SHIFT; > @@ -425,18 +423,17 @@ static inline dma_addr_t page_pool_get_dma_addr(struct page *page) > static inline bool page_pool_set_dma_addr_netmem(netmem_ref netmem, > dma_addr_t addr) > { > - struct page *page = netmem_to_page(netmem); > - > if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA) { > - page->dma_addr = addr >> PAGE_SHIFT; > + netmem_set_dma_addr(netmem, addr >> PAGE_SHIFT); > > /* We assume page alignment to shave off bottom bits, > * if this "compression" doesn't work we need to drop. > */ > - return addr != (dma_addr_t)page->dma_addr << PAGE_SHIFT; > + return addr != (dma_addr_t)netmem_get_dma_addr(netmem) > + << PAGE_SHIFT; > } > > - page->dma_addr = addr; > + netmem_set_dma_addr(netmem, addr); > return false; > } > > diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h > index 0d164624f16d..f04af1613f59 100644 > --- a/include/net/page_pool/types.h > +++ b/include/net/page_pool/types.h > @@ -6,6 +6,7 @@ > #include <linux/dma-direction.h> > #include <linux/ptr_ring.h> > #include <linux/types.h> > +#include <net/netmem.h> > > #define PP_FLAG_DMA_MAP BIT(0) /* Should page_pool do the DMA > * map/unmap > diff --git a/net/core/page_pool.c b/net/core/page_pool.c > index c8125be3a6e2..c7bffd08218b 100644 > --- a/net/core/page_pool.c > +++ b/net/core/page_pool.c > @@ -25,7 +25,7 @@ > > #include "page_pool_priv.h" > > -static DEFINE_STATIC_KEY_FALSE(page_pool_mem_providers); > +DEFINE_STATIC_KEY_FALSE(page_pool_mem_providers); > > #define DEFER_TIME (msecs_to_jiffies(1000)) > #define DEFER_WARN_INTERVAL (60 * HZ) > @@ -359,7 +359,7 @@ static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool) > if (unlikely(!netmem)) > break; > > - if (likely(page_to_nid(netmem_to_page(netmem)) == pref_nid)) { > + if (likely(netmem_is_pref_nid(netmem, pref_nid))) { > pool->alloc.cache[pool->alloc.count++] = netmem; > } else { > /* NUMA mismatch; > @@ -446,10 +446,8 @@ static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem) > > static void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem) > { > - struct page *page = netmem_to_page(netmem); > - > - page->pp = pool; > - page->pp_magic |= PP_SIGNATURE; > + netmem_set_pp(netmem, pool); > + netmem_or_pp_magic(netmem, PP_SIGNATURE); > > /* Ensuring all pages have been split into one fragment initially: > * page_pool_set_pp_info() is only called once for every page when it > @@ -464,10 +462,8 @@ static void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem) > > static void page_pool_clear_pp_info(netmem_ref netmem) > { > - struct page *page = netmem_to_page(netmem); > - > - page->pp_magic = 0; > - page->pp = NULL; > + netmem_clear_pp_magic(netmem); > + netmem_set_pp(netmem, NULL); > } > > static struct page *__page_pool_alloc_page_order(struct page_pool *pool, > @@ -695,8 +691,9 @@ static bool page_pool_recycle_in_cache(netmem_ref netmem, > > static bool __page_pool_page_can_be_recycled(netmem_ref netmem) > { > - return page_ref_count(netmem_to_page(netmem)) == 1 && > - !page_is_pfmemalloc(netmem_to_page(netmem)); > + return netmem_is_net_iov(netmem) || > + (page_ref_count(netmem_to_page(netmem)) == 1 && > + !page_is_pfmemalloc(netmem_to_page(netmem))); > } > > /* If the page refcnt == 1, this will try to recycle the page. > @@ -718,7 +715,7 @@ __page_pool_put_page(struct page_pool *pool, netmem_ref netmem, > * refcnt == 1 means page_pool owns page, and can recycle it. > * > * page is NOT reusable when allocated when system is under > - * some pressure. (page_is_pfmemalloc) > + * some pressure. (page_pool_page_is_pfmemalloc) > */ > if (likely(__page_pool_page_can_be_recycled(netmem))) { > /* Read barrier done in page_ref_count / READ_ONCE */ > @@ -734,6 +731,7 @@ __page_pool_put_page(struct page_pool *pool, netmem_ref netmem, > /* Page found as candidate for recycling */ > return netmem; > } > + > /* Fallback/non-XDP mode: API user have elevated refcnt. > * > * Many drivers split up the page into fragments, and some > @@ -928,7 +926,7 @@ static void page_pool_empty_ring(struct page_pool *pool) > /* Empty recycle ring */ > while ((netmem = (__force netmem_ref)ptr_ring_consume_bh(&pool->ring))) { > /* Verify the refcnt invariant of cached pages */ > - if (!(page_ref_count(netmem_to_page(netmem)) == 1)) > + if (!(netmem_ref_count(netmem) == 1)) > pr_crit("%s() page_pool refcnt %d violation\n", > __func__, netmem_ref_count(netmem)); > > diff --git a/net/core/skbuff.c b/net/core/skbuff.c > index 7193ee9737a0..b7e974f0ae51 100644 > --- a/net/core/skbuff.c > +++ b/net/core/skbuff.c > @@ -907,9 +907,9 @@ static void skb_clone_fraglist(struct sk_buff *skb) > skb_get(list); > } > > -static bool is_pp_page(struct page *page) > +static bool is_pp_netmem(netmem_ref netmem) > { > - return (page->pp_magic & ~0x3UL) == PP_SIGNATURE; > + return (netmem_get_pp_magic(netmem) & ~0x3UL) == PP_SIGNATURE; > } > > int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, > @@ -1007,11 +1007,10 @@ EXPORT_SYMBOL(skb_cow_data_for_xdp); > #if IS_ENABLED(CONFIG_PAGE_POOL) > bool napi_pp_put_page(netmem_ref netmem, bool napi_safe) > { > - struct page *page = netmem_to_page(netmem); > bool allow_direct = false; > struct page_pool *pp; > > - page = compound_head(page); > + netmem = netmem_compound_head(netmem); > > /* page->pp_magic is OR'ed with PP_SIGNATURE after the allocation > * in order to preserve any existing bits, such as bit 0 for the > @@ -1020,10 +1019,10 @@ bool napi_pp_put_page(netmem_ref netmem, bool napi_safe) > * and page_is_pfmemalloc() is checked in __page_pool_put_page() > * to avoid recycling the pfmemalloc page. > */ > - if (unlikely(!is_pp_page(page))) > + if (unlikely(!is_pp_netmem(netmem))) > return false; > > - pp = page->pp; > + pp = netmem_get_pp(netmem); > > /* Allow direct recycle if we have reasons to believe that we are > * in the same context as the consumer would run, so there's > @@ -1044,7 +1043,7 @@ bool napi_pp_put_page(netmem_ref netmem, bool napi_safe) > * The page will be returned to the pool here regardless of the > * 'flipped' fragment being in use or not. > */ > - page_pool_put_full_netmem(pp, page_to_netmem(page), allow_direct); > + page_pool_put_full_netmem(pp, netmem, allow_direct); > > return true; > } > @@ -1071,7 +1070,7 @@ static bool skb_pp_recycle(struct sk_buff *skb, void *data, bool napi_safe) > static int skb_pp_frag_ref(struct sk_buff *skb) > { > struct skb_shared_info *shinfo; > - struct page *head_page; > + netmem_ref head_netmem; > int i; > > if (!skb->pp_recycle) > @@ -1080,11 +1079,11 @@ static int skb_pp_frag_ref(struct sk_buff *skb) > shinfo = skb_shinfo(skb); > > for (i = 0; i < shinfo->nr_frags; i++) { > - head_page = compound_head(skb_frag_page(&shinfo->frags[i])); > - if (likely(is_pp_page(head_page))) > - page_pool_ref_page(head_page); > + head_netmem = netmem_compound_head(shinfo->frags[i].netmem); > + if (likely(is_pp_netmem(head_netmem))) > + page_pool_ref_netmem(head_netmem); > else > - page_ref_inc(head_page); > + page_ref_inc(netmem_to_page(head_netmem)); > } > return 0; > } > -- > 2.44.0.396.g6e790dbe36-goog >
RFC v7: ======= Major Changes: -------------- This revision largely rebases on top of net-next and addresses the feedback RFCv6 received from folks, namely Jakub, Yunsheng, Arnd, David, & Pavel. The series remains in RFC because the queue-API ndos defined in this series are not yet implemented. I have a GVE implementation I carry out of tree for my testing. A upstreamable GVE implementation is in the works. Aside from that, in my estimation all the patches are ready for review/merge. Please do take a look. As usual the full devmem TCP changes including the full GVE driver implementation is here: https://github.com/mina/linux/commits/tcpdevmem-v7/ Detailed changelog: - Use admin-perm in netlink API. - Addressed feedback from Jakub with regards to netlink API implementation. - Renamed devmem.c functions to something more appropriate for that file. - Improve the performance seen through the page_pool benchmark. - Fix the value definition of all the SO_DEVMEM_* uapi. - Various fixes to documentation. Perf - page-pool benchmark: --------------------------- Improved performance of bench_page_pool_simple.ko tests compared to v6: https://pastebin.com/raw/v5dYRg8L net-next base: 8 cycle fast path. RFC v6: 10 cycle fast path. RFC v7: 9 cycle fast path. RFC v7 with CONFIG_DMA_SHARED_BUFFER disabled: 8 cycle fast path, same as baseline. Perf - Devmem TCP benchmark: --------------------- Perf is about the same regardless of the changes in v7, namely the removal of the static_branch_unlikely to improve the page_pool benchmark performance: 189/200gbps bi-directional throughput with RX devmem TCP and regular TCP TX i.e. ~95% line rate. RFC v6: ======= Major Changes: -------------- This revision largely rebases on top of net-next and addresses the little feedback RFCv5 received. The series remains in RFC because the queue-API ndos defined in this series are not yet implemented. I have a GVE implementation I carry out of tree for my testing. A upstreamable GVE implementation is in the works. Aside from that, in my estimation all the patches are ready for review/merge. Please do take a look. As usual the full devmem TCP changes including the full GVE driver implementation is here: https://github.com/mina/linux/commits/tcpdevmem-v6/ This version also comes with some performance data recorded in the cover letter (see below changelog). Detailed changelog: - Rebased on top of the merged netmem_ref changes. - Converted skb->dmabuf to skb->readable (Pavel). Pavel's original suggestion was to remove the skb->dmabuf flag entirely, but when I looked into it closely, I found the issue that if we remove the flag we have to dereference the shinfo(skb) pointer to obtain the first frag to tell whether an skb is readable or not. This can cause a performance regression if it dirties the cache line when the shinfo(skb) was not really needed. Instead, I converted the skb->dmabuf flag into a generic skb->readable flag which can be re-used by io_uring 0-copy RX. - Squashed a few locking optimizations from Eric Dumazet in the RX path and the DEVMEM_DONTNEED setsockopt. - Expanded the tests a bit. Added validation for invalid scenarios and added some more coverage. Perf - page-pool benchmark: --------------------------- bench_page_pool_simple.ko tests with and without these changes: https://pastebin.com/raw/ncHDwAbn AFAIK the number that really matters in the perf tests is the 'tasklet_page_pool01_fast_path Per elem'. This one measures at about 8 cycles without the changes but there is some 1 cycle noise in some results. With the patches this regresses to 9 cycles with the changes but there is 1 cycle noise occasionally running this test repeatedly. Lastly I tried disable the static_branch_unlikely() in netmem_is_net_iov() check. To my surprise disabling the static_branch_unlikely() check reduces the fast path back to 8 cycles, but the 1 cycle noise remains. Perf - Devmem TCP benchmark: --------------------- 189/200gbps bi-directional throughput with RX devmem TCP and regular TCP TX i.e. ~95% line rate. Major changes in RFC v5: ======================== 1. Rebased on top of 'Abstract page from net stack' series and used the new netmem type to refer to LSB set pointers instead of re-using struct page. 2. Downgraded this series back to RFC and called it RFC v5. This is because this series is now dependent on 'Abstract page from net stack'[1] and the queue API. Both are removed from the series to reduce the patch # and those bits are fairly independent or pre-requisite work. 3. Reworked the page_pool devmem support to use netmem and for some more unified handling. 4. Reworked the reference counting of net_iov (renamed from page_pool_iov) to use pp_ref_count for refcounting. The full changes including the dependent series and GVE page pool support is here: https://github.com/mina/linux/commits/tcpdevmem-rfcv5/ [1] https://patchwork.kernel.org/project/netdevbpf/list/?series=810774 Major changes in v1: ==================== 1. Implemented MVP queue API ndos to remove the userspace-visible driver reset. 2. Fixed issues in the napi_pp_put_page() devmem frag unref path. 3. Removed RFC tag. Many smaller addressed comments across all the patches (patches have individual change log). Full tree including the rest of the GVE driver changes: https://github.com/mina/linux/commits/tcpdevmem-v1 Changes in RFC v3: ================== 1. Pulled in the memory-provider dependency from Jakub's RFC[1] to make the series reviewable and mergeable. 2. Implemented multi-rx-queue binding which was a todo in v2. 3. Fix to cmsg handling. The sticking point in RFC v2[2] was the device reset required to refill the device rx-queues after the dmabuf bind/unbind. The solution suggested as I understand is a subset of the per-queue management ops Jakub suggested or similar: https://lore.kernel.org/netdev/20230815171638.4c057dcd@kernel.org/ This is not addressed in this revision, because: 1. This point was discussed at netconf & netdev and there is openness to using the current approach of requiring a device reset. 2. Implementing individual queue resetting seems to be difficult for my test bed with GVE. My prototype to test this ran into issues with the rx-queues not coming back up properly if reset individually. At the moment I'm unsure if it's a mistake in the POC or a genuine issue in the virtualization stack behind GVE, which currently doesn't test individual rx-queue restart. 3. Our usecases are not bothered by requiring a device reset to refill the buffer queues, and we'd like to support NICs that run into this limitation with resetting individual queues. My thought is that drivers that have trouble with per-queue configs can use the support in this series, while drivers that support new netdev ops to reset individual queues can automatically reset the queue as part of the dma-buf bind/unbind. The same approach with device resets is presented again for consideration with other sticking points addressed. This proposal includes the rx devmem path only proposed for merge. For a snapshot of my entire tree which includes the GVE POC page pool support & device memory support: https://github.com/torvalds/linux/compare/master...mina:linux:tcpdevmem-v3 [1] https://lore.kernel.org/netdev/f8270765-a27b-6ccf-33ea-cda097168d79@redhat.com/T/ [2] https://lore.kernel.org/netdev/CAHS8izOVJGJH5WF68OsRWFKJid1_huzzUK+hpKbLcL4pSOD1Jw@mail.gmail.com/T/ Changes in RFC v2: ================== The sticking point in RFC v1[1] was the dma-buf pages approach we used to deliver the device memory to the TCP stack. RFC v2 is a proof-of-concept that attempts to resolve this by implementing scatterlist support in the networking stack, such that we can import the dma-buf scatterlist directly. This is the approach proposed at a high level here[2]. Detailed changes: 1. Replaced dma-buf pages approach with importing scatterlist into the page pool. 2. Replace the dma-buf pages centric API with a netlink API. 3. Removed the TX path implementation - there is no issue with implementing the TX path with scatterlist approach, but leaving out the TX path makes it easier to review. 4. Functionality is tested with this proposal, but I have not conducted perf testing yet. I'm not sure there are regressions, but I removed perf claims from the cover letter until they can be re-confirmed. 5. Added Signed-off-by: contributors to the implementation. 6. Fixed some bugs with the RX path since RFC v1. Any feedback welcome, but specifically the biggest pending questions needing feedback IMO are: 1. Feedback on the scatterlist-based approach in general. 2. Netlink API (Patch 1 & 2). 3. Approach to handle all the drivers that expect to receive pages from the page pool (Patch 6). [1] https://lore.kernel.org/netdev/dfe4bae7-13a0-3c5d-d671-f61b375cb0b4@gmail.com/T/ [2] https://lore.kernel.org/netdev/CAHS8izPm6XRS54LdCDZVd0C75tA1zHSu6jLVO8nzTLXCc=H7Nw@mail.gmail.com/ ================== * TL;DR: Device memory TCP (devmem TCP) is a proposal for transferring data to and/or from device memory efficiently, without bouncing the data to a host memory buffer. * Problem: A large amount of data transfers have device memory as the source and/or destination. Accelerators drastically increased the volume of such transfers. Some examples include: - ML accelerators transferring large amounts of training data from storage into GPU/TPU memory. In some cases ML training setup time can be as long as 50% of TPU compute time, improving data transfer throughput & efficiency can help improving GPU/TPU utilization. - Distributed training, where ML accelerators, such as GPUs on different hosts, exchange data among them. - Distributed raw block storage applications transfer large amounts of data with remote SSDs, much of this data does not require host processing. Today, the majority of the Device-to-Device data transfers the network are implemented as the following low level operations: Device-to-Host copy, Host-to-Host network transfer, and Host-to-Device copy. The implementation is suboptimal, especially for bulk data transfers, and can put significant strains on system resources, such as host memory bandwidth, PCIe bandwidth, etc. One important reason behind the current state is the kernel’s lack of semantics to express device to network transfers. * Proposal: In this patch series we attempt to optimize this use case by implementing socket APIs that enable the user to: 1. send device memory across the network directly, and 2. receive incoming network packets directly into device memory. Packet _payloads_ go directly from the NIC to device memory for receive and from device memory to NIC for transmit. Packet _headers_ go to/from host memory and are processed by the TCP/IP stack normally. The NIC _must_ support header split to achieve this. Advantages: - Alleviate host memory bandwidth pressure, compared to existing network-transfer + device-copy semantics. - Alleviate PCIe BW pressure, by limiting data transfer to the lowest level of the PCIe tree, compared to traditional path which sends data through the root complex. * Patch overview: ** Part 1: netlink API Gives user ability to bind dma-buf to an RX queue. ** Part 2: scatterlist support Currently the standard for device memory sharing is DMABUF, which doesn't generate struct pages. On the other hand, networking stack (skbs, drivers, and page pool) operate on pages. We have 2 options: 1. Generate struct pages for dmabuf device memory, or, 2. Modify the networking stack to process scatterlist. Approach #1 was attempted in RFC v1. RFC v2 implements approach #2. ** part 3: page pool support We piggy back on page pool memory providers proposal: https://github.com/kuba-moo/linux/tree/pp-providers It allows the page pool to define a memory provider that provides the page allocation and freeing. It helps abstract most of the device memory TCP changes from the driver. ** part 4: support for unreadable skb frags Page pool iovs are not accessible by the host; we implement changes throughput the networking stack to correctly handle skbs with unreadable frags. ** Part 5: recvmsg() APIs We define user APIs for the user to send and receive device memory. Not included with this series is the GVE devmem TCP support, just to simplify the review. Code available here if desired: https://github.com/mina/linux/tree/tcpdevmem This series is built on top of net-next with Jakub's pp-providers changes cherry-picked. * NIC dependencies: 1. (strict) Devmem TCP require the NIC to support header split, i.e. the capability to split incoming packets into a header + payload and to put each into a separate buffer. Devmem TCP works by using device memory for the packet payload, and host memory for the packet headers. 2. (optional) Devmem TCP works better with flow steering support & RSS support, i.e. the NIC's ability to steer flows into certain rx queues. This allows the sysadmin to enable devmem TCP on a subset of the rx queues, and steer devmem TCP traffic onto these queues and non devmem TCP elsewhere. The NIC I have access to with these properties is the GVE with DQO support running in Google Cloud, but any NIC that supports these features would suffice. I may be able to help reviewers bring up devmem TCP on their NICs. * Testing: The series includes a udmabuf kselftest that show a simple use case of devmem TCP and validates the entire data path end to end without a dependency on a specific dmabuf provider. ** Test Setup Kernel: net-next with this series and memory provider API cherry-picked locally. Hardware: Google Cloud A3 VMs. NIC: GVE with header split & RSS & flow steering support. Cc: Pavel Begunkov <asml.silence@gmail.com> Cc: David Wei <dw@davidwei.uk> Cc: Jason Gunthorpe <jgg@ziepe.ca> Cc: Yunsheng Lin <linyunsheng@huawei.com> Cc: Shailend Chand <shailend@google.com> Cc: Harshitha Ramamurthy <hramamurthy@google.com> Cc: Shakeel Butt <shakeel.butt@linux.dev> Cc: Jeroen de Borst <jeroendb@google.com> Cc: Praveen Kaligineedi <pkaligineedi@google.com> Jakub Kicinski (1): net: page_pool: create hooks for custom page providers Mina Almasry (13): queue_api: define queue api net: netdev netlink api to bind dma-buf to a net device netdev: support binding dma-buf to netdevice netdev: netdevice devmem allocator page_pool: convert to use netmem page_pool: devmem support memory-provider: dmabuf devmem memory provider net: support non paged skb frags net: add support for skbs with unreadable frags tcp: RX path for devmem TCP net: add SO_DEVMEM_DONTNEED setsockopt to release RX frags net: add devmem TCP documentation selftests: add ncdevmem, netcat for devmem TCP Documentation/netlink/specs/netdev.yaml | 57 +++ Documentation/networking/devmem.rst | 256 +++++++++++ Documentation/networking/index.rst | 1 + arch/alpha/include/uapi/asm/socket.h | 6 + arch/mips/include/uapi/asm/socket.h | 6 + arch/parisc/include/uapi/asm/socket.h | 6 + arch/sparc/include/uapi/asm/socket.h | 6 + include/linux/netdevice.h | 3 + include/linux/skbuff.h | 73 +++- include/linux/socket.h | 1 + include/net/devmem.h | 124 ++++++ include/net/netdev_queues.h | 27 ++ include/net/netdev_rx_queue.h | 2 + include/net/netmem.h | 234 +++++++++- include/net/page_pool/helpers.h | 155 +++++-- include/net/page_pool/types.h | 33 +- include/net/sock.h | 2 + include/net/tcp.h | 5 +- include/trace/events/page_pool.h | 29 +- include/uapi/asm-generic/socket.h | 6 + include/uapi/linux/netdev.h | 19 + include/uapi/linux/uio.h | 17 + net/bpf/test_run.c | 5 +- net/core/Makefile | 2 +- net/core/datagram.c | 6 + net/core/dev.c | 6 +- net/core/devmem.c | 425 ++++++++++++++++++ net/core/gro.c | 8 +- net/core/netdev-genl-gen.c | 23 + net/core/netdev-genl-gen.h | 6 + net/core/netdev-genl.c | 107 +++++ net/core/page_pool.c | 364 +++++++++------- net/core/skbuff.c | 110 ++++- net/core/sock.c | 61 +++ net/ipv4/esp4.c | 2 +- net/ipv4/tcp.c | 254 ++++++++++- net/ipv4/tcp_input.c | 13 +- net/ipv4/tcp_ipv4.c | 9 + net/ipv4/tcp_minisocks.c | 2 + net/ipv4/tcp_output.c | 5 +- net/ipv6/esp6.c | 2 +- net/packet/af_packet.c | 4 +- tools/include/uapi/linux/netdev.h | 19 + tools/testing/selftests/net/.gitignore | 1 + tools/testing/selftests/net/Makefile | 5 + tools/testing/selftests/net/ncdevmem.c | 546 ++++++++++++++++++++++++ 46 files changed, 2776 insertions(+), 277 deletions(-) create mode 100644 Documentation/networking/devmem.rst create mode 100644 include/net/devmem.h create mode 100644 net/core/devmem.c create mode 100644 tools/testing/selftests/net/ncdevmem.c