From patchwork Tue Jan 19 20:20:09 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lorenzo Bianconi X-Patchwork-Id: 366704 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-19.2 required=3.0 tests=BAYES_00,DKIMWL_WL_HIGH, DKIM_SIGNED, DKIM_VALID, DKIM_VALID_AU, INCLUDES_CR_TRAILER, INCLUDES_PATCH, MAILING_LIST_MULTI, SPF_HELO_NONE, SPF_PASS, USER_AGENT_GIT autolearn=ham autolearn_force=no version=3.4.0 Received: from mail.kernel.org (mail.kernel.org [198.145.29.99]) by smtp.lore.kernel.org (Postfix) with ESMTP id 01695C433DB for ; Tue, 19 Jan 2021 20:21:58 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id 9CE5B23104 for ; Tue, 19 Jan 2021 20:21:57 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726208AbhASUVz (ORCPT ); Tue, 19 Jan 2021 15:21:55 -0500 Received: from mail.kernel.org ([198.145.29.99]:55994 "EHLO mail.kernel.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1730126AbhASUVZ (ORCPT ); Tue, 19 Jan 2021 15:21:25 -0500 Received: by mail.kernel.org (Postfix) with ESMTPSA id 777AB2310D; Tue, 19 Jan 2021 20:20:39 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1611087642; bh=+08pvCIfgsqYJ8S06q2qM20qFEOiUxlYPFCxQCxJDY0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=CpvLLBnPEJYz5s+nRgj2Y/tdzW3/dSHqIkjQ37V4Kf4zXW754O9tD/NfezCMtBNYc RSCT6n5VDmHHnxQXt/oLAaL+zoB0IsQymaPO5i7JbJmJ8O3fdFEu/icR2+yo+Udwha RWs/SL8xmbCJJrvZ5QSXdtr4cfvcGW5rlmIuN5BOGqloA4bZ0oKpw453Mc9ldN8k6c ilPWmSvT4sQcu7vAtnm4jtGxWHJl3z+4mq+Im6obEzHK9pc4PTElOcSpKZO3H/9PLL p72VJd9tWjTmRK5/mq0+EmQpt9u6c61J7GkyV541xyqyM9p2CzcH0xpbKN9YgNtedB 2gg2rwghSC+Sw== From: Lorenzo Bianconi To: bpf@vger.kernel.org, netdev@vger.kernel.org Cc: lorenzo.bianconi@redhat.com, davem@davemloft.net, kuba@kernel.org, ast@kernel.org, daniel@iogearbox.net, shayagr@amazon.com, john.fastabend@gmail.com, dsahern@kernel.org, brouer@redhat.com, echaudro@redhat.com, jasowang@redhat.com, alexander.duyck@gmail.com, saeed@kernel.org, maciej.fijalkowski@intel.com, sameehj@amazon.com Subject: [PATCH v6 bpf-next 3/8] net: mvneta: update mb bit before passing the xdp buffer to eBPF layer Date: Tue, 19 Jan 2021 21:20:09 +0100 Message-Id: X-Mailer: git-send-email 2.29.2 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: netdev@vger.kernel.org Update multi-buffer bit (mb) in xdp_buff to notify XDP/eBPF layer and XDP remote drivers if this is a "non-linear" XDP buffer. Access xdp_shared_info only if xdp_buff mb is set. Signed-off-by: Lorenzo Bianconi --- drivers/net/ethernet/marvell/mvneta.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index ed986ffff96b..3df0602239ad 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -2038,12 +2038,16 @@ mvneta_xdp_put_buff(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, { int i; + if (likely(!xdp->mb)) + goto out; + for (i = 0; i < xdp_sinfo->nr_frags; i++) { skb_frag_t *frag = &xdp_sinfo->frags[i]; page_pool_put_full_page(rxq->page_pool, xdp_get_frag_page(frag), true); } +out: page_pool_put_page(rxq->page_pool, virt_to_head_page(xdp->data), sync_len, true); } @@ -2244,7 +2248,6 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, { unsigned char *data = page_address(page); int data_len = -MVNETA_MH_SIZE, len; - struct xdp_shared_info *xdp_sinfo; struct net_device *dev = pp->dev; enum dma_data_direction dma_dir; @@ -2268,9 +2271,6 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, prefetch(data); xdp_prepare_buff(xdp, data, pp->rx_offset_correction + MVNETA_MH_SIZE, data_len, false); - - xdp_sinfo = xdp_get_shared_info_from_buff(xdp); - xdp_sinfo->nr_frags = 0; } static void @@ -2305,12 +2305,18 @@ mvneta_swbm_add_rx_fragment(struct mvneta_port *pp, xdp_set_frag_size(frag, data_len); xdp_set_frag_page(frag, page); + if (!xdp->mb) { + xdp_sinfo->data_length = *size; + xdp->mb = 1; + } /* last fragment */ if (len == *size) { struct xdp_shared_info *sinfo; sinfo = xdp_get_shared_info_from_buff(xdp); sinfo->nr_frags = xdp_sinfo->nr_frags; + sinfo->data_length = xdp_sinfo->data_length; + memcpy(sinfo->frags, xdp_sinfo->frags, sinfo->nr_frags * sizeof(skb_frag_t)); } @@ -2325,11 +2331,15 @@ mvneta_swbm_build_skb(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, struct xdp_buff *xdp, u32 desc_status) { struct xdp_shared_info *xdp_sinfo = xdp_get_shared_info_from_buff(xdp); - int i, num_frags = xdp_sinfo->nr_frags; skb_frag_t frag_list[MAX_SKB_FRAGS]; + int i, num_frags = 0; struct sk_buff *skb; - memcpy(frag_list, xdp_sinfo->frags, sizeof(skb_frag_t) * num_frags); + if (unlikely(xdp->mb)) { + num_frags = xdp_sinfo->nr_frags; + memcpy(frag_list, xdp_sinfo->frags, + sizeof(skb_frag_t) * num_frags); + } skb = build_skb(xdp->data_hard_start, PAGE_SIZE); if (!skb) @@ -2341,6 +2351,9 @@ mvneta_swbm_build_skb(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, skb_put(skb, xdp->data_end - xdp->data); mvneta_rx_csum(pp, desc_status, skb); + if (likely(!xdp->mb)) + return skb; + for (i = 0; i < num_frags; i++) { struct page *page = xdp_get_frag_page(&frag_list[i]); @@ -2402,6 +2415,7 @@ static int mvneta_rx_swbm(struct napi_struct *napi, frame_sz = size - ETH_FCS_LEN; desc_status = rx_status; + xdp_buf.mb = 0; mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf, &size, page); } else { From patchwork Tue Jan 19 20:20:10 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lorenzo Bianconi X-Patchwork-Id: 366702 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-19.2 required=3.0 tests=BAYES_00,DKIMWL_WL_HIGH, DKIM_SIGNED, DKIM_VALID, DKIM_VALID_AU, INCLUDES_CR_TRAILER, INCLUDES_PATCH, MAILING_LIST_MULTI,SPF_HELO_NONE,SPF_PASS,USER_AGENT_GIT autolearn=unavailable autolearn_force=no version=3.4.0 Received: from mail.kernel.org (mail.kernel.org [198.145.29.99]) by smtp.lore.kernel.org (Postfix) with ESMTP id 54C01C433DB for ; Tue, 19 Jan 2021 20:24:21 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id DE3002313A for ; Tue, 19 Jan 2021 20:24:20 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S2390206AbhASUYP (ORCPT ); Tue, 19 Jan 2021 15:24:15 -0500 Received: from mail.kernel.org ([198.145.29.99]:56206 "EHLO mail.kernel.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1729879AbhASUWQ (ORCPT ); Tue, 19 Jan 2021 15:22:16 -0500 Received: by mail.kernel.org (Postfix) with ESMTPSA id CF79323119; Tue, 19 Jan 2021 20:20:43 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1611087647; bh=xpdKrm8ZcTc0HwbnQvHlKBZmoqejubUtzFtDYJ82BdU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=a0+hSu4mV3DvlS6bIAyIR1RCM4FyszhhkgkeFoTIluuTPzz3HxNWKKaFFKNK4r7hm 8IN7vnHSVyoYyygVHvtctxgqHtDM2Fn1kqfXsTHvCN2Eg4QDwE8MSixhK56zZI6UcR iCDQ9jEmbEs3vqLIyB4Od2IL1Oh31splpI0ifV1IbbVSKHtcCOE7M+exAPLH63NsmM 1W7/kFI34pdhLC3m6TuC8MNOmCXZ9QcuZwbrjyQKCgtwdlYZtrVjX0IK1KUO6+MSNL VAtnU859OPvW6V9TnBby1Esn59C5NEM1TnFMX8+NJzpbX+tfwysY7PMoUaUEMmZnY4 YzXor2U3EVzig== From: Lorenzo Bianconi To: bpf@vger.kernel.org, netdev@vger.kernel.org Cc: lorenzo.bianconi@redhat.com, davem@davemloft.net, kuba@kernel.org, ast@kernel.org, daniel@iogearbox.net, shayagr@amazon.com, john.fastabend@gmail.com, dsahern@kernel.org, brouer@redhat.com, echaudro@redhat.com, jasowang@redhat.com, alexander.duyck@gmail.com, saeed@kernel.org, maciej.fijalkowski@intel.com, sameehj@amazon.com Subject: [PATCH v6 bpf-next 4/8] xdp: add multi-buff support to xdp_return_{buff/frame} Date: Tue, 19 Jan 2021 21:20:10 +0100 Message-Id: <1542d03761990429276d72fcb94f7e91d63e7e1f.1611086134.git.lorenzo@kernel.org> X-Mailer: git-send-email 2.29.2 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: netdev@vger.kernel.org Take into account if the received xdp_buff/xdp_frame is non-linear recycling/returning the frame memory to the allocator or into xdp_frame_bulk. Introduce xdp_return_num_frags_from_buff to return a given number of fragments from a xdp multi-buff starting from the tail. Signed-off-by: Lorenzo Bianconi --- include/net/xdp.h | 19 ++++++++++-- net/core/xdp.c | 76 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/include/net/xdp.h b/include/net/xdp.h index ca7c4fb30b4e..a2e09031b346 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -286,6 +286,7 @@ void xdp_return_buff(struct xdp_buff *xdp); void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq); void xdp_return_frame_bulk(struct xdp_frame *xdpf, struct xdp_frame_bulk *bq); +void xdp_return_num_frags_from_buff(struct xdp_buff *xdp, u16 num_frags); /* When sending xdp_frame into the network stack, then there is no * return point callback, which is needed to release e.g. DMA-mapping @@ -296,10 +297,24 @@ void __xdp_release_frame(void *data, struct xdp_mem_info *mem); static inline void xdp_release_frame(struct xdp_frame *xdpf) { struct xdp_mem_info *mem = &xdpf->mem; + struct xdp_shared_info *xdp_sinfo; + int i; /* Curr only page_pool needs this */ - if (mem->type == MEM_TYPE_PAGE_POOL) - __xdp_release_frame(xdpf->data, mem); + if (mem->type != MEM_TYPE_PAGE_POOL) + return; + + if (likely(!xdpf->mb)) + goto out; + + xdp_sinfo = xdp_get_shared_info_from_frame(xdpf); + for (i = 0; i < xdp_sinfo->nr_frags; i++) { + struct page *page = xdp_get_frag_page(&xdp_sinfo->frags[i]); + + __xdp_release_frame(page_address(page), mem); + } +out: + __xdp_release_frame(xdpf->data, mem); } int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, diff --git a/net/core/xdp.c b/net/core/xdp.c index 0d2630a35c3e..af9b9c1194fc 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -374,12 +374,38 @@ static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct, void xdp_return_frame(struct xdp_frame *xdpf) { + struct xdp_shared_info *xdp_sinfo; + int i; + + if (likely(!xdpf->mb)) + goto out; + + xdp_sinfo = xdp_get_shared_info_from_frame(xdpf); + for (i = 0; i < xdp_sinfo->nr_frags; i++) { + struct page *page = xdp_get_frag_page(&xdp_sinfo->frags[i]); + + __xdp_return(page_address(page), &xdpf->mem, false, NULL); + } +out: __xdp_return(xdpf->data, &xdpf->mem, false, NULL); } EXPORT_SYMBOL_GPL(xdp_return_frame); void xdp_return_frame_rx_napi(struct xdp_frame *xdpf) { + struct xdp_shared_info *xdp_sinfo; + int i; + + if (likely(!xdpf->mb)) + goto out; + + xdp_sinfo = xdp_get_shared_info_from_frame(xdpf); + for (i = 0; i < xdp_sinfo->nr_frags; i++) { + struct page *page = xdp_get_frag_page(&xdp_sinfo->frags[i]); + + __xdp_return(page_address(page), &xdpf->mem, true, NULL); + } +out: __xdp_return(xdpf->data, &xdpf->mem, true, NULL); } EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi); @@ -415,7 +441,7 @@ void xdp_return_frame_bulk(struct xdp_frame *xdpf, struct xdp_mem_allocator *xa; if (mem->type != MEM_TYPE_PAGE_POOL) { - __xdp_return(xdpf->data, &xdpf->mem, false, NULL); + xdp_return_frame(xdpf); return; } @@ -434,15 +460,63 @@ void xdp_return_frame_bulk(struct xdp_frame *xdpf, bq->xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params); } + if (unlikely(xdpf->mb)) { + struct xdp_shared_info *xdp_sinfo; + int i; + + xdp_sinfo = xdp_get_shared_info_from_frame(xdpf); + for (i = 0; i < xdp_sinfo->nr_frags; i++) { + skb_frag_t *frag = &xdp_sinfo->frags[i]; + + bq->q[bq->count++] = xdp_get_frag_address(frag); + if (bq->count == XDP_BULK_QUEUE_SIZE) + xdp_flush_frame_bulk(bq); + } + } bq->q[bq->count++] = xdpf->data; } EXPORT_SYMBOL_GPL(xdp_return_frame_bulk); void xdp_return_buff(struct xdp_buff *xdp) { + struct xdp_shared_info *xdp_sinfo; + int i; + + if (likely(!xdp->mb)) + goto out; + + xdp_sinfo = xdp_get_shared_info_from_buff(xdp); + for (i = 0; i < xdp_sinfo->nr_frags; i++) { + struct page *page = xdp_get_frag_page(&xdp_sinfo->frags[i]); + + __xdp_return(page_address(page), &xdp->rxq->mem, true, xdp); + } +out: __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp); } +void xdp_return_num_frags_from_buff(struct xdp_buff *xdp, u16 num_frags) +{ + struct xdp_shared_info *xdp_sinfo; + int i; + + if (unlikely(!xdp->mb)) + return; + + xdp_sinfo = xdp_get_shared_info_from_buff(xdp); + num_frags = min_t(u16, num_frags, xdp_sinfo->nr_frags); + for (i = 1; i <= num_frags; i++) { + skb_frag_t *frag = &xdp_sinfo->frags[xdp_sinfo->nr_frags - i]; + struct page *page = xdp_get_frag_page(frag); + + xdp_sinfo->data_length -= xdp_get_frag_size(frag); + __xdp_return(page_address(page), &xdp->rxq->mem, false, NULL); + } + xdp_sinfo->nr_frags -= num_frags; + xdp->mb = !!xdp_sinfo->nr_frags; +} +EXPORT_SYMBOL_GPL(xdp_return_num_frags_from_buff); + /* Only called for MEM_TYPE_PAGE_POOL see xdp.h */ void __xdp_release_frame(void *data, struct xdp_mem_info *mem) { From patchwork Tue Jan 19 20:20:11 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lorenzo Bianconi X-Patchwork-Id: 366703 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-16.4 required=3.0 tests=BAYES_00,DKIMWL_WL_HIGH, DKIM_SIGNED, DKIM_VALID, DKIM_VALID_AU, INCLUDES_CR_TRAILER, INCLUDES_PATCH, MAILING_LIST_MULTI,SPF_HELO_NONE,SPF_PASS,UNWANTED_LANGUAGE_BODY, USER_AGENT_GIT autolearn=unavailable autolearn_force=no version=3.4.0 Received: from mail.kernel.org (mail.kernel.org [198.145.29.99]) by smtp.lore.kernel.org (Postfix) with ESMTP id 764C2C433E0 for ; Tue, 19 Jan 2021 20:23:12 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id 11FCD23133 for ; Tue, 19 Jan 2021 20:23:12 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1730456AbhASUWb (ORCPT ); Tue, 19 Jan 2021 15:22:31 -0500 Received: from mail.kernel.org ([198.145.29.99]:56200 "EHLO mail.kernel.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1729302AbhASUWQ (ORCPT ); Tue, 19 Jan 2021 15:22:16 -0500 Received: by mail.kernel.org (Postfix) with ESMTPSA id 114682312B; Tue, 19 Jan 2021 20:20:47 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1611087651; bh=wYcilxXfq5lFMwCFOPI6lGUmo/1e45Q8wUEYBB5EZcc=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=F2cCtbzily9udQwI/lKyVx4/MPe1vbRB5ftJSNsOW16s5ylFUTPFazZq37XKl3HFZ pEn4CIwoRMjHMwfTEDSd2yHMDtMRUyBZ/Z++rxEY0XsZ/d2ii5uRHKAvKUfgXNv6Or YGqmpJKlHeDEvAWenuGfM3Tae0ru2RiQjxByuxFKFGHDmXGLUAg7M+AqJO8ZnYJEjN FDvYnTYOjC11WiEJKqjyT9/QnBenj6MWUym1jXIUARLdh0VW83VyqZsTY3q+jBPANw XcxXNGtNTvObwhbT9ERFFWJtYaTQobGTU7HJRng3QLQYDC2odB2EjVtJYprLjcht1i Jsr/5VAm3ywtg== From: Lorenzo Bianconi To: bpf@vger.kernel.org, netdev@vger.kernel.org Cc: lorenzo.bianconi@redhat.com, davem@davemloft.net, kuba@kernel.org, ast@kernel.org, daniel@iogearbox.net, shayagr@amazon.com, john.fastabend@gmail.com, dsahern@kernel.org, brouer@redhat.com, echaudro@redhat.com, jasowang@redhat.com, alexander.duyck@gmail.com, saeed@kernel.org, maciej.fijalkowski@intel.com, sameehj@amazon.com Subject: [PATCH v6 bpf-next 5/8] net: mvneta: add multi buffer support to XDP_TX Date: Tue, 19 Jan 2021 21:20:11 +0100 Message-Id: X-Mailer: git-send-email 2.29.2 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: netdev@vger.kernel.org Introduce the capability to map non-linear xdp buffer running mvneta_xdp_submit_frame() for XDP_TX and XDP_REDIRECT Signed-off-by: Lorenzo Bianconi --- drivers/net/ethernet/marvell/mvneta.c | 94 ++++++++++++++++----------- 1 file changed, 56 insertions(+), 38 deletions(-) diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 3df0602239ad..c273e674e3de 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -1857,8 +1857,8 @@ static void mvneta_txq_bufs_free(struct mvneta_port *pp, bytes_compl += buf->skb->len; pkts_compl++; dev_kfree_skb_any(buf->skb); - } else if (buf->type == MVNETA_TYPE_XDP_TX || - buf->type == MVNETA_TYPE_XDP_NDO) { + } else if ((buf->type == MVNETA_TYPE_XDP_TX || + buf->type == MVNETA_TYPE_XDP_NDO) && buf->xdpf) { if (napi && buf->type == MVNETA_TYPE_XDP_TX) xdp_return_frame_rx_napi(buf->xdpf); else @@ -2054,45 +2054,64 @@ mvneta_xdp_put_buff(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, static int mvneta_xdp_submit_frame(struct mvneta_port *pp, struct mvneta_tx_queue *txq, - struct xdp_frame *xdpf, bool dma_map) + struct xdp_frame *xdpf, int *nxmit_byte, bool dma_map) { - struct mvneta_tx_desc *tx_desc; - struct mvneta_tx_buf *buf; - dma_addr_t dma_addr; + struct xdp_shared_info *xdp_sinfo = xdp_get_shared_info_from_frame(xdpf); + int i, num_frames = xdpf->mb ? xdp_sinfo->nr_frags + 1 : 1; + struct mvneta_tx_desc *tx_desc = NULL; + struct page *page; - if (txq->count >= txq->tx_stop_threshold) + if (txq->count + num_frames >= txq->size) return MVNETA_XDP_DROPPED; - tx_desc = mvneta_txq_next_desc_get(txq); + for (i = 0; i < num_frames; i++) { + struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index]; + skb_frag_t *frag = i ? &xdp_sinfo->frags[i - 1] : NULL; + int len = i ? xdp_get_frag_size(frag) : xdpf->len; + dma_addr_t dma_addr; - buf = &txq->buf[txq->txq_put_index]; - if (dma_map) { - /* ndo_xdp_xmit */ - dma_addr = dma_map_single(pp->dev->dev.parent, xdpf->data, - xdpf->len, DMA_TO_DEVICE); - if (dma_mapping_error(pp->dev->dev.parent, dma_addr)) { - mvneta_txq_desc_put(txq); - return MVNETA_XDP_DROPPED; + tx_desc = mvneta_txq_next_desc_get(txq); + if (dma_map) { + /* ndo_xdp_xmit */ + void *data; + + data = frag ? xdp_get_frag_address(frag) : xdpf->data; + dma_addr = dma_map_single(pp->dev->dev.parent, data, + len, DMA_TO_DEVICE); + if (dma_mapping_error(pp->dev->dev.parent, dma_addr)) { + for (; i >= 0; i--) + mvneta_txq_desc_put(txq); + return MVNETA_XDP_DROPPED; + } + buf->type = MVNETA_TYPE_XDP_NDO; + } else { + page = frag ? xdp_get_frag_page(frag) + : virt_to_page(xdpf->data); + dma_addr = page_pool_get_dma_addr(page); + if (frag) + dma_addr += xdp_get_frag_offset(frag); + else + dma_addr += sizeof(*xdpf) + xdpf->headroom; + dma_sync_single_for_device(pp->dev->dev.parent, + dma_addr, len, + DMA_BIDIRECTIONAL); + buf->type = MVNETA_TYPE_XDP_TX; } - buf->type = MVNETA_TYPE_XDP_NDO; - } else { - struct page *page = virt_to_page(xdpf->data); + buf->xdpf = i ? NULL : xdpf; + + tx_desc->command = !i ? MVNETA_TXD_F_DESC : 0; + tx_desc->buf_phys_addr = dma_addr; + tx_desc->data_size = len; + *nxmit_byte += len; - dma_addr = page_pool_get_dma_addr(page) + - sizeof(*xdpf) + xdpf->headroom; - dma_sync_single_for_device(pp->dev->dev.parent, dma_addr, - xdpf->len, DMA_BIDIRECTIONAL); - buf->type = MVNETA_TYPE_XDP_TX; + mvneta_txq_inc_put(txq); } - buf->xdpf = xdpf; - tx_desc->command = MVNETA_TXD_FLZ_DESC; - tx_desc->buf_phys_addr = dma_addr; - tx_desc->data_size = xdpf->len; + /*last descriptor */ + tx_desc->command |= MVNETA_TXD_L_DESC | MVNETA_TXD_Z_PAD; - mvneta_txq_inc_put(txq); - txq->pending++; - txq->count++; + txq->pending += num_frames; + txq->count += num_frames; return MVNETA_XDP_TX; } @@ -2103,8 +2122,8 @@ mvneta_xdp_xmit_back(struct mvneta_port *pp, struct xdp_buff *xdp) struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); struct mvneta_tx_queue *txq; struct netdev_queue *nq; + int cpu, nxmit_byte = 0; struct xdp_frame *xdpf; - int cpu; u32 ret; xdpf = xdp_convert_buff_to_frame(xdp); @@ -2116,10 +2135,10 @@ mvneta_xdp_xmit_back(struct mvneta_port *pp, struct xdp_buff *xdp) nq = netdev_get_tx_queue(pp->dev, txq->id); __netif_tx_lock(nq, cpu); - ret = mvneta_xdp_submit_frame(pp, txq, xdpf, false); + ret = mvneta_xdp_submit_frame(pp, txq, xdpf, &nxmit_byte, false); if (ret == MVNETA_XDP_TX) { u64_stats_update_begin(&stats->syncp); - stats->es.ps.tx_bytes += xdpf->len; + stats->es.ps.tx_bytes += nxmit_byte; stats->es.ps.tx_packets++; stats->es.ps.xdp_tx++; u64_stats_update_end(&stats->syncp); @@ -2158,10 +2177,9 @@ mvneta_xdp_xmit(struct net_device *dev, int num_frame, __netif_tx_lock(nq, cpu); for (i = 0; i < num_frame; i++) { - ret = mvneta_xdp_submit_frame(pp, txq, frames[i], true); - if (ret == MVNETA_XDP_TX) { - nxmit_byte += frames[i]->len; - } else { + ret = mvneta_xdp_submit_frame(pp, txq, frames[i], &nxmit_byte, + true); + if (ret != MVNETA_XDP_TX) { xdp_return_frame_rx_napi(frames[i]); nxmit--; }