@@ -273,13 +273,43 @@ static int dpaa2_eth_xdp_flush(struct dpaa2_eth_priv *priv,
return total_enqueued;
}
-static int xdp_enqueue(struct dpaa2_eth_priv *priv, struct dpaa2_fd *fd,
- void *buf_start, u16 queue_id)
+static void xdp_tx_flush(struct dpaa2_eth_priv *priv,
+ struct dpaa2_eth_channel *ch,
+ struct dpaa2_eth_fq *fq)
+{
+ struct rtnl_link_stats64 *percpu_stats;
+ struct dpaa2_fd *fds;
+ int enqueued, i;
+
+ percpu_stats = this_cpu_ptr(priv->percpu_stats);
+
+ // enqueue the array of XDP_TX frames
+ enqueued = dpaa2_eth_xdp_flush(priv, fq, &fq->xdp_tx_fds);
+
+ /* update statistics */
+ percpu_stats->tx_packets += enqueued;
+ fds = fq->xdp_tx_fds.fds;
+ for (i = 0; i < enqueued; i++) {
+ percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]);
+ ch->stats.xdp_tx++;
+ }
+ for (i = enqueued; i < fq->xdp_tx_fds.num; i++) {
+ xdp_release_buf(priv, ch, dpaa2_fd_get_addr(&fds[i]));
+ percpu_stats->tx_errors++;
+ ch->stats.xdp_tx_err++;
+ }
+ fq->xdp_tx_fds.num = 0;
+}
+
+static void xdp_enqueue(struct dpaa2_eth_priv *priv,
+ struct dpaa2_eth_channel *ch,
+ struct dpaa2_fd *fd,
+ void *buf_start, u16 queue_id)
{
- struct dpaa2_eth_fq *fq;
struct dpaa2_faead *faead;
+ struct dpaa2_fd *dest_fd;
+ struct dpaa2_eth_fq *fq;
u32 ctrl, frc;
- int i, err;
/* Mark the egress frame hardware annotation area as valid */
frc = dpaa2_fd_get_frc(fd);
@@ -296,13 +326,13 @@ static int xdp_enqueue(struct dpaa2_eth_priv *priv, struct dpaa2_fd *fd,
faead->conf_fqid = 0;
fq = &priv->fq[queue_id];
- for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) {
- err = priv->enqueue(priv, fq, fd, 0, 1, NULL);
- if (err != -EBUSY)
- break;
- }
+ dest_fd = &fq->xdp_tx_fds.fds[fq->xdp_tx_fds.num++];
+ memcpy(dest_fd, fd, sizeof(*dest_fd));
- return err;
+ if (fq->xdp_tx_fds.num < DEV_MAP_BULK_SIZE)
+ return;
+
+ xdp_tx_flush(priv, ch, fq);
}
static u32 run_xdp(struct dpaa2_eth_priv *priv,
@@ -311,14 +341,11 @@ static u32 run_xdp(struct dpaa2_eth_priv *priv,
struct dpaa2_fd *fd, void *vaddr)
{
dma_addr_t addr = dpaa2_fd_get_addr(fd);
- struct rtnl_link_stats64 *percpu_stats;
struct bpf_prog *xdp_prog;
struct xdp_buff xdp;
u32 xdp_act = XDP_PASS;
int err;
- percpu_stats = this_cpu_ptr(priv->percpu_stats);
-
rcu_read_lock();
xdp_prog = READ_ONCE(ch->xdp.prog);
@@ -341,16 +368,7 @@ static u32 run_xdp(struct dpaa2_eth_priv *priv,
case XDP_PASS:
break;
case XDP_TX:
- err = xdp_enqueue(priv, fd, vaddr, rx_fq->flowid);
- if (err) {
- xdp_release_buf(priv, ch, addr);
- percpu_stats->tx_errors++;
- ch->stats.xdp_tx_err++;
- } else {
- percpu_stats->tx_packets++;
- percpu_stats->tx_bytes += dpaa2_fd_get_len(fd);
- ch->stats.xdp_tx++;
- }
+ xdp_enqueue(priv, ch, fd, vaddr, rx_fq->flowid);
break;
default:
bpf_warn_invalid_xdp_action(xdp_act);
@@ -1168,6 +1186,7 @@ static int dpaa2_eth_poll(struct napi_struct *napi, int budget)
int store_cleaned, work_done;
struct list_head rx_list;
int retries = 0;
+ u16 flowid;
int err;
ch = container_of(napi, struct dpaa2_eth_channel, napi);
@@ -1190,6 +1209,7 @@ static int dpaa2_eth_poll(struct napi_struct *napi, int budget)
break;
if (fq->type == DPAA2_RX_FQ) {
rx_cleaned += store_cleaned;
+ flowid = fq->flowid;
} else {
txconf_cleaned += store_cleaned;
/* We have a single Tx conf FQ on this channel */
@@ -1232,6 +1252,8 @@ static int dpaa2_eth_poll(struct napi_struct *napi, int budget)
if (ch->xdp.res & XDP_REDIRECT)
xdp_do_flush_map();
+ else if (rx_cleaned && ch->xdp.res & XDP_TX)
+ xdp_tx_flush(priv, ch, &priv->fq[flowid]);
return work_done;
}
@@ -334,6 +334,7 @@ struct dpaa2_eth_fq {
struct dpaa2_eth_fq_stats stats;
struct dpaa2_eth_xdp_fds xdp_redirect_fds;
+ struct dpaa2_eth_xdp_fds xdp_tx_fds;
};
struct dpaa2_eth_ch_xdp {
Add driver level bulking to the XDP_TX action. An array of frame descriptors is held for each Tx frame queue and populated accordingly when the action returned by the XDP program is XDP_TX. The frames will be actually enqueued only when the array is filled. At the end of the NAPI cycle a flush on the queued frames is performed in order to enqueue the remaining FDs. Signed-off-by: Ioana Ciornei <ioana.ciornei@nxp.com> --- .../net/ethernet/freescale/dpaa2/dpaa2-eth.c | 68 ++++++++++++------- .../net/ethernet/freescale/dpaa2/dpaa2-eth.h | 1 + 2 files changed, 46 insertions(+), 23 deletions(-)