diff mbox

[API-NEXT,PATCHv3] linux-generic: packet: minimize use of atomic ops on free paths

Message ID 20170217205703.7230-1-bill.fischofer@linaro.org
State New
Headers show

Commit Message

Bill Fischofer Feb. 17, 2017, 8:57 p.m. UTC 
While reference counts are implemented as atomics, try to avoid
unnecessary atomic decrement operations in free paths. This streamlines
free processing in the expected case where packets do not contain
shared segments.

Signed-off-by: Bill Fischofer <bill.fischofer@linaro.org>

---
 platform/linux-generic/include/odp_packet_internal.h |  6 +++++-
 platform/linux-generic/odp_packet.c                  | 20 +++++++++++---------
 2 files changed, 16 insertions(+), 10 deletions(-)

-- 
2.12.0.rc1
diff mbox

Patch

diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h
index 593f30dd..5864785a 100644
--- a/platform/linux-generic/include/odp_packet_internal.h
+++ b/platform/linux-generic/include/odp_packet_internal.h
@@ -257,7 +257,11 @@  static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr)
 
 static inline uint32_t packet_ref_count(odp_packet_hdr_t *pkt_hdr)
 {
-	return odp_atomic_load_u32(&pkt_hdr->ref_count);
+	/* Breach the atomic type to do a peek at the ref count. This
+	 * is used to bypass atomic operations if ref_count == 1 for
+	 * performance reasons.
+	 */
+	return pkt_hdr->ref_count.v;
 }
 
 static inline void packet_ref_count_set(odp_packet_hdr_t *pkt_hdr, uint32_t n)
diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c
index 4d2cde12..81bbcedd 100644
--- a/platform/linux-generic/odp_packet.c
+++ b/platform/linux-generic/odp_packet.c
@@ -477,7 +477,7 @@  static inline void free_bufs(odp_packet_hdr_t *pkt_hdr, int first, int num)
 	for (i = 0, nfree = 0; i < num; i++) {
 		odp_packet_hdr_t *hdr = pkt_hdr->buf_hdr.seg[first + i].hdr;
 
-		if (packet_ref_dec(hdr) == 1)
+		if (packet_ref_count(hdr) == 1 || packet_ref_dec(hdr) == 1)
 			buf[nfree++] = buffer_handle(hdr);
 	}
 
@@ -499,7 +499,8 @@  static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr,
 		for (i = 0, nfree = 0; i < num; i++) {
 			new_hdr = pkt_hdr->buf_hdr.seg[i].hdr;
 
-			if (packet_ref_dec(new_hdr) == 1)
+			if (packet_ref_count(new_hdr) == 1 ||
+			    packet_ref_dec(new_hdr) == 1)
 				buf[nfree++] = buffer_handle(new_hdr);
 		}
 
@@ -643,12 +644,12 @@  static inline odp_packet_hdr_t *packet_free_to_list(odp_packet_hdr_t *pkt_hdr,
 	int num_seg, i;
 
 	do {
-		ref_count = packet_ref_count(pkt_hdr) - 1;
+		ref_count = packet_ref_count(pkt_hdr);
 		num_seg = pkt_hdr->buf_hdr.segcount;
 		ref_hdr = pkt_hdr->ref_hdr;
 
 		if (odp_likely((CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1) &&
-			       ref_count == 0)) {
+			       ref_count == 1)) {
 			if (*nfree >= nbufs)
 				break;
 
@@ -661,11 +662,12 @@  static inline odp_packet_hdr_t *packet_free_to_list(odp_packet_hdr_t *pkt_hdr,
 				odp_packet_hdr_t *hdr =
 					pkt_hdr->buf_hdr.seg[i].hdr;
 
-				if (packet_ref_dec(hdr) == 1)
+				if (packet_ref_count(hdr) == 1 ||
+				    packet_ref_dec(hdr) == 1)
 					buf[(*nfree)++] = buffer_handle(hdr);
 			}
 
-			if (ref_count == 1)
+			if (ref_count == 2)
 				pkt_hdr->unshared_len = pkt_hdr->frame_len;
 		}
 
@@ -682,18 +684,18 @@  static inline void packet_free(odp_packet_hdr_t *pkt_hdr)
 	int num_seg;
 
 	do {
-		ref_count = packet_ref_count(pkt_hdr) - 1;
+		ref_count = packet_ref_count(pkt_hdr);
 		num_seg = pkt_hdr->buf_hdr.segcount;
 		ref_hdr = pkt_hdr->ref_hdr;
 
 		if (odp_likely((CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1) &&
-			       ref_count == 0)) {
+			       ref_count == 1)) {
 			buffer_free_multi((odp_buffer_t *)
 					  &pkt_hdr->buf_hdr.handle.handle, 1);
 		} else {
 			free_bufs(pkt_hdr, 0, num_seg);
 
-			if (ref_count == 1)
+			if (ref_count == 2)
 				pkt_hdr->unshared_len = pkt_hdr->frame_len;
 		}