@@ -62,11 +62,16 @@ struct odp_buffer_hdr_t {
/* Last header of the segment list */
void *last_seg;
- /* Initial buffer data pointer and length */
+ /* Initial buffer data pointer */
uint8_t *base_data;
- uint8_t *buf_end;
- /* --- 40 bytes --- */
+ /* Reference count */
+ odp_atomic_u32_t ref_cnt;
+
+ /* Event type. Maybe different than pool type (crypto compl event) */
+ int8_t event_type;
+
+ /* --- 37 bytes --- */
/* Segments */
seg_entry_t seg[CONFIG_PACKET_MAX_SEGS];
@@ -93,14 +98,17 @@ struct odp_buffer_hdr_t {
/* Pool pointer */
void *pool_ptr;
+ /* Initial buffer tail pointer */
+ uint8_t *buf_end;
+
/* User area pointer */
void *uarea_addr;
/* User area size */
uint32_t uarea_size;
- /* Event type. Maybe different than pool type (crypto compl event) */
- int8_t event_type;
+ /* Max data size */
+ uint32_t size;
/* ipc mapped process can not walk over pointers,
* offset has to be used */
@@ -110,9 +118,6 @@ struct odp_buffer_hdr_t {
* inlining */
odp_pool_t pool_hdl;
- /* Max data size */
- uint32_t size;
-
/* Data or next header */
uint8_t data[0];
} ODP_ALIGNED_CACHE;
@@ -116,11 +116,13 @@ typedef struct {
packet_parser_t p;
+ uint32_t frame_len;
+
odp_pktio_t input;
- uint32_t frame_len;
uint32_t headroom;
uint32_t tailroom;
+ uint32_t shared_len;
/*
* Members below are not initialized by packet_init()
@@ -217,6 +219,7 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len)
* segment occupied by the allocated length.
*/
pkt_hdr->frame_len = len;
+ pkt_hdr->shared_len = 0;
pkt_hdr->headroom = CONFIG_PACKET_HEADROOM;
pkt_hdr->tailroom = CONFIG_PACKET_MAX_SEG_LEN - seg_len +
CONFIG_PACKET_TAILROOM;
@@ -491,6 +491,36 @@ static inline void copy_buf_hdr(odp_packet_hdr_t *pkt_hdr, int first, int num,
}
}
+static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num)
+{
+ int i;
+ uint32_t ref_cnt;
+ int num_ref = 0;
+
+ for (i = 0; i < num; i++) {
+ /* Zero when reference API has not been used */
+ ref_cnt = odp_atomic_load_u32(&hdr[i]->ref_cnt);
+
+ if (odp_unlikely(ref_cnt)) {
+ ref_cnt = odp_atomic_fetch_dec_u32(&hdr[i]->ref_cnt);
+
+ if (ref_cnt > 1) {
+ num_ref++;
+ continue;
+ }
+ }
+
+ /* Skip references and pack to be freed headers to array head */
+ if (odp_unlikely(num_ref))
+ hdr[i - num_ref] = hdr[i];
+ }
+
+ num -= num_ref;
+
+ if (odp_likely(num))
+ buffer_free_multi(hdr, num);
+}
+
static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num)
{
seg_entry_t *seg;
@@ -503,7 +533,7 @@ static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num)
buf_hdr[i] = seg->hdr;
}
- buffer_free_multi(buf_hdr, num);
+ packet_free_multi(buf_hdr, num);
}
static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr,
@@ -560,7 +590,7 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr,
pkt_hdr = new_hdr;
- buffer_free_multi(buf_hdr, num);
+ packet_free_multi(buf_hdr, num);
} else {
/* Free last 'num' bufs.
* First, find the last remaining header. */
@@ -575,7 +605,7 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr,
buf_hdr[i] = seg->hdr;
}
- buffer_free_multi(buf_hdr, num);
+ packet_free_multi(buf_hdr, num);
/* Head segment remains, no need to copy or update majority
* of the metadata. */
@@ -699,7 +729,7 @@ void odp_packet_free(odp_packet_t pkt)
int num_seg = pkt_hdr->buf_hdr.segcount;
if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1))
- buffer_free_multi((odp_buffer_hdr_t **)&hdl, 1);
+ packet_free_multi((odp_buffer_hdr_t **)&hdl, 1);
else
free_all_segments(pkt_hdr, num_seg);
}
@@ -707,7 +737,7 @@ void odp_packet_free(odp_packet_t pkt)
void odp_packet_free_multi(const odp_packet_t pkt[], int num)
{
if (CONFIG_PACKET_MAX_SEGS == 1) {
- buffer_free_multi((odp_buffer_hdr_t **)(uintptr_t)pkt, num);
+ packet_free_multi((odp_buffer_hdr_t **)(uintptr_t)pkt, num);
} else {
odp_buffer_hdr_t *buf_hdr[num * CONFIG_PACKET_MAX_SEGS];
int i;
@@ -728,7 +758,7 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num)
bufs += num_seg - 1;
}
- buffer_free_multi(buf_hdr, bufs);
+ packet_free_multi(buf_hdr, bufs);
}
}
@@ -1937,9 +1967,37 @@ uint64_t odp_packet_seg_to_u64(odp_packet_seg_t hdl)
return _odp_pri(hdl);
}
+static inline void buffer_ref_inc(odp_buffer_hdr_t *buf_hdr)
+{
+ /* First count increment after alloc */
+ if (odp_atomic_load_u32(&buf_hdr->ref_cnt) == 0)
+ odp_atomic_store_u32(&buf_hdr->ref_cnt, 2);
+ else
+ odp_atomic_inc_u32(&buf_hdr->ref_cnt);
+}
+
+static inline void packet_ref_inc(odp_packet_hdr_t *pkt_hdr)
+{
+ seg_entry_t *seg;
+ int i;
+ int seg_count = pkt_hdr->buf_hdr.segcount;
+ odp_packet_hdr_t *hdr = pkt_hdr;
+ uint8_t idx = 0;
+
+ for (i = 0; i < seg_count; i++) {
+ seg = seg_entry_next(&hdr, &idx);
+ buffer_ref_inc(seg->hdr);
+ }
+}
+
odp_packet_t odp_packet_ref_static(odp_packet_t pkt)
{
- return odp_packet_copy(pkt, odp_packet_pool(pkt));
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+ packet_ref_inc(pkt_hdr);
+ pkt_hdr->shared_len = packet_len(pkt_hdr);
+
+ return pkt;
}
odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset)
@@ -2001,14 +2059,30 @@ odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset,
int odp_packet_has_ref(odp_packet_t pkt)
{
- (void)pkt;
+ odp_buffer_hdr_t *buf_hdr;
+ seg_entry_t *seg;
+ int i;
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+ int seg_count = pkt_hdr->buf_hdr.segcount;
+ odp_packet_hdr_t *hdr = pkt_hdr;
+ uint8_t idx = 0;
+
+ for (i = 0; i < seg_count; i++) {
+ seg = seg_entry_next(&hdr, &idx);
+ buf_hdr = seg->hdr;
+
+ if (odp_atomic_load_u32(&buf_hdr->ref_cnt) > 1)
+ return 1;
+ }
return 0;
}
uint32_t odp_packet_unshared_len(odp_packet_t pkt)
{
- return odp_packet_len(pkt);
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+ return packet_len(pkt_hdr) - pkt_hdr->shared_len;
}
/* Include non-inlined versions of API functions */
@@ -277,6 +277,8 @@ static void init_buffers(pool_t *pool)
buf_hdr->seg[0].data = &data[offset];
buf_hdr->seg[0].len = pool->data_size;
+ odp_atomic_init_u32(&buf_hdr->ref_cnt, 0);
+
/* Store base values for fast init */
buf_hdr->base_data = buf_hdr->seg[0].data;
buf_hdr->buf_end = &data[offset + pool->data_size +