| Message ID | 1487161772-632-4-git-send-email-petri.savolainen@linaro.org |
|---|---|
| State | New |
| Headers | show |
| Series | Packet references as copy | expand |
On 02/15/17 15:29, Petri Savolainen wrote: > This reverts commit 22b3986fea090986625f3255d57b64de35bbc475. > --- sign-off is missing. Checkpatch should warn about that. Maxim. > .../linux-generic/include/odp_packet_internal.h | 85 +--- > platform/linux-generic/odp_packet.c | 536 ++++----------------- > 2 files changed, 107 insertions(+), 514 deletions(-) > > diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h > index 42df5ac..e3ada5c 100644 > --- a/platform/linux-generic/include/odp_packet_internal.h > +++ b/platform/linux-generic/include/odp_packet_internal.h > @@ -19,7 +19,6 @@ extern "C" { > > #include <odp/api/align.h> > #include <odp/api/debug.h> > -#include <odp_debug_internal.h> > #include <odp_buffer_internal.h> > #include <odp_pool_internal.h> > #include <odp_buffer_inlines.h> > @@ -169,7 +168,7 @@ typedef struct { > * packet_init(). Because of this any new fields added must be reviewed for > * initialization requirements. > */ > -typedef struct odp_packet_hdr_t { > +typedef struct { > /* common buffer header */ > odp_buffer_hdr_t buf_hdr; > > @@ -185,13 +184,6 @@ typedef struct odp_packet_hdr_t { > uint32_t headroom; > uint32_t tailroom; > > - /* Fields used to support packet references */ > - uint32_t unshared_len; > - struct odp_packet_hdr_t *ref_hdr; > - uint32_t ref_offset; > - uint32_t ref_len; > - odp_atomic_u32_t ref_count; > - > /* > * Members below are not initialized by packet_init() > */ > @@ -220,55 +212,6 @@ static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) > return (odp_packet_hdr_t *)buf_hdl_to_hdr((odp_buffer_t)pkt); > } > > -static inline odp_packet_hdr_t *odp_packet_last_hdr(odp_packet_t pkt, > - uint32_t *offset) > -{ > - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - odp_packet_hdr_t *prev_hdr = pkt_hdr; > - uint32_t ref_offset = 0; > - > - while (pkt_hdr->ref_hdr) { > - ref_offset = pkt_hdr->ref_offset; > - prev_hdr = pkt_hdr; > - pkt_hdr = pkt_hdr->ref_hdr; > - } > - > - if (offset) { > - if (prev_hdr != pkt_hdr) > - ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; > - *offset = ref_offset; > - } > - > - return pkt_hdr; > -} > - > -static inline odp_packet_hdr_t *odp_packet_prev_hdr(odp_packet_hdr_t *pkt_hdr, > - odp_packet_hdr_t *cur_hdr, > - uint32_t *offset) > -{ > - uint32_t ref_offset = 0; > - odp_packet_hdr_t *prev_hdr = pkt_hdr; > - > - while (pkt_hdr->ref_hdr != cur_hdr) { > - ref_offset = pkt_hdr->ref_offset; > - prev_hdr = pkt_hdr; > - pkt_hdr = pkt_hdr->ref_hdr; > - } > - > - if (offset) { > - if (prev_hdr != pkt_hdr) > - ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; > - *offset = ref_offset; > - } > - > - return pkt_hdr; > -} > - > -static inline odp_packet_t _odp_packet_hdl(odp_packet_hdr_t *pkt_hdr) > -{ > - return (odp_packet_t)odp_hdr_to_buf(&pkt_hdr->buf_hdr); > -} > - > static inline void copy_packet_parser_metadata(odp_packet_hdr_t *src_hdr, > odp_packet_hdr_t *dst_hdr) > { > @@ -291,41 +234,17 @@ static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) > > pkt_hdr->tailroom += len; > pkt_hdr->frame_len -= len; > - pkt_hdr->unshared_len -= len; > pkt_hdr->buf_hdr.seg[last].len -= len; > } > > static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) > { > - uint32_t pkt_len = 0; > - uint32_t offset = 0; > - > - do { > - pkt_len += pkt_hdr->frame_len - offset; > - offset = pkt_hdr->ref_offset; > - if (pkt_hdr->ref_hdr) > - offset += (pkt_hdr->ref_hdr->frame_len - > - pkt_hdr->ref_len); > - pkt_hdr = pkt_hdr->ref_hdr; > - } while (pkt_hdr); > - > - return pkt_len; > -} > - > -static inline uint32_t packet_ref_count(odp_packet_hdr_t *pkt_hdr) > -{ > - return odp_atomic_load_u32(&pkt_hdr->ref_count); > -} > - > -static inline void packet_ref_count_set(odp_packet_hdr_t *pkt_hdr, uint32_t n) > -{ > - odp_atomic_init_u32(&pkt_hdr->ref_count, n); > + return pkt_hdr->frame_len; > } > > static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) > { > pkt_hdr->frame_len = len; > - pkt_hdr->unshared_len = len; > } > > static inline int packet_parse_l2_not_done(packet_parser_t *prs) > diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c > index 3aadc4d..024f694 100644 > --- a/platform/linux-generic/odp_packet.c > +++ b/platform/linux-generic/odp_packet.c > @@ -33,24 +33,13 @@ static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) > return pkt_hdr->buf_hdr.handle.handle; > } > > -static inline uint32_t packet_ref_inc(odp_packet_hdr_t *pkt_hdr) > -{ > - return odp_atomic_fetch_inc_u32(&pkt_hdr->ref_count); > -} > - > -static inline uint32_t packet_ref_dec(odp_packet_hdr_t *pkt_hdr) > -{ > - return odp_atomic_fetch_dec_u32(&pkt_hdr->ref_count); > -} > - > static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, > uint32_t seg_idx) > { > return pkt_hdr->buf_hdr.seg[seg_idx].len; > } > > -static inline uint8_t *packet_seg_data(odp_packet_hdr_t *pkt_hdr, > - uint32_t seg_idx) > +static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) > { > return pkt_hdr->buf_hdr.seg[seg_idx].data; > } > @@ -63,11 +52,6 @@ static inline int packet_last_seg(odp_packet_hdr_t *pkt_hdr) > return pkt_hdr->buf_hdr.segcount - 1; > } > > -static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) > -{ > - return pkt_hdr->buf_hdr.seg[0].data; > -} > - > static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr) > { > return packet_seg_len(pkt_hdr, 0); > @@ -80,6 +64,11 @@ static inline uint32_t packet_last_seg_len(odp_packet_hdr_t *pkt_hdr) > return packet_seg_len(pkt_hdr, last); > } > > +static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) > +{ > + return pkt_hdr->buf_hdr.seg[0].data; > +} > + > static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr) > { > int last = packet_last_seg(pkt_hdr); > @@ -110,7 +99,6 @@ static inline void push_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) > { > pkt_hdr->headroom -= len; > pkt_hdr->frame_len += len; > - pkt_hdr->unshared_len += len; > pkt_hdr->buf_hdr.seg[0].data -= len; > pkt_hdr->buf_hdr.seg[0].len += len; > } > @@ -119,7 +107,6 @@ static inline void pull_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) > { > pkt_hdr->headroom += len; > pkt_hdr->frame_len -= len; > - pkt_hdr->unshared_len -= len; > pkt_hdr->buf_hdr.seg[0].data += len; > pkt_hdr->buf_hdr.seg[0].len -= len; > } > @@ -130,7 +117,6 @@ static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) > > pkt_hdr->tailroom -= len; > pkt_hdr->frame_len += len; > - pkt_hdr->unshared_len += len; > pkt_hdr->buf_hdr.seg[last].len += len; > } > > @@ -158,10 +144,6 @@ static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, > dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr; > dst->buf_hdr.uarea_size = src->buf_hdr.uarea_size; > > - /* reference related metadata */ > - dst->ref_len = src->ref_len; > - dst->unshared_len = src->unshared_len; > - > /* segmentation data is not copied: > * buf_hdr.seg[] > * buf_hdr.segcount > @@ -176,15 +158,7 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, > int seg = 0; > int seg_count = pkt_hdr->buf_hdr.segcount; > > - /* Special processing for references */ > - while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { > - offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); > - offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); > - pkt_hdr = pkt_hdr->ref_hdr; > - seg_count = pkt_hdr->buf_hdr.segcount; > - } > - > - if (odp_unlikely(offset > pkt_hdr->frame_len)) > + if (odp_unlikely(offset >= pkt_hdr->frame_len)) > return NULL; > > if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || seg_count == 1)) { > @@ -233,9 +207,6 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) > pkt_hdr->p.l2_offset = 0; > pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; > pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; > - > - /* Ensure dummy pkt_hdrs used in I/O recv classification are valid */ > - pkt_hdr->ref_hdr = NULL; > } > > /** > @@ -281,10 +252,6 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len, > CONFIG_PACKET_TAILROOM; > > pkt_hdr->input = ODP_PKTIO_INVALID; > - > - /* By default packet has no references */ > - pkt_hdr->unshared_len = len; > - pkt_hdr->ref_hdr = NULL; > } > > static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) > @@ -297,7 +264,6 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) > > hdr->buf_hdr.seg[0].data = hdr->buf_hdr.base_data; > hdr->buf_hdr.seg[0].len = BASE_LEN; > - packet_ref_count_set(hdr, 1); > > /* Link segments */ > if (CONFIG_PACKET_MAX_SEGS != 1) { > @@ -307,7 +273,6 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) > for (i = 1; i < num; i++) { > odp_buffer_hdr_t *buf_hdr; > > - packet_ref_count_set(pkt_hdr[i], 1); > buf_hdr = &pkt_hdr[i]->buf_hdr; > hdr->buf_hdr.seg[i].hdr = buf_hdr; > hdr->buf_hdr.seg[i].data = buf_hdr->base_data; > @@ -411,10 +376,9 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, > new_hdr->buf_hdr.seg[0].len = seg_len; > > packet_seg_copy_md(new_hdr, pkt_hdr); > - new_hdr->frame_len = pkt_hdr->frame_len + len; > - new_hdr->unshared_len = pkt_hdr->unshared_len + len; > - new_hdr->headroom = pool->headroom + offset; > - new_hdr->tailroom = pkt_hdr->tailroom; > + new_hdr->frame_len = pkt_hdr->frame_len + len; > + new_hdr->headroom = pool->headroom + offset; > + new_hdr->tailroom = pkt_hdr->tailroom; > > pkt_hdr = new_hdr; > } else { > @@ -427,9 +391,8 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, > last = packet_last_seg(pkt_hdr); > pkt_hdr->buf_hdr.seg[last].len = seg_len; > > - pkt_hdr->frame_len += len; > - pkt_hdr->unshared_len += len; > - pkt_hdr->tailroom = pool->tailroom + offset; > + pkt_hdr->frame_len += len; > + pkt_hdr->tailroom = pool->tailroom + offset; > } > > return pkt_hdr; > @@ -437,18 +400,13 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, > > static inline void free_bufs(odp_packet_hdr_t *pkt_hdr, int first, int num) > { > - int i, nfree; > + int i; > odp_buffer_t buf[num]; > > - for (i = 0, nfree = 0; i < num; i++) { > - odp_packet_hdr_t *hdr = pkt_hdr->buf_hdr.seg[first + i].hdr; > - > - if (packet_ref_dec(hdr) == 1) > - buf[nfree++] = buffer_handle(hdr); > - } > + for (i = 0; i < num; i++) > + buf[i] = buffer_handle(pkt_hdr->buf_hdr.seg[first + i].hdr); > > - if (nfree > 0) > - buffer_free_multi(buf, nfree); > + buffer_free_multi(buf, num); > } > > static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, > @@ -459,15 +417,11 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, > > if (head) { > odp_packet_hdr_t *new_hdr; > - int i, nfree; > + int i; > odp_buffer_t buf[num]; > > - for (i = 0, nfree = 0; i < num; i++) { > - new_hdr = pkt_hdr->buf_hdr.seg[i].hdr; > - > - if (packet_ref_dec(new_hdr) == 1) > - buf[nfree++] = buffer_handle(new_hdr); > - } > + for (i = 0; i < num; i++) > + buf[i] = buffer_handle(pkt_hdr->buf_hdr.seg[i].hdr); > > /* First remaining segment is the new packet descriptor */ > new_hdr = pkt_hdr->buf_hdr.seg[num].hdr; > @@ -476,17 +430,15 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, > packet_seg_copy_md(new_hdr, pkt_hdr); > > /* Tailroom not changed */ > - new_hdr->tailroom = pkt_hdr->tailroom; > - new_hdr->headroom = seg_headroom(new_hdr, 0); > - new_hdr->frame_len = pkt_hdr->frame_len - free_len; > - new_hdr->unshared_len = pkt_hdr->unshared_len - free_len; > + new_hdr->tailroom = pkt_hdr->tailroom; > + new_hdr->headroom = seg_headroom(new_hdr, 0); > + new_hdr->frame_len = pkt_hdr->frame_len - free_len; > > pull_head(new_hdr, pull_len); > > pkt_hdr = new_hdr; > > - if (nfree > 0) > - buffer_free_multi(buf, nfree); > + buffer_free_multi(buf, num); > } else { > /* Free last 'num' bufs */ > free_bufs(pkt_hdr, num_remain, num); > @@ -495,7 +447,6 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, > * of the metadata. */ > pkt_hdr->buf_hdr.segcount = num_remain; > pkt_hdr->frame_len -= free_len; > - pkt_hdr->unshared_len -= free_len; > pkt_hdr->tailroom = seg_tailroom(pkt_hdr, num_remain - 1); > > pull_tail(pkt_hdr, pull_len); > @@ -599,34 +550,45 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, > return num; > } > > -static inline void packet_free(odp_packet_hdr_t *pkt_hdr) > -{ > - odp_packet_hdr_t *ref_hdr; > - uint32_t ref_count; > - > - do { > - ref_hdr = pkt_hdr->ref_hdr; > - ref_count = packet_ref_count(pkt_hdr) - 1; > - free_bufs(pkt_hdr, 0, pkt_hdr->buf_hdr.segcount); > - > - if (ref_count == 1) > - pkt_hdr->unshared_len = pkt_hdr->frame_len; > - > - pkt_hdr = ref_hdr; > - } while (pkt_hdr); > -} > - > void odp_packet_free(odp_packet_t pkt) > { > - packet_free(odp_packet_hdr(pkt)); > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + int num_seg = pkt_hdr->buf_hdr.segcount; > + > + if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) > + buffer_free_multi((odp_buffer_t *)&pkt, 1); > + else > + free_bufs(pkt_hdr, 0, num_seg); > } > > void odp_packet_free_multi(const odp_packet_t pkt[], int num) > { > - int i; > + if (CONFIG_PACKET_MAX_SEGS == 1) { > + buffer_free_multi((const odp_buffer_t * const)pkt, num); > + } else { > + odp_buffer_t buf[num * CONFIG_PACKET_MAX_SEGS]; > + int i, j; > + int bufs = 0; > > - for (i = 0; i < num; i++) > - packet_free(odp_packet_hdr(pkt[i])); > + for (i = 0; i < num; i++) { > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt[i]); > + int num_seg = pkt_hdr->buf_hdr.segcount; > + odp_buffer_hdr_t *buf_hdr = &pkt_hdr->buf_hdr; > + > + buf[bufs] = (odp_buffer_t)pkt[i]; > + bufs++; > + > + if (odp_likely(num_seg == 1)) > + continue; > + > + for (j = 1; j < num_seg; j++) { > + buf[bufs] = buffer_handle(buf_hdr->seg[j].hdr); > + bufs++; > + } > + } > + > + buffer_free_multi(buf, bufs); > + } > } > > int odp_packet_reset(odp_packet_t pkt, uint32_t len) > @@ -637,9 +599,6 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) > if (len > pool->headroom + pool->data_size + pool->tailroom) > return -1; > > - if (pkt_hdr->ref_hdr) > - packet_free(pkt_hdr->ref_hdr); > - > packet_init(pkt_hdr, len, 0); > > return 0; > @@ -682,21 +641,15 @@ void *odp_packet_head(odp_packet_t pkt) > uint32_t odp_packet_buf_len(odp_packet_t pkt) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - uint32_t buf_len = 0; > > - do { > - buf_len += pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; > - pkt_hdr = pkt_hdr->ref_hdr; > - } while (pkt_hdr); > - > - return buf_len; > + return pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; > } > > void *odp_packet_data(odp_packet_t pkt) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - return packet_map(pkt_hdr, 0, NULL, NULL); > + return packet_data(pkt_hdr); > } > > uint32_t odp_packet_seg_len(odp_packet_t pkt) > @@ -708,32 +661,7 @@ uint32_t odp_packet_seg_len(odp_packet_t pkt) > > uint32_t odp_packet_len(odp_packet_t pkt) > { > - return packet_len(odp_packet_hdr(pkt)); > -} > - > -uint32_t odp_packet_unshared_len(odp_packet_t pkt) > -{ > - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - uint32_t pkt_len = 0, offset = 0; > - > - do { > - if (packet_ref_count(pkt_hdr) > 1) { > - if (offset == 0) > - pkt_len += pkt_hdr->unshared_len; > - break; > - } > - > - pkt_len += pkt_hdr->frame_len - offset; > - offset = pkt_hdr->ref_offset; > - > - if (pkt_hdr->ref_hdr) > - offset += (pkt_hdr->ref_hdr->frame_len - > - pkt_hdr->ref_len); > - > - pkt_hdr = pkt_hdr->ref_hdr; > - } while (pkt_hdr); > - > - return pkt_len; > + return odp_packet_hdr(pkt)->frame_len; > } > > uint32_t odp_packet_headroom(odp_packet_t pkt) > @@ -743,12 +671,12 @@ uint32_t odp_packet_headroom(odp_packet_t pkt) > > uint32_t odp_packet_tailroom(odp_packet_t pkt) > { > - return odp_packet_last_hdr(pkt, NULL)->tailroom; > + return odp_packet_hdr(pkt)->tailroom; > } > > void *odp_packet_tail(odp_packet_t pkt) > { > - odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(pkt, NULL); > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > return packet_tail(pkt_hdr); > } > @@ -942,7 +870,7 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); > uint32_t frame_len = pkt_hdr->frame_len; > - uint32_t headroom = pkt_hdr->headroom; > + uint32_t headroom = pkt_hdr->headroom; > int ret = 0; > > if (len > headroom) { > @@ -957,46 +885,6 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, > segs = pkt_hdr->buf_hdr.segcount; > > if (odp_unlikely((segs + num) > CONFIG_PACKET_MAX_SEGS)) { > - /* Handle recursively via references when > - * working with referenced packets since another > - * thread may be accessing it concurrently via > - * its reference to it. */ > - if (packet_ref_count(pkt_hdr) > 1) { > - odp_packet_t ref; > - uint32_t unshared_len; > - > - push_head(pkt_hdr, headroom); > - unshared_len = pkt_hdr->unshared_len; > - ref = odp_packet_ref(*pkt, 0); > - > - if (ref == ODP_PACKET_INVALID) { > - pull_head(pkt_hdr, headroom); > - return -1; > - } > - > - ret = odp_packet_extend_head(&ref, > - len - headroom, > - data_ptr, > - seg_len); > - > - if (ret < 0) { > - odp_packet_free(ref); > - pull_head(pkt_hdr, headroom); > - return -1; > - } > - > - /* Since this is a special ref, the > - * base pkt's unshared len is unchanged */ > - pkt_hdr->unshared_len = unshared_len; > - > - /* Remove extra ref to the base pkt */ > - odp_packet_free(*pkt); > - > - /* Return the ref as the extension result */ > - *pkt = ref; > - return 1; > - } > - > /* Cannot directly add new segments */ > odp_packet_hdr_t *new_hdr; > int new_segs = 0; > @@ -1048,7 +936,6 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, > > pkt_hdr->buf_hdr.segcount = segs; > pkt_hdr->frame_len = frame_len; > - pkt_hdr->unshared_len = frame_len; > pkt_hdr->headroom = offset + pool->headroom; > pkt_hdr->tailroom = pool->tailroom; > > @@ -1074,16 +961,11 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, > push_head(pkt_hdr, len); > } > > - if (data_ptr || seg_len) { > - uint32_t seg_ln = 0; > - void *data = packet_map(pkt_hdr, 0, &seg_ln, NULL); > - > - if (data_ptr) > - *data_ptr = data; > + if (data_ptr) > + *data_ptr = packet_data(pkt_hdr); > > - if (seg_len) > - *seg_len = seg_ln; > - } > + if (seg_len) > + *seg_len = packet_first_seg_len(pkt_hdr); > > return ret; > } > @@ -1095,8 +977,6 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) > if (len > pkt_hdr->frame_len) > return NULL; > > - ODP_ASSERT(len <= pkt_hdr->unshared_len); > - > pull_head(pkt_hdr, len); > return packet_data(pkt_hdr); > } > @@ -1104,35 +984,15 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) > int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, > void **data_ptr, uint32_t *seg_len_out) > { > - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt), *nxt_hdr; > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); > uint32_t seg_len = packet_first_seg_len(pkt_hdr); > - int ret = 0; > > - if (len > packet_len(pkt_hdr)) > + if (len > pkt_hdr->frame_len) > return -1; > > - ODP_ASSERT(len <= odp_packet_unshared_len(*pkt)); > - > - /* Special processing for references */ > - while (len >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { > - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); > - nxt_hdr = pkt_hdr->ref_hdr; > - len -= pkt_hdr->frame_len; > - len += pkt_hdr->ref_offset + > - (nxt_hdr->frame_len - pkt_hdr->ref_len); > - pkt_hdr->ref_hdr = NULL; > - packet_free(pkt_hdr); > - pkt_hdr = nxt_hdr; > - seg_len = packet_first_seg_len(pkt_hdr); > - *pkt = packet_handle(pkt_hdr); > - ret = 1; > - } > - > - if (CONFIG_PACKET_MAX_SEGS == 1 || > - len < seg_len || > - pkt_hdr->buf_hdr.segcount == 1) { > + if (len < seg_len) { > pull_head(pkt_hdr, len); > - } else { > + } else if (CONFIG_PACKET_MAX_SEGS != 1) { > int num = 0; > uint32_t pull_len = 0; > > @@ -1147,29 +1007,23 @@ int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, > *pkt = packet_handle(pkt_hdr); > } > > - if (data_ptr || seg_len_out) { > - void *data_head = packet_map(pkt_hdr, 0, &seg_len, NULL); > - > - if (data_ptr) > - *data_ptr = data_head; > + if (data_ptr) > + *data_ptr = packet_data(pkt_hdr); > > - if (seg_len_out) > - *seg_len_out = seg_len; > - } > + if (seg_len_out) > + *seg_len_out = packet_first_seg_len(pkt_hdr); > > - return ret; > + return 0; > } > > void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) > { > - odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(pkt, NULL); > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > void *old_tail; > > if (len > pkt_hdr->tailroom) > return NULL; > > - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); > - > old_tail = packet_tail(pkt_hdr); > push_tail(pkt_hdr, len); > > @@ -1179,14 +1033,12 @@ void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) > int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, > void **data_ptr, uint32_t *seg_len_out) > { > - odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(*pkt, NULL); > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); > uint32_t frame_len = pkt_hdr->frame_len; > uint32_t tailroom = pkt_hdr->tailroom; > uint32_t tail_off = frame_len; > int ret = 0; > > - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); > - > if (len > tailroom) { > pool_t *pool = pool_entry_from_hdl(pkt_hdr->buf_hdr.pool_hdl); > int num; > @@ -1277,7 +1129,6 @@ void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) > if (len > packet_last_seg_len(pkt_hdr)) > return NULL; > > - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); > pull_tail(pkt_hdr, len); > > return packet_tail(pkt_hdr); > @@ -1288,34 +1139,17 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, > { > int last; > uint32_t seg_len; > - uint32_t offset; > - odp_packet_hdr_t *first_hdr = odp_packet_hdr(*pkt); > - odp_packet_hdr_t *pkt_hdr, *prev_hdr; > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); > > - if (len > packet_len(first_hdr)) > + if (len > pkt_hdr->frame_len) > return -1; > > - pkt_hdr = odp_packet_last_hdr(*pkt, &offset); > - > - /* Special processing for references */ > - while (len >= pkt_hdr->frame_len - offset && first_hdr->ref_hdr) { > - len -= (pkt_hdr->frame_len - offset); > - prev_hdr = odp_packet_prev_hdr(first_hdr, pkt_hdr, &offset); > - ODP_ASSERT(packet_ref_count(prev_hdr) == 1); > - prev_hdr->ref_hdr = NULL; > - packet_free(pkt_hdr); > - pkt_hdr = prev_hdr; > - } > - > - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); > last = packet_last_seg(pkt_hdr); > seg_len = packet_seg_len(pkt_hdr, last); > > - if (CONFIG_PACKET_MAX_SEGS == 1 || > - len < seg_len || > - pkt_hdr->buf_hdr.segcount == 1) { > + if (len < seg_len) { > pull_tail(pkt_hdr, len); > - } else { > + } else if (CONFIG_PACKET_MAX_SEGS != 1) { > int num = 0; > uint32_t pull_len = 0; > > @@ -1522,50 +1356,35 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) > > int odp_packet_is_segmented(odp_packet_t pkt) > { > - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - > - return pkt_hdr->buf_hdr.segcount > 1 || pkt_hdr->ref_hdr != NULL; > + return odp_packet_hdr(pkt)->buf_hdr.segcount > 1; > } > > int odp_packet_num_segs(odp_packet_t pkt) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - uint32_t segcount = 0, i; > - uint32_t seg_offset = 0, offset; > - > - do { > - segcount += pkt_hdr->buf_hdr.segcount - seg_offset; > - offset = pkt_hdr->ref_offset; > - pkt_hdr = pkt_hdr->ref_hdr; > - if (pkt_hdr) { > - for (i = 0, seg_offset = 0; > - i < pkt_hdr->buf_hdr.segcount; > - i++, seg_offset++) { > - if (offset < pkt_hdr->buf_hdr.seg[i].len) > - break; > - offset -= pkt_hdr->buf_hdr.seg[i].len; > - } > - } > - } while (pkt_hdr); > > - return segcount; > + return pkt_hdr->buf_hdr.segcount; > } > > -odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt ODP_UNUSED) > +odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) > { > + (void)pkt; > + > return 0; > } > > odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) > { > - return (odp_packet_seg_t)(odp_packet_num_segs(pkt) - 1); > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + > + return packet_last_seg(pkt_hdr); > } > > odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - if (odp_unlikely(seg >= packet_last_seg(pkt_hdr))) > + if (odp_unlikely(seg >= (odp_packet_seg_t)packet_last_seg(pkt_hdr))) > return ODP_PACKET_SEG_INVALID; > > return seg + 1; > @@ -1581,51 +1400,21 @@ odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) > void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - uint32_t seg_offset = 0, offset = 0, i; > - > - while (seg >= pkt_hdr->buf_hdr.segcount - seg_offset && > - pkt_hdr->ref_hdr) { > - seg -= (pkt_hdr->buf_hdr.segcount - seg_offset); > - offset = pkt_hdr->ref_offset; > - pkt_hdr = pkt_hdr->ref_hdr; > - for (i = 0, seg_offset = 0; > - i < pkt_hdr->buf_hdr.segcount; > - i++, seg_offset++) { > - if (offset < pkt_hdr->buf_hdr.seg[i].len) > - break; > - offset -= pkt_hdr->buf_hdr.seg[i].len; > - } > - } > > - if (odp_unlikely(seg + seg_offset >= pkt_hdr->buf_hdr.segcount)) > + if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount)) > return NULL; > > - return packet_seg_data(pkt_hdr, seg + seg_offset) + offset; > + return packet_seg_data(pkt_hdr, seg); > } > > uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - uint32_t seg_offset = 0, offset = 0, i; > - > - while (seg >= pkt_hdr->buf_hdr.segcount - seg_offset && > - pkt_hdr->ref_hdr) { > - seg -= (pkt_hdr->buf_hdr.segcount - seg_offset); > - offset = pkt_hdr->ref_offset; > - pkt_hdr = pkt_hdr->ref_hdr; > - for (i = 0, seg_offset = 0; > - i < pkt_hdr->buf_hdr.segcount; > - i++, seg_offset++) { > - if (offset < pkt_hdr->buf_hdr.seg[i].len) > - break; > - offset -= pkt_hdr->buf_hdr.seg[i].len; > - } > - } > > - if (odp_unlikely(seg + seg_offset >= pkt_hdr->buf_hdr.segcount)) > + if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount)) > return 0; > > - return packet_seg_len(pkt_hdr, seg + seg_offset) - offset; > + return packet_seg_len(pkt_hdr, seg); > } > > /* > @@ -1639,14 +1428,12 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) > { > odp_packet_t pkt = *pkt_ptr; > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - uint32_t pktlen = packet_len(pkt_hdr); > + uint32_t pktlen = pkt_hdr->frame_len; > odp_packet_t newpkt; > > if (offset > pktlen) > return -1; > > - ODP_ASSERT(odp_packet_unshared_len(*pkt_ptr) >= offset); > - > newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen + len); > > if (newpkt == ODP_PACKET_INVALID) > @@ -1709,8 +1496,6 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len, > if (align > ODP_CACHE_LINE_SIZE) > return -1; > > - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); > - > if (seglen >= len) { > misalign = align <= 1 ? 0 : > ROUNDUP_ALIGN(uaddr, align) - uaddr; > @@ -1750,13 +1535,10 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) > uint32_t dst_len = dst_hdr->frame_len; > uint32_t src_len = src_hdr->frame_len; > > - ODP_ASSERT(packet_ref_count(dst_hdr) == 1); > - > /* Do a copy if resulting packet would be out of segments or packets > - * are from different pools or src is a reference. */ > + * are from different pools. */ > if (odp_unlikely((dst_segs + src_segs) > CONFIG_PACKET_MAX_SEGS) || > - odp_unlikely(dst_pool != src_pool) || > - odp_unlikely(packet_ref_count(src_hdr)) > 1) { > + odp_unlikely(dst_pool != src_pool)) { > if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) { > (void)odp_packet_copy_from_pkt(*dst, dst_len, > src, 0, src_len); > @@ -1771,9 +1553,8 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) > > add_all_segs(dst_hdr, src_hdr); > > - dst_hdr->frame_len = dst_len + src_len; > - dst_hdr->unshared_len = dst_len + src_len; > - dst_hdr->tailroom = src_hdr->tailroom; > + dst_hdr->frame_len = dst_len + src_len; > + dst_hdr->tailroom = src_hdr->tailroom; > > /* Data was not moved in memory */ > return 0; > @@ -1786,7 +1567,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) > if (len >= pktlen || tail == NULL) > return -1; > > - ODP_ASSERT(odp_packet_unshared_len(*pkt) >= len); > *tail = odp_packet_copy_part(*pkt, len, pktlen - len, > odp_packet_pool(*pkt)); > > @@ -1797,109 +1577,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) > } > > /* > - * References > - */ > - > -static inline void packet_ref(odp_packet_hdr_t *pkt_hdr) > -{ > - uint32_t i; > - odp_packet_hdr_t *hdr; > - > - do { > - for (i = 0; i < pkt_hdr->buf_hdr.segcount; i++) { > - hdr = pkt_hdr->buf_hdr.seg[i].hdr; > - packet_ref_inc(hdr); > - } > - > - pkt_hdr = pkt_hdr->ref_hdr; > - } while (pkt_hdr); > -} > - > -static inline odp_packet_t packet_splice(odp_packet_hdr_t *pkt_hdr, > - uint32_t offset, > - odp_packet_hdr_t *ref_hdr) > -{ > - /* Catch attempted references to stale handles in debug builds */ > - ODP_ASSERT(packet_ref_count(pkt_hdr) > 0); > - > - /* Splicing is from the last section of src pkt */ > - while (ref_hdr->ref_hdr) > - ref_hdr = ref_hdr->ref_hdr; > - > - /* Find section where splice begins */ > - while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { > - offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); > - offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); > - pkt_hdr = pkt_hdr->ref_hdr; > - } > - > - ref_hdr->ref_hdr = pkt_hdr; > - ref_hdr->ref_offset = offset; > - ref_hdr->ref_len = pkt_hdr->frame_len; > - > - if (offset < pkt_hdr->unshared_len) > - pkt_hdr->unshared_len = offset; > - > - packet_ref(pkt_hdr); > - return _odp_packet_hdl(ref_hdr); > -} > - > -odp_packet_t odp_packet_ref_static(odp_packet_t pkt) > -{ > - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - > - pkt_hdr->unshared_len = 0; > - packet_ref(pkt_hdr); > - return pkt; > -} > - > -odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) > -{ > - odp_packet_t hdr; > - odp_packet_hdr_t *pkt_hdr; > - > - if (pkt == ODP_PACKET_INVALID) > - return ODP_PACKET_INVALID; > - > - pkt_hdr = odp_packet_hdr(pkt); > - if (offset >= packet_len(pkt_hdr)) > - return ODP_PACKET_INVALID; > - > - hdr = odp_packet_alloc(odp_packet_pool(pkt), 0); > - > - if (hdr == ODP_PACKET_INVALID) > - return ODP_PACKET_INVALID; > - > - return packet_splice(pkt_hdr, offset, odp_packet_hdr(hdr)); > -} > - > -odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, > - odp_packet_t hdr) > -{ > - odp_packet_hdr_t *pkt_hdr; > - > - if (pkt == ODP_PACKET_INVALID || > - hdr == ODP_PACKET_INVALID || > - pkt == hdr) > - return ODP_PACKET_INVALID; > - > - ODP_ASSERT(odp_packet_has_ref(hdr) == 0); > - > - pkt_hdr = odp_packet_hdr(pkt); > - if (offset >= packet_len(pkt_hdr)) > - return ODP_PACKET_INVALID; > - > - return packet_splice(pkt_hdr, offset, odp_packet_hdr(hdr)); > -} > - > -int odp_packet_has_ref(odp_packet_t pkt) > -{ > - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - > - return pkt_hdr->ref_hdr != NULL || packet_ref_count(pkt_hdr) > 1; > -} > - > -/* > * > * Copy > * ******************************************************** > @@ -1908,7 +1585,8 @@ int odp_packet_has_ref(odp_packet_t pkt) > > odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool) > { > - uint32_t pktlen = odp_packet_len(pkt); > + odp_packet_hdr_t *srchdr = odp_packet_hdr(pkt); > + uint32_t pktlen = srchdr->frame_len; > odp_packet_t newpkt = odp_packet_alloc(pool, pktlen); > > if (newpkt != ODP_PACKET_INVALID) { > @@ -1947,7 +1625,7 @@ int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t offset, > uint8_t *dstaddr = (uint8_t *)dst; > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - if (offset + len > packet_len(pkt_hdr)) > + if (offset + len > pkt_hdr->frame_len) > return -1; > > while (len > 0) { > @@ -1971,11 +1649,9 @@ int odp_packet_copy_from_mem(odp_packet_t pkt, uint32_t offset, > const uint8_t *srcaddr = (const uint8_t *)src; > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - if (offset + len > packet_len(pkt_hdr)) > + if (offset + len > pkt_hdr->frame_len) > return -1; > > - ODP_ASSERT(odp_packet_unshared_len(pkt) >= offset + len); > - > while (len > 0) { > mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); > cpylen = len > seglen ? seglen : len; > @@ -2001,12 +1677,10 @@ int odp_packet_copy_from_pkt(odp_packet_t dst, uint32_t dst_offset, > uint32_t src_seglen = 0; /* GCC */ > int overlap; > > - if (dst_offset + len > packet_len(dst_hdr) || > - src_offset + len > packet_len(src_hdr)) > + if (dst_offset + len > dst_hdr->frame_len || > + src_offset + len > src_hdr->frame_len) > return -1; > > - ODP_ASSERT(odp_packet_unshared_len(dst) >= dst_offset + len); > - > overlap = (dst_hdr == src_hdr && > ((dst_offset <= src_offset && > dst_offset + len >= src_offset) || > @@ -2090,7 +1764,7 @@ void odp_packet_print(odp_packet_t pkt) > len += snprintf(&str[len], n - len, > " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); > len += snprintf(&str[len], n - len, > - " frame_len %" PRIu32 "\n", packet_len(hdr)); > + " frame_len %" PRIu32 "\n", hdr->frame_len); > len += snprintf(&str[len], n - len, > " input %" PRIu64 "\n", > odp_pktio_to_u64(hdr->input)); >
> -----Original Message----- > From: lng-odp [mailto:lng-odp-bounces@lists.linaro.org] On Behalf Of Maxim > Uvarov > Sent: Wednesday, February 15, 2017 6:19 PM > To: lng-odp@lists.linaro.org > Subject: Re: [lng-odp] [API-NEXT PATCH v2 3/4] Revert "linux-generic: > packet: implement reference apis" > > On 02/15/17 15:29, Petri Savolainen wrote: > > This reverts commit 22b3986fea090986625f3255d57b64de35bbc475. > > --- > > sign-off is missing. Checkpatch should warn about that. > > Maxim. This patch is output of the revert command as-is, and has couple of check patch warnings due to that. I can add sign-off there if that's needed, although I didn't do any modifications into the commit itself. Also 'make check' fails after this commit since the new implementation is added in the next commit. Anyway, I think this results cleaner git history as the revert is not modified (== not squashed with the new implementation). E.g. odp-dpdk can directly merge the new implementation, without the need to first merge the reverted one. -Petri
I'm going to explicitly NAK this part just in case there was any confusion. If we have a better replacement for the current packet reference implementation then it can be posted the same way we've revised the implementation of other APIs. Until then this is an unnecessary step backwards. Parts 1 and 2 are good and should be merged. On Thu, Feb 16, 2017 at 1:38 AM, Savolainen, Petri (Nokia - FI/Espoo) <petri.savolainen@nokia-bell-labs.com> wrote: > > >> -----Original Message----- >> From: lng-odp [mailto:lng-odp-bounces@lists.linaro.org] On Behalf Of Maxim >> Uvarov >> Sent: Wednesday, February 15, 2017 6:19 PM >> To: lng-odp@lists.linaro.org >> Subject: Re: [lng-odp] [API-NEXT PATCH v2 3/4] Revert "linux-generic: >> packet: implement reference apis" >> >> On 02/15/17 15:29, Petri Savolainen wrote: >> > This reverts commit 22b3986fea090986625f3255d57b64de35bbc475. >> > --- >> >> sign-off is missing. Checkpatch should warn about that. >> >> Maxim. > > This patch is output of the revert command as-is, and has couple of check patch warnings due to that. I can add sign-off there if that's needed, although I didn't do any modifications into the commit itself. > > Also 'make check' fails after this commit since the new implementation is added in the next commit. Anyway, I think this results cleaner git history as the revert is not modified (== not squashed with the new implementation). E.g. odp-dpdk can directly merge the new implementation, without the need to first merge the reverted one. > > -Petri >
> -----Original Message----- > From: Bill Fischofer [mailto:bill.fischofer@linaro.org] > Sent: Thursday, February 16, 2017 3:00 PM > To: Savolainen, Petri (Nokia - FI/Espoo) <petri.savolainen@nokia-bell- > labs.com> > Cc: Maxim Uvarov <maxim.uvarov@linaro.org>; lng-odp@lists.linaro.org > Subject: Re: [lng-odp] [API-NEXT PATCH v2 3/4] Revert "linux-generic: > packet: implement reference apis" > > I'm going to explicitly NAK this part just in case there was any > confusion. If we have a better replacement for the current packet > reference implementation then it can be posted the same way we've > revised the implementation of other APIs. Until then this is an > unnecessary step backwards. > > Parts 1 and 2 are good and should be merged. I did NAK the current implementation in the first place as being too complex for the task. Still it was merged. The same did happen with ordered queues. I did say that it is too complex and low performance for the job, but it was still merged. Later on, me and Matias did have to work a lot to re-implement it (get rid-off all unnecessary dependencies, etc). Just by comparing the volume of this reverted patch (3/4) and the next one (4/4), you get some idea which one is easier to maintain, extend and guarantee that it works in all cases. The first patch fills about 27 screens on the mail client, while the second one only 2. A proper reference implementation should be focused in couple of places where the counter is incremented and decremented. It may require some changes in the current packet/pool implementation to achieve that, but the changes should make the entire code more compact, cleaner and faster (== easier to maintain) - instead of larger, complex and slower (harder to maintain). We'll save everybody's time, if we do not merge these 500-600 lines spread over the packet/pool code, but the compact <70 lines (of patch 4/4) first - and then continue from there. Today, single reference packet performance is more important than multi-ref performance, since nobody is using multi-refs yet, and we do not have good test apps for it. Single ref packets are used everywhere, and the current code (this revert) is 25 - 60% slower in many of the packet API calls. -Petri
The packet reference code is already compact. The reason there are many lines is that we have many packet APIs and each of them must be prepared to accept references as well as non-references as input. If you want to be able to do zero-copy references (which we do) most of these changes are needed anyway. Can they be improved? Of course, but doing everything with copies because it has "zero impact" on non-references buys us nothing. It would be better to not put the packet reference code in at all than to put in non-performant stubs just to say we "support" these new APIs. Matias' measurements have been invaluable and will allow us to further tune this implementation. For example, from the current run odp_packet_alloc() is reported to be 40% greater pathlength than master, and yet when you look at the actual code the only difference is three header fields being initialized (unshared_len set to len, ref_hdr set to NULL, and ref_count set to 1). How can this be? Well, ref_count is an odp_atomic_u32_t and the initialization is done via odp_atomic_init(). Reference counts must be atomics because when you have actual shared segments two separate threads can be doing odp_packet_free() and/or odp_packet_ref() operations in parallel that hit the same segments and we don't use locks. However, when initially allocating segments we know they aren't shared so in this case it is safe to breach the atomic type and set the initial value directly without incurring the pipeline stalls and related sync overhead of an actual atomic operation. I've posted patch http://patches.opendataplane.org/patch/8143/ to make this one-line change and I suspect it will show that odp_packet_alloc() and odp_packet_alloc_multi() add negligible overhead compared to master. My guess is that similar small adjustments can address any other areas of concern. In the meantime if you have a better idea how to do references efficiently I'd be very interested in hearing it. On Thu, Feb 16, 2017 at 8:56 AM, Savolainen, Petri (Nokia - FI/Espoo) <petri.savolainen@nokia-bell-labs.com> wrote: > > >> -----Original Message----- >> From: Bill Fischofer [mailto:bill.fischofer@linaro.org] >> Sent: Thursday, February 16, 2017 3:00 PM >> To: Savolainen, Petri (Nokia - FI/Espoo) <petri.savolainen@nokia-bell- >> labs.com> >> Cc: Maxim Uvarov <maxim.uvarov@linaro.org>; lng-odp@lists.linaro.org >> Subject: Re: [lng-odp] [API-NEXT PATCH v2 3/4] Revert "linux-generic: >> packet: implement reference apis" >> >> I'm going to explicitly NAK this part just in case there was any >> confusion. If we have a better replacement for the current packet >> reference implementation then it can be posted the same way we've >> revised the implementation of other APIs. Until then this is an >> unnecessary step backwards. >> >> Parts 1 and 2 are good and should be merged. > > I did NAK the current implementation in the first place as being too complex for the task. Still it was merged. The same did happen with ordered queues. I did say that it is too complex and low performance for the job, but it was still merged. Later on, me and Matias did have to work a lot to re-implement it (get rid-off all unnecessary dependencies, etc). > > Just by comparing the volume of this reverted patch (3/4) and the next one (4/4), you get some idea which one is easier to maintain, extend and guarantee that it works in all cases. The first patch fills about 27 screens on the mail client, while the second one only 2. A proper reference implementation should be focused in couple of places where the counter is incremented and decremented. It may require some changes in the current packet/pool implementation to achieve that, but the changes should make the entire code more compact, cleaner and faster (== easier to maintain) - instead of larger, complex and slower (harder to maintain). > > We'll save everybody's time, if we do not merge these 500-600 lines spread over the packet/pool code, but the compact <70 lines (of patch 4/4) first - and then continue from there. > > Today, single reference packet performance is more important than multi-ref performance, since nobody is using multi-refs yet, and we do not have good test apps for it. Single ref packets are used everywhere, and the current code (this revert) is 25 - 60% slower in many of the packet API calls. > > -Petri > >
Continuing my efficiency review, I've posted patch http://patches.opendataplane.org/patch/8145/ that should help on the packet free side. If my theory is correct that adding atomic operations severely distorts these microbenchmarks because of its effect on pipelining, this second patch avoids atomic decrement operations on free paths unless the segment(s) being freed have a ref count > 1. I'd expect this and the previous patch to have a meaningful impact in reducing the measured overhead of api-next vs. master while still retaining the full zero-copy reference semantics that existed previously. On Thu, Feb 16, 2017 at 11:13 AM, Bill Fischofer <bill.fischofer@linaro.org> wrote: > The packet reference code is already compact. The reason there are > many lines is that we have many packet APIs and each of them must be > prepared to accept references as well as non-references as input. If > you want to be able to do zero-copy references (which we do) most of > these changes are needed anyway. Can they be improved? Of course, but > doing everything with copies because it has "zero impact" on > non-references buys us nothing. It would be better to not put the > packet reference code in at all than to put in non-performant stubs > just to say we "support" these new APIs. > > Matias' measurements have been invaluable and will allow us to further > tune this implementation. For example, from the current run > odp_packet_alloc() is reported to be 40% greater pathlength than > master, and yet when you look at the actual code the only difference > is three header fields being initialized (unshared_len set to len, > ref_hdr set to NULL, and ref_count set to 1). > > How can this be? Well, ref_count is an odp_atomic_u32_t and the > initialization is done via odp_atomic_init(). Reference counts must be > atomics because when you have actual shared segments two separate > threads can be doing odp_packet_free() and/or odp_packet_ref() > operations in parallel that hit the same segments and we don't use > locks. > > However, when initially allocating segments we know they aren't shared > so in this case it is safe to breach the atomic type and set the > initial value directly without incurring the pipeline stalls and > related sync overhead of an actual atomic operation. > > I've posted patch http://patches.opendataplane.org/patch/8143/ to make > this one-line change and I suspect it will show that > odp_packet_alloc() and odp_packet_alloc_multi() add negligible > overhead compared to master. My guess is that similar small > adjustments can address any other areas of concern. > > In the meantime if you have a better idea how to do references > efficiently I'd be very interested in hearing it. > > On Thu, Feb 16, 2017 at 8:56 AM, Savolainen, Petri (Nokia - FI/Espoo) > <petri.savolainen@nokia-bell-labs.com> wrote: >> >> >>> -----Original Message----- >>> From: Bill Fischofer [mailto:bill.fischofer@linaro.org] >>> Sent: Thursday, February 16, 2017 3:00 PM >>> To: Savolainen, Petri (Nokia - FI/Espoo) <petri.savolainen@nokia-bell- >>> labs.com> >>> Cc: Maxim Uvarov <maxim.uvarov@linaro.org>; lng-odp@lists.linaro.org >>> Subject: Re: [lng-odp] [API-NEXT PATCH v2 3/4] Revert "linux-generic: >>> packet: implement reference apis" >>> >>> I'm going to explicitly NAK this part just in case there was any >>> confusion. If we have a better replacement for the current packet >>> reference implementation then it can be posted the same way we've >>> revised the implementation of other APIs. Until then this is an >>> unnecessary step backwards. >>> >>> Parts 1 and 2 are good and should be merged. >> >> I did NAK the current implementation in the first place as being too complex for the task. Still it was merged. The same did happen with ordered queues. I did say that it is too complex and low performance for the job, but it was still merged. Later on, me and Matias did have to work a lot to re-implement it (get rid-off all unnecessary dependencies, etc). >> >> Just by comparing the volume of this reverted patch (3/4) and the next one (4/4), you get some idea which one is easier to maintain, extend and guarantee that it works in all cases. The first patch fills about 27 screens on the mail client, while the second one only 2. A proper reference implementation should be focused in couple of places where the counter is incremented and decremented. It may require some changes in the current packet/pool implementation to achieve that, but the changes should make the entire code more compact, cleaner and faster (== easier to maintain) - instead of larger, complex and slower (harder to maintain). >> >> We'll save everybody's time, if we do not merge these 500-600 lines spread over the packet/pool code, but the compact <70 lines (of patch 4/4) first - and then continue from there. >> >> Today, single reference packet performance is more important than multi-ref performance, since nobody is using multi-refs yet, and we do not have good test apps for it. Single ref packets are used everywhere, and the current code (this revert) is 25 - 60% slower in many of the packet API calls. >> >> -Petri >> >>
diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 42df5ac..e3ada5c 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -19,7 +19,6 @@ extern "C" { #include <odp/api/align.h> #include <odp/api/debug.h> -#include <odp_debug_internal.h> #include <odp_buffer_internal.h> #include <odp_pool_internal.h> #include <odp_buffer_inlines.h> @@ -169,7 +168,7 @@ typedef struct { * packet_init(). Because of this any new fields added must be reviewed for * initialization requirements. */ -typedef struct odp_packet_hdr_t { +typedef struct { /* common buffer header */ odp_buffer_hdr_t buf_hdr; @@ -185,13 +184,6 @@ typedef struct odp_packet_hdr_t { uint32_t headroom; uint32_t tailroom; - /* Fields used to support packet references */ - uint32_t unshared_len; - struct odp_packet_hdr_t *ref_hdr; - uint32_t ref_offset; - uint32_t ref_len; - odp_atomic_u32_t ref_count; - /* * Members below are not initialized by packet_init() */ @@ -220,55 +212,6 @@ static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) return (odp_packet_hdr_t *)buf_hdl_to_hdr((odp_buffer_t)pkt); } -static inline odp_packet_hdr_t *odp_packet_last_hdr(odp_packet_t pkt, - uint32_t *offset) -{ - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - odp_packet_hdr_t *prev_hdr = pkt_hdr; - uint32_t ref_offset = 0; - - while (pkt_hdr->ref_hdr) { - ref_offset = pkt_hdr->ref_offset; - prev_hdr = pkt_hdr; - pkt_hdr = pkt_hdr->ref_hdr; - } - - if (offset) { - if (prev_hdr != pkt_hdr) - ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; - *offset = ref_offset; - } - - return pkt_hdr; -} - -static inline odp_packet_hdr_t *odp_packet_prev_hdr(odp_packet_hdr_t *pkt_hdr, - odp_packet_hdr_t *cur_hdr, - uint32_t *offset) -{ - uint32_t ref_offset = 0; - odp_packet_hdr_t *prev_hdr = pkt_hdr; - - while (pkt_hdr->ref_hdr != cur_hdr) { - ref_offset = pkt_hdr->ref_offset; - prev_hdr = pkt_hdr; - pkt_hdr = pkt_hdr->ref_hdr; - } - - if (offset) { - if (prev_hdr != pkt_hdr) - ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; - *offset = ref_offset; - } - - return pkt_hdr; -} - -static inline odp_packet_t _odp_packet_hdl(odp_packet_hdr_t *pkt_hdr) -{ - return (odp_packet_t)odp_hdr_to_buf(&pkt_hdr->buf_hdr); -} - static inline void copy_packet_parser_metadata(odp_packet_hdr_t *src_hdr, odp_packet_hdr_t *dst_hdr) { @@ -291,41 +234,17 @@ static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) pkt_hdr->tailroom += len; pkt_hdr->frame_len -= len; - pkt_hdr->unshared_len -= len; pkt_hdr->buf_hdr.seg[last].len -= len; } static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) { - uint32_t pkt_len = 0; - uint32_t offset = 0; - - do { - pkt_len += pkt_hdr->frame_len - offset; - offset = pkt_hdr->ref_offset; - if (pkt_hdr->ref_hdr) - offset += (pkt_hdr->ref_hdr->frame_len - - pkt_hdr->ref_len); - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); - - return pkt_len; -} - -static inline uint32_t packet_ref_count(odp_packet_hdr_t *pkt_hdr) -{ - return odp_atomic_load_u32(&pkt_hdr->ref_count); -} - -static inline void packet_ref_count_set(odp_packet_hdr_t *pkt_hdr, uint32_t n) -{ - odp_atomic_init_u32(&pkt_hdr->ref_count, n); + return pkt_hdr->frame_len; } static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->frame_len = len; - pkt_hdr->unshared_len = len; } static inline int packet_parse_l2_not_done(packet_parser_t *prs) diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 3aadc4d..024f694 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -33,24 +33,13 @@ static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) return pkt_hdr->buf_hdr.handle.handle; } -static inline uint32_t packet_ref_inc(odp_packet_hdr_t *pkt_hdr) -{ - return odp_atomic_fetch_inc_u32(&pkt_hdr->ref_count); -} - -static inline uint32_t packet_ref_dec(odp_packet_hdr_t *pkt_hdr) -{ - return odp_atomic_fetch_dec_u32(&pkt_hdr->ref_count); -} - static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) { return pkt_hdr->buf_hdr.seg[seg_idx].len; } -static inline uint8_t *packet_seg_data(odp_packet_hdr_t *pkt_hdr, - uint32_t seg_idx) +static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) { return pkt_hdr->buf_hdr.seg[seg_idx].data; } @@ -63,11 +52,6 @@ static inline int packet_last_seg(odp_packet_hdr_t *pkt_hdr) return pkt_hdr->buf_hdr.segcount - 1; } -static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) -{ - return pkt_hdr->buf_hdr.seg[0].data; -} - static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr) { return packet_seg_len(pkt_hdr, 0); @@ -80,6 +64,11 @@ static inline uint32_t packet_last_seg_len(odp_packet_hdr_t *pkt_hdr) return packet_seg_len(pkt_hdr, last); } +static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) +{ + return pkt_hdr->buf_hdr.seg[0].data; +} + static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr) { int last = packet_last_seg(pkt_hdr); @@ -110,7 +99,6 @@ static inline void push_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->headroom -= len; pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; pkt_hdr->buf_hdr.seg[0].data -= len; pkt_hdr->buf_hdr.seg[0].len += len; } @@ -119,7 +107,6 @@ static inline void pull_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->headroom += len; pkt_hdr->frame_len -= len; - pkt_hdr->unshared_len -= len; pkt_hdr->buf_hdr.seg[0].data += len; pkt_hdr->buf_hdr.seg[0].len -= len; } @@ -130,7 +117,6 @@ static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) pkt_hdr->tailroom -= len; pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; pkt_hdr->buf_hdr.seg[last].len += len; } @@ -158,10 +144,6 @@ static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr; dst->buf_hdr.uarea_size = src->buf_hdr.uarea_size; - /* reference related metadata */ - dst->ref_len = src->ref_len; - dst->unshared_len = src->unshared_len; - /* segmentation data is not copied: * buf_hdr.seg[] * buf_hdr.segcount @@ -176,15 +158,7 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, int seg = 0; int seg_count = pkt_hdr->buf_hdr.segcount; - /* Special processing for references */ - while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); - offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr = pkt_hdr->ref_hdr; - seg_count = pkt_hdr->buf_hdr.segcount; - } - - if (odp_unlikely(offset > pkt_hdr->frame_len)) + if (odp_unlikely(offset >= pkt_hdr->frame_len)) return NULL; if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || seg_count == 1)) { @@ -233,9 +207,6 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) pkt_hdr->p.l2_offset = 0; pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; - - /* Ensure dummy pkt_hdrs used in I/O recv classification are valid */ - pkt_hdr->ref_hdr = NULL; } /** @@ -281,10 +252,6 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len, CONFIG_PACKET_TAILROOM; pkt_hdr->input = ODP_PKTIO_INVALID; - - /* By default packet has no references */ - pkt_hdr->unshared_len = len; - pkt_hdr->ref_hdr = NULL; } static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) @@ -297,7 +264,6 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) hdr->buf_hdr.seg[0].data = hdr->buf_hdr.base_data; hdr->buf_hdr.seg[0].len = BASE_LEN; - packet_ref_count_set(hdr, 1); /* Link segments */ if (CONFIG_PACKET_MAX_SEGS != 1) { @@ -307,7 +273,6 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) for (i = 1; i < num; i++) { odp_buffer_hdr_t *buf_hdr; - packet_ref_count_set(pkt_hdr[i], 1); buf_hdr = &pkt_hdr[i]->buf_hdr; hdr->buf_hdr.seg[i].hdr = buf_hdr; hdr->buf_hdr.seg[i].data = buf_hdr->base_data; @@ -411,10 +376,9 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, new_hdr->buf_hdr.seg[0].len = seg_len; packet_seg_copy_md(new_hdr, pkt_hdr); - new_hdr->frame_len = pkt_hdr->frame_len + len; - new_hdr->unshared_len = pkt_hdr->unshared_len + len; - new_hdr->headroom = pool->headroom + offset; - new_hdr->tailroom = pkt_hdr->tailroom; + new_hdr->frame_len = pkt_hdr->frame_len + len; + new_hdr->headroom = pool->headroom + offset; + new_hdr->tailroom = pkt_hdr->tailroom; pkt_hdr = new_hdr; } else { @@ -427,9 +391,8 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, last = packet_last_seg(pkt_hdr); pkt_hdr->buf_hdr.seg[last].len = seg_len; - pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; - pkt_hdr->tailroom = pool->tailroom + offset; + pkt_hdr->frame_len += len; + pkt_hdr->tailroom = pool->tailroom + offset; } return pkt_hdr; @@ -437,18 +400,13 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, static inline void free_bufs(odp_packet_hdr_t *pkt_hdr, int first, int num) { - int i, nfree; + int i; odp_buffer_t buf[num]; - for (i = 0, nfree = 0; i < num; i++) { - odp_packet_hdr_t *hdr = pkt_hdr->buf_hdr.seg[first + i].hdr; - - if (packet_ref_dec(hdr) == 1) - buf[nfree++] = buffer_handle(hdr); - } + for (i = 0; i < num; i++) + buf[i] = buffer_handle(pkt_hdr->buf_hdr.seg[first + i].hdr); - if (nfree > 0) - buffer_free_multi(buf, nfree); + buffer_free_multi(buf, num); } static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, @@ -459,15 +417,11 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, if (head) { odp_packet_hdr_t *new_hdr; - int i, nfree; + int i; odp_buffer_t buf[num]; - for (i = 0, nfree = 0; i < num; i++) { - new_hdr = pkt_hdr->buf_hdr.seg[i].hdr; - - if (packet_ref_dec(new_hdr) == 1) - buf[nfree++] = buffer_handle(new_hdr); - } + for (i = 0; i < num; i++) + buf[i] = buffer_handle(pkt_hdr->buf_hdr.seg[i].hdr); /* First remaining segment is the new packet descriptor */ new_hdr = pkt_hdr->buf_hdr.seg[num].hdr; @@ -476,17 +430,15 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, packet_seg_copy_md(new_hdr, pkt_hdr); /* Tailroom not changed */ - new_hdr->tailroom = pkt_hdr->tailroom; - new_hdr->headroom = seg_headroom(new_hdr, 0); - new_hdr->frame_len = pkt_hdr->frame_len - free_len; - new_hdr->unshared_len = pkt_hdr->unshared_len - free_len; + new_hdr->tailroom = pkt_hdr->tailroom; + new_hdr->headroom = seg_headroom(new_hdr, 0); + new_hdr->frame_len = pkt_hdr->frame_len - free_len; pull_head(new_hdr, pull_len); pkt_hdr = new_hdr; - if (nfree > 0) - buffer_free_multi(buf, nfree); + buffer_free_multi(buf, num); } else { /* Free last 'num' bufs */ free_bufs(pkt_hdr, num_remain, num); @@ -495,7 +447,6 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, * of the metadata. */ pkt_hdr->buf_hdr.segcount = num_remain; pkt_hdr->frame_len -= free_len; - pkt_hdr->unshared_len -= free_len; pkt_hdr->tailroom = seg_tailroom(pkt_hdr, num_remain - 1); pull_tail(pkt_hdr, pull_len); @@ -599,34 +550,45 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, return num; } -static inline void packet_free(odp_packet_hdr_t *pkt_hdr) -{ - odp_packet_hdr_t *ref_hdr; - uint32_t ref_count; - - do { - ref_hdr = pkt_hdr->ref_hdr; - ref_count = packet_ref_count(pkt_hdr) - 1; - free_bufs(pkt_hdr, 0, pkt_hdr->buf_hdr.segcount); - - if (ref_count == 1) - pkt_hdr->unshared_len = pkt_hdr->frame_len; - - pkt_hdr = ref_hdr; - } while (pkt_hdr); -} - void odp_packet_free(odp_packet_t pkt) { - packet_free(odp_packet_hdr(pkt)); + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + int num_seg = pkt_hdr->buf_hdr.segcount; + + if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) + buffer_free_multi((odp_buffer_t *)&pkt, 1); + else + free_bufs(pkt_hdr, 0, num_seg); } void odp_packet_free_multi(const odp_packet_t pkt[], int num) { - int i; + if (CONFIG_PACKET_MAX_SEGS == 1) { + buffer_free_multi((const odp_buffer_t * const)pkt, num); + } else { + odp_buffer_t buf[num * CONFIG_PACKET_MAX_SEGS]; + int i, j; + int bufs = 0; - for (i = 0; i < num; i++) - packet_free(odp_packet_hdr(pkt[i])); + for (i = 0; i < num; i++) { + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt[i]); + int num_seg = pkt_hdr->buf_hdr.segcount; + odp_buffer_hdr_t *buf_hdr = &pkt_hdr->buf_hdr; + + buf[bufs] = (odp_buffer_t)pkt[i]; + bufs++; + + if (odp_likely(num_seg == 1)) + continue; + + for (j = 1; j < num_seg; j++) { + buf[bufs] = buffer_handle(buf_hdr->seg[j].hdr); + bufs++; + } + } + + buffer_free_multi(buf, bufs); + } } int odp_packet_reset(odp_packet_t pkt, uint32_t len) @@ -637,9 +599,6 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) if (len > pool->headroom + pool->data_size + pool->tailroom) return -1; - if (pkt_hdr->ref_hdr) - packet_free(pkt_hdr->ref_hdr); - packet_init(pkt_hdr, len, 0); return 0; @@ -682,21 +641,15 @@ void *odp_packet_head(odp_packet_t pkt) uint32_t odp_packet_buf_len(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - uint32_t buf_len = 0; - do { - buf_len += pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); - - return buf_len; + return pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; } void *odp_packet_data(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - return packet_map(pkt_hdr, 0, NULL, NULL); + return packet_data(pkt_hdr); } uint32_t odp_packet_seg_len(odp_packet_t pkt) @@ -708,32 +661,7 @@ uint32_t odp_packet_seg_len(odp_packet_t pkt) uint32_t odp_packet_len(odp_packet_t pkt) { - return packet_len(odp_packet_hdr(pkt)); -} - -uint32_t odp_packet_unshared_len(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - uint32_t pkt_len = 0, offset = 0; - - do { - if (packet_ref_count(pkt_hdr) > 1) { - if (offset == 0) - pkt_len += pkt_hdr->unshared_len; - break; - } - - pkt_len += pkt_hdr->frame_len - offset; - offset = pkt_hdr->ref_offset; - - if (pkt_hdr->ref_hdr) - offset += (pkt_hdr->ref_hdr->frame_len - - pkt_hdr->ref_len); - - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); - - return pkt_len; + return odp_packet_hdr(pkt)->frame_len; } uint32_t odp_packet_headroom(odp_packet_t pkt) @@ -743,12 +671,12 @@ uint32_t odp_packet_headroom(odp_packet_t pkt) uint32_t odp_packet_tailroom(odp_packet_t pkt) { - return odp_packet_last_hdr(pkt, NULL)->tailroom; + return odp_packet_hdr(pkt)->tailroom; } void *odp_packet_tail(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(pkt, NULL); + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); return packet_tail(pkt_hdr); } @@ -942,7 +870,7 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); uint32_t frame_len = pkt_hdr->frame_len; - uint32_t headroom = pkt_hdr->headroom; + uint32_t headroom = pkt_hdr->headroom; int ret = 0; if (len > headroom) { @@ -957,46 +885,6 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, segs = pkt_hdr->buf_hdr.segcount; if (odp_unlikely((segs + num) > CONFIG_PACKET_MAX_SEGS)) { - /* Handle recursively via references when - * working with referenced packets since another - * thread may be accessing it concurrently via - * its reference to it. */ - if (packet_ref_count(pkt_hdr) > 1) { - odp_packet_t ref; - uint32_t unshared_len; - - push_head(pkt_hdr, headroom); - unshared_len = pkt_hdr->unshared_len; - ref = odp_packet_ref(*pkt, 0); - - if (ref == ODP_PACKET_INVALID) { - pull_head(pkt_hdr, headroom); - return -1; - } - - ret = odp_packet_extend_head(&ref, - len - headroom, - data_ptr, - seg_len); - - if (ret < 0) { - odp_packet_free(ref); - pull_head(pkt_hdr, headroom); - return -1; - } - - /* Since this is a special ref, the - * base pkt's unshared len is unchanged */ - pkt_hdr->unshared_len = unshared_len; - - /* Remove extra ref to the base pkt */ - odp_packet_free(*pkt); - - /* Return the ref as the extension result */ - *pkt = ref; - return 1; - } - /* Cannot directly add new segments */ odp_packet_hdr_t *new_hdr; int new_segs = 0; @@ -1048,7 +936,6 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, pkt_hdr->buf_hdr.segcount = segs; pkt_hdr->frame_len = frame_len; - pkt_hdr->unshared_len = frame_len; pkt_hdr->headroom = offset + pool->headroom; pkt_hdr->tailroom = pool->tailroom; @@ -1074,16 +961,11 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, push_head(pkt_hdr, len); } - if (data_ptr || seg_len) { - uint32_t seg_ln = 0; - void *data = packet_map(pkt_hdr, 0, &seg_ln, NULL); - - if (data_ptr) - *data_ptr = data; + if (data_ptr) + *data_ptr = packet_data(pkt_hdr); - if (seg_len) - *seg_len = seg_ln; - } + if (seg_len) + *seg_len = packet_first_seg_len(pkt_hdr); return ret; } @@ -1095,8 +977,6 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) if (len > pkt_hdr->frame_len) return NULL; - ODP_ASSERT(len <= pkt_hdr->unshared_len); - pull_head(pkt_hdr, len); return packet_data(pkt_hdr); } @@ -1104,35 +984,15 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len_out) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt), *nxt_hdr; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); uint32_t seg_len = packet_first_seg_len(pkt_hdr); - int ret = 0; - if (len > packet_len(pkt_hdr)) + if (len > pkt_hdr->frame_len) return -1; - ODP_ASSERT(len <= odp_packet_unshared_len(*pkt)); - - /* Special processing for references */ - while (len >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - nxt_hdr = pkt_hdr->ref_hdr; - len -= pkt_hdr->frame_len; - len += pkt_hdr->ref_offset + - (nxt_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr->ref_hdr = NULL; - packet_free(pkt_hdr); - pkt_hdr = nxt_hdr; - seg_len = packet_first_seg_len(pkt_hdr); - *pkt = packet_handle(pkt_hdr); - ret = 1; - } - - if (CONFIG_PACKET_MAX_SEGS == 1 || - len < seg_len || - pkt_hdr->buf_hdr.segcount == 1) { + if (len < seg_len) { pull_head(pkt_hdr, len); - } else { + } else if (CONFIG_PACKET_MAX_SEGS != 1) { int num = 0; uint32_t pull_len = 0; @@ -1147,29 +1007,23 @@ int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, *pkt = packet_handle(pkt_hdr); } - if (data_ptr || seg_len_out) { - void *data_head = packet_map(pkt_hdr, 0, &seg_len, NULL); - - if (data_ptr) - *data_ptr = data_head; + if (data_ptr) + *data_ptr = packet_data(pkt_hdr); - if (seg_len_out) - *seg_len_out = seg_len; - } + if (seg_len_out) + *seg_len_out = packet_first_seg_len(pkt_hdr); - return ret; + return 0; } void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) { - odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(pkt, NULL); + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); void *old_tail; if (len > pkt_hdr->tailroom) return NULL; - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - old_tail = packet_tail(pkt_hdr); push_tail(pkt_hdr, len); @@ -1179,14 +1033,12 @@ void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len_out) { - odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(*pkt, NULL); + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); uint32_t frame_len = pkt_hdr->frame_len; uint32_t tailroom = pkt_hdr->tailroom; uint32_t tail_off = frame_len; int ret = 0; - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - if (len > tailroom) { pool_t *pool = pool_entry_from_hdl(pkt_hdr->buf_hdr.pool_hdl); int num; @@ -1277,7 +1129,6 @@ void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) if (len > packet_last_seg_len(pkt_hdr)) return NULL; - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); pull_tail(pkt_hdr, len); return packet_tail(pkt_hdr); @@ -1288,34 +1139,17 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, { int last; uint32_t seg_len; - uint32_t offset; - odp_packet_hdr_t *first_hdr = odp_packet_hdr(*pkt); - odp_packet_hdr_t *pkt_hdr, *prev_hdr; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); - if (len > packet_len(first_hdr)) + if (len > pkt_hdr->frame_len) return -1; - pkt_hdr = odp_packet_last_hdr(*pkt, &offset); - - /* Special processing for references */ - while (len >= pkt_hdr->frame_len - offset && first_hdr->ref_hdr) { - len -= (pkt_hdr->frame_len - offset); - prev_hdr = odp_packet_prev_hdr(first_hdr, pkt_hdr, &offset); - ODP_ASSERT(packet_ref_count(prev_hdr) == 1); - prev_hdr->ref_hdr = NULL; - packet_free(pkt_hdr); - pkt_hdr = prev_hdr; - } - - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); last = packet_last_seg(pkt_hdr); seg_len = packet_seg_len(pkt_hdr, last); - if (CONFIG_PACKET_MAX_SEGS == 1 || - len < seg_len || - pkt_hdr->buf_hdr.segcount == 1) { + if (len < seg_len) { pull_tail(pkt_hdr, len); - } else { + } else if (CONFIG_PACKET_MAX_SEGS != 1) { int num = 0; uint32_t pull_len = 0; @@ -1522,50 +1356,35 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) int odp_packet_is_segmented(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - - return pkt_hdr->buf_hdr.segcount > 1 || pkt_hdr->ref_hdr != NULL; + return odp_packet_hdr(pkt)->buf_hdr.segcount > 1; } int odp_packet_num_segs(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - uint32_t segcount = 0, i; - uint32_t seg_offset = 0, offset; - - do { - segcount += pkt_hdr->buf_hdr.segcount - seg_offset; - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - if (pkt_hdr) { - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - } - } while (pkt_hdr); - return segcount; + return pkt_hdr->buf_hdr.segcount; } -odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt ODP_UNUSED) +odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) { + (void)pkt; + return 0; } odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) { - return (odp_packet_seg_t)(odp_packet_num_segs(pkt) - 1); + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + return packet_last_seg(pkt_hdr); } odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (odp_unlikely(seg >= packet_last_seg(pkt_hdr))) + if (odp_unlikely(seg >= (odp_packet_seg_t)packet_last_seg(pkt_hdr))) return ODP_PACKET_SEG_INVALID; return seg + 1; @@ -1581,51 +1400,21 @@ odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - uint32_t seg_offset = 0, offset = 0, i; - - while (seg >= pkt_hdr->buf_hdr.segcount - seg_offset && - pkt_hdr->ref_hdr) { - seg -= (pkt_hdr->buf_hdr.segcount - seg_offset); - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - } - if (odp_unlikely(seg + seg_offset >= pkt_hdr->buf_hdr.segcount)) + if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount)) return NULL; - return packet_seg_data(pkt_hdr, seg + seg_offset) + offset; + return packet_seg_data(pkt_hdr, seg); } uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - uint32_t seg_offset = 0, offset = 0, i; - - while (seg >= pkt_hdr->buf_hdr.segcount - seg_offset && - pkt_hdr->ref_hdr) { - seg -= (pkt_hdr->buf_hdr.segcount - seg_offset); - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - } - if (odp_unlikely(seg + seg_offset >= pkt_hdr->buf_hdr.segcount)) + if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount)) return 0; - return packet_seg_len(pkt_hdr, seg + seg_offset) - offset; + return packet_seg_len(pkt_hdr, seg); } /* @@ -1639,14 +1428,12 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) { odp_packet_t pkt = *pkt_ptr; odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - uint32_t pktlen = packet_len(pkt_hdr); + uint32_t pktlen = pkt_hdr->frame_len; odp_packet_t newpkt; if (offset > pktlen) return -1; - ODP_ASSERT(odp_packet_unshared_len(*pkt_ptr) >= offset); - newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen + len); if (newpkt == ODP_PACKET_INVALID) @@ -1709,8 +1496,6 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len, if (align > ODP_CACHE_LINE_SIZE) return -1; - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); - if (seglen >= len) { misalign = align <= 1 ? 0 : ROUNDUP_ALIGN(uaddr, align) - uaddr; @@ -1750,13 +1535,10 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) uint32_t dst_len = dst_hdr->frame_len; uint32_t src_len = src_hdr->frame_len; - ODP_ASSERT(packet_ref_count(dst_hdr) == 1); - /* Do a copy if resulting packet would be out of segments or packets - * are from different pools or src is a reference. */ + * are from different pools. */ if (odp_unlikely((dst_segs + src_segs) > CONFIG_PACKET_MAX_SEGS) || - odp_unlikely(dst_pool != src_pool) || - odp_unlikely(packet_ref_count(src_hdr)) > 1) { + odp_unlikely(dst_pool != src_pool)) { if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) { (void)odp_packet_copy_from_pkt(*dst, dst_len, src, 0, src_len); @@ -1771,9 +1553,8 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) add_all_segs(dst_hdr, src_hdr); - dst_hdr->frame_len = dst_len + src_len; - dst_hdr->unshared_len = dst_len + src_len; - dst_hdr->tailroom = src_hdr->tailroom; + dst_hdr->frame_len = dst_len + src_len; + dst_hdr->tailroom = src_hdr->tailroom; /* Data was not moved in memory */ return 0; @@ -1786,7 +1567,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) if (len >= pktlen || tail == NULL) return -1; - ODP_ASSERT(odp_packet_unshared_len(*pkt) >= len); *tail = odp_packet_copy_part(*pkt, len, pktlen - len, odp_packet_pool(*pkt)); @@ -1797,109 +1577,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) } /* - * References - */ - -static inline void packet_ref(odp_packet_hdr_t *pkt_hdr) -{ - uint32_t i; - odp_packet_hdr_t *hdr; - - do { - for (i = 0; i < pkt_hdr->buf_hdr.segcount; i++) { - hdr = pkt_hdr->buf_hdr.seg[i].hdr; - packet_ref_inc(hdr); - } - - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); -} - -static inline odp_packet_t packet_splice(odp_packet_hdr_t *pkt_hdr, - uint32_t offset, - odp_packet_hdr_t *ref_hdr) -{ - /* Catch attempted references to stale handles in debug builds */ - ODP_ASSERT(packet_ref_count(pkt_hdr) > 0); - - /* Splicing is from the last section of src pkt */ - while (ref_hdr->ref_hdr) - ref_hdr = ref_hdr->ref_hdr; - - /* Find section where splice begins */ - while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); - offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr = pkt_hdr->ref_hdr; - } - - ref_hdr->ref_hdr = pkt_hdr; - ref_hdr->ref_offset = offset; - ref_hdr->ref_len = pkt_hdr->frame_len; - - if (offset < pkt_hdr->unshared_len) - pkt_hdr->unshared_len = offset; - - packet_ref(pkt_hdr); - return _odp_packet_hdl(ref_hdr); -} - -odp_packet_t odp_packet_ref_static(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - - pkt_hdr->unshared_len = 0; - packet_ref(pkt_hdr); - return pkt; -} - -odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) -{ - odp_packet_t hdr; - odp_packet_hdr_t *pkt_hdr; - - if (pkt == ODP_PACKET_INVALID) - return ODP_PACKET_INVALID; - - pkt_hdr = odp_packet_hdr(pkt); - if (offset >= packet_len(pkt_hdr)) - return ODP_PACKET_INVALID; - - hdr = odp_packet_alloc(odp_packet_pool(pkt), 0); - - if (hdr == ODP_PACKET_INVALID) - return ODP_PACKET_INVALID; - - return packet_splice(pkt_hdr, offset, odp_packet_hdr(hdr)); -} - -odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, - odp_packet_t hdr) -{ - odp_packet_hdr_t *pkt_hdr; - - if (pkt == ODP_PACKET_INVALID || - hdr == ODP_PACKET_INVALID || - pkt == hdr) - return ODP_PACKET_INVALID; - - ODP_ASSERT(odp_packet_has_ref(hdr) == 0); - - pkt_hdr = odp_packet_hdr(pkt); - if (offset >= packet_len(pkt_hdr)) - return ODP_PACKET_INVALID; - - return packet_splice(pkt_hdr, offset, odp_packet_hdr(hdr)); -} - -int odp_packet_has_ref(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - - return pkt_hdr->ref_hdr != NULL || packet_ref_count(pkt_hdr) > 1; -} - -/* * * Copy * ******************************************************** @@ -1908,7 +1585,8 @@ int odp_packet_has_ref(odp_packet_t pkt) odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool) { - uint32_t pktlen = odp_packet_len(pkt); + odp_packet_hdr_t *srchdr = odp_packet_hdr(pkt); + uint32_t pktlen = srchdr->frame_len; odp_packet_t newpkt = odp_packet_alloc(pool, pktlen); if (newpkt != ODP_PACKET_INVALID) { @@ -1947,7 +1625,7 @@ int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t offset, uint8_t *dstaddr = (uint8_t *)dst; odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (offset + len > packet_len(pkt_hdr)) + if (offset + len > pkt_hdr->frame_len) return -1; while (len > 0) { @@ -1971,11 +1649,9 @@ int odp_packet_copy_from_mem(odp_packet_t pkt, uint32_t offset, const uint8_t *srcaddr = (const uint8_t *)src; odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (offset + len > packet_len(pkt_hdr)) + if (offset + len > pkt_hdr->frame_len) return -1; - ODP_ASSERT(odp_packet_unshared_len(pkt) >= offset + len); - while (len > 0) { mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); cpylen = len > seglen ? seglen : len; @@ -2001,12 +1677,10 @@ int odp_packet_copy_from_pkt(odp_packet_t dst, uint32_t dst_offset, uint32_t src_seglen = 0; /* GCC */ int overlap; - if (dst_offset + len > packet_len(dst_hdr) || - src_offset + len > packet_len(src_hdr)) + if (dst_offset + len > dst_hdr->frame_len || + src_offset + len > src_hdr->frame_len) return -1; - ODP_ASSERT(odp_packet_unshared_len(dst) >= dst_offset + len); - overlap = (dst_hdr == src_hdr && ((dst_offset <= src_offset && dst_offset + len >= src_offset) || @@ -2090,7 +1764,7 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n - len, " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); len += snprintf(&str[len], n - len, - " frame_len %" PRIu32 "\n", packet_len(hdr)); + " frame_len %" PRIu32 "\n", hdr->frame_len); len += snprintf(&str[len], n - len, " input %" PRIu64 "\n", odp_pktio_to_u64(hdr->input));