| Message ID | 1486990410-14928-4-git-send-email-petri.savolainen@linaro.org |
|---|---|
| State | Superseded |
| Headers | show |
| Series | Packet references as copy | expand |
This is a pretty big step backwards and without substantial justification I'm not sure why we would consider doing this. The performance characteristics of this base code has already been improved by both patch http://patches.opendataplane.org/patch/7879/ as well as patch series http://patches.opendataplane.org/project/lng-odp/list/?series=47 which are both still awaiting review. If further improvements are needed then we should do so, as we've done with all other odp-linux implementation work. If incremental improvement is not feasible and a complete rewrite is available, then it should be posted as a replacement patch for review the same way we did with ordered queues, pools, etc. Until then there's no need to make this type of drastic reversion without providing a better replacement implementation as part of the same patch series. On Mon, Feb 13, 2017 at 6:53 AM, Petri Savolainen <petri.savolainen@linaro.org> wrote: > This reverts commit 22b3986fea090986625f3255d57b64de35bbc475. > --- > .../linux-generic/include/odp_packet_internal.h | 85 +--- > platform/linux-generic/odp_packet.c | 536 ++++----------------- > 2 files changed, 107 insertions(+), 514 deletions(-) > > diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h > index 42df5ac..e3ada5c 100644 > --- a/platform/linux-generic/include/odp_packet_internal.h > +++ b/platform/linux-generic/include/odp_packet_internal.h > @@ -19,7 +19,6 @@ extern "C" { > > #include <odp/api/align.h> > #include <odp/api/debug.h> > -#include <odp_debug_internal.h> > #include <odp_buffer_internal.h> > #include <odp_pool_internal.h> > #include <odp_buffer_inlines.h> > @@ -169,7 +168,7 @@ typedef struct { > * packet_init(). Because of this any new fields added must be reviewed for > * initialization requirements. > */ > -typedef struct odp_packet_hdr_t { > +typedef struct { > /* common buffer header */ > odp_buffer_hdr_t buf_hdr; > > @@ -185,13 +184,6 @@ typedef struct odp_packet_hdr_t { > uint32_t headroom; > uint32_t tailroom; > > - /* Fields used to support packet references */ > - uint32_t unshared_len; > - struct odp_packet_hdr_t *ref_hdr; > - uint32_t ref_offset; > - uint32_t ref_len; > - odp_atomic_u32_t ref_count; > - > /* > * Members below are not initialized by packet_init() > */ > @@ -220,55 +212,6 @@ static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) > return (odp_packet_hdr_t *)buf_hdl_to_hdr((odp_buffer_t)pkt); > } > > -static inline odp_packet_hdr_t *odp_packet_last_hdr(odp_packet_t pkt, > - uint32_t *offset) > -{ > - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - odp_packet_hdr_t *prev_hdr = pkt_hdr; > - uint32_t ref_offset = 0; > - > - while (pkt_hdr->ref_hdr) { > - ref_offset = pkt_hdr->ref_offset; > - prev_hdr = pkt_hdr; > - pkt_hdr = pkt_hdr->ref_hdr; > - } > - > - if (offset) { > - if (prev_hdr != pkt_hdr) > - ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; > - *offset = ref_offset; > - } > - > - return pkt_hdr; > -} > - > -static inline odp_packet_hdr_t *odp_packet_prev_hdr(odp_packet_hdr_t *pkt_hdr, > - odp_packet_hdr_t *cur_hdr, > - uint32_t *offset) > -{ > - uint32_t ref_offset = 0; > - odp_packet_hdr_t *prev_hdr = pkt_hdr; > - > - while (pkt_hdr->ref_hdr != cur_hdr) { > - ref_offset = pkt_hdr->ref_offset; > - prev_hdr = pkt_hdr; > - pkt_hdr = pkt_hdr->ref_hdr; > - } > - > - if (offset) { > - if (prev_hdr != pkt_hdr) > - ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; > - *offset = ref_offset; > - } > - > - return pkt_hdr; > -} > - > -static inline odp_packet_t _odp_packet_hdl(odp_packet_hdr_t *pkt_hdr) > -{ > - return (odp_packet_t)odp_hdr_to_buf(&pkt_hdr->buf_hdr); > -} > - > static inline void copy_packet_parser_metadata(odp_packet_hdr_t *src_hdr, > odp_packet_hdr_t *dst_hdr) > { > @@ -291,41 +234,17 @@ static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) > > pkt_hdr->tailroom += len; > pkt_hdr->frame_len -= len; > - pkt_hdr->unshared_len -= len; > pkt_hdr->buf_hdr.seg[last].len -= len; > } > > static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) > { > - uint32_t pkt_len = 0; > - uint32_t offset = 0; > - > - do { > - pkt_len += pkt_hdr->frame_len - offset; > - offset = pkt_hdr->ref_offset; > - if (pkt_hdr->ref_hdr) > - offset += (pkt_hdr->ref_hdr->frame_len - > - pkt_hdr->ref_len); > - pkt_hdr = pkt_hdr->ref_hdr; > - } while (pkt_hdr); > - > - return pkt_len; > -} > - > -static inline uint32_t packet_ref_count(odp_packet_hdr_t *pkt_hdr) > -{ > - return odp_atomic_load_u32(&pkt_hdr->ref_count); > -} > - > -static inline void packet_ref_count_set(odp_packet_hdr_t *pkt_hdr, uint32_t n) > -{ > - odp_atomic_init_u32(&pkt_hdr->ref_count, n); > + return pkt_hdr->frame_len; > } > > static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) > { > pkt_hdr->frame_len = len; > - pkt_hdr->unshared_len = len; > } > > static inline int packet_parse_l2_not_done(packet_parser_t *prs) > diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c > index 3aadc4d..024f694 100644 > --- a/platform/linux-generic/odp_packet.c > +++ b/platform/linux-generic/odp_packet.c > @@ -33,24 +33,13 @@ static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) > return pkt_hdr->buf_hdr.handle.handle; > } > > -static inline uint32_t packet_ref_inc(odp_packet_hdr_t *pkt_hdr) > -{ > - return odp_atomic_fetch_inc_u32(&pkt_hdr->ref_count); > -} > - > -static inline uint32_t packet_ref_dec(odp_packet_hdr_t *pkt_hdr) > -{ > - return odp_atomic_fetch_dec_u32(&pkt_hdr->ref_count); > -} > - > static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, > uint32_t seg_idx) > { > return pkt_hdr->buf_hdr.seg[seg_idx].len; > } > > -static inline uint8_t *packet_seg_data(odp_packet_hdr_t *pkt_hdr, > - uint32_t seg_idx) > +static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) > { > return pkt_hdr->buf_hdr.seg[seg_idx].data; > } > @@ -63,11 +52,6 @@ static inline int packet_last_seg(odp_packet_hdr_t *pkt_hdr) > return pkt_hdr->buf_hdr.segcount - 1; > } > > -static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) > -{ > - return pkt_hdr->buf_hdr.seg[0].data; > -} > - > static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr) > { > return packet_seg_len(pkt_hdr, 0); > @@ -80,6 +64,11 @@ static inline uint32_t packet_last_seg_len(odp_packet_hdr_t *pkt_hdr) > return packet_seg_len(pkt_hdr, last); > } > > +static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) > +{ > + return pkt_hdr->buf_hdr.seg[0].data; > +} > + > static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr) > { > int last = packet_last_seg(pkt_hdr); > @@ -110,7 +99,6 @@ static inline void push_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) > { > pkt_hdr->headroom -= len; > pkt_hdr->frame_len += len; > - pkt_hdr->unshared_len += len; > pkt_hdr->buf_hdr.seg[0].data -= len; > pkt_hdr->buf_hdr.seg[0].len += len; > } > @@ -119,7 +107,6 @@ static inline void pull_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) > { > pkt_hdr->headroom += len; > pkt_hdr->frame_len -= len; > - pkt_hdr->unshared_len -= len; > pkt_hdr->buf_hdr.seg[0].data += len; > pkt_hdr->buf_hdr.seg[0].len -= len; > } > @@ -130,7 +117,6 @@ static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) > > pkt_hdr->tailroom -= len; > pkt_hdr->frame_len += len; > - pkt_hdr->unshared_len += len; > pkt_hdr->buf_hdr.seg[last].len += len; > } > > @@ -158,10 +144,6 @@ static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, > dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr; > dst->buf_hdr.uarea_size = src->buf_hdr.uarea_size; > > - /* reference related metadata */ > - dst->ref_len = src->ref_len; > - dst->unshared_len = src->unshared_len; > - > /* segmentation data is not copied: > * buf_hdr.seg[] > * buf_hdr.segcount > @@ -176,15 +158,7 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, > int seg = 0; > int seg_count = pkt_hdr->buf_hdr.segcount; > > - /* Special processing for references */ > - while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { > - offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); > - offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); > - pkt_hdr = pkt_hdr->ref_hdr; > - seg_count = pkt_hdr->buf_hdr.segcount; > - } > - > - if (odp_unlikely(offset > pkt_hdr->frame_len)) > + if (odp_unlikely(offset >= pkt_hdr->frame_len)) > return NULL; > > if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || seg_count == 1)) { > @@ -233,9 +207,6 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) > pkt_hdr->p.l2_offset = 0; > pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; > pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; > - > - /* Ensure dummy pkt_hdrs used in I/O recv classification are valid */ > - pkt_hdr->ref_hdr = NULL; > } > > /** > @@ -281,10 +252,6 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len, > CONFIG_PACKET_TAILROOM; > > pkt_hdr->input = ODP_PKTIO_INVALID; > - > - /* By default packet has no references */ > - pkt_hdr->unshared_len = len; > - pkt_hdr->ref_hdr = NULL; > } > > static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) > @@ -297,7 +264,6 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) > > hdr->buf_hdr.seg[0].data = hdr->buf_hdr.base_data; > hdr->buf_hdr.seg[0].len = BASE_LEN; > - packet_ref_count_set(hdr, 1); > > /* Link segments */ > if (CONFIG_PACKET_MAX_SEGS != 1) { > @@ -307,7 +273,6 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) > for (i = 1; i < num; i++) { > odp_buffer_hdr_t *buf_hdr; > > - packet_ref_count_set(pkt_hdr[i], 1); > buf_hdr = &pkt_hdr[i]->buf_hdr; > hdr->buf_hdr.seg[i].hdr = buf_hdr; > hdr->buf_hdr.seg[i].data = buf_hdr->base_data; > @@ -411,10 +376,9 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, > new_hdr->buf_hdr.seg[0].len = seg_len; > > packet_seg_copy_md(new_hdr, pkt_hdr); > - new_hdr->frame_len = pkt_hdr->frame_len + len; > - new_hdr->unshared_len = pkt_hdr->unshared_len + len; > - new_hdr->headroom = pool->headroom + offset; > - new_hdr->tailroom = pkt_hdr->tailroom; > + new_hdr->frame_len = pkt_hdr->frame_len + len; > + new_hdr->headroom = pool->headroom + offset; > + new_hdr->tailroom = pkt_hdr->tailroom; > > pkt_hdr = new_hdr; > } else { > @@ -427,9 +391,8 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, > last = packet_last_seg(pkt_hdr); > pkt_hdr->buf_hdr.seg[last].len = seg_len; > > - pkt_hdr->frame_len += len; > - pkt_hdr->unshared_len += len; > - pkt_hdr->tailroom = pool->tailroom + offset; > + pkt_hdr->frame_len += len; > + pkt_hdr->tailroom = pool->tailroom + offset; > } > > return pkt_hdr; > @@ -437,18 +400,13 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, > > static inline void free_bufs(odp_packet_hdr_t *pkt_hdr, int first, int num) > { > - int i, nfree; > + int i; > odp_buffer_t buf[num]; > > - for (i = 0, nfree = 0; i < num; i++) { > - odp_packet_hdr_t *hdr = pkt_hdr->buf_hdr.seg[first + i].hdr; > - > - if (packet_ref_dec(hdr) == 1) > - buf[nfree++] = buffer_handle(hdr); > - } > + for (i = 0; i < num; i++) > + buf[i] = buffer_handle(pkt_hdr->buf_hdr.seg[first + i].hdr); > > - if (nfree > 0) > - buffer_free_multi(buf, nfree); > + buffer_free_multi(buf, num); > } > > static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, > @@ -459,15 +417,11 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, > > if (head) { > odp_packet_hdr_t *new_hdr; > - int i, nfree; > + int i; > odp_buffer_t buf[num]; > > - for (i = 0, nfree = 0; i < num; i++) { > - new_hdr = pkt_hdr->buf_hdr.seg[i].hdr; > - > - if (packet_ref_dec(new_hdr) == 1) > - buf[nfree++] = buffer_handle(new_hdr); > - } > + for (i = 0; i < num; i++) > + buf[i] = buffer_handle(pkt_hdr->buf_hdr.seg[i].hdr); > > /* First remaining segment is the new packet descriptor */ > new_hdr = pkt_hdr->buf_hdr.seg[num].hdr; > @@ -476,17 +430,15 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, > packet_seg_copy_md(new_hdr, pkt_hdr); > > /* Tailroom not changed */ > - new_hdr->tailroom = pkt_hdr->tailroom; > - new_hdr->headroom = seg_headroom(new_hdr, 0); > - new_hdr->frame_len = pkt_hdr->frame_len - free_len; > - new_hdr->unshared_len = pkt_hdr->unshared_len - free_len; > + new_hdr->tailroom = pkt_hdr->tailroom; > + new_hdr->headroom = seg_headroom(new_hdr, 0); > + new_hdr->frame_len = pkt_hdr->frame_len - free_len; > > pull_head(new_hdr, pull_len); > > pkt_hdr = new_hdr; > > - if (nfree > 0) > - buffer_free_multi(buf, nfree); > + buffer_free_multi(buf, num); > } else { > /* Free last 'num' bufs */ > free_bufs(pkt_hdr, num_remain, num); > @@ -495,7 +447,6 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, > * of the metadata. */ > pkt_hdr->buf_hdr.segcount = num_remain; > pkt_hdr->frame_len -= free_len; > - pkt_hdr->unshared_len -= free_len; > pkt_hdr->tailroom = seg_tailroom(pkt_hdr, num_remain - 1); > > pull_tail(pkt_hdr, pull_len); > @@ -599,34 +550,45 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, > return num; > } > > -static inline void packet_free(odp_packet_hdr_t *pkt_hdr) > -{ > - odp_packet_hdr_t *ref_hdr; > - uint32_t ref_count; > - > - do { > - ref_hdr = pkt_hdr->ref_hdr; > - ref_count = packet_ref_count(pkt_hdr) - 1; > - free_bufs(pkt_hdr, 0, pkt_hdr->buf_hdr.segcount); > - > - if (ref_count == 1) > - pkt_hdr->unshared_len = pkt_hdr->frame_len; > - > - pkt_hdr = ref_hdr; > - } while (pkt_hdr); > -} > - > void odp_packet_free(odp_packet_t pkt) > { > - packet_free(odp_packet_hdr(pkt)); > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + int num_seg = pkt_hdr->buf_hdr.segcount; > + > + if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) > + buffer_free_multi((odp_buffer_t *)&pkt, 1); > + else > + free_bufs(pkt_hdr, 0, num_seg); > } > > void odp_packet_free_multi(const odp_packet_t pkt[], int num) > { > - int i; > + if (CONFIG_PACKET_MAX_SEGS == 1) { > + buffer_free_multi((const odp_buffer_t * const)pkt, num); > + } else { > + odp_buffer_t buf[num * CONFIG_PACKET_MAX_SEGS]; > + int i, j; > + int bufs = 0; > > - for (i = 0; i < num; i++) > - packet_free(odp_packet_hdr(pkt[i])); > + for (i = 0; i < num; i++) { > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt[i]); > + int num_seg = pkt_hdr->buf_hdr.segcount; > + odp_buffer_hdr_t *buf_hdr = &pkt_hdr->buf_hdr; > + > + buf[bufs] = (odp_buffer_t)pkt[i]; > + bufs++; > + > + if (odp_likely(num_seg == 1)) > + continue; > + > + for (j = 1; j < num_seg; j++) { > + buf[bufs] = buffer_handle(buf_hdr->seg[j].hdr); > + bufs++; > + } > + } > + > + buffer_free_multi(buf, bufs); > + } > } > > int odp_packet_reset(odp_packet_t pkt, uint32_t len) > @@ -637,9 +599,6 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) > if (len > pool->headroom + pool->data_size + pool->tailroom) > return -1; > > - if (pkt_hdr->ref_hdr) > - packet_free(pkt_hdr->ref_hdr); > - > packet_init(pkt_hdr, len, 0); > > return 0; > @@ -682,21 +641,15 @@ void *odp_packet_head(odp_packet_t pkt) > uint32_t odp_packet_buf_len(odp_packet_t pkt) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - uint32_t buf_len = 0; > > - do { > - buf_len += pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; > - pkt_hdr = pkt_hdr->ref_hdr; > - } while (pkt_hdr); > - > - return buf_len; > + return pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; > } > > void *odp_packet_data(odp_packet_t pkt) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - return packet_map(pkt_hdr, 0, NULL, NULL); > + return packet_data(pkt_hdr); > } > > uint32_t odp_packet_seg_len(odp_packet_t pkt) > @@ -708,32 +661,7 @@ uint32_t odp_packet_seg_len(odp_packet_t pkt) > > uint32_t odp_packet_len(odp_packet_t pkt) > { > - return packet_len(odp_packet_hdr(pkt)); > -} > - > -uint32_t odp_packet_unshared_len(odp_packet_t pkt) > -{ > - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - uint32_t pkt_len = 0, offset = 0; > - > - do { > - if (packet_ref_count(pkt_hdr) > 1) { > - if (offset == 0) > - pkt_len += pkt_hdr->unshared_len; > - break; > - } > - > - pkt_len += pkt_hdr->frame_len - offset; > - offset = pkt_hdr->ref_offset; > - > - if (pkt_hdr->ref_hdr) > - offset += (pkt_hdr->ref_hdr->frame_len - > - pkt_hdr->ref_len); > - > - pkt_hdr = pkt_hdr->ref_hdr; > - } while (pkt_hdr); > - > - return pkt_len; > + return odp_packet_hdr(pkt)->frame_len; > } > > uint32_t odp_packet_headroom(odp_packet_t pkt) > @@ -743,12 +671,12 @@ uint32_t odp_packet_headroom(odp_packet_t pkt) > > uint32_t odp_packet_tailroom(odp_packet_t pkt) > { > - return odp_packet_last_hdr(pkt, NULL)->tailroom; > + return odp_packet_hdr(pkt)->tailroom; > } > > void *odp_packet_tail(odp_packet_t pkt) > { > - odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(pkt, NULL); > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > return packet_tail(pkt_hdr); > } > @@ -942,7 +870,7 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); > uint32_t frame_len = pkt_hdr->frame_len; > - uint32_t headroom = pkt_hdr->headroom; > + uint32_t headroom = pkt_hdr->headroom; > int ret = 0; > > if (len > headroom) { > @@ -957,46 +885,6 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, > segs = pkt_hdr->buf_hdr.segcount; > > if (odp_unlikely((segs + num) > CONFIG_PACKET_MAX_SEGS)) { > - /* Handle recursively via references when > - * working with referenced packets since another > - * thread may be accessing it concurrently via > - * its reference to it. */ > - if (packet_ref_count(pkt_hdr) > 1) { > - odp_packet_t ref; > - uint32_t unshared_len; > - > - push_head(pkt_hdr, headroom); > - unshared_len = pkt_hdr->unshared_len; > - ref = odp_packet_ref(*pkt, 0); > - > - if (ref == ODP_PACKET_INVALID) { > - pull_head(pkt_hdr, headroom); > - return -1; > - } > - > - ret = odp_packet_extend_head(&ref, > - len - headroom, > - data_ptr, > - seg_len); > - > - if (ret < 0) { > - odp_packet_free(ref); > - pull_head(pkt_hdr, headroom); > - return -1; > - } > - > - /* Since this is a special ref, the > - * base pkt's unshared len is unchanged */ > - pkt_hdr->unshared_len = unshared_len; > - > - /* Remove extra ref to the base pkt */ > - odp_packet_free(*pkt); > - > - /* Return the ref as the extension result */ > - *pkt = ref; > - return 1; > - } > - > /* Cannot directly add new segments */ > odp_packet_hdr_t *new_hdr; > int new_segs = 0; > @@ -1048,7 +936,6 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, > > pkt_hdr->buf_hdr.segcount = segs; > pkt_hdr->frame_len = frame_len; > - pkt_hdr->unshared_len = frame_len; > pkt_hdr->headroom = offset + pool->headroom; > pkt_hdr->tailroom = pool->tailroom; > > @@ -1074,16 +961,11 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, > push_head(pkt_hdr, len); > } > > - if (data_ptr || seg_len) { > - uint32_t seg_ln = 0; > - void *data = packet_map(pkt_hdr, 0, &seg_ln, NULL); > - > - if (data_ptr) > - *data_ptr = data; > + if (data_ptr) > + *data_ptr = packet_data(pkt_hdr); > > - if (seg_len) > - *seg_len = seg_ln; > - } > + if (seg_len) > + *seg_len = packet_first_seg_len(pkt_hdr); > > return ret; > } > @@ -1095,8 +977,6 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) > if (len > pkt_hdr->frame_len) > return NULL; > > - ODP_ASSERT(len <= pkt_hdr->unshared_len); > - > pull_head(pkt_hdr, len); > return packet_data(pkt_hdr); > } > @@ -1104,35 +984,15 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) > int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, > void **data_ptr, uint32_t *seg_len_out) > { > - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt), *nxt_hdr; > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); > uint32_t seg_len = packet_first_seg_len(pkt_hdr); > - int ret = 0; > > - if (len > packet_len(pkt_hdr)) > + if (len > pkt_hdr->frame_len) > return -1; > > - ODP_ASSERT(len <= odp_packet_unshared_len(*pkt)); > - > - /* Special processing for references */ > - while (len >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { > - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); > - nxt_hdr = pkt_hdr->ref_hdr; > - len -= pkt_hdr->frame_len; > - len += pkt_hdr->ref_offset + > - (nxt_hdr->frame_len - pkt_hdr->ref_len); > - pkt_hdr->ref_hdr = NULL; > - packet_free(pkt_hdr); > - pkt_hdr = nxt_hdr; > - seg_len = packet_first_seg_len(pkt_hdr); > - *pkt = packet_handle(pkt_hdr); > - ret = 1; > - } > - > - if (CONFIG_PACKET_MAX_SEGS == 1 || > - len < seg_len || > - pkt_hdr->buf_hdr.segcount == 1) { > + if (len < seg_len) { > pull_head(pkt_hdr, len); > - } else { > + } else if (CONFIG_PACKET_MAX_SEGS != 1) { > int num = 0; > uint32_t pull_len = 0; > > @@ -1147,29 +1007,23 @@ int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, > *pkt = packet_handle(pkt_hdr); > } > > - if (data_ptr || seg_len_out) { > - void *data_head = packet_map(pkt_hdr, 0, &seg_len, NULL); > - > - if (data_ptr) > - *data_ptr = data_head; > + if (data_ptr) > + *data_ptr = packet_data(pkt_hdr); > > - if (seg_len_out) > - *seg_len_out = seg_len; > - } > + if (seg_len_out) > + *seg_len_out = packet_first_seg_len(pkt_hdr); > > - return ret; > + return 0; > } > > void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) > { > - odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(pkt, NULL); > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > void *old_tail; > > if (len > pkt_hdr->tailroom) > return NULL; > > - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); > - > old_tail = packet_tail(pkt_hdr); > push_tail(pkt_hdr, len); > > @@ -1179,14 +1033,12 @@ void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) > int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, > void **data_ptr, uint32_t *seg_len_out) > { > - odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(*pkt, NULL); > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); > uint32_t frame_len = pkt_hdr->frame_len; > uint32_t tailroom = pkt_hdr->tailroom; > uint32_t tail_off = frame_len; > int ret = 0; > > - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); > - > if (len > tailroom) { > pool_t *pool = pool_entry_from_hdl(pkt_hdr->buf_hdr.pool_hdl); > int num; > @@ -1277,7 +1129,6 @@ void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) > if (len > packet_last_seg_len(pkt_hdr)) > return NULL; > > - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); > pull_tail(pkt_hdr, len); > > return packet_tail(pkt_hdr); > @@ -1288,34 +1139,17 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, > { > int last; > uint32_t seg_len; > - uint32_t offset; > - odp_packet_hdr_t *first_hdr = odp_packet_hdr(*pkt); > - odp_packet_hdr_t *pkt_hdr, *prev_hdr; > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); > > - if (len > packet_len(first_hdr)) > + if (len > pkt_hdr->frame_len) > return -1; > > - pkt_hdr = odp_packet_last_hdr(*pkt, &offset); > - > - /* Special processing for references */ > - while (len >= pkt_hdr->frame_len - offset && first_hdr->ref_hdr) { > - len -= (pkt_hdr->frame_len - offset); > - prev_hdr = odp_packet_prev_hdr(first_hdr, pkt_hdr, &offset); > - ODP_ASSERT(packet_ref_count(prev_hdr) == 1); > - prev_hdr->ref_hdr = NULL; > - packet_free(pkt_hdr); > - pkt_hdr = prev_hdr; > - } > - > - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); > last = packet_last_seg(pkt_hdr); > seg_len = packet_seg_len(pkt_hdr, last); > > - if (CONFIG_PACKET_MAX_SEGS == 1 || > - len < seg_len || > - pkt_hdr->buf_hdr.segcount == 1) { > + if (len < seg_len) { > pull_tail(pkt_hdr, len); > - } else { > + } else if (CONFIG_PACKET_MAX_SEGS != 1) { > int num = 0; > uint32_t pull_len = 0; > > @@ -1522,50 +1356,35 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) > > int odp_packet_is_segmented(odp_packet_t pkt) > { > - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - > - return pkt_hdr->buf_hdr.segcount > 1 || pkt_hdr->ref_hdr != NULL; > + return odp_packet_hdr(pkt)->buf_hdr.segcount > 1; > } > > int odp_packet_num_segs(odp_packet_t pkt) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - uint32_t segcount = 0, i; > - uint32_t seg_offset = 0, offset; > - > - do { > - segcount += pkt_hdr->buf_hdr.segcount - seg_offset; > - offset = pkt_hdr->ref_offset; > - pkt_hdr = pkt_hdr->ref_hdr; > - if (pkt_hdr) { > - for (i = 0, seg_offset = 0; > - i < pkt_hdr->buf_hdr.segcount; > - i++, seg_offset++) { > - if (offset < pkt_hdr->buf_hdr.seg[i].len) > - break; > - offset -= pkt_hdr->buf_hdr.seg[i].len; > - } > - } > - } while (pkt_hdr); > > - return segcount; > + return pkt_hdr->buf_hdr.segcount; > } > > -odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt ODP_UNUSED) > +odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) > { > + (void)pkt; > + > return 0; > } > > odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) > { > - return (odp_packet_seg_t)(odp_packet_num_segs(pkt) - 1); > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + > + return packet_last_seg(pkt_hdr); > } > > odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - if (odp_unlikely(seg >= packet_last_seg(pkt_hdr))) > + if (odp_unlikely(seg >= (odp_packet_seg_t)packet_last_seg(pkt_hdr))) > return ODP_PACKET_SEG_INVALID; > > return seg + 1; > @@ -1581,51 +1400,21 @@ odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) > void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - uint32_t seg_offset = 0, offset = 0, i; > - > - while (seg >= pkt_hdr->buf_hdr.segcount - seg_offset && > - pkt_hdr->ref_hdr) { > - seg -= (pkt_hdr->buf_hdr.segcount - seg_offset); > - offset = pkt_hdr->ref_offset; > - pkt_hdr = pkt_hdr->ref_hdr; > - for (i = 0, seg_offset = 0; > - i < pkt_hdr->buf_hdr.segcount; > - i++, seg_offset++) { > - if (offset < pkt_hdr->buf_hdr.seg[i].len) > - break; > - offset -= pkt_hdr->buf_hdr.seg[i].len; > - } > - } > > - if (odp_unlikely(seg + seg_offset >= pkt_hdr->buf_hdr.segcount)) > + if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount)) > return NULL; > > - return packet_seg_data(pkt_hdr, seg + seg_offset) + offset; > + return packet_seg_data(pkt_hdr, seg); > } > > uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - uint32_t seg_offset = 0, offset = 0, i; > - > - while (seg >= pkt_hdr->buf_hdr.segcount - seg_offset && > - pkt_hdr->ref_hdr) { > - seg -= (pkt_hdr->buf_hdr.segcount - seg_offset); > - offset = pkt_hdr->ref_offset; > - pkt_hdr = pkt_hdr->ref_hdr; > - for (i = 0, seg_offset = 0; > - i < pkt_hdr->buf_hdr.segcount; > - i++, seg_offset++) { > - if (offset < pkt_hdr->buf_hdr.seg[i].len) > - break; > - offset -= pkt_hdr->buf_hdr.seg[i].len; > - } > - } > > - if (odp_unlikely(seg + seg_offset >= pkt_hdr->buf_hdr.segcount)) > + if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount)) > return 0; > > - return packet_seg_len(pkt_hdr, seg + seg_offset) - offset; > + return packet_seg_len(pkt_hdr, seg); > } > > /* > @@ -1639,14 +1428,12 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) > { > odp_packet_t pkt = *pkt_ptr; > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - uint32_t pktlen = packet_len(pkt_hdr); > + uint32_t pktlen = pkt_hdr->frame_len; > odp_packet_t newpkt; > > if (offset > pktlen) > return -1; > > - ODP_ASSERT(odp_packet_unshared_len(*pkt_ptr) >= offset); > - > newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen + len); > > if (newpkt == ODP_PACKET_INVALID) > @@ -1709,8 +1496,6 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len, > if (align > ODP_CACHE_LINE_SIZE) > return -1; > > - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); > - > if (seglen >= len) { > misalign = align <= 1 ? 0 : > ROUNDUP_ALIGN(uaddr, align) - uaddr; > @@ -1750,13 +1535,10 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) > uint32_t dst_len = dst_hdr->frame_len; > uint32_t src_len = src_hdr->frame_len; > > - ODP_ASSERT(packet_ref_count(dst_hdr) == 1); > - > /* Do a copy if resulting packet would be out of segments or packets > - * are from different pools or src is a reference. */ > + * are from different pools. */ > if (odp_unlikely((dst_segs + src_segs) > CONFIG_PACKET_MAX_SEGS) || > - odp_unlikely(dst_pool != src_pool) || > - odp_unlikely(packet_ref_count(src_hdr)) > 1) { > + odp_unlikely(dst_pool != src_pool)) { > if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) { > (void)odp_packet_copy_from_pkt(*dst, dst_len, > src, 0, src_len); > @@ -1771,9 +1553,8 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) > > add_all_segs(dst_hdr, src_hdr); > > - dst_hdr->frame_len = dst_len + src_len; > - dst_hdr->unshared_len = dst_len + src_len; > - dst_hdr->tailroom = src_hdr->tailroom; > + dst_hdr->frame_len = dst_len + src_len; > + dst_hdr->tailroom = src_hdr->tailroom; > > /* Data was not moved in memory */ > return 0; > @@ -1786,7 +1567,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) > if (len >= pktlen || tail == NULL) > return -1; > > - ODP_ASSERT(odp_packet_unshared_len(*pkt) >= len); > *tail = odp_packet_copy_part(*pkt, len, pktlen - len, > odp_packet_pool(*pkt)); > > @@ -1797,109 +1577,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) > } > > /* > - * References > - */ > - > -static inline void packet_ref(odp_packet_hdr_t *pkt_hdr) > -{ > - uint32_t i; > - odp_packet_hdr_t *hdr; > - > - do { > - for (i = 0; i < pkt_hdr->buf_hdr.segcount; i++) { > - hdr = pkt_hdr->buf_hdr.seg[i].hdr; > - packet_ref_inc(hdr); > - } > - > - pkt_hdr = pkt_hdr->ref_hdr; > - } while (pkt_hdr); > -} > - > -static inline odp_packet_t packet_splice(odp_packet_hdr_t *pkt_hdr, > - uint32_t offset, > - odp_packet_hdr_t *ref_hdr) > -{ > - /* Catch attempted references to stale handles in debug builds */ > - ODP_ASSERT(packet_ref_count(pkt_hdr) > 0); > - > - /* Splicing is from the last section of src pkt */ > - while (ref_hdr->ref_hdr) > - ref_hdr = ref_hdr->ref_hdr; > - > - /* Find section where splice begins */ > - while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { > - offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); > - offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); > - pkt_hdr = pkt_hdr->ref_hdr; > - } > - > - ref_hdr->ref_hdr = pkt_hdr; > - ref_hdr->ref_offset = offset; > - ref_hdr->ref_len = pkt_hdr->frame_len; > - > - if (offset < pkt_hdr->unshared_len) > - pkt_hdr->unshared_len = offset; > - > - packet_ref(pkt_hdr); > - return _odp_packet_hdl(ref_hdr); > -} > - > -odp_packet_t odp_packet_ref_static(odp_packet_t pkt) > -{ > - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - > - pkt_hdr->unshared_len = 0; > - packet_ref(pkt_hdr); > - return pkt; > -} > - > -odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) > -{ > - odp_packet_t hdr; > - odp_packet_hdr_t *pkt_hdr; > - > - if (pkt == ODP_PACKET_INVALID) > - return ODP_PACKET_INVALID; > - > - pkt_hdr = odp_packet_hdr(pkt); > - if (offset >= packet_len(pkt_hdr)) > - return ODP_PACKET_INVALID; > - > - hdr = odp_packet_alloc(odp_packet_pool(pkt), 0); > - > - if (hdr == ODP_PACKET_INVALID) > - return ODP_PACKET_INVALID; > - > - return packet_splice(pkt_hdr, offset, odp_packet_hdr(hdr)); > -} > - > -odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, > - odp_packet_t hdr) > -{ > - odp_packet_hdr_t *pkt_hdr; > - > - if (pkt == ODP_PACKET_INVALID || > - hdr == ODP_PACKET_INVALID || > - pkt == hdr) > - return ODP_PACKET_INVALID; > - > - ODP_ASSERT(odp_packet_has_ref(hdr) == 0); > - > - pkt_hdr = odp_packet_hdr(pkt); > - if (offset >= packet_len(pkt_hdr)) > - return ODP_PACKET_INVALID; > - > - return packet_splice(pkt_hdr, offset, odp_packet_hdr(hdr)); > -} > - > -int odp_packet_has_ref(odp_packet_t pkt) > -{ > - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > - > - return pkt_hdr->ref_hdr != NULL || packet_ref_count(pkt_hdr) > 1; > -} > - > -/* > * > * Copy > * ******************************************************** > @@ -1908,7 +1585,8 @@ int odp_packet_has_ref(odp_packet_t pkt) > > odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool) > { > - uint32_t pktlen = odp_packet_len(pkt); > + odp_packet_hdr_t *srchdr = odp_packet_hdr(pkt); > + uint32_t pktlen = srchdr->frame_len; > odp_packet_t newpkt = odp_packet_alloc(pool, pktlen); > > if (newpkt != ODP_PACKET_INVALID) { > @@ -1947,7 +1625,7 @@ int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t offset, > uint8_t *dstaddr = (uint8_t *)dst; > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - if (offset + len > packet_len(pkt_hdr)) > + if (offset + len > pkt_hdr->frame_len) > return -1; > > while (len > 0) { > @@ -1971,11 +1649,9 @@ int odp_packet_copy_from_mem(odp_packet_t pkt, uint32_t offset, > const uint8_t *srcaddr = (const uint8_t *)src; > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - if (offset + len > packet_len(pkt_hdr)) > + if (offset + len > pkt_hdr->frame_len) > return -1; > > - ODP_ASSERT(odp_packet_unshared_len(pkt) >= offset + len); > - > while (len > 0) { > mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); > cpylen = len > seglen ? seglen : len; > @@ -2001,12 +1677,10 @@ int odp_packet_copy_from_pkt(odp_packet_t dst, uint32_t dst_offset, > uint32_t src_seglen = 0; /* GCC */ > int overlap; > > - if (dst_offset + len > packet_len(dst_hdr) || > - src_offset + len > packet_len(src_hdr)) > + if (dst_offset + len > dst_hdr->frame_len || > + src_offset + len > src_hdr->frame_len) > return -1; > > - ODP_ASSERT(odp_packet_unshared_len(dst) >= dst_offset + len); > - > overlap = (dst_hdr == src_hdr && > ((dst_offset <= src_offset && > dst_offset + len >= src_offset) || > @@ -2090,7 +1764,7 @@ void odp_packet_print(odp_packet_t pkt) > len += snprintf(&str[len], n - len, > " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); > len += snprintf(&str[len], n - len, > - " frame_len %" PRIu32 "\n", packet_len(hdr)); > + " frame_len %" PRIu32 "\n", hdr->frame_len); > len += snprintf(&str[len], n - len, > " input %" PRIu64 "\n", > odp_pktio_to_u64(hdr->input)); > -- > 2.8.1 >
diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 42df5ac..e3ada5c 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -19,7 +19,6 @@ extern "C" { #include <odp/api/align.h> #include <odp/api/debug.h> -#include <odp_debug_internal.h> #include <odp_buffer_internal.h> #include <odp_pool_internal.h> #include <odp_buffer_inlines.h> @@ -169,7 +168,7 @@ typedef struct { * packet_init(). Because of this any new fields added must be reviewed for * initialization requirements. */ -typedef struct odp_packet_hdr_t { +typedef struct { /* common buffer header */ odp_buffer_hdr_t buf_hdr; @@ -185,13 +184,6 @@ typedef struct odp_packet_hdr_t { uint32_t headroom; uint32_t tailroom; - /* Fields used to support packet references */ - uint32_t unshared_len; - struct odp_packet_hdr_t *ref_hdr; - uint32_t ref_offset; - uint32_t ref_len; - odp_atomic_u32_t ref_count; - /* * Members below are not initialized by packet_init() */ @@ -220,55 +212,6 @@ static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) return (odp_packet_hdr_t *)buf_hdl_to_hdr((odp_buffer_t)pkt); } -static inline odp_packet_hdr_t *odp_packet_last_hdr(odp_packet_t pkt, - uint32_t *offset) -{ - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - odp_packet_hdr_t *prev_hdr = pkt_hdr; - uint32_t ref_offset = 0; - - while (pkt_hdr->ref_hdr) { - ref_offset = pkt_hdr->ref_offset; - prev_hdr = pkt_hdr; - pkt_hdr = pkt_hdr->ref_hdr; - } - - if (offset) { - if (prev_hdr != pkt_hdr) - ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; - *offset = ref_offset; - } - - return pkt_hdr; -} - -static inline odp_packet_hdr_t *odp_packet_prev_hdr(odp_packet_hdr_t *pkt_hdr, - odp_packet_hdr_t *cur_hdr, - uint32_t *offset) -{ - uint32_t ref_offset = 0; - odp_packet_hdr_t *prev_hdr = pkt_hdr; - - while (pkt_hdr->ref_hdr != cur_hdr) { - ref_offset = pkt_hdr->ref_offset; - prev_hdr = pkt_hdr; - pkt_hdr = pkt_hdr->ref_hdr; - } - - if (offset) { - if (prev_hdr != pkt_hdr) - ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; - *offset = ref_offset; - } - - return pkt_hdr; -} - -static inline odp_packet_t _odp_packet_hdl(odp_packet_hdr_t *pkt_hdr) -{ - return (odp_packet_t)odp_hdr_to_buf(&pkt_hdr->buf_hdr); -} - static inline void copy_packet_parser_metadata(odp_packet_hdr_t *src_hdr, odp_packet_hdr_t *dst_hdr) { @@ -291,41 +234,17 @@ static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) pkt_hdr->tailroom += len; pkt_hdr->frame_len -= len; - pkt_hdr->unshared_len -= len; pkt_hdr->buf_hdr.seg[last].len -= len; } static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) { - uint32_t pkt_len = 0; - uint32_t offset = 0; - - do { - pkt_len += pkt_hdr->frame_len - offset; - offset = pkt_hdr->ref_offset; - if (pkt_hdr->ref_hdr) - offset += (pkt_hdr->ref_hdr->frame_len - - pkt_hdr->ref_len); - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); - - return pkt_len; -} - -static inline uint32_t packet_ref_count(odp_packet_hdr_t *pkt_hdr) -{ - return odp_atomic_load_u32(&pkt_hdr->ref_count); -} - -static inline void packet_ref_count_set(odp_packet_hdr_t *pkt_hdr, uint32_t n) -{ - odp_atomic_init_u32(&pkt_hdr->ref_count, n); + return pkt_hdr->frame_len; } static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->frame_len = len; - pkt_hdr->unshared_len = len; } static inline int packet_parse_l2_not_done(packet_parser_t *prs) diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 3aadc4d..024f694 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -33,24 +33,13 @@ static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) return pkt_hdr->buf_hdr.handle.handle; } -static inline uint32_t packet_ref_inc(odp_packet_hdr_t *pkt_hdr) -{ - return odp_atomic_fetch_inc_u32(&pkt_hdr->ref_count); -} - -static inline uint32_t packet_ref_dec(odp_packet_hdr_t *pkt_hdr) -{ - return odp_atomic_fetch_dec_u32(&pkt_hdr->ref_count); -} - static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) { return pkt_hdr->buf_hdr.seg[seg_idx].len; } -static inline uint8_t *packet_seg_data(odp_packet_hdr_t *pkt_hdr, - uint32_t seg_idx) +static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) { return pkt_hdr->buf_hdr.seg[seg_idx].data; } @@ -63,11 +52,6 @@ static inline int packet_last_seg(odp_packet_hdr_t *pkt_hdr) return pkt_hdr->buf_hdr.segcount - 1; } -static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) -{ - return pkt_hdr->buf_hdr.seg[0].data; -} - static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr) { return packet_seg_len(pkt_hdr, 0); @@ -80,6 +64,11 @@ static inline uint32_t packet_last_seg_len(odp_packet_hdr_t *pkt_hdr) return packet_seg_len(pkt_hdr, last); } +static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) +{ + return pkt_hdr->buf_hdr.seg[0].data; +} + static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr) { int last = packet_last_seg(pkt_hdr); @@ -110,7 +99,6 @@ static inline void push_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->headroom -= len; pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; pkt_hdr->buf_hdr.seg[0].data -= len; pkt_hdr->buf_hdr.seg[0].len += len; } @@ -119,7 +107,6 @@ static inline void pull_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->headroom += len; pkt_hdr->frame_len -= len; - pkt_hdr->unshared_len -= len; pkt_hdr->buf_hdr.seg[0].data += len; pkt_hdr->buf_hdr.seg[0].len -= len; } @@ -130,7 +117,6 @@ static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) pkt_hdr->tailroom -= len; pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; pkt_hdr->buf_hdr.seg[last].len += len; } @@ -158,10 +144,6 @@ static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr; dst->buf_hdr.uarea_size = src->buf_hdr.uarea_size; - /* reference related metadata */ - dst->ref_len = src->ref_len; - dst->unshared_len = src->unshared_len; - /* segmentation data is not copied: * buf_hdr.seg[] * buf_hdr.segcount @@ -176,15 +158,7 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, int seg = 0; int seg_count = pkt_hdr->buf_hdr.segcount; - /* Special processing for references */ - while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); - offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr = pkt_hdr->ref_hdr; - seg_count = pkt_hdr->buf_hdr.segcount; - } - - if (odp_unlikely(offset > pkt_hdr->frame_len)) + if (odp_unlikely(offset >= pkt_hdr->frame_len)) return NULL; if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || seg_count == 1)) { @@ -233,9 +207,6 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) pkt_hdr->p.l2_offset = 0; pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; - - /* Ensure dummy pkt_hdrs used in I/O recv classification are valid */ - pkt_hdr->ref_hdr = NULL; } /** @@ -281,10 +252,6 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len, CONFIG_PACKET_TAILROOM; pkt_hdr->input = ODP_PKTIO_INVALID; - - /* By default packet has no references */ - pkt_hdr->unshared_len = len; - pkt_hdr->ref_hdr = NULL; } static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) @@ -297,7 +264,6 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) hdr->buf_hdr.seg[0].data = hdr->buf_hdr.base_data; hdr->buf_hdr.seg[0].len = BASE_LEN; - packet_ref_count_set(hdr, 1); /* Link segments */ if (CONFIG_PACKET_MAX_SEGS != 1) { @@ -307,7 +273,6 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) for (i = 1; i < num; i++) { odp_buffer_hdr_t *buf_hdr; - packet_ref_count_set(pkt_hdr[i], 1); buf_hdr = &pkt_hdr[i]->buf_hdr; hdr->buf_hdr.seg[i].hdr = buf_hdr; hdr->buf_hdr.seg[i].data = buf_hdr->base_data; @@ -411,10 +376,9 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, new_hdr->buf_hdr.seg[0].len = seg_len; packet_seg_copy_md(new_hdr, pkt_hdr); - new_hdr->frame_len = pkt_hdr->frame_len + len; - new_hdr->unshared_len = pkt_hdr->unshared_len + len; - new_hdr->headroom = pool->headroom + offset; - new_hdr->tailroom = pkt_hdr->tailroom; + new_hdr->frame_len = pkt_hdr->frame_len + len; + new_hdr->headroom = pool->headroom + offset; + new_hdr->tailroom = pkt_hdr->tailroom; pkt_hdr = new_hdr; } else { @@ -427,9 +391,8 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, last = packet_last_seg(pkt_hdr); pkt_hdr->buf_hdr.seg[last].len = seg_len; - pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; - pkt_hdr->tailroom = pool->tailroom + offset; + pkt_hdr->frame_len += len; + pkt_hdr->tailroom = pool->tailroom + offset; } return pkt_hdr; @@ -437,18 +400,13 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, static inline void free_bufs(odp_packet_hdr_t *pkt_hdr, int first, int num) { - int i, nfree; + int i; odp_buffer_t buf[num]; - for (i = 0, nfree = 0; i < num; i++) { - odp_packet_hdr_t *hdr = pkt_hdr->buf_hdr.seg[first + i].hdr; - - if (packet_ref_dec(hdr) == 1) - buf[nfree++] = buffer_handle(hdr); - } + for (i = 0; i < num; i++) + buf[i] = buffer_handle(pkt_hdr->buf_hdr.seg[first + i].hdr); - if (nfree > 0) - buffer_free_multi(buf, nfree); + buffer_free_multi(buf, num); } static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, @@ -459,15 +417,11 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, if (head) { odp_packet_hdr_t *new_hdr; - int i, nfree; + int i; odp_buffer_t buf[num]; - for (i = 0, nfree = 0; i < num; i++) { - new_hdr = pkt_hdr->buf_hdr.seg[i].hdr; - - if (packet_ref_dec(new_hdr) == 1) - buf[nfree++] = buffer_handle(new_hdr); - } + for (i = 0; i < num; i++) + buf[i] = buffer_handle(pkt_hdr->buf_hdr.seg[i].hdr); /* First remaining segment is the new packet descriptor */ new_hdr = pkt_hdr->buf_hdr.seg[num].hdr; @@ -476,17 +430,15 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, packet_seg_copy_md(new_hdr, pkt_hdr); /* Tailroom not changed */ - new_hdr->tailroom = pkt_hdr->tailroom; - new_hdr->headroom = seg_headroom(new_hdr, 0); - new_hdr->frame_len = pkt_hdr->frame_len - free_len; - new_hdr->unshared_len = pkt_hdr->unshared_len - free_len; + new_hdr->tailroom = pkt_hdr->tailroom; + new_hdr->headroom = seg_headroom(new_hdr, 0); + new_hdr->frame_len = pkt_hdr->frame_len - free_len; pull_head(new_hdr, pull_len); pkt_hdr = new_hdr; - if (nfree > 0) - buffer_free_multi(buf, nfree); + buffer_free_multi(buf, num); } else { /* Free last 'num' bufs */ free_bufs(pkt_hdr, num_remain, num); @@ -495,7 +447,6 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, * of the metadata. */ pkt_hdr->buf_hdr.segcount = num_remain; pkt_hdr->frame_len -= free_len; - pkt_hdr->unshared_len -= free_len; pkt_hdr->tailroom = seg_tailroom(pkt_hdr, num_remain - 1); pull_tail(pkt_hdr, pull_len); @@ -599,34 +550,45 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, return num; } -static inline void packet_free(odp_packet_hdr_t *pkt_hdr) -{ - odp_packet_hdr_t *ref_hdr; - uint32_t ref_count; - - do { - ref_hdr = pkt_hdr->ref_hdr; - ref_count = packet_ref_count(pkt_hdr) - 1; - free_bufs(pkt_hdr, 0, pkt_hdr->buf_hdr.segcount); - - if (ref_count == 1) - pkt_hdr->unshared_len = pkt_hdr->frame_len; - - pkt_hdr = ref_hdr; - } while (pkt_hdr); -} - void odp_packet_free(odp_packet_t pkt) { - packet_free(odp_packet_hdr(pkt)); + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + int num_seg = pkt_hdr->buf_hdr.segcount; + + if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) + buffer_free_multi((odp_buffer_t *)&pkt, 1); + else + free_bufs(pkt_hdr, 0, num_seg); } void odp_packet_free_multi(const odp_packet_t pkt[], int num) { - int i; + if (CONFIG_PACKET_MAX_SEGS == 1) { + buffer_free_multi((const odp_buffer_t * const)pkt, num); + } else { + odp_buffer_t buf[num * CONFIG_PACKET_MAX_SEGS]; + int i, j; + int bufs = 0; - for (i = 0; i < num; i++) - packet_free(odp_packet_hdr(pkt[i])); + for (i = 0; i < num; i++) { + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt[i]); + int num_seg = pkt_hdr->buf_hdr.segcount; + odp_buffer_hdr_t *buf_hdr = &pkt_hdr->buf_hdr; + + buf[bufs] = (odp_buffer_t)pkt[i]; + bufs++; + + if (odp_likely(num_seg == 1)) + continue; + + for (j = 1; j < num_seg; j++) { + buf[bufs] = buffer_handle(buf_hdr->seg[j].hdr); + bufs++; + } + } + + buffer_free_multi(buf, bufs); + } } int odp_packet_reset(odp_packet_t pkt, uint32_t len) @@ -637,9 +599,6 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) if (len > pool->headroom + pool->data_size + pool->tailroom) return -1; - if (pkt_hdr->ref_hdr) - packet_free(pkt_hdr->ref_hdr); - packet_init(pkt_hdr, len, 0); return 0; @@ -682,21 +641,15 @@ void *odp_packet_head(odp_packet_t pkt) uint32_t odp_packet_buf_len(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - uint32_t buf_len = 0; - do { - buf_len += pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); - - return buf_len; + return pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; } void *odp_packet_data(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - return packet_map(pkt_hdr, 0, NULL, NULL); + return packet_data(pkt_hdr); } uint32_t odp_packet_seg_len(odp_packet_t pkt) @@ -708,32 +661,7 @@ uint32_t odp_packet_seg_len(odp_packet_t pkt) uint32_t odp_packet_len(odp_packet_t pkt) { - return packet_len(odp_packet_hdr(pkt)); -} - -uint32_t odp_packet_unshared_len(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - uint32_t pkt_len = 0, offset = 0; - - do { - if (packet_ref_count(pkt_hdr) > 1) { - if (offset == 0) - pkt_len += pkt_hdr->unshared_len; - break; - } - - pkt_len += pkt_hdr->frame_len - offset; - offset = pkt_hdr->ref_offset; - - if (pkt_hdr->ref_hdr) - offset += (pkt_hdr->ref_hdr->frame_len - - pkt_hdr->ref_len); - - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); - - return pkt_len; + return odp_packet_hdr(pkt)->frame_len; } uint32_t odp_packet_headroom(odp_packet_t pkt) @@ -743,12 +671,12 @@ uint32_t odp_packet_headroom(odp_packet_t pkt) uint32_t odp_packet_tailroom(odp_packet_t pkt) { - return odp_packet_last_hdr(pkt, NULL)->tailroom; + return odp_packet_hdr(pkt)->tailroom; } void *odp_packet_tail(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(pkt, NULL); + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); return packet_tail(pkt_hdr); } @@ -942,7 +870,7 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); uint32_t frame_len = pkt_hdr->frame_len; - uint32_t headroom = pkt_hdr->headroom; + uint32_t headroom = pkt_hdr->headroom; int ret = 0; if (len > headroom) { @@ -957,46 +885,6 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, segs = pkt_hdr->buf_hdr.segcount; if (odp_unlikely((segs + num) > CONFIG_PACKET_MAX_SEGS)) { - /* Handle recursively via references when - * working with referenced packets since another - * thread may be accessing it concurrently via - * its reference to it. */ - if (packet_ref_count(pkt_hdr) > 1) { - odp_packet_t ref; - uint32_t unshared_len; - - push_head(pkt_hdr, headroom); - unshared_len = pkt_hdr->unshared_len; - ref = odp_packet_ref(*pkt, 0); - - if (ref == ODP_PACKET_INVALID) { - pull_head(pkt_hdr, headroom); - return -1; - } - - ret = odp_packet_extend_head(&ref, - len - headroom, - data_ptr, - seg_len); - - if (ret < 0) { - odp_packet_free(ref); - pull_head(pkt_hdr, headroom); - return -1; - } - - /* Since this is a special ref, the - * base pkt's unshared len is unchanged */ - pkt_hdr->unshared_len = unshared_len; - - /* Remove extra ref to the base pkt */ - odp_packet_free(*pkt); - - /* Return the ref as the extension result */ - *pkt = ref; - return 1; - } - /* Cannot directly add new segments */ odp_packet_hdr_t *new_hdr; int new_segs = 0; @@ -1048,7 +936,6 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, pkt_hdr->buf_hdr.segcount = segs; pkt_hdr->frame_len = frame_len; - pkt_hdr->unshared_len = frame_len; pkt_hdr->headroom = offset + pool->headroom; pkt_hdr->tailroom = pool->tailroom; @@ -1074,16 +961,11 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, push_head(pkt_hdr, len); } - if (data_ptr || seg_len) { - uint32_t seg_ln = 0; - void *data = packet_map(pkt_hdr, 0, &seg_ln, NULL); - - if (data_ptr) - *data_ptr = data; + if (data_ptr) + *data_ptr = packet_data(pkt_hdr); - if (seg_len) - *seg_len = seg_ln; - } + if (seg_len) + *seg_len = packet_first_seg_len(pkt_hdr); return ret; } @@ -1095,8 +977,6 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) if (len > pkt_hdr->frame_len) return NULL; - ODP_ASSERT(len <= pkt_hdr->unshared_len); - pull_head(pkt_hdr, len); return packet_data(pkt_hdr); } @@ -1104,35 +984,15 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len_out) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt), *nxt_hdr; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); uint32_t seg_len = packet_first_seg_len(pkt_hdr); - int ret = 0; - if (len > packet_len(pkt_hdr)) + if (len > pkt_hdr->frame_len) return -1; - ODP_ASSERT(len <= odp_packet_unshared_len(*pkt)); - - /* Special processing for references */ - while (len >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - nxt_hdr = pkt_hdr->ref_hdr; - len -= pkt_hdr->frame_len; - len += pkt_hdr->ref_offset + - (nxt_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr->ref_hdr = NULL; - packet_free(pkt_hdr); - pkt_hdr = nxt_hdr; - seg_len = packet_first_seg_len(pkt_hdr); - *pkt = packet_handle(pkt_hdr); - ret = 1; - } - - if (CONFIG_PACKET_MAX_SEGS == 1 || - len < seg_len || - pkt_hdr->buf_hdr.segcount == 1) { + if (len < seg_len) { pull_head(pkt_hdr, len); - } else { + } else if (CONFIG_PACKET_MAX_SEGS != 1) { int num = 0; uint32_t pull_len = 0; @@ -1147,29 +1007,23 @@ int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, *pkt = packet_handle(pkt_hdr); } - if (data_ptr || seg_len_out) { - void *data_head = packet_map(pkt_hdr, 0, &seg_len, NULL); - - if (data_ptr) - *data_ptr = data_head; + if (data_ptr) + *data_ptr = packet_data(pkt_hdr); - if (seg_len_out) - *seg_len_out = seg_len; - } + if (seg_len_out) + *seg_len_out = packet_first_seg_len(pkt_hdr); - return ret; + return 0; } void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) { - odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(pkt, NULL); + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); void *old_tail; if (len > pkt_hdr->tailroom) return NULL; - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - old_tail = packet_tail(pkt_hdr); push_tail(pkt_hdr, len); @@ -1179,14 +1033,12 @@ void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len_out) { - odp_packet_hdr_t *pkt_hdr = odp_packet_last_hdr(*pkt, NULL); + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); uint32_t frame_len = pkt_hdr->frame_len; uint32_t tailroom = pkt_hdr->tailroom; uint32_t tail_off = frame_len; int ret = 0; - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - if (len > tailroom) { pool_t *pool = pool_entry_from_hdl(pkt_hdr->buf_hdr.pool_hdl); int num; @@ -1277,7 +1129,6 @@ void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) if (len > packet_last_seg_len(pkt_hdr)) return NULL; - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); pull_tail(pkt_hdr, len); return packet_tail(pkt_hdr); @@ -1288,34 +1139,17 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, { int last; uint32_t seg_len; - uint32_t offset; - odp_packet_hdr_t *first_hdr = odp_packet_hdr(*pkt); - odp_packet_hdr_t *pkt_hdr, *prev_hdr; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(*pkt); - if (len > packet_len(first_hdr)) + if (len > pkt_hdr->frame_len) return -1; - pkt_hdr = odp_packet_last_hdr(*pkt, &offset); - - /* Special processing for references */ - while (len >= pkt_hdr->frame_len - offset && first_hdr->ref_hdr) { - len -= (pkt_hdr->frame_len - offset); - prev_hdr = odp_packet_prev_hdr(first_hdr, pkt_hdr, &offset); - ODP_ASSERT(packet_ref_count(prev_hdr) == 1); - prev_hdr->ref_hdr = NULL; - packet_free(pkt_hdr); - pkt_hdr = prev_hdr; - } - - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); last = packet_last_seg(pkt_hdr); seg_len = packet_seg_len(pkt_hdr, last); - if (CONFIG_PACKET_MAX_SEGS == 1 || - len < seg_len || - pkt_hdr->buf_hdr.segcount == 1) { + if (len < seg_len) { pull_tail(pkt_hdr, len); - } else { + } else if (CONFIG_PACKET_MAX_SEGS != 1) { int num = 0; uint32_t pull_len = 0; @@ -1522,50 +1356,35 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) int odp_packet_is_segmented(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - - return pkt_hdr->buf_hdr.segcount > 1 || pkt_hdr->ref_hdr != NULL; + return odp_packet_hdr(pkt)->buf_hdr.segcount > 1; } int odp_packet_num_segs(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - uint32_t segcount = 0, i; - uint32_t seg_offset = 0, offset; - - do { - segcount += pkt_hdr->buf_hdr.segcount - seg_offset; - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - if (pkt_hdr) { - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - } - } while (pkt_hdr); - return segcount; + return pkt_hdr->buf_hdr.segcount; } -odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt ODP_UNUSED) +odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) { + (void)pkt; + return 0; } odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) { - return (odp_packet_seg_t)(odp_packet_num_segs(pkt) - 1); + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + return packet_last_seg(pkt_hdr); } odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (odp_unlikely(seg >= packet_last_seg(pkt_hdr))) + if (odp_unlikely(seg >= (odp_packet_seg_t)packet_last_seg(pkt_hdr))) return ODP_PACKET_SEG_INVALID; return seg + 1; @@ -1581,51 +1400,21 @@ odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - uint32_t seg_offset = 0, offset = 0, i; - - while (seg >= pkt_hdr->buf_hdr.segcount - seg_offset && - pkt_hdr->ref_hdr) { - seg -= (pkt_hdr->buf_hdr.segcount - seg_offset); - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - } - if (odp_unlikely(seg + seg_offset >= pkt_hdr->buf_hdr.segcount)) + if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount)) return NULL; - return packet_seg_data(pkt_hdr, seg + seg_offset) + offset; + return packet_seg_data(pkt_hdr, seg); } uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - uint32_t seg_offset = 0, offset = 0, i; - - while (seg >= pkt_hdr->buf_hdr.segcount - seg_offset && - pkt_hdr->ref_hdr) { - seg -= (pkt_hdr->buf_hdr.segcount - seg_offset); - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - } - if (odp_unlikely(seg + seg_offset >= pkt_hdr->buf_hdr.segcount)) + if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount)) return 0; - return packet_seg_len(pkt_hdr, seg + seg_offset) - offset; + return packet_seg_len(pkt_hdr, seg); } /* @@ -1639,14 +1428,12 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) { odp_packet_t pkt = *pkt_ptr; odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - uint32_t pktlen = packet_len(pkt_hdr); + uint32_t pktlen = pkt_hdr->frame_len; odp_packet_t newpkt; if (offset > pktlen) return -1; - ODP_ASSERT(odp_packet_unshared_len(*pkt_ptr) >= offset); - newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen + len); if (newpkt == ODP_PACKET_INVALID) @@ -1709,8 +1496,6 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len, if (align > ODP_CACHE_LINE_SIZE) return -1; - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); - if (seglen >= len) { misalign = align <= 1 ? 0 : ROUNDUP_ALIGN(uaddr, align) - uaddr; @@ -1750,13 +1535,10 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) uint32_t dst_len = dst_hdr->frame_len; uint32_t src_len = src_hdr->frame_len; - ODP_ASSERT(packet_ref_count(dst_hdr) == 1); - /* Do a copy if resulting packet would be out of segments or packets - * are from different pools or src is a reference. */ + * are from different pools. */ if (odp_unlikely((dst_segs + src_segs) > CONFIG_PACKET_MAX_SEGS) || - odp_unlikely(dst_pool != src_pool) || - odp_unlikely(packet_ref_count(src_hdr)) > 1) { + odp_unlikely(dst_pool != src_pool)) { if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) { (void)odp_packet_copy_from_pkt(*dst, dst_len, src, 0, src_len); @@ -1771,9 +1553,8 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) add_all_segs(dst_hdr, src_hdr); - dst_hdr->frame_len = dst_len + src_len; - dst_hdr->unshared_len = dst_len + src_len; - dst_hdr->tailroom = src_hdr->tailroom; + dst_hdr->frame_len = dst_len + src_len; + dst_hdr->tailroom = src_hdr->tailroom; /* Data was not moved in memory */ return 0; @@ -1786,7 +1567,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) if (len >= pktlen || tail == NULL) return -1; - ODP_ASSERT(odp_packet_unshared_len(*pkt) >= len); *tail = odp_packet_copy_part(*pkt, len, pktlen - len, odp_packet_pool(*pkt)); @@ -1797,109 +1577,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) } /* - * References - */ - -static inline void packet_ref(odp_packet_hdr_t *pkt_hdr) -{ - uint32_t i; - odp_packet_hdr_t *hdr; - - do { - for (i = 0; i < pkt_hdr->buf_hdr.segcount; i++) { - hdr = pkt_hdr->buf_hdr.seg[i].hdr; - packet_ref_inc(hdr); - } - - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); -} - -static inline odp_packet_t packet_splice(odp_packet_hdr_t *pkt_hdr, - uint32_t offset, - odp_packet_hdr_t *ref_hdr) -{ - /* Catch attempted references to stale handles in debug builds */ - ODP_ASSERT(packet_ref_count(pkt_hdr) > 0); - - /* Splicing is from the last section of src pkt */ - while (ref_hdr->ref_hdr) - ref_hdr = ref_hdr->ref_hdr; - - /* Find section where splice begins */ - while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); - offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr = pkt_hdr->ref_hdr; - } - - ref_hdr->ref_hdr = pkt_hdr; - ref_hdr->ref_offset = offset; - ref_hdr->ref_len = pkt_hdr->frame_len; - - if (offset < pkt_hdr->unshared_len) - pkt_hdr->unshared_len = offset; - - packet_ref(pkt_hdr); - return _odp_packet_hdl(ref_hdr); -} - -odp_packet_t odp_packet_ref_static(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - - pkt_hdr->unshared_len = 0; - packet_ref(pkt_hdr); - return pkt; -} - -odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) -{ - odp_packet_t hdr; - odp_packet_hdr_t *pkt_hdr; - - if (pkt == ODP_PACKET_INVALID) - return ODP_PACKET_INVALID; - - pkt_hdr = odp_packet_hdr(pkt); - if (offset >= packet_len(pkt_hdr)) - return ODP_PACKET_INVALID; - - hdr = odp_packet_alloc(odp_packet_pool(pkt), 0); - - if (hdr == ODP_PACKET_INVALID) - return ODP_PACKET_INVALID; - - return packet_splice(pkt_hdr, offset, odp_packet_hdr(hdr)); -} - -odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, - odp_packet_t hdr) -{ - odp_packet_hdr_t *pkt_hdr; - - if (pkt == ODP_PACKET_INVALID || - hdr == ODP_PACKET_INVALID || - pkt == hdr) - return ODP_PACKET_INVALID; - - ODP_ASSERT(odp_packet_has_ref(hdr) == 0); - - pkt_hdr = odp_packet_hdr(pkt); - if (offset >= packet_len(pkt_hdr)) - return ODP_PACKET_INVALID; - - return packet_splice(pkt_hdr, offset, odp_packet_hdr(hdr)); -} - -int odp_packet_has_ref(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - - return pkt_hdr->ref_hdr != NULL || packet_ref_count(pkt_hdr) > 1; -} - -/* * * Copy * ******************************************************** @@ -1908,7 +1585,8 @@ int odp_packet_has_ref(odp_packet_t pkt) odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool) { - uint32_t pktlen = odp_packet_len(pkt); + odp_packet_hdr_t *srchdr = odp_packet_hdr(pkt); + uint32_t pktlen = srchdr->frame_len; odp_packet_t newpkt = odp_packet_alloc(pool, pktlen); if (newpkt != ODP_PACKET_INVALID) { @@ -1947,7 +1625,7 @@ int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t offset, uint8_t *dstaddr = (uint8_t *)dst; odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (offset + len > packet_len(pkt_hdr)) + if (offset + len > pkt_hdr->frame_len) return -1; while (len > 0) { @@ -1971,11 +1649,9 @@ int odp_packet_copy_from_mem(odp_packet_t pkt, uint32_t offset, const uint8_t *srcaddr = (const uint8_t *)src; odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (offset + len > packet_len(pkt_hdr)) + if (offset + len > pkt_hdr->frame_len) return -1; - ODP_ASSERT(odp_packet_unshared_len(pkt) >= offset + len); - while (len > 0) { mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); cpylen = len > seglen ? seglen : len; @@ -2001,12 +1677,10 @@ int odp_packet_copy_from_pkt(odp_packet_t dst, uint32_t dst_offset, uint32_t src_seglen = 0; /* GCC */ int overlap; - if (dst_offset + len > packet_len(dst_hdr) || - src_offset + len > packet_len(src_hdr)) + if (dst_offset + len > dst_hdr->frame_len || + src_offset + len > src_hdr->frame_len) return -1; - ODP_ASSERT(odp_packet_unshared_len(dst) >= dst_offset + len); - overlap = (dst_hdr == src_hdr && ((dst_offset <= src_offset && dst_offset + len >= src_offset) || @@ -2090,7 +1764,7 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n - len, " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); len += snprintf(&str[len], n - len, - " frame_len %" PRIu32 "\n", packet_len(hdr)); + " frame_len %" PRIu32 "\n", hdr->frame_len); len += snprintf(&str[len], n - len, " input %" PRIu64 "\n", odp_pktio_to_u64(hdr->input));