Message ID | 1415678595-31405-11-git-send-email-bill.fischofer@linaro.org |
---|---|
State | New |
Headers | show |
On 11 November 2014 09:33, Bill Fischofer <bill.fischofer@linaro.org> wrote: > Signed-off-by: Bill Fischofer <bill.fischofer@linaro.org> > --- > platform/linux-generic/odp_buffer.c | 263 ++++++- > platform/linux-generic/odp_buffer_pool.c | 661 +++++++--------- > platform/linux-generic/odp_packet.c | 1200 > +++++++++++++++++++++++------ > platform/linux-generic/odp_packet_flags.c | 202 ----- > 4 files changed, 1455 insertions(+), 871 deletions(-) > delete mode 100644 platform/linux-generic/odp_packet_flags.c > > diff --git a/platform/linux-generic/odp_buffer.c > b/platform/linux-generic/odp_buffer.c > index e54e0e7..e47c722 100644 > --- a/platform/linux-generic/odp_buffer.c > +++ b/platform/linux-generic/odp_buffer.c > @@ -5,46 +5,259 @@ > */ > > #include <odp_buffer.h> > -#include <odp_buffer_internal.h> > #include <odp_buffer_pool_internal.h> > +#include <odp_buffer_internal.h> > +#include <odp_buffer_inlines.h> > > #include <string.h> > #include <stdio.h> > > +void *odp_buffer_offset_map(odp_buffer_t buf, size_t offset, size_t > *seglen) > +{ > + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); > > -void *odp_buffer_addr(odp_buffer_t buf) > + if (offset > buf_hdr->size) > + return NULL; > + > + return buffer_map(buf_hdr, offset, seglen, buf_hdr->size); > +} > + > +void odp_buffer_offset_unmap(odp_buffer_t buf ODP_UNUSED, > + size_t offset ODP_UNUSED) > { > - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); > +} > > - return hdr->addr; > +size_t odp_buffer_segment_count(odp_buffer_t buf) > +{ > + return odp_buf_to_hdr(buf)->segcount; > } > > +int odp_buffer_is_segmented(odp_buffer_t buf) > +{ > + return odp_buf_to_hdr(buf)->segcount > 1; > +} > > -size_t odp_buffer_size(odp_buffer_t buf) > +odp_buffer_segment_t odp_buffer_segment_by_index(odp_buffer_t buf, > + size_t ndx) > { > - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); > + return buffer_segment(odp_buf_to_hdr(buf), ndx); > +} > > - return hdr->size; > +odp_buffer_segment_t odp_buffer_segment_next(odp_buffer_t buf, > + odp_buffer_segment_t seg) > +{ > + return segment_next(odp_buf_to_hdr(buf), seg); > } > > +void *odp_buffer_segment_map(odp_buffer_t buf, odp_buffer_segment_t seg, > + size_t *seglen) > +{ > + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); > > -int odp_buffer_type(odp_buffer_t buf) > + return segment_map(buf_hdr, seg, seglen, buf_hdr->size); > +} > + > +void odp_buffer_segment_unmap(odp_buffer_segment_t seg ODP_UNUSED) > { > - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); > +} > + > +odp_buffer_t odp_buffer_split(odp_buffer_t buf, size_t offset) > +{ > + size_t size = odp_buffer_size(buf); > + odp_buffer_pool_t pool = odp_buffer_pool(buf); > + odp_buffer_t splitbuf; > + size_t splitsize = size - offset; > + > + if (offset >= size) > + return ODP_BUFFER_INVALID; > + > + splitbuf = buffer_alloc(pool, splitsize); > + > + if (splitbuf != ODP_BUFFER_INVALID) { > + buffer_copy_to_buffer(splitbuf, 0, buf, splitsize, > splitsize); > + odp_buffer_trim(buf, splitsize); > + } > > - return hdr->type; > + return splitbuf; > } > > +odp_buffer_t odp_buffer_join(odp_buffer_t buf1, odp_buffer_t buf2) > +{ > + size_t size1 = odp_buffer_size(buf1); > + size_t size2 = odp_buffer_size(buf2); > + odp_buffer_t joinbuf; > + void *udata1, *udata_join; > + size_t udata_size1, udata_size_join; > > -int odp_buffer_is_valid(odp_buffer_t buf) > + joinbuf = buffer_alloc(odp_buffer_pool(buf1), size1 + size2); > + > + if (joinbuf != ODP_BUFFER_INVALID) { > + buffer_copy_to_buffer(joinbuf, 0, buf1, 0, size1); > + buffer_copy_to_buffer(joinbuf, size1, buf2, 0, size2); > + > + /* Copy user metadata if present */ > + udata1 = odp_buffer_udata(buf1, &udata_size1); > + udata_join = odp_buffer_udata(joinbuf, &udata_size_join); > + > + if (udata1 != NULL && udata_join != NULL) > + memcpy(udata_join, udata1, > + udata_size_join > udata_size1 ? > + udata_size1 : udata_size_join); > + > + odp_buffer_free(buf1); > + odp_buffer_free(buf2); > + } > + > + return joinbuf; > +} > + > +odp_buffer_t odp_buffer_trim(odp_buffer_t buf, size_t bytes) > { > - odp_buffer_bits_t handle; > + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); > + > + if (bytes >= buf_hdr->size) > + return ODP_BUFFER_INVALID; > + > + buf_hdr->size -= bytes; > + size_t bufsegs = buf_hdr->size / buf_hdr->segsize; > + > + if (buf_hdr->size % buf_hdr->segsize > 0) > + bufsegs++; > + > + pool_entry_t *pool = odp_pool_to_entry(buf_hdr->pool_hdl); > > - handle.u32 = buf; > + /* Return excess segments back to block list */ > + while (bufsegs > buf_hdr->segcount) > + ret_blk(&pool->s, buf_hdr->addr[--buf_hdr->segcount]); > > - return (handle.index != ODP_BUFFER_INVALID_INDEX); > + return buf; > } > > +odp_buffer_t odp_buffer_extend(odp_buffer_t buf, size_t bytes) > +{ > + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); > + > + size_t lastseg = buf_hdr->size % buf_hdr->segsize; > + > + if (bytes <= buf_hdr->segsize - lastseg) { > + buf_hdr->size += bytes; > + return buf; > + } > + > + pool_entry_t *pool = odp_pool_to_entry(buf_hdr->pool_hdl); > + int extsize = buf_hdr->size + bytes; > + > + /* Extend buffer by adding additional segments to it */ > + if (extsize > ODP_CONFIG_BUF_MAX_SIZE || pool->s.flags.unsegmented) > + return ODP_BUFFER_INVALID; > + > + size_t segcount = buf_hdr->segcount; > + size_t extsegs = extsize / buf_hdr->segsize; > + > + if (extsize % buf_hdr->segsize > 0) > + extsize++; > + > + while (extsegs > buf_hdr->segcount) { > + void *newblk = get_blk(&pool->s); > + > + /* If we fail to get all the blocks we need, back out */ > + if (newblk == NULL) { > + while (segcount > buf_hdr->segcount) > + ret_blk(&pool->s, > buf_hdr->addr[--segcount]); > + return ODP_BUFFER_INVALID; > + } > + > + buf_hdr->addr[segcount++] = newblk; > + } > + > + buf_hdr->segcount = segcount; > + buf_hdr->size = extsize; > + > + return buf; > +} > + > +odp_buffer_t odp_buffer_clone(odp_buffer_t buf) > +{ > + return odp_buffer_copy(buf, odp_buf_to_hdr(buf)->pool_hdl); > +} > + > +odp_buffer_t odp_buffer_copy(odp_buffer_t buf, odp_buffer_pool_t pool) > +{ > + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); > + size_t len = buf_hdr->size; > + odp_buffer_t cpybuf = buffer_alloc(pool, len); > + > + if (cpybuf == ODP_BUFFER_INVALID) > + return ODP_BUFFER_INVALID; > + > + if (buffer_copy_to_buffer(cpybuf, 0, buf, 0, len) == 0) > + return cpybuf; > + > + odp_buffer_free(cpybuf); > + return ODP_BUFFER_INVALID; > +} > + > +int buffer_copy_to_buffer(odp_buffer_t dstbuf, size_t dstoffset, > + odp_buffer_t srcbuf, size_t srcoffset, > + size_t len) > +{ > + void *dstmap; > + void *srcmap; > + size_t cpylen, minseg, dstseglen, srcseglen; > + > + while (len > 0) { > + dstmap = odp_buffer_offset_map(dstbuf, dstoffset, > &dstseglen); > + srcmap = odp_buffer_offset_map(srcbuf, srcoffset, > &srcseglen); > + if (dstmap == NULL || srcmap == NULL) > + return -1; > + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; > + cpylen = len > minseg ? minseg : len; > + memcpy(dstmap, srcmap, cpylen); > + srcoffset += cpylen; > + dstoffset += cpylen; > + len -= cpylen; > + } > + > + return 0; > +} > + > +void *odp_buffer_addr(odp_buffer_t buf) > +{ > + return odp_buf_to_hdr(buf)->addr[0]; > +} > + > +odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf) > +{ > + return odp_buf_to_hdr(buf)->pool_hdl; > +} > + > +size_t odp_buffer_size(odp_buffer_t buf) > +{ > + return odp_buf_to_hdr(buf)->size; > +} > + > +odp_buffer_type_e odp_buffer_type(odp_buffer_t buf) > +{ > + return odp_buf_to_hdr(buf)->type; > +} > + > +void *odp_buffer_udata(odp_buffer_t buf, size_t *usize) > +{ > + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); > + > + *usize = hdr->udata_size; > + return hdr->udata_addr; > +} > + > +void *odp_buffer_udata_addr(odp_buffer_t buf) > +{ > + return odp_buf_to_hdr(buf)->udata_addr; > +} > + > +int odp_buffer_is_valid(odp_buffer_t buf) > +{ > + return validate_buf(buf) != NULL; > +} > > int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf) > { > @@ -63,27 +276,13 @@ int odp_buffer_snprint(char *str, size_t n, > odp_buffer_t buf) > len += snprintf(&str[len], n-len, > " pool %i\n", hdr->pool_hdl); > len += snprintf(&str[len], n-len, > - " index %"PRIu32"\n", hdr->index); > - len += snprintf(&str[len], n-len, > - " phy_addr %"PRIu64"\n", hdr->phys_addr); > - len += snprintf(&str[len], n-len, > - " addr %p\n", hdr->addr); > + " addr %p\n", hdr->addr[0]); > len += snprintf(&str[len], n-len, > " size %zu\n", hdr->size); > len += snprintf(&str[len], n-len, > - " cur_offset %zu\n", hdr->cur_offset); > - len += snprintf(&str[len], n-len, > " ref_count %i\n", hdr->ref_count); > len += snprintf(&str[len], n-len, > " type %i\n", hdr->type); > - len += snprintf(&str[len], n-len, > - " Scatter list\n"); > - len += snprintf(&str[len], n-len, > - " num_bufs %i\n", > hdr->scatter.num_bufs); > - len += snprintf(&str[len], n-len, > - " pos %i\n", hdr->scatter.pos); > - len += snprintf(&str[len], n-len, > - " total_len %zu\n", > hdr->scatter.total_len); > > return len; > } > @@ -100,9 +299,3 @@ void odp_buffer_print(odp_buffer_t buf) > > printf("\n%s\n", str); > } > - > -void odp_buffer_copy_scatter(odp_buffer_t buf_dst, odp_buffer_t buf_src) > -{ > - (void)buf_dst; > - (void)buf_src; > -} > diff --git a/platform/linux-generic/odp_buffer_pool.c > b/platform/linux-generic/odp_buffer_pool.c > index a48d7d6..f6161bb 100644 > --- a/platform/linux-generic/odp_buffer_pool.c > +++ b/platform/linux-generic/odp_buffer_pool.c > @@ -6,8 +6,9 @@ > > #include <odp_std_types.h> > #include <odp_buffer_pool.h> > -#include <odp_buffer_pool_internal.h> > #include <odp_buffer_internal.h> > +#include <odp_buffer_pool_internal.h> > +#include <odp_buffer_inlines.h> > #include <odp_packet_internal.h> > #include <odp_timer_internal.h> > #include <odp_shared_memory.h> > @@ -16,6 +17,7 @@ > #include <odp_config.h> > #include <odp_hints.h> > #include <odp_debug.h> > +#include <odph_eth.h> > > #include <string.h> > #include <stdlib.h> > @@ -33,36 +35,26 @@ > #define LOCK_INIT(a) odp_spinlock_init(a) > #endif > > - > -#if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS > -#error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS > -#endif > - > -#define NULL_INDEX ((uint32_t)-1) > - > -union buffer_type_any_u { > +typedef union buffer_type_any_u { > odp_buffer_hdr_t buf; > odp_packet_hdr_t pkt; > odp_timeout_hdr_t tmo; > -}; > - > -ODP_STATIC_ASSERT((sizeof(union buffer_type_any_u) % 8) == 0, > - "BUFFER_TYPE_ANY_U__SIZE_ERR"); > +} odp_anybuf_t; > > /* Any buffer type header */ > typedef struct { > union buffer_type_any_u any_hdr; /* any buffer type */ > - uint8_t buf_data[]; /* start of buffer data area */ > } odp_any_buffer_hdr_t; > > +typedef struct odp_any_hdr_stride { > + uint8_t > pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_any_buffer_hdr_t))]; > +} odp_any_hdr_stride; > > -typedef union pool_entry_u { > - struct pool_entry_s s; > - > - uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct > pool_entry_s))]; > - > -} pool_entry_t; > +#if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS > +#error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS > +#endif > > +#define NULL_INDEX ((uint32_t)-1) > > typedef struct pool_table_t { > pool_entry_t pool[ODP_CONFIG_BUFFER_POOLS]; > @@ -76,39 +68,6 @@ static pool_table_t *pool_tbl; > /* Pool entry pointers (for inlining) */ > void *pool_entry_ptr[ODP_CONFIG_BUFFER_POOLS]; > > - > -static __thread odp_buffer_chunk_hdr_t > *local_chunk[ODP_CONFIG_BUFFER_POOLS]; > - > - > -static inline odp_buffer_pool_t pool_index_to_handle(uint32_t pool_id) > -{ > - return pool_id + 1; > -} > - > - > -static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl) > -{ > - return pool_hdl -1; > -} > - > - > -static inline void set_handle(odp_buffer_hdr_t *hdr, > - pool_entry_t *pool, uint32_t index) > -{ > - odp_buffer_pool_t pool_hdl = pool->s.pool_hdl; > - uint32_t pool_id = pool_handle_to_index(pool_hdl); > - > - if (pool_id >= ODP_CONFIG_BUFFER_POOLS) > - ODP_ABORT("set_handle: Bad pool handle %u\n", pool_hdl); > - > - if (index > ODP_BUFFER_MAX_INDEX) > - ODP_ERR("set_handle: Bad buffer index\n"); > - > - hdr->handle.pool_id = pool_id; > - hdr->handle.index = index; > -} > - > - > int odp_buffer_pool_init_global(void) > { > uint32_t i; > @@ -142,269 +101,167 @@ int odp_buffer_pool_init_global(void) > return 0; > } > > - > -static odp_buffer_hdr_t *index_to_hdr(pool_entry_t *pool, uint32_t index) > -{ > - odp_buffer_hdr_t *hdr; > - > - hdr = (odp_buffer_hdr_t *)(pool->s.buf_base + index * > pool->s.buf_size); > - return hdr; > -} > - > - > -static void add_buf_index(odp_buffer_chunk_hdr_t *chunk_hdr, uint32_t > index) > -{ > - uint32_t i = chunk_hdr->chunk.num_bufs; > - chunk_hdr->chunk.buf_index[i] = index; > - chunk_hdr->chunk.num_bufs++; > -} > - > - > -static uint32_t rem_buf_index(odp_buffer_chunk_hdr_t *chunk_hdr) > +/** > + * Buffer pool creation > + */ > +odp_buffer_pool_t odp_buffer_pool_create(const char *name, > + odp_buffer_pool_param_t *args, > + odp_buffer_pool_init_t *init_args) > { > - uint32_t index; > + odp_buffer_pool_t pool_hdl = ODP_BUFFER_POOL_INVALID; > + pool_entry_t *pool; > uint32_t i; > + odp_buffer_pool_param_t *params; > + odp_buffer_pool_init_t *init_params; > > - i = chunk_hdr->chunk.num_bufs - 1; > - index = chunk_hdr->chunk.buf_index[i]; > - chunk_hdr->chunk.num_bufs--; > - return index; > -} > - > - > -static odp_buffer_chunk_hdr_t *next_chunk(pool_entry_t *pool, > - odp_buffer_chunk_hdr_t > *chunk_hdr) > -{ > - uint32_t index; > - > - index = chunk_hdr->chunk.buf_index[ODP_BUFS_PER_CHUNK-1]; > - if (index == NULL_INDEX) > - return NULL; > - else > - return (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, index); > -} > - > - > -static odp_buffer_chunk_hdr_t *rem_chunk(pool_entry_t *pool) > -{ > - odp_buffer_chunk_hdr_t *chunk_hdr; > - > - chunk_hdr = pool->s.head; > - if (chunk_hdr == NULL) { > - /* Pool is empty */ > - return NULL; > - } > - > - pool->s.head = next_chunk(pool, chunk_hdr); > - pool->s.free_bufs -= ODP_BUFS_PER_CHUNK; > - > - /* unlink */ > - rem_buf_index(chunk_hdr); > - return chunk_hdr; > -} > - > - > -static void add_chunk(pool_entry_t *pool, odp_buffer_chunk_hdr_t > *chunk_hdr) > -{ > - if (pool->s.head) /* link pool head to the chunk */ > - add_buf_index(chunk_hdr, pool->s.head->buf_hdr.index); > - else > - add_buf_index(chunk_hdr, NULL_INDEX); > - > - pool->s.head = chunk_hdr; > - pool->s.free_bufs += ODP_BUFS_PER_CHUNK; > -} > + /* Default pool creation arguments */ > + static odp_buffer_pool_param_t default_params = { > + .buf_num = 1024, > + .buf_size = ODP_CONFIG_BUF_SEG_SIZE, > + .buf_type = ODP_BUFFER_TYPE_PACKET, > + .buf_opts = ODP_BUFFER_OPTS_UNSEGMENTED, > + }; > Do we need to have this behaviour of allocating default params if the input params are NULL. It would be better if the API returns error. > > + /* Default initialization paramters */ > + static odp_buffer_pool_init_t default_init_params = { > + .udata_size = 0, > + .buf_init = NULL, > + .buf_init_arg = NULL, > + }; > > -static void check_align(pool_entry_t *pool, odp_buffer_hdr_t *hdr) > -{ > - if (!ODP_ALIGNED_CHECK_POWER_2(hdr->addr, pool->s.user_align)) { > - ODP_ABORT("check_align: user data align error %p, align > %zu\n", > - hdr->addr, pool->s.user_align); > - } > + /* Handle NULL input parameters */ > + params = args == NULL ? &default_params : args; > + init_params = init_args == NULL ? &default_init_params > + : init_args; > > - if (!ODP_ALIGNED_CHECK_POWER_2(hdr, ODP_CACHE_LINE_SIZE)) { > - ODP_ABORT("check_align: hdr align error %p, align %i\n", > - hdr, ODP_CACHE_LINE_SIZE); > - } > -} > + if (params->buf_type != ODP_BUFFER_TYPE_PACKET) > + params->buf_opts |= ODP_BUFFER_OPTS_UNSEGMENTED; > > + int unsegmented = ((params->buf_opts & > ODP_BUFFER_OPTS_UNSEGMENTED) == > + ODP_BUFFER_OPTS_UNSEGMENTED); > > -static void fill_hdr(void *ptr, pool_entry_t *pool, uint32_t index, > - int buf_type) > -{ > - odp_buffer_hdr_t *hdr = (odp_buffer_hdr_t *)ptr; > - size_t size = pool->s.hdr_size; > - uint8_t *buf_data; > + uint32_t udata_stride = > + ODP_CACHE_LINE_SIZE_ROUNDUP(init_params->udata_size); > > - if (buf_type == ODP_BUFFER_TYPE_CHUNK) > - size = sizeof(odp_buffer_chunk_hdr_t); > - > - switch (pool->s.buf_type) { > - odp_raw_buffer_hdr_t *raw_hdr; > - odp_packet_hdr_t *packet_hdr; > - odp_timeout_hdr_t *tmo_hdr; > - odp_any_buffer_hdr_t *any_hdr; > + uint32_t blk_size, buf_stride; > > + switch (params->buf_type) { > case ODP_BUFFER_TYPE_RAW: > - raw_hdr = ptr; > - buf_data = raw_hdr->buf_data; > + blk_size = params->buf_size; > + buf_stride = sizeof(odp_buffer_hdr_stride); > break; > + > case ODP_BUFFER_TYPE_PACKET: > - packet_hdr = ptr; > - buf_data = packet_hdr->buf_data; > + if (unsegmented) > + blk_size = > + > ODP_CACHE_LINE_SIZE_ROUNDUP(params->buf_size); > + else > + blk_size = ODP_ALIGN_ROUNDUP(params->buf_size, > + > ODP_CONFIG_BUF_SEG_SIZE); > + buf_stride = sizeof(odp_packet_hdr_stride); > break; > + > case ODP_BUFFER_TYPE_TIMEOUT: > - tmo_hdr = ptr; > - buf_data = tmo_hdr->buf_data; > + blk_size = 0; /* Timeouts have no block data, only > metadata */ > + buf_stride = sizeof(odp_timeout_hdr_stride); > break; > + > case ODP_BUFFER_TYPE_ANY: > - any_hdr = ptr; > - buf_data = any_hdr->buf_data; > + if (unsegmented) > + blk_size = > + > ODP_CACHE_LINE_SIZE_ROUNDUP(params->buf_size); > + else > + blk_size = ODP_ALIGN_ROUNDUP(params->buf_size, > + > ODP_CONFIG_BUF_SEG_SIZE); > + buf_stride = sizeof(odp_any_hdr_stride); > break; > - default: > - ODP_ABORT("Bad buffer type\n"); > - } > - > - memset(hdr, 0, size); > - > - set_handle(hdr, pool, index); > - > - hdr->addr = &buf_data[pool->s.buf_offset - pool->s.hdr_size]; > - hdr->index = index; > - hdr->size = pool->s.user_size; > - hdr->pool_hdl = pool->s.pool_hdl; > - hdr->type = buf_type; > - > - check_align(pool, hdr); > -} > - > - > -static void link_bufs(pool_entry_t *pool) > -{ > - odp_buffer_chunk_hdr_t *chunk_hdr; > - size_t hdr_size; > - size_t data_size; > - size_t data_align; > - size_t tot_size; > - size_t offset; > - size_t min_size; > - uint64_t pool_size; > - uintptr_t buf_base; > - uint32_t index; > - uintptr_t pool_base; > - int buf_type; > - > - buf_type = pool->s.buf_type; > - data_size = pool->s.user_size; > - data_align = pool->s.user_align; > - pool_size = pool->s.pool_size; > - pool_base = (uintptr_t) pool->s.pool_base_addr; > - > - if (buf_type == ODP_BUFFER_TYPE_RAW) { > - hdr_size = sizeof(odp_raw_buffer_hdr_t); > - } else if (buf_type == ODP_BUFFER_TYPE_PACKET) { > - hdr_size = sizeof(odp_packet_hdr_t); > - } else if (buf_type == ODP_BUFFER_TYPE_TIMEOUT) { > - hdr_size = sizeof(odp_timeout_hdr_t); > - } else if (buf_type == ODP_BUFFER_TYPE_ANY) { > - hdr_size = sizeof(odp_any_buffer_hdr_t); > - } else > - ODP_ABORT("odp_buffer_pool_create: Bad type %i\n", > buf_type); > - > - > - /* Chunk must fit into buffer data area.*/ > - min_size = sizeof(odp_buffer_chunk_hdr_t) - hdr_size; > - if (data_size < min_size) > - data_size = min_size; > - > - /* Roundup data size to full cachelines */ > - data_size = ODP_CACHE_LINE_SIZE_ROUNDUP(data_size); > - > - /* Min cacheline alignment for buffer header and data */ > - data_align = ODP_CACHE_LINE_SIZE_ROUNDUP(data_align); > - offset = ODP_CACHE_LINE_SIZE_ROUNDUP(hdr_size); > - > - /* Multiples of cacheline size */ > - if (data_size > data_align) > - tot_size = data_size + offset; > - else > - tot_size = data_align + offset; > - > - /* First buffer */ > - buf_base = ODP_ALIGN_ROUNDUP(pool_base + offset, data_align) - > offset; > - > - pool->s.hdr_size = hdr_size; > - pool->s.buf_base = buf_base; > - pool->s.buf_size = tot_size; > - pool->s.buf_offset = offset; > - index = 0; > - > - chunk_hdr = (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, index); > - pool->s.head = NULL; > - pool_size -= buf_base - pool_base; > - > - while (pool_size > ODP_BUFS_PER_CHUNK * tot_size) { > - int i; > - > - fill_hdr(chunk_hdr, pool, index, ODP_BUFFER_TYPE_CHUNK); > - > - index++; > - > - for (i = 0; i < ODP_BUFS_PER_CHUNK - 1; i++) { > - odp_buffer_hdr_t *hdr = index_to_hdr(pool, index); > - > - fill_hdr(hdr, pool, index, buf_type); > - > - add_buf_index(chunk_hdr, index); > - index++; > - } > - > - add_chunk(pool, chunk_hdr); > > - chunk_hdr = (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, > - index); > - pool->s.num_bufs += ODP_BUFS_PER_CHUNK; > - pool_size -= ODP_BUFS_PER_CHUNK * tot_size; > + default: > + return ODP_BUFFER_POOL_INVALID; > } > -} > - > - > -odp_buffer_pool_t odp_buffer_pool_create(const char *name, > - void *base_addr, uint64_t size, > - size_t buf_size, size_t buf_align, > - int buf_type) > -{ > - odp_buffer_pool_t pool_hdl = ODP_BUFFER_POOL_INVALID; > - pool_entry_t *pool; > - uint32_t i; > > for (i = 0; i < ODP_CONFIG_BUFFER_POOLS; i++) { > pool = get_pool_entry(i); > > LOCK(&pool->s.lock); > + if (pool->s.shm != ODP_SHM_INVALID) { > + UNLOCK(&pool->s.lock); > + continue; > + } > > - if (pool->s.buf_base == 0) { > - /* found free pool */ > + /* found free pool */ > + size_t block_size, mdata_size, udata_size; > > - strncpy(pool->s.name, name, > - ODP_BUFFER_POOL_NAME_LEN - 1); > - pool->s.name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; > - pool->s.pool_base_addr = base_addr; > - pool->s.pool_size = size; > - pool->s.user_size = buf_size; > - pool->s.user_align = buf_align; > - pool->s.buf_type = buf_type; > + strncpy(pool->s.name, name, > + ODP_BUFFER_POOL_NAME_LEN - 1); > + pool->s.name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; > > - link_bufs(pool); > + pool->s.params = *params; > + pool->s.init_params = *init_params; > > - UNLOCK(&pool->s.lock); > + mdata_size = params->buf_num * buf_stride; > + udata_size = params->buf_num * udata_stride; > + block_size = params->buf_num * blk_size; > + > + pool->s.pool_size = ODP_PAGE_SIZE_ROUNDUP(mdata_size + > + udata_size + > + block_size); > > - pool_hdl = pool->s.pool_hdl; > - break; > + pool->s.shm = odp_shm_reserve(pool->s.name, > pool->s.pool_size, > + ODP_PAGE_SIZE, 0); > + if (pool->s.shm == ODP_SHM_INVALID) { > + UNLOCK(&pool->s.lock); > + return ODP_BUFFER_POOL_INVALID; > } > > + pool->s.pool_base_addr = (uint8_t > *)odp_shm_addr(pool->s.shm); > + pool->s.flags.unsegmented = unsegmented; > + pool->s.seg_size = unsegmented ? > + blk_size : ODP_CONFIG_BUF_SEG_SIZE; > + uint8_t *udata_base_addr = pool->s.pool_base_addr + > mdata_size; > + uint8_t *block_base_addr = udata_base_addr + udata_size; > + > + pool->s.bufcount = 0; > + pool->s.buf_freelist = NULL; > + pool->s.blk_freelist = NULL; > + > + uint8_t *buf = pool->s.udata_base_addr - buf_stride; > + uint8_t *udat = (udata_stride == 0) ? NULL : > + block_base_addr - udata_stride; > + > + /* Init buffer common header and add to pool buffer > freelist */ > + do { > + odp_buffer_hdr_t *tmp = > + (odp_buffer_hdr_t *)(void *)buf; > + tmp->pool_hdl = pool->s.pool_hdl; > + tmp->size = 0; > + tmp->type = params->buf_type; > + tmp->udata_addr = (void *)udat; > + tmp->udata_size = init_params->udata_size; > + tmp->segcount = 0; > + tmp->segsize = pool->s.seg_size; > + tmp->buf_hdl.handle = > + odp_buffer_encode_handle((odp_buffer_hdr_t > *) > + (void *)buf); > + ret_buf(&pool->s, tmp); > + buf -= buf_stride; > + udat -= udata_stride; > + } while (buf >= pool->s.pool_base_addr); > + > + /* Form block freelist for pool */ > + uint8_t *blk = pool->s.pool_base_addr + pool->s.pool_size - > + pool->s.seg_size; > + > + if (blk_size > 0) > + do { > + ret_blk(&pool->s, blk); > + blk -= pool->s.seg_size; > + } while (blk >= block_base_addr); > + > UNLOCK(&pool->s.lock); > + > + pool_hdl = pool->s.pool_hdl; > + break; > } > > return pool_hdl; > @@ -431,93 +288,172 @@ odp_buffer_pool_t odp_buffer_pool_lookup(const char > *name) > return ODP_BUFFER_POOL_INVALID; > } > > - > -odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_hdl) > +odp_buffer_pool_t odp_buffer_pool_next(odp_buffer_pool_t pool_hdl, > + char *name, > + size_t *udata_size, > + odp_buffer_pool_param_t *params, > + int *predef) > { > - pool_entry_t *pool; > - odp_buffer_chunk_hdr_t *chunk; > - odp_buffer_bits_t handle; > - uint32_t pool_id = pool_handle_to_index(pool_hdl); > - > - pool = get_pool_entry(pool_id); > - chunk = local_chunk[pool_id]; > - > - if (chunk == NULL) { > - LOCK(&pool->s.lock); > - chunk = rem_chunk(pool); > - UNLOCK(&pool->s.lock); > + pool_entry_t *next_pool; > + uint32_t pool_id; > > - if (chunk == NULL) > - return ODP_BUFFER_INVALID; > + /* NULL input means first element */ > + if (pool_hdl == ODP_BUFFER_POOL_INVALID) { > + pool_id = 0; > + next_pool = get_pool_entry(pool_id); > + } else { > + pool_id = pool_handle_to_index(pool_hdl); > > - local_chunk[pool_id] = chunk; > + if (pool_id == ODP_CONFIG_BUFFER_POOLS) > + return ODP_BUFFER_POOL_INVALID; > + else > + next_pool = get_pool_entry(++pool_id); > } > > - if (chunk->chunk.num_bufs == 0) { > - /* give the chunk buffer */ > - local_chunk[pool_id] = NULL; > - chunk->buf_hdr.type = pool->s.buf_type; > + /* Only interested in pools that exist */ > + while (next_pool->s.shm == ODP_SHM_INVALID) { > + if (pool_id == ODP_CONFIG_BUFFER_POOLS) > + return ODP_BUFFER_POOL_INVALID; > + else > + next_pool = get_pool_entry(++pool_id); > + } > > - handle = chunk->buf_hdr.handle; > - } else { > - odp_buffer_hdr_t *hdr; > - uint32_t index; > - index = rem_buf_index(chunk); > - hdr = index_to_hdr(pool, index); > + /* Found the next pool, so return info about it */ > + strncpy(name, next_pool->s.name, ODP_BUFFER_POOL_NAME_LEN - 1); > + name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; > > - handle = hdr->handle; > - } > + *udata_size = next_pool->s.init_params.udata_size; > + *params = next_pool->s.params; > + *predef = next_pool->s.flags.predefined; > > - return handle.u32; > + return next_pool->s.pool_hdl; > } > > - > -void odp_buffer_free(odp_buffer_t buf) > +int odp_buffer_pool_destroy(odp_buffer_pool_t pool_hdl) > { > - odp_buffer_hdr_t *hdr; > - uint32_t pool_id; > - pool_entry_t *pool; > - odp_buffer_chunk_hdr_t *chunk_hdr; > + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); > > - hdr = odp_buf_to_hdr(buf); > - pool_id = pool_handle_to_index(hdr->pool_hdl); > - pool = get_pool_entry(pool_id); > - chunk_hdr = local_chunk[pool_id]; > + if (pool == NULL) > + return -1; > > - if (chunk_hdr && chunk_hdr->chunk.num_bufs == ODP_BUFS_PER_CHUNK - > 1) { > - /* Current chunk is full. Push back to the pool */ > - LOCK(&pool->s.lock); > - add_chunk(pool, chunk_hdr); > + LOCK(&pool->s.lock); > + > + if (pool->s.shm == ODP_SHM_INVALID || > + pool->s.bufcount > 0 || > + pool->s.flags.predefined) { > UNLOCK(&pool->s.lock); > - chunk_hdr = NULL; > + return -1; > } > > - if (chunk_hdr == NULL) { > - /* Use this buffer */ > - chunk_hdr = (odp_buffer_chunk_hdr_t *)hdr; > - local_chunk[pool_id] = chunk_hdr; > - chunk_hdr->chunk.num_bufs = 0; > + odp_shm_free(pool->s.shm); > + > + pool->s.shm = ODP_SHM_INVALID; > + UNLOCK(&pool->s.lock); > + > + return 0; > +} > + > +size_t odp_buffer_pool_headroom(odp_buffer_pool_t pool_hdl) > +{ > + return odp_pool_to_entry(pool_hdl)->s.headroom; > +} > + > +int odp_buffer_pool_set_headroom(odp_buffer_pool_t pool_hdl, size_t hr) > +{ > + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); > + > + if (hr >= pool->s.seg_size/2) { > + return -1; > } else { > - /* Add to current chunk */ > - add_buf_index(chunk_hdr, hdr->index); > + pool->s.headroom = hr; > + return 0; > } > } > > +size_t odp_buffer_pool_tailroom(odp_buffer_pool_t pool_hdl) > +{ > + return odp_pool_to_entry(pool_hdl)->s.tailroom; > +} > > -odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf) > +int odp_buffer_pool_set_tailroom(odp_buffer_pool_t pool_hdl, size_t tr) > { > - odp_buffer_hdr_t *hdr; > + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); > > - hdr = odp_buf_to_hdr(buf); > - return hdr->pool_hdl; > + if (tr >= pool->s.seg_size/2) { > + return -1; > + } else { > + pool->s.tailroom = tr; > + return 0; > + } > } > > +odp_buffer_t buffer_alloc(odp_buffer_pool_t pool_hdl, size_t size) > +{ > + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); > + size_t totsize = pool->s.headroom + size + pool->s.tailroom; > + odp_anybuf_t *buf; > + uint8_t *blk; > + > + if ((pool->s.flags.unsegmented && totsize > pool->s.seg_size) || > + (!pool->s.flags.unsegmented && totsize > > ODP_CONFIG_BUF_MAX_SIZE)) > + return ODP_BUFFER_INVALID; > + > + buf = (odp_anybuf_t *)(void *)get_buf(&pool->s); > + > + if (buf == NULL) > + return ODP_BUFFER_INVALID; > + > + /* Get blocks for this buffer, if pool uses application data */ > + if (buf->buf.segsize > 0) > + do { > + blk = get_blk(&pool->s); > + if (blk == NULL) { > + ret_buf(&pool->s, &buf->buf); > + return ODP_BUFFER_INVALID; > + } > + buf->buf.addr[buf->buf.segcount++] = blk; > + totsize = totsize < pool->s.seg_size ? 0 : > + totsize - pool->s.seg_size; > + } while (totsize > 0); > + > + switch (buf->buf.type) { > + case ODP_BUFFER_TYPE_RAW: > + break; > + > + case ODP_BUFFER_TYPE_PACKET: > + packet_init(pool, &buf->pkt, size); > + if (pool->s.init_params.buf_init != NULL) > + (*pool->s.init_params.buf_init) > + (buf->buf.buf_hdl.handle, > + pool->s.init_params.buf_init_arg); > + break; > + > + case ODP_BUFFER_TYPE_TIMEOUT: > + break; > + > + default: > + ret_buf(&pool->s, &buf->buf); > + return ODP_BUFFER_INVALID; > + } > + > + return odp_hdr_to_buf(&buf->buf); > +} > + > +odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_hdl) > +{ > + return buffer_alloc(pool_hdl, 0); > +} > + > +void odp_buffer_free(odp_buffer_t buf) > +{ > + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); > + pool_entry_t *pool = odp_buf_to_pool(hdr); > + ret_buf(&pool->s, hdr); > +} > > void odp_buffer_pool_print(odp_buffer_pool_t pool_hdl) > { > pool_entry_t *pool; > - odp_buffer_chunk_hdr_t *chunk_hdr; > - uint32_t i; > uint32_t pool_id; > > pool_id = pool_handle_to_index(pool_hdl); > @@ -528,47 +464,4 @@ void odp_buffer_pool_print(odp_buffer_pool_t pool_hdl) > printf(" pool %i\n", pool->s.pool_hdl); > printf(" name %s\n", pool->s.name); > printf(" pool base %p\n", pool->s.pool_base_addr); > - printf(" buf base 0x%"PRIxPTR"\n", pool->s.buf_base); > - printf(" pool size 0x%"PRIx64"\n", pool->s.pool_size); > - printf(" buf size %zu\n", pool->s.user_size); > - printf(" buf align %zu\n", pool->s.user_align); > - printf(" hdr size %zu\n", pool->s.hdr_size); > - printf(" alloc size %zu\n", pool->s.buf_size); > - printf(" offset to hdr %zu\n", pool->s.buf_offset); > - printf(" num bufs %"PRIu64"\n", pool->s.num_bufs); > - printf(" free bufs %"PRIu64"\n", pool->s.free_bufs); > - > - /* first chunk */ > - chunk_hdr = pool->s.head; > - > - if (chunk_hdr == NULL) { > - ODP_ERR(" POOL EMPTY\n"); > - return; > - } > - > - printf("\n First chunk\n"); > - > - for (i = 0; i < chunk_hdr->chunk.num_bufs - 1; i++) { > - uint32_t index; > - odp_buffer_hdr_t *hdr; > - > - index = chunk_hdr->chunk.buf_index[i]; > - hdr = index_to_hdr(pool, index); > - > - printf(" [%i] addr %p, id %"PRIu32"\n", i, hdr->addr, > index); > - } > - > - printf(" [%i] addr %p, id %"PRIu32"\n", i, > chunk_hdr->buf_hdr.addr, > - chunk_hdr->buf_hdr.index); > - > - /* next chunk */ > - chunk_hdr = next_chunk(pool, chunk_hdr); > - > - if (chunk_hdr) { > - printf(" Next chunk\n"); > - printf(" addr %p, id %"PRIu32"\n", > chunk_hdr->buf_hdr.addr, > - chunk_hdr->buf_hdr.index); > - } > - > - printf("\n"); > } > diff --git a/platform/linux-generic/odp_packet.c > b/platform/linux-generic/odp_packet.c > index 82ea879..45ac8e5 100644 > --- a/platform/linux-generic/odp_packet.c > +++ b/platform/linux-generic/odp_packet.c > @@ -11,29 +11,31 @@ > > #include <odph_eth.h> > #include <odph_ip.h> > +#include <odph_tcp.h> > +#include <odph_udp.h> > > #include <string.h> > #include <stdio.h> > > -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, > - odph_ipv4hdr_t *ipv4, size_t *offset_out); > -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, > - odph_ipv6hdr_t *ipv6, size_t *offset_out); > - > void odp_packet_init(odp_packet_t pkt) > { > odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); > + pool_entry_t *pool = odp_buf_to_pool(&pkt_hdr->buf_hdr); > const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, > buf_hdr); > uint8_t *start; > size_t len; > > start = (uint8_t *)pkt_hdr + start_offset; > - len = ODP_OFFSETOF(odp_packet_hdr_t, buf_data) - start_offset; > + len = sizeof(odp_packet_hdr_t) - start_offset; > memset(start, 0, len); > > pkt_hdr->l2_offset = ODP_PACKET_OFFSET_INVALID; > pkt_hdr->l3_offset = ODP_PACKET_OFFSET_INVALID; > pkt_hdr->l4_offset = ODP_PACKET_OFFSET_INVALID; > + pkt_hdr->payload_offset = ODP_PACKET_OFFSET_INVALID; > + > + pkt_hdr->headroom = pool->s.headroom; > + pkt_hdr->tailroom = pool->s.tailroom; > } > > odp_packet_t odp_packet_from_buffer(odp_buffer_t buf) > @@ -46,55 +48,112 @@ odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt) > return (odp_buffer_t)pkt; > } > > -void odp_packet_set_len(odp_packet_t pkt, size_t len) > +size_t odp_packet_len(odp_packet_t pkt) > { > - odp_packet_hdr(pkt)->frame_len = len; > + return odp_packet_hdr(pkt)->frame_len; > } > > -size_t odp_packet_get_len(odp_packet_t pkt) > +void *odp_packet_offset_map(odp_packet_t pkt, size_t offset, size_t > *seglen) > { > - return odp_packet_hdr(pkt)->frame_len; > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + > + if (offset >= pkt_hdr->frame_len) > + return NULL; > + > + return buffer_map(&pkt_hdr->buf_hdr, > + pkt_hdr->headroom + offset, > + seglen, pkt_hdr->frame_len); > } > > -uint8_t *odp_packet_addr(odp_packet_t pkt) > +void odp_packet_offset_unmap(odp_packet_t pkt ODP_UNUSED, > + size_t offset ODP_UNUSED) > { > - return odp_buffer_addr(odp_packet_to_buffer(pkt)); > } > > -uint8_t *odp_packet_data(odp_packet_t pkt) > +void *odp_packet_map(odp_packet_t pkt, size_t *seglen) > { > - return odp_packet_addr(pkt) + odp_packet_hdr(pkt)->frame_offset; > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + > + return buffer_map(&pkt_hdr->buf_hdr, > + 0, seglen, pkt_hdr->frame_len); > } > > +void *odp_packet_addr(odp_packet_t pkt) > +{ > + size_t seglen; > + return odp_packet_map(pkt, &seglen); > +} > > -uint8_t *odp_packet_l2(odp_packet_t pkt) > +odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt) > { > - const size_t offset = odp_packet_l2_offset(pkt); > + return odp_packet_hdr(pkt)->buf_hdr.pool_hdl; > +} > > - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) > - return NULL; > +odp_packet_segment_t odp_packet_segment_by_index(odp_packet_t pkt, > + size_t ndx) > +{ > > - return odp_packet_addr(pkt) + offset; > + return (odp_packet_segment_t) > + buffer_segment(&odp_packet_hdr(pkt)->buf_hdr, ndx); > } > > -size_t odp_packet_l2_offset(odp_packet_t pkt) > +odp_packet_segment_t odp_packet_segment_next(odp_packet_t pkt, > + odp_packet_segment_t seg) > { > - return odp_packet_hdr(pkt)->l2_offset; > + return (odp_packet_segment_t) > + segment_next(&odp_packet_hdr(pkt)->buf_hdr, seg); > } > > -void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset) > +void *odp_packet_segment_map(odp_packet_t pkt, odp_packet_segment_t seg, > + size_t *seglen) > { > - odp_packet_hdr(pkt)->l2_offset = offset; > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + > + return segment_map(&pkt_hdr->buf_hdr, seg, > + seglen, pkt_hdr->frame_len); > } > > -uint8_t *odp_packet_l3(odp_packet_t pkt) > +void odp_packet_segment_unmap(odp_packet_segment_t seg ODP_UNUSED) > { > - const size_t offset = odp_packet_l3_offset(pkt); > +} > > - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) > - return NULL; > +void *odp_packet_udata(odp_packet_t pkt, size_t *len) > +{ > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + > + *len = pkt_hdr->buf_hdr.udata_size; > + return pkt_hdr->buf_hdr.udata_addr; > +} > + > +void *odp_packet_udata_addr(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->buf_hdr.udata_addr; > +} > + > +void *odp_packet_l2_map(odp_packet_t pkt, size_t *seglen) > +{ > + return odp_packet_offset_map(pkt, odp_packet_l2_offset(pkt), > seglen); > +} > + > +size_t odp_packet_l2_offset(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->l2_offset; > +} > + > +int odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset) > +{ > + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); > + > + if (offset >= hdr->frame_len) > + return -1; > + > + hdr->l2_offset = offset; > + return 0; > +} > > - return odp_packet_addr(pkt) + offset; > +void *odp_packet_l3_map(odp_packet_t pkt, size_t *seglen) > +{ > + return odp_packet_offset_map(pkt, odp_packet_l3_offset(pkt), > seglen); > } > > size_t odp_packet_l3_offset(odp_packet_t pkt) > @@ -102,19 +161,35 @@ size_t odp_packet_l3_offset(odp_packet_t pkt) > return odp_packet_hdr(pkt)->l3_offset; > } > > -void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset) > +int odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset) > { > - odp_packet_hdr(pkt)->l3_offset = offset; > + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); > + > + if (offset >= hdr->frame_len) > + return -1; > + > + hdr->l3_offset = offset; > + return 0; > } > > -uint8_t *odp_packet_l4(odp_packet_t pkt) > +uint32_t odp_packet_l3_protocol(odp_packet_t pkt) > { > - const size_t offset = odp_packet_l4_offset(pkt); > + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); > > - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) > - return NULL; > + if (hdr->input_flags.l3) > + return hdr->l3_protocol; > + else > + return -1; > +} > + > +void odp_packet_set_l3_protocol(odp_packet_t pkt, uint16_t protocol) > +{ > + odp_packet_hdr(pkt)->l3_protocol = protocol; > +} > > - return odp_packet_addr(pkt) + offset; > +void *odp_packet_l4_map(odp_packet_t pkt, size_t *seglen) > +{ > + return odp_packet_offset_map(pkt, odp_packet_l4_offset(pkt), > seglen); > } > > size_t odp_packet_l4_offset(odp_packet_t pkt) > @@ -122,277 +197,902 @@ size_t odp_packet_l4_offset(odp_packet_t pkt) > return odp_packet_hdr(pkt)->l4_offset; > } > > -void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset) > +int odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset) > { > - odp_packet_hdr(pkt)->l4_offset = offset; > -} > + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); > + > + if (offset >= hdr->frame_len) > + return -1; > > + hdr->l4_offset = offset; > + return 0; > +} > > -int odp_packet_is_segmented(odp_packet_t pkt) > +uint32_t odp_packet_l4_protocol(odp_packet_t pkt) > { > - odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr((odp_buffer_t)pkt); > + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); > > - if (buf_hdr->scatter.num_bufs == 0) > - return 0; > + if (hdr->input_flags.l4) > + return hdr->l4_protocol; > else > - return 1; > + return -1; > } > > +void odp_packet_set_l4_protocol(odp_packet_t pkt, uint8_t protocol) > +{ > + odp_packet_hdr(pkt)->l4_protocol = protocol; > +} > > -int odp_packet_seg_count(odp_packet_t pkt) > +void *odp_packet_payload_map(odp_packet_t pkt, size_t *seglen) > { > - odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr((odp_buffer_t)pkt); > + return odp_packet_offset_map(pkt, odp_packet_payload_offset(pkt), > + seglen); > +} > > - return (int)buf_hdr->scatter.num_bufs + 1; > +size_t odp_packet_payload_offset(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->payload_offset; > } > > +int odp_packet_set_payload_offset(odp_packet_t pkt, size_t offset) > +{ > + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); > > -/** > - * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP > - * > - * Internal function: caller is resposible for passing only valid packet > handles > - * , lengths and offsets (usually done&called in packet input). > - * > - * @param pkt Packet handle > - * @param len Packet length in bytes > - * @param frame_offset Byte offset to L2 header > - */ > -void odp_packet_parse(odp_packet_t pkt, size_t len, size_t frame_offset) > + if (offset >= hdr->frame_len) > + return -1; > + > + hdr->payload_offset = offset; > + return 0; > +} > + > +int odp_packet_error(odp_packet_t pkt) > { > - odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); > - odph_ethhdr_t *eth; > - odph_vlanhdr_t *vlan; > - odph_ipv4hdr_t *ipv4; > - odph_ipv6hdr_t *ipv6; > - uint16_t ethtype; > - size_t offset = 0; > - uint8_t ip_proto = 0; > + return odp_packet_hdr(pkt)->error_flags.all != 0; > +} > > - pkt_hdr->input_flags.eth = 1; > - pkt_hdr->frame_offset = frame_offset; > - pkt_hdr->frame_len = len; > +void odp_packet_set_error(odp_packet_t pkt, int val) > +{ > + odp_packet_hdr(pkt)->error_flags.app_error = val; > +} > > - if (odp_unlikely(len < ODPH_ETH_LEN_MIN)) { > - pkt_hdr->error_flags.frame_len = 1; > - return; > - } else if (len > ODPH_ETH_LEN_MAX) { > - pkt_hdr->input_flags.jumbo = 1; > - } > +int odp_packet_inflag_l2(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->input_flags.l2; > +} > > - /* Assume valid L2 header, no CRC/FCS check in SW */ > - pkt_hdr->input_flags.l2 = 1; > - pkt_hdr->l2_offset = frame_offset; > +void odp_packet_set_inflag_l2(odp_packet_t pkt, int val) > +{ > + odp_packet_hdr(pkt)->input_flags.l2 = val; > +} > > - eth = (odph_ethhdr_t *)odp_packet_data(pkt); > - ethtype = odp_be_to_cpu_16(eth->type); > - vlan = (odph_vlanhdr_t *)ð->type; > +int odp_packet_inflag_l3(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->input_flags.l3; > +} > > - if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { > - pkt_hdr->input_flags.vlan_qinq = 1; > - ethtype = odp_be_to_cpu_16(vlan->tpid); > - offset += sizeof(odph_vlanhdr_t); > - vlan = &vlan[1]; > - } > +void odp_packet_set_inflag_l3(odp_packet_t pkt, int val) > +{ > + odp_packet_hdr(pkt)->input_flags.l3 = val; > +} > > - if (ethtype == ODPH_ETHTYPE_VLAN) { > - pkt_hdr->input_flags.vlan = 1; > - ethtype = odp_be_to_cpu_16(vlan->tpid); > - offset += sizeof(odph_vlanhdr_t); > - } > +int odp_packet_inflag_l4(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->input_flags.l4; > +} > > - /* Set l3_offset+flag only for known ethtypes */ > - switch (ethtype) { > - case ODPH_ETHTYPE_IPV4: > - pkt_hdr->input_flags.ipv4 = 1; > - pkt_hdr->input_flags.l3 = 1; > - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + > offset; > - ipv4 = (odph_ipv4hdr_t *)odp_packet_l3(pkt); > - ip_proto = parse_ipv4(pkt_hdr, ipv4, &offset); > - break; > - case ODPH_ETHTYPE_IPV6: > - pkt_hdr->input_flags.ipv6 = 1; > - pkt_hdr->input_flags.l3 = 1; > - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + > offset; > - ipv6 = (odph_ipv6hdr_t *)odp_packet_l3(pkt); > - ip_proto = parse_ipv6(pkt_hdr, ipv6, &offset); > - break; > - case ODPH_ETHTYPE_ARP: > - pkt_hdr->input_flags.arp = 1; > - /* fall through */ > - default: > - ip_proto = 0; > - break; > - } > +void odp_packet_set_inflag_l4(odp_packet_t pkt, int val) > +{ > + odp_packet_hdr(pkt)->input_flags.l4 = val; > +} > > - switch (ip_proto) { > - case ODPH_IPPROTO_UDP: > - pkt_hdr->input_flags.udp = 1; > - pkt_hdr->input_flags.l4 = 1; > - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; > - break; > - case ODPH_IPPROTO_TCP: > - pkt_hdr->input_flags.tcp = 1; > - pkt_hdr->input_flags.l4 = 1; > - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; > - break; > - case ODPH_IPPROTO_SCTP: > - pkt_hdr->input_flags.sctp = 1; > - pkt_hdr->input_flags.l4 = 1; > - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; > - break; > - case ODPH_IPPROTO_ICMP: > - pkt_hdr->input_flags.icmp = 1; > - pkt_hdr->input_flags.l4 = 1; > - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; > - break; > - default: > - /* 0 or unhandled IP protocols, don't set L4 flag+offset */ > - if (pkt_hdr->input_flags.ipv6) { > - /* IPv6 next_hdr is not L4, mark as IP-option > instead */ > - pkt_hdr->input_flags.ipopt = 1; > - } > - break; > - } > +int odp_packet_inflag_eth(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->input_flags.eth; > } > > -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, > - odph_ipv4hdr_t *ipv4, size_t *offset_out) > +void odp_packet_set_inflag_eth(odp_packet_t pkt, int val) > { > - uint8_t ihl; > - uint16_t frag_offset; > + odp_packet_hdr(pkt)->input_flags.eth = val; > +} > > - ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); > - if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN)) { > - pkt_hdr->error_flags.ip_err = 1; > - return 0; > - } > +int odp_packet_inflag_jumbo(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->input_flags.jumbo; > +} > > - if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) { > - pkt_hdr->input_flags.ipopt = 1; > - return 0; > - } > +void odp_packet_set_inflag_jumbo(odp_packet_t pkt, int val) > +{ > + odp_packet_hdr(pkt)->input_flags.jumbo = val; > +} > > - /* A packet is a fragment if: > - * "more fragments" flag is set (all fragments except the last) > - * OR > - * "fragment offset" field is nonzero (all fragments except the > first) > - */ > - frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); > - if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) { > - pkt_hdr->input_flags.ipfrag = 1; > - return 0; > - } > +int odp_packet_inflag_vlan(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->input_flags.vlan; > +} > > - if (ipv4->proto == ODPH_IPPROTO_ESP || > - ipv4->proto == ODPH_IPPROTO_AH) { > - pkt_hdr->input_flags.ipsec = 1; > - return 0; > - } > +void odp_packet_set_inflag_vlan(odp_packet_t pkt, int val) > +{ > + odp_packet_hdr(pkt)->input_flags.vlan = val; > +} > > - /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after > return */ > +int odp_packet_inflag_vlan_qinq(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->input_flags.vlan_qinq; > +} > > - *offset_out = sizeof(uint32_t) * ihl; > - return ipv4->proto; > +void odp_packet_set_inflag_vlan_qinq(odp_packet_t pkt, int val) > +{ > + odp_packet_hdr(pkt)->input_flags.vlan_qinq = val; > } > > -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, > - odph_ipv6hdr_t *ipv6, size_t *offset_out) > +int odp_packet_inflag_snap(odp_packet_t pkt) > { > - if (ipv6->next_hdr == ODPH_IPPROTO_ESP || > - ipv6->next_hdr == ODPH_IPPROTO_AH) { > - pkt_hdr->input_flags.ipopt = 1; > - pkt_hdr->input_flags.ipsec = 1; > - return 0; > - } > + return odp_packet_hdr(pkt)->input_flags.snap; > +} > > - if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { > - pkt_hdr->input_flags.ipopt = 1; > - pkt_hdr->input_flags.ipfrag = 1; > - return 0; > - } > +void odp_packet_set_inflag_snap(odp_packet_t pkt, int val) > +{ > + odp_packet_hdr(pkt)->input_flags.snap = val; > +} > > - /* Don't step through more extensions */ > - *offset_out = ODPH_IPV6HDR_LEN; > - return ipv6->next_hdr; > +int odp_packet_inflag_arp(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->input_flags.arp; > } > > -void odp_packet_print(odp_packet_t pkt) > +void odp_packet_set_inflag_arp(odp_packet_t pkt, int val) > { > - int max_len = 512; > - char str[max_len]; > - int len = 0; > - int n = max_len-1; > - odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); > + odp_packet_hdr(pkt)->input_flags.arp = val; > +} > > - len += snprintf(&str[len], n-len, "Packet "); > - len += odp_buffer_snprint(&str[len], n-len, (odp_buffer_t) pkt); > - len += snprintf(&str[len], n-len, > - " input_flags 0x%x\n", hdr->input_flags.all); > - len += snprintf(&str[len], n-len, > - " error_flags 0x%x\n", hdr->error_flags.all); > - len += snprintf(&str[len], n-len, > - " output_flags 0x%x\n", hdr->output_flags.all); > - len += snprintf(&str[len], n-len, > - " frame_offset %u\n", hdr->frame_offset); > - len += snprintf(&str[len], n-len, > - " l2_offset %u\n", hdr->l2_offset); > - len += snprintf(&str[len], n-len, > - " l3_offset %u\n", hdr->l3_offset); > - len += snprintf(&str[len], n-len, > - " l4_offset %u\n", hdr->l4_offset); > - len += snprintf(&str[len], n-len, > - " frame_len %u\n", hdr->frame_len); > - len += snprintf(&str[len], n-len, > - " input %u\n", hdr->input); > - str[len] = '\0'; > +int odp_packet_inflag_ipv4(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->input_flags.ipv4; > +} > > - printf("\n%s\n", str); > +void odp_packet_set_inflag_ipv4(odp_packet_t pkt, int val) > +{ > + odp_packet_hdr(pkt)->input_flags.ipv4 = val; > } > > -int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src) > +int odp_packet_inflag_ipv6(odp_packet_t pkt) > { > - odp_packet_hdr_t *const pkt_hdr_dst = odp_packet_hdr(pkt_dst); > - odp_packet_hdr_t *const pkt_hdr_src = odp_packet_hdr(pkt_src); > - const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, > buf_hdr); > - uint8_t *start_src; > - uint8_t *start_dst; > - size_t len; > + return odp_packet_hdr(pkt)->input_flags.ipv6; > +} > > - if (pkt_dst == ODP_PACKET_INVALID || pkt_src == ODP_PACKET_INVALID) > - return -1; > +void odp_packet_set_inflag_ipv6(odp_packet_t pkt, int val) > +{ > + odp_packet_hdr(pkt)->input_flags.ipv6 = val; > +} > > - if (pkt_hdr_dst->buf_hdr.size < > - pkt_hdr_src->frame_len + pkt_hdr_src->frame_offset) > - return -1; > +int odp_packet_inflag_ipfrag(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->input_flags.ipfrag; > +} > + > +void odp_packet_set_inflag_ipfrag(odp_packet_t pkt, int val) > +{ > + odp_packet_hdr(pkt)->input_flags.ipfrag = val; > +} > + > +int odp_packet_inflag_ipopt(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->input_flags.ipopt; > +} > > - /* Copy packet header */ > - start_dst = (uint8_t *)pkt_hdr_dst + start_offset; > - start_src = (uint8_t *)pkt_hdr_src + start_offset; > - len = ODP_OFFSETOF(odp_packet_hdr_t, buf_data) - start_offset; > - memcpy(start_dst, start_src, len); > +void odp_packet_set_inflag_ipopt(odp_packet_t pkt, int val) > +{ > + odp_packet_hdr(pkt)->input_flags.ipopt = val; > +} > > - /* Copy frame payload */ > - start_dst = (uint8_t *)odp_packet_data(pkt_dst); > - start_src = (uint8_t *)odp_packet_data(pkt_src); > - len = pkt_hdr_src->frame_len; > - memcpy(start_dst, start_src, len); > +int odp_packet_inflag_ipsec(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->input_flags.ipsec; > +} > > - /* Copy useful things from the buffer header */ > - pkt_hdr_dst->buf_hdr.cur_offset = pkt_hdr_src->buf_hdr.cur_offset; > +void odp_packet_set_inflag_ipsec(odp_packet_t pkt, int val) > +{ > + odp_packet_hdr(pkt)->input_flags.ipsec = val; > +} > > - /* Create a copy of the scatter list */ > - odp_buffer_copy_scatter(odp_packet_to_buffer(pkt_dst), > - odp_packet_to_buffer(pkt_src)); > +int odp_packet_inflag_udp(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->input_flags.udp; > +} > > - return 0; > +void odp_packet_set_inflag_udp(odp_packet_t pkt, int val) > +{ > + odp_packet_hdr(pkt)->input_flags.udp = val; > } > > -void odp_packet_set_ctx(odp_packet_t pkt, const void *ctx) > +int odp_packet_inflag_tcp(odp_packet_t pkt) > { > - odp_packet_hdr(pkt)->user_ctx = (intptr_t)ctx; > + return odp_packet_hdr(pkt)->input_flags.tcp; > } > > -void *odp_packet_get_ctx(odp_packet_t pkt) > +void odp_packet_set_inflag_tcp(odp_packet_t pkt, int val) > { > - return (void *)(intptr_t)odp_packet_hdr(pkt)->user_ctx; > + odp_packet_hdr(pkt)->input_flags.tcp = val; > +} > + > +int odp_packet_inflag_tcpopt(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->input_flags.tcpopt; > +} > + > +void odp_packet_set_inflag_tcpopt(odp_packet_t pkt, int val) > +{ > + odp_packet_hdr(pkt)->input_flags.tcpopt = val; > +} > + > +int odp_packet_inflag_icmp(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->input_flags.icmp; > +} > + > +void odp_packet_set_inflag_icmp(odp_packet_t pkt, int val) > +{ > + odp_packet_hdr(pkt)->input_flags.icmp = val; > +} > + > +int odp_packet_is_valid(odp_packet_t pkt) > +{ > + odp_buffer_hdr_t *buf = validate_buf((odp_buffer_t)pkt); > + > + if (buf == NULL) > + return 0; > + else > + return buf->type == ODP_BUFFER_TYPE_PACKET; > +} > + > +int odp_packet_is_segmented(odp_packet_t pkt) > +{ > + return (odp_packet_hdr(pkt)->buf_hdr.segcount > 1); > +} > + > +int odp_packet_segment_count(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->buf_hdr.segcount; > +} > + > +size_t odp_packet_headroom(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->headroom; > +} > + > +size_t odp_packet_tailroom(odp_packet_t pkt) > +{ > + return odp_packet_hdr(pkt)->tailroom; > +} > + > +int odp_packet_push_head(odp_packet_t pkt, size_t len) > +{ > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + > + if (len > pkt_hdr->headroom) > + return -1; > + > + push_head(pkt_hdr, len); > + return 0; > +} > + > +void *odp_packet_push_head_and_map(odp_packet_t pkt, size_t len, size_t > *seglen) > +{ > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + > + if (len > pkt_hdr->headroom) > + return NULL; > + > + push_head(pkt_hdr, len); > + > + return buffer_map(&pkt_hdr->buf_hdr, 0, seglen, > pkt_hdr->frame_len); > +} > + > +int odp_packet_pull_head(odp_packet_t pkt, size_t len) > +{ > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + > + if (len > pkt_hdr->frame_len) > + return -1; > + > + pull_head(pkt_hdr, len); > + return 0; > +} > + > +void *odp_packet_pull_head_and_map(odp_packet_t pkt, size_t len, size_t > *seglen) > +{ > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + > + if (len > pkt_hdr->frame_len) > + return NULL; > + > + pull_head(pkt_hdr, len); > + return buffer_map(&pkt_hdr->buf_hdr, 0, seglen, > pkt_hdr->frame_len); > +} > + > +int odp_packet_push_tail(odp_packet_t pkt, size_t len) > +{ > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + > + if (len > pkt_hdr->tailroom) > + return -1; > + > + push_tail(pkt_hdr, len); > + return 0; > +} > + > +void *odp_packet_push_tail_and_map(odp_packet_t pkt, size_t len, size_t > *seglen) > +{ > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + size_t origin = pkt_hdr->frame_len; > + > + if (len > pkt_hdr->tailroom) > + return NULL; > + > + push_tail(pkt_hdr, len); > + > + return buffer_map(&pkt_hdr->buf_hdr, origin, seglen, > + pkt_hdr->frame_len); > +} > + > +int odp_packet_pull_tail(odp_packet_t pkt, size_t len) > +{ > + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > + > + if (len > pkt_hdr->frame_len) > + return -1; > + > + pull_tail(pkt_hdr, len); > + return 0; > +} > + > +void odp_packet_print(odp_packet_t pkt) > +{ > + int max_len = 512; > + char str[max_len]; > + int len = 0; > + int n = max_len-1; > + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); > + > + len += snprintf(&str[len], n-len, "Packet "); > + len += odp_buffer_snprint(&str[len], n-len, (odp_buffer_t) pkt); > + len += snprintf(&str[len], n-len, > + " input_flags 0x%x\n", hdr->input_flags.all); > + len += snprintf(&str[len], n-len, > + " error_flags 0x%x\n", hdr->error_flags.all); > + len += snprintf(&str[len], n-len, > + " output_flags 0x%x\n", hdr->output_flags.all); > + len += snprintf(&str[len], n-len, > + " l2_offset %u\n", hdr->l2_offset); > + len += snprintf(&str[len], n-len, > + " l3_offset %u\n", hdr->l3_offset); > + len += snprintf(&str[len], n-len, > + " l3_len %u\n", hdr->l3_len); > + len += snprintf(&str[len], n-len, > + " l3_protocol 0x%x\n", hdr->l3_protocol); > + len += snprintf(&str[len], n-len, > + " l4_offset %u\n", hdr->l4_offset); > + len += snprintf(&str[len], n-len, > + " l4_len %u\n", hdr->l4_len); > + len += snprintf(&str[len], n-len, > + " l4_protocol %u\n", hdr->l4_protocol); > + len += snprintf(&str[len], n-len, > + " payload_offset %u\n", hdr->payload_offset); > + len += snprintf(&str[len], n-len, > + " frame_len %u\n", hdr->frame_len); > + str[len] = '\0'; > + > + printf("\n%s\n", str); > +} > + > +int odp_packet_copy_to_packet(odp_packet_t dstpkt, size_t dstoffset, > + odp_packet_t srcpkt, size_t srcoffset, > + size_t len) > +{ > + void *dstmap; > + void *srcmap; > + size_t cpylen, minseg, dstseglen, srcseglen; > + > + while (len > 0) { > + dstmap = odp_packet_offset_map(dstpkt, dstoffset, > &dstseglen); > + srcmap = odp_packet_offset_map(srcpkt, srcoffset, > &srcseglen); > + if (dstmap == NULL || srcmap == NULL) > + return -1; > + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; > + cpylen = len > minseg ? minseg : len; > + memcpy(dstmap, srcmap, cpylen); > + srcoffset += cpylen; > + dstoffset += cpylen; > + len -= cpylen; > + } > + > + return 0; > +} > + > +int odp_packet_copy_to_memory(void *dstmem, odp_packet_t srcpkt, > + size_t srcoffset, size_t dstlen) > +{ > + void *mapaddr; > + size_t seglen, cpylen; > + uint8_t *dstaddr = (uint8_t *)dstmem; > + > + while (dstlen > 0) { > + mapaddr = odp_packet_offset_map(srcpkt, srcoffset, > &seglen); > + if (mapaddr == NULL) > + return -1; > + cpylen = dstlen > seglen ? seglen : dstlen; > + memcpy(dstaddr, mapaddr, cpylen); > + srcoffset += cpylen; > + dstaddr += cpylen; > + dstlen -= cpylen; > + } > + > + return 0; > +} > + > +int odp_packet_copy_from_memory(odp_packet_t dstpkt, size_t dstoffset, > + void *srcmem, size_t srclen) > +{ > + void *mapaddr; > + size_t seglen, cpylen; > + uint8_t *srcaddr = (uint8_t *)srcmem; > + > + while (srclen > 0) { > + mapaddr = odp_packet_offset_map(dstpkt, dstoffset, > &seglen); > + if (mapaddr == NULL) > + return -1; > + cpylen = srclen > seglen ? seglen : srclen; > + memcpy(mapaddr, srcaddr, cpylen); > + dstoffset += cpylen; > + srcaddr += cpylen; > + srclen -= cpylen; > + } > + > + return 0; > +} > + > +odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool) > +{ > + size_t pktlen = odp_packet_len(pkt); > + const size_t meta_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, > buf_hdr); > + odp_packet_t newpkt = odp_packet_alloc_len(pool, pktlen); > + odp_packet_hdr_t *newhdr, *srchdr; > + uint8_t *newstart, *srcstart; > + > + if (newpkt != ODP_PACKET_INVALID) { > + /* Must copy meta data first, followed by packet data */ > + srchdr = odp_packet_hdr(pkt); > + newhdr = odp_packet_hdr(newpkt); > + newstart = (uint8_t *)newhdr + meta_offset; > + srcstart = (uint8_t *)srchdr + meta_offset; > + > + memcpy(newstart, srcstart, > + sizeof(odp_packet_hdr_t) - meta_offset); > + > + if (odp_packet_copy_to_packet(newpkt, 0, pkt, 0, pktlen) > != 0) { > + odp_packet_free(newpkt); > + newpkt = ODP_PACKET_INVALID; > + } > + } > + > + return newpkt; > +} > + > +odp_packet_t odp_packet_clone(odp_packet_t pkt) > +{ > + return odp_packet_copy(pkt, odp_packet_hdr(pkt)->buf_hdr.pool_hdl); > +} > + > +odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool) > +{ > + if (odp_pool_to_entry(pool)->s.params.buf_type != > + ODP_BUFFER_TYPE_PACKET) > + return ODP_PACKET_INVALID; > + > + return (odp_packet_t)buffer_alloc(pool, 0); > +} > + > +odp_packet_t odp_packet_alloc_len(odp_buffer_pool_t pool, size_t len) > +{ > + if (odp_pool_to_entry(pool)->s.params.buf_type != > + ODP_BUFFER_TYPE_PACKET) > + return ODP_PACKET_INVALID; > + > + return (odp_packet_t)buffer_alloc(pool, len); > +} > + > +void odp_packet_free(odp_packet_t pkt) > +{ > + odp_buffer_free((odp_buffer_t)pkt); > +} > + > +odp_packet_t odp_packet_split(odp_packet_t pkt, size_t offset, > + size_t hr, size_t tr) > +{ > + size_t len = odp_packet_len(pkt); > + odp_buffer_pool_t pool = odp_packet_pool(pkt); > + size_t pool_hr = odp_buffer_pool_headroom(pool); > + size_t pkt_tr = odp_packet_tailroom(pkt); > + odp_packet_t splitpkt; > + size_t splitlen = len - offset; > + > + if (offset >= len) > + return ODP_PACKET_INVALID; > + > + /* Erratum: We don't handle this rare corner case */ > + if (tr > pkt_tr + splitlen) > + return ODP_PACKET_INVALID; > + > + if (hr > pool_hr) > + splitlen += (hr - pool_hr); > + > + splitpkt = odp_packet_alloc_len(pool, splitlen); > + > + if (splitpkt != ODP_PACKET_INVALID) { > + if (hr > pool_hr) { > + odp_packet_pull_head(splitpkt, hr - pool_hr); > + splitlen -= (hr - pool_hr); > + } > + odp_packet_copy_to_packet(splitpkt, 0, > + pkt, offset, splitlen); > + odp_packet_pull_tail(pkt, splitlen); > + } > + > + return splitpkt; > +} > + > +odp_packet_t odp_packet_join(odp_packet_t pkt1, odp_packet_t pkt2) > +{ > + size_t len1 = odp_packet_len(pkt1); > + size_t len2 = odp_packet_len(pkt2); > + odp_packet_t joinpkt; > + void *udata1, *udata_join; > + size_t udata_size1, udata_size_join; > + > + /* Optimize if pkt1 is big enough to hold pkt2 */ > + if (odp_packet_push_tail(pkt1, len2) == 0) { > + odp_packet_copy_to_packet(pkt1, len1, > + pkt2, 0, len2); > + odp_packet_free(pkt2); > + return pkt1; > + } > + > + /* Otherwise join into a new packet */ > + joinpkt = odp_packet_alloc_len(odp_packet_pool(pkt1), > + len1 + len2); > + > + if (joinpkt != ODP_PACKET_INVALID) { > + odp_packet_copy_to_packet(joinpkt, 0, pkt1, 0, len1); > + odp_packet_copy_to_packet(joinpkt, len1, pkt2, 0, len2); > + > + /* Copy user metadata if present */ > + udata1 = odp_packet_udata(pkt1, &udata_size1); > + udata_join = odp_packet_udata(joinpkt, &udata_size_join); > + > + if (udata1 != NULL && udata_join != NULL) > + memcpy(udata_join, udata1, > + udata_size_join > udata_size1 ? > + udata_size1 : udata_size_join); > + > + odp_packet_free(pkt1); > + odp_packet_free(pkt2); > + } > + > + return joinpkt; > +} > + > +uint32_t odp_packet_refcount(odp_packet_t pkt) > +{ > + return odp_buffer_refcount(&odp_packet_hdr(pkt)->buf_hdr); > +} > + > +uint32_t odp_packet_incr_refcount(odp_packet_t pkt, uint32_t val) > +{ > + return odp_buffer_incr_refcount(&odp_packet_hdr(pkt)->buf_hdr, > val); > +} > + > +uint32_t odp_packet_decr_refcount(odp_packet_t pkt, uint32_t val) > +{ > + return odp_buffer_decr_refcount(&odp_packet_hdr(pkt)->buf_hdr, > val); > +} > + > +/** > + * Parser helper function for IPv4 > + */ > +static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, > + uint8_t **parseptr, size_t *offset) > +{ > + odph_ipv4hdr_t *ipv4 = (odph_ipv4hdr_t *)*parseptr; > + uint8_t ver = ODPH_IPV4HDR_VER(ipv4->ver_ihl); > + uint8_t ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); > + uint16_t frag_offset; > + > + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv4->tot_len); > + > + if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN) || > + odp_unlikely(ver != 4) || > + (pkt_hdr->l3_len > pkt_hdr->frame_len - *offset)) { > + pkt_hdr->error_flags.ip_err = 1; > + return 0; > + } > + > + *offset += ihl * 4; > + *parseptr += ihl * 4; > + > + if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) > + pkt_hdr->input_flags.ipopt = 1; > + > + /* A packet is a fragment if: > + * "more fragments" flag is set (all fragments except the last) > + * OR > + * "fragment offset" field is nonzero (all fragments except the > first) > + */ > + frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); > + if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) > + pkt_hdr->input_flags.ipfrag = 1; > + > + if (ipv4->proto == ODPH_IPPROTO_ESP || > + ipv4->proto == ODPH_IPPROTO_AH) { > + pkt_hdr->input_flags.ipsec = 1; > + return 0; > + } > + > + /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after > return */ > + > + *offset = sizeof(uint32_t) * ihl; > + return ipv4->proto; > +} > + > +/** > + * Parser helper function for IPv6 > + */ > +static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, > + uint8_t **parseptr, size_t *offset) > +{ > + odph_ipv6hdr_t *ipv6 = (odph_ipv6hdr_t *)*parseptr; > + odph_ipv6hdr_ext_t *ipv6ext; > + > + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv6->payload_len); > + > + /* Basic sanity checks on IPv6 header */ > + if (ipv6->ver != 6 || > + pkt_hdr->l3_len > pkt_hdr->frame_len - *offset) { > + pkt_hdr->error_flags.ip_err = 1; > + return 0; > + } > + > + /* Skip past IPv6 header */ > + *offset += sizeof(odph_ipv6hdr_t); > + *parseptr += sizeof(odph_ipv6hdr_t); > + > + > + /* Skip past any IPv6 extension headers */ > + if (ipv6->next_hdr == ODPH_IPPROTO_HOPOPTS || > + ipv6->next_hdr == ODPH_IPPROTO_ROUTE) { > + pkt_hdr->input_flags.ipopt = 1; > + > + do { > + ipv6ext = (odph_ipv6hdr_ext_t *)*parseptr; > + uint16_t extlen = 8 + ipv6ext->ext_len * 8; > + > + *offset += extlen; > + *parseptr += extlen; > + } while ((ipv6ext->next_hdr == ODPH_IPPROTO_HOPOPTS || > + ipv6ext->next_hdr == ODPH_IPPROTO_ROUTE) && > + *offset < pkt_hdr->frame_len); > + > + if (*offset >= pkt_hdr->l3_offset + ipv6->payload_len) { > + pkt_hdr->error_flags.ip_err = 1; > + return 0; > + } > + > + if (ipv6ext->next_hdr == ODPH_IPPROTO_FRAG) > + pkt_hdr->input_flags.ipfrag = 1; > + > + return ipv6ext->next_hdr; > + } > + > + if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { > + pkt_hdr->input_flags.ipopt = 1; > + pkt_hdr->input_flags.ipfrag = 1; > + } > + > + return ipv6->next_hdr; > +} > + > +/** > + * Parser helper function for TCP > + */ > +static inline void parse_tcp(odp_packet_hdr_t *pkt_hdr, > + uint8_t **parseptr, size_t *offset) > +{ > + odph_tcphdr_t *tcp = (odph_tcphdr_t *)*parseptr; > + > + if (tcp->hl < sizeof(odph_tcphdr_t)/sizeof(uint32_t)) > + pkt_hdr->error_flags.tcp_err = 1; > + else if ((uint32_t)tcp->hl * 4 > sizeof(odph_tcphdr_t)) > + pkt_hdr->input_flags.tcpopt = 1; > + > + pkt_hdr->l4_len = pkt_hdr->l3_len + > + pkt_hdr->l3_offset - pkt_hdr->l4_offset; > + > + *offset += sizeof(odph_tcphdr_t); > + *parseptr += sizeof(odph_tcphdr_t); > +} > + > +/** > + * Parser helper function for UDP > + */ > +static inline void parse_udp(odp_packet_hdr_t *pkt_hdr, > + uint8_t **parseptr, size_t *offset) > +{ > + odph_udphdr_t *udp = (odph_udphdr_t *)*parseptr; > + uint32_t udplen = odp_be_to_cpu_16(udp->length); > + > + if (udplen < sizeof(odph_udphdr_t) || > + udplen > (pkt_hdr->l3_len + > + pkt_hdr->l3_offset - pkt_hdr->l4_offset)) { > + pkt_hdr->error_flags.udp_err = 1; > + } > + > + pkt_hdr->l4_len = udplen; > + > + *offset += sizeof(odph_udphdr_t); > + *parseptr += sizeof(odph_udphdr_t); > +} > + > +/** > + * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP > + * > + * Internal function: caller is resposible for passing only valid packet > handles > + * , lengths and offsets (usually done&called in packet input). > + * > + * @param pkt Packet handle > + */ > +int odp_packet_parse(odp_packet_t pkt) > +{ > + odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); > + odph_ethhdr_t *eth; > + odph_vlanhdr_t *vlan; > + uint16_t ethtype; > + uint8_t *parseptr; > + size_t offset, seglen; > + uint8_t ip_proto = 0; > + > + /* Reset parser metadata for new parse */ > + pkt_hdr->error_flags.all = 0; > + pkt_hdr->input_flags.all = 0; > + pkt_hdr->output_flags.all = 0; > + pkt_hdr->l2_offset = 0; > + pkt_hdr->l3_offset = 0; > + pkt_hdr->l4_offset = 0; > + pkt_hdr->payload_offset = 0; > + pkt_hdr->vlan_s_tag = 0; > + pkt_hdr->vlan_c_tag = 0; > + pkt_hdr->l3_protocol = 0; > + pkt_hdr->l4_protocol = 0; > + > + /* We only support Ethernet for now */ > + pkt_hdr->input_flags.eth = 1; > + > + if (odp_unlikely(pkt_hdr->frame_len < ODPH_ETH_LEN_MIN)) { > + pkt_hdr->error_flags.frame_len = 1; > + goto parse_exit; > + } else if (pkt_hdr->frame_len > ODPH_ETH_LEN_MAX) { > + pkt_hdr->input_flags.jumbo = 1; > + } > + > + /* Assume valid L2 header, no CRC/FCS check in SW */ > + pkt_hdr->input_flags.l2 = 1; > + > + eth = (odph_ethhdr_t *)odp_packet_map(pkt, &seglen); > + offset = sizeof(odph_ethhdr_t); > + parseptr = (uint8_t *)ð->type; > + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); > + > + /* Parse the VLAN header(s), if present */ > + if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { > + pkt_hdr->input_flags.vlan_qinq = 1; > + pkt_hdr->input_flags.vlan = 1; > + vlan = (odph_vlanhdr_t *)(void *)parseptr; > + pkt_hdr->vlan_s_tag = ((ethtype << 16) | > + odp_be_to_cpu_16(vlan->tci)); > + offset += sizeof(odph_vlanhdr_t); > + parseptr += sizeof(odph_vlanhdr_t); > + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void > *)parseptr)); > + } > + > + if (ethtype == ODPH_ETHTYPE_VLAN) { > + pkt_hdr->input_flags.vlan = 1; > + vlan = (odph_vlanhdr_t *)(void *)parseptr; > + pkt_hdr->vlan_c_tag = ((ethtype << 16) | > + odp_be_to_cpu_16(vlan->tci)); > + offset += sizeof(odph_vlanhdr_t); > + parseptr += sizeof(odph_vlanhdr_t); > + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void > *)parseptr)); > + } > + > + /* Check for SNAP vs. DIX */ > + if (ethtype < ODPH_ETH_LEN_MAX) { > + pkt_hdr->input_flags.snap = 1; > + if (ethtype > pkt_hdr->frame_len - offset) { > + pkt_hdr->error_flags.snap_len = 1; > + goto parse_exit; > + } > + offset += 8; > + parseptr += 8; > + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void > *)parseptr)); > + } > + > + /* Consume Ethertype for Layer 3 parse */ > + parseptr += 2; > + > + /* Set l3_offset+flag only for known ethtypes */ > + pkt_hdr->input_flags.l3 = 1; > + pkt_hdr->l3_offset = offset; > + pkt_hdr->l3_protocol = ethtype; > + > + /* Parse Layer 3 headers */ > + switch (ethtype) { > + case ODPH_ETHTYPE_IPV4: > + pkt_hdr->input_flags.ipv4 = 1; > + ip_proto = parse_ipv4(pkt_hdr, &parseptr, &offset); > + break; > + > + case ODPH_ETHTYPE_IPV6: > + pkt_hdr->input_flags.ipv6 = 1; > + ip_proto = parse_ipv6(pkt_hdr, &parseptr, &offset); > + break; > + > + case ODPH_ETHTYPE_ARP: > + pkt_hdr->input_flags.arp = 1; > + ip_proto = 255; /* Reserved invalid by IANA */ > + break; > + > + default: > + pkt_hdr->input_flags.l3 = 0; > + ip_proto = 255; /* Reserved invalid by IANA */ > + } > + > + /* Set l4_offset+flag only for known ip_proto */ > + pkt_hdr->input_flags.l4 = 1; > + pkt_hdr->l4_offset = offset; > + pkt_hdr->l4_protocol = ip_proto; > + > + /* Parse Layer 4 headers */ > + switch (ip_proto) { > + case ODPH_IPPROTO_ICMP: > + pkt_hdr->input_flags.icmp = 1; > + break; > + > + case ODPH_IPPROTO_TCP: > + pkt_hdr->input_flags.tcp = 1; > + parse_tcp(pkt_hdr, &parseptr, &offset); > + break; > + > + case ODPH_IPPROTO_UDP: > + pkt_hdr->input_flags.udp = 1; > + parse_udp(pkt_hdr, &parseptr, &offset); > + break; > + > + case ODPH_IPPROTO_AH: > + case ODPH_IPPROTO_ESP: > + pkt_hdr->input_flags.ipsec = 1; > + break; > + > + default: > + pkt_hdr->input_flags.l4 = 0; > + break; > + } > + > + /* > + * Anything beyond what we parse here is considered payload. > + * Note: Payload is really only relevant for TCP and UDP. For > + * all other protocols, the payload offset will point to the > + * final header (ARP, ICMP, AH, ESP, or IP Fragment. > + */ > + pkt_hdr->payload_offset = offset; > + > +parse_exit: > + return pkt_hdr->error_flags.all != 0; > } > diff --git a/platform/linux-generic/odp_packet_flags.c > b/platform/linux-generic/odp_packet_flags.c > deleted file mode 100644 > index 06fdeed..0000000 > --- a/platform/linux-generic/odp_packet_flags.c > +++ /dev/null > @@ -1,202 +0,0 @@ > -/* Copyright (c) 2014, Linaro Limited > - * All rights reserved. > - * > - * SPDX-License-Identifier: BSD-3-Clause > - */ > - > -#include <odp_packet_flags.h> > -#include <odp_packet_internal.h> > - > - > -int odp_packet_error(odp_packet_t pkt) > -{ > - return (odp_packet_hdr(pkt)->error_flags.all != 0); > -} > - > -/* Get Error Flags */ > - > -int odp_packet_errflag_frame_len(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->error_flags.frame_len; > -} > - > -/* Get Input Flags */ > - > -int odp_packet_inflag_l2(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.l2; > -} > - > -int odp_packet_inflag_l3(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.l3; > -} > - > -int odp_packet_inflag_l4(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.l4; > -} > - > -int odp_packet_inflag_eth(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.eth; > -} > - > -int odp_packet_inflag_jumbo(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.jumbo; > -} > - > -int odp_packet_inflag_vlan(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.vlan; > -} > - > -int odp_packet_inflag_vlan_qinq(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.vlan_qinq; > -} > - > -int odp_packet_inflag_arp(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.arp; > -} > - > -int odp_packet_inflag_ipv4(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.ipv4; > -} > - > -int odp_packet_inflag_ipv6(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.ipv6; > -} > - > -int odp_packet_inflag_ipfrag(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.ipfrag; > -} > - > -int odp_packet_inflag_ipopt(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.ipopt; > -} > - > -int odp_packet_inflag_ipsec(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.ipsec; > -} > - > -int odp_packet_inflag_udp(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.udp; > -} > - > -int odp_packet_inflag_tcp(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.tcp; > -} > - > -int odp_packet_inflag_sctp(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.sctp; > -} > - > -int odp_packet_inflag_icmp(odp_packet_t pkt) > -{ > - return odp_packet_hdr(pkt)->input_flags.icmp; > -} > - > -/* Set Output Flags */ > - > -void odp_packet_outflag_l4_chksum(odp_packet_t pkt) > -{ > - odp_packet_hdr(pkt)->output_flags.l4_chksum = 1; > -} > - > -/* Set Input Flags */ > - > -void odp_packet_set_inflag_l2(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.l2 = val; > -} > - > -void odp_packet_set_inflag_l3(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.l3 = val; > -} > - > -void odp_packet_set_inflag_l4(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.l4 = val; > -} > - > -void odp_packet_set_inflag_eth(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.eth = val; > -} > - > -void odp_packet_set_inflag_jumbo(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.jumbo = val; > -} > - > -void odp_packet_set_inflag_vlan(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.vlan = val; > -} > - > -void odp_packet_set_inflag_vlan_qinq(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.vlan_qinq = val; > -} > - > -void odp_packet_set_inflag_arp(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.arp = val; > -} > - > -void odp_packet_set_inflag_ipv4(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.ipv4 = val; > -} > - > -void odp_packet_set_inflag_ipv6(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.ipv6 = val; > -} > - > -void odp_packet_set_inflag_ipfrag(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.ipfrag = val; > -} > - > -void odp_packet_set_inflag_ipopt(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.ipopt = val; > -} > - > -void odp_packet_set_inflag_ipsec(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.ipsec = val; > -} > - > -void odp_packet_set_inflag_udp(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.udp = val; > -} > - > -void odp_packet_set_inflag_tcp(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.tcp = val; > -} > - > -void odp_packet_set_inflag_sctp(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.sctp = val; > -} > - > -void odp_packet_set_inflag_icmp(odp_packet_t pkt, int val) > -{ > - odp_packet_hdr(pkt)->input_flags.icmp = val; > -} > -- > 1.8.3.2 > > > _______________________________________________ > lng-odp mailing list > lng-odp@lists.linaro.org > http://lists.linaro.org/mailman/listinfo/lng-odp >
The API allows the init_params to be NULL if user meta data is not required. I just added the corresponding support to the params to have the same behavior. I have no problem failing the call if that parameter is missing. Bill On Tue, Nov 11, 2014 at 10:34 AM, Bala Manoharan <bala.manoharan@linaro.org> wrote: > > > On 11 November 2014 09:33, Bill Fischofer <bill.fischofer@linaro.org> > wrote: > >> Signed-off-by: Bill Fischofer <bill.fischofer@linaro.org> >> --- >> platform/linux-generic/odp_buffer.c | 263 ++++++- >> platform/linux-generic/odp_buffer_pool.c | 661 +++++++--------- >> platform/linux-generic/odp_packet.c | 1200 >> +++++++++++++++++++++++------ >> platform/linux-generic/odp_packet_flags.c | 202 ----- >> 4 files changed, 1455 insertions(+), 871 deletions(-) >> delete mode 100644 platform/linux-generic/odp_packet_flags.c >> >> diff --git a/platform/linux-generic/odp_buffer.c >> b/platform/linux-generic/odp_buffer.c >> index e54e0e7..e47c722 100644 >> --- a/platform/linux-generic/odp_buffer.c >> +++ b/platform/linux-generic/odp_buffer.c >> @@ -5,46 +5,259 @@ >> */ >> >> #include <odp_buffer.h> >> -#include <odp_buffer_internal.h> >> #include <odp_buffer_pool_internal.h> >> +#include <odp_buffer_internal.h> >> +#include <odp_buffer_inlines.h> >> >> #include <string.h> >> #include <stdio.h> >> >> +void *odp_buffer_offset_map(odp_buffer_t buf, size_t offset, size_t >> *seglen) >> +{ >> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >> >> -void *odp_buffer_addr(odp_buffer_t buf) >> + if (offset > buf_hdr->size) >> + return NULL; >> + >> + return buffer_map(buf_hdr, offset, seglen, buf_hdr->size); >> +} >> + >> +void odp_buffer_offset_unmap(odp_buffer_t buf ODP_UNUSED, >> + size_t offset ODP_UNUSED) >> { >> - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >> +} >> >> - return hdr->addr; >> +size_t odp_buffer_segment_count(odp_buffer_t buf) >> +{ >> + return odp_buf_to_hdr(buf)->segcount; >> } >> >> +int odp_buffer_is_segmented(odp_buffer_t buf) >> +{ >> + return odp_buf_to_hdr(buf)->segcount > 1; >> +} >> >> -size_t odp_buffer_size(odp_buffer_t buf) >> +odp_buffer_segment_t odp_buffer_segment_by_index(odp_buffer_t buf, >> + size_t ndx) >> { >> - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >> + return buffer_segment(odp_buf_to_hdr(buf), ndx); >> +} >> >> - return hdr->size; >> +odp_buffer_segment_t odp_buffer_segment_next(odp_buffer_t buf, >> + odp_buffer_segment_t seg) >> +{ >> + return segment_next(odp_buf_to_hdr(buf), seg); >> } >> >> +void *odp_buffer_segment_map(odp_buffer_t buf, odp_buffer_segment_t seg, >> + size_t *seglen) >> +{ >> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >> >> -int odp_buffer_type(odp_buffer_t buf) >> + return segment_map(buf_hdr, seg, seglen, buf_hdr->size); >> +} >> + >> +void odp_buffer_segment_unmap(odp_buffer_segment_t seg ODP_UNUSED) >> { >> - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >> +} >> + >> +odp_buffer_t odp_buffer_split(odp_buffer_t buf, size_t offset) >> +{ >> + size_t size = odp_buffer_size(buf); >> + odp_buffer_pool_t pool = odp_buffer_pool(buf); >> + odp_buffer_t splitbuf; >> + size_t splitsize = size - offset; >> + >> + if (offset >= size) >> + return ODP_BUFFER_INVALID; >> + >> + splitbuf = buffer_alloc(pool, splitsize); >> + >> + if (splitbuf != ODP_BUFFER_INVALID) { >> + buffer_copy_to_buffer(splitbuf, 0, buf, splitsize, >> splitsize); >> + odp_buffer_trim(buf, splitsize); >> + } >> >> - return hdr->type; >> + return splitbuf; >> } >> >> +odp_buffer_t odp_buffer_join(odp_buffer_t buf1, odp_buffer_t buf2) >> +{ >> + size_t size1 = odp_buffer_size(buf1); >> + size_t size2 = odp_buffer_size(buf2); >> + odp_buffer_t joinbuf; >> + void *udata1, *udata_join; >> + size_t udata_size1, udata_size_join; >> >> -int odp_buffer_is_valid(odp_buffer_t buf) >> + joinbuf = buffer_alloc(odp_buffer_pool(buf1), size1 + size2); >> + >> + if (joinbuf != ODP_BUFFER_INVALID) { >> + buffer_copy_to_buffer(joinbuf, 0, buf1, 0, size1); >> + buffer_copy_to_buffer(joinbuf, size1, buf2, 0, size2); >> + >> + /* Copy user metadata if present */ >> + udata1 = odp_buffer_udata(buf1, &udata_size1); >> + udata_join = odp_buffer_udata(joinbuf, &udata_size_join); >> + >> + if (udata1 != NULL && udata_join != NULL) >> + memcpy(udata_join, udata1, >> + udata_size_join > udata_size1 ? >> + udata_size1 : udata_size_join); >> + >> + odp_buffer_free(buf1); >> + odp_buffer_free(buf2); >> + } >> + >> + return joinbuf; >> +} >> + >> +odp_buffer_t odp_buffer_trim(odp_buffer_t buf, size_t bytes) >> { >> - odp_buffer_bits_t handle; >> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >> + >> + if (bytes >= buf_hdr->size) >> + return ODP_BUFFER_INVALID; >> + >> + buf_hdr->size -= bytes; >> + size_t bufsegs = buf_hdr->size / buf_hdr->segsize; >> + >> + if (buf_hdr->size % buf_hdr->segsize > 0) >> + bufsegs++; >> + >> + pool_entry_t *pool = odp_pool_to_entry(buf_hdr->pool_hdl); >> >> - handle.u32 = buf; >> + /* Return excess segments back to block list */ >> + while (bufsegs > buf_hdr->segcount) >> + ret_blk(&pool->s, buf_hdr->addr[--buf_hdr->segcount]); >> >> - return (handle.index != ODP_BUFFER_INVALID_INDEX); >> + return buf; >> } >> >> +odp_buffer_t odp_buffer_extend(odp_buffer_t buf, size_t bytes) >> +{ >> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >> + >> + size_t lastseg = buf_hdr->size % buf_hdr->segsize; >> + >> + if (bytes <= buf_hdr->segsize - lastseg) { >> + buf_hdr->size += bytes; >> + return buf; >> + } >> + >> + pool_entry_t *pool = odp_pool_to_entry(buf_hdr->pool_hdl); >> + int extsize = buf_hdr->size + bytes; >> + >> + /* Extend buffer by adding additional segments to it */ >> + if (extsize > ODP_CONFIG_BUF_MAX_SIZE || >> pool->s.flags.unsegmented) >> + return ODP_BUFFER_INVALID; >> + >> + size_t segcount = buf_hdr->segcount; >> + size_t extsegs = extsize / buf_hdr->segsize; >> + >> + if (extsize % buf_hdr->segsize > 0) >> + extsize++; >> + >> + while (extsegs > buf_hdr->segcount) { >> + void *newblk = get_blk(&pool->s); >> + >> + /* If we fail to get all the blocks we need, back out */ >> + if (newblk == NULL) { >> + while (segcount > buf_hdr->segcount) >> + ret_blk(&pool->s, >> buf_hdr->addr[--segcount]); >> + return ODP_BUFFER_INVALID; >> + } >> + >> + buf_hdr->addr[segcount++] = newblk; >> + } >> + >> + buf_hdr->segcount = segcount; >> + buf_hdr->size = extsize; >> + >> + return buf; >> +} >> + >> +odp_buffer_t odp_buffer_clone(odp_buffer_t buf) >> +{ >> + return odp_buffer_copy(buf, odp_buf_to_hdr(buf)->pool_hdl); >> +} >> + >> +odp_buffer_t odp_buffer_copy(odp_buffer_t buf, odp_buffer_pool_t pool) >> +{ >> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >> + size_t len = buf_hdr->size; >> + odp_buffer_t cpybuf = buffer_alloc(pool, len); >> + >> + if (cpybuf == ODP_BUFFER_INVALID) >> + return ODP_BUFFER_INVALID; >> + >> + if (buffer_copy_to_buffer(cpybuf, 0, buf, 0, len) == 0) >> + return cpybuf; >> + >> + odp_buffer_free(cpybuf); >> + return ODP_BUFFER_INVALID; >> +} >> + >> +int buffer_copy_to_buffer(odp_buffer_t dstbuf, size_t dstoffset, >> + odp_buffer_t srcbuf, size_t srcoffset, >> + size_t len) >> +{ >> + void *dstmap; >> + void *srcmap; >> + size_t cpylen, minseg, dstseglen, srcseglen; >> + >> + while (len > 0) { >> + dstmap = odp_buffer_offset_map(dstbuf, dstoffset, >> &dstseglen); >> + srcmap = odp_buffer_offset_map(srcbuf, srcoffset, >> &srcseglen); >> + if (dstmap == NULL || srcmap == NULL) >> + return -1; >> + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; >> + cpylen = len > minseg ? minseg : len; >> + memcpy(dstmap, srcmap, cpylen); >> + srcoffset += cpylen; >> + dstoffset += cpylen; >> + len -= cpylen; >> + } >> + >> + return 0; >> +} >> + >> +void *odp_buffer_addr(odp_buffer_t buf) >> +{ >> + return odp_buf_to_hdr(buf)->addr[0]; >> +} >> + >> +odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf) >> +{ >> + return odp_buf_to_hdr(buf)->pool_hdl; >> +} >> + >> +size_t odp_buffer_size(odp_buffer_t buf) >> +{ >> + return odp_buf_to_hdr(buf)->size; >> +} >> + >> +odp_buffer_type_e odp_buffer_type(odp_buffer_t buf) >> +{ >> + return odp_buf_to_hdr(buf)->type; >> +} >> + >> +void *odp_buffer_udata(odp_buffer_t buf, size_t *usize) >> +{ >> + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >> + >> + *usize = hdr->udata_size; >> + return hdr->udata_addr; >> +} >> + >> +void *odp_buffer_udata_addr(odp_buffer_t buf) >> +{ >> + return odp_buf_to_hdr(buf)->udata_addr; >> +} >> + >> +int odp_buffer_is_valid(odp_buffer_t buf) >> +{ >> + return validate_buf(buf) != NULL; >> +} >> >> int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf) >> { >> @@ -63,27 +276,13 @@ int odp_buffer_snprint(char *str, size_t n, >> odp_buffer_t buf) >> len += snprintf(&str[len], n-len, >> " pool %i\n", hdr->pool_hdl); >> len += snprintf(&str[len], n-len, >> - " index %"PRIu32"\n", hdr->index); >> - len += snprintf(&str[len], n-len, >> - " phy_addr %"PRIu64"\n", hdr->phys_addr); >> - len += snprintf(&str[len], n-len, >> - " addr %p\n", hdr->addr); >> + " addr %p\n", hdr->addr[0]); >> len += snprintf(&str[len], n-len, >> " size %zu\n", hdr->size); >> len += snprintf(&str[len], n-len, >> - " cur_offset %zu\n", hdr->cur_offset); >> - len += snprintf(&str[len], n-len, >> " ref_count %i\n", hdr->ref_count); >> len += snprintf(&str[len], n-len, >> " type %i\n", hdr->type); >> - len += snprintf(&str[len], n-len, >> - " Scatter list\n"); >> - len += snprintf(&str[len], n-len, >> - " num_bufs %i\n", >> hdr->scatter.num_bufs); >> - len += snprintf(&str[len], n-len, >> - " pos %i\n", hdr->scatter.pos); >> - len += snprintf(&str[len], n-len, >> - " total_len %zu\n", >> hdr->scatter.total_len); >> >> return len; >> } >> @@ -100,9 +299,3 @@ void odp_buffer_print(odp_buffer_t buf) >> >> printf("\n%s\n", str); >> } >> - >> -void odp_buffer_copy_scatter(odp_buffer_t buf_dst, odp_buffer_t buf_src) >> -{ >> - (void)buf_dst; >> - (void)buf_src; >> -} >> diff --git a/platform/linux-generic/odp_buffer_pool.c >> b/platform/linux-generic/odp_buffer_pool.c >> index a48d7d6..f6161bb 100644 >> --- a/platform/linux-generic/odp_buffer_pool.c >> +++ b/platform/linux-generic/odp_buffer_pool.c >> @@ -6,8 +6,9 @@ >> >> #include <odp_std_types.h> >> #include <odp_buffer_pool.h> >> -#include <odp_buffer_pool_internal.h> >> #include <odp_buffer_internal.h> >> +#include <odp_buffer_pool_internal.h> >> +#include <odp_buffer_inlines.h> >> #include <odp_packet_internal.h> >> #include <odp_timer_internal.h> >> #include <odp_shared_memory.h> >> @@ -16,6 +17,7 @@ >> #include <odp_config.h> >> #include <odp_hints.h> >> #include <odp_debug.h> >> +#include <odph_eth.h> >> >> #include <string.h> >> #include <stdlib.h> >> @@ -33,36 +35,26 @@ >> #define LOCK_INIT(a) odp_spinlock_init(a) >> #endif >> >> - >> -#if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >> -#error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >> -#endif >> - >> -#define NULL_INDEX ((uint32_t)-1) >> - >> -union buffer_type_any_u { >> +typedef union buffer_type_any_u { >> odp_buffer_hdr_t buf; >> odp_packet_hdr_t pkt; >> odp_timeout_hdr_t tmo; >> -}; >> - >> -ODP_STATIC_ASSERT((sizeof(union buffer_type_any_u) % 8) == 0, >> - "BUFFER_TYPE_ANY_U__SIZE_ERR"); >> +} odp_anybuf_t; >> >> /* Any buffer type header */ >> typedef struct { >> union buffer_type_any_u any_hdr; /* any buffer type */ >> - uint8_t buf_data[]; /* start of buffer data area >> */ >> } odp_any_buffer_hdr_t; >> >> +typedef struct odp_any_hdr_stride { >> + uint8_t >> pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_any_buffer_hdr_t))]; >> +} odp_any_hdr_stride; >> >> -typedef union pool_entry_u { >> - struct pool_entry_s s; >> - >> - uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct >> pool_entry_s))]; >> - >> -} pool_entry_t; >> +#if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >> +#error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >> +#endif >> >> +#define NULL_INDEX ((uint32_t)-1) >> >> typedef struct pool_table_t { >> pool_entry_t pool[ODP_CONFIG_BUFFER_POOLS]; >> @@ -76,39 +68,6 @@ static pool_table_t *pool_tbl; >> /* Pool entry pointers (for inlining) */ >> void *pool_entry_ptr[ODP_CONFIG_BUFFER_POOLS]; >> >> - >> -static __thread odp_buffer_chunk_hdr_t >> *local_chunk[ODP_CONFIG_BUFFER_POOLS]; >> - >> - >> -static inline odp_buffer_pool_t pool_index_to_handle(uint32_t pool_id) >> -{ >> - return pool_id + 1; >> -} >> - >> - >> -static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl) >> -{ >> - return pool_hdl -1; >> -} >> - >> - >> -static inline void set_handle(odp_buffer_hdr_t *hdr, >> - pool_entry_t *pool, uint32_t index) >> -{ >> - odp_buffer_pool_t pool_hdl = pool->s.pool_hdl; >> - uint32_t pool_id = pool_handle_to_index(pool_hdl); >> - >> - if (pool_id >= ODP_CONFIG_BUFFER_POOLS) >> - ODP_ABORT("set_handle: Bad pool handle %u\n", pool_hdl); >> - >> - if (index > ODP_BUFFER_MAX_INDEX) >> - ODP_ERR("set_handle: Bad buffer index\n"); >> - >> - hdr->handle.pool_id = pool_id; >> - hdr->handle.index = index; >> -} >> - >> - >> int odp_buffer_pool_init_global(void) >> { >> uint32_t i; >> @@ -142,269 +101,167 @@ int odp_buffer_pool_init_global(void) >> return 0; >> } >> >> - >> -static odp_buffer_hdr_t *index_to_hdr(pool_entry_t *pool, uint32_t index) >> -{ >> - odp_buffer_hdr_t *hdr; >> - >> - hdr = (odp_buffer_hdr_t *)(pool->s.buf_base + index * >> pool->s.buf_size); >> - return hdr; >> -} >> - >> - >> -static void add_buf_index(odp_buffer_chunk_hdr_t *chunk_hdr, uint32_t >> index) >> -{ >> - uint32_t i = chunk_hdr->chunk.num_bufs; >> - chunk_hdr->chunk.buf_index[i] = index; >> - chunk_hdr->chunk.num_bufs++; >> -} >> - >> - >> -static uint32_t rem_buf_index(odp_buffer_chunk_hdr_t *chunk_hdr) >> +/** >> + * Buffer pool creation >> + */ >> +odp_buffer_pool_t odp_buffer_pool_create(const char *name, >> + odp_buffer_pool_param_t *args, >> + odp_buffer_pool_init_t >> *init_args) >> { >> - uint32_t index; >> + odp_buffer_pool_t pool_hdl = ODP_BUFFER_POOL_INVALID; >> + pool_entry_t *pool; >> uint32_t i; >> + odp_buffer_pool_param_t *params; >> + odp_buffer_pool_init_t *init_params; >> >> - i = chunk_hdr->chunk.num_bufs - 1; >> - index = chunk_hdr->chunk.buf_index[i]; >> - chunk_hdr->chunk.num_bufs--; >> - return index; >> -} >> - >> - >> -static odp_buffer_chunk_hdr_t *next_chunk(pool_entry_t *pool, >> - odp_buffer_chunk_hdr_t >> *chunk_hdr) >> -{ >> - uint32_t index; >> - >> - index = chunk_hdr->chunk.buf_index[ODP_BUFS_PER_CHUNK-1]; >> - if (index == NULL_INDEX) >> - return NULL; >> - else >> - return (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, >> index); >> -} >> - >> - >> -static odp_buffer_chunk_hdr_t *rem_chunk(pool_entry_t *pool) >> -{ >> - odp_buffer_chunk_hdr_t *chunk_hdr; >> - >> - chunk_hdr = pool->s.head; >> - if (chunk_hdr == NULL) { >> - /* Pool is empty */ >> - return NULL; >> - } >> - >> - pool->s.head = next_chunk(pool, chunk_hdr); >> - pool->s.free_bufs -= ODP_BUFS_PER_CHUNK; >> - >> - /* unlink */ >> - rem_buf_index(chunk_hdr); >> - return chunk_hdr; >> -} >> - >> - >> -static void add_chunk(pool_entry_t *pool, odp_buffer_chunk_hdr_t >> *chunk_hdr) >> -{ >> - if (pool->s.head) /* link pool head to the chunk */ >> - add_buf_index(chunk_hdr, pool->s.head->buf_hdr.index); >> - else >> - add_buf_index(chunk_hdr, NULL_INDEX); >> - >> - pool->s.head = chunk_hdr; >> - pool->s.free_bufs += ODP_BUFS_PER_CHUNK; >> -} >> + /* Default pool creation arguments */ >> + static odp_buffer_pool_param_t default_params = { >> + .buf_num = 1024, >> + .buf_size = ODP_CONFIG_BUF_SEG_SIZE, >> + .buf_type = ODP_BUFFER_TYPE_PACKET, >> + .buf_opts = ODP_BUFFER_OPTS_UNSEGMENTED, >> + }; >> > Do we need to have this behaviour of allocating default params if the > input params are NULL. > It would be better if the API returns error. > >> >> + /* Default initialization paramters */ >> + static odp_buffer_pool_init_t default_init_params = { >> + .udata_size = 0, >> + .buf_init = NULL, >> + .buf_init_arg = NULL, >> + }; >> >> -static void check_align(pool_entry_t *pool, odp_buffer_hdr_t *hdr) >> -{ >> - if (!ODP_ALIGNED_CHECK_POWER_2(hdr->addr, pool->s.user_align)) { >> - ODP_ABORT("check_align: user data align error %p, align >> %zu\n", >> - hdr->addr, pool->s.user_align); >> - } >> + /* Handle NULL input parameters */ >> + params = args == NULL ? &default_params : args; >> + init_params = init_args == NULL ? &default_init_params >> + : init_args; >> >> - if (!ODP_ALIGNED_CHECK_POWER_2(hdr, ODP_CACHE_LINE_SIZE)) { >> - ODP_ABORT("check_align: hdr align error %p, align %i\n", >> - hdr, ODP_CACHE_LINE_SIZE); >> - } >> -} >> + if (params->buf_type != ODP_BUFFER_TYPE_PACKET) >> + params->buf_opts |= ODP_BUFFER_OPTS_UNSEGMENTED; >> >> + int unsegmented = ((params->buf_opts & >> ODP_BUFFER_OPTS_UNSEGMENTED) == >> + ODP_BUFFER_OPTS_UNSEGMENTED); >> >> -static void fill_hdr(void *ptr, pool_entry_t *pool, uint32_t index, >> - int buf_type) >> -{ >> - odp_buffer_hdr_t *hdr = (odp_buffer_hdr_t *)ptr; >> - size_t size = pool->s.hdr_size; >> - uint8_t *buf_data; >> + uint32_t udata_stride = >> + ODP_CACHE_LINE_SIZE_ROUNDUP(init_params->udata_size); >> >> - if (buf_type == ODP_BUFFER_TYPE_CHUNK) >> - size = sizeof(odp_buffer_chunk_hdr_t); >> - >> - switch (pool->s.buf_type) { >> - odp_raw_buffer_hdr_t *raw_hdr; >> - odp_packet_hdr_t *packet_hdr; >> - odp_timeout_hdr_t *tmo_hdr; >> - odp_any_buffer_hdr_t *any_hdr; >> + uint32_t blk_size, buf_stride; >> >> + switch (params->buf_type) { >> case ODP_BUFFER_TYPE_RAW: >> - raw_hdr = ptr; >> - buf_data = raw_hdr->buf_data; >> + blk_size = params->buf_size; >> + buf_stride = sizeof(odp_buffer_hdr_stride); >> break; >> + >> case ODP_BUFFER_TYPE_PACKET: >> - packet_hdr = ptr; >> - buf_data = packet_hdr->buf_data; >> + if (unsegmented) >> + blk_size = >> + >> ODP_CACHE_LINE_SIZE_ROUNDUP(params->buf_size); >> + else >> + blk_size = ODP_ALIGN_ROUNDUP(params->buf_size, >> + >> ODP_CONFIG_BUF_SEG_SIZE); >> + buf_stride = sizeof(odp_packet_hdr_stride); >> break; >> + >> case ODP_BUFFER_TYPE_TIMEOUT: >> - tmo_hdr = ptr; >> - buf_data = tmo_hdr->buf_data; >> + blk_size = 0; /* Timeouts have no block data, only >> metadata */ >> + buf_stride = sizeof(odp_timeout_hdr_stride); >> break; >> + >> case ODP_BUFFER_TYPE_ANY: >> - any_hdr = ptr; >> - buf_data = any_hdr->buf_data; >> + if (unsegmented) >> + blk_size = >> + >> ODP_CACHE_LINE_SIZE_ROUNDUP(params->buf_size); >> + else >> + blk_size = ODP_ALIGN_ROUNDUP(params->buf_size, >> + >> ODP_CONFIG_BUF_SEG_SIZE); >> + buf_stride = sizeof(odp_any_hdr_stride); >> break; >> - default: >> - ODP_ABORT("Bad buffer type\n"); >> - } >> - >> - memset(hdr, 0, size); >> - >> - set_handle(hdr, pool, index); >> - >> - hdr->addr = &buf_data[pool->s.buf_offset - pool->s.hdr_size]; >> - hdr->index = index; >> - hdr->size = pool->s.user_size; >> - hdr->pool_hdl = pool->s.pool_hdl; >> - hdr->type = buf_type; >> - >> - check_align(pool, hdr); >> -} >> - >> - >> -static void link_bufs(pool_entry_t *pool) >> -{ >> - odp_buffer_chunk_hdr_t *chunk_hdr; >> - size_t hdr_size; >> - size_t data_size; >> - size_t data_align; >> - size_t tot_size; >> - size_t offset; >> - size_t min_size; >> - uint64_t pool_size; >> - uintptr_t buf_base; >> - uint32_t index; >> - uintptr_t pool_base; >> - int buf_type; >> - >> - buf_type = pool->s.buf_type; >> - data_size = pool->s.user_size; >> - data_align = pool->s.user_align; >> - pool_size = pool->s.pool_size; >> - pool_base = (uintptr_t) pool->s.pool_base_addr; >> - >> - if (buf_type == ODP_BUFFER_TYPE_RAW) { >> - hdr_size = sizeof(odp_raw_buffer_hdr_t); >> - } else if (buf_type == ODP_BUFFER_TYPE_PACKET) { >> - hdr_size = sizeof(odp_packet_hdr_t); >> - } else if (buf_type == ODP_BUFFER_TYPE_TIMEOUT) { >> - hdr_size = sizeof(odp_timeout_hdr_t); >> - } else if (buf_type == ODP_BUFFER_TYPE_ANY) { >> - hdr_size = sizeof(odp_any_buffer_hdr_t); >> - } else >> - ODP_ABORT("odp_buffer_pool_create: Bad type %i\n", >> buf_type); >> - >> - >> - /* Chunk must fit into buffer data area.*/ >> - min_size = sizeof(odp_buffer_chunk_hdr_t) - hdr_size; >> - if (data_size < min_size) >> - data_size = min_size; >> - >> - /* Roundup data size to full cachelines */ >> - data_size = ODP_CACHE_LINE_SIZE_ROUNDUP(data_size); >> - >> - /* Min cacheline alignment for buffer header and data */ >> - data_align = ODP_CACHE_LINE_SIZE_ROUNDUP(data_align); >> - offset = ODP_CACHE_LINE_SIZE_ROUNDUP(hdr_size); >> - >> - /* Multiples of cacheline size */ >> - if (data_size > data_align) >> - tot_size = data_size + offset; >> - else >> - tot_size = data_align + offset; >> - >> - /* First buffer */ >> - buf_base = ODP_ALIGN_ROUNDUP(pool_base + offset, data_align) - >> offset; >> - >> - pool->s.hdr_size = hdr_size; >> - pool->s.buf_base = buf_base; >> - pool->s.buf_size = tot_size; >> - pool->s.buf_offset = offset; >> - index = 0; >> - >> - chunk_hdr = (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, index); >> - pool->s.head = NULL; >> - pool_size -= buf_base - pool_base; >> - >> - while (pool_size > ODP_BUFS_PER_CHUNK * tot_size) { >> - int i; >> - >> - fill_hdr(chunk_hdr, pool, index, ODP_BUFFER_TYPE_CHUNK); >> - >> - index++; >> - >> - for (i = 0; i < ODP_BUFS_PER_CHUNK - 1; i++) { >> - odp_buffer_hdr_t *hdr = index_to_hdr(pool, index); >> - >> - fill_hdr(hdr, pool, index, buf_type); >> - >> - add_buf_index(chunk_hdr, index); >> - index++; >> - } >> - >> - add_chunk(pool, chunk_hdr); >> >> - chunk_hdr = (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, >> - index); >> - pool->s.num_bufs += ODP_BUFS_PER_CHUNK; >> - pool_size -= ODP_BUFS_PER_CHUNK * tot_size; >> + default: >> + return ODP_BUFFER_POOL_INVALID; >> } >> -} >> - >> - >> -odp_buffer_pool_t odp_buffer_pool_create(const char *name, >> - void *base_addr, uint64_t size, >> - size_t buf_size, size_t >> buf_align, >> - int buf_type) >> -{ >> - odp_buffer_pool_t pool_hdl = ODP_BUFFER_POOL_INVALID; >> - pool_entry_t *pool; >> - uint32_t i; >> >> for (i = 0; i < ODP_CONFIG_BUFFER_POOLS; i++) { >> pool = get_pool_entry(i); >> >> LOCK(&pool->s.lock); >> + if (pool->s.shm != ODP_SHM_INVALID) { >> + UNLOCK(&pool->s.lock); >> + continue; >> + } >> >> - if (pool->s.buf_base == 0) { >> - /* found free pool */ >> + /* found free pool */ >> + size_t block_size, mdata_size, udata_size; >> >> - strncpy(pool->s.name, name, >> - ODP_BUFFER_POOL_NAME_LEN - 1); >> - pool->s.name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; >> - pool->s.pool_base_addr = base_addr; >> - pool->s.pool_size = size; >> - pool->s.user_size = buf_size; >> - pool->s.user_align = buf_align; >> - pool->s.buf_type = buf_type; >> + strncpy(pool->s.name, name, >> + ODP_BUFFER_POOL_NAME_LEN - 1); >> + pool->s.name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; >> >> - link_bufs(pool); >> + pool->s.params = *params; >> + pool->s.init_params = *init_params; >> >> - UNLOCK(&pool->s.lock); >> + mdata_size = params->buf_num * buf_stride; >> + udata_size = params->buf_num * udata_stride; >> + block_size = params->buf_num * blk_size; >> + >> + pool->s.pool_size = ODP_PAGE_SIZE_ROUNDUP(mdata_size + >> + udata_size + >> + block_size); >> >> - pool_hdl = pool->s.pool_hdl; >> - break; >> + pool->s.shm = odp_shm_reserve(pool->s.name, >> pool->s.pool_size, >> + ODP_PAGE_SIZE, 0); >> + if (pool->s.shm == ODP_SHM_INVALID) { >> + UNLOCK(&pool->s.lock); >> + return ODP_BUFFER_POOL_INVALID; >> } >> >> + pool->s.pool_base_addr = (uint8_t >> *)odp_shm_addr(pool->s.shm); >> + pool->s.flags.unsegmented = unsegmented; >> + pool->s.seg_size = unsegmented ? >> + blk_size : ODP_CONFIG_BUF_SEG_SIZE; >> + uint8_t *udata_base_addr = pool->s.pool_base_addr + >> mdata_size; >> + uint8_t *block_base_addr = udata_base_addr + udata_size; >> + >> + pool->s.bufcount = 0; >> + pool->s.buf_freelist = NULL; >> + pool->s.blk_freelist = NULL; >> + >> + uint8_t *buf = pool->s.udata_base_addr - buf_stride; >> + uint8_t *udat = (udata_stride == 0) ? NULL : >> + block_base_addr - udata_stride; >> + >> + /* Init buffer common header and add to pool buffer >> freelist */ >> + do { >> + odp_buffer_hdr_t *tmp = >> + (odp_buffer_hdr_t *)(void *)buf; >> + tmp->pool_hdl = pool->s.pool_hdl; >> + tmp->size = 0; >> + tmp->type = params->buf_type; >> + tmp->udata_addr = (void *)udat; >> + tmp->udata_size = init_params->udata_size; >> + tmp->segcount = 0; >> + tmp->segsize = pool->s.seg_size; >> + tmp->buf_hdl.handle = >> + >> odp_buffer_encode_handle((odp_buffer_hdr_t *) >> + (void *)buf); >> + ret_buf(&pool->s, tmp); >> + buf -= buf_stride; >> + udat -= udata_stride; >> + } while (buf >= pool->s.pool_base_addr); >> + >> + /* Form block freelist for pool */ >> + uint8_t *blk = pool->s.pool_base_addr + pool->s.pool_size >> - >> + pool->s.seg_size; >> + >> + if (blk_size > 0) >> + do { >> + ret_blk(&pool->s, blk); >> + blk -= pool->s.seg_size; >> + } while (blk >= block_base_addr); >> + >> UNLOCK(&pool->s.lock); >> + >> + pool_hdl = pool->s.pool_hdl; >> + break; >> } >> >> return pool_hdl; >> @@ -431,93 +288,172 @@ odp_buffer_pool_t odp_buffer_pool_lookup(const >> char *name) >> return ODP_BUFFER_POOL_INVALID; >> } >> >> - >> -odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_hdl) >> +odp_buffer_pool_t odp_buffer_pool_next(odp_buffer_pool_t pool_hdl, >> + char *name, >> + size_t *udata_size, >> + odp_buffer_pool_param_t *params, >> + int *predef) >> { >> - pool_entry_t *pool; >> - odp_buffer_chunk_hdr_t *chunk; >> - odp_buffer_bits_t handle; >> - uint32_t pool_id = pool_handle_to_index(pool_hdl); >> - >> - pool = get_pool_entry(pool_id); >> - chunk = local_chunk[pool_id]; >> - >> - if (chunk == NULL) { >> - LOCK(&pool->s.lock); >> - chunk = rem_chunk(pool); >> - UNLOCK(&pool->s.lock); >> + pool_entry_t *next_pool; >> + uint32_t pool_id; >> >> - if (chunk == NULL) >> - return ODP_BUFFER_INVALID; >> + /* NULL input means first element */ >> + if (pool_hdl == ODP_BUFFER_POOL_INVALID) { >> + pool_id = 0; >> + next_pool = get_pool_entry(pool_id); >> + } else { >> + pool_id = pool_handle_to_index(pool_hdl); >> >> - local_chunk[pool_id] = chunk; >> + if (pool_id == ODP_CONFIG_BUFFER_POOLS) >> + return ODP_BUFFER_POOL_INVALID; >> + else >> + next_pool = get_pool_entry(++pool_id); >> } >> >> - if (chunk->chunk.num_bufs == 0) { >> - /* give the chunk buffer */ >> - local_chunk[pool_id] = NULL; >> - chunk->buf_hdr.type = pool->s.buf_type; >> + /* Only interested in pools that exist */ >> + while (next_pool->s.shm == ODP_SHM_INVALID) { >> + if (pool_id == ODP_CONFIG_BUFFER_POOLS) >> + return ODP_BUFFER_POOL_INVALID; >> + else >> + next_pool = get_pool_entry(++pool_id); >> + } >> >> - handle = chunk->buf_hdr.handle; >> - } else { >> - odp_buffer_hdr_t *hdr; >> - uint32_t index; >> - index = rem_buf_index(chunk); >> - hdr = index_to_hdr(pool, index); >> + /* Found the next pool, so return info about it */ >> + strncpy(name, next_pool->s.name, ODP_BUFFER_POOL_NAME_LEN - 1); >> + name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; >> >> - handle = hdr->handle; >> - } >> + *udata_size = next_pool->s.init_params.udata_size; >> + *params = next_pool->s.params; >> + *predef = next_pool->s.flags.predefined; >> >> - return handle.u32; >> + return next_pool->s.pool_hdl; >> } >> >> - >> -void odp_buffer_free(odp_buffer_t buf) >> +int odp_buffer_pool_destroy(odp_buffer_pool_t pool_hdl) >> { >> - odp_buffer_hdr_t *hdr; >> - uint32_t pool_id; >> - pool_entry_t *pool; >> - odp_buffer_chunk_hdr_t *chunk_hdr; >> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >> >> - hdr = odp_buf_to_hdr(buf); >> - pool_id = pool_handle_to_index(hdr->pool_hdl); >> - pool = get_pool_entry(pool_id); >> - chunk_hdr = local_chunk[pool_id]; >> + if (pool == NULL) >> + return -1; >> >> - if (chunk_hdr && chunk_hdr->chunk.num_bufs == ODP_BUFS_PER_CHUNK >> - 1) { >> - /* Current chunk is full. Push back to the pool */ >> - LOCK(&pool->s.lock); >> - add_chunk(pool, chunk_hdr); >> + LOCK(&pool->s.lock); >> + >> + if (pool->s.shm == ODP_SHM_INVALID || >> + pool->s.bufcount > 0 || >> + pool->s.flags.predefined) { >> UNLOCK(&pool->s.lock); >> - chunk_hdr = NULL; >> + return -1; >> } >> >> - if (chunk_hdr == NULL) { >> - /* Use this buffer */ >> - chunk_hdr = (odp_buffer_chunk_hdr_t *)hdr; >> - local_chunk[pool_id] = chunk_hdr; >> - chunk_hdr->chunk.num_bufs = 0; >> + odp_shm_free(pool->s.shm); >> + >> + pool->s.shm = ODP_SHM_INVALID; >> + UNLOCK(&pool->s.lock); >> + >> + return 0; >> +} >> + >> +size_t odp_buffer_pool_headroom(odp_buffer_pool_t pool_hdl) >> +{ >> + return odp_pool_to_entry(pool_hdl)->s.headroom; >> +} >> + >> +int odp_buffer_pool_set_headroom(odp_buffer_pool_t pool_hdl, size_t hr) >> +{ >> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >> + >> + if (hr >= pool->s.seg_size/2) { >> + return -1; >> } else { >> - /* Add to current chunk */ >> - add_buf_index(chunk_hdr, hdr->index); >> + pool->s.headroom = hr; >> + return 0; >> } >> } >> >> +size_t odp_buffer_pool_tailroom(odp_buffer_pool_t pool_hdl) >> +{ >> + return odp_pool_to_entry(pool_hdl)->s.tailroom; >> +} >> >> -odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf) >> +int odp_buffer_pool_set_tailroom(odp_buffer_pool_t pool_hdl, size_t tr) >> { >> - odp_buffer_hdr_t *hdr; >> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >> >> - hdr = odp_buf_to_hdr(buf); >> - return hdr->pool_hdl; >> + if (tr >= pool->s.seg_size/2) { >> + return -1; >> + } else { >> + pool->s.tailroom = tr; >> + return 0; >> + } >> } >> >> +odp_buffer_t buffer_alloc(odp_buffer_pool_t pool_hdl, size_t size) >> +{ >> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >> + size_t totsize = pool->s.headroom + size + pool->s.tailroom; >> + odp_anybuf_t *buf; >> + uint8_t *blk; >> + >> + if ((pool->s.flags.unsegmented && totsize > pool->s.seg_size) || >> + (!pool->s.flags.unsegmented && totsize > >> ODP_CONFIG_BUF_MAX_SIZE)) >> + return ODP_BUFFER_INVALID; >> + >> + buf = (odp_anybuf_t *)(void *)get_buf(&pool->s); >> + >> + if (buf == NULL) >> + return ODP_BUFFER_INVALID; >> + >> + /* Get blocks for this buffer, if pool uses application data */ >> + if (buf->buf.segsize > 0) >> + do { >> + blk = get_blk(&pool->s); >> + if (blk == NULL) { >> + ret_buf(&pool->s, &buf->buf); >> + return ODP_BUFFER_INVALID; >> + } >> + buf->buf.addr[buf->buf.segcount++] = blk; >> + totsize = totsize < pool->s.seg_size ? 0 : >> + totsize - pool->s.seg_size; >> + } while (totsize > 0); >> + >> + switch (buf->buf.type) { >> + case ODP_BUFFER_TYPE_RAW: >> + break; >> + >> + case ODP_BUFFER_TYPE_PACKET: >> + packet_init(pool, &buf->pkt, size); >> + if (pool->s.init_params.buf_init != NULL) >> + (*pool->s.init_params.buf_init) >> + (buf->buf.buf_hdl.handle, >> + pool->s.init_params.buf_init_arg); >> + break; >> + >> + case ODP_BUFFER_TYPE_TIMEOUT: >> + break; >> + >> + default: >> + ret_buf(&pool->s, &buf->buf); >> + return ODP_BUFFER_INVALID; >> + } >> + >> + return odp_hdr_to_buf(&buf->buf); >> +} >> + >> +odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_hdl) >> +{ >> + return buffer_alloc(pool_hdl, 0); >> +} >> + >> +void odp_buffer_free(odp_buffer_t buf) >> +{ >> + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >> + pool_entry_t *pool = odp_buf_to_pool(hdr); >> + ret_buf(&pool->s, hdr); >> +} >> >> void odp_buffer_pool_print(odp_buffer_pool_t pool_hdl) >> { >> pool_entry_t *pool; >> - odp_buffer_chunk_hdr_t *chunk_hdr; >> - uint32_t i; >> uint32_t pool_id; >> >> pool_id = pool_handle_to_index(pool_hdl); >> @@ -528,47 +464,4 @@ void odp_buffer_pool_print(odp_buffer_pool_t >> pool_hdl) >> printf(" pool %i\n", pool->s.pool_hdl); >> printf(" name %s\n", pool->s.name); >> printf(" pool base %p\n", pool->s.pool_base_addr); >> - printf(" buf base 0x%"PRIxPTR"\n", pool->s.buf_base); >> - printf(" pool size 0x%"PRIx64"\n", pool->s.pool_size); >> - printf(" buf size %zu\n", pool->s.user_size); >> - printf(" buf align %zu\n", pool->s.user_align); >> - printf(" hdr size %zu\n", pool->s.hdr_size); >> - printf(" alloc size %zu\n", pool->s.buf_size); >> - printf(" offset to hdr %zu\n", pool->s.buf_offset); >> - printf(" num bufs %"PRIu64"\n", pool->s.num_bufs); >> - printf(" free bufs %"PRIu64"\n", pool->s.free_bufs); >> - >> - /* first chunk */ >> - chunk_hdr = pool->s.head; >> - >> - if (chunk_hdr == NULL) { >> - ODP_ERR(" POOL EMPTY\n"); >> - return; >> - } >> - >> - printf("\n First chunk\n"); >> - >> - for (i = 0; i < chunk_hdr->chunk.num_bufs - 1; i++) { >> - uint32_t index; >> - odp_buffer_hdr_t *hdr; >> - >> - index = chunk_hdr->chunk.buf_index[i]; >> - hdr = index_to_hdr(pool, index); >> - >> - printf(" [%i] addr %p, id %"PRIu32"\n", i, hdr->addr, >> index); >> - } >> - >> - printf(" [%i] addr %p, id %"PRIu32"\n", i, >> chunk_hdr->buf_hdr.addr, >> - chunk_hdr->buf_hdr.index); >> - >> - /* next chunk */ >> - chunk_hdr = next_chunk(pool, chunk_hdr); >> - >> - if (chunk_hdr) { >> - printf(" Next chunk\n"); >> - printf(" addr %p, id %"PRIu32"\n", >> chunk_hdr->buf_hdr.addr, >> - chunk_hdr->buf_hdr.index); >> - } >> - >> - printf("\n"); >> } >> diff --git a/platform/linux-generic/odp_packet.c >> b/platform/linux-generic/odp_packet.c >> index 82ea879..45ac8e5 100644 >> --- a/platform/linux-generic/odp_packet.c >> +++ b/platform/linux-generic/odp_packet.c >> @@ -11,29 +11,31 @@ >> >> #include <odph_eth.h> >> #include <odph_ip.h> >> +#include <odph_tcp.h> >> +#include <odph_udp.h> >> >> #include <string.h> >> #include <stdio.h> >> >> -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >> - odph_ipv4hdr_t *ipv4, size_t >> *offset_out); >> -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >> - odph_ipv6hdr_t *ipv6, size_t >> *offset_out); >> - >> void odp_packet_init(odp_packet_t pkt) >> { >> odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >> + pool_entry_t *pool = odp_buf_to_pool(&pkt_hdr->buf_hdr); >> const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, >> buf_hdr); >> uint8_t *start; >> size_t len; >> >> start = (uint8_t *)pkt_hdr + start_offset; >> - len = ODP_OFFSETOF(odp_packet_hdr_t, buf_data) - start_offset; >> + len = sizeof(odp_packet_hdr_t) - start_offset; >> memset(start, 0, len); >> >> pkt_hdr->l2_offset = ODP_PACKET_OFFSET_INVALID; >> pkt_hdr->l3_offset = ODP_PACKET_OFFSET_INVALID; >> pkt_hdr->l4_offset = ODP_PACKET_OFFSET_INVALID; >> + pkt_hdr->payload_offset = ODP_PACKET_OFFSET_INVALID; >> + >> + pkt_hdr->headroom = pool->s.headroom; >> + pkt_hdr->tailroom = pool->s.tailroom; >> } >> >> odp_packet_t odp_packet_from_buffer(odp_buffer_t buf) >> @@ -46,55 +48,112 @@ odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt) >> return (odp_buffer_t)pkt; >> } >> >> -void odp_packet_set_len(odp_packet_t pkt, size_t len) >> +size_t odp_packet_len(odp_packet_t pkt) >> { >> - odp_packet_hdr(pkt)->frame_len = len; >> + return odp_packet_hdr(pkt)->frame_len; >> } >> >> -size_t odp_packet_get_len(odp_packet_t pkt) >> +void *odp_packet_offset_map(odp_packet_t pkt, size_t offset, size_t >> *seglen) >> { >> - return odp_packet_hdr(pkt)->frame_len; >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + if (offset >= pkt_hdr->frame_len) >> + return NULL; >> + >> + return buffer_map(&pkt_hdr->buf_hdr, >> + pkt_hdr->headroom + offset, >> + seglen, pkt_hdr->frame_len); >> } >> >> -uint8_t *odp_packet_addr(odp_packet_t pkt) >> +void odp_packet_offset_unmap(odp_packet_t pkt ODP_UNUSED, >> + size_t offset ODP_UNUSED) >> { >> - return odp_buffer_addr(odp_packet_to_buffer(pkt)); >> } >> >> -uint8_t *odp_packet_data(odp_packet_t pkt) >> +void *odp_packet_map(odp_packet_t pkt, size_t *seglen) >> { >> - return odp_packet_addr(pkt) + odp_packet_hdr(pkt)->frame_offset; >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + return buffer_map(&pkt_hdr->buf_hdr, >> + 0, seglen, pkt_hdr->frame_len); >> } >> >> +void *odp_packet_addr(odp_packet_t pkt) >> +{ >> + size_t seglen; >> + return odp_packet_map(pkt, &seglen); >> +} >> >> -uint8_t *odp_packet_l2(odp_packet_t pkt) >> +odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt) >> { >> - const size_t offset = odp_packet_l2_offset(pkt); >> + return odp_packet_hdr(pkt)->buf_hdr.pool_hdl; >> +} >> >> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >> - return NULL; >> +odp_packet_segment_t odp_packet_segment_by_index(odp_packet_t pkt, >> + size_t ndx) >> +{ >> >> - return odp_packet_addr(pkt) + offset; >> + return (odp_packet_segment_t) >> + buffer_segment(&odp_packet_hdr(pkt)->buf_hdr, ndx); >> } >> >> -size_t odp_packet_l2_offset(odp_packet_t pkt) >> +odp_packet_segment_t odp_packet_segment_next(odp_packet_t pkt, >> + odp_packet_segment_t seg) >> { >> - return odp_packet_hdr(pkt)->l2_offset; >> + return (odp_packet_segment_t) >> + segment_next(&odp_packet_hdr(pkt)->buf_hdr, seg); >> } >> >> -void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset) >> +void *odp_packet_segment_map(odp_packet_t pkt, odp_packet_segment_t seg, >> + size_t *seglen) >> { >> - odp_packet_hdr(pkt)->l2_offset = offset; >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + return segment_map(&pkt_hdr->buf_hdr, seg, >> + seglen, pkt_hdr->frame_len); >> } >> >> -uint8_t *odp_packet_l3(odp_packet_t pkt) >> +void odp_packet_segment_unmap(odp_packet_segment_t seg ODP_UNUSED) >> { >> - const size_t offset = odp_packet_l3_offset(pkt); >> +} >> >> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >> - return NULL; >> +void *odp_packet_udata(odp_packet_t pkt, size_t *len) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + *len = pkt_hdr->buf_hdr.udata_size; >> + return pkt_hdr->buf_hdr.udata_addr; >> +} >> + >> +void *odp_packet_udata_addr(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->buf_hdr.udata_addr; >> +} >> + >> +void *odp_packet_l2_map(odp_packet_t pkt, size_t *seglen) >> +{ >> + return odp_packet_offset_map(pkt, odp_packet_l2_offset(pkt), >> seglen); >> +} >> + >> +size_t odp_packet_l2_offset(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->l2_offset; >> +} >> + >> +int odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset) >> +{ >> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >> + >> + if (offset >= hdr->frame_len) >> + return -1; >> + >> + hdr->l2_offset = offset; >> + return 0; >> +} >> >> - return odp_packet_addr(pkt) + offset; >> +void *odp_packet_l3_map(odp_packet_t pkt, size_t *seglen) >> +{ >> + return odp_packet_offset_map(pkt, odp_packet_l3_offset(pkt), >> seglen); >> } >> >> size_t odp_packet_l3_offset(odp_packet_t pkt) >> @@ -102,19 +161,35 @@ size_t odp_packet_l3_offset(odp_packet_t pkt) >> return odp_packet_hdr(pkt)->l3_offset; >> } >> >> -void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset) >> +int odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset) >> { >> - odp_packet_hdr(pkt)->l3_offset = offset; >> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >> + >> + if (offset >= hdr->frame_len) >> + return -1; >> + >> + hdr->l3_offset = offset; >> + return 0; >> } >> >> -uint8_t *odp_packet_l4(odp_packet_t pkt) >> +uint32_t odp_packet_l3_protocol(odp_packet_t pkt) >> { >> - const size_t offset = odp_packet_l4_offset(pkt); >> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >> >> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >> - return NULL; >> + if (hdr->input_flags.l3) >> + return hdr->l3_protocol; >> + else >> + return -1; >> +} >> + >> +void odp_packet_set_l3_protocol(odp_packet_t pkt, uint16_t protocol) >> +{ >> + odp_packet_hdr(pkt)->l3_protocol = protocol; >> +} >> >> - return odp_packet_addr(pkt) + offset; >> +void *odp_packet_l4_map(odp_packet_t pkt, size_t *seglen) >> +{ >> + return odp_packet_offset_map(pkt, odp_packet_l4_offset(pkt), >> seglen); >> } >> >> size_t odp_packet_l4_offset(odp_packet_t pkt) >> @@ -122,277 +197,902 @@ size_t odp_packet_l4_offset(odp_packet_t pkt) >> return odp_packet_hdr(pkt)->l4_offset; >> } >> >> -void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset) >> +int odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset) >> { >> - odp_packet_hdr(pkt)->l4_offset = offset; >> -} >> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >> + >> + if (offset >= hdr->frame_len) >> + return -1; >> >> + hdr->l4_offset = offset; >> + return 0; >> +} >> >> -int odp_packet_is_segmented(odp_packet_t pkt) >> +uint32_t odp_packet_l4_protocol(odp_packet_t pkt) >> { >> - odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr((odp_buffer_t)pkt); >> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >> >> - if (buf_hdr->scatter.num_bufs == 0) >> - return 0; >> + if (hdr->input_flags.l4) >> + return hdr->l4_protocol; >> else >> - return 1; >> + return -1; >> } >> >> +void odp_packet_set_l4_protocol(odp_packet_t pkt, uint8_t protocol) >> +{ >> + odp_packet_hdr(pkt)->l4_protocol = protocol; >> +} >> >> -int odp_packet_seg_count(odp_packet_t pkt) >> +void *odp_packet_payload_map(odp_packet_t pkt, size_t *seglen) >> { >> - odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr((odp_buffer_t)pkt); >> + return odp_packet_offset_map(pkt, odp_packet_payload_offset(pkt), >> + seglen); >> +} >> >> - return (int)buf_hdr->scatter.num_bufs + 1; >> +size_t odp_packet_payload_offset(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->payload_offset; >> } >> >> +int odp_packet_set_payload_offset(odp_packet_t pkt, size_t offset) >> +{ >> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >> >> -/** >> - * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP >> - * >> - * Internal function: caller is resposible for passing only valid packet >> handles >> - * , lengths and offsets (usually done&called in packet input). >> - * >> - * @param pkt Packet handle >> - * @param len Packet length in bytes >> - * @param frame_offset Byte offset to L2 header >> - */ >> -void odp_packet_parse(odp_packet_t pkt, size_t len, size_t frame_offset) >> + if (offset >= hdr->frame_len) >> + return -1; >> + >> + hdr->payload_offset = offset; >> + return 0; >> +} >> + >> +int odp_packet_error(odp_packet_t pkt) >> { >> - odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >> - odph_ethhdr_t *eth; >> - odph_vlanhdr_t *vlan; >> - odph_ipv4hdr_t *ipv4; >> - odph_ipv6hdr_t *ipv6; >> - uint16_t ethtype; >> - size_t offset = 0; >> - uint8_t ip_proto = 0; >> + return odp_packet_hdr(pkt)->error_flags.all != 0; >> +} >> >> - pkt_hdr->input_flags.eth = 1; >> - pkt_hdr->frame_offset = frame_offset; >> - pkt_hdr->frame_len = len; >> +void odp_packet_set_error(odp_packet_t pkt, int val) >> +{ >> + odp_packet_hdr(pkt)->error_flags.app_error = val; >> +} >> >> - if (odp_unlikely(len < ODPH_ETH_LEN_MIN)) { >> - pkt_hdr->error_flags.frame_len = 1; >> - return; >> - } else if (len > ODPH_ETH_LEN_MAX) { >> - pkt_hdr->input_flags.jumbo = 1; >> - } >> +int odp_packet_inflag_l2(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->input_flags.l2; >> +} >> >> - /* Assume valid L2 header, no CRC/FCS check in SW */ >> - pkt_hdr->input_flags.l2 = 1; >> - pkt_hdr->l2_offset = frame_offset; >> +void odp_packet_set_inflag_l2(odp_packet_t pkt, int val) >> +{ >> + odp_packet_hdr(pkt)->input_flags.l2 = val; >> +} >> >> - eth = (odph_ethhdr_t *)odp_packet_data(pkt); >> - ethtype = odp_be_to_cpu_16(eth->type); >> - vlan = (odph_vlanhdr_t *)ð->type; >> +int odp_packet_inflag_l3(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->input_flags.l3; >> +} >> >> - if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { >> - pkt_hdr->input_flags.vlan_qinq = 1; >> - ethtype = odp_be_to_cpu_16(vlan->tpid); >> - offset += sizeof(odph_vlanhdr_t); >> - vlan = &vlan[1]; >> - } >> +void odp_packet_set_inflag_l3(odp_packet_t pkt, int val) >> +{ >> + odp_packet_hdr(pkt)->input_flags.l3 = val; >> +} >> >> - if (ethtype == ODPH_ETHTYPE_VLAN) { >> - pkt_hdr->input_flags.vlan = 1; >> - ethtype = odp_be_to_cpu_16(vlan->tpid); >> - offset += sizeof(odph_vlanhdr_t); >> - } >> +int odp_packet_inflag_l4(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->input_flags.l4; >> +} >> >> - /* Set l3_offset+flag only for known ethtypes */ >> - switch (ethtype) { >> - case ODPH_ETHTYPE_IPV4: >> - pkt_hdr->input_flags.ipv4 = 1; >> - pkt_hdr->input_flags.l3 = 1; >> - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + >> offset; >> - ipv4 = (odph_ipv4hdr_t *)odp_packet_l3(pkt); >> - ip_proto = parse_ipv4(pkt_hdr, ipv4, &offset); >> - break; >> - case ODPH_ETHTYPE_IPV6: >> - pkt_hdr->input_flags.ipv6 = 1; >> - pkt_hdr->input_flags.l3 = 1; >> - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + >> offset; >> - ipv6 = (odph_ipv6hdr_t *)odp_packet_l3(pkt); >> - ip_proto = parse_ipv6(pkt_hdr, ipv6, &offset); >> - break; >> - case ODPH_ETHTYPE_ARP: >> - pkt_hdr->input_flags.arp = 1; >> - /* fall through */ >> - default: >> - ip_proto = 0; >> - break; >> - } >> +void odp_packet_set_inflag_l4(odp_packet_t pkt, int val) >> +{ >> + odp_packet_hdr(pkt)->input_flags.l4 = val; >> +} >> >> - switch (ip_proto) { >> - case ODPH_IPPROTO_UDP: >> - pkt_hdr->input_flags.udp = 1; >> - pkt_hdr->input_flags.l4 = 1; >> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >> - break; >> - case ODPH_IPPROTO_TCP: >> - pkt_hdr->input_flags.tcp = 1; >> - pkt_hdr->input_flags.l4 = 1; >> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >> - break; >> - case ODPH_IPPROTO_SCTP: >> - pkt_hdr->input_flags.sctp = 1; >> - pkt_hdr->input_flags.l4 = 1; >> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >> - break; >> - case ODPH_IPPROTO_ICMP: >> - pkt_hdr->input_flags.icmp = 1; >> - pkt_hdr->input_flags.l4 = 1; >> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >> - break; >> - default: >> - /* 0 or unhandled IP protocols, don't set L4 flag+offset >> */ >> - if (pkt_hdr->input_flags.ipv6) { >> - /* IPv6 next_hdr is not L4, mark as IP-option >> instead */ >> - pkt_hdr->input_flags.ipopt = 1; >> - } >> - break; >> - } >> +int odp_packet_inflag_eth(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->input_flags.eth; >> } >> >> -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >> - odph_ipv4hdr_t *ipv4, size_t *offset_out) >> +void odp_packet_set_inflag_eth(odp_packet_t pkt, int val) >> { >> - uint8_t ihl; >> - uint16_t frag_offset; >> + odp_packet_hdr(pkt)->input_flags.eth = val; >> +} >> >> - ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); >> - if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN)) { >> - pkt_hdr->error_flags.ip_err = 1; >> - return 0; >> - } >> +int odp_packet_inflag_jumbo(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->input_flags.jumbo; >> +} >> >> - if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) { >> - pkt_hdr->input_flags.ipopt = 1; >> - return 0; >> - } >> +void odp_packet_set_inflag_jumbo(odp_packet_t pkt, int val) >> +{ >> + odp_packet_hdr(pkt)->input_flags.jumbo = val; >> +} >> >> - /* A packet is a fragment if: >> - * "more fragments" flag is set (all fragments except the last) >> - * OR >> - * "fragment offset" field is nonzero (all fragments except the >> first) >> - */ >> - frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); >> - if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) { >> - pkt_hdr->input_flags.ipfrag = 1; >> - return 0; >> - } >> +int odp_packet_inflag_vlan(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->input_flags.vlan; >> +} >> >> - if (ipv4->proto == ODPH_IPPROTO_ESP || >> - ipv4->proto == ODPH_IPPROTO_AH) { >> - pkt_hdr->input_flags.ipsec = 1; >> - return 0; >> - } >> +void odp_packet_set_inflag_vlan(odp_packet_t pkt, int val) >> +{ >> + odp_packet_hdr(pkt)->input_flags.vlan = val; >> +} >> >> - /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after >> return */ >> +int odp_packet_inflag_vlan_qinq(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->input_flags.vlan_qinq; >> +} >> >> - *offset_out = sizeof(uint32_t) * ihl; >> - return ipv4->proto; >> +void odp_packet_set_inflag_vlan_qinq(odp_packet_t pkt, int val) >> +{ >> + odp_packet_hdr(pkt)->input_flags.vlan_qinq = val; >> } >> >> -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >> - odph_ipv6hdr_t *ipv6, size_t *offset_out) >> +int odp_packet_inflag_snap(odp_packet_t pkt) >> { >> - if (ipv6->next_hdr == ODPH_IPPROTO_ESP || >> - ipv6->next_hdr == ODPH_IPPROTO_AH) { >> - pkt_hdr->input_flags.ipopt = 1; >> - pkt_hdr->input_flags.ipsec = 1; >> - return 0; >> - } >> + return odp_packet_hdr(pkt)->input_flags.snap; >> +} >> >> - if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { >> - pkt_hdr->input_flags.ipopt = 1; >> - pkt_hdr->input_flags.ipfrag = 1; >> - return 0; >> - } >> +void odp_packet_set_inflag_snap(odp_packet_t pkt, int val) >> +{ >> + odp_packet_hdr(pkt)->input_flags.snap = val; >> +} >> >> - /* Don't step through more extensions */ >> - *offset_out = ODPH_IPV6HDR_LEN; >> - return ipv6->next_hdr; >> +int odp_packet_inflag_arp(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->input_flags.arp; >> } >> >> -void odp_packet_print(odp_packet_t pkt) >> +void odp_packet_set_inflag_arp(odp_packet_t pkt, int val) >> { >> - int max_len = 512; >> - char str[max_len]; >> - int len = 0; >> - int n = max_len-1; >> - odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >> + odp_packet_hdr(pkt)->input_flags.arp = val; >> +} >> >> - len += snprintf(&str[len], n-len, "Packet "); >> - len += odp_buffer_snprint(&str[len], n-len, (odp_buffer_t) pkt); >> - len += snprintf(&str[len], n-len, >> - " input_flags 0x%x\n", hdr->input_flags.all); >> - len += snprintf(&str[len], n-len, >> - " error_flags 0x%x\n", hdr->error_flags.all); >> - len += snprintf(&str[len], n-len, >> - " output_flags 0x%x\n", hdr->output_flags.all); >> - len += snprintf(&str[len], n-len, >> - " frame_offset %u\n", hdr->frame_offset); >> - len += snprintf(&str[len], n-len, >> - " l2_offset %u\n", hdr->l2_offset); >> - len += snprintf(&str[len], n-len, >> - " l3_offset %u\n", hdr->l3_offset); >> - len += snprintf(&str[len], n-len, >> - " l4_offset %u\n", hdr->l4_offset); >> - len += snprintf(&str[len], n-len, >> - " frame_len %u\n", hdr->frame_len); >> - len += snprintf(&str[len], n-len, >> - " input %u\n", hdr->input); >> - str[len] = '\0'; >> +int odp_packet_inflag_ipv4(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->input_flags.ipv4; >> +} >> >> - printf("\n%s\n", str); >> +void odp_packet_set_inflag_ipv4(odp_packet_t pkt, int val) >> +{ >> + odp_packet_hdr(pkt)->input_flags.ipv4 = val; >> } >> >> -int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src) >> +int odp_packet_inflag_ipv6(odp_packet_t pkt) >> { >> - odp_packet_hdr_t *const pkt_hdr_dst = odp_packet_hdr(pkt_dst); >> - odp_packet_hdr_t *const pkt_hdr_src = odp_packet_hdr(pkt_src); >> - const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, >> buf_hdr); >> - uint8_t *start_src; >> - uint8_t *start_dst; >> - size_t len; >> + return odp_packet_hdr(pkt)->input_flags.ipv6; >> +} >> >> - if (pkt_dst == ODP_PACKET_INVALID || pkt_src == >> ODP_PACKET_INVALID) >> - return -1; >> +void odp_packet_set_inflag_ipv6(odp_packet_t pkt, int val) >> +{ >> + odp_packet_hdr(pkt)->input_flags.ipv6 = val; >> +} >> >> - if (pkt_hdr_dst->buf_hdr.size < >> - pkt_hdr_src->frame_len + pkt_hdr_src->frame_offset) >> - return -1; >> +int odp_packet_inflag_ipfrag(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->input_flags.ipfrag; >> +} >> + >> +void odp_packet_set_inflag_ipfrag(odp_packet_t pkt, int val) >> +{ >> + odp_packet_hdr(pkt)->input_flags.ipfrag = val; >> +} >> + >> +int odp_packet_inflag_ipopt(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->input_flags.ipopt; >> +} >> >> - /* Copy packet header */ >> - start_dst = (uint8_t *)pkt_hdr_dst + start_offset; >> - start_src = (uint8_t *)pkt_hdr_src + start_offset; >> - len = ODP_OFFSETOF(odp_packet_hdr_t, buf_data) - start_offset; >> - memcpy(start_dst, start_src, len); >> +void odp_packet_set_inflag_ipopt(odp_packet_t pkt, int val) >> +{ >> + odp_packet_hdr(pkt)->input_flags.ipopt = val; >> +} >> >> - /* Copy frame payload */ >> - start_dst = (uint8_t *)odp_packet_data(pkt_dst); >> - start_src = (uint8_t *)odp_packet_data(pkt_src); >> - len = pkt_hdr_src->frame_len; >> - memcpy(start_dst, start_src, len); >> +int odp_packet_inflag_ipsec(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->input_flags.ipsec; >> +} >> >> - /* Copy useful things from the buffer header */ >> - pkt_hdr_dst->buf_hdr.cur_offset = pkt_hdr_src->buf_hdr.cur_offset; >> +void odp_packet_set_inflag_ipsec(odp_packet_t pkt, int val) >> +{ >> + odp_packet_hdr(pkt)->input_flags.ipsec = val; >> +} >> >> - /* Create a copy of the scatter list */ >> - odp_buffer_copy_scatter(odp_packet_to_buffer(pkt_dst), >> - odp_packet_to_buffer(pkt_src)); >> +int odp_packet_inflag_udp(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->input_flags.udp; >> +} >> >> - return 0; >> +void odp_packet_set_inflag_udp(odp_packet_t pkt, int val) >> +{ >> + odp_packet_hdr(pkt)->input_flags.udp = val; >> } >> >> -void odp_packet_set_ctx(odp_packet_t pkt, const void *ctx) >> +int odp_packet_inflag_tcp(odp_packet_t pkt) >> { >> - odp_packet_hdr(pkt)->user_ctx = (intptr_t)ctx; >> + return odp_packet_hdr(pkt)->input_flags.tcp; >> } >> >> -void *odp_packet_get_ctx(odp_packet_t pkt) >> +void odp_packet_set_inflag_tcp(odp_packet_t pkt, int val) >> { >> - return (void *)(intptr_t)odp_packet_hdr(pkt)->user_ctx; >> + odp_packet_hdr(pkt)->input_flags.tcp = val; >> +} >> + >> +int odp_packet_inflag_tcpopt(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->input_flags.tcpopt; >> +} >> + >> +void odp_packet_set_inflag_tcpopt(odp_packet_t pkt, int val) >> +{ >> + odp_packet_hdr(pkt)->input_flags.tcpopt = val; >> +} >> + >> +int odp_packet_inflag_icmp(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->input_flags.icmp; >> +} >> + >> +void odp_packet_set_inflag_icmp(odp_packet_t pkt, int val) >> +{ >> + odp_packet_hdr(pkt)->input_flags.icmp = val; >> +} >> + >> +int odp_packet_is_valid(odp_packet_t pkt) >> +{ >> + odp_buffer_hdr_t *buf = validate_buf((odp_buffer_t)pkt); >> + >> + if (buf == NULL) >> + return 0; >> + else >> + return buf->type == ODP_BUFFER_TYPE_PACKET; >> +} >> + >> +int odp_packet_is_segmented(odp_packet_t pkt) >> +{ >> + return (odp_packet_hdr(pkt)->buf_hdr.segcount > 1); >> +} >> + >> +int odp_packet_segment_count(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->buf_hdr.segcount; >> +} >> + >> +size_t odp_packet_headroom(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->headroom; >> +} >> + >> +size_t odp_packet_tailroom(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->tailroom; >> +} >> + >> +int odp_packet_push_head(odp_packet_t pkt, size_t len) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + if (len > pkt_hdr->headroom) >> + return -1; >> + >> + push_head(pkt_hdr, len); >> + return 0; >> +} >> + >> +void *odp_packet_push_head_and_map(odp_packet_t pkt, size_t len, size_t >> *seglen) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + if (len > pkt_hdr->headroom) >> + return NULL; >> + >> + push_head(pkt_hdr, len); >> + >> + return buffer_map(&pkt_hdr->buf_hdr, 0, seglen, >> pkt_hdr->frame_len); >> +} >> + >> +int odp_packet_pull_head(odp_packet_t pkt, size_t len) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + if (len > pkt_hdr->frame_len) >> + return -1; >> + >> + pull_head(pkt_hdr, len); >> + return 0; >> +} >> + >> +void *odp_packet_pull_head_and_map(odp_packet_t pkt, size_t len, size_t >> *seglen) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + if (len > pkt_hdr->frame_len) >> + return NULL; >> + >> + pull_head(pkt_hdr, len); >> + return buffer_map(&pkt_hdr->buf_hdr, 0, seglen, >> pkt_hdr->frame_len); >> +} >> + >> +int odp_packet_push_tail(odp_packet_t pkt, size_t len) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + if (len > pkt_hdr->tailroom) >> + return -1; >> + >> + push_tail(pkt_hdr, len); >> + return 0; >> +} >> + >> +void *odp_packet_push_tail_and_map(odp_packet_t pkt, size_t len, size_t >> *seglen) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + size_t origin = pkt_hdr->frame_len; >> + >> + if (len > pkt_hdr->tailroom) >> + return NULL; >> + >> + push_tail(pkt_hdr, len); >> + >> + return buffer_map(&pkt_hdr->buf_hdr, origin, seglen, >> + pkt_hdr->frame_len); >> +} >> + >> +int odp_packet_pull_tail(odp_packet_t pkt, size_t len) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + if (len > pkt_hdr->frame_len) >> + return -1; >> + >> + pull_tail(pkt_hdr, len); >> + return 0; >> +} >> + >> +void odp_packet_print(odp_packet_t pkt) >> +{ >> + int max_len = 512; >> + char str[max_len]; >> + int len = 0; >> + int n = max_len-1; >> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >> + >> + len += snprintf(&str[len], n-len, "Packet "); >> + len += odp_buffer_snprint(&str[len], n-len, (odp_buffer_t) pkt); >> + len += snprintf(&str[len], n-len, >> + " input_flags 0x%x\n", hdr->input_flags.all); >> + len += snprintf(&str[len], n-len, >> + " error_flags 0x%x\n", hdr->error_flags.all); >> + len += snprintf(&str[len], n-len, >> + " output_flags 0x%x\n", hdr->output_flags.all); >> + len += snprintf(&str[len], n-len, >> + " l2_offset %u\n", hdr->l2_offset); >> + len += snprintf(&str[len], n-len, >> + " l3_offset %u\n", hdr->l3_offset); >> + len += snprintf(&str[len], n-len, >> + " l3_len %u\n", hdr->l3_len); >> + len += snprintf(&str[len], n-len, >> + " l3_protocol 0x%x\n", hdr->l3_protocol); >> + len += snprintf(&str[len], n-len, >> + " l4_offset %u\n", hdr->l4_offset); >> + len += snprintf(&str[len], n-len, >> + " l4_len %u\n", hdr->l4_len); >> + len += snprintf(&str[len], n-len, >> + " l4_protocol %u\n", hdr->l4_protocol); >> + len += snprintf(&str[len], n-len, >> + " payload_offset %u\n", hdr->payload_offset); >> + len += snprintf(&str[len], n-len, >> + " frame_len %u\n", hdr->frame_len); >> + str[len] = '\0'; >> + >> + printf("\n%s\n", str); >> +} >> + >> +int odp_packet_copy_to_packet(odp_packet_t dstpkt, size_t dstoffset, >> + odp_packet_t srcpkt, size_t srcoffset, >> + size_t len) >> +{ >> + void *dstmap; >> + void *srcmap; >> + size_t cpylen, minseg, dstseglen, srcseglen; >> + >> + while (len > 0) { >> + dstmap = odp_packet_offset_map(dstpkt, dstoffset, >> &dstseglen); >> + srcmap = odp_packet_offset_map(srcpkt, srcoffset, >> &srcseglen); >> + if (dstmap == NULL || srcmap == NULL) >> + return -1; >> + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; >> + cpylen = len > minseg ? minseg : len; >> + memcpy(dstmap, srcmap, cpylen); >> + srcoffset += cpylen; >> + dstoffset += cpylen; >> + len -= cpylen; >> + } >> + >> + return 0; >> +} >> + >> +int odp_packet_copy_to_memory(void *dstmem, odp_packet_t srcpkt, >> + size_t srcoffset, size_t dstlen) >> +{ >> + void *mapaddr; >> + size_t seglen, cpylen; >> + uint8_t *dstaddr = (uint8_t *)dstmem; >> + >> + while (dstlen > 0) { >> + mapaddr = odp_packet_offset_map(srcpkt, srcoffset, >> &seglen); >> + if (mapaddr == NULL) >> + return -1; >> + cpylen = dstlen > seglen ? seglen : dstlen; >> + memcpy(dstaddr, mapaddr, cpylen); >> + srcoffset += cpylen; >> + dstaddr += cpylen; >> + dstlen -= cpylen; >> + } >> + >> + return 0; >> +} >> + >> +int odp_packet_copy_from_memory(odp_packet_t dstpkt, size_t dstoffset, >> + void *srcmem, size_t srclen) >> +{ >> + void *mapaddr; >> + size_t seglen, cpylen; >> + uint8_t *srcaddr = (uint8_t *)srcmem; >> + >> + while (srclen > 0) { >> + mapaddr = odp_packet_offset_map(dstpkt, dstoffset, >> &seglen); >> + if (mapaddr == NULL) >> + return -1; >> + cpylen = srclen > seglen ? seglen : srclen; >> + memcpy(mapaddr, srcaddr, cpylen); >> + dstoffset += cpylen; >> + srcaddr += cpylen; >> + srclen -= cpylen; >> + } >> + >> + return 0; >> +} >> + >> +odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool) >> +{ >> + size_t pktlen = odp_packet_len(pkt); >> + const size_t meta_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, >> buf_hdr); >> + odp_packet_t newpkt = odp_packet_alloc_len(pool, pktlen); >> + odp_packet_hdr_t *newhdr, *srchdr; >> + uint8_t *newstart, *srcstart; >> + >> + if (newpkt != ODP_PACKET_INVALID) { >> + /* Must copy meta data first, followed by packet data */ >> + srchdr = odp_packet_hdr(pkt); >> + newhdr = odp_packet_hdr(newpkt); >> + newstart = (uint8_t *)newhdr + meta_offset; >> + srcstart = (uint8_t *)srchdr + meta_offset; >> + >> + memcpy(newstart, srcstart, >> + sizeof(odp_packet_hdr_t) - meta_offset); >> + >> + if (odp_packet_copy_to_packet(newpkt, 0, pkt, 0, pktlen) >> != 0) { >> + odp_packet_free(newpkt); >> + newpkt = ODP_PACKET_INVALID; >> + } >> + } >> + >> + return newpkt; >> +} >> + >> +odp_packet_t odp_packet_clone(odp_packet_t pkt) >> +{ >> + return odp_packet_copy(pkt, >> odp_packet_hdr(pkt)->buf_hdr.pool_hdl); >> +} >> + >> +odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool) >> +{ >> + if (odp_pool_to_entry(pool)->s.params.buf_type != >> + ODP_BUFFER_TYPE_PACKET) >> + return ODP_PACKET_INVALID; >> + >> + return (odp_packet_t)buffer_alloc(pool, 0); >> +} >> + >> +odp_packet_t odp_packet_alloc_len(odp_buffer_pool_t pool, size_t len) >> +{ >> + if (odp_pool_to_entry(pool)->s.params.buf_type != >> + ODP_BUFFER_TYPE_PACKET) >> + return ODP_PACKET_INVALID; >> + >> + return (odp_packet_t)buffer_alloc(pool, len); >> +} >> + >> +void odp_packet_free(odp_packet_t pkt) >> +{ >> + odp_buffer_free((odp_buffer_t)pkt); >> +} >> + >> +odp_packet_t odp_packet_split(odp_packet_t pkt, size_t offset, >> + size_t hr, size_t tr) >> +{ >> + size_t len = odp_packet_len(pkt); >> + odp_buffer_pool_t pool = odp_packet_pool(pkt); >> + size_t pool_hr = odp_buffer_pool_headroom(pool); >> + size_t pkt_tr = odp_packet_tailroom(pkt); >> + odp_packet_t splitpkt; >> + size_t splitlen = len - offset; >> + >> + if (offset >= len) >> + return ODP_PACKET_INVALID; >> + >> + /* Erratum: We don't handle this rare corner case */ >> + if (tr > pkt_tr + splitlen) >> + return ODP_PACKET_INVALID; >> + >> + if (hr > pool_hr) >> + splitlen += (hr - pool_hr); >> + >> + splitpkt = odp_packet_alloc_len(pool, splitlen); >> + >> + if (splitpkt != ODP_PACKET_INVALID) { >> + if (hr > pool_hr) { >> + odp_packet_pull_head(splitpkt, hr - pool_hr); >> + splitlen -= (hr - pool_hr); >> + } >> + odp_packet_copy_to_packet(splitpkt, 0, >> + pkt, offset, splitlen); >> + odp_packet_pull_tail(pkt, splitlen); >> + } >> + >> + return splitpkt; >> +} >> + >> +odp_packet_t odp_packet_join(odp_packet_t pkt1, odp_packet_t pkt2) >> +{ >> + size_t len1 = odp_packet_len(pkt1); >> + size_t len2 = odp_packet_len(pkt2); >> + odp_packet_t joinpkt; >> + void *udata1, *udata_join; >> + size_t udata_size1, udata_size_join; >> + >> + /* Optimize if pkt1 is big enough to hold pkt2 */ >> + if (odp_packet_push_tail(pkt1, len2) == 0) { >> + odp_packet_copy_to_packet(pkt1, len1, >> + pkt2, 0, len2); >> + odp_packet_free(pkt2); >> + return pkt1; >> + } >> + >> + /* Otherwise join into a new packet */ >> + joinpkt = odp_packet_alloc_len(odp_packet_pool(pkt1), >> + len1 + len2); >> + >> + if (joinpkt != ODP_PACKET_INVALID) { >> + odp_packet_copy_to_packet(joinpkt, 0, pkt1, 0, len1); >> + odp_packet_copy_to_packet(joinpkt, len1, pkt2, 0, len2); >> + >> + /* Copy user metadata if present */ >> + udata1 = odp_packet_udata(pkt1, &udata_size1); >> + udata_join = odp_packet_udata(joinpkt, &udata_size_join); >> + >> + if (udata1 != NULL && udata_join != NULL) >> + memcpy(udata_join, udata1, >> + udata_size_join > udata_size1 ? >> + udata_size1 : udata_size_join); >> + >> + odp_packet_free(pkt1); >> + odp_packet_free(pkt2); >> + } >> + >> + return joinpkt; >> +} >> + >> +uint32_t odp_packet_refcount(odp_packet_t pkt) >> +{ >> + return odp_buffer_refcount(&odp_packet_hdr(pkt)->buf_hdr); >> +} >> + >> +uint32_t odp_packet_incr_refcount(odp_packet_t pkt, uint32_t val) >> +{ >> + return odp_buffer_incr_refcount(&odp_packet_hdr(pkt)->buf_hdr, >> val); >> +} >> + >> +uint32_t odp_packet_decr_refcount(odp_packet_t pkt, uint32_t val) >> +{ >> + return odp_buffer_decr_refcount(&odp_packet_hdr(pkt)->buf_hdr, >> val); >> +} >> + >> +/** >> + * Parser helper function for IPv4 >> + */ >> +static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >> + uint8_t **parseptr, size_t *offset) >> +{ >> + odph_ipv4hdr_t *ipv4 = (odph_ipv4hdr_t *)*parseptr; >> + uint8_t ver = ODPH_IPV4HDR_VER(ipv4->ver_ihl); >> + uint8_t ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); >> + uint16_t frag_offset; >> + >> + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv4->tot_len); >> + >> + if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN) || >> + odp_unlikely(ver != 4) || >> + (pkt_hdr->l3_len > pkt_hdr->frame_len - *offset)) { >> + pkt_hdr->error_flags.ip_err = 1; >> + return 0; >> + } >> + >> + *offset += ihl * 4; >> + *parseptr += ihl * 4; >> + >> + if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) >> + pkt_hdr->input_flags.ipopt = 1; >> + >> + /* A packet is a fragment if: >> + * "more fragments" flag is set (all fragments except the last) >> + * OR >> + * "fragment offset" field is nonzero (all fragments except the >> first) >> + */ >> + frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); >> + if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) >> + pkt_hdr->input_flags.ipfrag = 1; >> + >> + if (ipv4->proto == ODPH_IPPROTO_ESP || >> + ipv4->proto == ODPH_IPPROTO_AH) { >> + pkt_hdr->input_flags.ipsec = 1; >> + return 0; >> + } >> + >> + /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after >> return */ >> + >> + *offset = sizeof(uint32_t) * ihl; >> + return ipv4->proto; >> +} >> + >> +/** >> + * Parser helper function for IPv6 >> + */ >> +static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >> + uint8_t **parseptr, size_t *offset) >> +{ >> + odph_ipv6hdr_t *ipv6 = (odph_ipv6hdr_t *)*parseptr; >> + odph_ipv6hdr_ext_t *ipv6ext; >> + >> + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv6->payload_len); >> + >> + /* Basic sanity checks on IPv6 header */ >> + if (ipv6->ver != 6 || >> + pkt_hdr->l3_len > pkt_hdr->frame_len - *offset) { >> + pkt_hdr->error_flags.ip_err = 1; >> + return 0; >> + } >> + >> + /* Skip past IPv6 header */ >> + *offset += sizeof(odph_ipv6hdr_t); >> + *parseptr += sizeof(odph_ipv6hdr_t); >> + >> + >> + /* Skip past any IPv6 extension headers */ >> + if (ipv6->next_hdr == ODPH_IPPROTO_HOPOPTS || >> + ipv6->next_hdr == ODPH_IPPROTO_ROUTE) { >> + pkt_hdr->input_flags.ipopt = 1; >> + >> + do { >> + ipv6ext = (odph_ipv6hdr_ext_t *)*parseptr; >> + uint16_t extlen = 8 + ipv6ext->ext_len * 8; >> + >> + *offset += extlen; >> + *parseptr += extlen; >> + } while ((ipv6ext->next_hdr == ODPH_IPPROTO_HOPOPTS || >> + ipv6ext->next_hdr == ODPH_IPPROTO_ROUTE) && >> + *offset < pkt_hdr->frame_len); >> + >> + if (*offset >= pkt_hdr->l3_offset + ipv6->payload_len) { >> + pkt_hdr->error_flags.ip_err = 1; >> + return 0; >> + } >> + >> + if (ipv6ext->next_hdr == ODPH_IPPROTO_FRAG) >> + pkt_hdr->input_flags.ipfrag = 1; >> + >> + return ipv6ext->next_hdr; >> + } >> + >> + if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { >> + pkt_hdr->input_flags.ipopt = 1; >> + pkt_hdr->input_flags.ipfrag = 1; >> + } >> + >> + return ipv6->next_hdr; >> +} >> + >> +/** >> + * Parser helper function for TCP >> + */ >> +static inline void parse_tcp(odp_packet_hdr_t *pkt_hdr, >> + uint8_t **parseptr, size_t *offset) >> +{ >> + odph_tcphdr_t *tcp = (odph_tcphdr_t *)*parseptr; >> + >> + if (tcp->hl < sizeof(odph_tcphdr_t)/sizeof(uint32_t)) >> + pkt_hdr->error_flags.tcp_err = 1; >> + else if ((uint32_t)tcp->hl * 4 > sizeof(odph_tcphdr_t)) >> + pkt_hdr->input_flags.tcpopt = 1; >> + >> + pkt_hdr->l4_len = pkt_hdr->l3_len + >> + pkt_hdr->l3_offset - pkt_hdr->l4_offset; >> + >> + *offset += sizeof(odph_tcphdr_t); >> + *parseptr += sizeof(odph_tcphdr_t); >> +} >> + >> +/** >> + * Parser helper function for UDP >> + */ >> +static inline void parse_udp(odp_packet_hdr_t *pkt_hdr, >> + uint8_t **parseptr, size_t *offset) >> +{ >> + odph_udphdr_t *udp = (odph_udphdr_t *)*parseptr; >> + uint32_t udplen = odp_be_to_cpu_16(udp->length); >> + >> + if (udplen < sizeof(odph_udphdr_t) || >> + udplen > (pkt_hdr->l3_len + >> + pkt_hdr->l3_offset - pkt_hdr->l4_offset)) { >> + pkt_hdr->error_flags.udp_err = 1; >> + } >> + >> + pkt_hdr->l4_len = udplen; >> + >> + *offset += sizeof(odph_udphdr_t); >> + *parseptr += sizeof(odph_udphdr_t); >> +} >> + >> +/** >> + * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP >> + * >> + * Internal function: caller is resposible for passing only valid packet >> handles >> + * , lengths and offsets (usually done&called in packet input). >> + * >> + * @param pkt Packet handle >> + */ >> +int odp_packet_parse(odp_packet_t pkt) >> +{ >> + odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >> + odph_ethhdr_t *eth; >> + odph_vlanhdr_t *vlan; >> + uint16_t ethtype; >> + uint8_t *parseptr; >> + size_t offset, seglen; >> + uint8_t ip_proto = 0; >> + >> + /* Reset parser metadata for new parse */ >> + pkt_hdr->error_flags.all = 0; >> + pkt_hdr->input_flags.all = 0; >> + pkt_hdr->output_flags.all = 0; >> + pkt_hdr->l2_offset = 0; >> + pkt_hdr->l3_offset = 0; >> + pkt_hdr->l4_offset = 0; >> + pkt_hdr->payload_offset = 0; >> + pkt_hdr->vlan_s_tag = 0; >> + pkt_hdr->vlan_c_tag = 0; >> + pkt_hdr->l3_protocol = 0; >> + pkt_hdr->l4_protocol = 0; >> + >> + /* We only support Ethernet for now */ >> + pkt_hdr->input_flags.eth = 1; >> + >> + if (odp_unlikely(pkt_hdr->frame_len < ODPH_ETH_LEN_MIN)) { >> + pkt_hdr->error_flags.frame_len = 1; >> + goto parse_exit; >> + } else if (pkt_hdr->frame_len > ODPH_ETH_LEN_MAX) { >> + pkt_hdr->input_flags.jumbo = 1; >> + } >> + >> + /* Assume valid L2 header, no CRC/FCS check in SW */ >> + pkt_hdr->input_flags.l2 = 1; >> + >> + eth = (odph_ethhdr_t *)odp_packet_map(pkt, &seglen); >> + offset = sizeof(odph_ethhdr_t); >> + parseptr = (uint8_t *)ð->type; >> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); >> + >> + /* Parse the VLAN header(s), if present */ >> + if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { >> + pkt_hdr->input_flags.vlan_qinq = 1; >> + pkt_hdr->input_flags.vlan = 1; >> + vlan = (odph_vlanhdr_t *)(void *)parseptr; >> + pkt_hdr->vlan_s_tag = ((ethtype << 16) | >> + odp_be_to_cpu_16(vlan->tci)); >> + offset += sizeof(odph_vlanhdr_t); >> + parseptr += sizeof(odph_vlanhdr_t); >> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >> *)parseptr)); >> + } >> + >> + if (ethtype == ODPH_ETHTYPE_VLAN) { >> + pkt_hdr->input_flags.vlan = 1; >> + vlan = (odph_vlanhdr_t *)(void *)parseptr; >> + pkt_hdr->vlan_c_tag = ((ethtype << 16) | >> + odp_be_to_cpu_16(vlan->tci)); >> + offset += sizeof(odph_vlanhdr_t); >> + parseptr += sizeof(odph_vlanhdr_t); >> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >> *)parseptr)); >> + } >> + >> + /* Check for SNAP vs. DIX */ >> + if (ethtype < ODPH_ETH_LEN_MAX) { >> + pkt_hdr->input_flags.snap = 1; >> + if (ethtype > pkt_hdr->frame_len - offset) { >> + pkt_hdr->error_flags.snap_len = 1; >> + goto parse_exit; >> + } >> + offset += 8; >> + parseptr += 8; >> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >> *)parseptr)); >> + } >> + >> + /* Consume Ethertype for Layer 3 parse */ >> + parseptr += 2; >> + >> + /* Set l3_offset+flag only for known ethtypes */ >> + pkt_hdr->input_flags.l3 = 1; >> + pkt_hdr->l3_offset = offset; >> + pkt_hdr->l3_protocol = ethtype; >> + >> + /* Parse Layer 3 headers */ >> + switch (ethtype) { >> + case ODPH_ETHTYPE_IPV4: >> + pkt_hdr->input_flags.ipv4 = 1; >> + ip_proto = parse_ipv4(pkt_hdr, &parseptr, &offset); >> + break; >> + >> + case ODPH_ETHTYPE_IPV6: >> + pkt_hdr->input_flags.ipv6 = 1; >> + ip_proto = parse_ipv6(pkt_hdr, &parseptr, &offset); >> + break; >> + >> + case ODPH_ETHTYPE_ARP: >> + pkt_hdr->input_flags.arp = 1; >> + ip_proto = 255; /* Reserved invalid by IANA */ >> + break; >> + >> + default: >> + pkt_hdr->input_flags.l3 = 0; >> + ip_proto = 255; /* Reserved invalid by IANA */ >> + } >> + >> + /* Set l4_offset+flag only for known ip_proto */ >> + pkt_hdr->input_flags.l4 = 1; >> + pkt_hdr->l4_offset = offset; >> + pkt_hdr->l4_protocol = ip_proto; >> + >> + /* Parse Layer 4 headers */ >> + switch (ip_proto) { >> + case ODPH_IPPROTO_ICMP: >> + pkt_hdr->input_flags.icmp = 1; >> + break; >> + >> + case ODPH_IPPROTO_TCP: >> + pkt_hdr->input_flags.tcp = 1; >> + parse_tcp(pkt_hdr, &parseptr, &offset); >> + break; >> + >> + case ODPH_IPPROTO_UDP: >> + pkt_hdr->input_flags.udp = 1; >> + parse_udp(pkt_hdr, &parseptr, &offset); >> + break; >> + >> + case ODPH_IPPROTO_AH: >> + case ODPH_IPPROTO_ESP: >> + pkt_hdr->input_flags.ipsec = 1; >> + break; >> + >> + default: >> + pkt_hdr->input_flags.l4 = 0; >> + break; >> + } >> + >> + /* >> + * Anything beyond what we parse here is considered payload. >> + * Note: Payload is really only relevant for TCP and UDP. For >> + * all other protocols, the payload offset will point to the >> + * final header (ARP, ICMP, AH, ESP, or IP Fragment. >> + */ >> + pkt_hdr->payload_offset = offset; >> + >> +parse_exit: >> + return pkt_hdr->error_flags.all != 0; >> } >> diff --git a/platform/linux-generic/odp_packet_flags.c >> b/platform/linux-generic/odp_packet_flags.c >> deleted file mode 100644 >> index 06fdeed..0000000 >> --- a/platform/linux-generic/odp_packet_flags.c >> +++ /dev/null >> @@ -1,202 +0,0 @@ >> -/* Copyright (c) 2014, Linaro Limited >> - * All rights reserved. >> - * >> - * SPDX-License-Identifier: BSD-3-Clause >> - */ >> - >> -#include <odp_packet_flags.h> >> -#include <odp_packet_internal.h> >> - >> - >> -int odp_packet_error(odp_packet_t pkt) >> -{ >> - return (odp_packet_hdr(pkt)->error_flags.all != 0); >> -} >> - >> -/* Get Error Flags */ >> - >> -int odp_packet_errflag_frame_len(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->error_flags.frame_len; >> -} >> - >> -/* Get Input Flags */ >> - >> -int odp_packet_inflag_l2(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.l2; >> -} >> - >> -int odp_packet_inflag_l3(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.l3; >> -} >> - >> -int odp_packet_inflag_l4(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.l4; >> -} >> - >> -int odp_packet_inflag_eth(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.eth; >> -} >> - >> -int odp_packet_inflag_jumbo(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.jumbo; >> -} >> - >> -int odp_packet_inflag_vlan(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.vlan; >> -} >> - >> -int odp_packet_inflag_vlan_qinq(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.vlan_qinq; >> -} >> - >> -int odp_packet_inflag_arp(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.arp; >> -} >> - >> -int odp_packet_inflag_ipv4(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.ipv4; >> -} >> - >> -int odp_packet_inflag_ipv6(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.ipv6; >> -} >> - >> -int odp_packet_inflag_ipfrag(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.ipfrag; >> -} >> - >> -int odp_packet_inflag_ipopt(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.ipopt; >> -} >> - >> -int odp_packet_inflag_ipsec(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.ipsec; >> -} >> - >> -int odp_packet_inflag_udp(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.udp; >> -} >> - >> -int odp_packet_inflag_tcp(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.tcp; >> -} >> - >> -int odp_packet_inflag_sctp(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.sctp; >> -} >> - >> -int odp_packet_inflag_icmp(odp_packet_t pkt) >> -{ >> - return odp_packet_hdr(pkt)->input_flags.icmp; >> -} >> - >> -/* Set Output Flags */ >> - >> -void odp_packet_outflag_l4_chksum(odp_packet_t pkt) >> -{ >> - odp_packet_hdr(pkt)->output_flags.l4_chksum = 1; >> -} >> - >> -/* Set Input Flags */ >> - >> -void odp_packet_set_inflag_l2(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.l2 = val; >> -} >> - >> -void odp_packet_set_inflag_l3(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.l3 = val; >> -} >> - >> -void odp_packet_set_inflag_l4(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.l4 = val; >> -} >> - >> -void odp_packet_set_inflag_eth(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.eth = val; >> -} >> - >> -void odp_packet_set_inflag_jumbo(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.jumbo = val; >> -} >> - >> -void odp_packet_set_inflag_vlan(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.vlan = val; >> -} >> - >> -void odp_packet_set_inflag_vlan_qinq(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.vlan_qinq = val; >> -} >> - >> -void odp_packet_set_inflag_arp(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.arp = val; >> -} >> - >> -void odp_packet_set_inflag_ipv4(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.ipv4 = val; >> -} >> - >> -void odp_packet_set_inflag_ipv6(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.ipv6 = val; >> -} >> - >> -void odp_packet_set_inflag_ipfrag(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.ipfrag = val; >> -} >> - >> -void odp_packet_set_inflag_ipopt(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.ipopt = val; >> -} >> - >> -void odp_packet_set_inflag_ipsec(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.ipsec = val; >> -} >> - >> -void odp_packet_set_inflag_udp(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.udp = val; >> -} >> - >> -void odp_packet_set_inflag_tcp(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.tcp = val; >> -} >> - >> -void odp_packet_set_inflag_sctp(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.sctp = val; >> -} >> - >> -void odp_packet_set_inflag_icmp(odp_packet_t pkt, int val) >> -{ >> - odp_packet_hdr(pkt)->input_flags.icmp = val; >> -} >> -- >> 1.8.3.2 >> >> >> _______________________________________________ >> lng-odp mailing list >> lng-odp@lists.linaro.org >> http://lists.linaro.org/mailman/listinfo/lng-odp >> > >
I am okay with the behaviour for "init_params" value, my concern is with "params" value. Since this params deals with the number of buffer in the pool and buf_size. I would like to return an error if the params value is NULL. Regards, Bala On 11 November 2014 22:09, Bill Fischofer <bill.fischofer@linaro.org> wrote: > The API allows the init_params to be NULL if user meta data is not > required. I just added the corresponding support to the params to have the > same behavior. I have no problem failing the call if that parameter is > missing. > > Bill > > On Tue, Nov 11, 2014 at 10:34 AM, Bala Manoharan < > bala.manoharan@linaro.org> wrote: > >> >> >> On 11 November 2014 09:33, Bill Fischofer <bill.fischofer@linaro.org> >> wrote: >> >>> Signed-off-by: Bill Fischofer <bill.fischofer@linaro.org> >>> --- >>> platform/linux-generic/odp_buffer.c | 263 ++++++- >>> platform/linux-generic/odp_buffer_pool.c | 661 +++++++--------- >>> platform/linux-generic/odp_packet.c | 1200 >>> +++++++++++++++++++++++------ >>> platform/linux-generic/odp_packet_flags.c | 202 ----- >>> 4 files changed, 1455 insertions(+), 871 deletions(-) >>> delete mode 100644 platform/linux-generic/odp_packet_flags.c >>> >>> diff --git a/platform/linux-generic/odp_buffer.c >>> b/platform/linux-generic/odp_buffer.c >>> index e54e0e7..e47c722 100644 >>> --- a/platform/linux-generic/odp_buffer.c >>> +++ b/platform/linux-generic/odp_buffer.c >>> @@ -5,46 +5,259 @@ >>> */ >>> >>> #include <odp_buffer.h> >>> -#include <odp_buffer_internal.h> >>> #include <odp_buffer_pool_internal.h> >>> +#include <odp_buffer_internal.h> >>> +#include <odp_buffer_inlines.h> >>> >>> #include <string.h> >>> #include <stdio.h> >>> >>> +void *odp_buffer_offset_map(odp_buffer_t buf, size_t offset, size_t >>> *seglen) >>> +{ >>> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >>> >>> -void *odp_buffer_addr(odp_buffer_t buf) >>> + if (offset > buf_hdr->size) >>> + return NULL; >>> + >>> + return buffer_map(buf_hdr, offset, seglen, buf_hdr->size); >>> +} >>> + >>> +void odp_buffer_offset_unmap(odp_buffer_t buf ODP_UNUSED, >>> + size_t offset ODP_UNUSED) >>> { >>> - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >>> +} >>> >>> - return hdr->addr; >>> +size_t odp_buffer_segment_count(odp_buffer_t buf) >>> +{ >>> + return odp_buf_to_hdr(buf)->segcount; >>> } >>> >>> +int odp_buffer_is_segmented(odp_buffer_t buf) >>> +{ >>> + return odp_buf_to_hdr(buf)->segcount > 1; >>> +} >>> >>> -size_t odp_buffer_size(odp_buffer_t buf) >>> +odp_buffer_segment_t odp_buffer_segment_by_index(odp_buffer_t buf, >>> + size_t ndx) >>> { >>> - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >>> + return buffer_segment(odp_buf_to_hdr(buf), ndx); >>> +} >>> >>> - return hdr->size; >>> +odp_buffer_segment_t odp_buffer_segment_next(odp_buffer_t buf, >>> + odp_buffer_segment_t seg) >>> +{ >>> + return segment_next(odp_buf_to_hdr(buf), seg); >>> } >>> >>> +void *odp_buffer_segment_map(odp_buffer_t buf, odp_buffer_segment_t seg, >>> + size_t *seglen) >>> +{ >>> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >>> >>> -int odp_buffer_type(odp_buffer_t buf) >>> + return segment_map(buf_hdr, seg, seglen, buf_hdr->size); >>> +} >>> + >>> +void odp_buffer_segment_unmap(odp_buffer_segment_t seg ODP_UNUSED) >>> { >>> - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >>> +} >>> + >>> +odp_buffer_t odp_buffer_split(odp_buffer_t buf, size_t offset) >>> +{ >>> + size_t size = odp_buffer_size(buf); >>> + odp_buffer_pool_t pool = odp_buffer_pool(buf); >>> + odp_buffer_t splitbuf; >>> + size_t splitsize = size - offset; >>> + >>> + if (offset >= size) >>> + return ODP_BUFFER_INVALID; >>> + >>> + splitbuf = buffer_alloc(pool, splitsize); >>> + >>> + if (splitbuf != ODP_BUFFER_INVALID) { >>> + buffer_copy_to_buffer(splitbuf, 0, buf, splitsize, >>> splitsize); >>> + odp_buffer_trim(buf, splitsize); >>> + } >>> >>> - return hdr->type; >>> + return splitbuf; >>> } >>> >>> +odp_buffer_t odp_buffer_join(odp_buffer_t buf1, odp_buffer_t buf2) >>> +{ >>> + size_t size1 = odp_buffer_size(buf1); >>> + size_t size2 = odp_buffer_size(buf2); >>> + odp_buffer_t joinbuf; >>> + void *udata1, *udata_join; >>> + size_t udata_size1, udata_size_join; >>> >>> -int odp_buffer_is_valid(odp_buffer_t buf) >>> + joinbuf = buffer_alloc(odp_buffer_pool(buf1), size1 + size2); >>> + >>> + if (joinbuf != ODP_BUFFER_INVALID) { >>> + buffer_copy_to_buffer(joinbuf, 0, buf1, 0, size1); >>> + buffer_copy_to_buffer(joinbuf, size1, buf2, 0, size2); >>> + >>> + /* Copy user metadata if present */ >>> + udata1 = odp_buffer_udata(buf1, &udata_size1); >>> + udata_join = odp_buffer_udata(joinbuf, &udata_size_join); >>> + >>> + if (udata1 != NULL && udata_join != NULL) >>> + memcpy(udata_join, udata1, >>> + udata_size_join > udata_size1 ? >>> + udata_size1 : udata_size_join); >>> + >>> + odp_buffer_free(buf1); >>> + odp_buffer_free(buf2); >>> + } >>> + >>> + return joinbuf; >>> +} >>> + >>> +odp_buffer_t odp_buffer_trim(odp_buffer_t buf, size_t bytes) >>> { >>> - odp_buffer_bits_t handle; >>> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >>> + >>> + if (bytes >= buf_hdr->size) >>> + return ODP_BUFFER_INVALID; >>> + >>> + buf_hdr->size -= bytes; >>> + size_t bufsegs = buf_hdr->size / buf_hdr->segsize; >>> + >>> + if (buf_hdr->size % buf_hdr->segsize > 0) >>> + bufsegs++; >>> + >>> + pool_entry_t *pool = odp_pool_to_entry(buf_hdr->pool_hdl); >>> >>> - handle.u32 = buf; >>> + /* Return excess segments back to block list */ >>> + while (bufsegs > buf_hdr->segcount) >>> + ret_blk(&pool->s, buf_hdr->addr[--buf_hdr->segcount]); >>> >>> - return (handle.index != ODP_BUFFER_INVALID_INDEX); >>> + return buf; >>> } >>> >>> +odp_buffer_t odp_buffer_extend(odp_buffer_t buf, size_t bytes) >>> +{ >>> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >>> + >>> + size_t lastseg = buf_hdr->size % buf_hdr->segsize; >>> + >>> + if (bytes <= buf_hdr->segsize - lastseg) { >>> + buf_hdr->size += bytes; >>> + return buf; >>> + } >>> + >>> + pool_entry_t *pool = odp_pool_to_entry(buf_hdr->pool_hdl); >>> + int extsize = buf_hdr->size + bytes; >>> + >>> + /* Extend buffer by adding additional segments to it */ >>> + if (extsize > ODP_CONFIG_BUF_MAX_SIZE || >>> pool->s.flags.unsegmented) >>> + return ODP_BUFFER_INVALID; >>> + >>> + size_t segcount = buf_hdr->segcount; >>> + size_t extsegs = extsize / buf_hdr->segsize; >>> + >>> + if (extsize % buf_hdr->segsize > 0) >>> + extsize++; >>> + >>> + while (extsegs > buf_hdr->segcount) { >>> + void *newblk = get_blk(&pool->s); >>> + >>> + /* If we fail to get all the blocks we need, back out */ >>> + if (newblk == NULL) { >>> + while (segcount > buf_hdr->segcount) >>> + ret_blk(&pool->s, >>> buf_hdr->addr[--segcount]); >>> + return ODP_BUFFER_INVALID; >>> + } >>> + >>> + buf_hdr->addr[segcount++] = newblk; >>> + } >>> + >>> + buf_hdr->segcount = segcount; >>> + buf_hdr->size = extsize; >>> + >>> + return buf; >>> +} >>> + >>> +odp_buffer_t odp_buffer_clone(odp_buffer_t buf) >>> +{ >>> + return odp_buffer_copy(buf, odp_buf_to_hdr(buf)->pool_hdl); >>> +} >>> + >>> +odp_buffer_t odp_buffer_copy(odp_buffer_t buf, odp_buffer_pool_t pool) >>> +{ >>> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >>> + size_t len = buf_hdr->size; >>> + odp_buffer_t cpybuf = buffer_alloc(pool, len); >>> + >>> + if (cpybuf == ODP_BUFFER_INVALID) >>> + return ODP_BUFFER_INVALID; >>> + >>> + if (buffer_copy_to_buffer(cpybuf, 0, buf, 0, len) == 0) >>> + return cpybuf; >>> + >>> + odp_buffer_free(cpybuf); >>> + return ODP_BUFFER_INVALID; >>> +} >>> + >>> +int buffer_copy_to_buffer(odp_buffer_t dstbuf, size_t dstoffset, >>> + odp_buffer_t srcbuf, size_t srcoffset, >>> + size_t len) >>> +{ >>> + void *dstmap; >>> + void *srcmap; >>> + size_t cpylen, minseg, dstseglen, srcseglen; >>> + >>> + while (len > 0) { >>> + dstmap = odp_buffer_offset_map(dstbuf, dstoffset, >>> &dstseglen); >>> + srcmap = odp_buffer_offset_map(srcbuf, srcoffset, >>> &srcseglen); >>> + if (dstmap == NULL || srcmap == NULL) >>> + return -1; >>> + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; >>> + cpylen = len > minseg ? minseg : len; >>> + memcpy(dstmap, srcmap, cpylen); >>> + srcoffset += cpylen; >>> + dstoffset += cpylen; >>> + len -= cpylen; >>> + } >>> + >>> + return 0; >>> +} >>> + >>> +void *odp_buffer_addr(odp_buffer_t buf) >>> +{ >>> + return odp_buf_to_hdr(buf)->addr[0]; >>> +} >>> + >>> +odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf) >>> +{ >>> + return odp_buf_to_hdr(buf)->pool_hdl; >>> +} >>> + >>> +size_t odp_buffer_size(odp_buffer_t buf) >>> +{ >>> + return odp_buf_to_hdr(buf)->size; >>> +} >>> + >>> +odp_buffer_type_e odp_buffer_type(odp_buffer_t buf) >>> +{ >>> + return odp_buf_to_hdr(buf)->type; >>> +} >>> + >>> +void *odp_buffer_udata(odp_buffer_t buf, size_t *usize) >>> +{ >>> + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >>> + >>> + *usize = hdr->udata_size; >>> + return hdr->udata_addr; >>> +} >>> + >>> +void *odp_buffer_udata_addr(odp_buffer_t buf) >>> +{ >>> + return odp_buf_to_hdr(buf)->udata_addr; >>> +} >>> + >>> +int odp_buffer_is_valid(odp_buffer_t buf) >>> +{ >>> + return validate_buf(buf) != NULL; >>> +} >>> >>> int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf) >>> { >>> @@ -63,27 +276,13 @@ int odp_buffer_snprint(char *str, size_t n, >>> odp_buffer_t buf) >>> len += snprintf(&str[len], n-len, >>> " pool %i\n", hdr->pool_hdl); >>> len += snprintf(&str[len], n-len, >>> - " index %"PRIu32"\n", hdr->index); >>> - len += snprintf(&str[len], n-len, >>> - " phy_addr %"PRIu64"\n", hdr->phys_addr); >>> - len += snprintf(&str[len], n-len, >>> - " addr %p\n", hdr->addr); >>> + " addr %p\n", hdr->addr[0]); >>> len += snprintf(&str[len], n-len, >>> " size %zu\n", hdr->size); >>> len += snprintf(&str[len], n-len, >>> - " cur_offset %zu\n", hdr->cur_offset); >>> - len += snprintf(&str[len], n-len, >>> " ref_count %i\n", hdr->ref_count); >>> len += snprintf(&str[len], n-len, >>> " type %i\n", hdr->type); >>> - len += snprintf(&str[len], n-len, >>> - " Scatter list\n"); >>> - len += snprintf(&str[len], n-len, >>> - " num_bufs %i\n", >>> hdr->scatter.num_bufs); >>> - len += snprintf(&str[len], n-len, >>> - " pos %i\n", hdr->scatter.pos); >>> - len += snprintf(&str[len], n-len, >>> - " total_len %zu\n", >>> hdr->scatter.total_len); >>> >>> return len; >>> } >>> @@ -100,9 +299,3 @@ void odp_buffer_print(odp_buffer_t buf) >>> >>> printf("\n%s\n", str); >>> } >>> - >>> -void odp_buffer_copy_scatter(odp_buffer_t buf_dst, odp_buffer_t buf_src) >>> -{ >>> - (void)buf_dst; >>> - (void)buf_src; >>> -} >>> diff --git a/platform/linux-generic/odp_buffer_pool.c >>> b/platform/linux-generic/odp_buffer_pool.c >>> index a48d7d6..f6161bb 100644 >>> --- a/platform/linux-generic/odp_buffer_pool.c >>> +++ b/platform/linux-generic/odp_buffer_pool.c >>> @@ -6,8 +6,9 @@ >>> >>> #include <odp_std_types.h> >>> #include <odp_buffer_pool.h> >>> -#include <odp_buffer_pool_internal.h> >>> #include <odp_buffer_internal.h> >>> +#include <odp_buffer_pool_internal.h> >>> +#include <odp_buffer_inlines.h> >>> #include <odp_packet_internal.h> >>> #include <odp_timer_internal.h> >>> #include <odp_shared_memory.h> >>> @@ -16,6 +17,7 @@ >>> #include <odp_config.h> >>> #include <odp_hints.h> >>> #include <odp_debug.h> >>> +#include <odph_eth.h> >>> >>> #include <string.h> >>> #include <stdlib.h> >>> @@ -33,36 +35,26 @@ >>> #define LOCK_INIT(a) odp_spinlock_init(a) >>> #endif >>> >>> - >>> -#if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >>> -#error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >>> -#endif >>> - >>> -#define NULL_INDEX ((uint32_t)-1) >>> - >>> -union buffer_type_any_u { >>> +typedef union buffer_type_any_u { >>> odp_buffer_hdr_t buf; >>> odp_packet_hdr_t pkt; >>> odp_timeout_hdr_t tmo; >>> -}; >>> - >>> -ODP_STATIC_ASSERT((sizeof(union buffer_type_any_u) % 8) == 0, >>> - "BUFFER_TYPE_ANY_U__SIZE_ERR"); >>> +} odp_anybuf_t; >>> >>> /* Any buffer type header */ >>> typedef struct { >>> union buffer_type_any_u any_hdr; /* any buffer type */ >>> - uint8_t buf_data[]; /* start of buffer data area >>> */ >>> } odp_any_buffer_hdr_t; >>> >>> +typedef struct odp_any_hdr_stride { >>> + uint8_t >>> pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_any_buffer_hdr_t))]; >>> +} odp_any_hdr_stride; >>> >>> -typedef union pool_entry_u { >>> - struct pool_entry_s s; >>> - >>> - uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct >>> pool_entry_s))]; >>> - >>> -} pool_entry_t; >>> +#if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >>> +#error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >>> +#endif >>> >>> +#define NULL_INDEX ((uint32_t)-1) >>> >>> typedef struct pool_table_t { >>> pool_entry_t pool[ODP_CONFIG_BUFFER_POOLS]; >>> @@ -76,39 +68,6 @@ static pool_table_t *pool_tbl; >>> /* Pool entry pointers (for inlining) */ >>> void *pool_entry_ptr[ODP_CONFIG_BUFFER_POOLS]; >>> >>> - >>> -static __thread odp_buffer_chunk_hdr_t >>> *local_chunk[ODP_CONFIG_BUFFER_POOLS]; >>> - >>> - >>> -static inline odp_buffer_pool_t pool_index_to_handle(uint32_t pool_id) >>> -{ >>> - return pool_id + 1; >>> -} >>> - >>> - >>> -static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl) >>> -{ >>> - return pool_hdl -1; >>> -} >>> - >>> - >>> -static inline void set_handle(odp_buffer_hdr_t *hdr, >>> - pool_entry_t *pool, uint32_t index) >>> -{ >>> - odp_buffer_pool_t pool_hdl = pool->s.pool_hdl; >>> - uint32_t pool_id = pool_handle_to_index(pool_hdl); >>> - >>> - if (pool_id >= ODP_CONFIG_BUFFER_POOLS) >>> - ODP_ABORT("set_handle: Bad pool handle %u\n", pool_hdl); >>> - >>> - if (index > ODP_BUFFER_MAX_INDEX) >>> - ODP_ERR("set_handle: Bad buffer index\n"); >>> - >>> - hdr->handle.pool_id = pool_id; >>> - hdr->handle.index = index; >>> -} >>> - >>> - >>> int odp_buffer_pool_init_global(void) >>> { >>> uint32_t i; >>> @@ -142,269 +101,167 @@ int odp_buffer_pool_init_global(void) >>> return 0; >>> } >>> >>> - >>> -static odp_buffer_hdr_t *index_to_hdr(pool_entry_t *pool, uint32_t >>> index) >>> -{ >>> - odp_buffer_hdr_t *hdr; >>> - >>> - hdr = (odp_buffer_hdr_t *)(pool->s.buf_base + index * >>> pool->s.buf_size); >>> - return hdr; >>> -} >>> - >>> - >>> -static void add_buf_index(odp_buffer_chunk_hdr_t *chunk_hdr, uint32_t >>> index) >>> -{ >>> - uint32_t i = chunk_hdr->chunk.num_bufs; >>> - chunk_hdr->chunk.buf_index[i] = index; >>> - chunk_hdr->chunk.num_bufs++; >>> -} >>> - >>> - >>> -static uint32_t rem_buf_index(odp_buffer_chunk_hdr_t *chunk_hdr) >>> +/** >>> + * Buffer pool creation >>> + */ >>> +odp_buffer_pool_t odp_buffer_pool_create(const char *name, >>> + odp_buffer_pool_param_t *args, >>> + odp_buffer_pool_init_t >>> *init_args) >>> { >>> - uint32_t index; >>> + odp_buffer_pool_t pool_hdl = ODP_BUFFER_POOL_INVALID; >>> + pool_entry_t *pool; >>> uint32_t i; >>> + odp_buffer_pool_param_t *params; >>> + odp_buffer_pool_init_t *init_params; >>> >>> - i = chunk_hdr->chunk.num_bufs - 1; >>> - index = chunk_hdr->chunk.buf_index[i]; >>> - chunk_hdr->chunk.num_bufs--; >>> - return index; >>> -} >>> - >>> - >>> -static odp_buffer_chunk_hdr_t *next_chunk(pool_entry_t *pool, >>> - odp_buffer_chunk_hdr_t >>> *chunk_hdr) >>> -{ >>> - uint32_t index; >>> - >>> - index = chunk_hdr->chunk.buf_index[ODP_BUFS_PER_CHUNK-1]; >>> - if (index == NULL_INDEX) >>> - return NULL; >>> - else >>> - return (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, >>> index); >>> -} >>> - >>> - >>> -static odp_buffer_chunk_hdr_t *rem_chunk(pool_entry_t *pool) >>> -{ >>> - odp_buffer_chunk_hdr_t *chunk_hdr; >>> - >>> - chunk_hdr = pool->s.head; >>> - if (chunk_hdr == NULL) { >>> - /* Pool is empty */ >>> - return NULL; >>> - } >>> - >>> - pool->s.head = next_chunk(pool, chunk_hdr); >>> - pool->s.free_bufs -= ODP_BUFS_PER_CHUNK; >>> - >>> - /* unlink */ >>> - rem_buf_index(chunk_hdr); >>> - return chunk_hdr; >>> -} >>> - >>> - >>> -static void add_chunk(pool_entry_t *pool, odp_buffer_chunk_hdr_t >>> *chunk_hdr) >>> -{ >>> - if (pool->s.head) /* link pool head to the chunk */ >>> - add_buf_index(chunk_hdr, pool->s.head->buf_hdr.index); >>> - else >>> - add_buf_index(chunk_hdr, NULL_INDEX); >>> - >>> - pool->s.head = chunk_hdr; >>> - pool->s.free_bufs += ODP_BUFS_PER_CHUNK; >>> -} >>> + /* Default pool creation arguments */ >>> + static odp_buffer_pool_param_t default_params = { >>> + .buf_num = 1024, >>> + .buf_size = ODP_CONFIG_BUF_SEG_SIZE, >>> + .buf_type = ODP_BUFFER_TYPE_PACKET, >>> + .buf_opts = ODP_BUFFER_OPTS_UNSEGMENTED, >>> + }; >>> >> Do we need to have this behaviour of allocating default params if the >> input params are NULL. >> It would be better if the API returns error. >> >>> >>> + /* Default initialization paramters */ >>> + static odp_buffer_pool_init_t default_init_params = { >>> + .udata_size = 0, >>> + .buf_init = NULL, >>> + .buf_init_arg = NULL, >>> + }; >>> >>> -static void check_align(pool_entry_t *pool, odp_buffer_hdr_t *hdr) >>> -{ >>> - if (!ODP_ALIGNED_CHECK_POWER_2(hdr->addr, pool->s.user_align)) { >>> - ODP_ABORT("check_align: user data align error %p, align >>> %zu\n", >>> - hdr->addr, pool->s.user_align); >>> - } >>> + /* Handle NULL input parameters */ >>> + params = args == NULL ? &default_params : args; >>> + init_params = init_args == NULL ? &default_init_params >>> + : init_args; >>> >>> - if (!ODP_ALIGNED_CHECK_POWER_2(hdr, ODP_CACHE_LINE_SIZE)) { >>> - ODP_ABORT("check_align: hdr align error %p, align %i\n", >>> - hdr, ODP_CACHE_LINE_SIZE); >>> - } >>> -} >>> + if (params->buf_type != ODP_BUFFER_TYPE_PACKET) >>> + params->buf_opts |= ODP_BUFFER_OPTS_UNSEGMENTED; >>> >>> + int unsegmented = ((params->buf_opts & >>> ODP_BUFFER_OPTS_UNSEGMENTED) == >>> + ODP_BUFFER_OPTS_UNSEGMENTED); >>> >>> -static void fill_hdr(void *ptr, pool_entry_t *pool, uint32_t index, >>> - int buf_type) >>> -{ >>> - odp_buffer_hdr_t *hdr = (odp_buffer_hdr_t *)ptr; >>> - size_t size = pool->s.hdr_size; >>> - uint8_t *buf_data; >>> + uint32_t udata_stride = >>> + ODP_CACHE_LINE_SIZE_ROUNDUP(init_params->udata_size); >>> >>> - if (buf_type == ODP_BUFFER_TYPE_CHUNK) >>> - size = sizeof(odp_buffer_chunk_hdr_t); >>> - >>> - switch (pool->s.buf_type) { >>> - odp_raw_buffer_hdr_t *raw_hdr; >>> - odp_packet_hdr_t *packet_hdr; >>> - odp_timeout_hdr_t *tmo_hdr; >>> - odp_any_buffer_hdr_t *any_hdr; >>> + uint32_t blk_size, buf_stride; >>> >>> + switch (params->buf_type) { >>> case ODP_BUFFER_TYPE_RAW: >>> - raw_hdr = ptr; >>> - buf_data = raw_hdr->buf_data; >>> + blk_size = params->buf_size; >>> + buf_stride = sizeof(odp_buffer_hdr_stride); >>> break; >>> + >>> case ODP_BUFFER_TYPE_PACKET: >>> - packet_hdr = ptr; >>> - buf_data = packet_hdr->buf_data; >>> + if (unsegmented) >>> + blk_size = >>> + >>> ODP_CACHE_LINE_SIZE_ROUNDUP(params->buf_size); >>> + else >>> + blk_size = ODP_ALIGN_ROUNDUP(params->buf_size, >>> + >>> ODP_CONFIG_BUF_SEG_SIZE); >>> + buf_stride = sizeof(odp_packet_hdr_stride); >>> break; >>> + >>> case ODP_BUFFER_TYPE_TIMEOUT: >>> - tmo_hdr = ptr; >>> - buf_data = tmo_hdr->buf_data; >>> + blk_size = 0; /* Timeouts have no block data, only >>> metadata */ >>> + buf_stride = sizeof(odp_timeout_hdr_stride); >>> break; >>> + >>> case ODP_BUFFER_TYPE_ANY: >>> - any_hdr = ptr; >>> - buf_data = any_hdr->buf_data; >>> + if (unsegmented) >>> + blk_size = >>> + >>> ODP_CACHE_LINE_SIZE_ROUNDUP(params->buf_size); >>> + else >>> + blk_size = ODP_ALIGN_ROUNDUP(params->buf_size, >>> + >>> ODP_CONFIG_BUF_SEG_SIZE); >>> + buf_stride = sizeof(odp_any_hdr_stride); >>> break; >>> - default: >>> - ODP_ABORT("Bad buffer type\n"); >>> - } >>> - >>> - memset(hdr, 0, size); >>> - >>> - set_handle(hdr, pool, index); >>> - >>> - hdr->addr = &buf_data[pool->s.buf_offset - pool->s.hdr_size]; >>> - hdr->index = index; >>> - hdr->size = pool->s.user_size; >>> - hdr->pool_hdl = pool->s.pool_hdl; >>> - hdr->type = buf_type; >>> - >>> - check_align(pool, hdr); >>> -} >>> - >>> - >>> -static void link_bufs(pool_entry_t *pool) >>> -{ >>> - odp_buffer_chunk_hdr_t *chunk_hdr; >>> - size_t hdr_size; >>> - size_t data_size; >>> - size_t data_align; >>> - size_t tot_size; >>> - size_t offset; >>> - size_t min_size; >>> - uint64_t pool_size; >>> - uintptr_t buf_base; >>> - uint32_t index; >>> - uintptr_t pool_base; >>> - int buf_type; >>> - >>> - buf_type = pool->s.buf_type; >>> - data_size = pool->s.user_size; >>> - data_align = pool->s.user_align; >>> - pool_size = pool->s.pool_size; >>> - pool_base = (uintptr_t) pool->s.pool_base_addr; >>> - >>> - if (buf_type == ODP_BUFFER_TYPE_RAW) { >>> - hdr_size = sizeof(odp_raw_buffer_hdr_t); >>> - } else if (buf_type == ODP_BUFFER_TYPE_PACKET) { >>> - hdr_size = sizeof(odp_packet_hdr_t); >>> - } else if (buf_type == ODP_BUFFER_TYPE_TIMEOUT) { >>> - hdr_size = sizeof(odp_timeout_hdr_t); >>> - } else if (buf_type == ODP_BUFFER_TYPE_ANY) { >>> - hdr_size = sizeof(odp_any_buffer_hdr_t); >>> - } else >>> - ODP_ABORT("odp_buffer_pool_create: Bad type %i\n", >>> buf_type); >>> - >>> - >>> - /* Chunk must fit into buffer data area.*/ >>> - min_size = sizeof(odp_buffer_chunk_hdr_t) - hdr_size; >>> - if (data_size < min_size) >>> - data_size = min_size; >>> - >>> - /* Roundup data size to full cachelines */ >>> - data_size = ODP_CACHE_LINE_SIZE_ROUNDUP(data_size); >>> - >>> - /* Min cacheline alignment for buffer header and data */ >>> - data_align = ODP_CACHE_LINE_SIZE_ROUNDUP(data_align); >>> - offset = ODP_CACHE_LINE_SIZE_ROUNDUP(hdr_size); >>> - >>> - /* Multiples of cacheline size */ >>> - if (data_size > data_align) >>> - tot_size = data_size + offset; >>> - else >>> - tot_size = data_align + offset; >>> - >>> - /* First buffer */ >>> - buf_base = ODP_ALIGN_ROUNDUP(pool_base + offset, data_align) - >>> offset; >>> - >>> - pool->s.hdr_size = hdr_size; >>> - pool->s.buf_base = buf_base; >>> - pool->s.buf_size = tot_size; >>> - pool->s.buf_offset = offset; >>> - index = 0; >>> - >>> - chunk_hdr = (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, index); >>> - pool->s.head = NULL; >>> - pool_size -= buf_base - pool_base; >>> - >>> - while (pool_size > ODP_BUFS_PER_CHUNK * tot_size) { >>> - int i; >>> - >>> - fill_hdr(chunk_hdr, pool, index, ODP_BUFFER_TYPE_CHUNK); >>> - >>> - index++; >>> - >>> - for (i = 0; i < ODP_BUFS_PER_CHUNK - 1; i++) { >>> - odp_buffer_hdr_t *hdr = index_to_hdr(pool, >>> index); >>> - >>> - fill_hdr(hdr, pool, index, buf_type); >>> - >>> - add_buf_index(chunk_hdr, index); >>> - index++; >>> - } >>> - >>> - add_chunk(pool, chunk_hdr); >>> >>> - chunk_hdr = (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, >>> - >>> index); >>> - pool->s.num_bufs += ODP_BUFS_PER_CHUNK; >>> - pool_size -= ODP_BUFS_PER_CHUNK * tot_size; >>> + default: >>> + return ODP_BUFFER_POOL_INVALID; >>> } >>> -} >>> - >>> - >>> -odp_buffer_pool_t odp_buffer_pool_create(const char *name, >>> - void *base_addr, uint64_t size, >>> - size_t buf_size, size_t >>> buf_align, >>> - int buf_type) >>> -{ >>> - odp_buffer_pool_t pool_hdl = ODP_BUFFER_POOL_INVALID; >>> - pool_entry_t *pool; >>> - uint32_t i; >>> >>> for (i = 0; i < ODP_CONFIG_BUFFER_POOLS; i++) { >>> pool = get_pool_entry(i); >>> >>> LOCK(&pool->s.lock); >>> + if (pool->s.shm != ODP_SHM_INVALID) { >>> + UNLOCK(&pool->s.lock); >>> + continue; >>> + } >>> >>> - if (pool->s.buf_base == 0) { >>> - /* found free pool */ >>> + /* found free pool */ >>> + size_t block_size, mdata_size, udata_size; >>> >>> - strncpy(pool->s.name, name, >>> - ODP_BUFFER_POOL_NAME_LEN - 1); >>> - pool->s.name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; >>> - pool->s.pool_base_addr = base_addr; >>> - pool->s.pool_size = size; >>> - pool->s.user_size = buf_size; >>> - pool->s.user_align = buf_align; >>> - pool->s.buf_type = buf_type; >>> + strncpy(pool->s.name, name, >>> + ODP_BUFFER_POOL_NAME_LEN - 1); >>> + pool->s.name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; >>> >>> - link_bufs(pool); >>> + pool->s.params = *params; >>> + pool->s.init_params = *init_params; >>> >>> - UNLOCK(&pool->s.lock); >>> + mdata_size = params->buf_num * buf_stride; >>> + udata_size = params->buf_num * udata_stride; >>> + block_size = params->buf_num * blk_size; >>> + >>> + pool->s.pool_size = ODP_PAGE_SIZE_ROUNDUP(mdata_size + >>> + udata_size + >>> + block_size); >>> >>> - pool_hdl = pool->s.pool_hdl; >>> - break; >>> + pool->s.shm = odp_shm_reserve(pool->s.name, >>> pool->s.pool_size, >>> + ODP_PAGE_SIZE, 0); >>> + if (pool->s.shm == ODP_SHM_INVALID) { >>> + UNLOCK(&pool->s.lock); >>> + return ODP_BUFFER_POOL_INVALID; >>> } >>> >>> + pool->s.pool_base_addr = (uint8_t >>> *)odp_shm_addr(pool->s.shm); >>> + pool->s.flags.unsegmented = unsegmented; >>> + pool->s.seg_size = unsegmented ? >>> + blk_size : ODP_CONFIG_BUF_SEG_SIZE; >>> + uint8_t *udata_base_addr = pool->s.pool_base_addr + >>> mdata_size; >>> + uint8_t *block_base_addr = udata_base_addr + udata_size; >>> + >>> + pool->s.bufcount = 0; >>> + pool->s.buf_freelist = NULL; >>> + pool->s.blk_freelist = NULL; >>> + >>> + uint8_t *buf = pool->s.udata_base_addr - buf_stride; >>> + uint8_t *udat = (udata_stride == 0) ? NULL : >>> + block_base_addr - udata_stride; >>> + >>> + /* Init buffer common header and add to pool buffer >>> freelist */ >>> + do { >>> + odp_buffer_hdr_t *tmp = >>> + (odp_buffer_hdr_t *)(void *)buf; >>> + tmp->pool_hdl = pool->s.pool_hdl; >>> + tmp->size = 0; >>> + tmp->type = params->buf_type; >>> + tmp->udata_addr = (void *)udat; >>> + tmp->udata_size = init_params->udata_size; >>> + tmp->segcount = 0; >>> + tmp->segsize = pool->s.seg_size; >>> + tmp->buf_hdl.handle = >>> + >>> odp_buffer_encode_handle((odp_buffer_hdr_t *) >>> + (void *)buf); >>> + ret_buf(&pool->s, tmp); >>> + buf -= buf_stride; >>> + udat -= udata_stride; >>> + } while (buf >= pool->s.pool_base_addr); >>> + >>> + /* Form block freelist for pool */ >>> + uint8_t *blk = pool->s.pool_base_addr + >>> pool->s.pool_size - >>> + pool->s.seg_size; >>> + >>> + if (blk_size > 0) >>> + do { >>> + ret_blk(&pool->s, blk); >>> + blk -= pool->s.seg_size; >>> + } while (blk >= block_base_addr); >>> + >>> UNLOCK(&pool->s.lock); >>> + >>> + pool_hdl = pool->s.pool_hdl; >>> + break; >>> } >>> >>> return pool_hdl; >>> @@ -431,93 +288,172 @@ odp_buffer_pool_t odp_buffer_pool_lookup(const >>> char *name) >>> return ODP_BUFFER_POOL_INVALID; >>> } >>> >>> - >>> -odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_hdl) >>> +odp_buffer_pool_t odp_buffer_pool_next(odp_buffer_pool_t pool_hdl, >>> + char *name, >>> + size_t *udata_size, >>> + odp_buffer_pool_param_t *params, >>> + int *predef) >>> { >>> - pool_entry_t *pool; >>> - odp_buffer_chunk_hdr_t *chunk; >>> - odp_buffer_bits_t handle; >>> - uint32_t pool_id = pool_handle_to_index(pool_hdl); >>> - >>> - pool = get_pool_entry(pool_id); >>> - chunk = local_chunk[pool_id]; >>> - >>> - if (chunk == NULL) { >>> - LOCK(&pool->s.lock); >>> - chunk = rem_chunk(pool); >>> - UNLOCK(&pool->s.lock); >>> + pool_entry_t *next_pool; >>> + uint32_t pool_id; >>> >>> - if (chunk == NULL) >>> - return ODP_BUFFER_INVALID; >>> + /* NULL input means first element */ >>> + if (pool_hdl == ODP_BUFFER_POOL_INVALID) { >>> + pool_id = 0; >>> + next_pool = get_pool_entry(pool_id); >>> + } else { >>> + pool_id = pool_handle_to_index(pool_hdl); >>> >>> - local_chunk[pool_id] = chunk; >>> + if (pool_id == ODP_CONFIG_BUFFER_POOLS) >>> + return ODP_BUFFER_POOL_INVALID; >>> + else >>> + next_pool = get_pool_entry(++pool_id); >>> } >>> >>> - if (chunk->chunk.num_bufs == 0) { >>> - /* give the chunk buffer */ >>> - local_chunk[pool_id] = NULL; >>> - chunk->buf_hdr.type = pool->s.buf_type; >>> + /* Only interested in pools that exist */ >>> + while (next_pool->s.shm == ODP_SHM_INVALID) { >>> + if (pool_id == ODP_CONFIG_BUFFER_POOLS) >>> + return ODP_BUFFER_POOL_INVALID; >>> + else >>> + next_pool = get_pool_entry(++pool_id); >>> + } >>> >>> - handle = chunk->buf_hdr.handle; >>> - } else { >>> - odp_buffer_hdr_t *hdr; >>> - uint32_t index; >>> - index = rem_buf_index(chunk); >>> - hdr = index_to_hdr(pool, index); >>> + /* Found the next pool, so return info about it */ >>> + strncpy(name, next_pool->s.name, ODP_BUFFER_POOL_NAME_LEN - 1); >>> + name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; >>> >>> - handle = hdr->handle; >>> - } >>> + *udata_size = next_pool->s.init_params.udata_size; >>> + *params = next_pool->s.params; >>> + *predef = next_pool->s.flags.predefined; >>> >>> - return handle.u32; >>> + return next_pool->s.pool_hdl; >>> } >>> >>> - >>> -void odp_buffer_free(odp_buffer_t buf) >>> +int odp_buffer_pool_destroy(odp_buffer_pool_t pool_hdl) >>> { >>> - odp_buffer_hdr_t *hdr; >>> - uint32_t pool_id; >>> - pool_entry_t *pool; >>> - odp_buffer_chunk_hdr_t *chunk_hdr; >>> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >>> >>> - hdr = odp_buf_to_hdr(buf); >>> - pool_id = pool_handle_to_index(hdr->pool_hdl); >>> - pool = get_pool_entry(pool_id); >>> - chunk_hdr = local_chunk[pool_id]; >>> + if (pool == NULL) >>> + return -1; >>> >>> - if (chunk_hdr && chunk_hdr->chunk.num_bufs == ODP_BUFS_PER_CHUNK >>> - 1) { >>> - /* Current chunk is full. Push back to the pool */ >>> - LOCK(&pool->s.lock); >>> - add_chunk(pool, chunk_hdr); >>> + LOCK(&pool->s.lock); >>> + >>> + if (pool->s.shm == ODP_SHM_INVALID || >>> + pool->s.bufcount > 0 || >>> + pool->s.flags.predefined) { >>> UNLOCK(&pool->s.lock); >>> - chunk_hdr = NULL; >>> + return -1; >>> } >>> >>> - if (chunk_hdr == NULL) { >>> - /* Use this buffer */ >>> - chunk_hdr = (odp_buffer_chunk_hdr_t *)hdr; >>> - local_chunk[pool_id] = chunk_hdr; >>> - chunk_hdr->chunk.num_bufs = 0; >>> + odp_shm_free(pool->s.shm); >>> + >>> + pool->s.shm = ODP_SHM_INVALID; >>> + UNLOCK(&pool->s.lock); >>> + >>> + return 0; >>> +} >>> + >>> +size_t odp_buffer_pool_headroom(odp_buffer_pool_t pool_hdl) >>> +{ >>> + return odp_pool_to_entry(pool_hdl)->s.headroom; >>> +} >>> + >>> +int odp_buffer_pool_set_headroom(odp_buffer_pool_t pool_hdl, size_t hr) >>> +{ >>> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >>> + >>> + if (hr >= pool->s.seg_size/2) { >>> + return -1; >>> } else { >>> - /* Add to current chunk */ >>> - add_buf_index(chunk_hdr, hdr->index); >>> + pool->s.headroom = hr; >>> + return 0; >>> } >>> } >>> >>> +size_t odp_buffer_pool_tailroom(odp_buffer_pool_t pool_hdl) >>> +{ >>> + return odp_pool_to_entry(pool_hdl)->s.tailroom; >>> +} >>> >>> -odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf) >>> +int odp_buffer_pool_set_tailroom(odp_buffer_pool_t pool_hdl, size_t tr) >>> { >>> - odp_buffer_hdr_t *hdr; >>> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >>> >>> - hdr = odp_buf_to_hdr(buf); >>> - return hdr->pool_hdl; >>> + if (tr >= pool->s.seg_size/2) { >>> + return -1; >>> + } else { >>> + pool->s.tailroom = tr; >>> + return 0; >>> + } >>> } >>> >>> +odp_buffer_t buffer_alloc(odp_buffer_pool_t pool_hdl, size_t size) >>> +{ >>> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >>> + size_t totsize = pool->s.headroom + size + pool->s.tailroom; >>> + odp_anybuf_t *buf; >>> + uint8_t *blk; >>> + >>> + if ((pool->s.flags.unsegmented && totsize > pool->s.seg_size) || >>> + (!pool->s.flags.unsegmented && totsize > >>> ODP_CONFIG_BUF_MAX_SIZE)) >>> + return ODP_BUFFER_INVALID; >>> + >>> + buf = (odp_anybuf_t *)(void *)get_buf(&pool->s); >>> + >>> + if (buf == NULL) >>> + return ODP_BUFFER_INVALID; >>> + >>> + /* Get blocks for this buffer, if pool uses application data */ >>> + if (buf->buf.segsize > 0) >>> + do { >>> + blk = get_blk(&pool->s); >>> + if (blk == NULL) { >>> + ret_buf(&pool->s, &buf->buf); >>> + return ODP_BUFFER_INVALID; >>> + } >>> + buf->buf.addr[buf->buf.segcount++] = blk; >>> + totsize = totsize < pool->s.seg_size ? 0 : >>> + totsize - pool->s.seg_size; >>> + } while (totsize > 0); >>> + >>> + switch (buf->buf.type) { >>> + case ODP_BUFFER_TYPE_RAW: >>> + break; >>> + >>> + case ODP_BUFFER_TYPE_PACKET: >>> + packet_init(pool, &buf->pkt, size); >>> + if (pool->s.init_params.buf_init != NULL) >>> + (*pool->s.init_params.buf_init) >>> + (buf->buf.buf_hdl.handle, >>> + pool->s.init_params.buf_init_arg); >>> + break; >>> + >>> + case ODP_BUFFER_TYPE_TIMEOUT: >>> + break; >>> + >>> + default: >>> + ret_buf(&pool->s, &buf->buf); >>> + return ODP_BUFFER_INVALID; >>> + } >>> + >>> + return odp_hdr_to_buf(&buf->buf); >>> +} >>> + >>> +odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_hdl) >>> +{ >>> + return buffer_alloc(pool_hdl, 0); >>> +} >>> + >>> +void odp_buffer_free(odp_buffer_t buf) >>> +{ >>> + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >>> + pool_entry_t *pool = odp_buf_to_pool(hdr); >>> + ret_buf(&pool->s, hdr); >>> +} >>> >>> void odp_buffer_pool_print(odp_buffer_pool_t pool_hdl) >>> { >>> pool_entry_t *pool; >>> - odp_buffer_chunk_hdr_t *chunk_hdr; >>> - uint32_t i; >>> uint32_t pool_id; >>> >>> pool_id = pool_handle_to_index(pool_hdl); >>> @@ -528,47 +464,4 @@ void odp_buffer_pool_print(odp_buffer_pool_t >>> pool_hdl) >>> printf(" pool %i\n", pool->s.pool_hdl); >>> printf(" name %s\n", pool->s.name); >>> printf(" pool base %p\n", pool->s.pool_base_addr); >>> - printf(" buf base 0x%"PRIxPTR"\n", pool->s.buf_base); >>> - printf(" pool size 0x%"PRIx64"\n", pool->s.pool_size); >>> - printf(" buf size %zu\n", pool->s.user_size); >>> - printf(" buf align %zu\n", pool->s.user_align); >>> - printf(" hdr size %zu\n", pool->s.hdr_size); >>> - printf(" alloc size %zu\n", pool->s.buf_size); >>> - printf(" offset to hdr %zu\n", pool->s.buf_offset); >>> - printf(" num bufs %"PRIu64"\n", pool->s.num_bufs); >>> - printf(" free bufs %"PRIu64"\n", pool->s.free_bufs); >>> - >>> - /* first chunk */ >>> - chunk_hdr = pool->s.head; >>> - >>> - if (chunk_hdr == NULL) { >>> - ODP_ERR(" POOL EMPTY\n"); >>> - return; >>> - } >>> - >>> - printf("\n First chunk\n"); >>> - >>> - for (i = 0; i < chunk_hdr->chunk.num_bufs - 1; i++) { >>> - uint32_t index; >>> - odp_buffer_hdr_t *hdr; >>> - >>> - index = chunk_hdr->chunk.buf_index[i]; >>> - hdr = index_to_hdr(pool, index); >>> - >>> - printf(" [%i] addr %p, id %"PRIu32"\n", i, hdr->addr, >>> index); >>> - } >>> - >>> - printf(" [%i] addr %p, id %"PRIu32"\n", i, >>> chunk_hdr->buf_hdr.addr, >>> - chunk_hdr->buf_hdr.index); >>> - >>> - /* next chunk */ >>> - chunk_hdr = next_chunk(pool, chunk_hdr); >>> - >>> - if (chunk_hdr) { >>> - printf(" Next chunk\n"); >>> - printf(" addr %p, id %"PRIu32"\n", >>> chunk_hdr->buf_hdr.addr, >>> - chunk_hdr->buf_hdr.index); >>> - } >>> - >>> - printf("\n"); >>> } >>> diff --git a/platform/linux-generic/odp_packet.c >>> b/platform/linux-generic/odp_packet.c >>> index 82ea879..45ac8e5 100644 >>> --- a/platform/linux-generic/odp_packet.c >>> +++ b/platform/linux-generic/odp_packet.c >>> @@ -11,29 +11,31 @@ >>> >>> #include <odph_eth.h> >>> #include <odph_ip.h> >>> +#include <odph_tcp.h> >>> +#include <odph_udp.h> >>> >>> #include <string.h> >>> #include <stdio.h> >>> >>> -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >>> - odph_ipv4hdr_t *ipv4, size_t >>> *offset_out); >>> -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >>> - odph_ipv6hdr_t *ipv6, size_t >>> *offset_out); >>> - >>> void odp_packet_init(odp_packet_t pkt) >>> { >>> odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >>> + pool_entry_t *pool = odp_buf_to_pool(&pkt_hdr->buf_hdr); >>> const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, >>> buf_hdr); >>> uint8_t *start; >>> size_t len; >>> >>> start = (uint8_t *)pkt_hdr + start_offset; >>> - len = ODP_OFFSETOF(odp_packet_hdr_t, buf_data) - start_offset; >>> + len = sizeof(odp_packet_hdr_t) - start_offset; >>> memset(start, 0, len); >>> >>> pkt_hdr->l2_offset = ODP_PACKET_OFFSET_INVALID; >>> pkt_hdr->l3_offset = ODP_PACKET_OFFSET_INVALID; >>> pkt_hdr->l4_offset = ODP_PACKET_OFFSET_INVALID; >>> + pkt_hdr->payload_offset = ODP_PACKET_OFFSET_INVALID; >>> + >>> + pkt_hdr->headroom = pool->s.headroom; >>> + pkt_hdr->tailroom = pool->s.tailroom; >>> } >>> >>> odp_packet_t odp_packet_from_buffer(odp_buffer_t buf) >>> @@ -46,55 +48,112 @@ odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt) >>> return (odp_buffer_t)pkt; >>> } >>> >>> -void odp_packet_set_len(odp_packet_t pkt, size_t len) >>> +size_t odp_packet_len(odp_packet_t pkt) >>> { >>> - odp_packet_hdr(pkt)->frame_len = len; >>> + return odp_packet_hdr(pkt)->frame_len; >>> } >>> >>> -size_t odp_packet_get_len(odp_packet_t pkt) >>> +void *odp_packet_offset_map(odp_packet_t pkt, size_t offset, size_t >>> *seglen) >>> { >>> - return odp_packet_hdr(pkt)->frame_len; >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + if (offset >= pkt_hdr->frame_len) >>> + return NULL; >>> + >>> + return buffer_map(&pkt_hdr->buf_hdr, >>> + pkt_hdr->headroom + offset, >>> + seglen, pkt_hdr->frame_len); >>> } >>> >>> -uint8_t *odp_packet_addr(odp_packet_t pkt) >>> +void odp_packet_offset_unmap(odp_packet_t pkt ODP_UNUSED, >>> + size_t offset ODP_UNUSED) >>> { >>> - return odp_buffer_addr(odp_packet_to_buffer(pkt)); >>> } >>> >>> -uint8_t *odp_packet_data(odp_packet_t pkt) >>> +void *odp_packet_map(odp_packet_t pkt, size_t *seglen) >>> { >>> - return odp_packet_addr(pkt) + odp_packet_hdr(pkt)->frame_offset; >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + return buffer_map(&pkt_hdr->buf_hdr, >>> + 0, seglen, pkt_hdr->frame_len); >>> } >>> >>> +void *odp_packet_addr(odp_packet_t pkt) >>> +{ >>> + size_t seglen; >>> + return odp_packet_map(pkt, &seglen); >>> +} >>> >>> -uint8_t *odp_packet_l2(odp_packet_t pkt) >>> +odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt) >>> { >>> - const size_t offset = odp_packet_l2_offset(pkt); >>> + return odp_packet_hdr(pkt)->buf_hdr.pool_hdl; >>> +} >>> >>> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >>> - return NULL; >>> +odp_packet_segment_t odp_packet_segment_by_index(odp_packet_t pkt, >>> + size_t ndx) >>> +{ >>> >>> - return odp_packet_addr(pkt) + offset; >>> + return (odp_packet_segment_t) >>> + buffer_segment(&odp_packet_hdr(pkt)->buf_hdr, ndx); >>> } >>> >>> -size_t odp_packet_l2_offset(odp_packet_t pkt) >>> +odp_packet_segment_t odp_packet_segment_next(odp_packet_t pkt, >>> + odp_packet_segment_t seg) >>> { >>> - return odp_packet_hdr(pkt)->l2_offset; >>> + return (odp_packet_segment_t) >>> + segment_next(&odp_packet_hdr(pkt)->buf_hdr, seg); >>> } >>> >>> -void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset) >>> +void *odp_packet_segment_map(odp_packet_t pkt, odp_packet_segment_t seg, >>> + size_t *seglen) >>> { >>> - odp_packet_hdr(pkt)->l2_offset = offset; >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + return segment_map(&pkt_hdr->buf_hdr, seg, >>> + seglen, pkt_hdr->frame_len); >>> } >>> >>> -uint8_t *odp_packet_l3(odp_packet_t pkt) >>> +void odp_packet_segment_unmap(odp_packet_segment_t seg ODP_UNUSED) >>> { >>> - const size_t offset = odp_packet_l3_offset(pkt); >>> +} >>> >>> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >>> - return NULL; >>> +void *odp_packet_udata(odp_packet_t pkt, size_t *len) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + *len = pkt_hdr->buf_hdr.udata_size; >>> + return pkt_hdr->buf_hdr.udata_addr; >>> +} >>> + >>> +void *odp_packet_udata_addr(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->buf_hdr.udata_addr; >>> +} >>> + >>> +void *odp_packet_l2_map(odp_packet_t pkt, size_t *seglen) >>> +{ >>> + return odp_packet_offset_map(pkt, odp_packet_l2_offset(pkt), >>> seglen); >>> +} >>> + >>> +size_t odp_packet_l2_offset(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->l2_offset; >>> +} >>> + >>> +int odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset) >>> +{ >>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>> + >>> + if (offset >= hdr->frame_len) >>> + return -1; >>> + >>> + hdr->l2_offset = offset; >>> + return 0; >>> +} >>> >>> - return odp_packet_addr(pkt) + offset; >>> +void *odp_packet_l3_map(odp_packet_t pkt, size_t *seglen) >>> +{ >>> + return odp_packet_offset_map(pkt, odp_packet_l3_offset(pkt), >>> seglen); >>> } >>> >>> size_t odp_packet_l3_offset(odp_packet_t pkt) >>> @@ -102,19 +161,35 @@ size_t odp_packet_l3_offset(odp_packet_t pkt) >>> return odp_packet_hdr(pkt)->l3_offset; >>> } >>> >>> -void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset) >>> +int odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset) >>> { >>> - odp_packet_hdr(pkt)->l3_offset = offset; >>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>> + >>> + if (offset >= hdr->frame_len) >>> + return -1; >>> + >>> + hdr->l3_offset = offset; >>> + return 0; >>> } >>> >>> -uint8_t *odp_packet_l4(odp_packet_t pkt) >>> +uint32_t odp_packet_l3_protocol(odp_packet_t pkt) >>> { >>> - const size_t offset = odp_packet_l4_offset(pkt); >>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>> >>> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >>> - return NULL; >>> + if (hdr->input_flags.l3) >>> + return hdr->l3_protocol; >>> + else >>> + return -1; >>> +} >>> + >>> +void odp_packet_set_l3_protocol(odp_packet_t pkt, uint16_t protocol) >>> +{ >>> + odp_packet_hdr(pkt)->l3_protocol = protocol; >>> +} >>> >>> - return odp_packet_addr(pkt) + offset; >>> +void *odp_packet_l4_map(odp_packet_t pkt, size_t *seglen) >>> +{ >>> + return odp_packet_offset_map(pkt, odp_packet_l4_offset(pkt), >>> seglen); >>> } >>> >>> size_t odp_packet_l4_offset(odp_packet_t pkt) >>> @@ -122,277 +197,902 @@ size_t odp_packet_l4_offset(odp_packet_t pkt) >>> return odp_packet_hdr(pkt)->l4_offset; >>> } >>> >>> -void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset) >>> +int odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset) >>> { >>> - odp_packet_hdr(pkt)->l4_offset = offset; >>> -} >>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>> + >>> + if (offset >= hdr->frame_len) >>> + return -1; >>> >>> + hdr->l4_offset = offset; >>> + return 0; >>> +} >>> >>> -int odp_packet_is_segmented(odp_packet_t pkt) >>> +uint32_t odp_packet_l4_protocol(odp_packet_t pkt) >>> { >>> - odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr((odp_buffer_t)pkt); >>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>> >>> - if (buf_hdr->scatter.num_bufs == 0) >>> - return 0; >>> + if (hdr->input_flags.l4) >>> + return hdr->l4_protocol; >>> else >>> - return 1; >>> + return -1; >>> } >>> >>> +void odp_packet_set_l4_protocol(odp_packet_t pkt, uint8_t protocol) >>> +{ >>> + odp_packet_hdr(pkt)->l4_protocol = protocol; >>> +} >>> >>> -int odp_packet_seg_count(odp_packet_t pkt) >>> +void *odp_packet_payload_map(odp_packet_t pkt, size_t *seglen) >>> { >>> - odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr((odp_buffer_t)pkt); >>> + return odp_packet_offset_map(pkt, odp_packet_payload_offset(pkt), >>> + seglen); >>> +} >>> >>> - return (int)buf_hdr->scatter.num_bufs + 1; >>> +size_t odp_packet_payload_offset(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->payload_offset; >>> } >>> >>> +int odp_packet_set_payload_offset(odp_packet_t pkt, size_t offset) >>> +{ >>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>> >>> -/** >>> - * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP >>> - * >>> - * Internal function: caller is resposible for passing only valid >>> packet handles >>> - * , lengths and offsets (usually done&called in packet input). >>> - * >>> - * @param pkt Packet handle >>> - * @param len Packet length in bytes >>> - * @param frame_offset Byte offset to L2 header >>> - */ >>> -void odp_packet_parse(odp_packet_t pkt, size_t len, size_t frame_offset) >>> + if (offset >= hdr->frame_len) >>> + return -1; >>> + >>> + hdr->payload_offset = offset; >>> + return 0; >>> +} >>> + >>> +int odp_packet_error(odp_packet_t pkt) >>> { >>> - odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >>> - odph_ethhdr_t *eth; >>> - odph_vlanhdr_t *vlan; >>> - odph_ipv4hdr_t *ipv4; >>> - odph_ipv6hdr_t *ipv6; >>> - uint16_t ethtype; >>> - size_t offset = 0; >>> - uint8_t ip_proto = 0; >>> + return odp_packet_hdr(pkt)->error_flags.all != 0; >>> +} >>> >>> - pkt_hdr->input_flags.eth = 1; >>> - pkt_hdr->frame_offset = frame_offset; >>> - pkt_hdr->frame_len = len; >>> +void odp_packet_set_error(odp_packet_t pkt, int val) >>> +{ >>> + odp_packet_hdr(pkt)->error_flags.app_error = val; >>> +} >>> >>> - if (odp_unlikely(len < ODPH_ETH_LEN_MIN)) { >>> - pkt_hdr->error_flags.frame_len = 1; >>> - return; >>> - } else if (len > ODPH_ETH_LEN_MAX) { >>> - pkt_hdr->input_flags.jumbo = 1; >>> - } >>> +int odp_packet_inflag_l2(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->input_flags.l2; >>> +} >>> >>> - /* Assume valid L2 header, no CRC/FCS check in SW */ >>> - pkt_hdr->input_flags.l2 = 1; >>> - pkt_hdr->l2_offset = frame_offset; >>> +void odp_packet_set_inflag_l2(odp_packet_t pkt, int val) >>> +{ >>> + odp_packet_hdr(pkt)->input_flags.l2 = val; >>> +} >>> >>> - eth = (odph_ethhdr_t *)odp_packet_data(pkt); >>> - ethtype = odp_be_to_cpu_16(eth->type); >>> - vlan = (odph_vlanhdr_t *)ð->type; >>> +int odp_packet_inflag_l3(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->input_flags.l3; >>> +} >>> >>> - if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { >>> - pkt_hdr->input_flags.vlan_qinq = 1; >>> - ethtype = odp_be_to_cpu_16(vlan->tpid); >>> - offset += sizeof(odph_vlanhdr_t); >>> - vlan = &vlan[1]; >>> - } >>> +void odp_packet_set_inflag_l3(odp_packet_t pkt, int val) >>> +{ >>> + odp_packet_hdr(pkt)->input_flags.l3 = val; >>> +} >>> >>> - if (ethtype == ODPH_ETHTYPE_VLAN) { >>> - pkt_hdr->input_flags.vlan = 1; >>> - ethtype = odp_be_to_cpu_16(vlan->tpid); >>> - offset += sizeof(odph_vlanhdr_t); >>> - } >>> +int odp_packet_inflag_l4(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->input_flags.l4; >>> +} >>> >>> - /* Set l3_offset+flag only for known ethtypes */ >>> - switch (ethtype) { >>> - case ODPH_ETHTYPE_IPV4: >>> - pkt_hdr->input_flags.ipv4 = 1; >>> - pkt_hdr->input_flags.l3 = 1; >>> - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + >>> offset; >>> - ipv4 = (odph_ipv4hdr_t *)odp_packet_l3(pkt); >>> - ip_proto = parse_ipv4(pkt_hdr, ipv4, &offset); >>> - break; >>> - case ODPH_ETHTYPE_IPV6: >>> - pkt_hdr->input_flags.ipv6 = 1; >>> - pkt_hdr->input_flags.l3 = 1; >>> - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + >>> offset; >>> - ipv6 = (odph_ipv6hdr_t *)odp_packet_l3(pkt); >>> - ip_proto = parse_ipv6(pkt_hdr, ipv6, &offset); >>> - break; >>> - case ODPH_ETHTYPE_ARP: >>> - pkt_hdr->input_flags.arp = 1; >>> - /* fall through */ >>> - default: >>> - ip_proto = 0; >>> - break; >>> - } >>> +void odp_packet_set_inflag_l4(odp_packet_t pkt, int val) >>> +{ >>> + odp_packet_hdr(pkt)->input_flags.l4 = val; >>> +} >>> >>> - switch (ip_proto) { >>> - case ODPH_IPPROTO_UDP: >>> - pkt_hdr->input_flags.udp = 1; >>> - pkt_hdr->input_flags.l4 = 1; >>> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >>> - break; >>> - case ODPH_IPPROTO_TCP: >>> - pkt_hdr->input_flags.tcp = 1; >>> - pkt_hdr->input_flags.l4 = 1; >>> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >>> - break; >>> - case ODPH_IPPROTO_SCTP: >>> - pkt_hdr->input_flags.sctp = 1; >>> - pkt_hdr->input_flags.l4 = 1; >>> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >>> - break; >>> - case ODPH_IPPROTO_ICMP: >>> - pkt_hdr->input_flags.icmp = 1; >>> - pkt_hdr->input_flags.l4 = 1; >>> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >>> - break; >>> - default: >>> - /* 0 or unhandled IP protocols, don't set L4 flag+offset >>> */ >>> - if (pkt_hdr->input_flags.ipv6) { >>> - /* IPv6 next_hdr is not L4, mark as IP-option >>> instead */ >>> - pkt_hdr->input_flags.ipopt = 1; >>> - } >>> - break; >>> - } >>> +int odp_packet_inflag_eth(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->input_flags.eth; >>> } >>> >>> -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >>> - odph_ipv4hdr_t *ipv4, size_t >>> *offset_out) >>> +void odp_packet_set_inflag_eth(odp_packet_t pkt, int val) >>> { >>> - uint8_t ihl; >>> - uint16_t frag_offset; >>> + odp_packet_hdr(pkt)->input_flags.eth = val; >>> +} >>> >>> - ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); >>> - if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN)) { >>> - pkt_hdr->error_flags.ip_err = 1; >>> - return 0; >>> - } >>> +int odp_packet_inflag_jumbo(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->input_flags.jumbo; >>> +} >>> >>> - if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) { >>> - pkt_hdr->input_flags.ipopt = 1; >>> - return 0; >>> - } >>> +void odp_packet_set_inflag_jumbo(odp_packet_t pkt, int val) >>> +{ >>> + odp_packet_hdr(pkt)->input_flags.jumbo = val; >>> +} >>> >>> - /* A packet is a fragment if: >>> - * "more fragments" flag is set (all fragments except the last) >>> - * OR >>> - * "fragment offset" field is nonzero (all fragments except the >>> first) >>> - */ >>> - frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); >>> - if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) { >>> - pkt_hdr->input_flags.ipfrag = 1; >>> - return 0; >>> - } >>> +int odp_packet_inflag_vlan(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->input_flags.vlan; >>> +} >>> >>> - if (ipv4->proto == ODPH_IPPROTO_ESP || >>> - ipv4->proto == ODPH_IPPROTO_AH) { >>> - pkt_hdr->input_flags.ipsec = 1; >>> - return 0; >>> - } >>> +void odp_packet_set_inflag_vlan(odp_packet_t pkt, int val) >>> +{ >>> + odp_packet_hdr(pkt)->input_flags.vlan = val; >>> +} >>> >>> - /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after >>> return */ >>> +int odp_packet_inflag_vlan_qinq(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->input_flags.vlan_qinq; >>> +} >>> >>> - *offset_out = sizeof(uint32_t) * ihl; >>> - return ipv4->proto; >>> +void odp_packet_set_inflag_vlan_qinq(odp_packet_t pkt, int val) >>> +{ >>> + odp_packet_hdr(pkt)->input_flags.vlan_qinq = val; >>> } >>> >>> -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >>> - odph_ipv6hdr_t *ipv6, size_t >>> *offset_out) >>> +int odp_packet_inflag_snap(odp_packet_t pkt) >>> { >>> - if (ipv6->next_hdr == ODPH_IPPROTO_ESP || >>> - ipv6->next_hdr == ODPH_IPPROTO_AH) { >>> - pkt_hdr->input_flags.ipopt = 1; >>> - pkt_hdr->input_flags.ipsec = 1; >>> - return 0; >>> - } >>> + return odp_packet_hdr(pkt)->input_flags.snap; >>> +} >>> >>> - if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { >>> - pkt_hdr->input_flags.ipopt = 1; >>> - pkt_hdr->input_flags.ipfrag = 1; >>> - return 0; >>> - } >>> +void odp_packet_set_inflag_snap(odp_packet_t pkt, int val) >>> +{ >>> + odp_packet_hdr(pkt)->input_flags.snap = val; >>> +} >>> >>> - /* Don't step through more extensions */ >>> - *offset_out = ODPH_IPV6HDR_LEN; >>> - return ipv6->next_hdr; >>> +int odp_packet_inflag_arp(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->input_flags.arp; >>> } >>> >>> -void odp_packet_print(odp_packet_t pkt) >>> +void odp_packet_set_inflag_arp(odp_packet_t pkt, int val) >>> { >>> - int max_len = 512; >>> - char str[max_len]; >>> - int len = 0; >>> - int n = max_len-1; >>> - odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>> + odp_packet_hdr(pkt)->input_flags.arp = val; >>> +} >>> >>> - len += snprintf(&str[len], n-len, "Packet "); >>> - len += odp_buffer_snprint(&str[len], n-len, (odp_buffer_t) pkt); >>> - len += snprintf(&str[len], n-len, >>> - " input_flags 0x%x\n", hdr->input_flags.all); >>> - len += snprintf(&str[len], n-len, >>> - " error_flags 0x%x\n", hdr->error_flags.all); >>> - len += snprintf(&str[len], n-len, >>> - " output_flags 0x%x\n", hdr->output_flags.all); >>> - len += snprintf(&str[len], n-len, >>> - " frame_offset %u\n", hdr->frame_offset); >>> - len += snprintf(&str[len], n-len, >>> - " l2_offset %u\n", hdr->l2_offset); >>> - len += snprintf(&str[len], n-len, >>> - " l3_offset %u\n", hdr->l3_offset); >>> - len += snprintf(&str[len], n-len, >>> - " l4_offset %u\n", hdr->l4_offset); >>> - len += snprintf(&str[len], n-len, >>> - " frame_len %u\n", hdr->frame_len); >>> - len += snprintf(&str[len], n-len, >>> - " input %u\n", hdr->input); >>> - str[len] = '\0'; >>> +int odp_packet_inflag_ipv4(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->input_flags.ipv4; >>> +} >>> >>> - printf("\n%s\n", str); >>> +void odp_packet_set_inflag_ipv4(odp_packet_t pkt, int val) >>> +{ >>> + odp_packet_hdr(pkt)->input_flags.ipv4 = val; >>> } >>> >>> -int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src) >>> +int odp_packet_inflag_ipv6(odp_packet_t pkt) >>> { >>> - odp_packet_hdr_t *const pkt_hdr_dst = odp_packet_hdr(pkt_dst); >>> - odp_packet_hdr_t *const pkt_hdr_src = odp_packet_hdr(pkt_src); >>> - const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, >>> buf_hdr); >>> - uint8_t *start_src; >>> - uint8_t *start_dst; >>> - size_t len; >>> + return odp_packet_hdr(pkt)->input_flags.ipv6; >>> +} >>> >>> - if (pkt_dst == ODP_PACKET_INVALID || pkt_src == >>> ODP_PACKET_INVALID) >>> - return -1; >>> +void odp_packet_set_inflag_ipv6(odp_packet_t pkt, int val) >>> +{ >>> + odp_packet_hdr(pkt)->input_flags.ipv6 = val; >>> +} >>> >>> - if (pkt_hdr_dst->buf_hdr.size < >>> - pkt_hdr_src->frame_len + pkt_hdr_src->frame_offset) >>> - return -1; >>> +int odp_packet_inflag_ipfrag(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->input_flags.ipfrag; >>> +} >>> + >>> +void odp_packet_set_inflag_ipfrag(odp_packet_t pkt, int val) >>> +{ >>> + odp_packet_hdr(pkt)->input_flags.ipfrag = val; >>> +} >>> + >>> +int odp_packet_inflag_ipopt(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->input_flags.ipopt; >>> +} >>> >>> - /* Copy packet header */ >>> - start_dst = (uint8_t *)pkt_hdr_dst + start_offset; >>> - start_src = (uint8_t *)pkt_hdr_src + start_offset; >>> - len = ODP_OFFSETOF(odp_packet_hdr_t, buf_data) - start_offset; >>> - memcpy(start_dst, start_src, len); >>> +void odp_packet_set_inflag_ipopt(odp_packet_t pkt, int val) >>> +{ >>> + odp_packet_hdr(pkt)->input_flags.ipopt = val; >>> +} >>> >>> - /* Copy frame payload */ >>> - start_dst = (uint8_t *)odp_packet_data(pkt_dst); >>> - start_src = (uint8_t *)odp_packet_data(pkt_src); >>> - len = pkt_hdr_src->frame_len; >>> - memcpy(start_dst, start_src, len); >>> +int odp_packet_inflag_ipsec(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->input_flags.ipsec; >>> +} >>> >>> - /* Copy useful things from the buffer header */ >>> - pkt_hdr_dst->buf_hdr.cur_offset = >>> pkt_hdr_src->buf_hdr.cur_offset; >>> +void odp_packet_set_inflag_ipsec(odp_packet_t pkt, int val) >>> +{ >>> + odp_packet_hdr(pkt)->input_flags.ipsec = val; >>> +} >>> >>> - /* Create a copy of the scatter list */ >>> - odp_buffer_copy_scatter(odp_packet_to_buffer(pkt_dst), >>> - odp_packet_to_buffer(pkt_src)); >>> +int odp_packet_inflag_udp(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->input_flags.udp; >>> +} >>> >>> - return 0; >>> +void odp_packet_set_inflag_udp(odp_packet_t pkt, int val) >>> +{ >>> + odp_packet_hdr(pkt)->input_flags.udp = val; >>> } >>> >>> -void odp_packet_set_ctx(odp_packet_t pkt, const void *ctx) >>> +int odp_packet_inflag_tcp(odp_packet_t pkt) >>> { >>> - odp_packet_hdr(pkt)->user_ctx = (intptr_t)ctx; >>> + return odp_packet_hdr(pkt)->input_flags.tcp; >>> } >>> >>> -void *odp_packet_get_ctx(odp_packet_t pkt) >>> +void odp_packet_set_inflag_tcp(odp_packet_t pkt, int val) >>> { >>> - return (void *)(intptr_t)odp_packet_hdr(pkt)->user_ctx; >>> + odp_packet_hdr(pkt)->input_flags.tcp = val; >>> +} >>> + >>> +int odp_packet_inflag_tcpopt(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->input_flags.tcpopt; >>> +} >>> + >>> +void odp_packet_set_inflag_tcpopt(odp_packet_t pkt, int val) >>> +{ >>> + odp_packet_hdr(pkt)->input_flags.tcpopt = val; >>> +} >>> + >>> +int odp_packet_inflag_icmp(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->input_flags.icmp; >>> +} >>> + >>> +void odp_packet_set_inflag_icmp(odp_packet_t pkt, int val) >>> +{ >>> + odp_packet_hdr(pkt)->input_flags.icmp = val; >>> +} >>> + >>> +int odp_packet_is_valid(odp_packet_t pkt) >>> +{ >>> + odp_buffer_hdr_t *buf = validate_buf((odp_buffer_t)pkt); >>> + >>> + if (buf == NULL) >>> + return 0; >>> + else >>> + return buf->type == ODP_BUFFER_TYPE_PACKET; >>> +} >>> + >>> +int odp_packet_is_segmented(odp_packet_t pkt) >>> +{ >>> + return (odp_packet_hdr(pkt)->buf_hdr.segcount > 1); >>> +} >>> + >>> +int odp_packet_segment_count(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->buf_hdr.segcount; >>> +} >>> + >>> +size_t odp_packet_headroom(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->headroom; >>> +} >>> + >>> +size_t odp_packet_tailroom(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->tailroom; >>> +} >>> + >>> +int odp_packet_push_head(odp_packet_t pkt, size_t len) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + if (len > pkt_hdr->headroom) >>> + return -1; >>> + >>> + push_head(pkt_hdr, len); >>> + return 0; >>> +} >>> + >>> +void *odp_packet_push_head_and_map(odp_packet_t pkt, size_t len, size_t >>> *seglen) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + if (len > pkt_hdr->headroom) >>> + return NULL; >>> + >>> + push_head(pkt_hdr, len); >>> + >>> + return buffer_map(&pkt_hdr->buf_hdr, 0, seglen, >>> pkt_hdr->frame_len); >>> +} >>> + >>> +int odp_packet_pull_head(odp_packet_t pkt, size_t len) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + if (len > pkt_hdr->frame_len) >>> + return -1; >>> + >>> + pull_head(pkt_hdr, len); >>> + return 0; >>> +} >>> + >>> +void *odp_packet_pull_head_and_map(odp_packet_t pkt, size_t len, size_t >>> *seglen) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + if (len > pkt_hdr->frame_len) >>> + return NULL; >>> + >>> + pull_head(pkt_hdr, len); >>> + return buffer_map(&pkt_hdr->buf_hdr, 0, seglen, >>> pkt_hdr->frame_len); >>> +} >>> + >>> +int odp_packet_push_tail(odp_packet_t pkt, size_t len) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + if (len > pkt_hdr->tailroom) >>> + return -1; >>> + >>> + push_tail(pkt_hdr, len); >>> + return 0; >>> +} >>> + >>> +void *odp_packet_push_tail_and_map(odp_packet_t pkt, size_t len, size_t >>> *seglen) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + size_t origin = pkt_hdr->frame_len; >>> + >>> + if (len > pkt_hdr->tailroom) >>> + return NULL; >>> + >>> + push_tail(pkt_hdr, len); >>> + >>> + return buffer_map(&pkt_hdr->buf_hdr, origin, seglen, >>> + pkt_hdr->frame_len); >>> +} >>> + >>> +int odp_packet_pull_tail(odp_packet_t pkt, size_t len) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + if (len > pkt_hdr->frame_len) >>> + return -1; >>> + >>> + pull_tail(pkt_hdr, len); >>> + return 0; >>> +} >>> + >>> +void odp_packet_print(odp_packet_t pkt) >>> +{ >>> + int max_len = 512; >>> + char str[max_len]; >>> + int len = 0; >>> + int n = max_len-1; >>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>> + >>> + len += snprintf(&str[len], n-len, "Packet "); >>> + len += odp_buffer_snprint(&str[len], n-len, (odp_buffer_t) pkt); >>> + len += snprintf(&str[len], n-len, >>> + " input_flags 0x%x\n", hdr->input_flags.all); >>> + len += snprintf(&str[len], n-len, >>> + " error_flags 0x%x\n", hdr->error_flags.all); >>> + len += snprintf(&str[len], n-len, >>> + " output_flags 0x%x\n", hdr->output_flags.all); >>> + len += snprintf(&str[len], n-len, >>> + " l2_offset %u\n", hdr->l2_offset); >>> + len += snprintf(&str[len], n-len, >>> + " l3_offset %u\n", hdr->l3_offset); >>> + len += snprintf(&str[len], n-len, >>> + " l3_len %u\n", hdr->l3_len); >>> + len += snprintf(&str[len], n-len, >>> + " l3_protocol 0x%x\n", hdr->l3_protocol); >>> + len += snprintf(&str[len], n-len, >>> + " l4_offset %u\n", hdr->l4_offset); >>> + len += snprintf(&str[len], n-len, >>> + " l4_len %u\n", hdr->l4_len); >>> + len += snprintf(&str[len], n-len, >>> + " l4_protocol %u\n", hdr->l4_protocol); >>> + len += snprintf(&str[len], n-len, >>> + " payload_offset %u\n", hdr->payload_offset); >>> + len += snprintf(&str[len], n-len, >>> + " frame_len %u\n", hdr->frame_len); >>> + str[len] = '\0'; >>> + >>> + printf("\n%s\n", str); >>> +} >>> + >>> +int odp_packet_copy_to_packet(odp_packet_t dstpkt, size_t dstoffset, >>> + odp_packet_t srcpkt, size_t srcoffset, >>> + size_t len) >>> +{ >>> + void *dstmap; >>> + void *srcmap; >>> + size_t cpylen, minseg, dstseglen, srcseglen; >>> + >>> + while (len > 0) { >>> + dstmap = odp_packet_offset_map(dstpkt, dstoffset, >>> &dstseglen); >>> + srcmap = odp_packet_offset_map(srcpkt, srcoffset, >>> &srcseglen); >>> + if (dstmap == NULL || srcmap == NULL) >>> + return -1; >>> + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; >>> + cpylen = len > minseg ? minseg : len; >>> + memcpy(dstmap, srcmap, cpylen); >>> + srcoffset += cpylen; >>> + dstoffset += cpylen; >>> + len -= cpylen; >>> + } >>> + >>> + return 0; >>> +} >>> + >>> +int odp_packet_copy_to_memory(void *dstmem, odp_packet_t srcpkt, >>> + size_t srcoffset, size_t dstlen) >>> +{ >>> + void *mapaddr; >>> + size_t seglen, cpylen; >>> + uint8_t *dstaddr = (uint8_t *)dstmem; >>> + >>> + while (dstlen > 0) { >>> + mapaddr = odp_packet_offset_map(srcpkt, srcoffset, >>> &seglen); >>> + if (mapaddr == NULL) >>> + return -1; >>> + cpylen = dstlen > seglen ? seglen : dstlen; >>> + memcpy(dstaddr, mapaddr, cpylen); >>> + srcoffset += cpylen; >>> + dstaddr += cpylen; >>> + dstlen -= cpylen; >>> + } >>> + >>> + return 0; >>> +} >>> + >>> +int odp_packet_copy_from_memory(odp_packet_t dstpkt, size_t dstoffset, >>> + void *srcmem, size_t srclen) >>> +{ >>> + void *mapaddr; >>> + size_t seglen, cpylen; >>> + uint8_t *srcaddr = (uint8_t *)srcmem; >>> + >>> + while (srclen > 0) { >>> + mapaddr = odp_packet_offset_map(dstpkt, dstoffset, >>> &seglen); >>> + if (mapaddr == NULL) >>> + return -1; >>> + cpylen = srclen > seglen ? seglen : srclen; >>> + memcpy(mapaddr, srcaddr, cpylen); >>> + dstoffset += cpylen; >>> + srcaddr += cpylen; >>> + srclen -= cpylen; >>> + } >>> + >>> + return 0; >>> +} >>> + >>> +odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool) >>> +{ >>> + size_t pktlen = odp_packet_len(pkt); >>> + const size_t meta_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, >>> buf_hdr); >>> + odp_packet_t newpkt = odp_packet_alloc_len(pool, pktlen); >>> + odp_packet_hdr_t *newhdr, *srchdr; >>> + uint8_t *newstart, *srcstart; >>> + >>> + if (newpkt != ODP_PACKET_INVALID) { >>> + /* Must copy meta data first, followed by packet data */ >>> + srchdr = odp_packet_hdr(pkt); >>> + newhdr = odp_packet_hdr(newpkt); >>> + newstart = (uint8_t *)newhdr + meta_offset; >>> + srcstart = (uint8_t *)srchdr + meta_offset; >>> + >>> + memcpy(newstart, srcstart, >>> + sizeof(odp_packet_hdr_t) - meta_offset); >>> + >>> + if (odp_packet_copy_to_packet(newpkt, 0, pkt, 0, pktlen) >>> != 0) { >>> + odp_packet_free(newpkt); >>> + newpkt = ODP_PACKET_INVALID; >>> + } >>> + } >>> + >>> + return newpkt; >>> +} >>> + >>> +odp_packet_t odp_packet_clone(odp_packet_t pkt) >>> +{ >>> + return odp_packet_copy(pkt, >>> odp_packet_hdr(pkt)->buf_hdr.pool_hdl); >>> +} >>> + >>> +odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool) >>> +{ >>> + if (odp_pool_to_entry(pool)->s.params.buf_type != >>> + ODP_BUFFER_TYPE_PACKET) >>> + return ODP_PACKET_INVALID; >>> + >>> + return (odp_packet_t)buffer_alloc(pool, 0); >>> +} >>> + >>> +odp_packet_t odp_packet_alloc_len(odp_buffer_pool_t pool, size_t len) >>> +{ >>> + if (odp_pool_to_entry(pool)->s.params.buf_type != >>> + ODP_BUFFER_TYPE_PACKET) >>> + return ODP_PACKET_INVALID; >>> + >>> + return (odp_packet_t)buffer_alloc(pool, len); >>> +} >>> + >>> +void odp_packet_free(odp_packet_t pkt) >>> +{ >>> + odp_buffer_free((odp_buffer_t)pkt); >>> +} >>> + >>> +odp_packet_t odp_packet_split(odp_packet_t pkt, size_t offset, >>> + size_t hr, size_t tr) >>> +{ >>> + size_t len = odp_packet_len(pkt); >>> + odp_buffer_pool_t pool = odp_packet_pool(pkt); >>> + size_t pool_hr = odp_buffer_pool_headroom(pool); >>> + size_t pkt_tr = odp_packet_tailroom(pkt); >>> + odp_packet_t splitpkt; >>> + size_t splitlen = len - offset; >>> + >>> + if (offset >= len) >>> + return ODP_PACKET_INVALID; >>> + >>> + /* Erratum: We don't handle this rare corner case */ >>> + if (tr > pkt_tr + splitlen) >>> + return ODP_PACKET_INVALID; >>> + >>> + if (hr > pool_hr) >>> + splitlen += (hr - pool_hr); >>> + >>> + splitpkt = odp_packet_alloc_len(pool, splitlen); >>> + >>> + if (splitpkt != ODP_PACKET_INVALID) { >>> + if (hr > pool_hr) { >>> + odp_packet_pull_head(splitpkt, hr - pool_hr); >>> + splitlen -= (hr - pool_hr); >>> + } >>> + odp_packet_copy_to_packet(splitpkt, 0, >>> + pkt, offset, splitlen); >>> + odp_packet_pull_tail(pkt, splitlen); >>> + } >>> + >>> + return splitpkt; >>> +} >>> + >>> +odp_packet_t odp_packet_join(odp_packet_t pkt1, odp_packet_t pkt2) >>> +{ >>> + size_t len1 = odp_packet_len(pkt1); >>> + size_t len2 = odp_packet_len(pkt2); >>> + odp_packet_t joinpkt; >>> + void *udata1, *udata_join; >>> + size_t udata_size1, udata_size_join; >>> + >>> + /* Optimize if pkt1 is big enough to hold pkt2 */ >>> + if (odp_packet_push_tail(pkt1, len2) == 0) { >>> + odp_packet_copy_to_packet(pkt1, len1, >>> + pkt2, 0, len2); >>> + odp_packet_free(pkt2); >>> + return pkt1; >>> + } >>> + >>> + /* Otherwise join into a new packet */ >>> + joinpkt = odp_packet_alloc_len(odp_packet_pool(pkt1), >>> + len1 + len2); >>> + >>> + if (joinpkt != ODP_PACKET_INVALID) { >>> + odp_packet_copy_to_packet(joinpkt, 0, pkt1, 0, len1); >>> + odp_packet_copy_to_packet(joinpkt, len1, pkt2, 0, len2); >>> + >>> + /* Copy user metadata if present */ >>> + udata1 = odp_packet_udata(pkt1, &udata_size1); >>> + udata_join = odp_packet_udata(joinpkt, &udata_size_join); >>> + >>> + if (udata1 != NULL && udata_join != NULL) >>> + memcpy(udata_join, udata1, >>> + udata_size_join > udata_size1 ? >>> + udata_size1 : udata_size_join); >>> + >>> + odp_packet_free(pkt1); >>> + odp_packet_free(pkt2); >>> + } >>> + >>> + return joinpkt; >>> +} >>> + >>> +uint32_t odp_packet_refcount(odp_packet_t pkt) >>> +{ >>> + return odp_buffer_refcount(&odp_packet_hdr(pkt)->buf_hdr); >>> +} >>> + >>> +uint32_t odp_packet_incr_refcount(odp_packet_t pkt, uint32_t val) >>> +{ >>> + return odp_buffer_incr_refcount(&odp_packet_hdr(pkt)->buf_hdr, >>> val); >>> +} >>> + >>> +uint32_t odp_packet_decr_refcount(odp_packet_t pkt, uint32_t val) >>> +{ >>> + return odp_buffer_decr_refcount(&odp_packet_hdr(pkt)->buf_hdr, >>> val); >>> +} >>> + >>> +/** >>> + * Parser helper function for IPv4 >>> + */ >>> +static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >>> + uint8_t **parseptr, size_t *offset) >>> +{ >>> + odph_ipv4hdr_t *ipv4 = (odph_ipv4hdr_t *)*parseptr; >>> + uint8_t ver = ODPH_IPV4HDR_VER(ipv4->ver_ihl); >>> + uint8_t ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); >>> + uint16_t frag_offset; >>> + >>> + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv4->tot_len); >>> + >>> + if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN) || >>> + odp_unlikely(ver != 4) || >>> + (pkt_hdr->l3_len > pkt_hdr->frame_len - *offset)) { >>> + pkt_hdr->error_flags.ip_err = 1; >>> + return 0; >>> + } >>> + >>> + *offset += ihl * 4; >>> + *parseptr += ihl * 4; >>> + >>> + if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) >>> + pkt_hdr->input_flags.ipopt = 1; >>> + >>> + /* A packet is a fragment if: >>> + * "more fragments" flag is set (all fragments except the last) >>> + * OR >>> + * "fragment offset" field is nonzero (all fragments except the >>> first) >>> + */ >>> + frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); >>> + if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) >>> + pkt_hdr->input_flags.ipfrag = 1; >>> + >>> + if (ipv4->proto == ODPH_IPPROTO_ESP || >>> + ipv4->proto == ODPH_IPPROTO_AH) { >>> + pkt_hdr->input_flags.ipsec = 1; >>> + return 0; >>> + } >>> + >>> + /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after >>> return */ >>> + >>> + *offset = sizeof(uint32_t) * ihl; >>> + return ipv4->proto; >>> +} >>> + >>> +/** >>> + * Parser helper function for IPv6 >>> + */ >>> +static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >>> + uint8_t **parseptr, size_t *offset) >>> +{ >>> + odph_ipv6hdr_t *ipv6 = (odph_ipv6hdr_t *)*parseptr; >>> + odph_ipv6hdr_ext_t *ipv6ext; >>> + >>> + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv6->payload_len); >>> + >>> + /* Basic sanity checks on IPv6 header */ >>> + if (ipv6->ver != 6 || >>> + pkt_hdr->l3_len > pkt_hdr->frame_len - *offset) { >>> + pkt_hdr->error_flags.ip_err = 1; >>> + return 0; >>> + } >>> + >>> + /* Skip past IPv6 header */ >>> + *offset += sizeof(odph_ipv6hdr_t); >>> + *parseptr += sizeof(odph_ipv6hdr_t); >>> + >>> + >>> + /* Skip past any IPv6 extension headers */ >>> + if (ipv6->next_hdr == ODPH_IPPROTO_HOPOPTS || >>> + ipv6->next_hdr == ODPH_IPPROTO_ROUTE) { >>> + pkt_hdr->input_flags.ipopt = 1; >>> + >>> + do { >>> + ipv6ext = (odph_ipv6hdr_ext_t *)*parseptr; >>> + uint16_t extlen = 8 + ipv6ext->ext_len * 8; >>> + >>> + *offset += extlen; >>> + *parseptr += extlen; >>> + } while ((ipv6ext->next_hdr == ODPH_IPPROTO_HOPOPTS || >>> + ipv6ext->next_hdr == ODPH_IPPROTO_ROUTE) && >>> + *offset < pkt_hdr->frame_len); >>> + >>> + if (*offset >= pkt_hdr->l3_offset + ipv6->payload_len) { >>> + pkt_hdr->error_flags.ip_err = 1; >>> + return 0; >>> + } >>> + >>> + if (ipv6ext->next_hdr == ODPH_IPPROTO_FRAG) >>> + pkt_hdr->input_flags.ipfrag = 1; >>> + >>> + return ipv6ext->next_hdr; >>> + } >>> + >>> + if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { >>> + pkt_hdr->input_flags.ipopt = 1; >>> + pkt_hdr->input_flags.ipfrag = 1; >>> + } >>> + >>> + return ipv6->next_hdr; >>> +} >>> + >>> +/** >>> + * Parser helper function for TCP >>> + */ >>> +static inline void parse_tcp(odp_packet_hdr_t *pkt_hdr, >>> + uint8_t **parseptr, size_t *offset) >>> +{ >>> + odph_tcphdr_t *tcp = (odph_tcphdr_t *)*parseptr; >>> + >>> + if (tcp->hl < sizeof(odph_tcphdr_t)/sizeof(uint32_t)) >>> + pkt_hdr->error_flags.tcp_err = 1; >>> + else if ((uint32_t)tcp->hl * 4 > sizeof(odph_tcphdr_t)) >>> + pkt_hdr->input_flags.tcpopt = 1; >>> + >>> + pkt_hdr->l4_len = pkt_hdr->l3_len + >>> + pkt_hdr->l3_offset - pkt_hdr->l4_offset; >>> + >>> + *offset += sizeof(odph_tcphdr_t); >>> + *parseptr += sizeof(odph_tcphdr_t); >>> +} >>> + >>> +/** >>> + * Parser helper function for UDP >>> + */ >>> +static inline void parse_udp(odp_packet_hdr_t *pkt_hdr, >>> + uint8_t **parseptr, size_t *offset) >>> +{ >>> + odph_udphdr_t *udp = (odph_udphdr_t *)*parseptr; >>> + uint32_t udplen = odp_be_to_cpu_16(udp->length); >>> + >>> + if (udplen < sizeof(odph_udphdr_t) || >>> + udplen > (pkt_hdr->l3_len + >>> + pkt_hdr->l3_offset - pkt_hdr->l4_offset)) { >>> + pkt_hdr->error_flags.udp_err = 1; >>> + } >>> + >>> + pkt_hdr->l4_len = udplen; >>> + >>> + *offset += sizeof(odph_udphdr_t); >>> + *parseptr += sizeof(odph_udphdr_t); >>> +} >>> + >>> +/** >>> + * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP >>> + * >>> + * Internal function: caller is resposible for passing only valid >>> packet handles >>> + * , lengths and offsets (usually done&called in packet input). >>> + * >>> + * @param pkt Packet handle >>> + */ >>> +int odp_packet_parse(odp_packet_t pkt) >>> +{ >>> + odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >>> + odph_ethhdr_t *eth; >>> + odph_vlanhdr_t *vlan; >>> + uint16_t ethtype; >>> + uint8_t *parseptr; >>> + size_t offset, seglen; >>> + uint8_t ip_proto = 0; >>> + >>> + /* Reset parser metadata for new parse */ >>> + pkt_hdr->error_flags.all = 0; >>> + pkt_hdr->input_flags.all = 0; >>> + pkt_hdr->output_flags.all = 0; >>> + pkt_hdr->l2_offset = 0; >>> + pkt_hdr->l3_offset = 0; >>> + pkt_hdr->l4_offset = 0; >>> + pkt_hdr->payload_offset = 0; >>> + pkt_hdr->vlan_s_tag = 0; >>> + pkt_hdr->vlan_c_tag = 0; >>> + pkt_hdr->l3_protocol = 0; >>> + pkt_hdr->l4_protocol = 0; >>> + >>> + /* We only support Ethernet for now */ >>> + pkt_hdr->input_flags.eth = 1; >>> + >>> + if (odp_unlikely(pkt_hdr->frame_len < ODPH_ETH_LEN_MIN)) { >>> + pkt_hdr->error_flags.frame_len = 1; >>> + goto parse_exit; >>> + } else if (pkt_hdr->frame_len > ODPH_ETH_LEN_MAX) { >>> + pkt_hdr->input_flags.jumbo = 1; >>> + } >>> + >>> + /* Assume valid L2 header, no CRC/FCS check in SW */ >>> + pkt_hdr->input_flags.l2 = 1; >>> + >>> + eth = (odph_ethhdr_t *)odp_packet_map(pkt, &seglen); >>> + offset = sizeof(odph_ethhdr_t); >>> + parseptr = (uint8_t *)ð->type; >>> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); >>> + >>> + /* Parse the VLAN header(s), if present */ >>> + if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { >>> + pkt_hdr->input_flags.vlan_qinq = 1; >>> + pkt_hdr->input_flags.vlan = 1; >>> + vlan = (odph_vlanhdr_t *)(void *)parseptr; >>> + pkt_hdr->vlan_s_tag = ((ethtype << 16) | >>> + odp_be_to_cpu_16(vlan->tci)); >>> + offset += sizeof(odph_vlanhdr_t); >>> + parseptr += sizeof(odph_vlanhdr_t); >>> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >>> *)parseptr)); >>> + } >>> + >>> + if (ethtype == ODPH_ETHTYPE_VLAN) { >>> + pkt_hdr->input_flags.vlan = 1; >>> + vlan = (odph_vlanhdr_t *)(void *)parseptr; >>> + pkt_hdr->vlan_c_tag = ((ethtype << 16) | >>> + odp_be_to_cpu_16(vlan->tci)); >>> + offset += sizeof(odph_vlanhdr_t); >>> + parseptr += sizeof(odph_vlanhdr_t); >>> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >>> *)parseptr)); >>> + } >>> + >>> + /* Check for SNAP vs. DIX */ >>> + if (ethtype < ODPH_ETH_LEN_MAX) { >>> + pkt_hdr->input_flags.snap = 1; >>> + if (ethtype > pkt_hdr->frame_len - offset) { >>> + pkt_hdr->error_flags.snap_len = 1; >>> + goto parse_exit; >>> + } >>> + offset += 8; >>> + parseptr += 8; >>> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >>> *)parseptr)); >>> + } >>> + >>> + /* Consume Ethertype for Layer 3 parse */ >>> + parseptr += 2; >>> + >>> + /* Set l3_offset+flag only for known ethtypes */ >>> + pkt_hdr->input_flags.l3 = 1; >>> + pkt_hdr->l3_offset = offset; >>> + pkt_hdr->l3_protocol = ethtype; >>> + >>> + /* Parse Layer 3 headers */ >>> + switch (ethtype) { >>> + case ODPH_ETHTYPE_IPV4: >>> + pkt_hdr->input_flags.ipv4 = 1; >>> + ip_proto = parse_ipv4(pkt_hdr, &parseptr, &offset); >>> + break; >>> + >>> + case ODPH_ETHTYPE_IPV6: >>> + pkt_hdr->input_flags.ipv6 = 1; >>> + ip_proto = parse_ipv6(pkt_hdr, &parseptr, &offset); >>> + break; >>> + >>> + case ODPH_ETHTYPE_ARP: >>> + pkt_hdr->input_flags.arp = 1; >>> + ip_proto = 255; /* Reserved invalid by IANA */ >>> + break; >>> + >>> + default: >>> + pkt_hdr->input_flags.l3 = 0; >>> + ip_proto = 255; /* Reserved invalid by IANA */ >>> + } >>> + >>> + /* Set l4_offset+flag only for known ip_proto */ >>> + pkt_hdr->input_flags.l4 = 1; >>> + pkt_hdr->l4_offset = offset; >>> + pkt_hdr->l4_protocol = ip_proto; >>> + >>> + /* Parse Layer 4 headers */ >>> + switch (ip_proto) { >>> + case ODPH_IPPROTO_ICMP: >>> + pkt_hdr->input_flags.icmp = 1; >>> + break; >>> + >>> + case ODPH_IPPROTO_TCP: >>> + pkt_hdr->input_flags.tcp = 1; >>> + parse_tcp(pkt_hdr, &parseptr, &offset); >>> + break; >>> + >>> + case ODPH_IPPROTO_UDP: >>> + pkt_hdr->input_flags.udp = 1; >>> + parse_udp(pkt_hdr, &parseptr, &offset); >>> + break; >>> + >>> + case ODPH_IPPROTO_AH: >>> + case ODPH_IPPROTO_ESP: >>> + pkt_hdr->input_flags.ipsec = 1; >>> + break; >>> + >>> + default: >>> + pkt_hdr->input_flags.l4 = 0; >>> + break; >>> + } >>> + >>> + /* >>> + * Anything beyond what we parse here is considered payload. >>> + * Note: Payload is really only relevant for TCP and UDP. For >>> + * all other protocols, the payload offset will point to the >>> + * final header (ARP, ICMP, AH, ESP, or IP Fragment. >>> + */ >>> + pkt_hdr->payload_offset = offset; >>> + >>> +parse_exit: >>> + return pkt_hdr->error_flags.all != 0; >>> } >>> diff --git a/platform/linux-generic/odp_packet_flags.c >>> b/platform/linux-generic/odp_packet_flags.c >>> deleted file mode 100644 >>> index 06fdeed..0000000 >>> --- a/platform/linux-generic/odp_packet_flags.c >>> +++ /dev/null >>> @@ -1,202 +0,0 @@ >>> -/* Copyright (c) 2014, Linaro Limited >>> - * All rights reserved. >>> - * >>> - * SPDX-License-Identifier: BSD-3-Clause >>> - */ >>> - >>> -#include <odp_packet_flags.h> >>> -#include <odp_packet_internal.h> >>> - >>> - >>> -int odp_packet_error(odp_packet_t pkt) >>> -{ >>> - return (odp_packet_hdr(pkt)->error_flags.all != 0); >>> -} >>> - >>> -/* Get Error Flags */ >>> - >>> -int odp_packet_errflag_frame_len(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->error_flags.frame_len; >>> -} >>> - >>> -/* Get Input Flags */ >>> - >>> -int odp_packet_inflag_l2(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.l2; >>> -} >>> - >>> -int odp_packet_inflag_l3(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.l3; >>> -} >>> - >>> -int odp_packet_inflag_l4(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.l4; >>> -} >>> - >>> -int odp_packet_inflag_eth(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.eth; >>> -} >>> - >>> -int odp_packet_inflag_jumbo(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.jumbo; >>> -} >>> - >>> -int odp_packet_inflag_vlan(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.vlan; >>> -} >>> - >>> -int odp_packet_inflag_vlan_qinq(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.vlan_qinq; >>> -} >>> - >>> -int odp_packet_inflag_arp(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.arp; >>> -} >>> - >>> -int odp_packet_inflag_ipv4(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.ipv4; >>> -} >>> - >>> -int odp_packet_inflag_ipv6(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.ipv6; >>> -} >>> - >>> -int odp_packet_inflag_ipfrag(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.ipfrag; >>> -} >>> - >>> -int odp_packet_inflag_ipopt(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.ipopt; >>> -} >>> - >>> -int odp_packet_inflag_ipsec(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.ipsec; >>> -} >>> - >>> -int odp_packet_inflag_udp(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.udp; >>> -} >>> - >>> -int odp_packet_inflag_tcp(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.tcp; >>> -} >>> - >>> -int odp_packet_inflag_sctp(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.sctp; >>> -} >>> - >>> -int odp_packet_inflag_icmp(odp_packet_t pkt) >>> -{ >>> - return odp_packet_hdr(pkt)->input_flags.icmp; >>> -} >>> - >>> -/* Set Output Flags */ >>> - >>> -void odp_packet_outflag_l4_chksum(odp_packet_t pkt) >>> -{ >>> - odp_packet_hdr(pkt)->output_flags.l4_chksum = 1; >>> -} >>> - >>> -/* Set Input Flags */ >>> - >>> -void odp_packet_set_inflag_l2(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.l2 = val; >>> -} >>> - >>> -void odp_packet_set_inflag_l3(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.l3 = val; >>> -} >>> - >>> -void odp_packet_set_inflag_l4(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.l4 = val; >>> -} >>> - >>> -void odp_packet_set_inflag_eth(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.eth = val; >>> -} >>> - >>> -void odp_packet_set_inflag_jumbo(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.jumbo = val; >>> -} >>> - >>> -void odp_packet_set_inflag_vlan(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.vlan = val; >>> -} >>> - >>> -void odp_packet_set_inflag_vlan_qinq(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.vlan_qinq = val; >>> -} >>> - >>> -void odp_packet_set_inflag_arp(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.arp = val; >>> -} >>> - >>> -void odp_packet_set_inflag_ipv4(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.ipv4 = val; >>> -} >>> - >>> -void odp_packet_set_inflag_ipv6(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.ipv6 = val; >>> -} >>> - >>> -void odp_packet_set_inflag_ipfrag(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.ipfrag = val; >>> -} >>> - >>> -void odp_packet_set_inflag_ipopt(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.ipopt = val; >>> -} >>> - >>> -void odp_packet_set_inflag_ipsec(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.ipsec = val; >>> -} >>> - >>> -void odp_packet_set_inflag_udp(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.udp = val; >>> -} >>> - >>> -void odp_packet_set_inflag_tcp(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.tcp = val; >>> -} >>> - >>> -void odp_packet_set_inflag_sctp(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.sctp = val; >>> -} >>> - >>> -void odp_packet_set_inflag_icmp(odp_packet_t pkt, int val) >>> -{ >>> - odp_packet_hdr(pkt)->input_flags.icmp = val; >>> -} >>> -- >>> 1.8.3.2 >>> >>> >>> _______________________________________________ >>> lng-odp mailing list >>> lng-odp@lists.linaro.org >>> http://lists.linaro.org/mailman/listinfo/lng-odp >>> >> >> >
Sounds reasonable. It will be in the next rev, along with Tara's latest bug fix. Thanks. Bill On Tue, Nov 11, 2014 at 10:42 AM, Bala Manoharan <bala.manoharan@linaro.org> wrote: > I am okay with the behaviour for "init_params" value, my concern is with > "params" value. > Since this params deals with the number of buffer in the pool and buf_size. > I would like to return an error if the params value is NULL. > > Regards, > Bala > > On 11 November 2014 22:09, Bill Fischofer <bill.fischofer@linaro.org> > wrote: > >> The API allows the init_params to be NULL if user meta data is not >> required. I just added the corresponding support to the params to have the >> same behavior. I have no problem failing the call if that parameter is >> missing. >> >> Bill >> >> On Tue, Nov 11, 2014 at 10:34 AM, Bala Manoharan < >> bala.manoharan@linaro.org> wrote: >> >>> >>> >>> On 11 November 2014 09:33, Bill Fischofer <bill.fischofer@linaro.org> >>> wrote: >>> >>>> Signed-off-by: Bill Fischofer <bill.fischofer@linaro.org> >>>> --- >>>> platform/linux-generic/odp_buffer.c | 263 ++++++- >>>> platform/linux-generic/odp_buffer_pool.c | 661 +++++++--------- >>>> platform/linux-generic/odp_packet.c | 1200 >>>> +++++++++++++++++++++++------ >>>> platform/linux-generic/odp_packet_flags.c | 202 ----- >>>> 4 files changed, 1455 insertions(+), 871 deletions(-) >>>> delete mode 100644 platform/linux-generic/odp_packet_flags.c >>>> >>>> diff --git a/platform/linux-generic/odp_buffer.c >>>> b/platform/linux-generic/odp_buffer.c >>>> index e54e0e7..e47c722 100644 >>>> --- a/platform/linux-generic/odp_buffer.c >>>> +++ b/platform/linux-generic/odp_buffer.c >>>> @@ -5,46 +5,259 @@ >>>> */ >>>> >>>> #include <odp_buffer.h> >>>> -#include <odp_buffer_internal.h> >>>> #include <odp_buffer_pool_internal.h> >>>> +#include <odp_buffer_internal.h> >>>> +#include <odp_buffer_inlines.h> >>>> >>>> #include <string.h> >>>> #include <stdio.h> >>>> >>>> +void *odp_buffer_offset_map(odp_buffer_t buf, size_t offset, size_t >>>> *seglen) >>>> +{ >>>> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >>>> >>>> -void *odp_buffer_addr(odp_buffer_t buf) >>>> + if (offset > buf_hdr->size) >>>> + return NULL; >>>> + >>>> + return buffer_map(buf_hdr, offset, seglen, buf_hdr->size); >>>> +} >>>> + >>>> +void odp_buffer_offset_unmap(odp_buffer_t buf ODP_UNUSED, >>>> + size_t offset ODP_UNUSED) >>>> { >>>> - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >>>> +} >>>> >>>> - return hdr->addr; >>>> +size_t odp_buffer_segment_count(odp_buffer_t buf) >>>> +{ >>>> + return odp_buf_to_hdr(buf)->segcount; >>>> } >>>> >>>> +int odp_buffer_is_segmented(odp_buffer_t buf) >>>> +{ >>>> + return odp_buf_to_hdr(buf)->segcount > 1; >>>> +} >>>> >>>> -size_t odp_buffer_size(odp_buffer_t buf) >>>> +odp_buffer_segment_t odp_buffer_segment_by_index(odp_buffer_t buf, >>>> + size_t ndx) >>>> { >>>> - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >>>> + return buffer_segment(odp_buf_to_hdr(buf), ndx); >>>> +} >>>> >>>> - return hdr->size; >>>> +odp_buffer_segment_t odp_buffer_segment_next(odp_buffer_t buf, >>>> + odp_buffer_segment_t seg) >>>> +{ >>>> + return segment_next(odp_buf_to_hdr(buf), seg); >>>> } >>>> >>>> +void *odp_buffer_segment_map(odp_buffer_t buf, odp_buffer_segment_t >>>> seg, >>>> + size_t *seglen) >>>> +{ >>>> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >>>> >>>> -int odp_buffer_type(odp_buffer_t buf) >>>> + return segment_map(buf_hdr, seg, seglen, buf_hdr->size); >>>> +} >>>> + >>>> +void odp_buffer_segment_unmap(odp_buffer_segment_t seg ODP_UNUSED) >>>> { >>>> - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >>>> +} >>>> + >>>> +odp_buffer_t odp_buffer_split(odp_buffer_t buf, size_t offset) >>>> +{ >>>> + size_t size = odp_buffer_size(buf); >>>> + odp_buffer_pool_t pool = odp_buffer_pool(buf); >>>> + odp_buffer_t splitbuf; >>>> + size_t splitsize = size - offset; >>>> + >>>> + if (offset >= size) >>>> + return ODP_BUFFER_INVALID; >>>> + >>>> + splitbuf = buffer_alloc(pool, splitsize); >>>> + >>>> + if (splitbuf != ODP_BUFFER_INVALID) { >>>> + buffer_copy_to_buffer(splitbuf, 0, buf, splitsize, >>>> splitsize); >>>> + odp_buffer_trim(buf, splitsize); >>>> + } >>>> >>>> - return hdr->type; >>>> + return splitbuf; >>>> } >>>> >>>> +odp_buffer_t odp_buffer_join(odp_buffer_t buf1, odp_buffer_t buf2) >>>> +{ >>>> + size_t size1 = odp_buffer_size(buf1); >>>> + size_t size2 = odp_buffer_size(buf2); >>>> + odp_buffer_t joinbuf; >>>> + void *udata1, *udata_join; >>>> + size_t udata_size1, udata_size_join; >>>> >>>> -int odp_buffer_is_valid(odp_buffer_t buf) >>>> + joinbuf = buffer_alloc(odp_buffer_pool(buf1), size1 + size2); >>>> + >>>> + if (joinbuf != ODP_BUFFER_INVALID) { >>>> + buffer_copy_to_buffer(joinbuf, 0, buf1, 0, size1); >>>> + buffer_copy_to_buffer(joinbuf, size1, buf2, 0, size2); >>>> + >>>> + /* Copy user metadata if present */ >>>> + udata1 = odp_buffer_udata(buf1, &udata_size1); >>>> + udata_join = odp_buffer_udata(joinbuf, >>>> &udata_size_join); >>>> + >>>> + if (udata1 != NULL && udata_join != NULL) >>>> + memcpy(udata_join, udata1, >>>> + udata_size_join > udata_size1 ? >>>> + udata_size1 : udata_size_join); >>>> + >>>> + odp_buffer_free(buf1); >>>> + odp_buffer_free(buf2); >>>> + } >>>> + >>>> + return joinbuf; >>>> +} >>>> + >>>> +odp_buffer_t odp_buffer_trim(odp_buffer_t buf, size_t bytes) >>>> { >>>> - odp_buffer_bits_t handle; >>>> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >>>> + >>>> + if (bytes >= buf_hdr->size) >>>> + return ODP_BUFFER_INVALID; >>>> + >>>> + buf_hdr->size -= bytes; >>>> + size_t bufsegs = buf_hdr->size / buf_hdr->segsize; >>>> + >>>> + if (buf_hdr->size % buf_hdr->segsize > 0) >>>> + bufsegs++; >>>> + >>>> + pool_entry_t *pool = odp_pool_to_entry(buf_hdr->pool_hdl); >>>> >>>> - handle.u32 = buf; >>>> + /* Return excess segments back to block list */ >>>> + while (bufsegs > buf_hdr->segcount) >>>> + ret_blk(&pool->s, buf_hdr->addr[--buf_hdr->segcount]); >>>> >>>> - return (handle.index != ODP_BUFFER_INVALID_INDEX); >>>> + return buf; >>>> } >>>> >>>> +odp_buffer_t odp_buffer_extend(odp_buffer_t buf, size_t bytes) >>>> +{ >>>> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >>>> + >>>> + size_t lastseg = buf_hdr->size % buf_hdr->segsize; >>>> + >>>> + if (bytes <= buf_hdr->segsize - lastseg) { >>>> + buf_hdr->size += bytes; >>>> + return buf; >>>> + } >>>> + >>>> + pool_entry_t *pool = odp_pool_to_entry(buf_hdr->pool_hdl); >>>> + int extsize = buf_hdr->size + bytes; >>>> + >>>> + /* Extend buffer by adding additional segments to it */ >>>> + if (extsize > ODP_CONFIG_BUF_MAX_SIZE || >>>> pool->s.flags.unsegmented) >>>> + return ODP_BUFFER_INVALID; >>>> + >>>> + size_t segcount = buf_hdr->segcount; >>>> + size_t extsegs = extsize / buf_hdr->segsize; >>>> + >>>> + if (extsize % buf_hdr->segsize > 0) >>>> + extsize++; >>>> + >>>> + while (extsegs > buf_hdr->segcount) { >>>> + void *newblk = get_blk(&pool->s); >>>> + >>>> + /* If we fail to get all the blocks we need, back out */ >>>> + if (newblk == NULL) { >>>> + while (segcount > buf_hdr->segcount) >>>> + ret_blk(&pool->s, >>>> buf_hdr->addr[--segcount]); >>>> + return ODP_BUFFER_INVALID; >>>> + } >>>> + >>>> + buf_hdr->addr[segcount++] = newblk; >>>> + } >>>> + >>>> + buf_hdr->segcount = segcount; >>>> + buf_hdr->size = extsize; >>>> + >>>> + return buf; >>>> +} >>>> + >>>> +odp_buffer_t odp_buffer_clone(odp_buffer_t buf) >>>> +{ >>>> + return odp_buffer_copy(buf, odp_buf_to_hdr(buf)->pool_hdl); >>>> +} >>>> + >>>> +odp_buffer_t odp_buffer_copy(odp_buffer_t buf, odp_buffer_pool_t pool) >>>> +{ >>>> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >>>> + size_t len = buf_hdr->size; >>>> + odp_buffer_t cpybuf = buffer_alloc(pool, len); >>>> + >>>> + if (cpybuf == ODP_BUFFER_INVALID) >>>> + return ODP_BUFFER_INVALID; >>>> + >>>> + if (buffer_copy_to_buffer(cpybuf, 0, buf, 0, len) == 0) >>>> + return cpybuf; >>>> + >>>> + odp_buffer_free(cpybuf); >>>> + return ODP_BUFFER_INVALID; >>>> +} >>>> + >>>> +int buffer_copy_to_buffer(odp_buffer_t dstbuf, size_t dstoffset, >>>> + odp_buffer_t srcbuf, size_t srcoffset, >>>> + size_t len) >>>> +{ >>>> + void *dstmap; >>>> + void *srcmap; >>>> + size_t cpylen, minseg, dstseglen, srcseglen; >>>> + >>>> + while (len > 0) { >>>> + dstmap = odp_buffer_offset_map(dstbuf, dstoffset, >>>> &dstseglen); >>>> + srcmap = odp_buffer_offset_map(srcbuf, srcoffset, >>>> &srcseglen); >>>> + if (dstmap == NULL || srcmap == NULL) >>>> + return -1; >>>> + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; >>>> + cpylen = len > minseg ? minseg : len; >>>> + memcpy(dstmap, srcmap, cpylen); >>>> + srcoffset += cpylen; >>>> + dstoffset += cpylen; >>>> + len -= cpylen; >>>> + } >>>> + >>>> + return 0; >>>> +} >>>> + >>>> +void *odp_buffer_addr(odp_buffer_t buf) >>>> +{ >>>> + return odp_buf_to_hdr(buf)->addr[0]; >>>> +} >>>> + >>>> +odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf) >>>> +{ >>>> + return odp_buf_to_hdr(buf)->pool_hdl; >>>> +} >>>> + >>>> +size_t odp_buffer_size(odp_buffer_t buf) >>>> +{ >>>> + return odp_buf_to_hdr(buf)->size; >>>> +} >>>> + >>>> +odp_buffer_type_e odp_buffer_type(odp_buffer_t buf) >>>> +{ >>>> + return odp_buf_to_hdr(buf)->type; >>>> +} >>>> + >>>> +void *odp_buffer_udata(odp_buffer_t buf, size_t *usize) >>>> +{ >>>> + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >>>> + >>>> + *usize = hdr->udata_size; >>>> + return hdr->udata_addr; >>>> +} >>>> + >>>> +void *odp_buffer_udata_addr(odp_buffer_t buf) >>>> +{ >>>> + return odp_buf_to_hdr(buf)->udata_addr; >>>> +} >>>> + >>>> +int odp_buffer_is_valid(odp_buffer_t buf) >>>> +{ >>>> + return validate_buf(buf) != NULL; >>>> +} >>>> >>>> int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf) >>>> { >>>> @@ -63,27 +276,13 @@ int odp_buffer_snprint(char *str, size_t n, >>>> odp_buffer_t buf) >>>> len += snprintf(&str[len], n-len, >>>> " pool %i\n", hdr->pool_hdl); >>>> len += snprintf(&str[len], n-len, >>>> - " index %"PRIu32"\n", hdr->index); >>>> - len += snprintf(&str[len], n-len, >>>> - " phy_addr %"PRIu64"\n", hdr->phys_addr); >>>> - len += snprintf(&str[len], n-len, >>>> - " addr %p\n", hdr->addr); >>>> + " addr %p\n", hdr->addr[0]); >>>> len += snprintf(&str[len], n-len, >>>> " size %zu\n", hdr->size); >>>> len += snprintf(&str[len], n-len, >>>> - " cur_offset %zu\n", hdr->cur_offset); >>>> - len += snprintf(&str[len], n-len, >>>> " ref_count %i\n", hdr->ref_count); >>>> len += snprintf(&str[len], n-len, >>>> " type %i\n", hdr->type); >>>> - len += snprintf(&str[len], n-len, >>>> - " Scatter list\n"); >>>> - len += snprintf(&str[len], n-len, >>>> - " num_bufs %i\n", >>>> hdr->scatter.num_bufs); >>>> - len += snprintf(&str[len], n-len, >>>> - " pos %i\n", hdr->scatter.pos); >>>> - len += snprintf(&str[len], n-len, >>>> - " total_len %zu\n", >>>> hdr->scatter.total_len); >>>> >>>> return len; >>>> } >>>> @@ -100,9 +299,3 @@ void odp_buffer_print(odp_buffer_t buf) >>>> >>>> printf("\n%s\n", str); >>>> } >>>> - >>>> -void odp_buffer_copy_scatter(odp_buffer_t buf_dst, odp_buffer_t >>>> buf_src) >>>> -{ >>>> - (void)buf_dst; >>>> - (void)buf_src; >>>> -} >>>> diff --git a/platform/linux-generic/odp_buffer_pool.c >>>> b/platform/linux-generic/odp_buffer_pool.c >>>> index a48d7d6..f6161bb 100644 >>>> --- a/platform/linux-generic/odp_buffer_pool.c >>>> +++ b/platform/linux-generic/odp_buffer_pool.c >>>> @@ -6,8 +6,9 @@ >>>> >>>> #include <odp_std_types.h> >>>> #include <odp_buffer_pool.h> >>>> -#include <odp_buffer_pool_internal.h> >>>> #include <odp_buffer_internal.h> >>>> +#include <odp_buffer_pool_internal.h> >>>> +#include <odp_buffer_inlines.h> >>>> #include <odp_packet_internal.h> >>>> #include <odp_timer_internal.h> >>>> #include <odp_shared_memory.h> >>>> @@ -16,6 +17,7 @@ >>>> #include <odp_config.h> >>>> #include <odp_hints.h> >>>> #include <odp_debug.h> >>>> +#include <odph_eth.h> >>>> >>>> #include <string.h> >>>> #include <stdlib.h> >>>> @@ -33,36 +35,26 @@ >>>> #define LOCK_INIT(a) odp_spinlock_init(a) >>>> #endif >>>> >>>> - >>>> -#if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >>>> -#error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >>>> -#endif >>>> - >>>> -#define NULL_INDEX ((uint32_t)-1) >>>> - >>>> -union buffer_type_any_u { >>>> +typedef union buffer_type_any_u { >>>> odp_buffer_hdr_t buf; >>>> odp_packet_hdr_t pkt; >>>> odp_timeout_hdr_t tmo; >>>> -}; >>>> - >>>> -ODP_STATIC_ASSERT((sizeof(union buffer_type_any_u) % 8) == 0, >>>> - "BUFFER_TYPE_ANY_U__SIZE_ERR"); >>>> +} odp_anybuf_t; >>>> >>>> /* Any buffer type header */ >>>> typedef struct { >>>> union buffer_type_any_u any_hdr; /* any buffer type */ >>>> - uint8_t buf_data[]; /* start of buffer data >>>> area */ >>>> } odp_any_buffer_hdr_t; >>>> >>>> +typedef struct odp_any_hdr_stride { >>>> + uint8_t >>>> pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_any_buffer_hdr_t))]; >>>> +} odp_any_hdr_stride; >>>> >>>> -typedef union pool_entry_u { >>>> - struct pool_entry_s s; >>>> - >>>> - uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct >>>> pool_entry_s))]; >>>> - >>>> -} pool_entry_t; >>>> +#if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >>>> +#error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >>>> +#endif >>>> >>>> +#define NULL_INDEX ((uint32_t)-1) >>>> >>>> typedef struct pool_table_t { >>>> pool_entry_t pool[ODP_CONFIG_BUFFER_POOLS]; >>>> @@ -76,39 +68,6 @@ static pool_table_t *pool_tbl; >>>> /* Pool entry pointers (for inlining) */ >>>> void *pool_entry_ptr[ODP_CONFIG_BUFFER_POOLS]; >>>> >>>> - >>>> -static __thread odp_buffer_chunk_hdr_t >>>> *local_chunk[ODP_CONFIG_BUFFER_POOLS]; >>>> - >>>> - >>>> -static inline odp_buffer_pool_t pool_index_to_handle(uint32_t pool_id) >>>> -{ >>>> - return pool_id + 1; >>>> -} >>>> - >>>> - >>>> -static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl) >>>> -{ >>>> - return pool_hdl -1; >>>> -} >>>> - >>>> - >>>> -static inline void set_handle(odp_buffer_hdr_t *hdr, >>>> - pool_entry_t *pool, uint32_t index) >>>> -{ >>>> - odp_buffer_pool_t pool_hdl = pool->s.pool_hdl; >>>> - uint32_t pool_id = pool_handle_to_index(pool_hdl); >>>> - >>>> - if (pool_id >= ODP_CONFIG_BUFFER_POOLS) >>>> - ODP_ABORT("set_handle: Bad pool handle %u\n", pool_hdl); >>>> - >>>> - if (index > ODP_BUFFER_MAX_INDEX) >>>> - ODP_ERR("set_handle: Bad buffer index\n"); >>>> - >>>> - hdr->handle.pool_id = pool_id; >>>> - hdr->handle.index = index; >>>> -} >>>> - >>>> - >>>> int odp_buffer_pool_init_global(void) >>>> { >>>> uint32_t i; >>>> @@ -142,269 +101,167 @@ int odp_buffer_pool_init_global(void) >>>> return 0; >>>> } >>>> >>>> - >>>> -static odp_buffer_hdr_t *index_to_hdr(pool_entry_t *pool, uint32_t >>>> index) >>>> -{ >>>> - odp_buffer_hdr_t *hdr; >>>> - >>>> - hdr = (odp_buffer_hdr_t *)(pool->s.buf_base + index * >>>> pool->s.buf_size); >>>> - return hdr; >>>> -} >>>> - >>>> - >>>> -static void add_buf_index(odp_buffer_chunk_hdr_t *chunk_hdr, uint32_t >>>> index) >>>> -{ >>>> - uint32_t i = chunk_hdr->chunk.num_bufs; >>>> - chunk_hdr->chunk.buf_index[i] = index; >>>> - chunk_hdr->chunk.num_bufs++; >>>> -} >>>> - >>>> - >>>> -static uint32_t rem_buf_index(odp_buffer_chunk_hdr_t *chunk_hdr) >>>> +/** >>>> + * Buffer pool creation >>>> + */ >>>> +odp_buffer_pool_t odp_buffer_pool_create(const char *name, >>>> + odp_buffer_pool_param_t *args, >>>> + odp_buffer_pool_init_t >>>> *init_args) >>>> { >>>> - uint32_t index; >>>> + odp_buffer_pool_t pool_hdl = ODP_BUFFER_POOL_INVALID; >>>> + pool_entry_t *pool; >>>> uint32_t i; >>>> + odp_buffer_pool_param_t *params; >>>> + odp_buffer_pool_init_t *init_params; >>>> >>>> - i = chunk_hdr->chunk.num_bufs - 1; >>>> - index = chunk_hdr->chunk.buf_index[i]; >>>> - chunk_hdr->chunk.num_bufs--; >>>> - return index; >>>> -} >>>> - >>>> - >>>> -static odp_buffer_chunk_hdr_t *next_chunk(pool_entry_t *pool, >>>> - odp_buffer_chunk_hdr_t >>>> *chunk_hdr) >>>> -{ >>>> - uint32_t index; >>>> - >>>> - index = chunk_hdr->chunk.buf_index[ODP_BUFS_PER_CHUNK-1]; >>>> - if (index == NULL_INDEX) >>>> - return NULL; >>>> - else >>>> - return (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, >>>> index); >>>> -} >>>> - >>>> - >>>> -static odp_buffer_chunk_hdr_t *rem_chunk(pool_entry_t *pool) >>>> -{ >>>> - odp_buffer_chunk_hdr_t *chunk_hdr; >>>> - >>>> - chunk_hdr = pool->s.head; >>>> - if (chunk_hdr == NULL) { >>>> - /* Pool is empty */ >>>> - return NULL; >>>> - } >>>> - >>>> - pool->s.head = next_chunk(pool, chunk_hdr); >>>> - pool->s.free_bufs -= ODP_BUFS_PER_CHUNK; >>>> - >>>> - /* unlink */ >>>> - rem_buf_index(chunk_hdr); >>>> - return chunk_hdr; >>>> -} >>>> - >>>> - >>>> -static void add_chunk(pool_entry_t *pool, odp_buffer_chunk_hdr_t >>>> *chunk_hdr) >>>> -{ >>>> - if (pool->s.head) /* link pool head to the chunk */ >>>> - add_buf_index(chunk_hdr, pool->s.head->buf_hdr.index); >>>> - else >>>> - add_buf_index(chunk_hdr, NULL_INDEX); >>>> - >>>> - pool->s.head = chunk_hdr; >>>> - pool->s.free_bufs += ODP_BUFS_PER_CHUNK; >>>> -} >>>> + /* Default pool creation arguments */ >>>> + static odp_buffer_pool_param_t default_params = { >>>> + .buf_num = 1024, >>>> + .buf_size = ODP_CONFIG_BUF_SEG_SIZE, >>>> + .buf_type = ODP_BUFFER_TYPE_PACKET, >>>> + .buf_opts = ODP_BUFFER_OPTS_UNSEGMENTED, >>>> + }; >>>> >>> Do we need to have this behaviour of allocating default params if the >>> input params are NULL. >>> It would be better if the API returns error. >>> >>>> >>>> + /* Default initialization paramters */ >>>> + static odp_buffer_pool_init_t default_init_params = { >>>> + .udata_size = 0, >>>> + .buf_init = NULL, >>>> + .buf_init_arg = NULL, >>>> + }; >>>> >>>> -static void check_align(pool_entry_t *pool, odp_buffer_hdr_t *hdr) >>>> -{ >>>> - if (!ODP_ALIGNED_CHECK_POWER_2(hdr->addr, pool->s.user_align)) { >>>> - ODP_ABORT("check_align: user data align error %p, align >>>> %zu\n", >>>> - hdr->addr, pool->s.user_align); >>>> - } >>>> + /* Handle NULL input parameters */ >>>> + params = args == NULL ? &default_params : args; >>>> + init_params = init_args == NULL ? &default_init_params >>>> + : init_args; >>>> >>>> - if (!ODP_ALIGNED_CHECK_POWER_2(hdr, ODP_CACHE_LINE_SIZE)) { >>>> - ODP_ABORT("check_align: hdr align error %p, align %i\n", >>>> - hdr, ODP_CACHE_LINE_SIZE); >>>> - } >>>> -} >>>> + if (params->buf_type != ODP_BUFFER_TYPE_PACKET) >>>> + params->buf_opts |= ODP_BUFFER_OPTS_UNSEGMENTED; >>>> >>>> + int unsegmented = ((params->buf_opts & >>>> ODP_BUFFER_OPTS_UNSEGMENTED) == >>>> + ODP_BUFFER_OPTS_UNSEGMENTED); >>>> >>>> -static void fill_hdr(void *ptr, pool_entry_t *pool, uint32_t index, >>>> - int buf_type) >>>> -{ >>>> - odp_buffer_hdr_t *hdr = (odp_buffer_hdr_t *)ptr; >>>> - size_t size = pool->s.hdr_size; >>>> - uint8_t *buf_data; >>>> + uint32_t udata_stride = >>>> + ODP_CACHE_LINE_SIZE_ROUNDUP(init_params->udata_size); >>>> >>>> - if (buf_type == ODP_BUFFER_TYPE_CHUNK) >>>> - size = sizeof(odp_buffer_chunk_hdr_t); >>>> - >>>> - switch (pool->s.buf_type) { >>>> - odp_raw_buffer_hdr_t *raw_hdr; >>>> - odp_packet_hdr_t *packet_hdr; >>>> - odp_timeout_hdr_t *tmo_hdr; >>>> - odp_any_buffer_hdr_t *any_hdr; >>>> + uint32_t blk_size, buf_stride; >>>> >>>> + switch (params->buf_type) { >>>> case ODP_BUFFER_TYPE_RAW: >>>> - raw_hdr = ptr; >>>> - buf_data = raw_hdr->buf_data; >>>> + blk_size = params->buf_size; >>>> + buf_stride = sizeof(odp_buffer_hdr_stride); >>>> break; >>>> + >>>> case ODP_BUFFER_TYPE_PACKET: >>>> - packet_hdr = ptr; >>>> - buf_data = packet_hdr->buf_data; >>>> + if (unsegmented) >>>> + blk_size = >>>> + >>>> ODP_CACHE_LINE_SIZE_ROUNDUP(params->buf_size); >>>> + else >>>> + blk_size = ODP_ALIGN_ROUNDUP(params->buf_size, >>>> + >>>> ODP_CONFIG_BUF_SEG_SIZE); >>>> + buf_stride = sizeof(odp_packet_hdr_stride); >>>> break; >>>> + >>>> case ODP_BUFFER_TYPE_TIMEOUT: >>>> - tmo_hdr = ptr; >>>> - buf_data = tmo_hdr->buf_data; >>>> + blk_size = 0; /* Timeouts have no block data, only >>>> metadata */ >>>> + buf_stride = sizeof(odp_timeout_hdr_stride); >>>> break; >>>> + >>>> case ODP_BUFFER_TYPE_ANY: >>>> - any_hdr = ptr; >>>> - buf_data = any_hdr->buf_data; >>>> + if (unsegmented) >>>> + blk_size = >>>> + >>>> ODP_CACHE_LINE_SIZE_ROUNDUP(params->buf_size); >>>> + else >>>> + blk_size = ODP_ALIGN_ROUNDUP(params->buf_size, >>>> + >>>> ODP_CONFIG_BUF_SEG_SIZE); >>>> + buf_stride = sizeof(odp_any_hdr_stride); >>>> break; >>>> - default: >>>> - ODP_ABORT("Bad buffer type\n"); >>>> - } >>>> - >>>> - memset(hdr, 0, size); >>>> - >>>> - set_handle(hdr, pool, index); >>>> - >>>> - hdr->addr = &buf_data[pool->s.buf_offset - >>>> pool->s.hdr_size]; >>>> - hdr->index = index; >>>> - hdr->size = pool->s.user_size; >>>> - hdr->pool_hdl = pool->s.pool_hdl; >>>> - hdr->type = buf_type; >>>> - >>>> - check_align(pool, hdr); >>>> -} >>>> - >>>> - >>>> -static void link_bufs(pool_entry_t *pool) >>>> -{ >>>> - odp_buffer_chunk_hdr_t *chunk_hdr; >>>> - size_t hdr_size; >>>> - size_t data_size; >>>> - size_t data_align; >>>> - size_t tot_size; >>>> - size_t offset; >>>> - size_t min_size; >>>> - uint64_t pool_size; >>>> - uintptr_t buf_base; >>>> - uint32_t index; >>>> - uintptr_t pool_base; >>>> - int buf_type; >>>> - >>>> - buf_type = pool->s.buf_type; >>>> - data_size = pool->s.user_size; >>>> - data_align = pool->s.user_align; >>>> - pool_size = pool->s.pool_size; >>>> - pool_base = (uintptr_t) pool->s.pool_base_addr; >>>> - >>>> - if (buf_type == ODP_BUFFER_TYPE_RAW) { >>>> - hdr_size = sizeof(odp_raw_buffer_hdr_t); >>>> - } else if (buf_type == ODP_BUFFER_TYPE_PACKET) { >>>> - hdr_size = sizeof(odp_packet_hdr_t); >>>> - } else if (buf_type == ODP_BUFFER_TYPE_TIMEOUT) { >>>> - hdr_size = sizeof(odp_timeout_hdr_t); >>>> - } else if (buf_type == ODP_BUFFER_TYPE_ANY) { >>>> - hdr_size = sizeof(odp_any_buffer_hdr_t); >>>> - } else >>>> - ODP_ABORT("odp_buffer_pool_create: Bad type %i\n", >>>> buf_type); >>>> - >>>> - >>>> - /* Chunk must fit into buffer data area.*/ >>>> - min_size = sizeof(odp_buffer_chunk_hdr_t) - hdr_size; >>>> - if (data_size < min_size) >>>> - data_size = min_size; >>>> - >>>> - /* Roundup data size to full cachelines */ >>>> - data_size = ODP_CACHE_LINE_SIZE_ROUNDUP(data_size); >>>> - >>>> - /* Min cacheline alignment for buffer header and data */ >>>> - data_align = ODP_CACHE_LINE_SIZE_ROUNDUP(data_align); >>>> - offset = ODP_CACHE_LINE_SIZE_ROUNDUP(hdr_size); >>>> - >>>> - /* Multiples of cacheline size */ >>>> - if (data_size > data_align) >>>> - tot_size = data_size + offset; >>>> - else >>>> - tot_size = data_align + offset; >>>> - >>>> - /* First buffer */ >>>> - buf_base = ODP_ALIGN_ROUNDUP(pool_base + offset, data_align) - >>>> offset; >>>> - >>>> - pool->s.hdr_size = hdr_size; >>>> - pool->s.buf_base = buf_base; >>>> - pool->s.buf_size = tot_size; >>>> - pool->s.buf_offset = offset; >>>> - index = 0; >>>> - >>>> - chunk_hdr = (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, index); >>>> - pool->s.head = NULL; >>>> - pool_size -= buf_base - pool_base; >>>> - >>>> - while (pool_size > ODP_BUFS_PER_CHUNK * tot_size) { >>>> - int i; >>>> - >>>> - fill_hdr(chunk_hdr, pool, index, ODP_BUFFER_TYPE_CHUNK); >>>> - >>>> - index++; >>>> - >>>> - for (i = 0; i < ODP_BUFS_PER_CHUNK - 1; i++) { >>>> - odp_buffer_hdr_t *hdr = index_to_hdr(pool, >>>> index); >>>> - >>>> - fill_hdr(hdr, pool, index, buf_type); >>>> - >>>> - add_buf_index(chunk_hdr, index); >>>> - index++; >>>> - } >>>> - >>>> - add_chunk(pool, chunk_hdr); >>>> >>>> - chunk_hdr = (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, >>>> - >>>> index); >>>> - pool->s.num_bufs += ODP_BUFS_PER_CHUNK; >>>> - pool_size -= ODP_BUFS_PER_CHUNK * tot_size; >>>> + default: >>>> + return ODP_BUFFER_POOL_INVALID; >>>> } >>>> -} >>>> - >>>> - >>>> -odp_buffer_pool_t odp_buffer_pool_create(const char *name, >>>> - void *base_addr, uint64_t size, >>>> - size_t buf_size, size_t >>>> buf_align, >>>> - int buf_type) >>>> -{ >>>> - odp_buffer_pool_t pool_hdl = ODP_BUFFER_POOL_INVALID; >>>> - pool_entry_t *pool; >>>> - uint32_t i; >>>> >>>> for (i = 0; i < ODP_CONFIG_BUFFER_POOLS; i++) { >>>> pool = get_pool_entry(i); >>>> >>>> LOCK(&pool->s.lock); >>>> + if (pool->s.shm != ODP_SHM_INVALID) { >>>> + UNLOCK(&pool->s.lock); >>>> + continue; >>>> + } >>>> >>>> - if (pool->s.buf_base == 0) { >>>> - /* found free pool */ >>>> + /* found free pool */ >>>> + size_t block_size, mdata_size, udata_size; >>>> >>>> - strncpy(pool->s.name, name, >>>> - ODP_BUFFER_POOL_NAME_LEN - 1); >>>> - pool->s.name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; >>>> - pool->s.pool_base_addr = base_addr; >>>> - pool->s.pool_size = size; >>>> - pool->s.user_size = buf_size; >>>> - pool->s.user_align = buf_align; >>>> - pool->s.buf_type = buf_type; >>>> + strncpy(pool->s.name, name, >>>> + ODP_BUFFER_POOL_NAME_LEN - 1); >>>> + pool->s.name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; >>>> >>>> - link_bufs(pool); >>>> + pool->s.params = *params; >>>> + pool->s.init_params = *init_params; >>>> >>>> - UNLOCK(&pool->s.lock); >>>> + mdata_size = params->buf_num * buf_stride; >>>> + udata_size = params->buf_num * udata_stride; >>>> + block_size = params->buf_num * blk_size; >>>> + >>>> + pool->s.pool_size = ODP_PAGE_SIZE_ROUNDUP(mdata_size + >>>> + udata_size + >>>> + block_size); >>>> >>>> - pool_hdl = pool->s.pool_hdl; >>>> - break; >>>> + pool->s.shm = odp_shm_reserve(pool->s.name, >>>> pool->s.pool_size, >>>> + ODP_PAGE_SIZE, 0); >>>> + if (pool->s.shm == ODP_SHM_INVALID) { >>>> + UNLOCK(&pool->s.lock); >>>> + return ODP_BUFFER_POOL_INVALID; >>>> } >>>> >>>> + pool->s.pool_base_addr = (uint8_t >>>> *)odp_shm_addr(pool->s.shm); >>>> + pool->s.flags.unsegmented = unsegmented; >>>> + pool->s.seg_size = unsegmented ? >>>> + blk_size : ODP_CONFIG_BUF_SEG_SIZE; >>>> + uint8_t *udata_base_addr = pool->s.pool_base_addr + >>>> mdata_size; >>>> + uint8_t *block_base_addr = udata_base_addr + udata_size; >>>> + >>>> + pool->s.bufcount = 0; >>>> + pool->s.buf_freelist = NULL; >>>> + pool->s.blk_freelist = NULL; >>>> + >>>> + uint8_t *buf = pool->s.udata_base_addr - buf_stride; >>>> + uint8_t *udat = (udata_stride == 0) ? NULL : >>>> + block_base_addr - udata_stride; >>>> + >>>> + /* Init buffer common header and add to pool buffer >>>> freelist */ >>>> + do { >>>> + odp_buffer_hdr_t *tmp = >>>> + (odp_buffer_hdr_t *)(void *)buf; >>>> + tmp->pool_hdl = pool->s.pool_hdl; >>>> + tmp->size = 0; >>>> + tmp->type = params->buf_type; >>>> + tmp->udata_addr = (void *)udat; >>>> + tmp->udata_size = init_params->udata_size; >>>> + tmp->segcount = 0; >>>> + tmp->segsize = pool->s.seg_size; >>>> + tmp->buf_hdl.handle = >>>> + >>>> odp_buffer_encode_handle((odp_buffer_hdr_t *) >>>> + (void *)buf); >>>> + ret_buf(&pool->s, tmp); >>>> + buf -= buf_stride; >>>> + udat -= udata_stride; >>>> + } while (buf >= pool->s.pool_base_addr); >>>> + >>>> + /* Form block freelist for pool */ >>>> + uint8_t *blk = pool->s.pool_base_addr + >>>> pool->s.pool_size - >>>> + pool->s.seg_size; >>>> + >>>> + if (blk_size > 0) >>>> + do { >>>> + ret_blk(&pool->s, blk); >>>> + blk -= pool->s.seg_size; >>>> + } while (blk >= block_base_addr); >>>> + >>>> UNLOCK(&pool->s.lock); >>>> + >>>> + pool_hdl = pool->s.pool_hdl; >>>> + break; >>>> } >>>> >>>> return pool_hdl; >>>> @@ -431,93 +288,172 @@ odp_buffer_pool_t odp_buffer_pool_lookup(const >>>> char *name) >>>> return ODP_BUFFER_POOL_INVALID; >>>> } >>>> >>>> - >>>> -odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_hdl) >>>> +odp_buffer_pool_t odp_buffer_pool_next(odp_buffer_pool_t pool_hdl, >>>> + char *name, >>>> + size_t *udata_size, >>>> + odp_buffer_pool_param_t *params, >>>> + int *predef) >>>> { >>>> - pool_entry_t *pool; >>>> - odp_buffer_chunk_hdr_t *chunk; >>>> - odp_buffer_bits_t handle; >>>> - uint32_t pool_id = pool_handle_to_index(pool_hdl); >>>> - >>>> - pool = get_pool_entry(pool_id); >>>> - chunk = local_chunk[pool_id]; >>>> - >>>> - if (chunk == NULL) { >>>> - LOCK(&pool->s.lock); >>>> - chunk = rem_chunk(pool); >>>> - UNLOCK(&pool->s.lock); >>>> + pool_entry_t *next_pool; >>>> + uint32_t pool_id; >>>> >>>> - if (chunk == NULL) >>>> - return ODP_BUFFER_INVALID; >>>> + /* NULL input means first element */ >>>> + if (pool_hdl == ODP_BUFFER_POOL_INVALID) { >>>> + pool_id = 0; >>>> + next_pool = get_pool_entry(pool_id); >>>> + } else { >>>> + pool_id = pool_handle_to_index(pool_hdl); >>>> >>>> - local_chunk[pool_id] = chunk; >>>> + if (pool_id == ODP_CONFIG_BUFFER_POOLS) >>>> + return ODP_BUFFER_POOL_INVALID; >>>> + else >>>> + next_pool = get_pool_entry(++pool_id); >>>> } >>>> >>>> - if (chunk->chunk.num_bufs == 0) { >>>> - /* give the chunk buffer */ >>>> - local_chunk[pool_id] = NULL; >>>> - chunk->buf_hdr.type = pool->s.buf_type; >>>> + /* Only interested in pools that exist */ >>>> + while (next_pool->s.shm == ODP_SHM_INVALID) { >>>> + if (pool_id == ODP_CONFIG_BUFFER_POOLS) >>>> + return ODP_BUFFER_POOL_INVALID; >>>> + else >>>> + next_pool = get_pool_entry(++pool_id); >>>> + } >>>> >>>> - handle = chunk->buf_hdr.handle; >>>> - } else { >>>> - odp_buffer_hdr_t *hdr; >>>> - uint32_t index; >>>> - index = rem_buf_index(chunk); >>>> - hdr = index_to_hdr(pool, index); >>>> + /* Found the next pool, so return info about it */ >>>> + strncpy(name, next_pool->s.name, ODP_BUFFER_POOL_NAME_LEN - 1); >>>> + name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; >>>> >>>> - handle = hdr->handle; >>>> - } >>>> + *udata_size = next_pool->s.init_params.udata_size; >>>> + *params = next_pool->s.params; >>>> + *predef = next_pool->s.flags.predefined; >>>> >>>> - return handle.u32; >>>> + return next_pool->s.pool_hdl; >>>> } >>>> >>>> - >>>> -void odp_buffer_free(odp_buffer_t buf) >>>> +int odp_buffer_pool_destroy(odp_buffer_pool_t pool_hdl) >>>> { >>>> - odp_buffer_hdr_t *hdr; >>>> - uint32_t pool_id; >>>> - pool_entry_t *pool; >>>> - odp_buffer_chunk_hdr_t *chunk_hdr; >>>> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >>>> >>>> - hdr = odp_buf_to_hdr(buf); >>>> - pool_id = pool_handle_to_index(hdr->pool_hdl); >>>> - pool = get_pool_entry(pool_id); >>>> - chunk_hdr = local_chunk[pool_id]; >>>> + if (pool == NULL) >>>> + return -1; >>>> >>>> - if (chunk_hdr && chunk_hdr->chunk.num_bufs == >>>> ODP_BUFS_PER_CHUNK - 1) { >>>> - /* Current chunk is full. Push back to the pool */ >>>> - LOCK(&pool->s.lock); >>>> - add_chunk(pool, chunk_hdr); >>>> + LOCK(&pool->s.lock); >>>> + >>>> + if (pool->s.shm == ODP_SHM_INVALID || >>>> + pool->s.bufcount > 0 || >>>> + pool->s.flags.predefined) { >>>> UNLOCK(&pool->s.lock); >>>> - chunk_hdr = NULL; >>>> + return -1; >>>> } >>>> >>>> - if (chunk_hdr == NULL) { >>>> - /* Use this buffer */ >>>> - chunk_hdr = (odp_buffer_chunk_hdr_t *)hdr; >>>> - local_chunk[pool_id] = chunk_hdr; >>>> - chunk_hdr->chunk.num_bufs = 0; >>>> + odp_shm_free(pool->s.shm); >>>> + >>>> + pool->s.shm = ODP_SHM_INVALID; >>>> + UNLOCK(&pool->s.lock); >>>> + >>>> + return 0; >>>> +} >>>> + >>>> +size_t odp_buffer_pool_headroom(odp_buffer_pool_t pool_hdl) >>>> +{ >>>> + return odp_pool_to_entry(pool_hdl)->s.headroom; >>>> +} >>>> + >>>> +int odp_buffer_pool_set_headroom(odp_buffer_pool_t pool_hdl, size_t hr) >>>> +{ >>>> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >>>> + >>>> + if (hr >= pool->s.seg_size/2) { >>>> + return -1; >>>> } else { >>>> - /* Add to current chunk */ >>>> - add_buf_index(chunk_hdr, hdr->index); >>>> + pool->s.headroom = hr; >>>> + return 0; >>>> } >>>> } >>>> >>>> +size_t odp_buffer_pool_tailroom(odp_buffer_pool_t pool_hdl) >>>> +{ >>>> + return odp_pool_to_entry(pool_hdl)->s.tailroom; >>>> +} >>>> >>>> -odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf) >>>> +int odp_buffer_pool_set_tailroom(odp_buffer_pool_t pool_hdl, size_t tr) >>>> { >>>> - odp_buffer_hdr_t *hdr; >>>> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >>>> >>>> - hdr = odp_buf_to_hdr(buf); >>>> - return hdr->pool_hdl; >>>> + if (tr >= pool->s.seg_size/2) { >>>> + return -1; >>>> + } else { >>>> + pool->s.tailroom = tr; >>>> + return 0; >>>> + } >>>> } >>>> >>>> +odp_buffer_t buffer_alloc(odp_buffer_pool_t pool_hdl, size_t size) >>>> +{ >>>> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >>>> + size_t totsize = pool->s.headroom + size + pool->s.tailroom; >>>> + odp_anybuf_t *buf; >>>> + uint8_t *blk; >>>> + >>>> + if ((pool->s.flags.unsegmented && totsize > pool->s.seg_size) || >>>> + (!pool->s.flags.unsegmented && totsize > >>>> ODP_CONFIG_BUF_MAX_SIZE)) >>>> + return ODP_BUFFER_INVALID; >>>> + >>>> + buf = (odp_anybuf_t *)(void *)get_buf(&pool->s); >>>> + >>>> + if (buf == NULL) >>>> + return ODP_BUFFER_INVALID; >>>> + >>>> + /* Get blocks for this buffer, if pool uses application data */ >>>> + if (buf->buf.segsize > 0) >>>> + do { >>>> + blk = get_blk(&pool->s); >>>> + if (blk == NULL) { >>>> + ret_buf(&pool->s, &buf->buf); >>>> + return ODP_BUFFER_INVALID; >>>> + } >>>> + buf->buf.addr[buf->buf.segcount++] = blk; >>>> + totsize = totsize < pool->s.seg_size ? 0 : >>>> + totsize - pool->s.seg_size; >>>> + } while (totsize > 0); >>>> + >>>> + switch (buf->buf.type) { >>>> + case ODP_BUFFER_TYPE_RAW: >>>> + break; >>>> + >>>> + case ODP_BUFFER_TYPE_PACKET: >>>> + packet_init(pool, &buf->pkt, size); >>>> + if (pool->s.init_params.buf_init != NULL) >>>> + (*pool->s.init_params.buf_init) >>>> + (buf->buf.buf_hdl.handle, >>>> + pool->s.init_params.buf_init_arg); >>>> + break; >>>> + >>>> + case ODP_BUFFER_TYPE_TIMEOUT: >>>> + break; >>>> + >>>> + default: >>>> + ret_buf(&pool->s, &buf->buf); >>>> + return ODP_BUFFER_INVALID; >>>> + } >>>> + >>>> + return odp_hdr_to_buf(&buf->buf); >>>> +} >>>> + >>>> +odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_hdl) >>>> +{ >>>> + return buffer_alloc(pool_hdl, 0); >>>> +} >>>> + >>>> +void odp_buffer_free(odp_buffer_t buf) >>>> +{ >>>> + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >>>> + pool_entry_t *pool = odp_buf_to_pool(hdr); >>>> + ret_buf(&pool->s, hdr); >>>> +} >>>> >>>> void odp_buffer_pool_print(odp_buffer_pool_t pool_hdl) >>>> { >>>> pool_entry_t *pool; >>>> - odp_buffer_chunk_hdr_t *chunk_hdr; >>>> - uint32_t i; >>>> uint32_t pool_id; >>>> >>>> pool_id = pool_handle_to_index(pool_hdl); >>>> @@ -528,47 +464,4 @@ void odp_buffer_pool_print(odp_buffer_pool_t >>>> pool_hdl) >>>> printf(" pool %i\n", pool->s.pool_hdl); >>>> printf(" name %s\n", pool->s.name); >>>> printf(" pool base %p\n", >>>> pool->s.pool_base_addr); >>>> - printf(" buf base 0x%"PRIxPTR"\n", pool->s.buf_base); >>>> - printf(" pool size 0x%"PRIx64"\n", pool->s.pool_size); >>>> - printf(" buf size %zu\n", pool->s.user_size); >>>> - printf(" buf align %zu\n", pool->s.user_align); >>>> - printf(" hdr size %zu\n", pool->s.hdr_size); >>>> - printf(" alloc size %zu\n", pool->s.buf_size); >>>> - printf(" offset to hdr %zu\n", pool->s.buf_offset); >>>> - printf(" num bufs %"PRIu64"\n", pool->s.num_bufs); >>>> - printf(" free bufs %"PRIu64"\n", pool->s.free_bufs); >>>> - >>>> - /* first chunk */ >>>> - chunk_hdr = pool->s.head; >>>> - >>>> - if (chunk_hdr == NULL) { >>>> - ODP_ERR(" POOL EMPTY\n"); >>>> - return; >>>> - } >>>> - >>>> - printf("\n First chunk\n"); >>>> - >>>> - for (i = 0; i < chunk_hdr->chunk.num_bufs - 1; i++) { >>>> - uint32_t index; >>>> - odp_buffer_hdr_t *hdr; >>>> - >>>> - index = chunk_hdr->chunk.buf_index[i]; >>>> - hdr = index_to_hdr(pool, index); >>>> - >>>> - printf(" [%i] addr %p, id %"PRIu32"\n", i, hdr->addr, >>>> index); >>>> - } >>>> - >>>> - printf(" [%i] addr %p, id %"PRIu32"\n", i, >>>> chunk_hdr->buf_hdr.addr, >>>> - chunk_hdr->buf_hdr.index); >>>> - >>>> - /* next chunk */ >>>> - chunk_hdr = next_chunk(pool, chunk_hdr); >>>> - >>>> - if (chunk_hdr) { >>>> - printf(" Next chunk\n"); >>>> - printf(" addr %p, id %"PRIu32"\n", >>>> chunk_hdr->buf_hdr.addr, >>>> - chunk_hdr->buf_hdr.index); >>>> - } >>>> - >>>> - printf("\n"); >>>> } >>>> diff --git a/platform/linux-generic/odp_packet.c >>>> b/platform/linux-generic/odp_packet.c >>>> index 82ea879..45ac8e5 100644 >>>> --- a/platform/linux-generic/odp_packet.c >>>> +++ b/platform/linux-generic/odp_packet.c >>>> @@ -11,29 +11,31 @@ >>>> >>>> #include <odph_eth.h> >>>> #include <odph_ip.h> >>>> +#include <odph_tcp.h> >>>> +#include <odph_udp.h> >>>> >>>> #include <string.h> >>>> #include <stdio.h> >>>> >>>> -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >>>> - odph_ipv4hdr_t *ipv4, size_t >>>> *offset_out); >>>> -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >>>> - odph_ipv6hdr_t *ipv6, size_t >>>> *offset_out); >>>> - >>>> void odp_packet_init(odp_packet_t pkt) >>>> { >>>> odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >>>> + pool_entry_t *pool = odp_buf_to_pool(&pkt_hdr->buf_hdr); >>>> const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, >>>> buf_hdr); >>>> uint8_t *start; >>>> size_t len; >>>> >>>> start = (uint8_t *)pkt_hdr + start_offset; >>>> - len = ODP_OFFSETOF(odp_packet_hdr_t, buf_data) - start_offset; >>>> + len = sizeof(odp_packet_hdr_t) - start_offset; >>>> memset(start, 0, len); >>>> >>>> pkt_hdr->l2_offset = ODP_PACKET_OFFSET_INVALID; >>>> pkt_hdr->l3_offset = ODP_PACKET_OFFSET_INVALID; >>>> pkt_hdr->l4_offset = ODP_PACKET_OFFSET_INVALID; >>>> + pkt_hdr->payload_offset = ODP_PACKET_OFFSET_INVALID; >>>> + >>>> + pkt_hdr->headroom = pool->s.headroom; >>>> + pkt_hdr->tailroom = pool->s.tailroom; >>>> } >>>> >>>> odp_packet_t odp_packet_from_buffer(odp_buffer_t buf) >>>> @@ -46,55 +48,112 @@ odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt) >>>> return (odp_buffer_t)pkt; >>>> } >>>> >>>> -void odp_packet_set_len(odp_packet_t pkt, size_t len) >>>> +size_t odp_packet_len(odp_packet_t pkt) >>>> { >>>> - odp_packet_hdr(pkt)->frame_len = len; >>>> + return odp_packet_hdr(pkt)->frame_len; >>>> } >>>> >>>> -size_t odp_packet_get_len(odp_packet_t pkt) >>>> +void *odp_packet_offset_map(odp_packet_t pkt, size_t offset, size_t >>>> *seglen) >>>> { >>>> - return odp_packet_hdr(pkt)->frame_len; >>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>> + >>>> + if (offset >= pkt_hdr->frame_len) >>>> + return NULL; >>>> + >>>> + return buffer_map(&pkt_hdr->buf_hdr, >>>> + pkt_hdr->headroom + offset, >>>> + seglen, pkt_hdr->frame_len); >>>> } >>>> >>>> -uint8_t *odp_packet_addr(odp_packet_t pkt) >>>> +void odp_packet_offset_unmap(odp_packet_t pkt ODP_UNUSED, >>>> + size_t offset ODP_UNUSED) >>>> { >>>> - return odp_buffer_addr(odp_packet_to_buffer(pkt)); >>>> } >>>> >>>> -uint8_t *odp_packet_data(odp_packet_t pkt) >>>> +void *odp_packet_map(odp_packet_t pkt, size_t *seglen) >>>> { >>>> - return odp_packet_addr(pkt) + odp_packet_hdr(pkt)->frame_offset; >>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>> + >>>> + return buffer_map(&pkt_hdr->buf_hdr, >>>> + 0, seglen, pkt_hdr->frame_len); >>>> } >>>> >>>> +void *odp_packet_addr(odp_packet_t pkt) >>>> +{ >>>> + size_t seglen; >>>> + return odp_packet_map(pkt, &seglen); >>>> +} >>>> >>>> -uint8_t *odp_packet_l2(odp_packet_t pkt) >>>> +odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt) >>>> { >>>> - const size_t offset = odp_packet_l2_offset(pkt); >>>> + return odp_packet_hdr(pkt)->buf_hdr.pool_hdl; >>>> +} >>>> >>>> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >>>> - return NULL; >>>> +odp_packet_segment_t odp_packet_segment_by_index(odp_packet_t pkt, >>>> + size_t ndx) >>>> +{ >>>> >>>> - return odp_packet_addr(pkt) + offset; >>>> + return (odp_packet_segment_t) >>>> + buffer_segment(&odp_packet_hdr(pkt)->buf_hdr, ndx); >>>> } >>>> >>>> -size_t odp_packet_l2_offset(odp_packet_t pkt) >>>> +odp_packet_segment_t odp_packet_segment_next(odp_packet_t pkt, >>>> + odp_packet_segment_t seg) >>>> { >>>> - return odp_packet_hdr(pkt)->l2_offset; >>>> + return (odp_packet_segment_t) >>>> + segment_next(&odp_packet_hdr(pkt)->buf_hdr, seg); >>>> } >>>> >>>> -void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset) >>>> +void *odp_packet_segment_map(odp_packet_t pkt, odp_packet_segment_t >>>> seg, >>>> + size_t *seglen) >>>> { >>>> - odp_packet_hdr(pkt)->l2_offset = offset; >>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>> + >>>> + return segment_map(&pkt_hdr->buf_hdr, seg, >>>> + seglen, pkt_hdr->frame_len); >>>> } >>>> >>>> -uint8_t *odp_packet_l3(odp_packet_t pkt) >>>> +void odp_packet_segment_unmap(odp_packet_segment_t seg ODP_UNUSED) >>>> { >>>> - const size_t offset = odp_packet_l3_offset(pkt); >>>> +} >>>> >>>> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >>>> - return NULL; >>>> +void *odp_packet_udata(odp_packet_t pkt, size_t *len) >>>> +{ >>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>> + >>>> + *len = pkt_hdr->buf_hdr.udata_size; >>>> + return pkt_hdr->buf_hdr.udata_addr; >>>> +} >>>> + >>>> +void *odp_packet_udata_addr(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->buf_hdr.udata_addr; >>>> +} >>>> + >>>> +void *odp_packet_l2_map(odp_packet_t pkt, size_t *seglen) >>>> +{ >>>> + return odp_packet_offset_map(pkt, odp_packet_l2_offset(pkt), >>>> seglen); >>>> +} >>>> + >>>> +size_t odp_packet_l2_offset(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->l2_offset; >>>> +} >>>> + >>>> +int odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset) >>>> +{ >>>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>>> + >>>> + if (offset >= hdr->frame_len) >>>> + return -1; >>>> + >>>> + hdr->l2_offset = offset; >>>> + return 0; >>>> +} >>>> >>>> - return odp_packet_addr(pkt) + offset; >>>> +void *odp_packet_l3_map(odp_packet_t pkt, size_t *seglen) >>>> +{ >>>> + return odp_packet_offset_map(pkt, odp_packet_l3_offset(pkt), >>>> seglen); >>>> } >>>> >>>> size_t odp_packet_l3_offset(odp_packet_t pkt) >>>> @@ -102,19 +161,35 @@ size_t odp_packet_l3_offset(odp_packet_t pkt) >>>> return odp_packet_hdr(pkt)->l3_offset; >>>> } >>>> >>>> -void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset) >>>> +int odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset) >>>> { >>>> - odp_packet_hdr(pkt)->l3_offset = offset; >>>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>>> + >>>> + if (offset >= hdr->frame_len) >>>> + return -1; >>>> + >>>> + hdr->l3_offset = offset; >>>> + return 0; >>>> } >>>> >>>> -uint8_t *odp_packet_l4(odp_packet_t pkt) >>>> +uint32_t odp_packet_l3_protocol(odp_packet_t pkt) >>>> { >>>> - const size_t offset = odp_packet_l4_offset(pkt); >>>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>>> >>>> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >>>> - return NULL; >>>> + if (hdr->input_flags.l3) >>>> + return hdr->l3_protocol; >>>> + else >>>> + return -1; >>>> +} >>>> + >>>> +void odp_packet_set_l3_protocol(odp_packet_t pkt, uint16_t protocol) >>>> +{ >>>> + odp_packet_hdr(pkt)->l3_protocol = protocol; >>>> +} >>>> >>>> - return odp_packet_addr(pkt) + offset; >>>> +void *odp_packet_l4_map(odp_packet_t pkt, size_t *seglen) >>>> +{ >>>> + return odp_packet_offset_map(pkt, odp_packet_l4_offset(pkt), >>>> seglen); >>>> } >>>> >>>> size_t odp_packet_l4_offset(odp_packet_t pkt) >>>> @@ -122,277 +197,902 @@ size_t odp_packet_l4_offset(odp_packet_t pkt) >>>> return odp_packet_hdr(pkt)->l4_offset; >>>> } >>>> >>>> -void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset) >>>> +int odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset) >>>> { >>>> - odp_packet_hdr(pkt)->l4_offset = offset; >>>> -} >>>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>>> + >>>> + if (offset >= hdr->frame_len) >>>> + return -1; >>>> >>>> + hdr->l4_offset = offset; >>>> + return 0; >>>> +} >>>> >>>> -int odp_packet_is_segmented(odp_packet_t pkt) >>>> +uint32_t odp_packet_l4_protocol(odp_packet_t pkt) >>>> { >>>> - odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr((odp_buffer_t)pkt); >>>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>>> >>>> - if (buf_hdr->scatter.num_bufs == 0) >>>> - return 0; >>>> + if (hdr->input_flags.l4) >>>> + return hdr->l4_protocol; >>>> else >>>> - return 1; >>>> + return -1; >>>> } >>>> >>>> +void odp_packet_set_l4_protocol(odp_packet_t pkt, uint8_t protocol) >>>> +{ >>>> + odp_packet_hdr(pkt)->l4_protocol = protocol; >>>> +} >>>> >>>> -int odp_packet_seg_count(odp_packet_t pkt) >>>> +void *odp_packet_payload_map(odp_packet_t pkt, size_t *seglen) >>>> { >>>> - odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr((odp_buffer_t)pkt); >>>> + return odp_packet_offset_map(pkt, >>>> odp_packet_payload_offset(pkt), >>>> + seglen); >>>> +} >>>> >>>> - return (int)buf_hdr->scatter.num_bufs + 1; >>>> +size_t odp_packet_payload_offset(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->payload_offset; >>>> } >>>> >>>> +int odp_packet_set_payload_offset(odp_packet_t pkt, size_t offset) >>>> +{ >>>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>>> >>>> -/** >>>> - * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP >>>> - * >>>> - * Internal function: caller is resposible for passing only valid >>>> packet handles >>>> - * , lengths and offsets (usually done&called in packet input). >>>> - * >>>> - * @param pkt Packet handle >>>> - * @param len Packet length in bytes >>>> - * @param frame_offset Byte offset to L2 header >>>> - */ >>>> -void odp_packet_parse(odp_packet_t pkt, size_t len, size_t >>>> frame_offset) >>>> + if (offset >= hdr->frame_len) >>>> + return -1; >>>> + >>>> + hdr->payload_offset = offset; >>>> + return 0; >>>> +} >>>> + >>>> +int odp_packet_error(odp_packet_t pkt) >>>> { >>>> - odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >>>> - odph_ethhdr_t *eth; >>>> - odph_vlanhdr_t *vlan; >>>> - odph_ipv4hdr_t *ipv4; >>>> - odph_ipv6hdr_t *ipv6; >>>> - uint16_t ethtype; >>>> - size_t offset = 0; >>>> - uint8_t ip_proto = 0; >>>> + return odp_packet_hdr(pkt)->error_flags.all != 0; >>>> +} >>>> >>>> - pkt_hdr->input_flags.eth = 1; >>>> - pkt_hdr->frame_offset = frame_offset; >>>> - pkt_hdr->frame_len = len; >>>> +void odp_packet_set_error(odp_packet_t pkt, int val) >>>> +{ >>>> + odp_packet_hdr(pkt)->error_flags.app_error = val; >>>> +} >>>> >>>> - if (odp_unlikely(len < ODPH_ETH_LEN_MIN)) { >>>> - pkt_hdr->error_flags.frame_len = 1; >>>> - return; >>>> - } else if (len > ODPH_ETH_LEN_MAX) { >>>> - pkt_hdr->input_flags.jumbo = 1; >>>> - } >>>> +int odp_packet_inflag_l2(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->input_flags.l2; >>>> +} >>>> >>>> - /* Assume valid L2 header, no CRC/FCS check in SW */ >>>> - pkt_hdr->input_flags.l2 = 1; >>>> - pkt_hdr->l2_offset = frame_offset; >>>> +void odp_packet_set_inflag_l2(odp_packet_t pkt, int val) >>>> +{ >>>> + odp_packet_hdr(pkt)->input_flags.l2 = val; >>>> +} >>>> >>>> - eth = (odph_ethhdr_t *)odp_packet_data(pkt); >>>> - ethtype = odp_be_to_cpu_16(eth->type); >>>> - vlan = (odph_vlanhdr_t *)ð->type; >>>> +int odp_packet_inflag_l3(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->input_flags.l3; >>>> +} >>>> >>>> - if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { >>>> - pkt_hdr->input_flags.vlan_qinq = 1; >>>> - ethtype = odp_be_to_cpu_16(vlan->tpid); >>>> - offset += sizeof(odph_vlanhdr_t); >>>> - vlan = &vlan[1]; >>>> - } >>>> +void odp_packet_set_inflag_l3(odp_packet_t pkt, int val) >>>> +{ >>>> + odp_packet_hdr(pkt)->input_flags.l3 = val; >>>> +} >>>> >>>> - if (ethtype == ODPH_ETHTYPE_VLAN) { >>>> - pkt_hdr->input_flags.vlan = 1; >>>> - ethtype = odp_be_to_cpu_16(vlan->tpid); >>>> - offset += sizeof(odph_vlanhdr_t); >>>> - } >>>> +int odp_packet_inflag_l4(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->input_flags.l4; >>>> +} >>>> >>>> - /* Set l3_offset+flag only for known ethtypes */ >>>> - switch (ethtype) { >>>> - case ODPH_ETHTYPE_IPV4: >>>> - pkt_hdr->input_flags.ipv4 = 1; >>>> - pkt_hdr->input_flags.l3 = 1; >>>> - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + >>>> offset; >>>> - ipv4 = (odph_ipv4hdr_t *)odp_packet_l3(pkt); >>>> - ip_proto = parse_ipv4(pkt_hdr, ipv4, &offset); >>>> - break; >>>> - case ODPH_ETHTYPE_IPV6: >>>> - pkt_hdr->input_flags.ipv6 = 1; >>>> - pkt_hdr->input_flags.l3 = 1; >>>> - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + >>>> offset; >>>> - ipv6 = (odph_ipv6hdr_t *)odp_packet_l3(pkt); >>>> - ip_proto = parse_ipv6(pkt_hdr, ipv6, &offset); >>>> - break; >>>> - case ODPH_ETHTYPE_ARP: >>>> - pkt_hdr->input_flags.arp = 1; >>>> - /* fall through */ >>>> - default: >>>> - ip_proto = 0; >>>> - break; >>>> - } >>>> +void odp_packet_set_inflag_l4(odp_packet_t pkt, int val) >>>> +{ >>>> + odp_packet_hdr(pkt)->input_flags.l4 = val; >>>> +} >>>> >>>> - switch (ip_proto) { >>>> - case ODPH_IPPROTO_UDP: >>>> - pkt_hdr->input_flags.udp = 1; >>>> - pkt_hdr->input_flags.l4 = 1; >>>> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >>>> - break; >>>> - case ODPH_IPPROTO_TCP: >>>> - pkt_hdr->input_flags.tcp = 1; >>>> - pkt_hdr->input_flags.l4 = 1; >>>> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >>>> - break; >>>> - case ODPH_IPPROTO_SCTP: >>>> - pkt_hdr->input_flags.sctp = 1; >>>> - pkt_hdr->input_flags.l4 = 1; >>>> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >>>> - break; >>>> - case ODPH_IPPROTO_ICMP: >>>> - pkt_hdr->input_flags.icmp = 1; >>>> - pkt_hdr->input_flags.l4 = 1; >>>> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >>>> - break; >>>> - default: >>>> - /* 0 or unhandled IP protocols, don't set L4 >>>> flag+offset */ >>>> - if (pkt_hdr->input_flags.ipv6) { >>>> - /* IPv6 next_hdr is not L4, mark as IP-option >>>> instead */ >>>> - pkt_hdr->input_flags.ipopt = 1; >>>> - } >>>> - break; >>>> - } >>>> +int odp_packet_inflag_eth(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->input_flags.eth; >>>> } >>>> >>>> -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >>>> - odph_ipv4hdr_t *ipv4, size_t >>>> *offset_out) >>>> +void odp_packet_set_inflag_eth(odp_packet_t pkt, int val) >>>> { >>>> - uint8_t ihl; >>>> - uint16_t frag_offset; >>>> + odp_packet_hdr(pkt)->input_flags.eth = val; >>>> +} >>>> >>>> - ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); >>>> - if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN)) { >>>> - pkt_hdr->error_flags.ip_err = 1; >>>> - return 0; >>>> - } >>>> +int odp_packet_inflag_jumbo(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->input_flags.jumbo; >>>> +} >>>> >>>> - if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) { >>>> - pkt_hdr->input_flags.ipopt = 1; >>>> - return 0; >>>> - } >>>> +void odp_packet_set_inflag_jumbo(odp_packet_t pkt, int val) >>>> +{ >>>> + odp_packet_hdr(pkt)->input_flags.jumbo = val; >>>> +} >>>> >>>> - /* A packet is a fragment if: >>>> - * "more fragments" flag is set (all fragments except the last) >>>> - * OR >>>> - * "fragment offset" field is nonzero (all fragments except the >>>> first) >>>> - */ >>>> - frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); >>>> - if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) { >>>> - pkt_hdr->input_flags.ipfrag = 1; >>>> - return 0; >>>> - } >>>> +int odp_packet_inflag_vlan(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->input_flags.vlan; >>>> +} >>>> >>>> - if (ipv4->proto == ODPH_IPPROTO_ESP || >>>> - ipv4->proto == ODPH_IPPROTO_AH) { >>>> - pkt_hdr->input_flags.ipsec = 1; >>>> - return 0; >>>> - } >>>> +void odp_packet_set_inflag_vlan(odp_packet_t pkt, int val) >>>> +{ >>>> + odp_packet_hdr(pkt)->input_flags.vlan = val; >>>> +} >>>> >>>> - /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after >>>> return */ >>>> +int odp_packet_inflag_vlan_qinq(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->input_flags.vlan_qinq; >>>> +} >>>> >>>> - *offset_out = sizeof(uint32_t) * ihl; >>>> - return ipv4->proto; >>>> +void odp_packet_set_inflag_vlan_qinq(odp_packet_t pkt, int val) >>>> +{ >>>> + odp_packet_hdr(pkt)->input_flags.vlan_qinq = val; >>>> } >>>> >>>> -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >>>> - odph_ipv6hdr_t *ipv6, size_t >>>> *offset_out) >>>> +int odp_packet_inflag_snap(odp_packet_t pkt) >>>> { >>>> - if (ipv6->next_hdr == ODPH_IPPROTO_ESP || >>>> - ipv6->next_hdr == ODPH_IPPROTO_AH) { >>>> - pkt_hdr->input_flags.ipopt = 1; >>>> - pkt_hdr->input_flags.ipsec = 1; >>>> - return 0; >>>> - } >>>> + return odp_packet_hdr(pkt)->input_flags.snap; >>>> +} >>>> >>>> - if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { >>>> - pkt_hdr->input_flags.ipopt = 1; >>>> - pkt_hdr->input_flags.ipfrag = 1; >>>> - return 0; >>>> - } >>>> +void odp_packet_set_inflag_snap(odp_packet_t pkt, int val) >>>> +{ >>>> + odp_packet_hdr(pkt)->input_flags.snap = val; >>>> +} >>>> >>>> - /* Don't step through more extensions */ >>>> - *offset_out = ODPH_IPV6HDR_LEN; >>>> - return ipv6->next_hdr; >>>> +int odp_packet_inflag_arp(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->input_flags.arp; >>>> } >>>> >>>> -void odp_packet_print(odp_packet_t pkt) >>>> +void odp_packet_set_inflag_arp(odp_packet_t pkt, int val) >>>> { >>>> - int max_len = 512; >>>> - char str[max_len]; >>>> - int len = 0; >>>> - int n = max_len-1; >>>> - odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>>> + odp_packet_hdr(pkt)->input_flags.arp = val; >>>> +} >>>> >>>> - len += snprintf(&str[len], n-len, "Packet "); >>>> - len += odp_buffer_snprint(&str[len], n-len, (odp_buffer_t) pkt); >>>> - len += snprintf(&str[len], n-len, >>>> - " input_flags 0x%x\n", hdr->input_flags.all); >>>> - len += snprintf(&str[len], n-len, >>>> - " error_flags 0x%x\n", hdr->error_flags.all); >>>> - len += snprintf(&str[len], n-len, >>>> - " output_flags 0x%x\n", hdr->output_flags.all); >>>> - len += snprintf(&str[len], n-len, >>>> - " frame_offset %u\n", hdr->frame_offset); >>>> - len += snprintf(&str[len], n-len, >>>> - " l2_offset %u\n", hdr->l2_offset); >>>> - len += snprintf(&str[len], n-len, >>>> - " l3_offset %u\n", hdr->l3_offset); >>>> - len += snprintf(&str[len], n-len, >>>> - " l4_offset %u\n", hdr->l4_offset); >>>> - len += snprintf(&str[len], n-len, >>>> - " frame_len %u\n", hdr->frame_len); >>>> - len += snprintf(&str[len], n-len, >>>> - " input %u\n", hdr->input); >>>> - str[len] = '\0'; >>>> +int odp_packet_inflag_ipv4(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->input_flags.ipv4; >>>> +} >>>> >>>> - printf("\n%s\n", str); >>>> +void odp_packet_set_inflag_ipv4(odp_packet_t pkt, int val) >>>> +{ >>>> + odp_packet_hdr(pkt)->input_flags.ipv4 = val; >>>> } >>>> >>>> -int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src) >>>> +int odp_packet_inflag_ipv6(odp_packet_t pkt) >>>> { >>>> - odp_packet_hdr_t *const pkt_hdr_dst = odp_packet_hdr(pkt_dst); >>>> - odp_packet_hdr_t *const pkt_hdr_src = odp_packet_hdr(pkt_src); >>>> - const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, >>>> buf_hdr); >>>> - uint8_t *start_src; >>>> - uint8_t *start_dst; >>>> - size_t len; >>>> + return odp_packet_hdr(pkt)->input_flags.ipv6; >>>> +} >>>> >>>> - if (pkt_dst == ODP_PACKET_INVALID || pkt_src == >>>> ODP_PACKET_INVALID) >>>> - return -1; >>>> +void odp_packet_set_inflag_ipv6(odp_packet_t pkt, int val) >>>> +{ >>>> + odp_packet_hdr(pkt)->input_flags.ipv6 = val; >>>> +} >>>> >>>> - if (pkt_hdr_dst->buf_hdr.size < >>>> - pkt_hdr_src->frame_len + pkt_hdr_src->frame_offset) >>>> - return -1; >>>> +int odp_packet_inflag_ipfrag(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->input_flags.ipfrag; >>>> +} >>>> + >>>> +void odp_packet_set_inflag_ipfrag(odp_packet_t pkt, int val) >>>> +{ >>>> + odp_packet_hdr(pkt)->input_flags.ipfrag = val; >>>> +} >>>> + >>>> +int odp_packet_inflag_ipopt(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->input_flags.ipopt; >>>> +} >>>> >>>> - /* Copy packet header */ >>>> - start_dst = (uint8_t *)pkt_hdr_dst + start_offset; >>>> - start_src = (uint8_t *)pkt_hdr_src + start_offset; >>>> - len = ODP_OFFSETOF(odp_packet_hdr_t, buf_data) - start_offset; >>>> - memcpy(start_dst, start_src, len); >>>> +void odp_packet_set_inflag_ipopt(odp_packet_t pkt, int val) >>>> +{ >>>> + odp_packet_hdr(pkt)->input_flags.ipopt = val; >>>> +} >>>> >>>> - /* Copy frame payload */ >>>> - start_dst = (uint8_t *)odp_packet_data(pkt_dst); >>>> - start_src = (uint8_t *)odp_packet_data(pkt_src); >>>> - len = pkt_hdr_src->frame_len; >>>> - memcpy(start_dst, start_src, len); >>>> +int odp_packet_inflag_ipsec(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->input_flags.ipsec; >>>> +} >>>> >>>> - /* Copy useful things from the buffer header */ >>>> - pkt_hdr_dst->buf_hdr.cur_offset = >>>> pkt_hdr_src->buf_hdr.cur_offset; >>>> +void odp_packet_set_inflag_ipsec(odp_packet_t pkt, int val) >>>> +{ >>>> + odp_packet_hdr(pkt)->input_flags.ipsec = val; >>>> +} >>>> >>>> - /* Create a copy of the scatter list */ >>>> - odp_buffer_copy_scatter(odp_packet_to_buffer(pkt_dst), >>>> - odp_packet_to_buffer(pkt_src)); >>>> +int odp_packet_inflag_udp(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->input_flags.udp; >>>> +} >>>> >>>> - return 0; >>>> +void odp_packet_set_inflag_udp(odp_packet_t pkt, int val) >>>> +{ >>>> + odp_packet_hdr(pkt)->input_flags.udp = val; >>>> } >>>> >>>> -void odp_packet_set_ctx(odp_packet_t pkt, const void *ctx) >>>> +int odp_packet_inflag_tcp(odp_packet_t pkt) >>>> { >>>> - odp_packet_hdr(pkt)->user_ctx = (intptr_t)ctx; >>>> + return odp_packet_hdr(pkt)->input_flags.tcp; >>>> } >>>> >>>> -void *odp_packet_get_ctx(odp_packet_t pkt) >>>> +void odp_packet_set_inflag_tcp(odp_packet_t pkt, int val) >>>> { >>>> - return (void *)(intptr_t)odp_packet_hdr(pkt)->user_ctx; >>>> + odp_packet_hdr(pkt)->input_flags.tcp = val; >>>> +} >>>> + >>>> +int odp_packet_inflag_tcpopt(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->input_flags.tcpopt; >>>> +} >>>> + >>>> +void odp_packet_set_inflag_tcpopt(odp_packet_t pkt, int val) >>>> +{ >>>> + odp_packet_hdr(pkt)->input_flags.tcpopt = val; >>>> +} >>>> + >>>> +int odp_packet_inflag_icmp(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->input_flags.icmp; >>>> +} >>>> + >>>> +void odp_packet_set_inflag_icmp(odp_packet_t pkt, int val) >>>> +{ >>>> + odp_packet_hdr(pkt)->input_flags.icmp = val; >>>> +} >>>> + >>>> +int odp_packet_is_valid(odp_packet_t pkt) >>>> +{ >>>> + odp_buffer_hdr_t *buf = validate_buf((odp_buffer_t)pkt); >>>> + >>>> + if (buf == NULL) >>>> + return 0; >>>> + else >>>> + return buf->type == ODP_BUFFER_TYPE_PACKET; >>>> +} >>>> + >>>> +int odp_packet_is_segmented(odp_packet_t pkt) >>>> +{ >>>> + return (odp_packet_hdr(pkt)->buf_hdr.segcount > 1); >>>> +} >>>> + >>>> +int odp_packet_segment_count(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->buf_hdr.segcount; >>>> +} >>>> + >>>> +size_t odp_packet_headroom(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->headroom; >>>> +} >>>> + >>>> +size_t odp_packet_tailroom(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_hdr(pkt)->tailroom; >>>> +} >>>> + >>>> +int odp_packet_push_head(odp_packet_t pkt, size_t len) >>>> +{ >>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>> + >>>> + if (len > pkt_hdr->headroom) >>>> + return -1; >>>> + >>>> + push_head(pkt_hdr, len); >>>> + return 0; >>>> +} >>>> + >>>> +void *odp_packet_push_head_and_map(odp_packet_t pkt, size_t len, >>>> size_t *seglen) >>>> +{ >>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>> + >>>> + if (len > pkt_hdr->headroom) >>>> + return NULL; >>>> + >>>> + push_head(pkt_hdr, len); >>>> + >>>> + return buffer_map(&pkt_hdr->buf_hdr, 0, seglen, >>>> pkt_hdr->frame_len); >>>> +} >>>> + >>>> +int odp_packet_pull_head(odp_packet_t pkt, size_t len) >>>> +{ >>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>> + >>>> + if (len > pkt_hdr->frame_len) >>>> + return -1; >>>> + >>>> + pull_head(pkt_hdr, len); >>>> + return 0; >>>> +} >>>> + >>>> +void *odp_packet_pull_head_and_map(odp_packet_t pkt, size_t len, >>>> size_t *seglen) >>>> +{ >>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>> + >>>> + if (len > pkt_hdr->frame_len) >>>> + return NULL; >>>> + >>>> + pull_head(pkt_hdr, len); >>>> + return buffer_map(&pkt_hdr->buf_hdr, 0, seglen, >>>> pkt_hdr->frame_len); >>>> +} >>>> + >>>> +int odp_packet_push_tail(odp_packet_t pkt, size_t len) >>>> +{ >>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>> + >>>> + if (len > pkt_hdr->tailroom) >>>> + return -1; >>>> + >>>> + push_tail(pkt_hdr, len); >>>> + return 0; >>>> +} >>>> + >>>> +void *odp_packet_push_tail_and_map(odp_packet_t pkt, size_t len, >>>> size_t *seglen) >>>> +{ >>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>> + size_t origin = pkt_hdr->frame_len; >>>> + >>>> + if (len > pkt_hdr->tailroom) >>>> + return NULL; >>>> + >>>> + push_tail(pkt_hdr, len); >>>> + >>>> + return buffer_map(&pkt_hdr->buf_hdr, origin, seglen, >>>> + pkt_hdr->frame_len); >>>> +} >>>> + >>>> +int odp_packet_pull_tail(odp_packet_t pkt, size_t len) >>>> +{ >>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>> + >>>> + if (len > pkt_hdr->frame_len) >>>> + return -1; >>>> + >>>> + pull_tail(pkt_hdr, len); >>>> + return 0; >>>> +} >>>> + >>>> +void odp_packet_print(odp_packet_t pkt) >>>> +{ >>>> + int max_len = 512; >>>> + char str[max_len]; >>>> + int len = 0; >>>> + int n = max_len-1; >>>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>>> + >>>> + len += snprintf(&str[len], n-len, "Packet "); >>>> + len += odp_buffer_snprint(&str[len], n-len, (odp_buffer_t) pkt); >>>> + len += snprintf(&str[len], n-len, >>>> + " input_flags 0x%x\n", hdr->input_flags.all); >>>> + len += snprintf(&str[len], n-len, >>>> + " error_flags 0x%x\n", hdr->error_flags.all); >>>> + len += snprintf(&str[len], n-len, >>>> + " output_flags 0x%x\n", hdr->output_flags.all); >>>> + len += snprintf(&str[len], n-len, >>>> + " l2_offset %u\n", hdr->l2_offset); >>>> + len += snprintf(&str[len], n-len, >>>> + " l3_offset %u\n", hdr->l3_offset); >>>> + len += snprintf(&str[len], n-len, >>>> + " l3_len %u\n", hdr->l3_len); >>>> + len += snprintf(&str[len], n-len, >>>> + " l3_protocol 0x%x\n", hdr->l3_protocol); >>>> + len += snprintf(&str[len], n-len, >>>> + " l4_offset %u\n", hdr->l4_offset); >>>> + len += snprintf(&str[len], n-len, >>>> + " l4_len %u\n", hdr->l4_len); >>>> + len += snprintf(&str[len], n-len, >>>> + " l4_protocol %u\n", hdr->l4_protocol); >>>> + len += snprintf(&str[len], n-len, >>>> + " payload_offset %u\n", >>>> hdr->payload_offset); >>>> + len += snprintf(&str[len], n-len, >>>> + " frame_len %u\n", hdr->frame_len); >>>> + str[len] = '\0'; >>>> + >>>> + printf("\n%s\n", str); >>>> +} >>>> + >>>> +int odp_packet_copy_to_packet(odp_packet_t dstpkt, size_t dstoffset, >>>> + odp_packet_t srcpkt, size_t srcoffset, >>>> + size_t len) >>>> +{ >>>> + void *dstmap; >>>> + void *srcmap; >>>> + size_t cpylen, minseg, dstseglen, srcseglen; >>>> + >>>> + while (len > 0) { >>>> + dstmap = odp_packet_offset_map(dstpkt, dstoffset, >>>> &dstseglen); >>>> + srcmap = odp_packet_offset_map(srcpkt, srcoffset, >>>> &srcseglen); >>>> + if (dstmap == NULL || srcmap == NULL) >>>> + return -1; >>>> + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; >>>> + cpylen = len > minseg ? minseg : len; >>>> + memcpy(dstmap, srcmap, cpylen); >>>> + srcoffset += cpylen; >>>> + dstoffset += cpylen; >>>> + len -= cpylen; >>>> + } >>>> + >>>> + return 0; >>>> +} >>>> + >>>> +int odp_packet_copy_to_memory(void *dstmem, odp_packet_t srcpkt, >>>> + size_t srcoffset, size_t dstlen) >>>> +{ >>>> + void *mapaddr; >>>> + size_t seglen, cpylen; >>>> + uint8_t *dstaddr = (uint8_t *)dstmem; >>>> + >>>> + while (dstlen > 0) { >>>> + mapaddr = odp_packet_offset_map(srcpkt, srcoffset, >>>> &seglen); >>>> + if (mapaddr == NULL) >>>> + return -1; >>>> + cpylen = dstlen > seglen ? seglen : dstlen; >>>> + memcpy(dstaddr, mapaddr, cpylen); >>>> + srcoffset += cpylen; >>>> + dstaddr += cpylen; >>>> + dstlen -= cpylen; >>>> + } >>>> + >>>> + return 0; >>>> +} >>>> + >>>> +int odp_packet_copy_from_memory(odp_packet_t dstpkt, size_t dstoffset, >>>> + void *srcmem, size_t srclen) >>>> +{ >>>> + void *mapaddr; >>>> + size_t seglen, cpylen; >>>> + uint8_t *srcaddr = (uint8_t *)srcmem; >>>> + >>>> + while (srclen > 0) { >>>> + mapaddr = odp_packet_offset_map(dstpkt, dstoffset, >>>> &seglen); >>>> + if (mapaddr == NULL) >>>> + return -1; >>>> + cpylen = srclen > seglen ? seglen : srclen; >>>> + memcpy(mapaddr, srcaddr, cpylen); >>>> + dstoffset += cpylen; >>>> + srcaddr += cpylen; >>>> + srclen -= cpylen; >>>> + } >>>> + >>>> + return 0; >>>> +} >>>> + >>>> +odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool) >>>> +{ >>>> + size_t pktlen = odp_packet_len(pkt); >>>> + const size_t meta_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, >>>> buf_hdr); >>>> + odp_packet_t newpkt = odp_packet_alloc_len(pool, pktlen); >>>> + odp_packet_hdr_t *newhdr, *srchdr; >>>> + uint8_t *newstart, *srcstart; >>>> + >>>> + if (newpkt != ODP_PACKET_INVALID) { >>>> + /* Must copy meta data first, followed by packet data */ >>>> + srchdr = odp_packet_hdr(pkt); >>>> + newhdr = odp_packet_hdr(newpkt); >>>> + newstart = (uint8_t *)newhdr + meta_offset; >>>> + srcstart = (uint8_t *)srchdr + meta_offset; >>>> + >>>> + memcpy(newstart, srcstart, >>>> + sizeof(odp_packet_hdr_t) - meta_offset); >>>> + >>>> + if (odp_packet_copy_to_packet(newpkt, 0, pkt, 0, >>>> pktlen) != 0) { >>>> + odp_packet_free(newpkt); >>>> + newpkt = ODP_PACKET_INVALID; >>>> + } >>>> + } >>>> + >>>> + return newpkt; >>>> +} >>>> + >>>> +odp_packet_t odp_packet_clone(odp_packet_t pkt) >>>> +{ >>>> + return odp_packet_copy(pkt, >>>> odp_packet_hdr(pkt)->buf_hdr.pool_hdl); >>>> +} >>>> + >>>> +odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool) >>>> +{ >>>> + if (odp_pool_to_entry(pool)->s.params.buf_type != >>>> + ODP_BUFFER_TYPE_PACKET) >>>> + return ODP_PACKET_INVALID; >>>> + >>>> + return (odp_packet_t)buffer_alloc(pool, 0); >>>> +} >>>> + >>>> +odp_packet_t odp_packet_alloc_len(odp_buffer_pool_t pool, size_t len) >>>> +{ >>>> + if (odp_pool_to_entry(pool)->s.params.buf_type != >>>> + ODP_BUFFER_TYPE_PACKET) >>>> + return ODP_PACKET_INVALID; >>>> + >>>> + return (odp_packet_t)buffer_alloc(pool, len); >>>> +} >>>> + >>>> +void odp_packet_free(odp_packet_t pkt) >>>> +{ >>>> + odp_buffer_free((odp_buffer_t)pkt); >>>> +} >>>> + >>>> +odp_packet_t odp_packet_split(odp_packet_t pkt, size_t offset, >>>> + size_t hr, size_t tr) >>>> +{ >>>> + size_t len = odp_packet_len(pkt); >>>> + odp_buffer_pool_t pool = odp_packet_pool(pkt); >>>> + size_t pool_hr = odp_buffer_pool_headroom(pool); >>>> + size_t pkt_tr = odp_packet_tailroom(pkt); >>>> + odp_packet_t splitpkt; >>>> + size_t splitlen = len - offset; >>>> + >>>> + if (offset >= len) >>>> + return ODP_PACKET_INVALID; >>>> + >>>> + /* Erratum: We don't handle this rare corner case */ >>>> + if (tr > pkt_tr + splitlen) >>>> + return ODP_PACKET_INVALID; >>>> + >>>> + if (hr > pool_hr) >>>> + splitlen += (hr - pool_hr); >>>> + >>>> + splitpkt = odp_packet_alloc_len(pool, splitlen); >>>> + >>>> + if (splitpkt != ODP_PACKET_INVALID) { >>>> + if (hr > pool_hr) { >>>> + odp_packet_pull_head(splitpkt, hr - pool_hr); >>>> + splitlen -= (hr - pool_hr); >>>> + } >>>> + odp_packet_copy_to_packet(splitpkt, 0, >>>> + pkt, offset, splitlen); >>>> + odp_packet_pull_tail(pkt, splitlen); >>>> + } >>>> + >>>> + return splitpkt; >>>> +} >>>> + >>>> +odp_packet_t odp_packet_join(odp_packet_t pkt1, odp_packet_t pkt2) >>>> +{ >>>> + size_t len1 = odp_packet_len(pkt1); >>>> + size_t len2 = odp_packet_len(pkt2); >>>> + odp_packet_t joinpkt; >>>> + void *udata1, *udata_join; >>>> + size_t udata_size1, udata_size_join; >>>> + >>>> + /* Optimize if pkt1 is big enough to hold pkt2 */ >>>> + if (odp_packet_push_tail(pkt1, len2) == 0) { >>>> + odp_packet_copy_to_packet(pkt1, len1, >>>> + pkt2, 0, len2); >>>> + odp_packet_free(pkt2); >>>> + return pkt1; >>>> + } >>>> + >>>> + /* Otherwise join into a new packet */ >>>> + joinpkt = odp_packet_alloc_len(odp_packet_pool(pkt1), >>>> + len1 + len2); >>>> + >>>> + if (joinpkt != ODP_PACKET_INVALID) { >>>> + odp_packet_copy_to_packet(joinpkt, 0, pkt1, 0, len1); >>>> + odp_packet_copy_to_packet(joinpkt, len1, pkt2, 0, len2); >>>> + >>>> + /* Copy user metadata if present */ >>>> + udata1 = odp_packet_udata(pkt1, &udata_size1); >>>> + udata_join = odp_packet_udata(joinpkt, >>>> &udata_size_join); >>>> + >>>> + if (udata1 != NULL && udata_join != NULL) >>>> + memcpy(udata_join, udata1, >>>> + udata_size_join > udata_size1 ? >>>> + udata_size1 : udata_size_join); >>>> + >>>> + odp_packet_free(pkt1); >>>> + odp_packet_free(pkt2); >>>> + } >>>> + >>>> + return joinpkt; >>>> +} >>>> + >>>> +uint32_t odp_packet_refcount(odp_packet_t pkt) >>>> +{ >>>> + return odp_buffer_refcount(&odp_packet_hdr(pkt)->buf_hdr); >>>> +} >>>> + >>>> +uint32_t odp_packet_incr_refcount(odp_packet_t pkt, uint32_t val) >>>> +{ >>>> + return odp_buffer_incr_refcount(&odp_packet_hdr(pkt)->buf_hdr, >>>> val); >>>> +} >>>> + >>>> +uint32_t odp_packet_decr_refcount(odp_packet_t pkt, uint32_t val) >>>> +{ >>>> + return odp_buffer_decr_refcount(&odp_packet_hdr(pkt)->buf_hdr, >>>> val); >>>> +} >>>> + >>>> +/** >>>> + * Parser helper function for IPv4 >>>> + */ >>>> +static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >>>> + uint8_t **parseptr, size_t *offset) >>>> +{ >>>> + odph_ipv4hdr_t *ipv4 = (odph_ipv4hdr_t *)*parseptr; >>>> + uint8_t ver = ODPH_IPV4HDR_VER(ipv4->ver_ihl); >>>> + uint8_t ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); >>>> + uint16_t frag_offset; >>>> + >>>> + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv4->tot_len); >>>> + >>>> + if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN) || >>>> + odp_unlikely(ver != 4) || >>>> + (pkt_hdr->l3_len > pkt_hdr->frame_len - *offset)) { >>>> + pkt_hdr->error_flags.ip_err = 1; >>>> + return 0; >>>> + } >>>> + >>>> + *offset += ihl * 4; >>>> + *parseptr += ihl * 4; >>>> + >>>> + if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) >>>> + pkt_hdr->input_flags.ipopt = 1; >>>> + >>>> + /* A packet is a fragment if: >>>> + * "more fragments" flag is set (all fragments except the last) >>>> + * OR >>>> + * "fragment offset" field is nonzero (all fragments except the >>>> first) >>>> + */ >>>> + frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); >>>> + if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) >>>> + pkt_hdr->input_flags.ipfrag = 1; >>>> + >>>> + if (ipv4->proto == ODPH_IPPROTO_ESP || >>>> + ipv4->proto == ODPH_IPPROTO_AH) { >>>> + pkt_hdr->input_flags.ipsec = 1; >>>> + return 0; >>>> + } >>>> + >>>> + /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after >>>> return */ >>>> + >>>> + *offset = sizeof(uint32_t) * ihl; >>>> + return ipv4->proto; >>>> +} >>>> + >>>> +/** >>>> + * Parser helper function for IPv6 >>>> + */ >>>> +static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >>>> + uint8_t **parseptr, size_t *offset) >>>> +{ >>>> + odph_ipv6hdr_t *ipv6 = (odph_ipv6hdr_t *)*parseptr; >>>> + odph_ipv6hdr_ext_t *ipv6ext; >>>> + >>>> + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv6->payload_len); >>>> + >>>> + /* Basic sanity checks on IPv6 header */ >>>> + if (ipv6->ver != 6 || >>>> + pkt_hdr->l3_len > pkt_hdr->frame_len - *offset) { >>>> + pkt_hdr->error_flags.ip_err = 1; >>>> + return 0; >>>> + } >>>> + >>>> + /* Skip past IPv6 header */ >>>> + *offset += sizeof(odph_ipv6hdr_t); >>>> + *parseptr += sizeof(odph_ipv6hdr_t); >>>> + >>>> + >>>> + /* Skip past any IPv6 extension headers */ >>>> + if (ipv6->next_hdr == ODPH_IPPROTO_HOPOPTS || >>>> + ipv6->next_hdr == ODPH_IPPROTO_ROUTE) { >>>> + pkt_hdr->input_flags.ipopt = 1; >>>> + >>>> + do { >>>> + ipv6ext = (odph_ipv6hdr_ext_t *)*parseptr; >>>> + uint16_t extlen = 8 + ipv6ext->ext_len * 8; >>>> + >>>> + *offset += extlen; >>>> + *parseptr += extlen; >>>> + } while ((ipv6ext->next_hdr == ODPH_IPPROTO_HOPOPTS || >>>> + ipv6ext->next_hdr == ODPH_IPPROTO_ROUTE) && >>>> + *offset < pkt_hdr->frame_len); >>>> + >>>> + if (*offset >= pkt_hdr->l3_offset + ipv6->payload_len) { >>>> + pkt_hdr->error_flags.ip_err = 1; >>>> + return 0; >>>> + } >>>> + >>>> + if (ipv6ext->next_hdr == ODPH_IPPROTO_FRAG) >>>> + pkt_hdr->input_flags.ipfrag = 1; >>>> + >>>> + return ipv6ext->next_hdr; >>>> + } >>>> + >>>> + if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { >>>> + pkt_hdr->input_flags.ipopt = 1; >>>> + pkt_hdr->input_flags.ipfrag = 1; >>>> + } >>>> + >>>> + return ipv6->next_hdr; >>>> +} >>>> + >>>> +/** >>>> + * Parser helper function for TCP >>>> + */ >>>> +static inline void parse_tcp(odp_packet_hdr_t *pkt_hdr, >>>> + uint8_t **parseptr, size_t *offset) >>>> +{ >>>> + odph_tcphdr_t *tcp = (odph_tcphdr_t *)*parseptr; >>>> + >>>> + if (tcp->hl < sizeof(odph_tcphdr_t)/sizeof(uint32_t)) >>>> + pkt_hdr->error_flags.tcp_err = 1; >>>> + else if ((uint32_t)tcp->hl * 4 > sizeof(odph_tcphdr_t)) >>>> + pkt_hdr->input_flags.tcpopt = 1; >>>> + >>>> + pkt_hdr->l4_len = pkt_hdr->l3_len + >>>> + pkt_hdr->l3_offset - pkt_hdr->l4_offset; >>>> + >>>> + *offset += sizeof(odph_tcphdr_t); >>>> + *parseptr += sizeof(odph_tcphdr_t); >>>> +} >>>> + >>>> +/** >>>> + * Parser helper function for UDP >>>> + */ >>>> +static inline void parse_udp(odp_packet_hdr_t *pkt_hdr, >>>> + uint8_t **parseptr, size_t *offset) >>>> +{ >>>> + odph_udphdr_t *udp = (odph_udphdr_t *)*parseptr; >>>> + uint32_t udplen = odp_be_to_cpu_16(udp->length); >>>> + >>>> + if (udplen < sizeof(odph_udphdr_t) || >>>> + udplen > (pkt_hdr->l3_len + >>>> + pkt_hdr->l3_offset - pkt_hdr->l4_offset)) { >>>> + pkt_hdr->error_flags.udp_err = 1; >>>> + } >>>> + >>>> + pkt_hdr->l4_len = udplen; >>>> + >>>> + *offset += sizeof(odph_udphdr_t); >>>> + *parseptr += sizeof(odph_udphdr_t); >>>> +} >>>> + >>>> +/** >>>> + * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP >>>> + * >>>> + * Internal function: caller is resposible for passing only valid >>>> packet handles >>>> + * , lengths and offsets (usually done&called in packet input). >>>> + * >>>> + * @param pkt Packet handle >>>> + */ >>>> +int odp_packet_parse(odp_packet_t pkt) >>>> +{ >>>> + odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >>>> + odph_ethhdr_t *eth; >>>> + odph_vlanhdr_t *vlan; >>>> + uint16_t ethtype; >>>> + uint8_t *parseptr; >>>> + size_t offset, seglen; >>>> + uint8_t ip_proto = 0; >>>> + >>>> + /* Reset parser metadata for new parse */ >>>> + pkt_hdr->error_flags.all = 0; >>>> + pkt_hdr->input_flags.all = 0; >>>> + pkt_hdr->output_flags.all = 0; >>>> + pkt_hdr->l2_offset = 0; >>>> + pkt_hdr->l3_offset = 0; >>>> + pkt_hdr->l4_offset = 0; >>>> + pkt_hdr->payload_offset = 0; >>>> + pkt_hdr->vlan_s_tag = 0; >>>> + pkt_hdr->vlan_c_tag = 0; >>>> + pkt_hdr->l3_protocol = 0; >>>> + pkt_hdr->l4_protocol = 0; >>>> + >>>> + /* We only support Ethernet for now */ >>>> + pkt_hdr->input_flags.eth = 1; >>>> + >>>> + if (odp_unlikely(pkt_hdr->frame_len < ODPH_ETH_LEN_MIN)) { >>>> + pkt_hdr->error_flags.frame_len = 1; >>>> + goto parse_exit; >>>> + } else if (pkt_hdr->frame_len > ODPH_ETH_LEN_MAX) { >>>> + pkt_hdr->input_flags.jumbo = 1; >>>> + } >>>> + >>>> + /* Assume valid L2 header, no CRC/FCS check in SW */ >>>> + pkt_hdr->input_flags.l2 = 1; >>>> + >>>> + eth = (odph_ethhdr_t *)odp_packet_map(pkt, &seglen); >>>> + offset = sizeof(odph_ethhdr_t); >>>> + parseptr = (uint8_t *)ð->type; >>>> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); >>>> + >>>> + /* Parse the VLAN header(s), if present */ >>>> + if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { >>>> + pkt_hdr->input_flags.vlan_qinq = 1; >>>> + pkt_hdr->input_flags.vlan = 1; >>>> + vlan = (odph_vlanhdr_t *)(void *)parseptr; >>>> + pkt_hdr->vlan_s_tag = ((ethtype << 16) | >>>> + odp_be_to_cpu_16(vlan->tci)); >>>> + offset += sizeof(odph_vlanhdr_t); >>>> + parseptr += sizeof(odph_vlanhdr_t); >>>> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >>>> *)parseptr)); >>>> + } >>>> + >>>> + if (ethtype == ODPH_ETHTYPE_VLAN) { >>>> + pkt_hdr->input_flags.vlan = 1; >>>> + vlan = (odph_vlanhdr_t *)(void *)parseptr; >>>> + pkt_hdr->vlan_c_tag = ((ethtype << 16) | >>>> + odp_be_to_cpu_16(vlan->tci)); >>>> + offset += sizeof(odph_vlanhdr_t); >>>> + parseptr += sizeof(odph_vlanhdr_t); >>>> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >>>> *)parseptr)); >>>> + } >>>> + >>>> + /* Check for SNAP vs. DIX */ >>>> + if (ethtype < ODPH_ETH_LEN_MAX) { >>>> + pkt_hdr->input_flags.snap = 1; >>>> + if (ethtype > pkt_hdr->frame_len - offset) { >>>> + pkt_hdr->error_flags.snap_len = 1; >>>> + goto parse_exit; >>>> + } >>>> + offset += 8; >>>> + parseptr += 8; >>>> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >>>> *)parseptr)); >>>> + } >>>> + >>>> + /* Consume Ethertype for Layer 3 parse */ >>>> + parseptr += 2; >>>> + >>>> + /* Set l3_offset+flag only for known ethtypes */ >>>> + pkt_hdr->input_flags.l3 = 1; >>>> + pkt_hdr->l3_offset = offset; >>>> + pkt_hdr->l3_protocol = ethtype; >>>> + >>>> + /* Parse Layer 3 headers */ >>>> + switch (ethtype) { >>>> + case ODPH_ETHTYPE_IPV4: >>>> + pkt_hdr->input_flags.ipv4 = 1; >>>> + ip_proto = parse_ipv4(pkt_hdr, &parseptr, &offset); >>>> + break; >>>> + >>>> + case ODPH_ETHTYPE_IPV6: >>>> + pkt_hdr->input_flags.ipv6 = 1; >>>> + ip_proto = parse_ipv6(pkt_hdr, &parseptr, &offset); >>>> + break; >>>> + >>>> + case ODPH_ETHTYPE_ARP: >>>> + pkt_hdr->input_flags.arp = 1; >>>> + ip_proto = 255; /* Reserved invalid by IANA */ >>>> + break; >>>> + >>>> + default: >>>> + pkt_hdr->input_flags.l3 = 0; >>>> + ip_proto = 255; /* Reserved invalid by IANA */ >>>> + } >>>> + >>>> + /* Set l4_offset+flag only for known ip_proto */ >>>> + pkt_hdr->input_flags.l4 = 1; >>>> + pkt_hdr->l4_offset = offset; >>>> + pkt_hdr->l4_protocol = ip_proto; >>>> + >>>> + /* Parse Layer 4 headers */ >>>> + switch (ip_proto) { >>>> + case ODPH_IPPROTO_ICMP: >>>> + pkt_hdr->input_flags.icmp = 1; >>>> + break; >>>> + >>>> + case ODPH_IPPROTO_TCP: >>>> + pkt_hdr->input_flags.tcp = 1; >>>> + parse_tcp(pkt_hdr, &parseptr, &offset); >>>> + break; >>>> + >>>> + case ODPH_IPPROTO_UDP: >>>> + pkt_hdr->input_flags.udp = 1; >>>> + parse_udp(pkt_hdr, &parseptr, &offset); >>>> + break; >>>> + >>>> + case ODPH_IPPROTO_AH: >>>> + case ODPH_IPPROTO_ESP: >>>> + pkt_hdr->input_flags.ipsec = 1; >>>> + break; >>>> + >>>> + default: >>>> + pkt_hdr->input_flags.l4 = 0; >>>> + break; >>>> + } >>>> + >>>> + /* >>>> + * Anything beyond what we parse here is considered payload. >>>> + * Note: Payload is really only relevant for TCP and UDP. For >>>> + * all other protocols, the payload offset will point to the >>>> + * final header (ARP, ICMP, AH, ESP, or IP Fragment. >>>> + */ >>>> + pkt_hdr->payload_offset = offset; >>>> + >>>> +parse_exit: >>>> + return pkt_hdr->error_flags.all != 0; >>>> } >>>> diff --git a/platform/linux-generic/odp_packet_flags.c >>>> b/platform/linux-generic/odp_packet_flags.c >>>> deleted file mode 100644 >>>> index 06fdeed..0000000 >>>> --- a/platform/linux-generic/odp_packet_flags.c >>>> +++ /dev/null >>>> @@ -1,202 +0,0 @@ >>>> -/* Copyright (c) 2014, Linaro Limited >>>> - * All rights reserved. >>>> - * >>>> - * SPDX-License-Identifier: BSD-3-Clause >>>> - */ >>>> - >>>> -#include <odp_packet_flags.h> >>>> -#include <odp_packet_internal.h> >>>> - >>>> - >>>> -int odp_packet_error(odp_packet_t pkt) >>>> -{ >>>> - return (odp_packet_hdr(pkt)->error_flags.all != 0); >>>> -} >>>> - >>>> -/* Get Error Flags */ >>>> - >>>> -int odp_packet_errflag_frame_len(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->error_flags.frame_len; >>>> -} >>>> - >>>> -/* Get Input Flags */ >>>> - >>>> -int odp_packet_inflag_l2(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.l2; >>>> -} >>>> - >>>> -int odp_packet_inflag_l3(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.l3; >>>> -} >>>> - >>>> -int odp_packet_inflag_l4(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.l4; >>>> -} >>>> - >>>> -int odp_packet_inflag_eth(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.eth; >>>> -} >>>> - >>>> -int odp_packet_inflag_jumbo(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.jumbo; >>>> -} >>>> - >>>> -int odp_packet_inflag_vlan(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.vlan; >>>> -} >>>> - >>>> -int odp_packet_inflag_vlan_qinq(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.vlan_qinq; >>>> -} >>>> - >>>> -int odp_packet_inflag_arp(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.arp; >>>> -} >>>> - >>>> -int odp_packet_inflag_ipv4(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.ipv4; >>>> -} >>>> - >>>> -int odp_packet_inflag_ipv6(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.ipv6; >>>> -} >>>> - >>>> -int odp_packet_inflag_ipfrag(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.ipfrag; >>>> -} >>>> - >>>> -int odp_packet_inflag_ipopt(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.ipopt; >>>> -} >>>> - >>>> -int odp_packet_inflag_ipsec(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.ipsec; >>>> -} >>>> - >>>> -int odp_packet_inflag_udp(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.udp; >>>> -} >>>> - >>>> -int odp_packet_inflag_tcp(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.tcp; >>>> -} >>>> - >>>> -int odp_packet_inflag_sctp(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.sctp; >>>> -} >>>> - >>>> -int odp_packet_inflag_icmp(odp_packet_t pkt) >>>> -{ >>>> - return odp_packet_hdr(pkt)->input_flags.icmp; >>>> -} >>>> - >>>> -/* Set Output Flags */ >>>> - >>>> -void odp_packet_outflag_l4_chksum(odp_packet_t pkt) >>>> -{ >>>> - odp_packet_hdr(pkt)->output_flags.l4_chksum = 1; >>>> -} >>>> - >>>> -/* Set Input Flags */ >>>> - >>>> -void odp_packet_set_inflag_l2(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.l2 = val; >>>> -} >>>> - >>>> -void odp_packet_set_inflag_l3(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.l3 = val; >>>> -} >>>> - >>>> -void odp_packet_set_inflag_l4(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.l4 = val; >>>> -} >>>> - >>>> -void odp_packet_set_inflag_eth(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.eth = val; >>>> -} >>>> - >>>> -void odp_packet_set_inflag_jumbo(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.jumbo = val; >>>> -} >>>> - >>>> -void odp_packet_set_inflag_vlan(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.vlan = val; >>>> -} >>>> - >>>> -void odp_packet_set_inflag_vlan_qinq(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.vlan_qinq = val; >>>> -} >>>> - >>>> -void odp_packet_set_inflag_arp(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.arp = val; >>>> -} >>>> - >>>> -void odp_packet_set_inflag_ipv4(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.ipv4 = val; >>>> -} >>>> - >>>> -void odp_packet_set_inflag_ipv6(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.ipv6 = val; >>>> -} >>>> - >>>> -void odp_packet_set_inflag_ipfrag(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.ipfrag = val; >>>> -} >>>> - >>>> -void odp_packet_set_inflag_ipopt(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.ipopt = val; >>>> -} >>>> - >>>> -void odp_packet_set_inflag_ipsec(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.ipsec = val; >>>> -} >>>> - >>>> -void odp_packet_set_inflag_udp(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.udp = val; >>>> -} >>>> - >>>> -void odp_packet_set_inflag_tcp(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.tcp = val; >>>> -} >>>> - >>>> -void odp_packet_set_inflag_sctp(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.sctp = val; >>>> -} >>>> - >>>> -void odp_packet_set_inflag_icmp(odp_packet_t pkt, int val) >>>> -{ >>>> - odp_packet_hdr(pkt)->input_flags.icmp = val; >>>> -} >>>> -- >>>> 1.8.3.2 >>>> >>>> >>>> _______________________________________________ >>>> lng-odp mailing list >>>> lng-odp@lists.linaro.org >>>> http://lists.linaro.org/mailman/listinfo/lng-odp >>>> >>> >>> >> >
v5 of part 9/9 resent with the above two fixes. On Tue, Nov 11, 2014 at 10:48 AM, Bill Fischofer <bill.fischofer@linaro.org> wrote: > Sounds reasonable. It will be in the next rev, along with Tara's latest > bug fix. > > Thanks. > > Bill > > On Tue, Nov 11, 2014 at 10:42 AM, Bala Manoharan < > bala.manoharan@linaro.org> wrote: > >> I am okay with the behaviour for "init_params" value, my concern is with >> "params" value. >> Since this params deals with the number of buffer in the pool and >> buf_size. >> I would like to return an error if the params value is NULL. >> >> Regards, >> Bala >> >> On 11 November 2014 22:09, Bill Fischofer <bill.fischofer@linaro.org> >> wrote: >> >>> The API allows the init_params to be NULL if user meta data is not >>> required. I just added the corresponding support to the params to have the >>> same behavior. I have no problem failing the call if that parameter is >>> missing. >>> >>> Bill >>> >>> On Tue, Nov 11, 2014 at 10:34 AM, Bala Manoharan < >>> bala.manoharan@linaro.org> wrote: >>> >>>> >>>> >>>> On 11 November 2014 09:33, Bill Fischofer <bill.fischofer@linaro.org> >>>> wrote: >>>> >>>>> Signed-off-by: Bill Fischofer <bill.fischofer@linaro.org> >>>>> --- >>>>> platform/linux-generic/odp_buffer.c | 263 ++++++- >>>>> platform/linux-generic/odp_buffer_pool.c | 661 +++++++--------- >>>>> platform/linux-generic/odp_packet.c | 1200 >>>>> +++++++++++++++++++++++------ >>>>> platform/linux-generic/odp_packet_flags.c | 202 ----- >>>>> 4 files changed, 1455 insertions(+), 871 deletions(-) >>>>> delete mode 100644 platform/linux-generic/odp_packet_flags.c >>>>> >>>>> diff --git a/platform/linux-generic/odp_buffer.c >>>>> b/platform/linux-generic/odp_buffer.c >>>>> index e54e0e7..e47c722 100644 >>>>> --- a/platform/linux-generic/odp_buffer.c >>>>> +++ b/platform/linux-generic/odp_buffer.c >>>>> @@ -5,46 +5,259 @@ >>>>> */ >>>>> >>>>> #include <odp_buffer.h> >>>>> -#include <odp_buffer_internal.h> >>>>> #include <odp_buffer_pool_internal.h> >>>>> +#include <odp_buffer_internal.h> >>>>> +#include <odp_buffer_inlines.h> >>>>> >>>>> #include <string.h> >>>>> #include <stdio.h> >>>>> >>>>> +void *odp_buffer_offset_map(odp_buffer_t buf, size_t offset, size_t >>>>> *seglen) >>>>> +{ >>>>> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >>>>> >>>>> -void *odp_buffer_addr(odp_buffer_t buf) >>>>> + if (offset > buf_hdr->size) >>>>> + return NULL; >>>>> + >>>>> + return buffer_map(buf_hdr, offset, seglen, buf_hdr->size); >>>>> +} >>>>> + >>>>> +void odp_buffer_offset_unmap(odp_buffer_t buf ODP_UNUSED, >>>>> + size_t offset ODP_UNUSED) >>>>> { >>>>> - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >>>>> +} >>>>> >>>>> - return hdr->addr; >>>>> +size_t odp_buffer_segment_count(odp_buffer_t buf) >>>>> +{ >>>>> + return odp_buf_to_hdr(buf)->segcount; >>>>> } >>>>> >>>>> +int odp_buffer_is_segmented(odp_buffer_t buf) >>>>> +{ >>>>> + return odp_buf_to_hdr(buf)->segcount > 1; >>>>> +} >>>>> >>>>> -size_t odp_buffer_size(odp_buffer_t buf) >>>>> +odp_buffer_segment_t odp_buffer_segment_by_index(odp_buffer_t buf, >>>>> + size_t ndx) >>>>> { >>>>> - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >>>>> + return buffer_segment(odp_buf_to_hdr(buf), ndx); >>>>> +} >>>>> >>>>> - return hdr->size; >>>>> +odp_buffer_segment_t odp_buffer_segment_next(odp_buffer_t buf, >>>>> + odp_buffer_segment_t seg) >>>>> +{ >>>>> + return segment_next(odp_buf_to_hdr(buf), seg); >>>>> } >>>>> >>>>> +void *odp_buffer_segment_map(odp_buffer_t buf, odp_buffer_segment_t >>>>> seg, >>>>> + size_t *seglen) >>>>> +{ >>>>> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >>>>> >>>>> -int odp_buffer_type(odp_buffer_t buf) >>>>> + return segment_map(buf_hdr, seg, seglen, buf_hdr->size); >>>>> +} >>>>> + >>>>> +void odp_buffer_segment_unmap(odp_buffer_segment_t seg ODP_UNUSED) >>>>> { >>>>> - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >>>>> +} >>>>> + >>>>> +odp_buffer_t odp_buffer_split(odp_buffer_t buf, size_t offset) >>>>> +{ >>>>> + size_t size = odp_buffer_size(buf); >>>>> + odp_buffer_pool_t pool = odp_buffer_pool(buf); >>>>> + odp_buffer_t splitbuf; >>>>> + size_t splitsize = size - offset; >>>>> + >>>>> + if (offset >= size) >>>>> + return ODP_BUFFER_INVALID; >>>>> + >>>>> + splitbuf = buffer_alloc(pool, splitsize); >>>>> + >>>>> + if (splitbuf != ODP_BUFFER_INVALID) { >>>>> + buffer_copy_to_buffer(splitbuf, 0, buf, splitsize, >>>>> splitsize); >>>>> + odp_buffer_trim(buf, splitsize); >>>>> + } >>>>> >>>>> - return hdr->type; >>>>> + return splitbuf; >>>>> } >>>>> >>>>> +odp_buffer_t odp_buffer_join(odp_buffer_t buf1, odp_buffer_t buf2) >>>>> +{ >>>>> + size_t size1 = odp_buffer_size(buf1); >>>>> + size_t size2 = odp_buffer_size(buf2); >>>>> + odp_buffer_t joinbuf; >>>>> + void *udata1, *udata_join; >>>>> + size_t udata_size1, udata_size_join; >>>>> >>>>> -int odp_buffer_is_valid(odp_buffer_t buf) >>>>> + joinbuf = buffer_alloc(odp_buffer_pool(buf1), size1 + size2); >>>>> + >>>>> + if (joinbuf != ODP_BUFFER_INVALID) { >>>>> + buffer_copy_to_buffer(joinbuf, 0, buf1, 0, size1); >>>>> + buffer_copy_to_buffer(joinbuf, size1, buf2, 0, size2); >>>>> + >>>>> + /* Copy user metadata if present */ >>>>> + udata1 = odp_buffer_udata(buf1, &udata_size1); >>>>> + udata_join = odp_buffer_udata(joinbuf, >>>>> &udata_size_join); >>>>> + >>>>> + if (udata1 != NULL && udata_join != NULL) >>>>> + memcpy(udata_join, udata1, >>>>> + udata_size_join > udata_size1 ? >>>>> + udata_size1 : udata_size_join); >>>>> + >>>>> + odp_buffer_free(buf1); >>>>> + odp_buffer_free(buf2); >>>>> + } >>>>> + >>>>> + return joinbuf; >>>>> +} >>>>> + >>>>> +odp_buffer_t odp_buffer_trim(odp_buffer_t buf, size_t bytes) >>>>> { >>>>> - odp_buffer_bits_t handle; >>>>> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >>>>> + >>>>> + if (bytes >= buf_hdr->size) >>>>> + return ODP_BUFFER_INVALID; >>>>> + >>>>> + buf_hdr->size -= bytes; >>>>> + size_t bufsegs = buf_hdr->size / buf_hdr->segsize; >>>>> + >>>>> + if (buf_hdr->size % buf_hdr->segsize > 0) >>>>> + bufsegs++; >>>>> + >>>>> + pool_entry_t *pool = odp_pool_to_entry(buf_hdr->pool_hdl); >>>>> >>>>> - handle.u32 = buf; >>>>> + /* Return excess segments back to block list */ >>>>> + while (bufsegs > buf_hdr->segcount) >>>>> + ret_blk(&pool->s, buf_hdr->addr[--buf_hdr->segcount]); >>>>> >>>>> - return (handle.index != ODP_BUFFER_INVALID_INDEX); >>>>> + return buf; >>>>> } >>>>> >>>>> +odp_buffer_t odp_buffer_extend(odp_buffer_t buf, size_t bytes) >>>>> +{ >>>>> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >>>>> + >>>>> + size_t lastseg = buf_hdr->size % buf_hdr->segsize; >>>>> + >>>>> + if (bytes <= buf_hdr->segsize - lastseg) { >>>>> + buf_hdr->size += bytes; >>>>> + return buf; >>>>> + } >>>>> + >>>>> + pool_entry_t *pool = odp_pool_to_entry(buf_hdr->pool_hdl); >>>>> + int extsize = buf_hdr->size + bytes; >>>>> + >>>>> + /* Extend buffer by adding additional segments to it */ >>>>> + if (extsize > ODP_CONFIG_BUF_MAX_SIZE || >>>>> pool->s.flags.unsegmented) >>>>> + return ODP_BUFFER_INVALID; >>>>> + >>>>> + size_t segcount = buf_hdr->segcount; >>>>> + size_t extsegs = extsize / buf_hdr->segsize; >>>>> + >>>>> + if (extsize % buf_hdr->segsize > 0) >>>>> + extsize++; >>>>> + >>>>> + while (extsegs > buf_hdr->segcount) { >>>>> + void *newblk = get_blk(&pool->s); >>>>> + >>>>> + /* If we fail to get all the blocks we need, back out >>>>> */ >>>>> + if (newblk == NULL) { >>>>> + while (segcount > buf_hdr->segcount) >>>>> + ret_blk(&pool->s, >>>>> buf_hdr->addr[--segcount]); >>>>> + return ODP_BUFFER_INVALID; >>>>> + } >>>>> + >>>>> + buf_hdr->addr[segcount++] = newblk; >>>>> + } >>>>> + >>>>> + buf_hdr->segcount = segcount; >>>>> + buf_hdr->size = extsize; >>>>> + >>>>> + return buf; >>>>> +} >>>>> + >>>>> +odp_buffer_t odp_buffer_clone(odp_buffer_t buf) >>>>> +{ >>>>> + return odp_buffer_copy(buf, odp_buf_to_hdr(buf)->pool_hdl); >>>>> +} >>>>> + >>>>> +odp_buffer_t odp_buffer_copy(odp_buffer_t buf, odp_buffer_pool_t pool) >>>>> +{ >>>>> + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); >>>>> + size_t len = buf_hdr->size; >>>>> + odp_buffer_t cpybuf = buffer_alloc(pool, len); >>>>> + >>>>> + if (cpybuf == ODP_BUFFER_INVALID) >>>>> + return ODP_BUFFER_INVALID; >>>>> + >>>>> + if (buffer_copy_to_buffer(cpybuf, 0, buf, 0, len) == 0) >>>>> + return cpybuf; >>>>> + >>>>> + odp_buffer_free(cpybuf); >>>>> + return ODP_BUFFER_INVALID; >>>>> +} >>>>> + >>>>> +int buffer_copy_to_buffer(odp_buffer_t dstbuf, size_t dstoffset, >>>>> + odp_buffer_t srcbuf, size_t srcoffset, >>>>> + size_t len) >>>>> +{ >>>>> + void *dstmap; >>>>> + void *srcmap; >>>>> + size_t cpylen, minseg, dstseglen, srcseglen; >>>>> + >>>>> + while (len > 0) { >>>>> + dstmap = odp_buffer_offset_map(dstbuf, dstoffset, >>>>> &dstseglen); >>>>> + srcmap = odp_buffer_offset_map(srcbuf, srcoffset, >>>>> &srcseglen); >>>>> + if (dstmap == NULL || srcmap == NULL) >>>>> + return -1; >>>>> + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; >>>>> + cpylen = len > minseg ? minseg : len; >>>>> + memcpy(dstmap, srcmap, cpylen); >>>>> + srcoffset += cpylen; >>>>> + dstoffset += cpylen; >>>>> + len -= cpylen; >>>>> + } >>>>> + >>>>> + return 0; >>>>> +} >>>>> + >>>>> +void *odp_buffer_addr(odp_buffer_t buf) >>>>> +{ >>>>> + return odp_buf_to_hdr(buf)->addr[0]; >>>>> +} >>>>> + >>>>> +odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf) >>>>> +{ >>>>> + return odp_buf_to_hdr(buf)->pool_hdl; >>>>> +} >>>>> + >>>>> +size_t odp_buffer_size(odp_buffer_t buf) >>>>> +{ >>>>> + return odp_buf_to_hdr(buf)->size; >>>>> +} >>>>> + >>>>> +odp_buffer_type_e odp_buffer_type(odp_buffer_t buf) >>>>> +{ >>>>> + return odp_buf_to_hdr(buf)->type; >>>>> +} >>>>> + >>>>> +void *odp_buffer_udata(odp_buffer_t buf, size_t *usize) >>>>> +{ >>>>> + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >>>>> + >>>>> + *usize = hdr->udata_size; >>>>> + return hdr->udata_addr; >>>>> +} >>>>> + >>>>> +void *odp_buffer_udata_addr(odp_buffer_t buf) >>>>> +{ >>>>> + return odp_buf_to_hdr(buf)->udata_addr; >>>>> +} >>>>> + >>>>> +int odp_buffer_is_valid(odp_buffer_t buf) >>>>> +{ >>>>> + return validate_buf(buf) != NULL; >>>>> +} >>>>> >>>>> int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf) >>>>> { >>>>> @@ -63,27 +276,13 @@ int odp_buffer_snprint(char *str, size_t n, >>>>> odp_buffer_t buf) >>>>> len += snprintf(&str[len], n-len, >>>>> " pool %i\n", hdr->pool_hdl); >>>>> len += snprintf(&str[len], n-len, >>>>> - " index %"PRIu32"\n", hdr->index); >>>>> - len += snprintf(&str[len], n-len, >>>>> - " phy_addr %"PRIu64"\n", hdr->phys_addr); >>>>> - len += snprintf(&str[len], n-len, >>>>> - " addr %p\n", hdr->addr); >>>>> + " addr %p\n", hdr->addr[0]); >>>>> len += snprintf(&str[len], n-len, >>>>> " size %zu\n", hdr->size); >>>>> len += snprintf(&str[len], n-len, >>>>> - " cur_offset %zu\n", hdr->cur_offset); >>>>> - len += snprintf(&str[len], n-len, >>>>> " ref_count %i\n", hdr->ref_count); >>>>> len += snprintf(&str[len], n-len, >>>>> " type %i\n", hdr->type); >>>>> - len += snprintf(&str[len], n-len, >>>>> - " Scatter list\n"); >>>>> - len += snprintf(&str[len], n-len, >>>>> - " num_bufs %i\n", >>>>> hdr->scatter.num_bufs); >>>>> - len += snprintf(&str[len], n-len, >>>>> - " pos %i\n", >>>>> hdr->scatter.pos); >>>>> - len += snprintf(&str[len], n-len, >>>>> - " total_len %zu\n", >>>>> hdr->scatter.total_len); >>>>> >>>>> return len; >>>>> } >>>>> @@ -100,9 +299,3 @@ void odp_buffer_print(odp_buffer_t buf) >>>>> >>>>> printf("\n%s\n", str); >>>>> } >>>>> - >>>>> -void odp_buffer_copy_scatter(odp_buffer_t buf_dst, odp_buffer_t >>>>> buf_src) >>>>> -{ >>>>> - (void)buf_dst; >>>>> - (void)buf_src; >>>>> -} >>>>> diff --git a/platform/linux-generic/odp_buffer_pool.c >>>>> b/platform/linux-generic/odp_buffer_pool.c >>>>> index a48d7d6..f6161bb 100644 >>>>> --- a/platform/linux-generic/odp_buffer_pool.c >>>>> +++ b/platform/linux-generic/odp_buffer_pool.c >>>>> @@ -6,8 +6,9 @@ >>>>> >>>>> #include <odp_std_types.h> >>>>> #include <odp_buffer_pool.h> >>>>> -#include <odp_buffer_pool_internal.h> >>>>> #include <odp_buffer_internal.h> >>>>> +#include <odp_buffer_pool_internal.h> >>>>> +#include <odp_buffer_inlines.h> >>>>> #include <odp_packet_internal.h> >>>>> #include <odp_timer_internal.h> >>>>> #include <odp_shared_memory.h> >>>>> @@ -16,6 +17,7 @@ >>>>> #include <odp_config.h> >>>>> #include <odp_hints.h> >>>>> #include <odp_debug.h> >>>>> +#include <odph_eth.h> >>>>> >>>>> #include <string.h> >>>>> #include <stdlib.h> >>>>> @@ -33,36 +35,26 @@ >>>>> #define LOCK_INIT(a) odp_spinlock_init(a) >>>>> #endif >>>>> >>>>> - >>>>> -#if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >>>>> -#error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >>>>> -#endif >>>>> - >>>>> -#define NULL_INDEX ((uint32_t)-1) >>>>> - >>>>> -union buffer_type_any_u { >>>>> +typedef union buffer_type_any_u { >>>>> odp_buffer_hdr_t buf; >>>>> odp_packet_hdr_t pkt; >>>>> odp_timeout_hdr_t tmo; >>>>> -}; >>>>> - >>>>> -ODP_STATIC_ASSERT((sizeof(union buffer_type_any_u) % 8) == 0, >>>>> - "BUFFER_TYPE_ANY_U__SIZE_ERR"); >>>>> +} odp_anybuf_t; >>>>> >>>>> /* Any buffer type header */ >>>>> typedef struct { >>>>> union buffer_type_any_u any_hdr; /* any buffer type */ >>>>> - uint8_t buf_data[]; /* start of buffer data >>>>> area */ >>>>> } odp_any_buffer_hdr_t; >>>>> >>>>> +typedef struct odp_any_hdr_stride { >>>>> + uint8_t >>>>> pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_any_buffer_hdr_t))]; >>>>> +} odp_any_hdr_stride; >>>>> >>>>> -typedef union pool_entry_u { >>>>> - struct pool_entry_s s; >>>>> - >>>>> - uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct >>>>> pool_entry_s))]; >>>>> - >>>>> -} pool_entry_t; >>>>> +#if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >>>>> +#error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >>>>> +#endif >>>>> >>>>> +#define NULL_INDEX ((uint32_t)-1) >>>>> >>>>> typedef struct pool_table_t { >>>>> pool_entry_t pool[ODP_CONFIG_BUFFER_POOLS]; >>>>> @@ -76,39 +68,6 @@ static pool_table_t *pool_tbl; >>>>> /* Pool entry pointers (for inlining) */ >>>>> void *pool_entry_ptr[ODP_CONFIG_BUFFER_POOLS]; >>>>> >>>>> - >>>>> -static __thread odp_buffer_chunk_hdr_t >>>>> *local_chunk[ODP_CONFIG_BUFFER_POOLS]; >>>>> - >>>>> - >>>>> -static inline odp_buffer_pool_t pool_index_to_handle(uint32_t pool_id) >>>>> -{ >>>>> - return pool_id + 1; >>>>> -} >>>>> - >>>>> - >>>>> -static inline uint32_t pool_handle_to_index(odp_buffer_pool_t >>>>> pool_hdl) >>>>> -{ >>>>> - return pool_hdl -1; >>>>> -} >>>>> - >>>>> - >>>>> -static inline void set_handle(odp_buffer_hdr_t *hdr, >>>>> - pool_entry_t *pool, uint32_t index) >>>>> -{ >>>>> - odp_buffer_pool_t pool_hdl = pool->s.pool_hdl; >>>>> - uint32_t pool_id = pool_handle_to_index(pool_hdl); >>>>> - >>>>> - if (pool_id >= ODP_CONFIG_BUFFER_POOLS) >>>>> - ODP_ABORT("set_handle: Bad pool handle %u\n", >>>>> pool_hdl); >>>>> - >>>>> - if (index > ODP_BUFFER_MAX_INDEX) >>>>> - ODP_ERR("set_handle: Bad buffer index\n"); >>>>> - >>>>> - hdr->handle.pool_id = pool_id; >>>>> - hdr->handle.index = index; >>>>> -} >>>>> - >>>>> - >>>>> int odp_buffer_pool_init_global(void) >>>>> { >>>>> uint32_t i; >>>>> @@ -142,269 +101,167 @@ int odp_buffer_pool_init_global(void) >>>>> return 0; >>>>> } >>>>> >>>>> - >>>>> -static odp_buffer_hdr_t *index_to_hdr(pool_entry_t *pool, uint32_t >>>>> index) >>>>> -{ >>>>> - odp_buffer_hdr_t *hdr; >>>>> - >>>>> - hdr = (odp_buffer_hdr_t *)(pool->s.buf_base + index * >>>>> pool->s.buf_size); >>>>> - return hdr; >>>>> -} >>>>> - >>>>> - >>>>> -static void add_buf_index(odp_buffer_chunk_hdr_t *chunk_hdr, uint32_t >>>>> index) >>>>> -{ >>>>> - uint32_t i = chunk_hdr->chunk.num_bufs; >>>>> - chunk_hdr->chunk.buf_index[i] = index; >>>>> - chunk_hdr->chunk.num_bufs++; >>>>> -} >>>>> - >>>>> - >>>>> -static uint32_t rem_buf_index(odp_buffer_chunk_hdr_t *chunk_hdr) >>>>> +/** >>>>> + * Buffer pool creation >>>>> + */ >>>>> +odp_buffer_pool_t odp_buffer_pool_create(const char *name, >>>>> + odp_buffer_pool_param_t *args, >>>>> + odp_buffer_pool_init_t >>>>> *init_args) >>>>> { >>>>> - uint32_t index; >>>>> + odp_buffer_pool_t pool_hdl = ODP_BUFFER_POOL_INVALID; >>>>> + pool_entry_t *pool; >>>>> uint32_t i; >>>>> + odp_buffer_pool_param_t *params; >>>>> + odp_buffer_pool_init_t *init_params; >>>>> >>>>> - i = chunk_hdr->chunk.num_bufs - 1; >>>>> - index = chunk_hdr->chunk.buf_index[i]; >>>>> - chunk_hdr->chunk.num_bufs--; >>>>> - return index; >>>>> -} >>>>> - >>>>> - >>>>> -static odp_buffer_chunk_hdr_t *next_chunk(pool_entry_t *pool, >>>>> - odp_buffer_chunk_hdr_t >>>>> *chunk_hdr) >>>>> -{ >>>>> - uint32_t index; >>>>> - >>>>> - index = chunk_hdr->chunk.buf_index[ODP_BUFS_PER_CHUNK-1]; >>>>> - if (index == NULL_INDEX) >>>>> - return NULL; >>>>> - else >>>>> - return (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, >>>>> index); >>>>> -} >>>>> - >>>>> - >>>>> -static odp_buffer_chunk_hdr_t *rem_chunk(pool_entry_t *pool) >>>>> -{ >>>>> - odp_buffer_chunk_hdr_t *chunk_hdr; >>>>> - >>>>> - chunk_hdr = pool->s.head; >>>>> - if (chunk_hdr == NULL) { >>>>> - /* Pool is empty */ >>>>> - return NULL; >>>>> - } >>>>> - >>>>> - pool->s.head = next_chunk(pool, chunk_hdr); >>>>> - pool->s.free_bufs -= ODP_BUFS_PER_CHUNK; >>>>> - >>>>> - /* unlink */ >>>>> - rem_buf_index(chunk_hdr); >>>>> - return chunk_hdr; >>>>> -} >>>>> - >>>>> - >>>>> -static void add_chunk(pool_entry_t *pool, odp_buffer_chunk_hdr_t >>>>> *chunk_hdr) >>>>> -{ >>>>> - if (pool->s.head) /* link pool head to the chunk */ >>>>> - add_buf_index(chunk_hdr, pool->s.head->buf_hdr.index); >>>>> - else >>>>> - add_buf_index(chunk_hdr, NULL_INDEX); >>>>> - >>>>> - pool->s.head = chunk_hdr; >>>>> - pool->s.free_bufs += ODP_BUFS_PER_CHUNK; >>>>> -} >>>>> + /* Default pool creation arguments */ >>>>> + static odp_buffer_pool_param_t default_params = { >>>>> + .buf_num = 1024, >>>>> + .buf_size = ODP_CONFIG_BUF_SEG_SIZE, >>>>> + .buf_type = ODP_BUFFER_TYPE_PACKET, >>>>> + .buf_opts = ODP_BUFFER_OPTS_UNSEGMENTED, >>>>> + }; >>>>> >>>> Do we need to have this behaviour of allocating default params if the >>>> input params are NULL. >>>> It would be better if the API returns error. >>>> >>>>> >>>>> + /* Default initialization paramters */ >>>>> + static odp_buffer_pool_init_t default_init_params = { >>>>> + .udata_size = 0, >>>>> + .buf_init = NULL, >>>>> + .buf_init_arg = NULL, >>>>> + }; >>>>> >>>>> -static void check_align(pool_entry_t *pool, odp_buffer_hdr_t *hdr) >>>>> -{ >>>>> - if (!ODP_ALIGNED_CHECK_POWER_2(hdr->addr, pool->s.user_align)) >>>>> { >>>>> - ODP_ABORT("check_align: user data align error %p, >>>>> align %zu\n", >>>>> - hdr->addr, pool->s.user_align); >>>>> - } >>>>> + /* Handle NULL input parameters */ >>>>> + params = args == NULL ? &default_params : args; >>>>> + init_params = init_args == NULL ? &default_init_params >>>>> + : init_args; >>>>> >>>>> - if (!ODP_ALIGNED_CHECK_POWER_2(hdr, ODP_CACHE_LINE_SIZE)) { >>>>> - ODP_ABORT("check_align: hdr align error %p, align >>>>> %i\n", >>>>> - hdr, ODP_CACHE_LINE_SIZE); >>>>> - } >>>>> -} >>>>> + if (params->buf_type != ODP_BUFFER_TYPE_PACKET) >>>>> + params->buf_opts |= ODP_BUFFER_OPTS_UNSEGMENTED; >>>>> >>>>> + int unsegmented = ((params->buf_opts & >>>>> ODP_BUFFER_OPTS_UNSEGMENTED) == >>>>> + ODP_BUFFER_OPTS_UNSEGMENTED); >>>>> >>>>> -static void fill_hdr(void *ptr, pool_entry_t *pool, uint32_t index, >>>>> - int buf_type) >>>>> -{ >>>>> - odp_buffer_hdr_t *hdr = (odp_buffer_hdr_t *)ptr; >>>>> - size_t size = pool->s.hdr_size; >>>>> - uint8_t *buf_data; >>>>> + uint32_t udata_stride = >>>>> + ODP_CACHE_LINE_SIZE_ROUNDUP(init_params->udata_size); >>>>> >>>>> - if (buf_type == ODP_BUFFER_TYPE_CHUNK) >>>>> - size = sizeof(odp_buffer_chunk_hdr_t); >>>>> - >>>>> - switch (pool->s.buf_type) { >>>>> - odp_raw_buffer_hdr_t *raw_hdr; >>>>> - odp_packet_hdr_t *packet_hdr; >>>>> - odp_timeout_hdr_t *tmo_hdr; >>>>> - odp_any_buffer_hdr_t *any_hdr; >>>>> + uint32_t blk_size, buf_stride; >>>>> >>>>> + switch (params->buf_type) { >>>>> case ODP_BUFFER_TYPE_RAW: >>>>> - raw_hdr = ptr; >>>>> - buf_data = raw_hdr->buf_data; >>>>> + blk_size = params->buf_size; >>>>> + buf_stride = sizeof(odp_buffer_hdr_stride); >>>>> break; >>>>> + >>>>> case ODP_BUFFER_TYPE_PACKET: >>>>> - packet_hdr = ptr; >>>>> - buf_data = packet_hdr->buf_data; >>>>> + if (unsegmented) >>>>> + blk_size = >>>>> + >>>>> ODP_CACHE_LINE_SIZE_ROUNDUP(params->buf_size); >>>>> + else >>>>> + blk_size = ODP_ALIGN_ROUNDUP(params->buf_size, >>>>> + >>>>> ODP_CONFIG_BUF_SEG_SIZE); >>>>> + buf_stride = sizeof(odp_packet_hdr_stride); >>>>> break; >>>>> + >>>>> case ODP_BUFFER_TYPE_TIMEOUT: >>>>> - tmo_hdr = ptr; >>>>> - buf_data = tmo_hdr->buf_data; >>>>> + blk_size = 0; /* Timeouts have no block data, only >>>>> metadata */ >>>>> + buf_stride = sizeof(odp_timeout_hdr_stride); >>>>> break; >>>>> + >>>>> case ODP_BUFFER_TYPE_ANY: >>>>> - any_hdr = ptr; >>>>> - buf_data = any_hdr->buf_data; >>>>> + if (unsegmented) >>>>> + blk_size = >>>>> + >>>>> ODP_CACHE_LINE_SIZE_ROUNDUP(params->buf_size); >>>>> + else >>>>> + blk_size = ODP_ALIGN_ROUNDUP(params->buf_size, >>>>> + >>>>> ODP_CONFIG_BUF_SEG_SIZE); >>>>> + buf_stride = sizeof(odp_any_hdr_stride); >>>>> break; >>>>> - default: >>>>> - ODP_ABORT("Bad buffer type\n"); >>>>> - } >>>>> - >>>>> - memset(hdr, 0, size); >>>>> - >>>>> - set_handle(hdr, pool, index); >>>>> - >>>>> - hdr->addr = &buf_data[pool->s.buf_offset - >>>>> pool->s.hdr_size]; >>>>> - hdr->index = index; >>>>> - hdr->size = pool->s.user_size; >>>>> - hdr->pool_hdl = pool->s.pool_hdl; >>>>> - hdr->type = buf_type; >>>>> - >>>>> - check_align(pool, hdr); >>>>> -} >>>>> - >>>>> - >>>>> -static void link_bufs(pool_entry_t *pool) >>>>> -{ >>>>> - odp_buffer_chunk_hdr_t *chunk_hdr; >>>>> - size_t hdr_size; >>>>> - size_t data_size; >>>>> - size_t data_align; >>>>> - size_t tot_size; >>>>> - size_t offset; >>>>> - size_t min_size; >>>>> - uint64_t pool_size; >>>>> - uintptr_t buf_base; >>>>> - uint32_t index; >>>>> - uintptr_t pool_base; >>>>> - int buf_type; >>>>> - >>>>> - buf_type = pool->s.buf_type; >>>>> - data_size = pool->s.user_size; >>>>> - data_align = pool->s.user_align; >>>>> - pool_size = pool->s.pool_size; >>>>> - pool_base = (uintptr_t) pool->s.pool_base_addr; >>>>> - >>>>> - if (buf_type == ODP_BUFFER_TYPE_RAW) { >>>>> - hdr_size = sizeof(odp_raw_buffer_hdr_t); >>>>> - } else if (buf_type == ODP_BUFFER_TYPE_PACKET) { >>>>> - hdr_size = sizeof(odp_packet_hdr_t); >>>>> - } else if (buf_type == ODP_BUFFER_TYPE_TIMEOUT) { >>>>> - hdr_size = sizeof(odp_timeout_hdr_t); >>>>> - } else if (buf_type == ODP_BUFFER_TYPE_ANY) { >>>>> - hdr_size = sizeof(odp_any_buffer_hdr_t); >>>>> - } else >>>>> - ODP_ABORT("odp_buffer_pool_create: Bad type %i\n", >>>>> buf_type); >>>>> - >>>>> - >>>>> - /* Chunk must fit into buffer data area.*/ >>>>> - min_size = sizeof(odp_buffer_chunk_hdr_t) - hdr_size; >>>>> - if (data_size < min_size) >>>>> - data_size = min_size; >>>>> - >>>>> - /* Roundup data size to full cachelines */ >>>>> - data_size = ODP_CACHE_LINE_SIZE_ROUNDUP(data_size); >>>>> - >>>>> - /* Min cacheline alignment for buffer header and data */ >>>>> - data_align = ODP_CACHE_LINE_SIZE_ROUNDUP(data_align); >>>>> - offset = ODP_CACHE_LINE_SIZE_ROUNDUP(hdr_size); >>>>> - >>>>> - /* Multiples of cacheline size */ >>>>> - if (data_size > data_align) >>>>> - tot_size = data_size + offset; >>>>> - else >>>>> - tot_size = data_align + offset; >>>>> - >>>>> - /* First buffer */ >>>>> - buf_base = ODP_ALIGN_ROUNDUP(pool_base + offset, data_align) - >>>>> offset; >>>>> - >>>>> - pool->s.hdr_size = hdr_size; >>>>> - pool->s.buf_base = buf_base; >>>>> - pool->s.buf_size = tot_size; >>>>> - pool->s.buf_offset = offset; >>>>> - index = 0; >>>>> - >>>>> - chunk_hdr = (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, >>>>> index); >>>>> - pool->s.head = NULL; >>>>> - pool_size -= buf_base - pool_base; >>>>> - >>>>> - while (pool_size > ODP_BUFS_PER_CHUNK * tot_size) { >>>>> - int i; >>>>> - >>>>> - fill_hdr(chunk_hdr, pool, index, >>>>> ODP_BUFFER_TYPE_CHUNK); >>>>> - >>>>> - index++; >>>>> - >>>>> - for (i = 0; i < ODP_BUFS_PER_CHUNK - 1; i++) { >>>>> - odp_buffer_hdr_t *hdr = index_to_hdr(pool, >>>>> index); >>>>> - >>>>> - fill_hdr(hdr, pool, index, buf_type); >>>>> - >>>>> - add_buf_index(chunk_hdr, index); >>>>> - index++; >>>>> - } >>>>> - >>>>> - add_chunk(pool, chunk_hdr); >>>>> >>>>> - chunk_hdr = (odp_buffer_chunk_hdr_t >>>>> *)index_to_hdr(pool, >>>>> - >>>>> index); >>>>> - pool->s.num_bufs += ODP_BUFS_PER_CHUNK; >>>>> - pool_size -= ODP_BUFS_PER_CHUNK * tot_size; >>>>> + default: >>>>> + return ODP_BUFFER_POOL_INVALID; >>>>> } >>>>> -} >>>>> - >>>>> - >>>>> -odp_buffer_pool_t odp_buffer_pool_create(const char *name, >>>>> - void *base_addr, uint64_t >>>>> size, >>>>> - size_t buf_size, size_t >>>>> buf_align, >>>>> - int buf_type) >>>>> -{ >>>>> - odp_buffer_pool_t pool_hdl = ODP_BUFFER_POOL_INVALID; >>>>> - pool_entry_t *pool; >>>>> - uint32_t i; >>>>> >>>>> for (i = 0; i < ODP_CONFIG_BUFFER_POOLS; i++) { >>>>> pool = get_pool_entry(i); >>>>> >>>>> LOCK(&pool->s.lock); >>>>> + if (pool->s.shm != ODP_SHM_INVALID) { >>>>> + UNLOCK(&pool->s.lock); >>>>> + continue; >>>>> + } >>>>> >>>>> - if (pool->s.buf_base == 0) { >>>>> - /* found free pool */ >>>>> + /* found free pool */ >>>>> + size_t block_size, mdata_size, udata_size; >>>>> >>>>> - strncpy(pool->s.name, name, >>>>> - ODP_BUFFER_POOL_NAME_LEN - 1); >>>>> - pool->s.name[ODP_BUFFER_POOL_NAME_LEN - 1] = >>>>> 0; >>>>> - pool->s.pool_base_addr = base_addr; >>>>> - pool->s.pool_size = size; >>>>> - pool->s.user_size = buf_size; >>>>> - pool->s.user_align = buf_align; >>>>> - pool->s.buf_type = buf_type; >>>>> + strncpy(pool->s.name, name, >>>>> + ODP_BUFFER_POOL_NAME_LEN - 1); >>>>> + pool->s.name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; >>>>> >>>>> - link_bufs(pool); >>>>> + pool->s.params = *params; >>>>> + pool->s.init_params = *init_params; >>>>> >>>>> - UNLOCK(&pool->s.lock); >>>>> + mdata_size = params->buf_num * buf_stride; >>>>> + udata_size = params->buf_num * udata_stride; >>>>> + block_size = params->buf_num * blk_size; >>>>> + >>>>> + pool->s.pool_size = ODP_PAGE_SIZE_ROUNDUP(mdata_size + >>>>> + udata_size + >>>>> + block_size); >>>>> >>>>> - pool_hdl = pool->s.pool_hdl; >>>>> - break; >>>>> + pool->s.shm = odp_shm_reserve(pool->s.name, >>>>> pool->s.pool_size, >>>>> + ODP_PAGE_SIZE, 0); >>>>> + if (pool->s.shm == ODP_SHM_INVALID) { >>>>> + UNLOCK(&pool->s.lock); >>>>> + return ODP_BUFFER_POOL_INVALID; >>>>> } >>>>> >>>>> + pool->s.pool_base_addr = (uint8_t >>>>> *)odp_shm_addr(pool->s.shm); >>>>> + pool->s.flags.unsegmented = unsegmented; >>>>> + pool->s.seg_size = unsegmented ? >>>>> + blk_size : ODP_CONFIG_BUF_SEG_SIZE; >>>>> + uint8_t *udata_base_addr = pool->s.pool_base_addr + >>>>> mdata_size; >>>>> + uint8_t *block_base_addr = udata_base_addr + >>>>> udata_size; >>>>> + >>>>> + pool->s.bufcount = 0; >>>>> + pool->s.buf_freelist = NULL; >>>>> + pool->s.blk_freelist = NULL; >>>>> + >>>>> + uint8_t *buf = pool->s.udata_base_addr - buf_stride; >>>>> + uint8_t *udat = (udata_stride == 0) ? NULL : >>>>> + block_base_addr - udata_stride; >>>>> + >>>>> + /* Init buffer common header and add to pool buffer >>>>> freelist */ >>>>> + do { >>>>> + odp_buffer_hdr_t *tmp = >>>>> + (odp_buffer_hdr_t *)(void *)buf; >>>>> + tmp->pool_hdl = pool->s.pool_hdl; >>>>> + tmp->size = 0; >>>>> + tmp->type = params->buf_type; >>>>> + tmp->udata_addr = (void *)udat; >>>>> + tmp->udata_size = init_params->udata_size; >>>>> + tmp->segcount = 0; >>>>> + tmp->segsize = pool->s.seg_size; >>>>> + tmp->buf_hdl.handle = >>>>> + >>>>> odp_buffer_encode_handle((odp_buffer_hdr_t *) >>>>> + (void *)buf); >>>>> + ret_buf(&pool->s, tmp); >>>>> + buf -= buf_stride; >>>>> + udat -= udata_stride; >>>>> + } while (buf >= pool->s.pool_base_addr); >>>>> + >>>>> + /* Form block freelist for pool */ >>>>> + uint8_t *blk = pool->s.pool_base_addr + >>>>> pool->s.pool_size - >>>>> + pool->s.seg_size; >>>>> + >>>>> + if (blk_size > 0) >>>>> + do { >>>>> + ret_blk(&pool->s, blk); >>>>> + blk -= pool->s.seg_size; >>>>> + } while (blk >= block_base_addr); >>>>> + >>>>> UNLOCK(&pool->s.lock); >>>>> + >>>>> + pool_hdl = pool->s.pool_hdl; >>>>> + break; >>>>> } >>>>> >>>>> return pool_hdl; >>>>> @@ -431,93 +288,172 @@ odp_buffer_pool_t odp_buffer_pool_lookup(const >>>>> char *name) >>>>> return ODP_BUFFER_POOL_INVALID; >>>>> } >>>>> >>>>> - >>>>> -odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_hdl) >>>>> +odp_buffer_pool_t odp_buffer_pool_next(odp_buffer_pool_t pool_hdl, >>>>> + char *name, >>>>> + size_t *udata_size, >>>>> + odp_buffer_pool_param_t *params, >>>>> + int *predef) >>>>> { >>>>> - pool_entry_t *pool; >>>>> - odp_buffer_chunk_hdr_t *chunk; >>>>> - odp_buffer_bits_t handle; >>>>> - uint32_t pool_id = pool_handle_to_index(pool_hdl); >>>>> - >>>>> - pool = get_pool_entry(pool_id); >>>>> - chunk = local_chunk[pool_id]; >>>>> - >>>>> - if (chunk == NULL) { >>>>> - LOCK(&pool->s.lock); >>>>> - chunk = rem_chunk(pool); >>>>> - UNLOCK(&pool->s.lock); >>>>> + pool_entry_t *next_pool; >>>>> + uint32_t pool_id; >>>>> >>>>> - if (chunk == NULL) >>>>> - return ODP_BUFFER_INVALID; >>>>> + /* NULL input means first element */ >>>>> + if (pool_hdl == ODP_BUFFER_POOL_INVALID) { >>>>> + pool_id = 0; >>>>> + next_pool = get_pool_entry(pool_id); >>>>> + } else { >>>>> + pool_id = pool_handle_to_index(pool_hdl); >>>>> >>>>> - local_chunk[pool_id] = chunk; >>>>> + if (pool_id == ODP_CONFIG_BUFFER_POOLS) >>>>> + return ODP_BUFFER_POOL_INVALID; >>>>> + else >>>>> + next_pool = get_pool_entry(++pool_id); >>>>> } >>>>> >>>>> - if (chunk->chunk.num_bufs == 0) { >>>>> - /* give the chunk buffer */ >>>>> - local_chunk[pool_id] = NULL; >>>>> - chunk->buf_hdr.type = pool->s.buf_type; >>>>> + /* Only interested in pools that exist */ >>>>> + while (next_pool->s.shm == ODP_SHM_INVALID) { >>>>> + if (pool_id == ODP_CONFIG_BUFFER_POOLS) >>>>> + return ODP_BUFFER_POOL_INVALID; >>>>> + else >>>>> + next_pool = get_pool_entry(++pool_id); >>>>> + } >>>>> >>>>> - handle = chunk->buf_hdr.handle; >>>>> - } else { >>>>> - odp_buffer_hdr_t *hdr; >>>>> - uint32_t index; >>>>> - index = rem_buf_index(chunk); >>>>> - hdr = index_to_hdr(pool, index); >>>>> + /* Found the next pool, so return info about it */ >>>>> + strncpy(name, next_pool->s.name, ODP_BUFFER_POOL_NAME_LEN - >>>>> 1); >>>>> + name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; >>>>> >>>>> - handle = hdr->handle; >>>>> - } >>>>> + *udata_size = next_pool->s.init_params.udata_size; >>>>> + *params = next_pool->s.params; >>>>> + *predef = next_pool->s.flags.predefined; >>>>> >>>>> - return handle.u32; >>>>> + return next_pool->s.pool_hdl; >>>>> } >>>>> >>>>> - >>>>> -void odp_buffer_free(odp_buffer_t buf) >>>>> +int odp_buffer_pool_destroy(odp_buffer_pool_t pool_hdl) >>>>> { >>>>> - odp_buffer_hdr_t *hdr; >>>>> - uint32_t pool_id; >>>>> - pool_entry_t *pool; >>>>> - odp_buffer_chunk_hdr_t *chunk_hdr; >>>>> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >>>>> >>>>> - hdr = odp_buf_to_hdr(buf); >>>>> - pool_id = pool_handle_to_index(hdr->pool_hdl); >>>>> - pool = get_pool_entry(pool_id); >>>>> - chunk_hdr = local_chunk[pool_id]; >>>>> + if (pool == NULL) >>>>> + return -1; >>>>> >>>>> - if (chunk_hdr && chunk_hdr->chunk.num_bufs == >>>>> ODP_BUFS_PER_CHUNK - 1) { >>>>> - /* Current chunk is full. Push back to the pool */ >>>>> - LOCK(&pool->s.lock); >>>>> - add_chunk(pool, chunk_hdr); >>>>> + LOCK(&pool->s.lock); >>>>> + >>>>> + if (pool->s.shm == ODP_SHM_INVALID || >>>>> + pool->s.bufcount > 0 || >>>>> + pool->s.flags.predefined) { >>>>> UNLOCK(&pool->s.lock); >>>>> - chunk_hdr = NULL; >>>>> + return -1; >>>>> } >>>>> >>>>> - if (chunk_hdr == NULL) { >>>>> - /* Use this buffer */ >>>>> - chunk_hdr = (odp_buffer_chunk_hdr_t *)hdr; >>>>> - local_chunk[pool_id] = chunk_hdr; >>>>> - chunk_hdr->chunk.num_bufs = 0; >>>>> + odp_shm_free(pool->s.shm); >>>>> + >>>>> + pool->s.shm = ODP_SHM_INVALID; >>>>> + UNLOCK(&pool->s.lock); >>>>> + >>>>> + return 0; >>>>> +} >>>>> + >>>>> +size_t odp_buffer_pool_headroom(odp_buffer_pool_t pool_hdl) >>>>> +{ >>>>> + return odp_pool_to_entry(pool_hdl)->s.headroom; >>>>> +} >>>>> + >>>>> +int odp_buffer_pool_set_headroom(odp_buffer_pool_t pool_hdl, size_t >>>>> hr) >>>>> +{ >>>>> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >>>>> + >>>>> + if (hr >= pool->s.seg_size/2) { >>>>> + return -1; >>>>> } else { >>>>> - /* Add to current chunk */ >>>>> - add_buf_index(chunk_hdr, hdr->index); >>>>> + pool->s.headroom = hr; >>>>> + return 0; >>>>> } >>>>> } >>>>> >>>>> +size_t odp_buffer_pool_tailroom(odp_buffer_pool_t pool_hdl) >>>>> +{ >>>>> + return odp_pool_to_entry(pool_hdl)->s.tailroom; >>>>> +} >>>>> >>>>> -odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf) >>>>> +int odp_buffer_pool_set_tailroom(odp_buffer_pool_t pool_hdl, size_t >>>>> tr) >>>>> { >>>>> - odp_buffer_hdr_t *hdr; >>>>> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >>>>> >>>>> - hdr = odp_buf_to_hdr(buf); >>>>> - return hdr->pool_hdl; >>>>> + if (tr >= pool->s.seg_size/2) { >>>>> + return -1; >>>>> + } else { >>>>> + pool->s.tailroom = tr; >>>>> + return 0; >>>>> + } >>>>> } >>>>> >>>>> +odp_buffer_t buffer_alloc(odp_buffer_pool_t pool_hdl, size_t size) >>>>> +{ >>>>> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >>>>> + size_t totsize = pool->s.headroom + size + pool->s.tailroom; >>>>> + odp_anybuf_t *buf; >>>>> + uint8_t *blk; >>>>> + >>>>> + if ((pool->s.flags.unsegmented && totsize > pool->s.seg_size) >>>>> || >>>>> + (!pool->s.flags.unsegmented && totsize > >>>>> ODP_CONFIG_BUF_MAX_SIZE)) >>>>> + return ODP_BUFFER_INVALID; >>>>> + >>>>> + buf = (odp_anybuf_t *)(void *)get_buf(&pool->s); >>>>> + >>>>> + if (buf == NULL) >>>>> + return ODP_BUFFER_INVALID; >>>>> + >>>>> + /* Get blocks for this buffer, if pool uses application data */ >>>>> + if (buf->buf.segsize > 0) >>>>> + do { >>>>> + blk = get_blk(&pool->s); >>>>> + if (blk == NULL) { >>>>> + ret_buf(&pool->s, &buf->buf); >>>>> + return ODP_BUFFER_INVALID; >>>>> + } >>>>> + buf->buf.addr[buf->buf.segcount++] = blk; >>>>> + totsize = totsize < pool->s.seg_size ? 0 : >>>>> + totsize - pool->s.seg_size; >>>>> + } while (totsize > 0); >>>>> + >>>>> + switch (buf->buf.type) { >>>>> + case ODP_BUFFER_TYPE_RAW: >>>>> + break; >>>>> + >>>>> + case ODP_BUFFER_TYPE_PACKET: >>>>> + packet_init(pool, &buf->pkt, size); >>>>> + if (pool->s.init_params.buf_init != NULL) >>>>> + (*pool->s.init_params.buf_init) >>>>> + (buf->buf.buf_hdl.handle, >>>>> + pool->s.init_params.buf_init_arg); >>>>> + break; >>>>> + >>>>> + case ODP_BUFFER_TYPE_TIMEOUT: >>>>> + break; >>>>> + >>>>> + default: >>>>> + ret_buf(&pool->s, &buf->buf); >>>>> + return ODP_BUFFER_INVALID; >>>>> + } >>>>> + >>>>> + return odp_hdr_to_buf(&buf->buf); >>>>> +} >>>>> + >>>>> +odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_hdl) >>>>> +{ >>>>> + return buffer_alloc(pool_hdl, 0); >>>>> +} >>>>> + >>>>> +void odp_buffer_free(odp_buffer_t buf) >>>>> +{ >>>>> + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); >>>>> + pool_entry_t *pool = odp_buf_to_pool(hdr); >>>>> + ret_buf(&pool->s, hdr); >>>>> +} >>>>> >>>>> void odp_buffer_pool_print(odp_buffer_pool_t pool_hdl) >>>>> { >>>>> pool_entry_t *pool; >>>>> - odp_buffer_chunk_hdr_t *chunk_hdr; >>>>> - uint32_t i; >>>>> uint32_t pool_id; >>>>> >>>>> pool_id = pool_handle_to_index(pool_hdl); >>>>> @@ -528,47 +464,4 @@ void odp_buffer_pool_print(odp_buffer_pool_t >>>>> pool_hdl) >>>>> printf(" pool %i\n", pool->s.pool_hdl); >>>>> printf(" name %s\n", pool->s.name); >>>>> printf(" pool base %p\n", >>>>> pool->s.pool_base_addr); >>>>> - printf(" buf base 0x%"PRIxPTR"\n", pool->s.buf_base); >>>>> - printf(" pool size 0x%"PRIx64"\n", pool->s.pool_size); >>>>> - printf(" buf size %zu\n", pool->s.user_size); >>>>> - printf(" buf align %zu\n", pool->s.user_align); >>>>> - printf(" hdr size %zu\n", pool->s.hdr_size); >>>>> - printf(" alloc size %zu\n", pool->s.buf_size); >>>>> - printf(" offset to hdr %zu\n", pool->s.buf_offset); >>>>> - printf(" num bufs %"PRIu64"\n", pool->s.num_bufs); >>>>> - printf(" free bufs %"PRIu64"\n", pool->s.free_bufs); >>>>> - >>>>> - /* first chunk */ >>>>> - chunk_hdr = pool->s.head; >>>>> - >>>>> - if (chunk_hdr == NULL) { >>>>> - ODP_ERR(" POOL EMPTY\n"); >>>>> - return; >>>>> - } >>>>> - >>>>> - printf("\n First chunk\n"); >>>>> - >>>>> - for (i = 0; i < chunk_hdr->chunk.num_bufs - 1; i++) { >>>>> - uint32_t index; >>>>> - odp_buffer_hdr_t *hdr; >>>>> - >>>>> - index = chunk_hdr->chunk.buf_index[i]; >>>>> - hdr = index_to_hdr(pool, index); >>>>> - >>>>> - printf(" [%i] addr %p, id %"PRIu32"\n", i, hdr->addr, >>>>> index); >>>>> - } >>>>> - >>>>> - printf(" [%i] addr %p, id %"PRIu32"\n", i, >>>>> chunk_hdr->buf_hdr.addr, >>>>> - chunk_hdr->buf_hdr.index); >>>>> - >>>>> - /* next chunk */ >>>>> - chunk_hdr = next_chunk(pool, chunk_hdr); >>>>> - >>>>> - if (chunk_hdr) { >>>>> - printf(" Next chunk\n"); >>>>> - printf(" addr %p, id %"PRIu32"\n", >>>>> chunk_hdr->buf_hdr.addr, >>>>> - chunk_hdr->buf_hdr.index); >>>>> - } >>>>> - >>>>> - printf("\n"); >>>>> } >>>>> diff --git a/platform/linux-generic/odp_packet.c >>>>> b/platform/linux-generic/odp_packet.c >>>>> index 82ea879..45ac8e5 100644 >>>>> --- a/platform/linux-generic/odp_packet.c >>>>> +++ b/platform/linux-generic/odp_packet.c >>>>> @@ -11,29 +11,31 @@ >>>>> >>>>> #include <odph_eth.h> >>>>> #include <odph_ip.h> >>>>> +#include <odph_tcp.h> >>>>> +#include <odph_udp.h> >>>>> >>>>> #include <string.h> >>>>> #include <stdio.h> >>>>> >>>>> -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >>>>> - odph_ipv4hdr_t *ipv4, size_t >>>>> *offset_out); >>>>> -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >>>>> - odph_ipv6hdr_t *ipv6, size_t >>>>> *offset_out); >>>>> - >>>>> void odp_packet_init(odp_packet_t pkt) >>>>> { >>>>> odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >>>>> + pool_entry_t *pool = odp_buf_to_pool(&pkt_hdr->buf_hdr); >>>>> const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, >>>>> buf_hdr); >>>>> uint8_t *start; >>>>> size_t len; >>>>> >>>>> start = (uint8_t *)pkt_hdr + start_offset; >>>>> - len = ODP_OFFSETOF(odp_packet_hdr_t, buf_data) - start_offset; >>>>> + len = sizeof(odp_packet_hdr_t) - start_offset; >>>>> memset(start, 0, len); >>>>> >>>>> pkt_hdr->l2_offset = ODP_PACKET_OFFSET_INVALID; >>>>> pkt_hdr->l3_offset = ODP_PACKET_OFFSET_INVALID; >>>>> pkt_hdr->l4_offset = ODP_PACKET_OFFSET_INVALID; >>>>> + pkt_hdr->payload_offset = ODP_PACKET_OFFSET_INVALID; >>>>> + >>>>> + pkt_hdr->headroom = pool->s.headroom; >>>>> + pkt_hdr->tailroom = pool->s.tailroom; >>>>> } >>>>> >>>>> odp_packet_t odp_packet_from_buffer(odp_buffer_t buf) >>>>> @@ -46,55 +48,112 @@ odp_buffer_t odp_packet_to_buffer(odp_packet_t >>>>> pkt) >>>>> return (odp_buffer_t)pkt; >>>>> } >>>>> >>>>> -void odp_packet_set_len(odp_packet_t pkt, size_t len) >>>>> +size_t odp_packet_len(odp_packet_t pkt) >>>>> { >>>>> - odp_packet_hdr(pkt)->frame_len = len; >>>>> + return odp_packet_hdr(pkt)->frame_len; >>>>> } >>>>> >>>>> -size_t odp_packet_get_len(odp_packet_t pkt) >>>>> +void *odp_packet_offset_map(odp_packet_t pkt, size_t offset, size_t >>>>> *seglen) >>>>> { >>>>> - return odp_packet_hdr(pkt)->frame_len; >>>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>>> + >>>>> + if (offset >= pkt_hdr->frame_len) >>>>> + return NULL; >>>>> + >>>>> + return buffer_map(&pkt_hdr->buf_hdr, >>>>> + pkt_hdr->headroom + offset, >>>>> + seglen, pkt_hdr->frame_len); >>>>> } >>>>> >>>>> -uint8_t *odp_packet_addr(odp_packet_t pkt) >>>>> +void odp_packet_offset_unmap(odp_packet_t pkt ODP_UNUSED, >>>>> + size_t offset ODP_UNUSED) >>>>> { >>>>> - return odp_buffer_addr(odp_packet_to_buffer(pkt)); >>>>> } >>>>> >>>>> -uint8_t *odp_packet_data(odp_packet_t pkt) >>>>> +void *odp_packet_map(odp_packet_t pkt, size_t *seglen) >>>>> { >>>>> - return odp_packet_addr(pkt) + >>>>> odp_packet_hdr(pkt)->frame_offset; >>>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>>> + >>>>> + return buffer_map(&pkt_hdr->buf_hdr, >>>>> + 0, seglen, pkt_hdr->frame_len); >>>>> } >>>>> >>>>> +void *odp_packet_addr(odp_packet_t pkt) >>>>> +{ >>>>> + size_t seglen; >>>>> + return odp_packet_map(pkt, &seglen); >>>>> +} >>>>> >>>>> -uint8_t *odp_packet_l2(odp_packet_t pkt) >>>>> +odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt) >>>>> { >>>>> - const size_t offset = odp_packet_l2_offset(pkt); >>>>> + return odp_packet_hdr(pkt)->buf_hdr.pool_hdl; >>>>> +} >>>>> >>>>> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >>>>> - return NULL; >>>>> +odp_packet_segment_t odp_packet_segment_by_index(odp_packet_t pkt, >>>>> + size_t ndx) >>>>> +{ >>>>> >>>>> - return odp_packet_addr(pkt) + offset; >>>>> + return (odp_packet_segment_t) >>>>> + buffer_segment(&odp_packet_hdr(pkt)->buf_hdr, ndx); >>>>> } >>>>> >>>>> -size_t odp_packet_l2_offset(odp_packet_t pkt) >>>>> +odp_packet_segment_t odp_packet_segment_next(odp_packet_t pkt, >>>>> + odp_packet_segment_t seg) >>>>> { >>>>> - return odp_packet_hdr(pkt)->l2_offset; >>>>> + return (odp_packet_segment_t) >>>>> + segment_next(&odp_packet_hdr(pkt)->buf_hdr, seg); >>>>> } >>>>> >>>>> -void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset) >>>>> +void *odp_packet_segment_map(odp_packet_t pkt, odp_packet_segment_t >>>>> seg, >>>>> + size_t *seglen) >>>>> { >>>>> - odp_packet_hdr(pkt)->l2_offset = offset; >>>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>>> + >>>>> + return segment_map(&pkt_hdr->buf_hdr, seg, >>>>> + seglen, pkt_hdr->frame_len); >>>>> } >>>>> >>>>> -uint8_t *odp_packet_l3(odp_packet_t pkt) >>>>> +void odp_packet_segment_unmap(odp_packet_segment_t seg ODP_UNUSED) >>>>> { >>>>> - const size_t offset = odp_packet_l3_offset(pkt); >>>>> +} >>>>> >>>>> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >>>>> - return NULL; >>>>> +void *odp_packet_udata(odp_packet_t pkt, size_t *len) >>>>> +{ >>>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>>> + >>>>> + *len = pkt_hdr->buf_hdr.udata_size; >>>>> + return pkt_hdr->buf_hdr.udata_addr; >>>>> +} >>>>> + >>>>> +void *odp_packet_udata_addr(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->buf_hdr.udata_addr; >>>>> +} >>>>> + >>>>> +void *odp_packet_l2_map(odp_packet_t pkt, size_t *seglen) >>>>> +{ >>>>> + return odp_packet_offset_map(pkt, odp_packet_l2_offset(pkt), >>>>> seglen); >>>>> +} >>>>> + >>>>> +size_t odp_packet_l2_offset(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->l2_offset; >>>>> +} >>>>> + >>>>> +int odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset) >>>>> +{ >>>>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>>>> + >>>>> + if (offset >= hdr->frame_len) >>>>> + return -1; >>>>> + >>>>> + hdr->l2_offset = offset; >>>>> + return 0; >>>>> +} >>>>> >>>>> - return odp_packet_addr(pkt) + offset; >>>>> +void *odp_packet_l3_map(odp_packet_t pkt, size_t *seglen) >>>>> +{ >>>>> + return odp_packet_offset_map(pkt, odp_packet_l3_offset(pkt), >>>>> seglen); >>>>> } >>>>> >>>>> size_t odp_packet_l3_offset(odp_packet_t pkt) >>>>> @@ -102,19 +161,35 @@ size_t odp_packet_l3_offset(odp_packet_t pkt) >>>>> return odp_packet_hdr(pkt)->l3_offset; >>>>> } >>>>> >>>>> -void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset) >>>>> +int odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset) >>>>> { >>>>> - odp_packet_hdr(pkt)->l3_offset = offset; >>>>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>>>> + >>>>> + if (offset >= hdr->frame_len) >>>>> + return -1; >>>>> + >>>>> + hdr->l3_offset = offset; >>>>> + return 0; >>>>> } >>>>> >>>>> -uint8_t *odp_packet_l4(odp_packet_t pkt) >>>>> +uint32_t odp_packet_l3_protocol(odp_packet_t pkt) >>>>> { >>>>> - const size_t offset = odp_packet_l4_offset(pkt); >>>>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>>>> >>>>> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >>>>> - return NULL; >>>>> + if (hdr->input_flags.l3) >>>>> + return hdr->l3_protocol; >>>>> + else >>>>> + return -1; >>>>> +} >>>>> + >>>>> +void odp_packet_set_l3_protocol(odp_packet_t pkt, uint16_t protocol) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->l3_protocol = protocol; >>>>> +} >>>>> >>>>> - return odp_packet_addr(pkt) + offset; >>>>> +void *odp_packet_l4_map(odp_packet_t pkt, size_t *seglen) >>>>> +{ >>>>> + return odp_packet_offset_map(pkt, odp_packet_l4_offset(pkt), >>>>> seglen); >>>>> } >>>>> >>>>> size_t odp_packet_l4_offset(odp_packet_t pkt) >>>>> @@ -122,277 +197,902 @@ size_t odp_packet_l4_offset(odp_packet_t pkt) >>>>> return odp_packet_hdr(pkt)->l4_offset; >>>>> } >>>>> >>>>> -void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset) >>>>> +int odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset) >>>>> { >>>>> - odp_packet_hdr(pkt)->l4_offset = offset; >>>>> -} >>>>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>>>> + >>>>> + if (offset >= hdr->frame_len) >>>>> + return -1; >>>>> >>>>> + hdr->l4_offset = offset; >>>>> + return 0; >>>>> +} >>>>> >>>>> -int odp_packet_is_segmented(odp_packet_t pkt) >>>>> +uint32_t odp_packet_l4_protocol(odp_packet_t pkt) >>>>> { >>>>> - odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr((odp_buffer_t)pkt); >>>>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>>>> >>>>> - if (buf_hdr->scatter.num_bufs == 0) >>>>> - return 0; >>>>> + if (hdr->input_flags.l4) >>>>> + return hdr->l4_protocol; >>>>> else >>>>> - return 1; >>>>> + return -1; >>>>> } >>>>> >>>>> +void odp_packet_set_l4_protocol(odp_packet_t pkt, uint8_t protocol) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->l4_protocol = protocol; >>>>> +} >>>>> >>>>> -int odp_packet_seg_count(odp_packet_t pkt) >>>>> +void *odp_packet_payload_map(odp_packet_t pkt, size_t *seglen) >>>>> { >>>>> - odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr((odp_buffer_t)pkt); >>>>> + return odp_packet_offset_map(pkt, >>>>> odp_packet_payload_offset(pkt), >>>>> + seglen); >>>>> +} >>>>> >>>>> - return (int)buf_hdr->scatter.num_bufs + 1; >>>>> +size_t odp_packet_payload_offset(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->payload_offset; >>>>> } >>>>> >>>>> +int odp_packet_set_payload_offset(odp_packet_t pkt, size_t offset) >>>>> +{ >>>>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>>>> >>>>> -/** >>>>> - * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP >>>>> - * >>>>> - * Internal function: caller is resposible for passing only valid >>>>> packet handles >>>>> - * , lengths and offsets (usually done&called in packet input). >>>>> - * >>>>> - * @param pkt Packet handle >>>>> - * @param len Packet length in bytes >>>>> - * @param frame_offset Byte offset to L2 header >>>>> - */ >>>>> -void odp_packet_parse(odp_packet_t pkt, size_t len, size_t >>>>> frame_offset) >>>>> + if (offset >= hdr->frame_len) >>>>> + return -1; >>>>> + >>>>> + hdr->payload_offset = offset; >>>>> + return 0; >>>>> +} >>>>> + >>>>> +int odp_packet_error(odp_packet_t pkt) >>>>> { >>>>> - odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >>>>> - odph_ethhdr_t *eth; >>>>> - odph_vlanhdr_t *vlan; >>>>> - odph_ipv4hdr_t *ipv4; >>>>> - odph_ipv6hdr_t *ipv6; >>>>> - uint16_t ethtype; >>>>> - size_t offset = 0; >>>>> - uint8_t ip_proto = 0; >>>>> + return odp_packet_hdr(pkt)->error_flags.all != 0; >>>>> +} >>>>> >>>>> - pkt_hdr->input_flags.eth = 1; >>>>> - pkt_hdr->frame_offset = frame_offset; >>>>> - pkt_hdr->frame_len = len; >>>>> +void odp_packet_set_error(odp_packet_t pkt, int val) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->error_flags.app_error = val; >>>>> +} >>>>> >>>>> - if (odp_unlikely(len < ODPH_ETH_LEN_MIN)) { >>>>> - pkt_hdr->error_flags.frame_len = 1; >>>>> - return; >>>>> - } else if (len > ODPH_ETH_LEN_MAX) { >>>>> - pkt_hdr->input_flags.jumbo = 1; >>>>> - } >>>>> +int odp_packet_inflag_l2(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->input_flags.l2; >>>>> +} >>>>> >>>>> - /* Assume valid L2 header, no CRC/FCS check in SW */ >>>>> - pkt_hdr->input_flags.l2 = 1; >>>>> - pkt_hdr->l2_offset = frame_offset; >>>>> +void odp_packet_set_inflag_l2(odp_packet_t pkt, int val) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->input_flags.l2 = val; >>>>> +} >>>>> >>>>> - eth = (odph_ethhdr_t *)odp_packet_data(pkt); >>>>> - ethtype = odp_be_to_cpu_16(eth->type); >>>>> - vlan = (odph_vlanhdr_t *)ð->type; >>>>> +int odp_packet_inflag_l3(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->input_flags.l3; >>>>> +} >>>>> >>>>> - if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { >>>>> - pkt_hdr->input_flags.vlan_qinq = 1; >>>>> - ethtype = odp_be_to_cpu_16(vlan->tpid); >>>>> - offset += sizeof(odph_vlanhdr_t); >>>>> - vlan = &vlan[1]; >>>>> - } >>>>> +void odp_packet_set_inflag_l3(odp_packet_t pkt, int val) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->input_flags.l3 = val; >>>>> +} >>>>> >>>>> - if (ethtype == ODPH_ETHTYPE_VLAN) { >>>>> - pkt_hdr->input_flags.vlan = 1; >>>>> - ethtype = odp_be_to_cpu_16(vlan->tpid); >>>>> - offset += sizeof(odph_vlanhdr_t); >>>>> - } >>>>> +int odp_packet_inflag_l4(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->input_flags.l4; >>>>> +} >>>>> >>>>> - /* Set l3_offset+flag only for known ethtypes */ >>>>> - switch (ethtype) { >>>>> - case ODPH_ETHTYPE_IPV4: >>>>> - pkt_hdr->input_flags.ipv4 = 1; >>>>> - pkt_hdr->input_flags.l3 = 1; >>>>> - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + >>>>> offset; >>>>> - ipv4 = (odph_ipv4hdr_t *)odp_packet_l3(pkt); >>>>> - ip_proto = parse_ipv4(pkt_hdr, ipv4, &offset); >>>>> - break; >>>>> - case ODPH_ETHTYPE_IPV6: >>>>> - pkt_hdr->input_flags.ipv6 = 1; >>>>> - pkt_hdr->input_flags.l3 = 1; >>>>> - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + >>>>> offset; >>>>> - ipv6 = (odph_ipv6hdr_t *)odp_packet_l3(pkt); >>>>> - ip_proto = parse_ipv6(pkt_hdr, ipv6, &offset); >>>>> - break; >>>>> - case ODPH_ETHTYPE_ARP: >>>>> - pkt_hdr->input_flags.arp = 1; >>>>> - /* fall through */ >>>>> - default: >>>>> - ip_proto = 0; >>>>> - break; >>>>> - } >>>>> +void odp_packet_set_inflag_l4(odp_packet_t pkt, int val) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->input_flags.l4 = val; >>>>> +} >>>>> >>>>> - switch (ip_proto) { >>>>> - case ODPH_IPPROTO_UDP: >>>>> - pkt_hdr->input_flags.udp = 1; >>>>> - pkt_hdr->input_flags.l4 = 1; >>>>> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >>>>> - break; >>>>> - case ODPH_IPPROTO_TCP: >>>>> - pkt_hdr->input_flags.tcp = 1; >>>>> - pkt_hdr->input_flags.l4 = 1; >>>>> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >>>>> - break; >>>>> - case ODPH_IPPROTO_SCTP: >>>>> - pkt_hdr->input_flags.sctp = 1; >>>>> - pkt_hdr->input_flags.l4 = 1; >>>>> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >>>>> - break; >>>>> - case ODPH_IPPROTO_ICMP: >>>>> - pkt_hdr->input_flags.icmp = 1; >>>>> - pkt_hdr->input_flags.l4 = 1; >>>>> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >>>>> - break; >>>>> - default: >>>>> - /* 0 or unhandled IP protocols, don't set L4 >>>>> flag+offset */ >>>>> - if (pkt_hdr->input_flags.ipv6) { >>>>> - /* IPv6 next_hdr is not L4, mark as IP-option >>>>> instead */ >>>>> - pkt_hdr->input_flags.ipopt = 1; >>>>> - } >>>>> - break; >>>>> - } >>>>> +int odp_packet_inflag_eth(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->input_flags.eth; >>>>> } >>>>> >>>>> -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >>>>> - odph_ipv4hdr_t *ipv4, size_t >>>>> *offset_out) >>>>> +void odp_packet_set_inflag_eth(odp_packet_t pkt, int val) >>>>> { >>>>> - uint8_t ihl; >>>>> - uint16_t frag_offset; >>>>> + odp_packet_hdr(pkt)->input_flags.eth = val; >>>>> +} >>>>> >>>>> - ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); >>>>> - if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN)) { >>>>> - pkt_hdr->error_flags.ip_err = 1; >>>>> - return 0; >>>>> - } >>>>> +int odp_packet_inflag_jumbo(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->input_flags.jumbo; >>>>> +} >>>>> >>>>> - if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) { >>>>> - pkt_hdr->input_flags.ipopt = 1; >>>>> - return 0; >>>>> - } >>>>> +void odp_packet_set_inflag_jumbo(odp_packet_t pkt, int val) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->input_flags.jumbo = val; >>>>> +} >>>>> >>>>> - /* A packet is a fragment if: >>>>> - * "more fragments" flag is set (all fragments except the last) >>>>> - * OR >>>>> - * "fragment offset" field is nonzero (all fragments except >>>>> the first) >>>>> - */ >>>>> - frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); >>>>> - if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) { >>>>> - pkt_hdr->input_flags.ipfrag = 1; >>>>> - return 0; >>>>> - } >>>>> +int odp_packet_inflag_vlan(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->input_flags.vlan; >>>>> +} >>>>> >>>>> - if (ipv4->proto == ODPH_IPPROTO_ESP || >>>>> - ipv4->proto == ODPH_IPPROTO_AH) { >>>>> - pkt_hdr->input_flags.ipsec = 1; >>>>> - return 0; >>>>> - } >>>>> +void odp_packet_set_inflag_vlan(odp_packet_t pkt, int val) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->input_flags.vlan = val; >>>>> +} >>>>> >>>>> - /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after >>>>> return */ >>>>> +int odp_packet_inflag_vlan_qinq(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->input_flags.vlan_qinq; >>>>> +} >>>>> >>>>> - *offset_out = sizeof(uint32_t) * ihl; >>>>> - return ipv4->proto; >>>>> +void odp_packet_set_inflag_vlan_qinq(odp_packet_t pkt, int val) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->input_flags.vlan_qinq = val; >>>>> } >>>>> >>>>> -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >>>>> - odph_ipv6hdr_t *ipv6, size_t >>>>> *offset_out) >>>>> +int odp_packet_inflag_snap(odp_packet_t pkt) >>>>> { >>>>> - if (ipv6->next_hdr == ODPH_IPPROTO_ESP || >>>>> - ipv6->next_hdr == ODPH_IPPROTO_AH) { >>>>> - pkt_hdr->input_flags.ipopt = 1; >>>>> - pkt_hdr->input_flags.ipsec = 1; >>>>> - return 0; >>>>> - } >>>>> + return odp_packet_hdr(pkt)->input_flags.snap; >>>>> +} >>>>> >>>>> - if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { >>>>> - pkt_hdr->input_flags.ipopt = 1; >>>>> - pkt_hdr->input_flags.ipfrag = 1; >>>>> - return 0; >>>>> - } >>>>> +void odp_packet_set_inflag_snap(odp_packet_t pkt, int val) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->input_flags.snap = val; >>>>> +} >>>>> >>>>> - /* Don't step through more extensions */ >>>>> - *offset_out = ODPH_IPV6HDR_LEN; >>>>> - return ipv6->next_hdr; >>>>> +int odp_packet_inflag_arp(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->input_flags.arp; >>>>> } >>>>> >>>>> -void odp_packet_print(odp_packet_t pkt) >>>>> +void odp_packet_set_inflag_arp(odp_packet_t pkt, int val) >>>>> { >>>>> - int max_len = 512; >>>>> - char str[max_len]; >>>>> - int len = 0; >>>>> - int n = max_len-1; >>>>> - odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>>>> + odp_packet_hdr(pkt)->input_flags.arp = val; >>>>> +} >>>>> >>>>> - len += snprintf(&str[len], n-len, "Packet "); >>>>> - len += odp_buffer_snprint(&str[len], n-len, (odp_buffer_t) >>>>> pkt); >>>>> - len += snprintf(&str[len], n-len, >>>>> - " input_flags 0x%x\n", hdr->input_flags.all); >>>>> - len += snprintf(&str[len], n-len, >>>>> - " error_flags 0x%x\n", hdr->error_flags.all); >>>>> - len += snprintf(&str[len], n-len, >>>>> - " output_flags 0x%x\n", >>>>> hdr->output_flags.all); >>>>> - len += snprintf(&str[len], n-len, >>>>> - " frame_offset %u\n", hdr->frame_offset); >>>>> - len += snprintf(&str[len], n-len, >>>>> - " l2_offset %u\n", hdr->l2_offset); >>>>> - len += snprintf(&str[len], n-len, >>>>> - " l3_offset %u\n", hdr->l3_offset); >>>>> - len += snprintf(&str[len], n-len, >>>>> - " l4_offset %u\n", hdr->l4_offset); >>>>> - len += snprintf(&str[len], n-len, >>>>> - " frame_len %u\n", hdr->frame_len); >>>>> - len += snprintf(&str[len], n-len, >>>>> - " input %u\n", hdr->input); >>>>> - str[len] = '\0'; >>>>> +int odp_packet_inflag_ipv4(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->input_flags.ipv4; >>>>> +} >>>>> >>>>> - printf("\n%s\n", str); >>>>> +void odp_packet_set_inflag_ipv4(odp_packet_t pkt, int val) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->input_flags.ipv4 = val; >>>>> } >>>>> >>>>> -int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src) >>>>> +int odp_packet_inflag_ipv6(odp_packet_t pkt) >>>>> { >>>>> - odp_packet_hdr_t *const pkt_hdr_dst = odp_packet_hdr(pkt_dst); >>>>> - odp_packet_hdr_t *const pkt_hdr_src = odp_packet_hdr(pkt_src); >>>>> - const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, >>>>> buf_hdr); >>>>> - uint8_t *start_src; >>>>> - uint8_t *start_dst; >>>>> - size_t len; >>>>> + return odp_packet_hdr(pkt)->input_flags.ipv6; >>>>> +} >>>>> >>>>> - if (pkt_dst == ODP_PACKET_INVALID || pkt_src == >>>>> ODP_PACKET_INVALID) >>>>> - return -1; >>>>> +void odp_packet_set_inflag_ipv6(odp_packet_t pkt, int val) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->input_flags.ipv6 = val; >>>>> +} >>>>> >>>>> - if (pkt_hdr_dst->buf_hdr.size < >>>>> - pkt_hdr_src->frame_len + pkt_hdr_src->frame_offset) >>>>> - return -1; >>>>> +int odp_packet_inflag_ipfrag(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->input_flags.ipfrag; >>>>> +} >>>>> + >>>>> +void odp_packet_set_inflag_ipfrag(odp_packet_t pkt, int val) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->input_flags.ipfrag = val; >>>>> +} >>>>> + >>>>> +int odp_packet_inflag_ipopt(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->input_flags.ipopt; >>>>> +} >>>>> >>>>> - /* Copy packet header */ >>>>> - start_dst = (uint8_t *)pkt_hdr_dst + start_offset; >>>>> - start_src = (uint8_t *)pkt_hdr_src + start_offset; >>>>> - len = ODP_OFFSETOF(odp_packet_hdr_t, buf_data) - start_offset; >>>>> - memcpy(start_dst, start_src, len); >>>>> +void odp_packet_set_inflag_ipopt(odp_packet_t pkt, int val) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->input_flags.ipopt = val; >>>>> +} >>>>> >>>>> - /* Copy frame payload */ >>>>> - start_dst = (uint8_t *)odp_packet_data(pkt_dst); >>>>> - start_src = (uint8_t *)odp_packet_data(pkt_src); >>>>> - len = pkt_hdr_src->frame_len; >>>>> - memcpy(start_dst, start_src, len); >>>>> +int odp_packet_inflag_ipsec(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->input_flags.ipsec; >>>>> +} >>>>> >>>>> - /* Copy useful things from the buffer header */ >>>>> - pkt_hdr_dst->buf_hdr.cur_offset = >>>>> pkt_hdr_src->buf_hdr.cur_offset; >>>>> +void odp_packet_set_inflag_ipsec(odp_packet_t pkt, int val) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->input_flags.ipsec = val; >>>>> +} >>>>> >>>>> - /* Create a copy of the scatter list */ >>>>> - odp_buffer_copy_scatter(odp_packet_to_buffer(pkt_dst), >>>>> - odp_packet_to_buffer(pkt_src)); >>>>> +int odp_packet_inflag_udp(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->input_flags.udp; >>>>> +} >>>>> >>>>> - return 0; >>>>> +void odp_packet_set_inflag_udp(odp_packet_t pkt, int val) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->input_flags.udp = val; >>>>> } >>>>> >>>>> -void odp_packet_set_ctx(odp_packet_t pkt, const void *ctx) >>>>> +int odp_packet_inflag_tcp(odp_packet_t pkt) >>>>> { >>>>> - odp_packet_hdr(pkt)->user_ctx = (intptr_t)ctx; >>>>> + return odp_packet_hdr(pkt)->input_flags.tcp; >>>>> } >>>>> >>>>> -void *odp_packet_get_ctx(odp_packet_t pkt) >>>>> +void odp_packet_set_inflag_tcp(odp_packet_t pkt, int val) >>>>> { >>>>> - return (void *)(intptr_t)odp_packet_hdr(pkt)->user_ctx; >>>>> + odp_packet_hdr(pkt)->input_flags.tcp = val; >>>>> +} >>>>> + >>>>> +int odp_packet_inflag_tcpopt(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->input_flags.tcpopt; >>>>> +} >>>>> + >>>>> +void odp_packet_set_inflag_tcpopt(odp_packet_t pkt, int val) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->input_flags.tcpopt = val; >>>>> +} >>>>> + >>>>> +int odp_packet_inflag_icmp(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->input_flags.icmp; >>>>> +} >>>>> + >>>>> +void odp_packet_set_inflag_icmp(odp_packet_t pkt, int val) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->input_flags.icmp = val; >>>>> +} >>>>> + >>>>> +int odp_packet_is_valid(odp_packet_t pkt) >>>>> +{ >>>>> + odp_buffer_hdr_t *buf = validate_buf((odp_buffer_t)pkt); >>>>> + >>>>> + if (buf == NULL) >>>>> + return 0; >>>>> + else >>>>> + return buf->type == ODP_BUFFER_TYPE_PACKET; >>>>> +} >>>>> + >>>>> +int odp_packet_is_segmented(odp_packet_t pkt) >>>>> +{ >>>>> + return (odp_packet_hdr(pkt)->buf_hdr.segcount > 1); >>>>> +} >>>>> + >>>>> +int odp_packet_segment_count(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->buf_hdr.segcount; >>>>> +} >>>>> + >>>>> +size_t odp_packet_headroom(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->headroom; >>>>> +} >>>>> + >>>>> +size_t odp_packet_tailroom(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_hdr(pkt)->tailroom; >>>>> +} >>>>> + >>>>> +int odp_packet_push_head(odp_packet_t pkt, size_t len) >>>>> +{ >>>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>>> + >>>>> + if (len > pkt_hdr->headroom) >>>>> + return -1; >>>>> + >>>>> + push_head(pkt_hdr, len); >>>>> + return 0; >>>>> +} >>>>> + >>>>> +void *odp_packet_push_head_and_map(odp_packet_t pkt, size_t len, >>>>> size_t *seglen) >>>>> +{ >>>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>>> + >>>>> + if (len > pkt_hdr->headroom) >>>>> + return NULL; >>>>> + >>>>> + push_head(pkt_hdr, len); >>>>> + >>>>> + return buffer_map(&pkt_hdr->buf_hdr, 0, seglen, >>>>> pkt_hdr->frame_len); >>>>> +} >>>>> + >>>>> +int odp_packet_pull_head(odp_packet_t pkt, size_t len) >>>>> +{ >>>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>>> + >>>>> + if (len > pkt_hdr->frame_len) >>>>> + return -1; >>>>> + >>>>> + pull_head(pkt_hdr, len); >>>>> + return 0; >>>>> +} >>>>> + >>>>> +void *odp_packet_pull_head_and_map(odp_packet_t pkt, size_t len, >>>>> size_t *seglen) >>>>> +{ >>>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>>> + >>>>> + if (len > pkt_hdr->frame_len) >>>>> + return NULL; >>>>> + >>>>> + pull_head(pkt_hdr, len); >>>>> + return buffer_map(&pkt_hdr->buf_hdr, 0, seglen, >>>>> pkt_hdr->frame_len); >>>>> +} >>>>> + >>>>> +int odp_packet_push_tail(odp_packet_t pkt, size_t len) >>>>> +{ >>>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>>> + >>>>> + if (len > pkt_hdr->tailroom) >>>>> + return -1; >>>>> + >>>>> + push_tail(pkt_hdr, len); >>>>> + return 0; >>>>> +} >>>>> + >>>>> +void *odp_packet_push_tail_and_map(odp_packet_t pkt, size_t len, >>>>> size_t *seglen) >>>>> +{ >>>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>>> + size_t origin = pkt_hdr->frame_len; >>>>> + >>>>> + if (len > pkt_hdr->tailroom) >>>>> + return NULL; >>>>> + >>>>> + push_tail(pkt_hdr, len); >>>>> + >>>>> + return buffer_map(&pkt_hdr->buf_hdr, origin, seglen, >>>>> + pkt_hdr->frame_len); >>>>> +} >>>>> + >>>>> +int odp_packet_pull_tail(odp_packet_t pkt, size_t len) >>>>> +{ >>>>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>>>> + >>>>> + if (len > pkt_hdr->frame_len) >>>>> + return -1; >>>>> + >>>>> + pull_tail(pkt_hdr, len); >>>>> + return 0; >>>>> +} >>>>> + >>>>> +void odp_packet_print(odp_packet_t pkt) >>>>> +{ >>>>> + int max_len = 512; >>>>> + char str[max_len]; >>>>> + int len = 0; >>>>> + int n = max_len-1; >>>>> + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>>>> + >>>>> + len += snprintf(&str[len], n-len, "Packet "); >>>>> + len += odp_buffer_snprint(&str[len], n-len, (odp_buffer_t) >>>>> pkt); >>>>> + len += snprintf(&str[len], n-len, >>>>> + " input_flags 0x%x\n", hdr->input_flags.all); >>>>> + len += snprintf(&str[len], n-len, >>>>> + " error_flags 0x%x\n", hdr->error_flags.all); >>>>> + len += snprintf(&str[len], n-len, >>>>> + " output_flags 0x%x\n", >>>>> hdr->output_flags.all); >>>>> + len += snprintf(&str[len], n-len, >>>>> + " l2_offset %u\n", hdr->l2_offset); >>>>> + len += snprintf(&str[len], n-len, >>>>> + " l3_offset %u\n", hdr->l3_offset); >>>>> + len += snprintf(&str[len], n-len, >>>>> + " l3_len %u\n", hdr->l3_len); >>>>> + len += snprintf(&str[len], n-len, >>>>> + " l3_protocol 0x%x\n", hdr->l3_protocol); >>>>> + len += snprintf(&str[len], n-len, >>>>> + " l4_offset %u\n", hdr->l4_offset); >>>>> + len += snprintf(&str[len], n-len, >>>>> + " l4_len %u\n", hdr->l4_len); >>>>> + len += snprintf(&str[len], n-len, >>>>> + " l4_protocol %u\n", hdr->l4_protocol); >>>>> + len += snprintf(&str[len], n-len, >>>>> + " payload_offset %u\n", >>>>> hdr->payload_offset); >>>>> + len += snprintf(&str[len], n-len, >>>>> + " frame_len %u\n", hdr->frame_len); >>>>> + str[len] = '\0'; >>>>> + >>>>> + printf("\n%s\n", str); >>>>> +} >>>>> + >>>>> +int odp_packet_copy_to_packet(odp_packet_t dstpkt, size_t dstoffset, >>>>> + odp_packet_t srcpkt, size_t srcoffset, >>>>> + size_t len) >>>>> +{ >>>>> + void *dstmap; >>>>> + void *srcmap; >>>>> + size_t cpylen, minseg, dstseglen, srcseglen; >>>>> + >>>>> + while (len > 0) { >>>>> + dstmap = odp_packet_offset_map(dstpkt, dstoffset, >>>>> &dstseglen); >>>>> + srcmap = odp_packet_offset_map(srcpkt, srcoffset, >>>>> &srcseglen); >>>>> + if (dstmap == NULL || srcmap == NULL) >>>>> + return -1; >>>>> + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; >>>>> + cpylen = len > minseg ? minseg : len; >>>>> + memcpy(dstmap, srcmap, cpylen); >>>>> + srcoffset += cpylen; >>>>> + dstoffset += cpylen; >>>>> + len -= cpylen; >>>>> + } >>>>> + >>>>> + return 0; >>>>> +} >>>>> + >>>>> +int odp_packet_copy_to_memory(void *dstmem, odp_packet_t srcpkt, >>>>> + size_t srcoffset, size_t dstlen) >>>>> +{ >>>>> + void *mapaddr; >>>>> + size_t seglen, cpylen; >>>>> + uint8_t *dstaddr = (uint8_t *)dstmem; >>>>> + >>>>> + while (dstlen > 0) { >>>>> + mapaddr = odp_packet_offset_map(srcpkt, srcoffset, >>>>> &seglen); >>>>> + if (mapaddr == NULL) >>>>> + return -1; >>>>> + cpylen = dstlen > seglen ? seglen : dstlen; >>>>> + memcpy(dstaddr, mapaddr, cpylen); >>>>> + srcoffset += cpylen; >>>>> + dstaddr += cpylen; >>>>> + dstlen -= cpylen; >>>>> + } >>>>> + >>>>> + return 0; >>>>> +} >>>>> + >>>>> +int odp_packet_copy_from_memory(odp_packet_t dstpkt, size_t dstoffset, >>>>> + void *srcmem, size_t srclen) >>>>> +{ >>>>> + void *mapaddr; >>>>> + size_t seglen, cpylen; >>>>> + uint8_t *srcaddr = (uint8_t *)srcmem; >>>>> + >>>>> + while (srclen > 0) { >>>>> + mapaddr = odp_packet_offset_map(dstpkt, dstoffset, >>>>> &seglen); >>>>> + if (mapaddr == NULL) >>>>> + return -1; >>>>> + cpylen = srclen > seglen ? seglen : srclen; >>>>> + memcpy(mapaddr, srcaddr, cpylen); >>>>> + dstoffset += cpylen; >>>>> + srcaddr += cpylen; >>>>> + srclen -= cpylen; >>>>> + } >>>>> + >>>>> + return 0; >>>>> +} >>>>> + >>>>> +odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool) >>>>> +{ >>>>> + size_t pktlen = odp_packet_len(pkt); >>>>> + const size_t meta_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, >>>>> buf_hdr); >>>>> + odp_packet_t newpkt = odp_packet_alloc_len(pool, pktlen); >>>>> + odp_packet_hdr_t *newhdr, *srchdr; >>>>> + uint8_t *newstart, *srcstart; >>>>> + >>>>> + if (newpkt != ODP_PACKET_INVALID) { >>>>> + /* Must copy meta data first, followed by packet data >>>>> */ >>>>> + srchdr = odp_packet_hdr(pkt); >>>>> + newhdr = odp_packet_hdr(newpkt); >>>>> + newstart = (uint8_t *)newhdr + meta_offset; >>>>> + srcstart = (uint8_t *)srchdr + meta_offset; >>>>> + >>>>> + memcpy(newstart, srcstart, >>>>> + sizeof(odp_packet_hdr_t) - meta_offset); >>>>> + >>>>> + if (odp_packet_copy_to_packet(newpkt, 0, pkt, 0, >>>>> pktlen) != 0) { >>>>> + odp_packet_free(newpkt); >>>>> + newpkt = ODP_PACKET_INVALID; >>>>> + } >>>>> + } >>>>> + >>>>> + return newpkt; >>>>> +} >>>>> + >>>>> +odp_packet_t odp_packet_clone(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_packet_copy(pkt, >>>>> odp_packet_hdr(pkt)->buf_hdr.pool_hdl); >>>>> +} >>>>> + >>>>> +odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool) >>>>> +{ >>>>> + if (odp_pool_to_entry(pool)->s.params.buf_type != >>>>> + ODP_BUFFER_TYPE_PACKET) >>>>> + return ODP_PACKET_INVALID; >>>>> + >>>>> + return (odp_packet_t)buffer_alloc(pool, 0); >>>>> +} >>>>> + >>>>> +odp_packet_t odp_packet_alloc_len(odp_buffer_pool_t pool, size_t len) >>>>> +{ >>>>> + if (odp_pool_to_entry(pool)->s.params.buf_type != >>>>> + ODP_BUFFER_TYPE_PACKET) >>>>> + return ODP_PACKET_INVALID; >>>>> + >>>>> + return (odp_packet_t)buffer_alloc(pool, len); >>>>> +} >>>>> + >>>>> +void odp_packet_free(odp_packet_t pkt) >>>>> +{ >>>>> + odp_buffer_free((odp_buffer_t)pkt); >>>>> +} >>>>> + >>>>> +odp_packet_t odp_packet_split(odp_packet_t pkt, size_t offset, >>>>> + size_t hr, size_t tr) >>>>> +{ >>>>> + size_t len = odp_packet_len(pkt); >>>>> + odp_buffer_pool_t pool = odp_packet_pool(pkt); >>>>> + size_t pool_hr = odp_buffer_pool_headroom(pool); >>>>> + size_t pkt_tr = odp_packet_tailroom(pkt); >>>>> + odp_packet_t splitpkt; >>>>> + size_t splitlen = len - offset; >>>>> + >>>>> + if (offset >= len) >>>>> + return ODP_PACKET_INVALID; >>>>> + >>>>> + /* Erratum: We don't handle this rare corner case */ >>>>> + if (tr > pkt_tr + splitlen) >>>>> + return ODP_PACKET_INVALID; >>>>> + >>>>> + if (hr > pool_hr) >>>>> + splitlen += (hr - pool_hr); >>>>> + >>>>> + splitpkt = odp_packet_alloc_len(pool, splitlen); >>>>> + >>>>> + if (splitpkt != ODP_PACKET_INVALID) { >>>>> + if (hr > pool_hr) { >>>>> + odp_packet_pull_head(splitpkt, hr - pool_hr); >>>>> + splitlen -= (hr - pool_hr); >>>>> + } >>>>> + odp_packet_copy_to_packet(splitpkt, 0, >>>>> + pkt, offset, splitlen); >>>>> + odp_packet_pull_tail(pkt, splitlen); >>>>> + } >>>>> + >>>>> + return splitpkt; >>>>> +} >>>>> + >>>>> +odp_packet_t odp_packet_join(odp_packet_t pkt1, odp_packet_t pkt2) >>>>> +{ >>>>> + size_t len1 = odp_packet_len(pkt1); >>>>> + size_t len2 = odp_packet_len(pkt2); >>>>> + odp_packet_t joinpkt; >>>>> + void *udata1, *udata_join; >>>>> + size_t udata_size1, udata_size_join; >>>>> + >>>>> + /* Optimize if pkt1 is big enough to hold pkt2 */ >>>>> + if (odp_packet_push_tail(pkt1, len2) == 0) { >>>>> + odp_packet_copy_to_packet(pkt1, len1, >>>>> + pkt2, 0, len2); >>>>> + odp_packet_free(pkt2); >>>>> + return pkt1; >>>>> + } >>>>> + >>>>> + /* Otherwise join into a new packet */ >>>>> + joinpkt = odp_packet_alloc_len(odp_packet_pool(pkt1), >>>>> + len1 + len2); >>>>> + >>>>> + if (joinpkt != ODP_PACKET_INVALID) { >>>>> + odp_packet_copy_to_packet(joinpkt, 0, pkt1, 0, len1); >>>>> + odp_packet_copy_to_packet(joinpkt, len1, pkt2, 0, >>>>> len2); >>>>> + >>>>> + /* Copy user metadata if present */ >>>>> + udata1 = odp_packet_udata(pkt1, &udata_size1); >>>>> + udata_join = odp_packet_udata(joinpkt, >>>>> &udata_size_join); >>>>> + >>>>> + if (udata1 != NULL && udata_join != NULL) >>>>> + memcpy(udata_join, udata1, >>>>> + udata_size_join > udata_size1 ? >>>>> + udata_size1 : udata_size_join); >>>>> + >>>>> + odp_packet_free(pkt1); >>>>> + odp_packet_free(pkt2); >>>>> + } >>>>> + >>>>> + return joinpkt; >>>>> +} >>>>> + >>>>> +uint32_t odp_packet_refcount(odp_packet_t pkt) >>>>> +{ >>>>> + return odp_buffer_refcount(&odp_packet_hdr(pkt)->buf_hdr); >>>>> +} >>>>> + >>>>> +uint32_t odp_packet_incr_refcount(odp_packet_t pkt, uint32_t val) >>>>> +{ >>>>> + return odp_buffer_incr_refcount(&odp_packet_hdr(pkt)->buf_hdr, >>>>> val); >>>>> +} >>>>> + >>>>> +uint32_t odp_packet_decr_refcount(odp_packet_t pkt, uint32_t val) >>>>> +{ >>>>> + return odp_buffer_decr_refcount(&odp_packet_hdr(pkt)->buf_hdr, >>>>> val); >>>>> +} >>>>> + >>>>> +/** >>>>> + * Parser helper function for IPv4 >>>>> + */ >>>>> +static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >>>>> + uint8_t **parseptr, size_t *offset) >>>>> +{ >>>>> + odph_ipv4hdr_t *ipv4 = (odph_ipv4hdr_t *)*parseptr; >>>>> + uint8_t ver = ODPH_IPV4HDR_VER(ipv4->ver_ihl); >>>>> + uint8_t ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); >>>>> + uint16_t frag_offset; >>>>> + >>>>> + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv4->tot_len); >>>>> + >>>>> + if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN) || >>>>> + odp_unlikely(ver != 4) || >>>>> + (pkt_hdr->l3_len > pkt_hdr->frame_len - *offset)) { >>>>> + pkt_hdr->error_flags.ip_err = 1; >>>>> + return 0; >>>>> + } >>>>> + >>>>> + *offset += ihl * 4; >>>>> + *parseptr += ihl * 4; >>>>> + >>>>> + if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) >>>>> + pkt_hdr->input_flags.ipopt = 1; >>>>> + >>>>> + /* A packet is a fragment if: >>>>> + * "more fragments" flag is set (all fragments except the last) >>>>> + * OR >>>>> + * "fragment offset" field is nonzero (all fragments except >>>>> the first) >>>>> + */ >>>>> + frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); >>>>> + if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) >>>>> + pkt_hdr->input_flags.ipfrag = 1; >>>>> + >>>>> + if (ipv4->proto == ODPH_IPPROTO_ESP || >>>>> + ipv4->proto == ODPH_IPPROTO_AH) { >>>>> + pkt_hdr->input_flags.ipsec = 1; >>>>> + return 0; >>>>> + } >>>>> + >>>>> + /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after >>>>> return */ >>>>> + >>>>> + *offset = sizeof(uint32_t) * ihl; >>>>> + return ipv4->proto; >>>>> +} >>>>> + >>>>> +/** >>>>> + * Parser helper function for IPv6 >>>>> + */ >>>>> +static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >>>>> + uint8_t **parseptr, size_t *offset) >>>>> +{ >>>>> + odph_ipv6hdr_t *ipv6 = (odph_ipv6hdr_t *)*parseptr; >>>>> + odph_ipv6hdr_ext_t *ipv6ext; >>>>> + >>>>> + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv6->payload_len); >>>>> + >>>>> + /* Basic sanity checks on IPv6 header */ >>>>> + if (ipv6->ver != 6 || >>>>> + pkt_hdr->l3_len > pkt_hdr->frame_len - *offset) { >>>>> + pkt_hdr->error_flags.ip_err = 1; >>>>> + return 0; >>>>> + } >>>>> + >>>>> + /* Skip past IPv6 header */ >>>>> + *offset += sizeof(odph_ipv6hdr_t); >>>>> + *parseptr += sizeof(odph_ipv6hdr_t); >>>>> + >>>>> + >>>>> + /* Skip past any IPv6 extension headers */ >>>>> + if (ipv6->next_hdr == ODPH_IPPROTO_HOPOPTS || >>>>> + ipv6->next_hdr == ODPH_IPPROTO_ROUTE) { >>>>> + pkt_hdr->input_flags.ipopt = 1; >>>>> + >>>>> + do { >>>>> + ipv6ext = (odph_ipv6hdr_ext_t *)*parseptr; >>>>> + uint16_t extlen = 8 + ipv6ext->ext_len * 8; >>>>> + >>>>> + *offset += extlen; >>>>> + *parseptr += extlen; >>>>> + } while ((ipv6ext->next_hdr == ODPH_IPPROTO_HOPOPTS || >>>>> + ipv6ext->next_hdr == ODPH_IPPROTO_ROUTE) && >>>>> + *offset < pkt_hdr->frame_len); >>>>> + >>>>> + if (*offset >= pkt_hdr->l3_offset + ipv6->payload_len) >>>>> { >>>>> + pkt_hdr->error_flags.ip_err = 1; >>>>> + return 0; >>>>> + } >>>>> + >>>>> + if (ipv6ext->next_hdr == ODPH_IPPROTO_FRAG) >>>>> + pkt_hdr->input_flags.ipfrag = 1; >>>>> + >>>>> + return ipv6ext->next_hdr; >>>>> + } >>>>> + >>>>> + if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { >>>>> + pkt_hdr->input_flags.ipopt = 1; >>>>> + pkt_hdr->input_flags.ipfrag = 1; >>>>> + } >>>>> + >>>>> + return ipv6->next_hdr; >>>>> +} >>>>> + >>>>> +/** >>>>> + * Parser helper function for TCP >>>>> + */ >>>>> +static inline void parse_tcp(odp_packet_hdr_t *pkt_hdr, >>>>> + uint8_t **parseptr, size_t *offset) >>>>> +{ >>>>> + odph_tcphdr_t *tcp = (odph_tcphdr_t *)*parseptr; >>>>> + >>>>> + if (tcp->hl < sizeof(odph_tcphdr_t)/sizeof(uint32_t)) >>>>> + pkt_hdr->error_flags.tcp_err = 1; >>>>> + else if ((uint32_t)tcp->hl * 4 > sizeof(odph_tcphdr_t)) >>>>> + pkt_hdr->input_flags.tcpopt = 1; >>>>> + >>>>> + pkt_hdr->l4_len = pkt_hdr->l3_len + >>>>> + pkt_hdr->l3_offset - pkt_hdr->l4_offset; >>>>> + >>>>> + *offset += sizeof(odph_tcphdr_t); >>>>> + *parseptr += sizeof(odph_tcphdr_t); >>>>> +} >>>>> + >>>>> +/** >>>>> + * Parser helper function for UDP >>>>> + */ >>>>> +static inline void parse_udp(odp_packet_hdr_t *pkt_hdr, >>>>> + uint8_t **parseptr, size_t *offset) >>>>> +{ >>>>> + odph_udphdr_t *udp = (odph_udphdr_t *)*parseptr; >>>>> + uint32_t udplen = odp_be_to_cpu_16(udp->length); >>>>> + >>>>> + if (udplen < sizeof(odph_udphdr_t) || >>>>> + udplen > (pkt_hdr->l3_len + >>>>> + pkt_hdr->l3_offset - pkt_hdr->l4_offset)) { >>>>> + pkt_hdr->error_flags.udp_err = 1; >>>>> + } >>>>> + >>>>> + pkt_hdr->l4_len = udplen; >>>>> + >>>>> + *offset += sizeof(odph_udphdr_t); >>>>> + *parseptr += sizeof(odph_udphdr_t); >>>>> +} >>>>> + >>>>> +/** >>>>> + * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP >>>>> + * >>>>> + * Internal function: caller is resposible for passing only valid >>>>> packet handles >>>>> + * , lengths and offsets (usually done&called in packet input). >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + */ >>>>> +int odp_packet_parse(odp_packet_t pkt) >>>>> +{ >>>>> + odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >>>>> + odph_ethhdr_t *eth; >>>>> + odph_vlanhdr_t *vlan; >>>>> + uint16_t ethtype; >>>>> + uint8_t *parseptr; >>>>> + size_t offset, seglen; >>>>> + uint8_t ip_proto = 0; >>>>> + >>>>> + /* Reset parser metadata for new parse */ >>>>> + pkt_hdr->error_flags.all = 0; >>>>> + pkt_hdr->input_flags.all = 0; >>>>> + pkt_hdr->output_flags.all = 0; >>>>> + pkt_hdr->l2_offset = 0; >>>>> + pkt_hdr->l3_offset = 0; >>>>> + pkt_hdr->l4_offset = 0; >>>>> + pkt_hdr->payload_offset = 0; >>>>> + pkt_hdr->vlan_s_tag = 0; >>>>> + pkt_hdr->vlan_c_tag = 0; >>>>> + pkt_hdr->l3_protocol = 0; >>>>> + pkt_hdr->l4_protocol = 0; >>>>> + >>>>> + /* We only support Ethernet for now */ >>>>> + pkt_hdr->input_flags.eth = 1; >>>>> + >>>>> + if (odp_unlikely(pkt_hdr->frame_len < ODPH_ETH_LEN_MIN)) { >>>>> + pkt_hdr->error_flags.frame_len = 1; >>>>> + goto parse_exit; >>>>> + } else if (pkt_hdr->frame_len > ODPH_ETH_LEN_MAX) { >>>>> + pkt_hdr->input_flags.jumbo = 1; >>>>> + } >>>>> + >>>>> + /* Assume valid L2 header, no CRC/FCS check in SW */ >>>>> + pkt_hdr->input_flags.l2 = 1; >>>>> + >>>>> + eth = (odph_ethhdr_t *)odp_packet_map(pkt, &seglen); >>>>> + offset = sizeof(odph_ethhdr_t); >>>>> + parseptr = (uint8_t *)ð->type; >>>>> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); >>>>> + >>>>> + /* Parse the VLAN header(s), if present */ >>>>> + if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { >>>>> + pkt_hdr->input_flags.vlan_qinq = 1; >>>>> + pkt_hdr->input_flags.vlan = 1; >>>>> + vlan = (odph_vlanhdr_t *)(void *)parseptr; >>>>> + pkt_hdr->vlan_s_tag = ((ethtype << 16) | >>>>> + odp_be_to_cpu_16(vlan->tci)); >>>>> + offset += sizeof(odph_vlanhdr_t); >>>>> + parseptr += sizeof(odph_vlanhdr_t); >>>>> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >>>>> *)parseptr)); >>>>> + } >>>>> + >>>>> + if (ethtype == ODPH_ETHTYPE_VLAN) { >>>>> + pkt_hdr->input_flags.vlan = 1; >>>>> + vlan = (odph_vlanhdr_t *)(void *)parseptr; >>>>> + pkt_hdr->vlan_c_tag = ((ethtype << 16) | >>>>> + odp_be_to_cpu_16(vlan->tci)); >>>>> + offset += sizeof(odph_vlanhdr_t); >>>>> + parseptr += sizeof(odph_vlanhdr_t); >>>>> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >>>>> *)parseptr)); >>>>> + } >>>>> + >>>>> + /* Check for SNAP vs. DIX */ >>>>> + if (ethtype < ODPH_ETH_LEN_MAX) { >>>>> + pkt_hdr->input_flags.snap = 1; >>>>> + if (ethtype > pkt_hdr->frame_len - offset) { >>>>> + pkt_hdr->error_flags.snap_len = 1; >>>>> + goto parse_exit; >>>>> + } >>>>> + offset += 8; >>>>> + parseptr += 8; >>>>> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >>>>> *)parseptr)); >>>>> + } >>>>> + >>>>> + /* Consume Ethertype for Layer 3 parse */ >>>>> + parseptr += 2; >>>>> + >>>>> + /* Set l3_offset+flag only for known ethtypes */ >>>>> + pkt_hdr->input_flags.l3 = 1; >>>>> + pkt_hdr->l3_offset = offset; >>>>> + pkt_hdr->l3_protocol = ethtype; >>>>> + >>>>> + /* Parse Layer 3 headers */ >>>>> + switch (ethtype) { >>>>> + case ODPH_ETHTYPE_IPV4: >>>>> + pkt_hdr->input_flags.ipv4 = 1; >>>>> + ip_proto = parse_ipv4(pkt_hdr, &parseptr, &offset); >>>>> + break; >>>>> + >>>>> + case ODPH_ETHTYPE_IPV6: >>>>> + pkt_hdr->input_flags.ipv6 = 1; >>>>> + ip_proto = parse_ipv6(pkt_hdr, &parseptr, &offset); >>>>> + break; >>>>> + >>>>> + case ODPH_ETHTYPE_ARP: >>>>> + pkt_hdr->input_flags.arp = 1; >>>>> + ip_proto = 255; /* Reserved invalid by IANA */ >>>>> + break; >>>>> + >>>>> + default: >>>>> + pkt_hdr->input_flags.l3 = 0; >>>>> + ip_proto = 255; /* Reserved invalid by IANA */ >>>>> + } >>>>> + >>>>> + /* Set l4_offset+flag only for known ip_proto */ >>>>> + pkt_hdr->input_flags.l4 = 1; >>>>> + pkt_hdr->l4_offset = offset; >>>>> + pkt_hdr->l4_protocol = ip_proto; >>>>> + >>>>> + /* Parse Layer 4 headers */ >>>>> + switch (ip_proto) { >>>>> + case ODPH_IPPROTO_ICMP: >>>>> + pkt_hdr->input_flags.icmp = 1; >>>>> + break; >>>>> + >>>>> + case ODPH_IPPROTO_TCP: >>>>> + pkt_hdr->input_flags.tcp = 1; >>>>> + parse_tcp(pkt_hdr, &parseptr, &offset); >>>>> + break; >>>>> + >>>>> + case ODPH_IPPROTO_UDP: >>>>> + pkt_hdr->input_flags.udp = 1; >>>>> + parse_udp(pkt_hdr, &parseptr, &offset); >>>>> + break; >>>>> + >>>>> + case ODPH_IPPROTO_AH: >>>>> + case ODPH_IPPROTO_ESP: >>>>> + pkt_hdr->input_flags.ipsec = 1; >>>>> + break; >>>>> + >>>>> + default: >>>>> + pkt_hdr->input_flags.l4 = 0; >>>>> + break; >>>>> + } >>>>> + >>>>> + /* >>>>> + * Anything beyond what we parse here is considered payload. >>>>> + * Note: Payload is really only relevant for TCP and UDP. For >>>>> + * all other protocols, the payload offset will point to the >>>>> + * final header (ARP, ICMP, AH, ESP, or IP Fragment. >>>>> + */ >>>>> + pkt_hdr->payload_offset = offset; >>>>> + >>>>> +parse_exit: >>>>> + return pkt_hdr->error_flags.all != 0; >>>>> } >>>>> diff --git a/platform/linux-generic/odp_packet_flags.c >>>>> b/platform/linux-generic/odp_packet_flags.c >>>>> deleted file mode 100644 >>>>> index 06fdeed..0000000 >>>>> --- a/platform/linux-generic/odp_packet_flags.c >>>>> +++ /dev/null >>>>> @@ -1,202 +0,0 @@ >>>>> -/* Copyright (c) 2014, Linaro Limited >>>>> - * All rights reserved. >>>>> - * >>>>> - * SPDX-License-Identifier: BSD-3-Clause >>>>> - */ >>>>> - >>>>> -#include <odp_packet_flags.h> >>>>> -#include <odp_packet_internal.h> >>>>> - >>>>> - >>>>> -int odp_packet_error(odp_packet_t pkt) >>>>> -{ >>>>> - return (odp_packet_hdr(pkt)->error_flags.all != 0); >>>>> -} >>>>> - >>>>> -/* Get Error Flags */ >>>>> - >>>>> -int odp_packet_errflag_frame_len(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->error_flags.frame_len; >>>>> -} >>>>> - >>>>> -/* Get Input Flags */ >>>>> - >>>>> -int odp_packet_inflag_l2(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.l2; >>>>> -} >>>>> - >>>>> -int odp_packet_inflag_l3(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.l3; >>>>> -} >>>>> - >>>>> -int odp_packet_inflag_l4(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.l4; >>>>> -} >>>>> - >>>>> -int odp_packet_inflag_eth(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.eth; >>>>> -} >>>>> - >>>>> -int odp_packet_inflag_jumbo(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.jumbo; >>>>> -} >>>>> - >>>>> -int odp_packet_inflag_vlan(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.vlan; >>>>> -} >>>>> - >>>>> -int odp_packet_inflag_vlan_qinq(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.vlan_qinq; >>>>> -} >>>>> - >>>>> -int odp_packet_inflag_arp(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.arp; >>>>> -} >>>>> - >>>>> -int odp_packet_inflag_ipv4(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.ipv4; >>>>> -} >>>>> - >>>>> -int odp_packet_inflag_ipv6(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.ipv6; >>>>> -} >>>>> - >>>>> -int odp_packet_inflag_ipfrag(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.ipfrag; >>>>> -} >>>>> - >>>>> -int odp_packet_inflag_ipopt(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.ipopt; >>>>> -} >>>>> - >>>>> -int odp_packet_inflag_ipsec(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.ipsec; >>>>> -} >>>>> - >>>>> -int odp_packet_inflag_udp(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.udp; >>>>> -} >>>>> - >>>>> -int odp_packet_inflag_tcp(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.tcp; >>>>> -} >>>>> - >>>>> -int odp_packet_inflag_sctp(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.sctp; >>>>> -} >>>>> - >>>>> -int odp_packet_inflag_icmp(odp_packet_t pkt) >>>>> -{ >>>>> - return odp_packet_hdr(pkt)->input_flags.icmp; >>>>> -} >>>>> - >>>>> -/* Set Output Flags */ >>>>> - >>>>> -void odp_packet_outflag_l4_chksum(odp_packet_t pkt) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->output_flags.l4_chksum = 1; >>>>> -} >>>>> - >>>>> -/* Set Input Flags */ >>>>> - >>>>> -void odp_packet_set_inflag_l2(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.l2 = val; >>>>> -} >>>>> - >>>>> -void odp_packet_set_inflag_l3(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.l3 = val; >>>>> -} >>>>> - >>>>> -void odp_packet_set_inflag_l4(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.l4 = val; >>>>> -} >>>>> - >>>>> -void odp_packet_set_inflag_eth(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.eth = val; >>>>> -} >>>>> - >>>>> -void odp_packet_set_inflag_jumbo(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.jumbo = val; >>>>> -} >>>>> - >>>>> -void odp_packet_set_inflag_vlan(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.vlan = val; >>>>> -} >>>>> - >>>>> -void odp_packet_set_inflag_vlan_qinq(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.vlan_qinq = val; >>>>> -} >>>>> - >>>>> -void odp_packet_set_inflag_arp(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.arp = val; >>>>> -} >>>>> - >>>>> -void odp_packet_set_inflag_ipv4(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.ipv4 = val; >>>>> -} >>>>> - >>>>> -void odp_packet_set_inflag_ipv6(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.ipv6 = val; >>>>> -} >>>>> - >>>>> -void odp_packet_set_inflag_ipfrag(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.ipfrag = val; >>>>> -} >>>>> - >>>>> -void odp_packet_set_inflag_ipopt(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.ipopt = val; >>>>> -} >>>>> - >>>>> -void odp_packet_set_inflag_ipsec(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.ipsec = val; >>>>> -} >>>>> - >>>>> -void odp_packet_set_inflag_udp(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.udp = val; >>>>> -} >>>>> - >>>>> -void odp_packet_set_inflag_tcp(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.tcp = val; >>>>> -} >>>>> - >>>>> -void odp_packet_set_inflag_sctp(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.sctp = val; >>>>> -} >>>>> - >>>>> -void odp_packet_set_inflag_icmp(odp_packet_t pkt, int val) >>>>> -{ >>>>> - odp_packet_hdr(pkt)->input_flags.icmp = val; >>>>> -} >>>>> -- >>>>> 1.8.3.2 >>>>> >>>>> >>>>> _______________________________________________ >>>>> lng-odp mailing list >>>>> lng-odp@lists.linaro.org >>>>> http://lists.linaro.org/mailman/listinfo/lng-odp >>>>> >>>> >>>> >>> >> >
diff --git a/platform/linux-generic/odp_buffer.c b/platform/linux-generic/odp_buffer.c index e54e0e7..e47c722 100644 --- a/platform/linux-generic/odp_buffer.c +++ b/platform/linux-generic/odp_buffer.c @@ -5,46 +5,259 @@ */ #include <odp_buffer.h> -#include <odp_buffer_internal.h> #include <odp_buffer_pool_internal.h> +#include <odp_buffer_internal.h> +#include <odp_buffer_inlines.h> #include <string.h> #include <stdio.h> +void *odp_buffer_offset_map(odp_buffer_t buf, size_t offset, size_t *seglen) +{ + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); -void *odp_buffer_addr(odp_buffer_t buf) + if (offset > buf_hdr->size) + return NULL; + + return buffer_map(buf_hdr, offset, seglen, buf_hdr->size); +} + +void odp_buffer_offset_unmap(odp_buffer_t buf ODP_UNUSED, + size_t offset ODP_UNUSED) { - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); +} - return hdr->addr; +size_t odp_buffer_segment_count(odp_buffer_t buf) +{ + return odp_buf_to_hdr(buf)->segcount; } +int odp_buffer_is_segmented(odp_buffer_t buf) +{ + return odp_buf_to_hdr(buf)->segcount > 1; +} -size_t odp_buffer_size(odp_buffer_t buf) +odp_buffer_segment_t odp_buffer_segment_by_index(odp_buffer_t buf, + size_t ndx) { - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); + return buffer_segment(odp_buf_to_hdr(buf), ndx); +} - return hdr->size; +odp_buffer_segment_t odp_buffer_segment_next(odp_buffer_t buf, + odp_buffer_segment_t seg) +{ + return segment_next(odp_buf_to_hdr(buf), seg); } +void *odp_buffer_segment_map(odp_buffer_t buf, odp_buffer_segment_t seg, + size_t *seglen) +{ + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); -int odp_buffer_type(odp_buffer_t buf) + return segment_map(buf_hdr, seg, seglen, buf_hdr->size); +} + +void odp_buffer_segment_unmap(odp_buffer_segment_t seg ODP_UNUSED) { - odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); +} + +odp_buffer_t odp_buffer_split(odp_buffer_t buf, size_t offset) +{ + size_t size = odp_buffer_size(buf); + odp_buffer_pool_t pool = odp_buffer_pool(buf); + odp_buffer_t splitbuf; + size_t splitsize = size - offset; + + if (offset >= size) + return ODP_BUFFER_INVALID; + + splitbuf = buffer_alloc(pool, splitsize); + + if (splitbuf != ODP_BUFFER_INVALID) { + buffer_copy_to_buffer(splitbuf, 0, buf, splitsize, splitsize); + odp_buffer_trim(buf, splitsize); + } - return hdr->type; + return splitbuf; } +odp_buffer_t odp_buffer_join(odp_buffer_t buf1, odp_buffer_t buf2) +{ + size_t size1 = odp_buffer_size(buf1); + size_t size2 = odp_buffer_size(buf2); + odp_buffer_t joinbuf; + void *udata1, *udata_join; + size_t udata_size1, udata_size_join; -int odp_buffer_is_valid(odp_buffer_t buf) + joinbuf = buffer_alloc(odp_buffer_pool(buf1), size1 + size2); + + if (joinbuf != ODP_BUFFER_INVALID) { + buffer_copy_to_buffer(joinbuf, 0, buf1, 0, size1); + buffer_copy_to_buffer(joinbuf, size1, buf2, 0, size2); + + /* Copy user metadata if present */ + udata1 = odp_buffer_udata(buf1, &udata_size1); + udata_join = odp_buffer_udata(joinbuf, &udata_size_join); + + if (udata1 != NULL && udata_join != NULL) + memcpy(udata_join, udata1, + udata_size_join > udata_size1 ? + udata_size1 : udata_size_join); + + odp_buffer_free(buf1); + odp_buffer_free(buf2); + } + + return joinbuf; +} + +odp_buffer_t odp_buffer_trim(odp_buffer_t buf, size_t bytes) { - odp_buffer_bits_t handle; + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); + + if (bytes >= buf_hdr->size) + return ODP_BUFFER_INVALID; + + buf_hdr->size -= bytes; + size_t bufsegs = buf_hdr->size / buf_hdr->segsize; + + if (buf_hdr->size % buf_hdr->segsize > 0) + bufsegs++; + + pool_entry_t *pool = odp_pool_to_entry(buf_hdr->pool_hdl); - handle.u32 = buf; + /* Return excess segments back to block list */ + while (bufsegs > buf_hdr->segcount) + ret_blk(&pool->s, buf_hdr->addr[--buf_hdr->segcount]); - return (handle.index != ODP_BUFFER_INVALID_INDEX); + return buf; } +odp_buffer_t odp_buffer_extend(odp_buffer_t buf, size_t bytes) +{ + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); + + size_t lastseg = buf_hdr->size % buf_hdr->segsize; + + if (bytes <= buf_hdr->segsize - lastseg) { + buf_hdr->size += bytes; + return buf; + } + + pool_entry_t *pool = odp_pool_to_entry(buf_hdr->pool_hdl); + int extsize = buf_hdr->size + bytes; + + /* Extend buffer by adding additional segments to it */ + if (extsize > ODP_CONFIG_BUF_MAX_SIZE || pool->s.flags.unsegmented) + return ODP_BUFFER_INVALID; + + size_t segcount = buf_hdr->segcount; + size_t extsegs = extsize / buf_hdr->segsize; + + if (extsize % buf_hdr->segsize > 0) + extsize++; + + while (extsegs > buf_hdr->segcount) { + void *newblk = get_blk(&pool->s); + + /* If we fail to get all the blocks we need, back out */ + if (newblk == NULL) { + while (segcount > buf_hdr->segcount) + ret_blk(&pool->s, buf_hdr->addr[--segcount]); + return ODP_BUFFER_INVALID; + } + + buf_hdr->addr[segcount++] = newblk; + } + + buf_hdr->segcount = segcount; + buf_hdr->size = extsize; + + return buf; +} + +odp_buffer_t odp_buffer_clone(odp_buffer_t buf) +{ + return odp_buffer_copy(buf, odp_buf_to_hdr(buf)->pool_hdl); +} + +odp_buffer_t odp_buffer_copy(odp_buffer_t buf, odp_buffer_pool_t pool) +{ + odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); + size_t len = buf_hdr->size; + odp_buffer_t cpybuf = buffer_alloc(pool, len); + + if (cpybuf == ODP_BUFFER_INVALID) + return ODP_BUFFER_INVALID; + + if (buffer_copy_to_buffer(cpybuf, 0, buf, 0, len) == 0) + return cpybuf; + + odp_buffer_free(cpybuf); + return ODP_BUFFER_INVALID; +} + +int buffer_copy_to_buffer(odp_buffer_t dstbuf, size_t dstoffset, + odp_buffer_t srcbuf, size_t srcoffset, + size_t len) +{ + void *dstmap; + void *srcmap; + size_t cpylen, minseg, dstseglen, srcseglen; + + while (len > 0) { + dstmap = odp_buffer_offset_map(dstbuf, dstoffset, &dstseglen); + srcmap = odp_buffer_offset_map(srcbuf, srcoffset, &srcseglen); + if (dstmap == NULL || srcmap == NULL) + return -1; + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; + cpylen = len > minseg ? minseg : len; + memcpy(dstmap, srcmap, cpylen); + srcoffset += cpylen; + dstoffset += cpylen; + len -= cpylen; + } + + return 0; +} + +void *odp_buffer_addr(odp_buffer_t buf) +{ + return odp_buf_to_hdr(buf)->addr[0]; +} + +odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf) +{ + return odp_buf_to_hdr(buf)->pool_hdl; +} + +size_t odp_buffer_size(odp_buffer_t buf) +{ + return odp_buf_to_hdr(buf)->size; +} + +odp_buffer_type_e odp_buffer_type(odp_buffer_t buf) +{ + return odp_buf_to_hdr(buf)->type; +} + +void *odp_buffer_udata(odp_buffer_t buf, size_t *usize) +{ + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); + + *usize = hdr->udata_size; + return hdr->udata_addr; +} + +void *odp_buffer_udata_addr(odp_buffer_t buf) +{ + return odp_buf_to_hdr(buf)->udata_addr; +} + +int odp_buffer_is_valid(odp_buffer_t buf) +{ + return validate_buf(buf) != NULL; +} int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf) { @@ -63,27 +276,13 @@ int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf) len += snprintf(&str[len], n-len, " pool %i\n", hdr->pool_hdl); len += snprintf(&str[len], n-len, - " index %"PRIu32"\n", hdr->index); - len += snprintf(&str[len], n-len, - " phy_addr %"PRIu64"\n", hdr->phys_addr); - len += snprintf(&str[len], n-len, - " addr %p\n", hdr->addr); + " addr %p\n", hdr->addr[0]); len += snprintf(&str[len], n-len, " size %zu\n", hdr->size); len += snprintf(&str[len], n-len, - " cur_offset %zu\n", hdr->cur_offset); - len += snprintf(&str[len], n-len, " ref_count %i\n", hdr->ref_count); len += snprintf(&str[len], n-len, " type %i\n", hdr->type); - len += snprintf(&str[len], n-len, - " Scatter list\n"); - len += snprintf(&str[len], n-len, - " num_bufs %i\n", hdr->scatter.num_bufs); - len += snprintf(&str[len], n-len, - " pos %i\n", hdr->scatter.pos); - len += snprintf(&str[len], n-len, - " total_len %zu\n", hdr->scatter.total_len); return len; } @@ -100,9 +299,3 @@ void odp_buffer_print(odp_buffer_t buf) printf("\n%s\n", str); } - -void odp_buffer_copy_scatter(odp_buffer_t buf_dst, odp_buffer_t buf_src) -{ - (void)buf_dst; - (void)buf_src; -} diff --git a/platform/linux-generic/odp_buffer_pool.c b/platform/linux-generic/odp_buffer_pool.c index a48d7d6..f6161bb 100644 --- a/platform/linux-generic/odp_buffer_pool.c +++ b/platform/linux-generic/odp_buffer_pool.c @@ -6,8 +6,9 @@ #include <odp_std_types.h> #include <odp_buffer_pool.h> -#include <odp_buffer_pool_internal.h> #include <odp_buffer_internal.h> +#include <odp_buffer_pool_internal.h> +#include <odp_buffer_inlines.h> #include <odp_packet_internal.h> #include <odp_timer_internal.h> #include <odp_shared_memory.h> @@ -16,6 +17,7 @@ #include <odp_config.h> #include <odp_hints.h> #include <odp_debug.h> +#include <odph_eth.h> #include <string.h> #include <stdlib.h> @@ -33,36 +35,26 @@ #define LOCK_INIT(a) odp_spinlock_init(a) #endif - -#if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS -#error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS -#endif - -#define NULL_INDEX ((uint32_t)-1) - -union buffer_type_any_u { +typedef union buffer_type_any_u { odp_buffer_hdr_t buf; odp_packet_hdr_t pkt; odp_timeout_hdr_t tmo; -}; - -ODP_STATIC_ASSERT((sizeof(union buffer_type_any_u) % 8) == 0, - "BUFFER_TYPE_ANY_U__SIZE_ERR"); +} odp_anybuf_t; /* Any buffer type header */ typedef struct { union buffer_type_any_u any_hdr; /* any buffer type */ - uint8_t buf_data[]; /* start of buffer data area */ } odp_any_buffer_hdr_t; +typedef struct odp_any_hdr_stride { + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_any_buffer_hdr_t))]; +} odp_any_hdr_stride; -typedef union pool_entry_u { - struct pool_entry_s s; - - uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct pool_entry_s))]; - -} pool_entry_t; +#if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS +#error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS +#endif +#define NULL_INDEX ((uint32_t)-1) typedef struct pool_table_t { pool_entry_t pool[ODP_CONFIG_BUFFER_POOLS]; @@ -76,39 +68,6 @@ static pool_table_t *pool_tbl; /* Pool entry pointers (for inlining) */ void *pool_entry_ptr[ODP_CONFIG_BUFFER_POOLS]; - -static __thread odp_buffer_chunk_hdr_t *local_chunk[ODP_CONFIG_BUFFER_POOLS]; - - -static inline odp_buffer_pool_t pool_index_to_handle(uint32_t pool_id) -{ - return pool_id + 1; -} - - -static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl) -{ - return pool_hdl -1; -} - - -static inline void set_handle(odp_buffer_hdr_t *hdr, - pool_entry_t *pool, uint32_t index) -{ - odp_buffer_pool_t pool_hdl = pool->s.pool_hdl; - uint32_t pool_id = pool_handle_to_index(pool_hdl); - - if (pool_id >= ODP_CONFIG_BUFFER_POOLS) - ODP_ABORT("set_handle: Bad pool handle %u\n", pool_hdl); - - if (index > ODP_BUFFER_MAX_INDEX) - ODP_ERR("set_handle: Bad buffer index\n"); - - hdr->handle.pool_id = pool_id; - hdr->handle.index = index; -} - - int odp_buffer_pool_init_global(void) { uint32_t i; @@ -142,269 +101,167 @@ int odp_buffer_pool_init_global(void) return 0; } - -static odp_buffer_hdr_t *index_to_hdr(pool_entry_t *pool, uint32_t index) -{ - odp_buffer_hdr_t *hdr; - - hdr = (odp_buffer_hdr_t *)(pool->s.buf_base + index * pool->s.buf_size); - return hdr; -} - - -static void add_buf_index(odp_buffer_chunk_hdr_t *chunk_hdr, uint32_t index) -{ - uint32_t i = chunk_hdr->chunk.num_bufs; - chunk_hdr->chunk.buf_index[i] = index; - chunk_hdr->chunk.num_bufs++; -} - - -static uint32_t rem_buf_index(odp_buffer_chunk_hdr_t *chunk_hdr) +/** + * Buffer pool creation + */ +odp_buffer_pool_t odp_buffer_pool_create(const char *name, + odp_buffer_pool_param_t *args, + odp_buffer_pool_init_t *init_args) { - uint32_t index; + odp_buffer_pool_t pool_hdl = ODP_BUFFER_POOL_INVALID; + pool_entry_t *pool; uint32_t i; + odp_buffer_pool_param_t *params; + odp_buffer_pool_init_t *init_params; - i = chunk_hdr->chunk.num_bufs - 1; - index = chunk_hdr->chunk.buf_index[i]; - chunk_hdr->chunk.num_bufs--; - return index; -} - - -static odp_buffer_chunk_hdr_t *next_chunk(pool_entry_t *pool, - odp_buffer_chunk_hdr_t *chunk_hdr) -{ - uint32_t index; - - index = chunk_hdr->chunk.buf_index[ODP_BUFS_PER_CHUNK-1]; - if (index == NULL_INDEX) - return NULL; - else - return (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, index); -} - - -static odp_buffer_chunk_hdr_t *rem_chunk(pool_entry_t *pool) -{ - odp_buffer_chunk_hdr_t *chunk_hdr; - - chunk_hdr = pool->s.head; - if (chunk_hdr == NULL) { - /* Pool is empty */ - return NULL; - } - - pool->s.head = next_chunk(pool, chunk_hdr); - pool->s.free_bufs -= ODP_BUFS_PER_CHUNK; - - /* unlink */ - rem_buf_index(chunk_hdr); - return chunk_hdr; -} - - -static void add_chunk(pool_entry_t *pool, odp_buffer_chunk_hdr_t *chunk_hdr) -{ - if (pool->s.head) /* link pool head to the chunk */ - add_buf_index(chunk_hdr, pool->s.head->buf_hdr.index); - else - add_buf_index(chunk_hdr, NULL_INDEX); - - pool->s.head = chunk_hdr; - pool->s.free_bufs += ODP_BUFS_PER_CHUNK; -} + /* Default pool creation arguments */ + static odp_buffer_pool_param_t default_params = { + .buf_num = 1024, + .buf_size = ODP_CONFIG_BUF_SEG_SIZE, + .buf_type = ODP_BUFFER_TYPE_PACKET, + .buf_opts = ODP_BUFFER_OPTS_UNSEGMENTED, + }; + /* Default initialization paramters */ + static odp_buffer_pool_init_t default_init_params = { + .udata_size = 0, + .buf_init = NULL, + .buf_init_arg = NULL, + }; -static void check_align(pool_entry_t *pool, odp_buffer_hdr_t *hdr) -{ - if (!ODP_ALIGNED_CHECK_POWER_2(hdr->addr, pool->s.user_align)) { - ODP_ABORT("check_align: user data align error %p, align %zu\n", - hdr->addr, pool->s.user_align); - } + /* Handle NULL input parameters */ + params = args == NULL ? &default_params : args; + init_params = init_args == NULL ? &default_init_params + : init_args; - if (!ODP_ALIGNED_CHECK_POWER_2(hdr, ODP_CACHE_LINE_SIZE)) { - ODP_ABORT("check_align: hdr align error %p, align %i\n", - hdr, ODP_CACHE_LINE_SIZE); - } -} + if (params->buf_type != ODP_BUFFER_TYPE_PACKET) + params->buf_opts |= ODP_BUFFER_OPTS_UNSEGMENTED; + int unsegmented = ((params->buf_opts & ODP_BUFFER_OPTS_UNSEGMENTED) == + ODP_BUFFER_OPTS_UNSEGMENTED); -static void fill_hdr(void *ptr, pool_entry_t *pool, uint32_t index, - int buf_type) -{ - odp_buffer_hdr_t *hdr = (odp_buffer_hdr_t *)ptr; - size_t size = pool->s.hdr_size; - uint8_t *buf_data; + uint32_t udata_stride = + ODP_CACHE_LINE_SIZE_ROUNDUP(init_params->udata_size); - if (buf_type == ODP_BUFFER_TYPE_CHUNK) - size = sizeof(odp_buffer_chunk_hdr_t); - - switch (pool->s.buf_type) { - odp_raw_buffer_hdr_t *raw_hdr; - odp_packet_hdr_t *packet_hdr; - odp_timeout_hdr_t *tmo_hdr; - odp_any_buffer_hdr_t *any_hdr; + uint32_t blk_size, buf_stride; + switch (params->buf_type) { case ODP_BUFFER_TYPE_RAW: - raw_hdr = ptr; - buf_data = raw_hdr->buf_data; + blk_size = params->buf_size; + buf_stride = sizeof(odp_buffer_hdr_stride); break; + case ODP_BUFFER_TYPE_PACKET: - packet_hdr = ptr; - buf_data = packet_hdr->buf_data; + if (unsegmented) + blk_size = + ODP_CACHE_LINE_SIZE_ROUNDUP(params->buf_size); + else + blk_size = ODP_ALIGN_ROUNDUP(params->buf_size, + ODP_CONFIG_BUF_SEG_SIZE); + buf_stride = sizeof(odp_packet_hdr_stride); break; + case ODP_BUFFER_TYPE_TIMEOUT: - tmo_hdr = ptr; - buf_data = tmo_hdr->buf_data; + blk_size = 0; /* Timeouts have no block data, only metadata */ + buf_stride = sizeof(odp_timeout_hdr_stride); break; + case ODP_BUFFER_TYPE_ANY: - any_hdr = ptr; - buf_data = any_hdr->buf_data; + if (unsegmented) + blk_size = + ODP_CACHE_LINE_SIZE_ROUNDUP(params->buf_size); + else + blk_size = ODP_ALIGN_ROUNDUP(params->buf_size, + ODP_CONFIG_BUF_SEG_SIZE); + buf_stride = sizeof(odp_any_hdr_stride); break; - default: - ODP_ABORT("Bad buffer type\n"); - } - - memset(hdr, 0, size); - - set_handle(hdr, pool, index); - - hdr->addr = &buf_data[pool->s.buf_offset - pool->s.hdr_size]; - hdr->index = index; - hdr->size = pool->s.user_size; - hdr->pool_hdl = pool->s.pool_hdl; - hdr->type = buf_type; - - check_align(pool, hdr); -} - - -static void link_bufs(pool_entry_t *pool) -{ - odp_buffer_chunk_hdr_t *chunk_hdr; - size_t hdr_size; - size_t data_size; - size_t data_align; - size_t tot_size; - size_t offset; - size_t min_size; - uint64_t pool_size; - uintptr_t buf_base; - uint32_t index; - uintptr_t pool_base; - int buf_type; - - buf_type = pool->s.buf_type; - data_size = pool->s.user_size; - data_align = pool->s.user_align; - pool_size = pool->s.pool_size; - pool_base = (uintptr_t) pool->s.pool_base_addr; - - if (buf_type == ODP_BUFFER_TYPE_RAW) { - hdr_size = sizeof(odp_raw_buffer_hdr_t); - } else if (buf_type == ODP_BUFFER_TYPE_PACKET) { - hdr_size = sizeof(odp_packet_hdr_t); - } else if (buf_type == ODP_BUFFER_TYPE_TIMEOUT) { - hdr_size = sizeof(odp_timeout_hdr_t); - } else if (buf_type == ODP_BUFFER_TYPE_ANY) { - hdr_size = sizeof(odp_any_buffer_hdr_t); - } else - ODP_ABORT("odp_buffer_pool_create: Bad type %i\n", buf_type); - - - /* Chunk must fit into buffer data area.*/ - min_size = sizeof(odp_buffer_chunk_hdr_t) - hdr_size; - if (data_size < min_size) - data_size = min_size; - - /* Roundup data size to full cachelines */ - data_size = ODP_CACHE_LINE_SIZE_ROUNDUP(data_size); - - /* Min cacheline alignment for buffer header and data */ - data_align = ODP_CACHE_LINE_SIZE_ROUNDUP(data_align); - offset = ODP_CACHE_LINE_SIZE_ROUNDUP(hdr_size); - - /* Multiples of cacheline size */ - if (data_size > data_align) - tot_size = data_size + offset; - else - tot_size = data_align + offset; - - /* First buffer */ - buf_base = ODP_ALIGN_ROUNDUP(pool_base + offset, data_align) - offset; - - pool->s.hdr_size = hdr_size; - pool->s.buf_base = buf_base; - pool->s.buf_size = tot_size; - pool->s.buf_offset = offset; - index = 0; - - chunk_hdr = (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, index); - pool->s.head = NULL; - pool_size -= buf_base - pool_base; - - while (pool_size > ODP_BUFS_PER_CHUNK * tot_size) { - int i; - - fill_hdr(chunk_hdr, pool, index, ODP_BUFFER_TYPE_CHUNK); - - index++; - - for (i = 0; i < ODP_BUFS_PER_CHUNK - 1; i++) { - odp_buffer_hdr_t *hdr = index_to_hdr(pool, index); - - fill_hdr(hdr, pool, index, buf_type); - - add_buf_index(chunk_hdr, index); - index++; - } - - add_chunk(pool, chunk_hdr); - chunk_hdr = (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, - index); - pool->s.num_bufs += ODP_BUFS_PER_CHUNK; - pool_size -= ODP_BUFS_PER_CHUNK * tot_size; + default: + return ODP_BUFFER_POOL_INVALID; } -} - - -odp_buffer_pool_t odp_buffer_pool_create(const char *name, - void *base_addr, uint64_t size, - size_t buf_size, size_t buf_align, - int buf_type) -{ - odp_buffer_pool_t pool_hdl = ODP_BUFFER_POOL_INVALID; - pool_entry_t *pool; - uint32_t i; for (i = 0; i < ODP_CONFIG_BUFFER_POOLS; i++) { pool = get_pool_entry(i); LOCK(&pool->s.lock); + if (pool->s.shm != ODP_SHM_INVALID) { + UNLOCK(&pool->s.lock); + continue; + } - if (pool->s.buf_base == 0) { - /* found free pool */ + /* found free pool */ + size_t block_size, mdata_size, udata_size; - strncpy(pool->s.name, name, - ODP_BUFFER_POOL_NAME_LEN - 1); - pool->s.name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; - pool->s.pool_base_addr = base_addr; - pool->s.pool_size = size; - pool->s.user_size = buf_size; - pool->s.user_align = buf_align; - pool->s.buf_type = buf_type; + strncpy(pool->s.name, name, + ODP_BUFFER_POOL_NAME_LEN - 1); + pool->s.name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; - link_bufs(pool); + pool->s.params = *params; + pool->s.init_params = *init_params; - UNLOCK(&pool->s.lock); + mdata_size = params->buf_num * buf_stride; + udata_size = params->buf_num * udata_stride; + block_size = params->buf_num * blk_size; + + pool->s.pool_size = ODP_PAGE_SIZE_ROUNDUP(mdata_size + + udata_size + + block_size); - pool_hdl = pool->s.pool_hdl; - break; + pool->s.shm = odp_shm_reserve(pool->s.name, pool->s.pool_size, + ODP_PAGE_SIZE, 0); + if (pool->s.shm == ODP_SHM_INVALID) { + UNLOCK(&pool->s.lock); + return ODP_BUFFER_POOL_INVALID; } + pool->s.pool_base_addr = (uint8_t *)odp_shm_addr(pool->s.shm); + pool->s.flags.unsegmented = unsegmented; + pool->s.seg_size = unsegmented ? + blk_size : ODP_CONFIG_BUF_SEG_SIZE; + uint8_t *udata_base_addr = pool->s.pool_base_addr + mdata_size; + uint8_t *block_base_addr = udata_base_addr + udata_size; + + pool->s.bufcount = 0; + pool->s.buf_freelist = NULL; + pool->s.blk_freelist = NULL; + + uint8_t *buf = pool->s.udata_base_addr - buf_stride; + uint8_t *udat = (udata_stride == 0) ? NULL : + block_base_addr - udata_stride; + + /* Init buffer common header and add to pool buffer freelist */ + do { + odp_buffer_hdr_t *tmp = + (odp_buffer_hdr_t *)(void *)buf; + tmp->pool_hdl = pool->s.pool_hdl; + tmp->size = 0; + tmp->type = params->buf_type; + tmp->udata_addr = (void *)udat; + tmp->udata_size = init_params->udata_size; + tmp->segcount = 0; + tmp->segsize = pool->s.seg_size; + tmp->buf_hdl.handle = + odp_buffer_encode_handle((odp_buffer_hdr_t *) + (void *)buf); + ret_buf(&pool->s, tmp); + buf -= buf_stride; + udat -= udata_stride; + } while (buf >= pool->s.pool_base_addr); + + /* Form block freelist for pool */ + uint8_t *blk = pool->s.pool_base_addr + pool->s.pool_size - + pool->s.seg_size; + + if (blk_size > 0) + do { + ret_blk(&pool->s, blk); + blk -= pool->s.seg_size; + } while (blk >= block_base_addr); + UNLOCK(&pool->s.lock); + + pool_hdl = pool->s.pool_hdl; + break; } return pool_hdl; @@ -431,93 +288,172 @@ odp_buffer_pool_t odp_buffer_pool_lookup(const char *name) return ODP_BUFFER_POOL_INVALID; } - -odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_hdl) +odp_buffer_pool_t odp_buffer_pool_next(odp_buffer_pool_t pool_hdl, + char *name, + size_t *udata_size, + odp_buffer_pool_param_t *params, + int *predef) { - pool_entry_t *pool; - odp_buffer_chunk_hdr_t *chunk; - odp_buffer_bits_t handle; - uint32_t pool_id = pool_handle_to_index(pool_hdl); - - pool = get_pool_entry(pool_id); - chunk = local_chunk[pool_id]; - - if (chunk == NULL) { - LOCK(&pool->s.lock); - chunk = rem_chunk(pool); - UNLOCK(&pool->s.lock); + pool_entry_t *next_pool; + uint32_t pool_id; - if (chunk == NULL) - return ODP_BUFFER_INVALID; + /* NULL input means first element */ + if (pool_hdl == ODP_BUFFER_POOL_INVALID) { + pool_id = 0; + next_pool = get_pool_entry(pool_id); + } else { + pool_id = pool_handle_to_index(pool_hdl); - local_chunk[pool_id] = chunk; + if (pool_id == ODP_CONFIG_BUFFER_POOLS) + return ODP_BUFFER_POOL_INVALID; + else + next_pool = get_pool_entry(++pool_id); } - if (chunk->chunk.num_bufs == 0) { - /* give the chunk buffer */ - local_chunk[pool_id] = NULL; - chunk->buf_hdr.type = pool->s.buf_type; + /* Only interested in pools that exist */ + while (next_pool->s.shm == ODP_SHM_INVALID) { + if (pool_id == ODP_CONFIG_BUFFER_POOLS) + return ODP_BUFFER_POOL_INVALID; + else + next_pool = get_pool_entry(++pool_id); + } - handle = chunk->buf_hdr.handle; - } else { - odp_buffer_hdr_t *hdr; - uint32_t index; - index = rem_buf_index(chunk); - hdr = index_to_hdr(pool, index); + /* Found the next pool, so return info about it */ + strncpy(name, next_pool->s.name, ODP_BUFFER_POOL_NAME_LEN - 1); + name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0; - handle = hdr->handle; - } + *udata_size = next_pool->s.init_params.udata_size; + *params = next_pool->s.params; + *predef = next_pool->s.flags.predefined; - return handle.u32; + return next_pool->s.pool_hdl; } - -void odp_buffer_free(odp_buffer_t buf) +int odp_buffer_pool_destroy(odp_buffer_pool_t pool_hdl) { - odp_buffer_hdr_t *hdr; - uint32_t pool_id; - pool_entry_t *pool; - odp_buffer_chunk_hdr_t *chunk_hdr; + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); - hdr = odp_buf_to_hdr(buf); - pool_id = pool_handle_to_index(hdr->pool_hdl); - pool = get_pool_entry(pool_id); - chunk_hdr = local_chunk[pool_id]; + if (pool == NULL) + return -1; - if (chunk_hdr && chunk_hdr->chunk.num_bufs == ODP_BUFS_PER_CHUNK - 1) { - /* Current chunk is full. Push back to the pool */ - LOCK(&pool->s.lock); - add_chunk(pool, chunk_hdr); + LOCK(&pool->s.lock); + + if (pool->s.shm == ODP_SHM_INVALID || + pool->s.bufcount > 0 || + pool->s.flags.predefined) { UNLOCK(&pool->s.lock); - chunk_hdr = NULL; + return -1; } - if (chunk_hdr == NULL) { - /* Use this buffer */ - chunk_hdr = (odp_buffer_chunk_hdr_t *)hdr; - local_chunk[pool_id] = chunk_hdr; - chunk_hdr->chunk.num_bufs = 0; + odp_shm_free(pool->s.shm); + + pool->s.shm = ODP_SHM_INVALID; + UNLOCK(&pool->s.lock); + + return 0; +} + +size_t odp_buffer_pool_headroom(odp_buffer_pool_t pool_hdl) +{ + return odp_pool_to_entry(pool_hdl)->s.headroom; +} + +int odp_buffer_pool_set_headroom(odp_buffer_pool_t pool_hdl, size_t hr) +{ + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); + + if (hr >= pool->s.seg_size/2) { + return -1; } else { - /* Add to current chunk */ - add_buf_index(chunk_hdr, hdr->index); + pool->s.headroom = hr; + return 0; } } +size_t odp_buffer_pool_tailroom(odp_buffer_pool_t pool_hdl) +{ + return odp_pool_to_entry(pool_hdl)->s.tailroom; +} -odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf) +int odp_buffer_pool_set_tailroom(odp_buffer_pool_t pool_hdl, size_t tr) { - odp_buffer_hdr_t *hdr; + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); - hdr = odp_buf_to_hdr(buf); - return hdr->pool_hdl; + if (tr >= pool->s.seg_size/2) { + return -1; + } else { + pool->s.tailroom = tr; + return 0; + } } +odp_buffer_t buffer_alloc(odp_buffer_pool_t pool_hdl, size_t size) +{ + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); + size_t totsize = pool->s.headroom + size + pool->s.tailroom; + odp_anybuf_t *buf; + uint8_t *blk; + + if ((pool->s.flags.unsegmented && totsize > pool->s.seg_size) || + (!pool->s.flags.unsegmented && totsize > ODP_CONFIG_BUF_MAX_SIZE)) + return ODP_BUFFER_INVALID; + + buf = (odp_anybuf_t *)(void *)get_buf(&pool->s); + + if (buf == NULL) + return ODP_BUFFER_INVALID; + + /* Get blocks for this buffer, if pool uses application data */ + if (buf->buf.segsize > 0) + do { + blk = get_blk(&pool->s); + if (blk == NULL) { + ret_buf(&pool->s, &buf->buf); + return ODP_BUFFER_INVALID; + } + buf->buf.addr[buf->buf.segcount++] = blk; + totsize = totsize < pool->s.seg_size ? 0 : + totsize - pool->s.seg_size; + } while (totsize > 0); + + switch (buf->buf.type) { + case ODP_BUFFER_TYPE_RAW: + break; + + case ODP_BUFFER_TYPE_PACKET: + packet_init(pool, &buf->pkt, size); + if (pool->s.init_params.buf_init != NULL) + (*pool->s.init_params.buf_init) + (buf->buf.buf_hdl.handle, + pool->s.init_params.buf_init_arg); + break; + + case ODP_BUFFER_TYPE_TIMEOUT: + break; + + default: + ret_buf(&pool->s, &buf->buf); + return ODP_BUFFER_INVALID; + } + + return odp_hdr_to_buf(&buf->buf); +} + +odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_hdl) +{ + return buffer_alloc(pool_hdl, 0); +} + +void odp_buffer_free(odp_buffer_t buf) +{ + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); + pool_entry_t *pool = odp_buf_to_pool(hdr); + ret_buf(&pool->s, hdr); +} void odp_buffer_pool_print(odp_buffer_pool_t pool_hdl) { pool_entry_t *pool; - odp_buffer_chunk_hdr_t *chunk_hdr; - uint32_t i; uint32_t pool_id; pool_id = pool_handle_to_index(pool_hdl); @@ -528,47 +464,4 @@ void odp_buffer_pool_print(odp_buffer_pool_t pool_hdl) printf(" pool %i\n", pool->s.pool_hdl); printf(" name %s\n", pool->s.name); printf(" pool base %p\n", pool->s.pool_base_addr); - printf(" buf base 0x%"PRIxPTR"\n", pool->s.buf_base); - printf(" pool size 0x%"PRIx64"\n", pool->s.pool_size); - printf(" buf size %zu\n", pool->s.user_size); - printf(" buf align %zu\n", pool->s.user_align); - printf(" hdr size %zu\n", pool->s.hdr_size); - printf(" alloc size %zu\n", pool->s.buf_size); - printf(" offset to hdr %zu\n", pool->s.buf_offset); - printf(" num bufs %"PRIu64"\n", pool->s.num_bufs); - printf(" free bufs %"PRIu64"\n", pool->s.free_bufs); - - /* first chunk */ - chunk_hdr = pool->s.head; - - if (chunk_hdr == NULL) { - ODP_ERR(" POOL EMPTY\n"); - return; - } - - printf("\n First chunk\n"); - - for (i = 0; i < chunk_hdr->chunk.num_bufs - 1; i++) { - uint32_t index; - odp_buffer_hdr_t *hdr; - - index = chunk_hdr->chunk.buf_index[i]; - hdr = index_to_hdr(pool, index); - - printf(" [%i] addr %p, id %"PRIu32"\n", i, hdr->addr, index); - } - - printf(" [%i] addr %p, id %"PRIu32"\n", i, chunk_hdr->buf_hdr.addr, - chunk_hdr->buf_hdr.index); - - /* next chunk */ - chunk_hdr = next_chunk(pool, chunk_hdr); - - if (chunk_hdr) { - printf(" Next chunk\n"); - printf(" addr %p, id %"PRIu32"\n", chunk_hdr->buf_hdr.addr, - chunk_hdr->buf_hdr.index); - } - - printf("\n"); } diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 82ea879..45ac8e5 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -11,29 +11,31 @@ #include <odph_eth.h> #include <odph_ip.h> +#include <odph_tcp.h> +#include <odph_udp.h> #include <string.h> #include <stdio.h> -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, - odph_ipv4hdr_t *ipv4, size_t *offset_out); -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, - odph_ipv6hdr_t *ipv6, size_t *offset_out); - void odp_packet_init(odp_packet_t pkt) { odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); + pool_entry_t *pool = odp_buf_to_pool(&pkt_hdr->buf_hdr); const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, buf_hdr); uint8_t *start; size_t len; start = (uint8_t *)pkt_hdr + start_offset; - len = ODP_OFFSETOF(odp_packet_hdr_t, buf_data) - start_offset; + len = sizeof(odp_packet_hdr_t) - start_offset; memset(start, 0, len); pkt_hdr->l2_offset = ODP_PACKET_OFFSET_INVALID; pkt_hdr->l3_offset = ODP_PACKET_OFFSET_INVALID; pkt_hdr->l4_offset = ODP_PACKET_OFFSET_INVALID; + pkt_hdr->payload_offset = ODP_PACKET_OFFSET_INVALID; + + pkt_hdr->headroom = pool->s.headroom; + pkt_hdr->tailroom = pool->s.tailroom; } odp_packet_t odp_packet_from_buffer(odp_buffer_t buf) @@ -46,55 +48,112 @@ odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt) return (odp_buffer_t)pkt; } -void odp_packet_set_len(odp_packet_t pkt, size_t len) +size_t odp_packet_len(odp_packet_t pkt) { - odp_packet_hdr(pkt)->frame_len = len; + return odp_packet_hdr(pkt)->frame_len; } -size_t odp_packet_get_len(odp_packet_t pkt) +void *odp_packet_offset_map(odp_packet_t pkt, size_t offset, size_t *seglen) { - return odp_packet_hdr(pkt)->frame_len; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (offset >= pkt_hdr->frame_len) + return NULL; + + return buffer_map(&pkt_hdr->buf_hdr, + pkt_hdr->headroom + offset, + seglen, pkt_hdr->frame_len); } -uint8_t *odp_packet_addr(odp_packet_t pkt) +void odp_packet_offset_unmap(odp_packet_t pkt ODP_UNUSED, + size_t offset ODP_UNUSED) { - return odp_buffer_addr(odp_packet_to_buffer(pkt)); } -uint8_t *odp_packet_data(odp_packet_t pkt) +void *odp_packet_map(odp_packet_t pkt, size_t *seglen) { - return odp_packet_addr(pkt) + odp_packet_hdr(pkt)->frame_offset; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + return buffer_map(&pkt_hdr->buf_hdr, + 0, seglen, pkt_hdr->frame_len); } +void *odp_packet_addr(odp_packet_t pkt) +{ + size_t seglen; + return odp_packet_map(pkt, &seglen); +} -uint8_t *odp_packet_l2(odp_packet_t pkt) +odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt) { - const size_t offset = odp_packet_l2_offset(pkt); + return odp_packet_hdr(pkt)->buf_hdr.pool_hdl; +} - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) - return NULL; +odp_packet_segment_t odp_packet_segment_by_index(odp_packet_t pkt, + size_t ndx) +{ - return odp_packet_addr(pkt) + offset; + return (odp_packet_segment_t) + buffer_segment(&odp_packet_hdr(pkt)->buf_hdr, ndx); } -size_t odp_packet_l2_offset(odp_packet_t pkt) +odp_packet_segment_t odp_packet_segment_next(odp_packet_t pkt, + odp_packet_segment_t seg) { - return odp_packet_hdr(pkt)->l2_offset; + return (odp_packet_segment_t) + segment_next(&odp_packet_hdr(pkt)->buf_hdr, seg); } -void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset) +void *odp_packet_segment_map(odp_packet_t pkt, odp_packet_segment_t seg, + size_t *seglen) { - odp_packet_hdr(pkt)->l2_offset = offset; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + return segment_map(&pkt_hdr->buf_hdr, seg, + seglen, pkt_hdr->frame_len); } -uint8_t *odp_packet_l3(odp_packet_t pkt) +void odp_packet_segment_unmap(odp_packet_segment_t seg ODP_UNUSED) { - const size_t offset = odp_packet_l3_offset(pkt); +} - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) - return NULL; +void *odp_packet_udata(odp_packet_t pkt, size_t *len) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + *len = pkt_hdr->buf_hdr.udata_size; + return pkt_hdr->buf_hdr.udata_addr; +} + +void *odp_packet_udata_addr(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->buf_hdr.udata_addr; +} + +void *odp_packet_l2_map(odp_packet_t pkt, size_t *seglen) +{ + return odp_packet_offset_map(pkt, odp_packet_l2_offset(pkt), seglen); +} + +size_t odp_packet_l2_offset(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->l2_offset; +} + +int odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset) +{ + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); + + if (offset >= hdr->frame_len) + return -1; + + hdr->l2_offset = offset; + return 0; +} - return odp_packet_addr(pkt) + offset; +void *odp_packet_l3_map(odp_packet_t pkt, size_t *seglen) +{ + return odp_packet_offset_map(pkt, odp_packet_l3_offset(pkt), seglen); } size_t odp_packet_l3_offset(odp_packet_t pkt) @@ -102,19 +161,35 @@ size_t odp_packet_l3_offset(odp_packet_t pkt) return odp_packet_hdr(pkt)->l3_offset; } -void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset) +int odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset) { - odp_packet_hdr(pkt)->l3_offset = offset; + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); + + if (offset >= hdr->frame_len) + return -1; + + hdr->l3_offset = offset; + return 0; } -uint8_t *odp_packet_l4(odp_packet_t pkt) +uint32_t odp_packet_l3_protocol(odp_packet_t pkt) { - const size_t offset = odp_packet_l4_offset(pkt); + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) - return NULL; + if (hdr->input_flags.l3) + return hdr->l3_protocol; + else + return -1; +} + +void odp_packet_set_l3_protocol(odp_packet_t pkt, uint16_t protocol) +{ + odp_packet_hdr(pkt)->l3_protocol = protocol; +} - return odp_packet_addr(pkt) + offset; +void *odp_packet_l4_map(odp_packet_t pkt, size_t *seglen) +{ + return odp_packet_offset_map(pkt, odp_packet_l4_offset(pkt), seglen); } size_t odp_packet_l4_offset(odp_packet_t pkt) @@ -122,277 +197,902 @@ size_t odp_packet_l4_offset(odp_packet_t pkt) return odp_packet_hdr(pkt)->l4_offset; } -void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset) +int odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset) { - odp_packet_hdr(pkt)->l4_offset = offset; -} + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); + + if (offset >= hdr->frame_len) + return -1; + hdr->l4_offset = offset; + return 0; +} -int odp_packet_is_segmented(odp_packet_t pkt) +uint32_t odp_packet_l4_protocol(odp_packet_t pkt) { - odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr((odp_buffer_t)pkt); + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); - if (buf_hdr->scatter.num_bufs == 0) - return 0; + if (hdr->input_flags.l4) + return hdr->l4_protocol; else - return 1; + return -1; } +void odp_packet_set_l4_protocol(odp_packet_t pkt, uint8_t protocol) +{ + odp_packet_hdr(pkt)->l4_protocol = protocol; +} -int odp_packet_seg_count(odp_packet_t pkt) +void *odp_packet_payload_map(odp_packet_t pkt, size_t *seglen) { - odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr((odp_buffer_t)pkt); + return odp_packet_offset_map(pkt, odp_packet_payload_offset(pkt), + seglen); +} - return (int)buf_hdr->scatter.num_bufs + 1; +size_t odp_packet_payload_offset(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->payload_offset; } +int odp_packet_set_payload_offset(odp_packet_t pkt, size_t offset) +{ + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); -/** - * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP - * - * Internal function: caller is resposible for passing only valid packet handles - * , lengths and offsets (usually done&called in packet input). - * - * @param pkt Packet handle - * @param len Packet length in bytes - * @param frame_offset Byte offset to L2 header - */ -void odp_packet_parse(odp_packet_t pkt, size_t len, size_t frame_offset) + if (offset >= hdr->frame_len) + return -1; + + hdr->payload_offset = offset; + return 0; +} + +int odp_packet_error(odp_packet_t pkt) { - odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); - odph_ethhdr_t *eth; - odph_vlanhdr_t *vlan; - odph_ipv4hdr_t *ipv4; - odph_ipv6hdr_t *ipv6; - uint16_t ethtype; - size_t offset = 0; - uint8_t ip_proto = 0; + return odp_packet_hdr(pkt)->error_flags.all != 0; +} - pkt_hdr->input_flags.eth = 1; - pkt_hdr->frame_offset = frame_offset; - pkt_hdr->frame_len = len; +void odp_packet_set_error(odp_packet_t pkt, int val) +{ + odp_packet_hdr(pkt)->error_flags.app_error = val; +} - if (odp_unlikely(len < ODPH_ETH_LEN_MIN)) { - pkt_hdr->error_flags.frame_len = 1; - return; - } else if (len > ODPH_ETH_LEN_MAX) { - pkt_hdr->input_flags.jumbo = 1; - } +int odp_packet_inflag_l2(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->input_flags.l2; +} - /* Assume valid L2 header, no CRC/FCS check in SW */ - pkt_hdr->input_flags.l2 = 1; - pkt_hdr->l2_offset = frame_offset; +void odp_packet_set_inflag_l2(odp_packet_t pkt, int val) +{ + odp_packet_hdr(pkt)->input_flags.l2 = val; +} - eth = (odph_ethhdr_t *)odp_packet_data(pkt); - ethtype = odp_be_to_cpu_16(eth->type); - vlan = (odph_vlanhdr_t *)ð->type; +int odp_packet_inflag_l3(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->input_flags.l3; +} - if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { - pkt_hdr->input_flags.vlan_qinq = 1; - ethtype = odp_be_to_cpu_16(vlan->tpid); - offset += sizeof(odph_vlanhdr_t); - vlan = &vlan[1]; - } +void odp_packet_set_inflag_l3(odp_packet_t pkt, int val) +{ + odp_packet_hdr(pkt)->input_flags.l3 = val; +} - if (ethtype == ODPH_ETHTYPE_VLAN) { - pkt_hdr->input_flags.vlan = 1; - ethtype = odp_be_to_cpu_16(vlan->tpid); - offset += sizeof(odph_vlanhdr_t); - } +int odp_packet_inflag_l4(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->input_flags.l4; +} - /* Set l3_offset+flag only for known ethtypes */ - switch (ethtype) { - case ODPH_ETHTYPE_IPV4: - pkt_hdr->input_flags.ipv4 = 1; - pkt_hdr->input_flags.l3 = 1; - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + offset; - ipv4 = (odph_ipv4hdr_t *)odp_packet_l3(pkt); - ip_proto = parse_ipv4(pkt_hdr, ipv4, &offset); - break; - case ODPH_ETHTYPE_IPV6: - pkt_hdr->input_flags.ipv6 = 1; - pkt_hdr->input_flags.l3 = 1; - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + offset; - ipv6 = (odph_ipv6hdr_t *)odp_packet_l3(pkt); - ip_proto = parse_ipv6(pkt_hdr, ipv6, &offset); - break; - case ODPH_ETHTYPE_ARP: - pkt_hdr->input_flags.arp = 1; - /* fall through */ - default: - ip_proto = 0; - break; - } +void odp_packet_set_inflag_l4(odp_packet_t pkt, int val) +{ + odp_packet_hdr(pkt)->input_flags.l4 = val; +} - switch (ip_proto) { - case ODPH_IPPROTO_UDP: - pkt_hdr->input_flags.udp = 1; - pkt_hdr->input_flags.l4 = 1; - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; - break; - case ODPH_IPPROTO_TCP: - pkt_hdr->input_flags.tcp = 1; - pkt_hdr->input_flags.l4 = 1; - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; - break; - case ODPH_IPPROTO_SCTP: - pkt_hdr->input_flags.sctp = 1; - pkt_hdr->input_flags.l4 = 1; - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; - break; - case ODPH_IPPROTO_ICMP: - pkt_hdr->input_flags.icmp = 1; - pkt_hdr->input_flags.l4 = 1; - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; - break; - default: - /* 0 or unhandled IP protocols, don't set L4 flag+offset */ - if (pkt_hdr->input_flags.ipv6) { - /* IPv6 next_hdr is not L4, mark as IP-option instead */ - pkt_hdr->input_flags.ipopt = 1; - } - break; - } +int odp_packet_inflag_eth(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->input_flags.eth; } -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, - odph_ipv4hdr_t *ipv4, size_t *offset_out) +void odp_packet_set_inflag_eth(odp_packet_t pkt, int val) { - uint8_t ihl; - uint16_t frag_offset; + odp_packet_hdr(pkt)->input_flags.eth = val; +} - ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); - if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN)) { - pkt_hdr->error_flags.ip_err = 1; - return 0; - } +int odp_packet_inflag_jumbo(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->input_flags.jumbo; +} - if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) { - pkt_hdr->input_flags.ipopt = 1; - return 0; - } +void odp_packet_set_inflag_jumbo(odp_packet_t pkt, int val) +{ + odp_packet_hdr(pkt)->input_flags.jumbo = val; +} - /* A packet is a fragment if: - * "more fragments" flag is set (all fragments except the last) - * OR - * "fragment offset" field is nonzero (all fragments except the first) - */ - frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); - if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) { - pkt_hdr->input_flags.ipfrag = 1; - return 0; - } +int odp_packet_inflag_vlan(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->input_flags.vlan; +} - if (ipv4->proto == ODPH_IPPROTO_ESP || - ipv4->proto == ODPH_IPPROTO_AH) { - pkt_hdr->input_flags.ipsec = 1; - return 0; - } +void odp_packet_set_inflag_vlan(odp_packet_t pkt, int val) +{ + odp_packet_hdr(pkt)->input_flags.vlan = val; +} - /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after return */ +int odp_packet_inflag_vlan_qinq(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->input_flags.vlan_qinq; +} - *offset_out = sizeof(uint32_t) * ihl; - return ipv4->proto; +void odp_packet_set_inflag_vlan_qinq(odp_packet_t pkt, int val) +{ + odp_packet_hdr(pkt)->input_flags.vlan_qinq = val; } -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, - odph_ipv6hdr_t *ipv6, size_t *offset_out) +int odp_packet_inflag_snap(odp_packet_t pkt) { - if (ipv6->next_hdr == ODPH_IPPROTO_ESP || - ipv6->next_hdr == ODPH_IPPROTO_AH) { - pkt_hdr->input_flags.ipopt = 1; - pkt_hdr->input_flags.ipsec = 1; - return 0; - } + return odp_packet_hdr(pkt)->input_flags.snap; +} - if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { - pkt_hdr->input_flags.ipopt = 1; - pkt_hdr->input_flags.ipfrag = 1; - return 0; - } +void odp_packet_set_inflag_snap(odp_packet_t pkt, int val) +{ + odp_packet_hdr(pkt)->input_flags.snap = val; +} - /* Don't step through more extensions */ - *offset_out = ODPH_IPV6HDR_LEN; - return ipv6->next_hdr; +int odp_packet_inflag_arp(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->input_flags.arp; } -void odp_packet_print(odp_packet_t pkt) +void odp_packet_set_inflag_arp(odp_packet_t pkt, int val) { - int max_len = 512; - char str[max_len]; - int len = 0; - int n = max_len-1; - odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); + odp_packet_hdr(pkt)->input_flags.arp = val; +} - len += snprintf(&str[len], n-len, "Packet "); - len += odp_buffer_snprint(&str[len], n-len, (odp_buffer_t) pkt); - len += snprintf(&str[len], n-len, - " input_flags 0x%x\n", hdr->input_flags.all); - len += snprintf(&str[len], n-len, - " error_flags 0x%x\n", hdr->error_flags.all); - len += snprintf(&str[len], n-len, - " output_flags 0x%x\n", hdr->output_flags.all); - len += snprintf(&str[len], n-len, - " frame_offset %u\n", hdr->frame_offset); - len += snprintf(&str[len], n-len, - " l2_offset %u\n", hdr->l2_offset); - len += snprintf(&str[len], n-len, - " l3_offset %u\n", hdr->l3_offset); - len += snprintf(&str[len], n-len, - " l4_offset %u\n", hdr->l4_offset); - len += snprintf(&str[len], n-len, - " frame_len %u\n", hdr->frame_len); - len += snprintf(&str[len], n-len, - " input %u\n", hdr->input); - str[len] = '\0'; +int odp_packet_inflag_ipv4(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->input_flags.ipv4; +} - printf("\n%s\n", str); +void odp_packet_set_inflag_ipv4(odp_packet_t pkt, int val) +{ + odp_packet_hdr(pkt)->input_flags.ipv4 = val; } -int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src) +int odp_packet_inflag_ipv6(odp_packet_t pkt) { - odp_packet_hdr_t *const pkt_hdr_dst = odp_packet_hdr(pkt_dst); - odp_packet_hdr_t *const pkt_hdr_src = odp_packet_hdr(pkt_src); - const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, buf_hdr); - uint8_t *start_src; - uint8_t *start_dst; - size_t len; + return odp_packet_hdr(pkt)->input_flags.ipv6; +} - if (pkt_dst == ODP_PACKET_INVALID || pkt_src == ODP_PACKET_INVALID) - return -1; +void odp_packet_set_inflag_ipv6(odp_packet_t pkt, int val) +{ + odp_packet_hdr(pkt)->input_flags.ipv6 = val; +} - if (pkt_hdr_dst->buf_hdr.size < - pkt_hdr_src->frame_len + pkt_hdr_src->frame_offset) - return -1; +int odp_packet_inflag_ipfrag(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->input_flags.ipfrag; +} + +void odp_packet_set_inflag_ipfrag(odp_packet_t pkt, int val) +{ + odp_packet_hdr(pkt)->input_flags.ipfrag = val; +} + +int odp_packet_inflag_ipopt(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->input_flags.ipopt; +} - /* Copy packet header */ - start_dst = (uint8_t *)pkt_hdr_dst + start_offset; - start_src = (uint8_t *)pkt_hdr_src + start_offset; - len = ODP_OFFSETOF(odp_packet_hdr_t, buf_data) - start_offset; - memcpy(start_dst, start_src, len); +void odp_packet_set_inflag_ipopt(odp_packet_t pkt, int val) +{ + odp_packet_hdr(pkt)->input_flags.ipopt = val; +} - /* Copy frame payload */ - start_dst = (uint8_t *)odp_packet_data(pkt_dst); - start_src = (uint8_t *)odp_packet_data(pkt_src); - len = pkt_hdr_src->frame_len; - memcpy(start_dst, start_src, len); +int odp_packet_inflag_ipsec(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->input_flags.ipsec; +} - /* Copy useful things from the buffer header */ - pkt_hdr_dst->buf_hdr.cur_offset = pkt_hdr_src->buf_hdr.cur_offset; +void odp_packet_set_inflag_ipsec(odp_packet_t pkt, int val) +{ + odp_packet_hdr(pkt)->input_flags.ipsec = val; +} - /* Create a copy of the scatter list */ - odp_buffer_copy_scatter(odp_packet_to_buffer(pkt_dst), - odp_packet_to_buffer(pkt_src)); +int odp_packet_inflag_udp(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->input_flags.udp; +} - return 0; +void odp_packet_set_inflag_udp(odp_packet_t pkt, int val) +{ + odp_packet_hdr(pkt)->input_flags.udp = val; } -void odp_packet_set_ctx(odp_packet_t pkt, const void *ctx) +int odp_packet_inflag_tcp(odp_packet_t pkt) { - odp_packet_hdr(pkt)->user_ctx = (intptr_t)ctx; + return odp_packet_hdr(pkt)->input_flags.tcp; } -void *odp_packet_get_ctx(odp_packet_t pkt) +void odp_packet_set_inflag_tcp(odp_packet_t pkt, int val) { - return (void *)(intptr_t)odp_packet_hdr(pkt)->user_ctx; + odp_packet_hdr(pkt)->input_flags.tcp = val; +} + +int odp_packet_inflag_tcpopt(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->input_flags.tcpopt; +} + +void odp_packet_set_inflag_tcpopt(odp_packet_t pkt, int val) +{ + odp_packet_hdr(pkt)->input_flags.tcpopt = val; +} + +int odp_packet_inflag_icmp(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->input_flags.icmp; +} + +void odp_packet_set_inflag_icmp(odp_packet_t pkt, int val) +{ + odp_packet_hdr(pkt)->input_flags.icmp = val; +} + +int odp_packet_is_valid(odp_packet_t pkt) +{ + odp_buffer_hdr_t *buf = validate_buf((odp_buffer_t)pkt); + + if (buf == NULL) + return 0; + else + return buf->type == ODP_BUFFER_TYPE_PACKET; +} + +int odp_packet_is_segmented(odp_packet_t pkt) +{ + return (odp_packet_hdr(pkt)->buf_hdr.segcount > 1); +} + +int odp_packet_segment_count(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->buf_hdr.segcount; +} + +size_t odp_packet_headroom(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->headroom; +} + +size_t odp_packet_tailroom(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->tailroom; +} + +int odp_packet_push_head(odp_packet_t pkt, size_t len) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (len > pkt_hdr->headroom) + return -1; + + push_head(pkt_hdr, len); + return 0; +} + +void *odp_packet_push_head_and_map(odp_packet_t pkt, size_t len, size_t *seglen) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (len > pkt_hdr->headroom) + return NULL; + + push_head(pkt_hdr, len); + + return buffer_map(&pkt_hdr->buf_hdr, 0, seglen, pkt_hdr->frame_len); +} + +int odp_packet_pull_head(odp_packet_t pkt, size_t len) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (len > pkt_hdr->frame_len) + return -1; + + pull_head(pkt_hdr, len); + return 0; +} + +void *odp_packet_pull_head_and_map(odp_packet_t pkt, size_t len, size_t *seglen) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (len > pkt_hdr->frame_len) + return NULL; + + pull_head(pkt_hdr, len); + return buffer_map(&pkt_hdr->buf_hdr, 0, seglen, pkt_hdr->frame_len); +} + +int odp_packet_push_tail(odp_packet_t pkt, size_t len) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (len > pkt_hdr->tailroom) + return -1; + + push_tail(pkt_hdr, len); + return 0; +} + +void *odp_packet_push_tail_and_map(odp_packet_t pkt, size_t len, size_t *seglen) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + size_t origin = pkt_hdr->frame_len; + + if (len > pkt_hdr->tailroom) + return NULL; + + push_tail(pkt_hdr, len); + + return buffer_map(&pkt_hdr->buf_hdr, origin, seglen, + pkt_hdr->frame_len); +} + +int odp_packet_pull_tail(odp_packet_t pkt, size_t len) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (len > pkt_hdr->frame_len) + return -1; + + pull_tail(pkt_hdr, len); + return 0; +} + +void odp_packet_print(odp_packet_t pkt) +{ + int max_len = 512; + char str[max_len]; + int len = 0; + int n = max_len-1; + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); + + len += snprintf(&str[len], n-len, "Packet "); + len += odp_buffer_snprint(&str[len], n-len, (odp_buffer_t) pkt); + len += snprintf(&str[len], n-len, + " input_flags 0x%x\n", hdr->input_flags.all); + len += snprintf(&str[len], n-len, + " error_flags 0x%x\n", hdr->error_flags.all); + len += snprintf(&str[len], n-len, + " output_flags 0x%x\n", hdr->output_flags.all); + len += snprintf(&str[len], n-len, + " l2_offset %u\n", hdr->l2_offset); + len += snprintf(&str[len], n-len, + " l3_offset %u\n", hdr->l3_offset); + len += snprintf(&str[len], n-len, + " l3_len %u\n", hdr->l3_len); + len += snprintf(&str[len], n-len, + " l3_protocol 0x%x\n", hdr->l3_protocol); + len += snprintf(&str[len], n-len, + " l4_offset %u\n", hdr->l4_offset); + len += snprintf(&str[len], n-len, + " l4_len %u\n", hdr->l4_len); + len += snprintf(&str[len], n-len, + " l4_protocol %u\n", hdr->l4_protocol); + len += snprintf(&str[len], n-len, + " payload_offset %u\n", hdr->payload_offset); + len += snprintf(&str[len], n-len, + " frame_len %u\n", hdr->frame_len); + str[len] = '\0'; + + printf("\n%s\n", str); +} + +int odp_packet_copy_to_packet(odp_packet_t dstpkt, size_t dstoffset, + odp_packet_t srcpkt, size_t srcoffset, + size_t len) +{ + void *dstmap; + void *srcmap; + size_t cpylen, minseg, dstseglen, srcseglen; + + while (len > 0) { + dstmap = odp_packet_offset_map(dstpkt, dstoffset, &dstseglen); + srcmap = odp_packet_offset_map(srcpkt, srcoffset, &srcseglen); + if (dstmap == NULL || srcmap == NULL) + return -1; + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; + cpylen = len > minseg ? minseg : len; + memcpy(dstmap, srcmap, cpylen); + srcoffset += cpylen; + dstoffset += cpylen; + len -= cpylen; + } + + return 0; +} + +int odp_packet_copy_to_memory(void *dstmem, odp_packet_t srcpkt, + size_t srcoffset, size_t dstlen) +{ + void *mapaddr; + size_t seglen, cpylen; + uint8_t *dstaddr = (uint8_t *)dstmem; + + while (dstlen > 0) { + mapaddr = odp_packet_offset_map(srcpkt, srcoffset, &seglen); + if (mapaddr == NULL) + return -1; + cpylen = dstlen > seglen ? seglen : dstlen; + memcpy(dstaddr, mapaddr, cpylen); + srcoffset += cpylen; + dstaddr += cpylen; + dstlen -= cpylen; + } + + return 0; +} + +int odp_packet_copy_from_memory(odp_packet_t dstpkt, size_t dstoffset, + void *srcmem, size_t srclen) +{ + void *mapaddr; + size_t seglen, cpylen; + uint8_t *srcaddr = (uint8_t *)srcmem; + + while (srclen > 0) { + mapaddr = odp_packet_offset_map(dstpkt, dstoffset, &seglen); + if (mapaddr == NULL) + return -1; + cpylen = srclen > seglen ? seglen : srclen; + memcpy(mapaddr, srcaddr, cpylen); + dstoffset += cpylen; + srcaddr += cpylen; + srclen -= cpylen; + } + + return 0; +} + +odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool) +{ + size_t pktlen = odp_packet_len(pkt); + const size_t meta_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, buf_hdr); + odp_packet_t newpkt = odp_packet_alloc_len(pool, pktlen); + odp_packet_hdr_t *newhdr, *srchdr; + uint8_t *newstart, *srcstart; + + if (newpkt != ODP_PACKET_INVALID) { + /* Must copy meta data first, followed by packet data */ + srchdr = odp_packet_hdr(pkt); + newhdr = odp_packet_hdr(newpkt); + newstart = (uint8_t *)newhdr + meta_offset; + srcstart = (uint8_t *)srchdr + meta_offset; + + memcpy(newstart, srcstart, + sizeof(odp_packet_hdr_t) - meta_offset); + + if (odp_packet_copy_to_packet(newpkt, 0, pkt, 0, pktlen) != 0) { + odp_packet_free(newpkt); + newpkt = ODP_PACKET_INVALID; + } + } + + return newpkt; +} + +odp_packet_t odp_packet_clone(odp_packet_t pkt) +{ + return odp_packet_copy(pkt, odp_packet_hdr(pkt)->buf_hdr.pool_hdl); +} + +odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool) +{ + if (odp_pool_to_entry(pool)->s.params.buf_type != + ODP_BUFFER_TYPE_PACKET) + return ODP_PACKET_INVALID; + + return (odp_packet_t)buffer_alloc(pool, 0); +} + +odp_packet_t odp_packet_alloc_len(odp_buffer_pool_t pool, size_t len) +{ + if (odp_pool_to_entry(pool)->s.params.buf_type != + ODP_BUFFER_TYPE_PACKET) + return ODP_PACKET_INVALID; + + return (odp_packet_t)buffer_alloc(pool, len); +} + +void odp_packet_free(odp_packet_t pkt) +{ + odp_buffer_free((odp_buffer_t)pkt); +} + +odp_packet_t odp_packet_split(odp_packet_t pkt, size_t offset, + size_t hr, size_t tr) +{ + size_t len = odp_packet_len(pkt); + odp_buffer_pool_t pool = odp_packet_pool(pkt); + size_t pool_hr = odp_buffer_pool_headroom(pool); + size_t pkt_tr = odp_packet_tailroom(pkt); + odp_packet_t splitpkt; + size_t splitlen = len - offset; + + if (offset >= len) + return ODP_PACKET_INVALID; + + /* Erratum: We don't handle this rare corner case */ + if (tr > pkt_tr + splitlen) + return ODP_PACKET_INVALID; + + if (hr > pool_hr) + splitlen += (hr - pool_hr); + + splitpkt = odp_packet_alloc_len(pool, splitlen); + + if (splitpkt != ODP_PACKET_INVALID) { + if (hr > pool_hr) { + odp_packet_pull_head(splitpkt, hr - pool_hr); + splitlen -= (hr - pool_hr); + } + odp_packet_copy_to_packet(splitpkt, 0, + pkt, offset, splitlen); + odp_packet_pull_tail(pkt, splitlen); + } + + return splitpkt; +} + +odp_packet_t odp_packet_join(odp_packet_t pkt1, odp_packet_t pkt2) +{ + size_t len1 = odp_packet_len(pkt1); + size_t len2 = odp_packet_len(pkt2); + odp_packet_t joinpkt; + void *udata1, *udata_join; + size_t udata_size1, udata_size_join; + + /* Optimize if pkt1 is big enough to hold pkt2 */ + if (odp_packet_push_tail(pkt1, len2) == 0) { + odp_packet_copy_to_packet(pkt1, len1, + pkt2, 0, len2); + odp_packet_free(pkt2); + return pkt1; + } + + /* Otherwise join into a new packet */ + joinpkt = odp_packet_alloc_len(odp_packet_pool(pkt1), + len1 + len2); + + if (joinpkt != ODP_PACKET_INVALID) { + odp_packet_copy_to_packet(joinpkt, 0, pkt1, 0, len1); + odp_packet_copy_to_packet(joinpkt, len1, pkt2, 0, len2); + + /* Copy user metadata if present */ + udata1 = odp_packet_udata(pkt1, &udata_size1); + udata_join = odp_packet_udata(joinpkt, &udata_size_join); + + if (udata1 != NULL && udata_join != NULL) + memcpy(udata_join, udata1, + udata_size_join > udata_size1 ? + udata_size1 : udata_size_join); + + odp_packet_free(pkt1); + odp_packet_free(pkt2); + } + + return joinpkt; +} + +uint32_t odp_packet_refcount(odp_packet_t pkt) +{ + return odp_buffer_refcount(&odp_packet_hdr(pkt)->buf_hdr); +} + +uint32_t odp_packet_incr_refcount(odp_packet_t pkt, uint32_t val) +{ + return odp_buffer_incr_refcount(&odp_packet_hdr(pkt)->buf_hdr, val); +} + +uint32_t odp_packet_decr_refcount(odp_packet_t pkt, uint32_t val) +{ + return odp_buffer_decr_refcount(&odp_packet_hdr(pkt)->buf_hdr, val); +} + +/** + * Parser helper function for IPv4 + */ +static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, + uint8_t **parseptr, size_t *offset) +{ + odph_ipv4hdr_t *ipv4 = (odph_ipv4hdr_t *)*parseptr; + uint8_t ver = ODPH_IPV4HDR_VER(ipv4->ver_ihl); + uint8_t ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); + uint16_t frag_offset; + + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv4->tot_len); + + if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN) || + odp_unlikely(ver != 4) || + (pkt_hdr->l3_len > pkt_hdr->frame_len - *offset)) { + pkt_hdr->error_flags.ip_err = 1; + return 0; + } + + *offset += ihl * 4; + *parseptr += ihl * 4; + + if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) + pkt_hdr->input_flags.ipopt = 1; + + /* A packet is a fragment if: + * "more fragments" flag is set (all fragments except the last) + * OR + * "fragment offset" field is nonzero (all fragments except the first) + */ + frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); + if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) + pkt_hdr->input_flags.ipfrag = 1; + + if (ipv4->proto == ODPH_IPPROTO_ESP || + ipv4->proto == ODPH_IPPROTO_AH) { + pkt_hdr->input_flags.ipsec = 1; + return 0; + } + + /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after return */ + + *offset = sizeof(uint32_t) * ihl; + return ipv4->proto; +} + +/** + * Parser helper function for IPv6 + */ +static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, + uint8_t **parseptr, size_t *offset) +{ + odph_ipv6hdr_t *ipv6 = (odph_ipv6hdr_t *)*parseptr; + odph_ipv6hdr_ext_t *ipv6ext; + + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv6->payload_len); + + /* Basic sanity checks on IPv6 header */ + if (ipv6->ver != 6 || + pkt_hdr->l3_len > pkt_hdr->frame_len - *offset) { + pkt_hdr->error_flags.ip_err = 1; + return 0; + } + + /* Skip past IPv6 header */ + *offset += sizeof(odph_ipv6hdr_t); + *parseptr += sizeof(odph_ipv6hdr_t); + + + /* Skip past any IPv6 extension headers */ + if (ipv6->next_hdr == ODPH_IPPROTO_HOPOPTS || + ipv6->next_hdr == ODPH_IPPROTO_ROUTE) { + pkt_hdr->input_flags.ipopt = 1; + + do { + ipv6ext = (odph_ipv6hdr_ext_t *)*parseptr; + uint16_t extlen = 8 + ipv6ext->ext_len * 8; + + *offset += extlen; + *parseptr += extlen; + } while ((ipv6ext->next_hdr == ODPH_IPPROTO_HOPOPTS || + ipv6ext->next_hdr == ODPH_IPPROTO_ROUTE) && + *offset < pkt_hdr->frame_len); + + if (*offset >= pkt_hdr->l3_offset + ipv6->payload_len) { + pkt_hdr->error_flags.ip_err = 1; + return 0; + } + + if (ipv6ext->next_hdr == ODPH_IPPROTO_FRAG) + pkt_hdr->input_flags.ipfrag = 1; + + return ipv6ext->next_hdr; + } + + if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { + pkt_hdr->input_flags.ipopt = 1; + pkt_hdr->input_flags.ipfrag = 1; + } + + return ipv6->next_hdr; +} + +/** + * Parser helper function for TCP + */ +static inline void parse_tcp(odp_packet_hdr_t *pkt_hdr, + uint8_t **parseptr, size_t *offset) +{ + odph_tcphdr_t *tcp = (odph_tcphdr_t *)*parseptr; + + if (tcp->hl < sizeof(odph_tcphdr_t)/sizeof(uint32_t)) + pkt_hdr->error_flags.tcp_err = 1; + else if ((uint32_t)tcp->hl * 4 > sizeof(odph_tcphdr_t)) + pkt_hdr->input_flags.tcpopt = 1; + + pkt_hdr->l4_len = pkt_hdr->l3_len + + pkt_hdr->l3_offset - pkt_hdr->l4_offset; + + *offset += sizeof(odph_tcphdr_t); + *parseptr += sizeof(odph_tcphdr_t); +} + +/** + * Parser helper function for UDP + */ +static inline void parse_udp(odp_packet_hdr_t *pkt_hdr, + uint8_t **parseptr, size_t *offset) +{ + odph_udphdr_t *udp = (odph_udphdr_t *)*parseptr; + uint32_t udplen = odp_be_to_cpu_16(udp->length); + + if (udplen < sizeof(odph_udphdr_t) || + udplen > (pkt_hdr->l3_len + + pkt_hdr->l3_offset - pkt_hdr->l4_offset)) { + pkt_hdr->error_flags.udp_err = 1; + } + + pkt_hdr->l4_len = udplen; + + *offset += sizeof(odph_udphdr_t); + *parseptr += sizeof(odph_udphdr_t); +} + +/** + * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP + * + * Internal function: caller is resposible for passing only valid packet handles + * , lengths and offsets (usually done&called in packet input). + * + * @param pkt Packet handle + */ +int odp_packet_parse(odp_packet_t pkt) +{ + odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); + odph_ethhdr_t *eth; + odph_vlanhdr_t *vlan; + uint16_t ethtype; + uint8_t *parseptr; + size_t offset, seglen; + uint8_t ip_proto = 0; + + /* Reset parser metadata for new parse */ + pkt_hdr->error_flags.all = 0; + pkt_hdr->input_flags.all = 0; + pkt_hdr->output_flags.all = 0; + pkt_hdr->l2_offset = 0; + pkt_hdr->l3_offset = 0; + pkt_hdr->l4_offset = 0; + pkt_hdr->payload_offset = 0; + pkt_hdr->vlan_s_tag = 0; + pkt_hdr->vlan_c_tag = 0; + pkt_hdr->l3_protocol = 0; + pkt_hdr->l4_protocol = 0; + + /* We only support Ethernet for now */ + pkt_hdr->input_flags.eth = 1; + + if (odp_unlikely(pkt_hdr->frame_len < ODPH_ETH_LEN_MIN)) { + pkt_hdr->error_flags.frame_len = 1; + goto parse_exit; + } else if (pkt_hdr->frame_len > ODPH_ETH_LEN_MAX) { + pkt_hdr->input_flags.jumbo = 1; + } + + /* Assume valid L2 header, no CRC/FCS check in SW */ + pkt_hdr->input_flags.l2 = 1; + + eth = (odph_ethhdr_t *)odp_packet_map(pkt, &seglen); + offset = sizeof(odph_ethhdr_t); + parseptr = (uint8_t *)ð->type; + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + + /* Parse the VLAN header(s), if present */ + if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { + pkt_hdr->input_flags.vlan_qinq = 1; + pkt_hdr->input_flags.vlan = 1; + vlan = (odph_vlanhdr_t *)(void *)parseptr; + pkt_hdr->vlan_s_tag = ((ethtype << 16) | + odp_be_to_cpu_16(vlan->tci)); + offset += sizeof(odph_vlanhdr_t); + parseptr += sizeof(odph_vlanhdr_t); + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + } + + if (ethtype == ODPH_ETHTYPE_VLAN) { + pkt_hdr->input_flags.vlan = 1; + vlan = (odph_vlanhdr_t *)(void *)parseptr; + pkt_hdr->vlan_c_tag = ((ethtype << 16) | + odp_be_to_cpu_16(vlan->tci)); + offset += sizeof(odph_vlanhdr_t); + parseptr += sizeof(odph_vlanhdr_t); + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + } + + /* Check for SNAP vs. DIX */ + if (ethtype < ODPH_ETH_LEN_MAX) { + pkt_hdr->input_flags.snap = 1; + if (ethtype > pkt_hdr->frame_len - offset) { + pkt_hdr->error_flags.snap_len = 1; + goto parse_exit; + } + offset += 8; + parseptr += 8; + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + } + + /* Consume Ethertype for Layer 3 parse */ + parseptr += 2; + + /* Set l3_offset+flag only for known ethtypes */ + pkt_hdr->input_flags.l3 = 1; + pkt_hdr->l3_offset = offset; + pkt_hdr->l3_protocol = ethtype; + + /* Parse Layer 3 headers */ + switch (ethtype) { + case ODPH_ETHTYPE_IPV4: + pkt_hdr->input_flags.ipv4 = 1; + ip_proto = parse_ipv4(pkt_hdr, &parseptr, &offset); + break; + + case ODPH_ETHTYPE_IPV6: + pkt_hdr->input_flags.ipv6 = 1; + ip_proto = parse_ipv6(pkt_hdr, &parseptr, &offset); + break; + + case ODPH_ETHTYPE_ARP: + pkt_hdr->input_flags.arp = 1; + ip_proto = 255; /* Reserved invalid by IANA */ + break; + + default: + pkt_hdr->input_flags.l3 = 0; + ip_proto = 255; /* Reserved invalid by IANA */ + } + + /* Set l4_offset+flag only for known ip_proto */ + pkt_hdr->input_flags.l4 = 1; + pkt_hdr->l4_offset = offset; + pkt_hdr->l4_protocol = ip_proto; + + /* Parse Layer 4 headers */ + switch (ip_proto) { + case ODPH_IPPROTO_ICMP: + pkt_hdr->input_flags.icmp = 1; + break; + + case ODPH_IPPROTO_TCP: + pkt_hdr->input_flags.tcp = 1; + parse_tcp(pkt_hdr, &parseptr, &offset); + break; + + case ODPH_IPPROTO_UDP: + pkt_hdr->input_flags.udp = 1; + parse_udp(pkt_hdr, &parseptr, &offset); + break; + + case ODPH_IPPROTO_AH: + case ODPH_IPPROTO_ESP: + pkt_hdr->input_flags.ipsec = 1; + break; + + default: + pkt_hdr->input_flags.l4 = 0; + break; + } + + /* + * Anything beyond what we parse here is considered payload. + * Note: Payload is really only relevant for TCP and UDP. For + * all other protocols, the payload offset will point to the + * final header (ARP, ICMP, AH, ESP, or IP Fragment. + */ + pkt_hdr->payload_offset = offset; + +parse_exit: + return pkt_hdr->error_flags.all != 0; } diff --git a/platform/linux-generic/odp_packet_flags.c b/platform/linux-generic/odp_packet_flags.c deleted file mode 100644 index 06fdeed..0000000 --- a/platform/linux-generic/odp_packet_flags.c +++ /dev/null @@ -1,202 +0,0 @@ -/* Copyright (c) 2014, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp_packet_flags.h> -#include <odp_packet_internal.h> - - -int odp_packet_error(odp_packet_t pkt) -{ - return (odp_packet_hdr(pkt)->error_flags.all != 0); -} - -/* Get Error Flags */ - -int odp_packet_errflag_frame_len(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->error_flags.frame_len; -} - -/* Get Input Flags */ - -int odp_packet_inflag_l2(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.l2; -} - -int odp_packet_inflag_l3(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.l3; -} - -int odp_packet_inflag_l4(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.l4; -} - -int odp_packet_inflag_eth(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.eth; -} - -int odp_packet_inflag_jumbo(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.jumbo; -} - -int odp_packet_inflag_vlan(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.vlan; -} - -int odp_packet_inflag_vlan_qinq(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.vlan_qinq; -} - -int odp_packet_inflag_arp(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.arp; -} - -int odp_packet_inflag_ipv4(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.ipv4; -} - -int odp_packet_inflag_ipv6(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.ipv6; -} - -int odp_packet_inflag_ipfrag(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.ipfrag; -} - -int odp_packet_inflag_ipopt(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.ipopt; -} - -int odp_packet_inflag_ipsec(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.ipsec; -} - -int odp_packet_inflag_udp(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.udp; -} - -int odp_packet_inflag_tcp(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.tcp; -} - -int odp_packet_inflag_sctp(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.sctp; -} - -int odp_packet_inflag_icmp(odp_packet_t pkt) -{ - return odp_packet_hdr(pkt)->input_flags.icmp; -} - -/* Set Output Flags */ - -void odp_packet_outflag_l4_chksum(odp_packet_t pkt) -{ - odp_packet_hdr(pkt)->output_flags.l4_chksum = 1; -} - -/* Set Input Flags */ - -void odp_packet_set_inflag_l2(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.l2 = val; -} - -void odp_packet_set_inflag_l3(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.l3 = val; -} - -void odp_packet_set_inflag_l4(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.l4 = val; -} - -void odp_packet_set_inflag_eth(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.eth = val; -} - -void odp_packet_set_inflag_jumbo(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.jumbo = val; -} - -void odp_packet_set_inflag_vlan(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.vlan = val; -} - -void odp_packet_set_inflag_vlan_qinq(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.vlan_qinq = val; -} - -void odp_packet_set_inflag_arp(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.arp = val; -} - -void odp_packet_set_inflag_ipv4(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.ipv4 = val; -} - -void odp_packet_set_inflag_ipv6(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.ipv6 = val; -} - -void odp_packet_set_inflag_ipfrag(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.ipfrag = val; -} - -void odp_packet_set_inflag_ipopt(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.ipopt = val; -} - -void odp_packet_set_inflag_ipsec(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.ipsec = val; -} - -void odp_packet_set_inflag_udp(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.udp = val; -} - -void odp_packet_set_inflag_tcp(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.tcp = val; -} - -void odp_packet_set_inflag_sctp(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.sctp = val; -} - -void odp_packet_set_inflag_icmp(odp_packet_t pkt, int val) -{ - odp_packet_hdr(pkt)->input_flags.icmp = val; -}
Signed-off-by: Bill Fischofer <bill.fischofer@linaro.org> --- platform/linux-generic/odp_buffer.c | 263 ++++++- platform/linux-generic/odp_buffer_pool.c | 661 +++++++--------- platform/linux-generic/odp_packet.c | 1200 +++++++++++++++++++++++------ platform/linux-generic/odp_packet_flags.c | 202 ----- 4 files changed, 1455 insertions(+), 871 deletions(-) delete mode 100644 platform/linux-generic/odp_packet_flags.c