| Message ID | 1423063092-3376-1-git-send-email-venkatesh.vivekanandan@linaro.org |
|---|---|
| State | New |
| Headers | show |
Venki - I think Zoltan is breaking his 0.6 patches up, andI think the right thing to do is create patches for all the changes as I believe ODP-DPDK will become a more important implementation to us this year. Maybe we do this at HKG15 to reduce the review cycle time ? On 4 February 2015 at 10:30, Venkatesh Vivekanandan < venkatesh.vivekanandan@linaro.org> wrote: > Hi All, > > Inorder to make the REVIEW PROCESS SIMPLER as it is a huge patch, I have > attached the changes that went in this patch on top of ODP 0.5.0 (diff > between platform/linux-generic and platform/linux-dpdk). Hope this helps. > > Thanks, > Venky > > On 4 February 2015 at 20:48, <venkatesh.vivekanandan@linaro.org> wrote: > >> From: Venkatesh Vivekanandan <venkatesh.vivekanandan@linaro.org> >> >> Signed-off-by: Venkatesh Vivekanandan <venkatesh.vivekanandan@linaro.org> >> --- >> -> This patch must be applied on odp v0.5.0 ontop of odp-dpdk >> -> odp_l2fwd, odp_pktio were tested both in burst and queue mode which >> worked >> fine. >> -> ipsec app was tested and working fine. For this to work, >> USE_MAC_ADDR_HACK has to be set to 0 for now. Will send out a patch for >> MAC address fix once this patch gets in. >> >> Checkpatch warning for camel case could not be avoided. >> WARNING: Avoid CamelCase: <_Static_assert> >> #373: FILE: platform/linux-dpdk/include/api/odp_debug.h:34: >> +#define _Static_assert(e, s) extern int (*static_assert_checker(void)) \ >> >> WARNING: storage class should be at the beginning of the declaration >> #373: FILE: platform/linux-dpdk/include/api/odp_debug.h:34: >> +#define _Static_assert(e, s) extern int (*static_assert_checker(void)) \ >> >> v2: Fixed odp_flush_caches compilation error >> v3: Removed debug code that slipped in >> >> platform/linux-dpdk/Makefile.am | 13 +- >> platform/linux-dpdk/include/api/odp_buffer.h | 30 +- >> platform/linux-dpdk/include/api/odp_buffer_pool.h | 124 ++- >> platform/linux-dpdk/include/api/odp_debug.h | 77 +- >> platform/linux-dpdk/include/api/odp_packet.h | 772 +++++++++++++-- >> platform/linux-dpdk/include/api/odp_packet_io.h | 92 +- >> platform/linux-dpdk/include/api/odp_pktio_types.h | 45 - >> .../linux-dpdk/include/api/odp_platform_types.h | 78 ++ >> platform/linux-dpdk/include/odp_buffer_inlines.h | 179 ++++ >> platform/linux-dpdk/include/odp_buffer_internal.h | 134 ++- >> .../linux-dpdk/include/odp_buffer_pool_internal.h | 333 ++++++- >> platform/linux-dpdk/include/odp_packet_dpdk.h | 2 - >> platform/linux-dpdk/include/odp_packet_internal.h | 135 ++- >> .../linux-dpdk/include/odp_packet_io_internal.h | 42 +- >> platform/linux-dpdk/odp_buffer.c | 14 +- >> platform/linux-dpdk/odp_buffer_pool.c | 153 +-- >> platform/linux-dpdk/odp_init.c | 35 +- >> platform/linux-dpdk/odp_linux.c | 117 ++- >> platform/linux-dpdk/odp_packet.c | 1001 >> +++++++++++++++----- >> platform/linux-dpdk/odp_packet_dpdk.c | 9 +- >> platform/linux-dpdk/odp_packet_io.c | 309 ++++-- >> platform/linux-dpdk/odp_queue.c | 3 +- >> platform/linux-dpdk/odp_schedule.c | 421 -------- >> 23 files changed, 2993 insertions(+), 1125 deletions(-) >> delete mode 100644 platform/linux-dpdk/include/api/odp_pktio_types.h >> create mode 100644 platform/linux-dpdk/include/api/odp_platform_types.h >> create mode 100644 platform/linux-dpdk/include/odp_buffer_inlines.h >> delete mode 100644 platform/linux-dpdk/odp_schedule.c >> >> diff --git a/platform/linux-dpdk/Makefile.am >> b/platform/linux-dpdk/Makefile.am >> index db11e0a..6491d3a 100644 >> --- a/platform/linux-dpdk/Makefile.am >> +++ b/platform/linux-dpdk/Makefile.am >> @@ -29,6 +29,7 @@ include_HEADERS = \ >> >> $(top_srcdir)/platform/linux-generic/include/api/odp_barrier.h \ >> $(srcdir)/include/api/odp_buffer_pool.h \ >> >> $(top_srcdir)/platform/linux-generic/include/api/odp_byteorder.h \ >> + >> $(top_srcdir)/platform/linux-generic/include/api/odp_classification.h \ >> >> $(top_srcdir)/platform/linux-generic/include/api/odp_compiler.h \ >> >> $(top_srcdir)/platform/linux-generic/include/api/odp_config.h \ >> >> $(top_srcdir)/platform/linux-generic/include/api/odp_coremask.h \ >> @@ -52,15 +53,16 @@ include_HEADERS = \ >> >> $(top_srcdir)/platform/linux-generic/include/api/odp_time.h \ >> >> $(top_srcdir)/platform/linux-generic/include/api/odp_timer.h \ >> >> $(top_srcdir)/platform/linux-generic/include/api/odp_version.h \ >> - $(srcdir)/include/api/odp_pktio_types.h >> + $(srcdir)/include/api/odp_platform_types.h >> >> subdirheadersdir = $(includedir)/helper >> subdirheaders_HEADERS = \ >> $(top_srcdir)/helper/include/odph_chksum.h \ >> $(top_srcdir)/helper/include/odph_eth.h \ >> + $(top_srcdir)/helper/include/odph_icmp.h \ >> $(top_srcdir)/helper/include/odph_ip.h \ >> + $(top_srcdir)/helper/include/odph_ipsec.h \ >> $(top_srcdir)/helper/include/odph_linux.h \ >> - $(top_srcdir)/helper/include/odph_packet.h \ >> $(top_srcdir)/helper/include/odph_ring.h \ >> $(top_srcdir)/helper/include/odph_udp.h >> >> @@ -68,9 +70,11 @@ __LIB__libodp_la_SOURCES = \ >> ../linux-generic/odp_barrier.c \ >> odp_buffer.c \ >> odp_buffer_pool.c \ >> + ../linux-generic/odp_classification.c \ >> ../linux-generic/odp_coremask.c \ >> ../linux-generic/odp_crypto.c \ >> odp_init.c \ >> + ../linux-generic/odp_impl.c \ >> odp_linux.c \ >> odp_packet.c \ >> odp_packet_dpdk.c \ >> @@ -80,11 +84,12 @@ __LIB__libodp_la_SOURCES = \ >> odp_queue.c \ >> ../linux-generic/odp_ring.c \ >> ../linux-generic/odp_rwlock.c \ >> - odp_schedule.c \ >> + ../linux-generic/odp_schedule.c \ >> ../linux-generic/odp_shared_memory.c \ >> ../linux-generic/odp_spinlock.c \ >> ../linux-generic/odp_system_info.c \ >> ../linux-generic/odp_thread.c \ >> ../linux-generic/odp_ticketlock.c \ >> ../linux-generic/odp_time.c \ >> - ../linux-generic/odp_timer.c >> + ../linux-generic/odp_timer.c \ >> + ../linux-generic/odp_weak.c >> diff --git a/platform/linux-dpdk/include/api/odp_buffer.h >> b/platform/linux-dpdk/include/api/odp_buffer.h >> index b2fbc76..3c23035 100644 >> --- a/platform/linux-dpdk/include/api/odp_buffer.h >> +++ b/platform/linux-dpdk/include/api/odp_buffer.h >> @@ -19,20 +19,13 @@ extern "C" { >> #endif >> >> >> - >> #include <odp_std_types.h> >> +#include <odp_platform_types.h> >> >> - >> - >> - >> - >> -/** >> - * ODP buffer >> +/** @defgroup odp_buffer ODP BUFFER >> + * Operations on a buffer. >> + * @{ >> */ >> -typedef unsigned long odp_buffer_t; >> - >> - >> -#define ODP_BUFFER_INVALID (unsigned long)(-1L) /**< Invalid buffer */ >> >> >> /** >> @@ -74,11 +67,21 @@ int odp_buffer_type(odp_buffer_t buf); >> * >> * @param buf Buffer handle >> * >> - * @return 1 if valid, otherwise 0 >> + * @retval 1 Buffer handle represents a valid buffer. >> + * @retval 0 Buffer handle does not represent a valid buffer. >> */ >> int odp_buffer_is_valid(odp_buffer_t buf); >> >> /** >> + * Buffer pool of the buffer >> + * >> + * @param buf Buffer handle >> + * >> + * @return Handle of buffer pool buffer belongs to >> + */ >> +odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf); >> + >> +/** >> * Print buffer metadata to STDOUT >> * >> * @param buf Buffer handle >> @@ -86,6 +89,9 @@ int odp_buffer_is_valid(odp_buffer_t buf); >> */ >> void odp_buffer_print(odp_buffer_t buf); >> >> +/** >> + * @} >> + */ >> >> #ifdef __cplusplus >> } >> diff --git a/platform/linux-dpdk/include/api/odp_buffer_pool.h >> b/platform/linux-dpdk/include/api/odp_buffer_pool.h >> index 382f4f0..4da5f84 100644 >> --- a/platform/linux-dpdk/include/api/odp_buffer_pool.h >> +++ b/platform/linux-dpdk/include/api/odp_buffer_pool.h >> @@ -21,66 +21,143 @@ extern "C" { >> >> >> #include <odp_std_types.h> >> +#include <odp_platform_types.h> >> #include <odp_buffer.h> >> >> +/** @addtogroup odp_buffer >> + * Operations on a buffer pool. >> + * @{ >> + */ >> + >> /** Maximum queue name lenght in chars */ >> #define ODP_BUFFER_POOL_NAME_LEN 32 >> >> -/** Invalid buffer pool */ >> -#define ODP_BUFFER_POOL_INVALID (unsigned long)(-1L) >> - >> -/** ODP buffer pool */ >> -typedef unsigned long odp_buffer_pool_t; >> - >> +/** >> + * Buffer pool parameters >> + * Used to communicate buffer pool creation options. >> + */ >> +typedef struct odp_buffer_pool_param_t { >> + uint32_t buf_size; /**< Buffer size in bytes. The maximum >> + number of bytes application will >> + store in each buffer. For packets, this >> + is the maximum packet data length, and >> + configured headroom and tailroom will be >> + added to this number */ >> + uint32_t buf_align; /**< Minimum buffer alignment in bytes. >> + Valid values are powers of two. Use 0 >> + for default alignment. Default will >> + always be a multiple of 8. */ >> + uint32_t num_bufs; /**< Number of buffers in the pool */ >> + int buf_type; /**< Buffer type */ >> +} odp_buffer_pool_param_t; >> >> /** >> * Create a buffer pool >> + * This routine is used to create a buffer pool. It take three >> + * arguments: the optional name of the pool to be created, an optional >> shared >> + * memory handle, and a parameter struct that describes the pool to be >> + * created. If a name is not specified the result is an anonymous pool >> that >> + * cannot be referenced by odp_buffer_pool_lookup(). >> + * >> + * @param name Name of the pool, max ODP_BUFFER_POOL_NAME_LEN-1 >> chars. >> + * May be specified as NULL for anonymous pools. >> + * >> + * @param shm The shared memory object in which to create the pool. >> + * Use ODP_SHM_NULL to reserve default memory type >> + * for the buffer type. >> * >> - * @param name Name of the pool (max ODP_BUFFER_POOL_NAME_LEN - 1 >> chars) >> - * @param base_addr Pool base address >> - * @param size Pool size in bytes >> - * @param buf_size Buffer size in bytes >> - * @param buf_align Minimum buffer alignment >> - * @param buf_type Buffer type >> + * @param params Buffer pool parameters. >> * >> - * @return Buffer pool handle >> + * @return Handle of the created buffer pool >> + * @retval ODP_BUFFER_POOL_INVALID Buffer pool could not be created >> */ >> + >> odp_buffer_pool_t odp_buffer_pool_create(const char *name, >> - void *base_addr, uint64_t size, >> - size_t buf_size, size_t >> buf_align, >> - int buf_type); >> + odp_shm_t shm, >> + odp_buffer_pool_param_t *params); >> >> +/** >> + * Destroy a buffer pool previously created by odp_buffer_pool_create() >> + * >> + * @param pool Handle of the buffer pool to be destroyed >> + * >> + * @retval 0 Success >> + * @retval -1 Failure >> + * >> + * @note This routine destroys a previously created buffer pool. This >> call >> + * does not destroy any shared memory object passed to >> + * odp_buffer_pool_create() used to store the buffer pool contents. The >> caller >> + * takes responsibility for that. If no shared memory object was passed >> as >> + * part of the create call, then this routine will destroy any internal >> shared >> + * memory objects associated with the buffer pool. Results are undefined >> if >> + * an attempt is made to destroy a buffer pool that contains allocated or >> + * otherwise active buffers. >> + */ >> +int odp_buffer_pool_destroy(odp_buffer_pool_t pool); >> >> /** >> * Find a buffer pool by name >> * >> * @param name Name of the pool >> * >> - * @return Buffer pool handle, or ODP_BUFFER_POOL_INVALID if not found. >> + * @return Handle of found buffer pool >> + * @retval ODP_BUFFER_POOL_INVALID Buffer pool could not be found >> + * >> + * @note This routine cannot be used to look up an anonymous pool (one >> created >> + * with no name). >> */ >> odp_buffer_pool_t odp_buffer_pool_lookup(const char *name); >> >> +/** >> + * Buffer pool information struct >> + * Used to get information about a buffer pool. >> + */ >> +typedef struct odp_buffer_pool_info_t { >> + const char *name; /**< pool name */ >> + odp_shm_t shm; /**< handle of shared memory >> area >> + supplied by application to >> + contain buffer pool, or >> + ODP_SHM_NULL if this pool is >> + managed by ODP */ >> + odp_buffer_pool_param_t params; /**< pool parameters */ >> +} odp_buffer_pool_info_t; >> + >> +/** >> + * Retrieve information about a buffer pool >> + * >> + * @param pool Buffer pool handle >> + * >> + * @param[out] info Receives an odp_buffer_pool_info_t object >> + * that describes the pool. >> + * >> + * @retval 0 Success >> + * @retval -1 Failure. Info could not be retrieved. >> + */ >> + >> +int odp_buffer_pool_info(odp_buffer_pool_t pool, >> + odp_buffer_pool_info_t *info); >> >> /** >> * Print buffer pool info >> * >> * @param pool Pool handle >> * >> + * @note This routine writes implementation-defined information about the >> + * specified buffer pool to the ODP log. The intended use is for >> debugging. >> */ >> void odp_buffer_pool_print(odp_buffer_pool_t pool); >> >> - >> - >> /** >> * Buffer alloc >> * >> + * The validity of a buffer can be cheked at any time with >> odp_buffer_is_valid() >> * @param pool Pool handle >> * >> - * @return Buffer handle or ODP_BUFFER_INVALID >> + * @return Handle of allocated buffer >> + * @retval ODP_BUFFER_INVALID Buffer could not be allocated >> */ >> odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool); >> >> - >> /** >> * Buffer free >> * >> @@ -89,8 +166,9 @@ odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool); >> */ >> void odp_buffer_free(odp_buffer_t buf); >> >> - >> - >> +/** >> + * @} >> + */ >> >> #ifdef __cplusplus >> } >> diff --git a/platform/linux-dpdk/include/api/odp_debug.h >> b/platform/linux-dpdk/include/api/odp_debug.h >> index 6b145b5..aed8a5f 100644 >> --- a/platform/linux-dpdk/include/api/odp_debug.h >> +++ b/platform/linux-dpdk/include/api/odp_debug.h >> @@ -12,73 +12,74 @@ >> #ifndef ODP_DEBUG_H_ >> #define ODP_DEBUG_H_ >> >> -#include <stdio.h> >> -#include <stdlib.h> >> >> #ifdef __cplusplus >> extern "C" { >> #endif >> >> +/** @addtogroup odp_ver_abt_log_dbg >> + * Macros that allows different messages. >> + * @{ >> + */ >> + >> #ifdef __GNUC__ >> >> -/** >> - * Indicate deprecated variables, functions or types >> - */ >> -#define ODP_DEPRECATED __attribute__((__deprecated__)) >> + >> +#if __GNUC__ < 4 || (__GNUC__ == 4 && (__GNUC_MINOR__ < 6)) >> >> /** >> - * Intentionally unused variables ot functions >> + * _Static_assert was only added in GCC 4.6. Provide a weak replacement >> + * for previous versions. >> */ >> -#define ODP_UNUSED __attribute__((__unused__)) >> +#define _Static_assert(e, s) extern int (*static_assert_checker(void)) \ >> + [sizeof(struct { unsigned int error_if_negative:(e) ? 1 : -1; })] >> >> -#else >> +#endif >> >> -#define ODP_DEPRECATED >> -#define ODP_UNUSED >> >> -#endif >> >> -/** >> - * Runtime assertion-macro - aborts if 'cond' is false. >> - */ >> -#ifndef ODP_NO_DEBUG >> -#define ODP_ASSERT(cond, msg) \ >> - do { if (!(cond)) {ODP_ERR("%s\n", msg); abort(); } } while (0) >> -#else >> -#define ODP_ASSERT(cond, msg) >> #endif >> >> + >> /** >> * Compile time assertion-macro - fail compilation if cond is false. >> * @note This macro has zero runtime overhead >> */ >> -#define ODP_STATIC_ASSERT(cond, msg) _Static_assert(1, msg) >> +#define _ODP_STATIC_ASSERT(cond, msg) _Static_assert(1, msg) >> >> /** >> - * Debug printing macro, which prints output when DEBUG flag is set. >> + * ODP log level. >> */ >> -#define ODP_DBG(fmt, ...) \ >> - do { if (ODP_DEBUG_PRINT == 1) \ >> - printf(fmt, ##__VA_ARGS__); \ >> - } while (0) >> +typedef enum odp_log_level { >> + ODP_LOG_DBG, >> + ODP_LOG_ERR, >> + ODP_LOG_UNIMPLEMENTED, >> + ODP_LOG_ABORT, >> + ODP_LOG_PRINT >> +} odp_log_level_e; >> >> /** >> - * Print output to stderr (file, line and function). >> + * ODP log function >> + * >> + * Instead of direct prints to stdout/stderr all logging in ODP >> implementation >> + * should be done via this function or its wrappers. >> + * ODP platform MUST provide a default *weak* implementation of this >> function. >> + * Application MAY override the function if needed by providing a strong >> + * function. >> + * >> + * @param[in] level Log level >> + * @param[in] fmt printf-style message format >> + * >> + * @return The number of characters logged if succeeded. Otherwise >> returns >> + * a negative number. >> */ >> -#define ODP_ERR(fmt, ...) \ >> -do { fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, \ >> - __LINE__, __func__, ##__VA_ARGS__); \ >> -} while (0) >> +extern int odp_override_log(odp_log_level_e level, const char *fmt, ...); >> + >> + >> >> /** >> - * Print output to stderr (file, line and function), >> - * then abort. >> + * @} >> */ >> -#define ODP_ABORT(fmt, ...) \ >> -do { fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, \ >> - __LINE__, __func__, ##__VA_ARGS__); \ >> - abort(); \ >> -} while (0) >> >> #ifdef __cplusplus >> } >> diff --git a/platform/linux-dpdk/include/api/odp_packet.h >> b/platform/linux-dpdk/include/api/odp_packet.h >> index 64a92e8..97c2cb6 100644 >> --- a/platform/linux-dpdk/include/api/odp_packet.h >> +++ b/platform/linux-dpdk/include/api/odp_packet.h >> @@ -1,7 +1,7 @@ >> /* Copyright (c) 2013, Linaro Limited >> * All rights reserved. >> * >> - * SPDX-License-Identifier: BSD-3-Clause >> + * SPDX-License-Identifier: BSD-3-Clause >> */ >> >> >> @@ -19,33 +19,84 @@ extern "C" { >> #endif >> >> #include <odp_buffer.h> >> +#include <odp_platform_types.h> >> >> +/** @defgroup odp_packet ODP PACKET >> + * Operations on a packet. >> + * @{ >> + */ >> >> -/** >> - * ODP packet descriptor >> + >> +/* >> + * Packet API v0.5 notes >> + * - Push/pull operations only on packet level >> + * - Push/pull within limits of segment headroom/tailroom/data lengths >> + * - Segment data length must be always at least one byte (i.e. there >> are no >> + * empty segments) >> + * - Head/tailroom content belong to packet content (in addition to data >> + * and meta-data) and thus is preserved over packet ownership changes. >> + * - _addr refer to a fixed address, which operations do not modify >> + * - _ptr refer to pointer to data, which may be modified by operations >> */ >> -typedef odp_buffer_t odp_packet_t; >> >> >> -/** Invalid packet */ >> -#define ODP_PACKET_INVALID ODP_BUFFER_INVALID >> +/* >> + * >> + * Alloc and free >> + * ******************************************************** >> + * >> + */ >> >> -/** Invalid offset */ >> -#define ODP_PACKET_OFFSET_INVALID ((size_t)-1) >> +/** >> + * Allocate a packet from a buffer pool >> + * >> + * Allocates a packet of the requested length from the specified buffer >> pool. >> + * Pool must have been created with buffer type ODP_BUFFER_TYPE_PACKET. >> The >> + * packet is initialized with data pointers and lengths set according to >> the >> + * specified len, and the default headroom and tailroom length settings. >> All >> + * other packet metadata are set to their default values. >> + * >> + * @param pool Pool handle >> + * @param len Packet data length >> + * >> + * @return Handle of allocated packet >> + * @retval ODP_PACKET_INVALID Packet could not be allocated >> + * >> + * @note The default headroom and tailroom used for packets is specified >> by >> + * the ODP_CONFIG_PACKET_HEADROOM and ODP_CONFIG_PACKET_TAILROOM defines >> in >> + * odp_config.h. >> + */ >> +odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool, uint32_t len); >> >> +/** >> + * Free packet >> + * >> + * Frees the packet into the buffer pool it was allocated from. >> + * >> + * @param pkt Packet handle >> + */ >> +void odp_packet_free(odp_packet_t pkt); >> >> /** >> - * Initialize the packet >> + * Reset packet >> * >> - * Needs to be called if the user allocates a packet buffer, i.e. the >> packet >> - * has not been received from I/O through ODP. >> + * Resets all packet meta-data to their default values. Packet length is >> used >> + * to initialize pointers and lengths. It must be less than the total >> buffer >> + * length of the packet minus the default headroom length. Packet is not >> + * modified on failure. >> * >> - * @param pkt Packet handle >> + * @param pkt Packet handle >> + * @param len Packet data length >> + * >> + * @retval 0 Success >> + * @retval Non-zero Failure >> + * >> + * @see odp_packet_buf_len() >> */ >> -void odp_packet_init(odp_packet_t pkt); >> +int odp_packet_reset(odp_packet_t pkt, uint32_t len); >> >> /** >> - * Convert from packet handle to buffer handle >> + * Convert a buffer handle to a packet handle >> * >> * @param buf Buffer handle >> * >> @@ -54,190 +105,737 @@ void odp_packet_init(odp_packet_t pkt); >> odp_packet_t odp_packet_from_buffer(odp_buffer_t buf); >> >> /** >> - * Convert from buffer handle to packet handle >> + * Convert a packet handle to a buffer handle >> * >> * @param pkt Packet handle >> * >> * @return Buffer handle >> */ >> -odp_buffer_t odp_buffer_from_packet(odp_packet_t pkt); >> +odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt); >> + >> + >> +/* >> + * >> + * Pointers and lengths >> + * ******************************************************** >> + * >> + */ >> + >> +/** >> + * Packet head address >> + * >> + * Returns start address of the first segment. Packet level headroom >> starts >> + * from here. Use odp_packet_data() or odp_packet_l2_ptr() to return the >> + * packet data start address. >> + * >> + * @param pkt Packet handle >> + * >> + * @return Pointer to the start address of the first packet segment >> + * >> + * @see odp_packet_data(), odp_packet_l2_ptr(), odp_packet_headroom() >> + */ >> +void *odp_packet_head(odp_packet_t pkt); >> + >> +/** >> + * Total packet buffer length >> + * >> + * Returns sum of buffer lengths over all packet segments. >> + * >> + * @param pkt Packet handle >> + * >> + * @return Total packet buffer length in bytes >> + * >> + * @see odp_packet_reset() >> + */ >> +uint32_t odp_packet_buf_len(odp_packet_t pkt); >> >> /** >> - * Set the packet length >> + * Packet data pointer >> + * >> + * Returns the current packet data pointer. When a packet is received >> + * from packet input, this points to the first byte of the received >> + * packet. Packet level offsets are calculated relative to this position. >> + * >> + * User can adjust the data pointer with head_push/head_pull (does not >> modify >> + * segmentation) and add_data/rem_data calls (may modify segmentation). >> + * >> + * @param pkt Packet handle >> + * >> + * @return Pointer to the packet data >> + * >> + * @see odp_packet_l2_ptr(), odp_packet_seg_len() >> + */ >> +void *odp_packet_data(odp_packet_t pkt); >> + >> +/** >> + * Packet segment data length >> + * >> + * Returns number of data bytes following the current data pointer >> + * (odp_packet_data()) location in the segment. >> * >> * @param pkt Packet handle >> - * @param len Length of packet in bytes >> + * >> + * @return Segment data length in bytes (pointed by odp_packet_data()) >> + * >> + * @see odp_packet_data() >> */ >> -void odp_packet_set_len(odp_packet_t pkt, size_t len); >> +uint32_t odp_packet_seg_len(odp_packet_t pkt); >> >> /** >> - * Get the packet length >> + * Packet data length >> + * >> + * Returns sum of data lengths over all packet segments. >> * >> * @param pkt Packet handle >> * >> - * @return Packet length in bytes >> + * @return Packet data length >> */ >> -size_t odp_packet_get_len(odp_packet_t pkt); >> +uint32_t odp_packet_len(odp_packet_t pkt); >> >> /** >> - * Set packet user context >> + * Packet headroom length >> * >> - * @param buf Packet handle >> - * @param ctx User context >> + * Returns the current packet level headroom length. >> + * >> + * @param pkt Packet handle >> * >> + * @return Headroom length >> */ >> -void odp_packet_set_ctx(odp_packet_t buf, const void *ctx); >> +uint32_t odp_packet_headroom(odp_packet_t pkt); >> >> /** >> - * Get packet user context >> + * Packet tailroom length >> * >> - * @param buf Packet handle >> + * Returns the current packet level tailroom length. >> * >> - * @return User context >> + * @param pkt Packet handle >> + * >> + * @return Tailroom length >> */ >> -void *odp_packet_get_ctx(odp_packet_t buf); >> +uint32_t odp_packet_tailroom(odp_packet_t pkt); >> >> /** >> - * Get address to the start of the packet buffer >> + * Packet tailroom pointer >> * >> - * The address of the packet buffer is not necessarily the same as the >> start >> - * address of the received frame, e.g. an eth frame may be offset by 2 >> or 6 >> - * bytes to ensure 32 or 64-bit alignment of the IP header. >> - * Use odp_packet_l2(pkt) to get the start address of a received valid >> frame >> - * or odp_packet_start(pkt) to get the start address even if no valid L2 >> header >> - * could be found. >> + * Returns pointer to the start of the current packet level tailroom. >> + * >> + * User can adjust the tail pointer with tail_push/tail_pull (does not >> modify >> + * segmentation) and add_data/rem_data calls (may modify segmentation). >> * >> * @param pkt Packet handle >> * >> - * @return Pointer to the start of the packet buffer >> + * @return Tailroom pointer >> * >> - * @see odp_packet_l2(), odp_packet_start() >> + * @see odp_packet_tailroom() >> */ >> -uint8_t *odp_packet_buf_addr(odp_packet_t pkt); >> +void *odp_packet_tail(odp_packet_t pkt); >> >> /** >> - * Get pointer to the start of the received frame >> + * Push out packet head >> + * >> + * Increase packet data length by moving packet head into packet >> headroom. >> + * Packet headroom is decreased with the same amount. The packet head >> may be >> + * pushed out up to 'headroom' bytes. Packet is not modified if there's >> not >> + * enough headroom space. >> * >> - * The address of the packet buffer is not necessarily the same as the >> start >> - * address of the received frame, e.g. an eth frame may be offset by 2 >> or 6 >> - * bytes to ensure 32 or 64-bit alignment of the IP header. >> - * Use odp_packet_l2(pkt) to get the start address of a received valid >> eth frame >> + * odp_packet_xxx: >> + * seg_len += len >> + * len += len >> + * headroom -= len >> + * data -= len >> * >> - * odp_packet_start() will always return a pointer to the start of the >> frame, >> - * even if the frame is unrecognized and no valid L2 header could be >> found. >> + * Operation does not modify packet segmentation or move data. Handles >> and >> + * pointers remain valid. User is responsible to update packet meta-data >> + * offsets when needed. >> * >> * @param pkt Packet handle >> + * @param len Number of bytes to push the head (0 ... headroom) >> * >> - * @return Pointer to the start of the received frame >> + * @return The new data pointer >> + * @retval NULL Requested offset exceeds available headroom >> * >> - * @see odp_packet_l2(), odp_packet_buf_addr() >> + * @see odp_packet_headroom(), odp_packet_pull_head() >> */ >> -uint8_t *odp_packet_start(odp_packet_t pkt); >> +void *odp_packet_push_head(odp_packet_t pkt, uint32_t len); >> >> /** >> - * Get pointer to the start of the L2 frame >> + * Pull in packet head >> * >> - * The L2 frame header address is not necessarily the same as the >> address of the >> - * packet buffer, see odp_packet_buf_addr() >> + * Decrease packet data length by removing data from the head of the >> packet. >> + * Packet headroom is increased with the same amount. Packet head may be >> pulled >> + * in up to seg_len - 1 bytes (i.e. packet data pointer must stay in the >> + * first segment). Packet is not modified if there's not enough data. >> + * >> + * odp_packet_xxx: >> + * seg_len -= len >> + * len -= len >> + * headroom += len >> + * data += len >> + * >> + * Operation does not modify packet segmentation or move data. Handles >> and >> + * pointers remain valid. User is responsible to update packet meta-data >> + * offsets when needed. >> * >> * @param pkt Packet handle >> + * @param len Number of bytes to pull the head (0 ... seg_len - 1) >> * >> - * @return Pointer to L2 header or NULL if not found >> + * @return The new data pointer, or NULL in case of an error. >> + * @retval NULL Requested offset exceeds packet segment length >> * >> - * @see odp_packet_buf_addr(), odp_packet_start() >> + * @see odp_packet_seg_len(), odp_packet_push_head() >> */ >> -uint8_t *odp_packet_l2(odp_packet_t pkt); >> +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len); >> >> /** >> - * Return the byte offset from the packet buffer to the L2 frame >> + * Push out packet tail >> + * >> + * Increase packet data length by moving packet tail into packet >> tailroom. >> + * Packet tailroom is decreased with the same amount. The packet tail >> may be >> + * pushed out up to 'tailroom' bytes. Packet is not modified if there's >> not >> + * enough tailroom. >> + * >> + * last_seg: >> + * data_len += len >> + * >> + * odp_packet_xxx: >> + * len += len >> + * tail += len >> + * tailroom -= len >> + * >> + * Operation does not modify packet segmentation or move data. Handles, >> + * pointers and offsets remain valid. >> * >> * @param pkt Packet handle >> + * @param len Number of bytes to push the tail (0 ... tailroom) >> * >> - * @return L2 byte offset or ODP_PACKET_OFFSET_INVALID if not found >> + * @return The old tail pointer >> + * @retval NULL Requested offset exceeds available tailroom >> + * >> + * @see odp_packet_tailroom(), odp_packet_pull_tail() >> */ >> -size_t odp_packet_l2_offset(odp_packet_t pkt); >> +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len); >> >> /** >> - * Set the byte offset to the L2 frame >> + * Pull in packet tail >> + * >> + * Decrease packet data length by removing data from the tail of the >> packet. >> + * Packet tailroom is increased with the same amount. Packet tail may be >> pulled >> + * in up to last segment data_len - 1 bytes. (i.e. packet tail must stay >> in the >> + * last segment). Packet is not modified if there's not enough data. >> + * >> + * last_seg: >> + * data_len -= len >> + * >> + * odp_packet_xxx: >> + * len -= len >> + * tail -= len >> + * tailroom += len >> + * >> + * Operation does not modify packet segmentation or move data. Handles >> and >> + * pointers remain valid. User is responsible to update packet meta-data >> + * offsets when needed. >> + * >> + * @param pkt Packet handle >> + * @param len Number of bytes to pull the tail (0 ... last_seg:data_len >> - 1) >> + * >> + * @return The new tail pointer >> + * @retval NULL The specified offset exceeds allowable data length >> + */ >> +void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len); >> + >> +/** >> + * Packet offset pointer >> + * >> + * Returns pointer to data in the packet offset. The packet level byte >> offset is >> + * calculated from the current odp_packet_data() position. Optionally >> outputs >> + * handle to the segment and number of data bytes in the segment >> following the >> + * pointer. >> + * >> + * @param pkt Packet handle >> + * @param offset Byte offset into the packet >> + * @param[out] len Number of data bytes remaining in the segment >> (output). >> + * Ignored when NULL. >> + * @param[out] seg Handle to the segment containing the address >> (output). >> + * Ignored when NULL. >> + * >> + * @return Pointer to the offset >> + * @retval NULL Requested offset exceeds packet length >> + */ >> +void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, >> + odp_packet_seg_t *seg); >> + >> +/* >> + * >> + * Meta-data >> + * ******************************************************** >> + * >> + */ >> + >> +/** >> + * Packet pool >> + * >> + * Returns handle to the buffer pool where the packet was allocated from. >> + * >> + * @param pkt Packet handle >> + * >> + * @return Buffer pool handle >> + */ >> +odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt); >> + >> +/** >> + * Packet input interface >> + * >> + * Returns handle to the packet IO interface which received the packet or >> + * ODP_PKTIO_INVALID when the packet was allocated/reset by the >> application. >> + * >> + * @param pkt Packet handle >> + * >> + * @return Packet interface handle >> + * @retval ODP_PKTIO_INVALID Packet was not received >> + */ >> +odp_pktio_t odp_packet_input(odp_packet_t pkt); >> + >> +/** >> + * User context pointer >> + * >> + * Return previously stored user context pointer. >> + * >> + * @param pkt Packet handle >> + * >> + * @return User context pointer >> + */ >> +void *odp_packet_user_ptr(odp_packet_t pkt); >> + >> +/** >> + * Set user context pointer >> + * >> + * Each packet has room for a user defined context. The context can be >> stored >> + * either as a pointer OR as a uint64_t value, but not both at the same >> time. >> + * The latest context set operation determines which one has been stored. >> + * >> + * @param pkt Packet handle >> + * @param ctx User context pointer >> + */ >> +void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ctx); >> + >> +/** >> + * User context data (uint64_t) >> + * >> + * Return previously stored user context uint64_t value. >> + * >> + * @param pkt Packet handle >> + * >> + * @return User context data >> + */ >> +uint64_t odp_packet_user_u64(odp_packet_t pkt); >> + >> +/** >> + * Set user context data (uint64_t) >> + * >> + * Each packet has room for a user defined context. The context can be >> stored >> + * either as a pointer OR as a uint64_t value, but not both at the same >> time. >> + * The latest context set operation determines which one has been stored. >> + * >> + * @param pkt Packet handle >> + * @param ctx User context data >> + */ >> +void odp_packet_user_u64_set(odp_packet_t pkt, uint64_t ctx); >> + >> +/** >> + * Layer 2 start pointer >> + * >> + * Returns pointer to the start of the layer 2 header. Optionally, >> outputs >> + * number of data bytes in the segment following the pointer. >> + * >> + * @param pkt Packet handle >> + * @param[out] len Number of data bytes remaining in the segment >> (output). >> + * Ignored when NULL. >> + * >> + * @return Layer 2 start pointer, or offset 0 by default >> + * >> + * @see odp_packet_l2_offset(), odp_packet_l2_offset_set() >> + */ >> +void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len); >> + >> +/** >> + * Layer 2 start offset >> + * >> + * Returns offset to the start of the layer 2 header. The offset is >> calculated >> + * from the current odp_packet_data() position in bytes. >> + * >> + * User is responsible to update the offset when modifying the packet >> data >> + * pointer position. >> + * >> + * @param pkt Packet handle >> + * >> + * @return Layer 2 start offset >> + */ >> +uint32_t odp_packet_l2_offset(odp_packet_t pkt); >> + >> +/** >> + * Set layer 2 start offset >> + * >> + * Set offset to the start of the layer 2 header. The offset is >> calculated from >> + * the current odp_packet_data() position in bytes. Offset must not >> exceed >> + * packet data length. Packet is not modified on an error. >> * >> * @param pkt Packet handle >> - * @param offset L2 byte offset >> + * @param offset Layer 2 start offset (0 ... odp_packet_len()-1) >> + * >> + * @retval 0 Success >> + * @retval Non-zero Failure >> */ >> -void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset); >> +int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset); >> >> +/** >> + * Layer 3 start pointer >> + * >> + * Returns pointer to the start of the layer 3 header. Optionally, >> outputs >> + * number of data bytes in the segment following the pointer. >> + * >> + * @param pkt Packet handle >> + * @param[out] len Number of data bytes remaining in the segment >> (output). >> + * Ignored when NULL. >> + * >> + * @return Layer 3 start pointer, or NULL >> + * >> + * @see odp_packet_l3_offset(), odp_packet_l3_offset_set() >> + */ >> +void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len); >> >> /** >> - * Get pointer to the start of the L3 packet >> + * Layer 3 start offset >> + * >> + * Returns offset to the start of the layer 3 header. The offset is >> calculated >> + * from the current odp_packet_data() position in bytes. >> + * >> + * User is responsible to update the offset when modifying the packet >> data >> + * pointer position. >> * >> * @param pkt Packet handle >> * >> - * @return Pointer to L3 packet or NULL if not found >> + * @return Layer 3 start offset or ODP_PACKET_OFFSET_INVALID if not >> found >> + */ >> +uint32_t odp_packet_l3_offset(odp_packet_t pkt); >> + >> +/** >> + * Set layer 3 start offset >> + * >> + * Set offset to the start of the layer 3 header. The offset is >> calculated from >> + * the current odp_packet_data() position in bytes. Offset must not >> exceed >> + * packet data length. Packet is not modified on an error. >> + * >> + * @param pkt Packet handle >> + * @param offset Layer 3 start offset (0 ... odp_packet_len()-1) >> * >> + * @retval 0 Success >> + * @retval Non-zero Failure >> */ >> -uint8_t *odp_packet_l3(odp_packet_t pkt); >> +int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset); >> >> /** >> - * Return the byte offset from the packet buffer to the L3 packet >> + * Layer 4 start pointer >> + * >> + * Returns pointer to the start of the layer 4 header. Optionally, >> outputs >> + * number of data bytes in the segment following the pointer. >> + * >> + * @param pkt Packet handle >> + * @param[out] len Number of data bytes remaining in the segment >> (output). >> + * Ignored when NULL. >> + * >> + * @return Layer 4 start pointer, or NULL >> + * >> + * @see odp_packet_l4_offset(), odp_packet_l4_offset_set() >> + */ >> +void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len); >> + >> +/** >> + * Layer 4 start offset >> + * >> + * Returns offset to the start of the layer 4 header. The offset is >> calculated >> + * from the current odp_packet_data() position in bytes. >> + * >> + * User is responsible to update the offset when modifying the packet >> data >> + * pointer position. >> * >> * @param pkt Packet handle >> * >> - * @return L3 byte offset or ODP_PACKET_OFFSET_INVALID if not found >> + * @return Layer 4 start offset or ODP_PACKET_OFFSET_INVALID if not >> found >> */ >> -size_t odp_packet_l3_offset(odp_packet_t pkt); >> +uint32_t odp_packet_l4_offset(odp_packet_t pkt); >> >> /** >> - * Set the byte offset to the L3 packet >> + * Set layer 4 start offset >> + * >> + * Set offset to the start of the layer 4 header. The offset is >> calculated from >> + * the current odp_packet_data() position in bytes. Offset must not >> exceed >> + * packet data length. Packet is not modified on an error. >> * >> * @param pkt Packet handle >> - * @param offset L3 byte offset >> + * @param offset Layer 4 start offset (0 ... odp_packet_len()-1) >> + * >> + * @retval 0 Success >> + * @retval Non-zero Failure >> + */ >> +int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset); >> + >> +/** >> + * Tests if packet is segmented >> + * >> + * @param pkt Packet handle >> + * >> + * @retval 0 Packet is not segmented >> + * @retval 1 Packet is segmented >> + */ >> +int odp_packet_is_segmented(odp_packet_t pkt); >> + >> +/** >> + * Number of segments >> + * >> + * Returns number of segments in the packet. A packet has always at >> least one >> + * segment. >> + * >> + * @param pkt Packet handle >> + * >> + * @return Number of segments (>0) >> + */ >> +int odp_packet_num_segs(odp_packet_t pkt); >> + >> +/** >> + * First segment in packet >> + * >> + * A packet has always the first segment (has at least one segment). >> + * >> + * @param pkt Packet handle >> + * >> + * @return Handle to the first segment >> + */ >> +odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt); >> + >> +/** >> + * Last segment in packet >> + * >> + * A packet has always the last segment (has at least one segment). >> + * >> + * @param pkt Packet handle >> + * >> + * @return Handle to the last segment >> + */ >> +odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt); >> + >> +/** >> + * Next segment in packet >> + * >> + * Returns handle to the next segment after the current segment, or >> + * ODP_PACKET_SEG_INVALID if there are no more segments. Use >> + * odp_packet_first_seg() to get handle to the first segment. >> + * >> + * @param pkt Packet handle >> + * @param seg Current segment handle >> + * >> + * @return Handle to the next segment, or ODP_PACKET_SEG_INVALID >> + */ >> +odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t >> seg); >> + >> + >> +/* >> + * >> + * Segment level >> + * ******************************************************** >> + * >> */ >> -void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset); >> >> +/** >> + * Segment buffer address >> + * >> + * Returns start address of the segment. >> + * >> + * @param pkt Packet handle >> + * @param seg Segment handle >> + * >> + * @return Start address of the segment, or NULL on an error >> + * >> + * @see odp_packet_seg_buf_len() >> + */ >> +void *odp_packet_seg_buf_addr(odp_packet_t pkt, odp_packet_seg_t seg); >> >> /** >> - * Get pointer to the start of the L4 packet >> + * Segment buffer length >> + * >> + * Returns segment buffer length in bytes. >> * >> * @param pkt Packet handle >> + * @param seg Segment handle >> * >> - * @return Pointer to L4 packet or NULL if not found >> + * @return Segment buffer length in bytes >> * >> + * @see odp_packet_seg_buf_addr() >> */ >> -uint8_t *odp_packet_l4(odp_packet_t pkt); >> +uint32_t odp_packet_seg_buf_len(odp_packet_t pkt, odp_packet_seg_t seg); >> >> /** >> - * Return the byte offset from the packet buffer to the L4 packet >> + * Segment data pointer >> + * >> + * Returns pointer to the first byte of data in the segment. >> + * >> + * @param pkt Packet handle >> + * @param seg Segment handle >> + * >> + * @return Pointer to the segment data, or NULL on an error >> + * >> + * @see odp_packet_seg_data_len() >> + */ >> +void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg); >> + >> +/** >> + * Segment data length >> + * >> + * Returns segment data length in bytes. >> * >> * @param pkt Packet handle >> + * @param seg Segment handle >> * >> - * @return L4 byte offset or ODP_PACKET_OFFSET_INVALID if not found >> + * @return Segment data length in bytes >> + * >> + * @see odp_packet_seg_data() >> */ >> -size_t odp_packet_l4_offset(odp_packet_t pkt); >> +uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg); >> + >> + >> +/* >> + * >> + * Manipulation >> + * ******************************************************** >> + * >> + */ >> + >> >> /** >> - * Set the byte offset to the L4 packet >> + * Add data into an offset >> + * >> + * Increases packet data length by adding new data area into the >> specified >> + * offset. The operation returns a new packet handle on success. It may >> modify >> + * packet segmentation and move data. Handles and pointers must be >> updated >> + * after the operation. User is responsible to update packet meta-data >> offsets >> + * when needed. The packet is not modified on an error. >> * >> * @param pkt Packet handle >> - * @param offset L4 byte offset >> + * @param offset Byte offset into the packet >> + * @param len Number of bytes to add into the offset >> + * >> + * @return New packet handle, or ODP_PACKET_INVALID in case of an error. >> */ >> -void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset); >> +odp_packet_t odp_packet_add_data(odp_packet_t pkt, uint32_t offset, >> + uint32_t len); >> >> /** >> - * Print (debug) information about the packet >> + * Remove data from an offset >> + * >> + * Decreases packet data length by removing data from the specified >> offset. >> + * The operation returns a new packet handle on success, and may modify >> + * packet segmentation and move data. Handles and pointers must be >> updated >> + * after the operation. User is responsible to update packet meta-data >> offsets >> + * when needed. The packet is not modified on an error. >> + * >> + * @param pkt Packet handle >> + * @param offset Byte offset into the packet >> + * @param len Number of bytes to remove from the offset >> + * >> + * @return New packet handle, or ODP_PACKET_INVALID in case of an error. >> + */ >> +odp_packet_t odp_packet_rem_data(odp_packet_t pkt, uint32_t offset, >> + uint32_t len); >> + >> + >> +/* >> + * >> + * Copy >> + * ******************************************************** >> + * >> + */ >> + >> +/** >> + * Copy packet >> + * >> + * Create a new copy of the packet. The new packet is exact copy of the >> source >> + * packet (incl. data and meta-data). The pool must have been created >> with >> + * buffer type ODP_BUFFER_TYPE_PACKET. >> + * >> + * @param pkt Packet handle >> + * @param pool Buffer pool for allocation of the new packet. >> + * >> + * @return Handle to the copy of the packet, or ODP_PACKET_INVALID >> + */ >> +odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool); >> + >> +/** >> + * Copy data from packet >> + * >> + * Copy 'len' bytes of data from the packet level offset to the >> destination >> + * address. >> + * >> + * @param pkt Packet handle >> + * @param offset Byte offset into the packet >> + * @param len Number of bytes to copy >> + * @param dst Destination address >> + * >> + * @retval 0 Success >> + * @retval Non-zero Failure >> + */ >> +int odp_packet_copydata_out(odp_packet_t pkt, uint32_t offset, >> + uint32_t len, void *dst); >> + >> +/** >> + * Copy data into packet >> + * >> + * Copy 'len' bytes of data from the source address into the packet >> level >> + * offset. Maximum number of bytes to copy is packet data length minus >> the >> + * offset. Packet is not modified on an error. >> + * >> + * @param pkt Packet handle >> + * @param offset Byte offset into the packet >> + * @param len Number of bytes to copy >> + * @param src Source address >> + * >> + * @retval 0 Success >> + * @retval Non-zero Failure >> + */ >> +int odp_packet_copydata_in(odp_packet_t pkt, uint32_t offset, >> + uint32_t len, const void *src); >> + >> +/* >> + * >> + * Debugging >> + * ******************************************************** >> + * >> + */ >> + >> +/** >> + * Print packet to the console >> + * >> + * Print all packet debug information to the console. >> * >> * @param pkt Packet handle >> */ >> void odp_packet_print(odp_packet_t pkt); >> >> /** >> - * Copy contents and metadata from pkt_src to pkt_dst >> - * Useful when creating copies of packets >> + * Perform full packet validity check >> * >> - * @param pkt_dst Destination packet >> - * @param pkt_src Source packet >> + * The operation may consume considerable number of cpu cycles depending >> on >> + * the check level. >> + * >> + * @param pkt Packet handle >> * >> - * @return 0 if successful >> + * @retval 0 Packet is not valid >> + * @retval 1 Packet is valid >> + */ >> +int odp_packet_is_valid(odp_packet_t pkt); >> + >> + >> +/** >> + * @} >> */ >> -int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src); >> >> #ifdef __cplusplus >> } >> diff --git a/platform/linux-dpdk/include/api/odp_packet_io.h >> b/platform/linux-dpdk/include/api/odp_packet_io.h >> index bd6868a..787df7d 100644 >> --- a/platform/linux-dpdk/include/api/odp_packet_io.h >> +++ b/platform/linux-dpdk/include/api/odp_packet_io.h >> @@ -1,4 +1,4 @@ >> -/* Copyright (c) 2014, Linaro Limited >> +/* Copyright (c) 2013, Linaro Limited >> * All rights reserved. >> * >> * SPDX-License-Identifier: BSD-3-Clause >> @@ -19,29 +19,29 @@ extern "C" { >> #endif >> >> #include <odp_std_types.h> >> +#include <odp_platform_types.h> >> #include <odp_buffer_pool.h> >> #include <odp_packet.h> >> #include <odp_queue.h> >> >> -#include <odp_pktio_types.h> >> - >> -/** ODP packet IO handle */ >> -typedef uint32_t odp_pktio_t; >> - >> -/** Invalid packet IO handle */ >> -#define ODP_PKTIO_INVALID 0 >> +/** @defgroup odp_packet_io ODP PACKET IO >> + * Operations on a packet. >> + * @{ >> + */ >> >> /** >> * Open an ODP packet IO instance >> * >> * @param dev Packet IO device >> * @param pool Pool to use for packet IO >> - * @param params Set of parameters to pass to the arch dependent >> implementation >> * >> * @return ODP packet IO handle or ODP_PKTIO_INVALID on error >> + * >> + * @note dev name loop is specially pktio reserved name for >> + * device used for testing. Usually it's loop back >> + * interface. >> */ >> -odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool, >> - odp_pktio_params_t *params); >> +odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool); >> >> /** >> * Close an ODP packet IO instance >> @@ -130,21 +130,77 @@ void odp_pktio_set_input(odp_packet_t pkt, >> odp_pktio_t id); >> odp_pktio_t odp_pktio_get_input(odp_packet_t pkt); >> >> /** >> - * Defines the maximum length of mac address supported by this platform >> + * Configure the MTU for a packet IO interface. >> + * >> + * @param[in] id ODP packet IO handle. >> + * @param[in] mtu The value of MTU that the interface will be >> configured to >> + * use. >> + * >> + * @retval 0 on success. >> + * @retval -1 if specified mtu can not be handled. >> + * @retval -1 on any other error or illegal input parameters. >> + */ >> +int odp_pktio_set_mtu(odp_pktio_t id, int mtu); >> + >> +/** >> + * Return the currently configured MTU value of a packet IO interface. >> + * >> + * @param[in] id ODP packet IO handle. >> + * >> + * @retval MTU value >0 on success. >> + * @retval -1 on any error or not existance pktio id. >> + */ >> +int odp_pktio_mtu(odp_pktio_t id); >> + >> +/** >> + * Enable/Disable promiscuous mode on a packet IO interface. >> + * >> + * @param[in] id ODP packet IO handle. >> + * @param[in] enable 1 to enable, 0 to disable. >> + * >> + * @retval 0 on success. >> + * @retval non-zero on any error. >> + */ >> +int odp_pktio_promisc_mode_set(odp_pktio_t id, odp_bool_t enable); >> + >> +/** >> + * Determine if promiscuous mode is enabled for a packet IO interface. >> + * >> + * @param[in] id ODP packet IO handle. >> + * >> + * @retval 1 if promiscuous mode is enabled. >> + * @retval 0 if promiscuous mode is disabled. >> + * @retval -1 on any error. >> +*/ >> +int odp_pktio_promisc_mode(odp_pktio_t id); >> + >> +/** >> + * Get the default MAC address of a packet IO interface. >> + * >> + * @param id ODP packet IO handle. >> + * @param[out] mac_addr Storage for MAC address of the packet IO >> interface. >> + * @param addr_size Storage size for the address >> + * >> + * @retval Number of bytes written on success, 0 on failure. >> */ >> -#define ODP_MAC_ADDR_MAX_LENGTH ETH_ALEN >> +size_t odp_pktio_mac_addr(odp_pktio_t id, void *mac_addr, >> + size_t addr_size); >> >> /** >> * Get mac address of the interface >> * >> - * @param id ODP packet IO handle >> - * @param mac_addr Storage for Mac address of the packet IO interface >> - * Storage provided by the caller should be equal >> - * to ODP_MAC_ADDR_MAX_LENGTH (filled by function) >> + * @param id ODP packet IO handle >> + * @param mac_addr Storage for Mac address of the packet IO interface >> + * Storage provided by the caller should be equal >> + * to ODP_MAC_ADDR_MAX_LENGTH (filled by function) >> * @return 0 on success or -1 on error >> -**/ >> + */ >> int odp_pktio_get_mac_addr(odp_pktio_t id, unsigned char *mac_addr); >> >> +/** >> + * @} >> + */ >> + >> #ifdef __cplusplus >> } >> #endif >> diff --git a/platform/linux-dpdk/include/api/odp_pktio_types.h >> b/platform/linux-dpdk/include/api/odp_pktio_types.h >> deleted file mode 100644 >> index b23e6da..0000000 >> --- a/platform/linux-dpdk/include/api/odp_pktio_types.h >> +++ /dev/null >> @@ -1,45 +0,0 @@ >> - >> -/* Copyright (c) 2013, Linaro Limited >> - * All rights reserved. >> - * >> - * SPDX-License-Identifier: BSD-3-Clause >> - */ >> - >> -#ifndef ODP_PKTIO_TYPES_H >> -#define ODP_PKTIO_TYPES_H >> - >> -#ifdef __cplusplus >> -extern "C" { >> -#endif >> - >> -/* We should ensure that future enum values will never overlap, otherwise >> - * applications that want netmap suport might get in trouble if the odp >> lib >> - * was not built with netmap support and there are more types define >> below >> - */ >> - >> -typedef enum { >> - ODP_PKTIO_TYPE_SOCKET_BASIC = 0x1, >> - ODP_PKTIO_TYPE_SOCKET_MMSG, >> - ODP_PKTIO_TYPE_SOCKET_MMAP, >> - ODP_PKTIO_TYPE_NETMAP, >> - ODP_PKTIO_TYPE_DPDK, >> -} odp_pktio_type_t; >> - >> -#include <odp_pktio_socket.h> >> -#ifdef ODP_HAVE_NETMAP >> -#include <odp_pktio_netmap.h> >> -#endif >> - >> -typedef union odp_pktio_params_t { >> - odp_pktio_type_t type; >> - socket_params_t sock_params; >> -#ifdef ODP_HAVE_NETMAP >> - netmap_params_t nm_params; >> -#endif >> -} odp_pktio_params_t; >> - >> -#ifdef __cplusplus >> -} >> -#endif >> - >> -#endif >> diff --git a/platform/linux-dpdk/include/api/odp_platform_types.h >> b/platform/linux-dpdk/include/api/odp_platform_types.h >> new file mode 100644 >> index 0000000..3bedb5d >> --- /dev/null >> +++ b/platform/linux-dpdk/include/api/odp_platform_types.h >> @@ -0,0 +1,78 @@ >> +/* Copyright (c) 2014, Linaro Limited >> + * All rights reserved. >> + * >> + * SPDX-License-Identifier: BSD-3-Clause >> + */ >> + >> + >> +/** >> + * @file >> + * ODP implementation types >> + * This file contains all of the implementation-defined types for ODP >> abstract >> + * definitions. Having this in one file means that other ODP API files >> are >> + * implementation-independent and avoids circular dependencies for files >> that >> + * refer to types managed by other components. Included here are >> typedefs and >> + * related typed constants that are referenced by other ODP API files. >> + */ >> + >> +#ifndef ODP_IMPL_TYPES_H_ >> +#define ODP_IMPL_TYPES_H_ >> + >> +/** @defgroup odp_platform_types ODP PLATFORM TYPES >> + * Implementation specific definitions for ODP abstract types. >> + * @{ >> + */ >> + >> +/** ODP Buffer pool */ >> +typedef unsigned long odp_buffer_pool_t; >> + >> +/** Invalid buffer pool */ >> +#define ODP_BUFFER_POOL_INVALID (0xffffffff) >> + >> +/** ODP buffer */ >> +typedef unsigned long odp_buffer_t; >> + >> +/** Invalid buffer */ >> +#define ODP_BUFFER_INVALID (0xffffffff) >> + >> +/** ODP buffer segment */ >> +typedef odp_buffer_t odp_buffer_seg_t; >> + >> +/** Invalid segment */ >> +#define ODP_SEGMENT_INVALID ODP_BUFFER_INVALID >> + >> +/** ODP packet */ >> +typedef odp_buffer_t odp_packet_t; >> + >> +/** Invalid packet */ >> +#define ODP_PACKET_INVALID ODP_BUFFER_INVALID >> + >> +/** ODP packet segment */ >> +typedef odp_buffer_t odp_packet_seg_t; >> + >> +/** Invalid packet segment */ >> +#define ODP_PACKET_SEG_INVALID ODP_BUFFER_INVALID >> + >> +/** ODP packet IO handle */ >> +typedef uint32_t odp_pktio_t; >> + >> +/** Invalid packet IO handle */ >> +#define ODP_PKTIO_INVALID 0 >> + >> +/** odp_pktio_t value to indicate any port */ >> +#define ODP_PKTIO_ANY ((odp_pktio_t)~0) >> + >> +/** >> + * ODP shared memory block >> + */ >> +typedef uint32_t odp_shm_t; >> + >> +/** Invalid shared memory block */ >> +#define ODP_SHM_INVALID 0 >> +#define ODP_SHM_NULL ODP_SHM_INVALID /**< Synonym for buffer pool use */ >> + >> +/** >> + * @} >> + */ >> + >> +#endif >> diff --git a/platform/linux-dpdk/include/odp_buffer_inlines.h >> b/platform/linux-dpdk/include/odp_buffer_inlines.h >> new file mode 100644 >> index 0000000..ebf600b >> --- /dev/null >> +++ b/platform/linux-dpdk/include/odp_buffer_inlines.h >> @@ -0,0 +1,179 @@ >> +/* Copyright (c) 2014, Linaro Limited >> + * All rights reserved. >> + * >> + * SPDX-License-Identifier: BSD-3-Clause >> + */ >> + >> +/** >> + * @file >> + * >> + * Inline functions for ODP buffer mgmt routines - implementation >> internal >> + */ >> + >> +#ifndef ODP_BUFFER_INLINES_H_ >> +#define ODP_BUFFER_INLINES_H_ >> + >> +#ifdef __cplusplus >> +extern "C" { >> +#endif >> + >> +static inline odp_buffer_t odp_buffer_encode_handle(odp_buffer_hdr_t >> *hdr) >> +{ >> + odp_buffer_bits_t handle; >> + uint32_t pool_id = pool_handle_to_index(hdr->pool_hdl); >> + struct pool_entry_s *pool = get_pool_entry(pool_id); >> + >> + handle.pool_id = pool_id; >> + handle.index = ((uint8_t *)hdr - pool->pool_mdata_addr) / >> + ODP_CACHE_LINE_SIZE; >> + handle.seg = 0; >> + >> + return handle.u32; >> +} >> + >> +static inline odp_buffer_t odp_hdr_to_buf(odp_buffer_hdr_t *hdr) >> +{ >> + return hdr->handle.handle; >> +} >> + >> +static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf) >> +{ >> + return (odp_buffer_hdr_t *)buf; >> +} >> + >> +static inline uint32_t odp_buffer_refcount(odp_buffer_hdr_t *buf) >> +{ >> + return odp_atomic_load_u32(&buf->ref_count); >> +} >> + >> +static inline uint32_t odp_buffer_incr_refcount(odp_buffer_hdr_t *buf, >> + uint32_t val) >> +{ >> + return odp_atomic_fetch_add_u32(&buf->ref_count, val) + val; >> +} >> + >> +static inline uint32_t odp_buffer_decr_refcount(odp_buffer_hdr_t *buf, >> + uint32_t val) >> +{ >> + uint32_t tmp; >> + >> + tmp = odp_atomic_fetch_sub_u32(&buf->ref_count, val); >> + >> + if (tmp < val) { >> + odp_atomic_fetch_add_u32(&buf->ref_count, val - tmp); >> + return 0; >> + } else { >> + return tmp - val; >> + } >> +} >> + >> +static inline odp_buffer_hdr_t *validate_buf(odp_buffer_t buf) >> +{ >> + odp_buffer_bits_t handle; >> + odp_buffer_hdr_t *buf_hdr; >> + handle.u32 = buf; >> + >> + /* For buffer handles, segment index must be 0 and pool id in >> range */ >> + if (handle.seg != 0 || handle.pool_id >= ODP_CONFIG_BUFFER_POOLS) >> + return NULL; >> + >> + pool_entry_t *pool = odp_pool_to_entry(handle.pool_id); >> + >> + /* If pool not created, handle is invalid */ >> + if (pool->s.pool_shm == ODP_SHM_INVALID) >> + return NULL; >> + >> + uint32_t buf_stride = pool->s.buf_stride / ODP_CACHE_LINE_SIZE; >> + >> + /* A valid buffer index must be on stride, and must be in range */ >> + if ((handle.index % buf_stride != 0) || >> + ((uint32_t)(handle.index / buf_stride) >= >> pool->s.params.num_bufs)) >> + return NULL; >> + >> + buf_hdr = (odp_buffer_hdr_t *)(void *) >> + (pool->s.pool_mdata_addr + >> + (handle.index * ODP_CACHE_LINE_SIZE)); >> + >> + /* Handle is valid, so buffer is valid if it is allocated */ >> + return buf_hdr->allocator == ODP_FREEBUF ? NULL : buf_hdr; >> +} >> + >> +int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf); >> + >> +static inline void *buffer_map(odp_buffer_hdr_t *buf, >> + uint32_t offset, >> + uint32_t *seglen, >> + uint32_t limit) >> +{ >> + int seg_index = offset / buf->segsize; >> + int seg_offset = offset % buf->segsize; >> + >> + if (seglen != NULL) { >> + uint32_t buf_left = limit - offset; >> + *seglen = buf_left < buf->segsize ? >> + buf_left : buf->segsize - seg_offset; >> + } >> + >> + return (void *)(seg_offset + (uint8_t *)buf->addr[seg_index]); >> +} >> + >> +static inline odp_buffer_seg_t segment_next(odp_buffer_hdr_t *buf, >> + odp_buffer_seg_t seg) >> +{ >> + odp_buffer_bits_t seghandle; >> + seghandle.u32 = seg; >> + >> + if (seg == ODP_SEGMENT_INVALID || >> + seghandle.prefix != buf->handle.prefix || >> + seghandle.seg >= buf->segcount - 1) >> + return ODP_SEGMENT_INVALID; >> + else { >> + seghandle.seg++; >> + return (odp_buffer_seg_t)seghandle.u32; >> + } >> +} >> + >> +static inline void *segment_map(odp_buffer_hdr_t *buf, >> + odp_buffer_seg_t seg, >> + uint32_t *seglen, >> + uint32_t limit, >> + uint32_t hr) >> +{ >> + uint32_t seg_offset, buf_left; >> + odp_buffer_bits_t seghandle; >> + uint8_t *seg_addr; >> + seghandle.u32 = seg; >> + >> + if (seghandle.prefix != buf->handle.prefix || >> + seghandle.seg >= buf->segcount) >> + return NULL; >> + >> + seg_addr = (uint8_t *)buf->addr[seghandle.seg]; >> + seg_offset = seghandle.seg * buf->segsize; >> + limit += hr; >> + >> + /* Can't map this segment if it's nothing but headroom or >> tailroom */ >> + if (hr >= seg_offset + buf->segsize || seg_offset > limit) >> + return NULL; >> + >> + /* Adjust address & offset if this segment contains any headroom >> */ >> + if (hr > seg_offset) { >> + seg_addr += hr % buf->segsize; >> + seg_offset += hr % buf->segsize; >> + } >> + >> + /* Set seglen if caller is asking for it */ >> + if (seglen != NULL) { >> + buf_left = limit - seg_offset; >> + *seglen = buf_left < buf->segsize ? buf_left : >> buf->segsize; >> + } >> + >> + return (void *)seg_addr; >> +} >> + >> + >> +#ifdef __cplusplus >> +} >> +#endif >> + >> +#endif >> diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h >> b/platform/linux-dpdk/include/odp_buffer_internal.h >> index 5406606..acc958b 100644 >> --- a/platform/linux-dpdk/include/odp_buffer_internal.h >> +++ b/platform/linux-dpdk/include/odp_buffer_internal.h >> @@ -24,50 +24,144 @@ extern "C" { >> #include <odp_buffer.h> >> #include <odp_debug.h> >> #include <odp_align.h> >> -#include <rte_mbuf.h> >> +#include <odp_align_internal.h> >> +#include <odp_config.h> >> +#include <odp_byteorder.h> >> +#include <odp_thread.h> >> >> -/* TODO: move these to correct files */ >> +/* DPDK */ >> +#include <rte_mbuf.h> >> >> -typedef uint64_t odp_phys_addr_t; >> +#define ODP_BITSIZE(x) \ >> + ((x) <= 2 ? 1 : \ >> + ((x) <= 4 ? 2 : \ >> + ((x) <= 8 ? 3 : \ >> + ((x) <= 16 ? 4 : \ >> + ((x) <= 32 ? 5 : \ >> + ((x) <= 64 ? 6 : \ >> + ((x) <= 128 ? 7 : \ >> + ((x) <= 256 ? 8 : \ >> + ((x) <= 512 ? 9 : \ >> + ((x) <= 1024 ? 10 : \ >> + ((x) <= 2048 ? 11 : \ >> + ((x) <= 4096 ? 12 : \ >> + ((x) <= 8196 ? 13 : \ >> + ((x) <= 16384 ? 14 : \ >> + ((x) <= 32768 ? 15 : \ >> + ((x) <= 65536 ? 16 : \ >> + (0/0))))))))))))))))) >> + >> +_ODP_STATIC_ASSERT(ODP_CONFIG_PACKET_BUF_LEN_MIN >= 256, >> + "ODP Segment size must be a minimum of 256 bytes"); >> + >> +_ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MIN % ODP_CACHE_LINE_SIZE) >> == 0, >> + "ODP Segment size must be a multiple of cache line >> size"); >> + >> +_ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MAX % >> + ODP_CONFIG_PACKET_BUF_LEN_MIN) == 0, >> + "Packet max size must be a multiple of segment size"); >> + >> +#define ODP_BUFFER_MAX_SEG \ >> + (ODP_CONFIG_PACKET_BUF_LEN_MAX / ODP_CONFIG_PACKET_BUF_LEN_MIN) >> + >> +/* We can optimize storage of small raw buffers within metadata area */ >> +#define ODP_MAX_INLINE_BUF ((sizeof(void *)) * (ODP_BUFFER_MAX_SEG - >> 1)) >> + >> +#define ODP_BUFFER_POOL_BITS ODP_BITSIZE(ODP_CONFIG_BUFFER_POOLS) >> +#define ODP_BUFFER_SEG_BITS ODP_BITSIZE(ODP_BUFFER_MAX_SEG) >> +#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS - >> ODP_BUFFER_SEG_BITS) >> +#define ODP_BUFFER_PREFIX_BITS (ODP_BUFFER_POOL_BITS + >> ODP_BUFFER_INDEX_BITS) >> +#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS) >> +#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS) >> >> #define ODP_BUFFER_MAX_INDEX (ODP_BUFFER_MAX_BUFFERS - 2) >> #define ODP_BUFFER_INVALID_INDEX (ODP_BUFFER_MAX_BUFFERS - 1) >> >> -#define ODP_BUFS_PER_CHUNK 16 >> -#define ODP_BUFS_PER_SCATTER 4 >> - >> -#define ODP_BUFFER_TYPE_CHUNK 0xffff >> - >> - >> -#define ODP_BUFFER_POOL_BITS 4 >> -#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS) >> -#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS) >> -#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS) >> - >> typedef union odp_buffer_bits_t { >> uint32_t u32; >> odp_buffer_t handle; >> >> struct { >> - uint32_t pool:ODP_BUFFER_POOL_BITS; >> +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN >> + uint32_t pool_id:ODP_BUFFER_POOL_BITS; >> + uint32_t index:ODP_BUFFER_INDEX_BITS; >> + uint32_t seg:ODP_BUFFER_SEG_BITS; >> +#else >> + uint32_t seg:ODP_BUFFER_SEG_BITS; >> uint32_t index:ODP_BUFFER_INDEX_BITS; >> + uint32_t pool_id:ODP_BUFFER_POOL_BITS; >> +#endif >> }; >> -} odp_buffer_bits_t; >> >> + struct { >> +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN >> + uint32_t prefix:ODP_BUFFER_PREFIX_BITS; >> + uint32_t pfxseg:ODP_BUFFER_SEG_BITS; >> +#else >> + uint32_t pfxseg:ODP_BUFFER_SEG_BITS; >> + uint32_t prefix:ODP_BUFFER_PREFIX_BITS; >> +#endif >> + }; >> +} odp_buffer_bits_t; >> >> /* forward declaration */ >> struct odp_buffer_hdr_t; >> >> - >> +/* Common buffer header */ >> typedef struct odp_buffer_hdr_t { >> + /* DPDK specific */ >> struct rte_mbuf mb; /* Underlying DPDK rte_mbuf */ >> - struct odp_buffer_hdr_t *next; /* Next buf in a list */ >> - int type; /* ODP buffer type; not DPDK buf >> type */ >> uint32_t index; /* Index in the rte_mempool */ >> + >> + struct odp_buffer_hdr_t *next; /* next buf in a list */ >> + int allocator; /* allocating thread id */ >> + odp_buffer_bits_t handle; /* handle */ >> + union { >> + uint32_t all; >> + struct { >> + uint32_t zeroized:1; /* Zeroize buf data on free >> */ >> + uint32_t hdrdata:1; /* Data is in buffer hdr */ >> + }; >> + } flags; >> + int type; /* buffer type */ >> + size_t size; /* max data size */ >> + odp_atomic_u32_t ref_count; /* reference count */ >> + odp_buffer_pool_t pool_hdl; /* buffer pool handle */ >> + union { >> + uint64_t buf_u64; /* user u64 */ >> + void *buf_ctx; /* user context */ >> + const void *buf_cctx; /* const alias for ctx */ >> + void *udata_addr; /* user metadata addr */ >> + }; >> + size_t udata_size; /* size of user metadata */ >> + uint32_t segcount; /* segment count */ >> + uint32_t segsize; /* segment size */ >> + void *addr[ODP_BUFFER_MAX_SEG]; /* block addrs >> */ >> + >> } odp_buffer_hdr_t; >> >> -int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf); >> +typedef struct odp_buffer_hdr_stride { >> + uint8_t >> pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_buffer_hdr_t))]; >> +} odp_buffer_hdr_stride; >> +/* Ensure next header starts from 8 byte align */ >> +_ODP_STATIC_ASSERT((sizeof(odp_buffer_hdr_t) % 8) == 0, >> + "ODP_BUFFER_HDR_T__SIZE_ERROR"); >> + >> +typedef struct odp_buf_blk_t { >> + struct odp_buf_blk_t *next; >> + struct odp_buf_blk_t *prev; >> +} odp_buf_blk_t; >> + >> +/* Raw buffer header */ >> +typedef struct { >> + odp_buffer_hdr_t buf_hdr; /* common buffer header */ >> +} odp_raw_buffer_hdr_t; >> + >> +/* Free buffer marker */ >> +#define ODP_FREEBUF -1 >> >> +/* Forward declarations */ >> +odp_buffer_t buffer_alloc(odp_buffer_pool_t pool, size_t size); >> >> #ifdef __cplusplus >> } >> diff --git a/platform/linux-dpdk/include/odp_buffer_pool_internal.h >> b/platform/linux-dpdk/include/odp_buffer_pool_internal.h >> index 1a36655..0f19f87 100644 >> --- a/platform/linux-dpdk/include/odp_buffer_pool_internal.h >> +++ b/platform/linux-dpdk/include/odp_buffer_pool_internal.h >> @@ -19,16 +19,48 @@ extern "C" { >> #endif >> >> #include <odp_std_types.h> >> +#include <odp_align.h> >> +#include <odp_align_internal.h> >> #include <odp_buffer_pool.h> >> #include <odp_buffer_internal.h> >> -#include <odp_align.h> >> #include <odp_hints.h> >> #include <odp_config.h> >> #include <odp_debug.h> >> +#include <odp_shared_memory.h> >> +#include <odp_atomic.h> >> +#include <odp_atomic_internal.h> >> +#include <string.h> >> >> -/* for DPDK */ >> +/* DPDK */ >> #include <rte_mempool.h> >> >> +/** >> + * Buffer initialization routine prototype >> + * >> + * @note Routines of this type MAY be passed as part of the >> + * _odp_buffer_pool_init_t structure to be called whenever a >> + * buffer is allocated to initialize the user metadata >> + * associated with that buffer. >> + */ >> +typedef void (_odp_buf_init_t)(odp_buffer_t buf, void *buf_init_arg); >> + >> +/** >> + * Buffer pool initialization parameters >> + * Used to communicate buffer pool initialization options. Internal for >> now. >> + */ >> +typedef struct _odp_buffer_pool_init_t { >> + size_t udata_size; /**< Size of user metadata for each >> buffer */ >> + _odp_buf_init_t *buf_init; /**< Buffer initialization routine to >> use */ >> + void *buf_init_arg; /**< Argument to be passed to >> buf_init() */ >> +} _odp_buffer_pool_init_t; /**< Type of buffer initialization >> struct */ >> + >> +/* Local cache for buffer alloc/free acceleration */ >> +typedef struct local_cache_t { >> + odp_buffer_hdr_t *buf_freelist; /* The local cache */ >> + uint64_t bufallocs; /* Local buffer alloc count */ >> + uint64_t buffrees; /* Local buffer free count */ >> +} local_cache_t; >> + >> /* Use ticketlock instead of spinlock */ >> #define POOL_USE_TICKETLOCK >> >> @@ -38,11 +70,16 @@ extern "C" { >> >> #ifdef POOL_USE_TICKETLOCK >> #include <odp_ticketlock.h> >> +#define POOL_LOCK(a) odp_ticketlock_lock(a) >> +#define POOL_UNLOCK(a) odp_ticketlock_unlock(a) >> +#define POOL_LOCK_INIT(a) odp_ticketlock_init(a) >> #else >> #include <odp_spinlock.h> >> +#define POOL_LOCK(a) odp_spinlock_lock(a) >> +#define POOL_UNLOCK(a) odp_spinlock_unlock(a) >> +#define POOL_LOCK_INIT(a) odp_spinlock_init(a) >> #endif >> >> - >> struct pool_entry_s { >> #ifdef POOL_USE_TICKETLOCK >> odp_ticketlock_t lock ODP_ALIGNED_CACHE; >> @@ -50,38 +87,294 @@ struct pool_entry_s { >> odp_spinlock_t lock ODP_ALIGNED_CACHE; >> #endif >> >> - uint64_t free_bufs; >> char name[ODP_BUFFER_POOL_NAME_LEN]; >> - >> - >> - odp_buffer_pool_t pool ODP_ALIGNED_CACHE; >> - uintptr_t buf_base; >> - size_t buf_size; >> - size_t buf_offset; >> - uint64_t num_bufs; >> - void *pool_base_addr; >> - uint64_t pool_size; >> - size_t payload_size; >> - size_t payload_align; >> - int buf_type; >> - size_t hdr_size; >> + odp_buffer_pool_param_t params; >> + _odp_buffer_pool_init_t init_params; >> + odp_buffer_pool_t pool_hdl; >> + uint32_t pool_id; >> + odp_shm_t pool_shm; >> + union { >> + uint32_t all; >> + struct { >> + uint32_t has_name:1; >> + uint32_t user_supplied_shm:1; >> + uint32_t unsegmented:1; >> + uint32_t zeroized:1; >> + uint32_t predefined:1; >> + }; >> + } flags; >> + uint32_t quiesced; >> + uint32_t low_wm_assert; >> + uint8_t *pool_base_addr; >> + uint8_t *pool_mdata_addr; >> + size_t pool_size; >> + uint32_t buf_align; >> + uint32_t buf_stride; >> + _odp_atomic_ptr_t buf_freelist; >> + _odp_atomic_ptr_t blk_freelist; >> + odp_atomic_u32_t bufcount; >> + odp_atomic_u32_t blkcount; >> + odp_atomic_u64_t bufallocs; >> + odp_atomic_u64_t buffrees; >> + odp_atomic_u64_t blkallocs; >> + odp_atomic_u64_t blkfrees; >> + odp_atomic_u64_t bufempty; >> + odp_atomic_u64_t blkempty; >> + odp_atomic_u64_t high_wm_count; >> + odp_atomic_u64_t low_wm_count; >> + uint32_t seg_size; >> + uint32_t high_wm; >> + uint32_t low_wm; >> + uint32_t headroom; >> + uint32_t tailroom; >> }; >> >> +typedef union pool_entry_u { >> + struct pool_entry_s s; >> + >> + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct >> pool_entry_s))]; >> +} pool_entry_t; >> >> extern void *pool_entry_ptr[]; >> >> +#if defined(ODP_CONFIG_SECURE_POOLS) && (ODP_CONFIG_SECURE_POOLS == 1) >> +#define buffer_is_secure(buf) (buf->flags.zeroized) >> +#define pool_is_secure(pool) (pool->flags.zeroized) >> +#else >> +#define buffer_is_secure(buf) 0 >> +#define pool_is_secure(pool) 0 >> +#endif >> + >> +#define TAG_ALIGN ((size_t)16) >> + >> +#define odp_cs(ptr, old, new) \ >> + _odp_atomic_ptr_cmp_xchg_strong(&ptr, (void **)&old, (void *)new, >> \ >> + _ODP_MEMMODEL_SC, \ >> + _ODP_MEMMODEL_SC) >> >> -static inline void *get_pool_entry(odp_buffer_pool_t pool_id) >> +/* Helper functions for pointer tagging to avoid ABA race conditions */ >> +#define odp_tag(ptr) \ >> + (((size_t)ptr) & (TAG_ALIGN - 1)) >> + >> +#define odp_detag(ptr) \ >> + ((void *)(((size_t)ptr) & -TAG_ALIGN)) >> + >> +#define odp_retag(ptr, tag) \ >> + ((void *)(((size_t)ptr) | odp_tag(tag))) >> + >> + >> +static inline void *get_blk(struct pool_entry_s *pool) >> +{ >> + void *oldhead, *myhead, *newhead; >> + >> + oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, >> _ODP_MEMMODEL_ACQ); >> + >> + do { >> + size_t tag = odp_tag(oldhead); >> + myhead = odp_detag(oldhead); >> + if (odp_unlikely(myhead == NULL)) >> + break; >> + newhead = odp_retag(((odp_buf_blk_t *)myhead)->next, tag >> + 1); >> + } while (odp_cs(pool->blk_freelist, oldhead, newhead) == 0); >> + >> + if (odp_unlikely(myhead == NULL)) >> + odp_atomic_inc_u64(&pool->blkempty); >> + else >> + odp_atomic_dec_u32(&pool->blkcount); >> + >> + return (void *)myhead; >> +} >> + >> +static inline void ret_blk(struct pool_entry_s *pool, void *block) >> +{ >> + void *oldhead, *myhead, *myblock; >> + >> + oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, >> _ODP_MEMMODEL_ACQ); >> + >> + do { >> + size_t tag = odp_tag(oldhead); >> + myhead = odp_detag(oldhead); >> + ((odp_buf_blk_t *)block)->next = myhead; >> + myblock = odp_retag(block, tag + 1); >> + } while (odp_cs(pool->blk_freelist, oldhead, myblock) == 0); >> + >> + odp_atomic_inc_u32(&pool->blkcount); >> + odp_atomic_inc_u64(&pool->blkfrees); >> +} >> + >> +static inline odp_buffer_hdr_t *get_buf(struct pool_entry_s *pool) >> +{ >> + odp_buffer_hdr_t *oldhead, *myhead, *newhead; >> + >> + oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, >> _ODP_MEMMODEL_ACQ); >> + >> + do { >> + size_t tag = odp_tag(oldhead); >> + myhead = odp_detag(oldhead); >> + if (odp_unlikely(myhead == NULL)) >> + break; >> + newhead = odp_retag(myhead->next, tag + 1); >> + } while (odp_cs(pool->buf_freelist, oldhead, newhead) == 0); >> + >> + if (odp_unlikely(myhead == NULL)) { >> + odp_atomic_inc_u64(&pool->bufempty); >> + } else { >> + uint64_t bufcount = >> + odp_atomic_fetch_sub_u32(&pool->bufcount, 1) - 1; >> + >> + /* Check for low watermark condition */ >> + if (bufcount == pool->low_wm && !pool->low_wm_assert) { >> + pool->low_wm_assert = 1; >> + odp_atomic_inc_u64(&pool->low_wm_count); >> + } >> + >> + odp_atomic_inc_u64(&pool->bufallocs); >> + myhead->next = myhead; /* Mark buffer allocated */ >> + myhead->allocator = odp_thread_id(); >> + } >> + >> + return (void *)myhead; >> +} >> + >> +static inline void ret_buf(struct pool_entry_s *pool, odp_buffer_hdr_t >> *buf) >> +{ >> + odp_buffer_hdr_t *oldhead, *myhead, *mybuf; >> + >> + buf->allocator = ODP_FREEBUF; /* Mark buffer free */ >> + >> + if (!buf->flags.hdrdata && buf->type != ODP_BUFFER_TYPE_RAW) { >> + while (buf->segcount > 0) { >> + if (buffer_is_secure(buf) || pool_is_secure(pool)) >> + memset(buf->addr[buf->segcount - 1], >> + 0, buf->segsize); >> + ret_blk(pool, buf->addr[--buf->segcount]); >> + } >> + buf->size = 0; >> + } >> + >> + oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, >> _ODP_MEMMODEL_ACQ); >> + >> + do { >> + size_t tag = odp_tag(oldhead); >> + myhead = odp_detag(oldhead); >> + buf->next = myhead; >> + mybuf = odp_retag(buf, tag + 1); >> + } while (odp_cs(pool->buf_freelist, oldhead, mybuf) == 0); >> + >> + uint64_t bufcount = odp_atomic_fetch_add_u32(&pool->bufcount, 1) >> + 1; >> + >> + /* Check if low watermark condition should be deasserted */ >> + if (bufcount == pool->high_wm && pool->low_wm_assert) { >> + pool->low_wm_assert = 0; >> + odp_atomic_inc_u64(&pool->high_wm_count); >> + } >> + >> + odp_atomic_inc_u64(&pool->buffrees); >> +} >> + >> +static inline void *get_local_buf(local_cache_t *buf_cache, >> + struct pool_entry_s *pool, >> + size_t totsize) >> +{ >> + odp_buffer_hdr_t *buf = buf_cache->buf_freelist; >> + >> + if (odp_likely(buf != NULL)) { >> + buf_cache->buf_freelist = buf->next; >> + >> + if (odp_unlikely(buf->size < totsize)) { >> + intmax_t needed = totsize - buf->size; >> + >> + do { >> + void *blk = get_blk(pool); >> + if (odp_unlikely(blk == NULL)) { >> + ret_buf(pool, buf); >> + buf_cache->buffrees--; >> + return NULL; >> + } >> + buf->addr[buf->segcount++] = blk; >> + needed -= pool->seg_size; >> + } while (needed > 0); >> + >> + buf->size = buf->segcount * pool->seg_size; >> + } >> + >> + buf_cache->bufallocs++; >> + buf->allocator = odp_thread_id(); /* Mark buffer >> allocated */ >> + } >> + >> + return buf; >> +} >> + >> +static inline void ret_local_buf(local_cache_t *buf_cache, >> + odp_buffer_hdr_t *buf) >> +{ >> + buf->allocator = ODP_FREEBUF; >> + buf->next = buf_cache->buf_freelist; >> + buf_cache->buf_freelist = buf; >> + >> + buf_cache->buffrees++; >> +} >> + >> +static inline void flush_cache(local_cache_t *buf_cache, >> + struct pool_entry_s *pool) >> +{ >> + odp_buffer_hdr_t *buf = buf_cache->buf_freelist; >> + uint32_t flush_count = 0; >> + >> + while (buf != NULL) { >> + odp_buffer_hdr_t *next = buf->next; >> + ret_buf(pool, buf); >> + buf = next; >> + flush_count++; >> + } >> + >> + odp_atomic_add_u64(&pool->bufallocs, buf_cache->bufallocs); >> + odp_atomic_add_u64(&pool->buffrees, buf_cache->buffrees - >> flush_count); >> + >> + buf_cache->buf_freelist = NULL; >> + buf_cache->bufallocs = 0; >> + buf_cache->buffrees = 0; >> +} >> + >> +static inline odp_buffer_pool_t pool_index_to_handle(uint32_t pool_id) >> +{ >> + return pool_id; >> +} >> + >> +static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl) >> +{ >> + return pool_hdl; >> +} >> + >> +static inline void *get_pool_entry(uint32_t pool_id) >> { >> return pool_entry_ptr[pool_id]; >> } >> >> +static inline pool_entry_t *odp_pool_to_entry(odp_buffer_pool_t pool) >> +{ >> + return (pool_entry_t *)get_pool_entry(pool_handle_to_index(pool)); >> +} >> + >> +static inline pool_entry_t *odp_buf_to_pool(odp_buffer_hdr_t *buf) >> +{ >> + return odp_pool_to_entry(buf->pool_hdl); >> +} >> >> -static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf) >> +static inline uint32_t odp_buffer_pool_segment_size(odp_buffer_pool_t >> pool) >> { >> - return (odp_buffer_hdr_t *)buf; >> + return odp_pool_to_entry(pool)->s.seg_size; >> } >> >> +static inline uint32_t odp_buffer_pool_headroom(odp_buffer_pool_t pool) >> +{ >> + return odp_pool_to_entry(pool)->s.headroom; >> +} >> + >> +static inline uint32_t odp_buffer_pool_tailroom(odp_buffer_pool_t pool) >> +{ >> + return odp_pool_to_entry(pool)->s.tailroom; >> +} >> >> #ifdef __cplusplus >> } >> diff --git a/platform/linux-dpdk/include/odp_packet_dpdk.h >> b/platform/linux-dpdk/include/odp_packet_dpdk.h >> index c3a2b70..c982d5e 100644 >> --- a/platform/linux-dpdk/include/odp_packet_dpdk.h >> +++ b/platform/linux-dpdk/include/odp_packet_dpdk.h >> @@ -8,10 +8,8 @@ >> #define ODP_PACKET_DPDK_H >> >> #include <stdint.h> >> -#include <net/if.h> >> >> #include <odph_eth.h> >> -#include <odph_packet.h> >> #include <odp_align.h> >> #include <odp_debug.h> >> #include <odp_packet.h> >> diff --git a/platform/linux-dpdk/include/odp_packet_internal.h >> b/platform/linux-dpdk/include/odp_packet_internal.h >> index 1cd44b1..24de33f 100644 >> --- a/platform/linux-dpdk/include/odp_packet_internal.h >> +++ b/platform/linux-dpdk/include/odp_packet_internal.h >> @@ -22,6 +22,7 @@ extern "C" { >> #include <odp_debug.h> >> #include <odp_buffer_internal.h> >> #include <odp_buffer_pool_internal.h> >> +#include <odp_buffer_inlines.h> >> #include <odp_packet.h> >> #include <odp_packet_io.h> >> >> @@ -43,6 +44,7 @@ typedef union { >> uint32_t vlan:1; /**< VLAN hdr found */ >> uint32_t vlan_qinq:1; /**< Stacked VLAN found, QinQ */ >> >> + uint32_t snap:1; /**< SNAP */ >> uint32_t arp:1; /**< ARP */ >> >> uint32_t ipv4:1; /**< IPv4 */ >> @@ -53,13 +55,14 @@ typedef union { >> >> uint32_t udp:1; /**< UDP */ >> uint32_t tcp:1; /**< TCP */ >> + uint32_t tcpopt:1; /**< TCP options present */ >> uint32_t sctp:1; /**< SCTP */ >> uint32_t icmp:1; /**< ICMP */ >> }; >> } input_flags_t; >> >> -ODP_STATIC_ASSERT(sizeof(input_flags_t) == sizeof(uint32_t), >> - "INPUT_FLAGS_SIZE_ERROR"); >> +_ODP_STATIC_ASSERT(sizeof(input_flags_t) == sizeof(uint32_t), >> + "INPUT_FLAGS_SIZE_ERROR"); >> >> /** >> * Packet error flags >> @@ -70,7 +73,9 @@ typedef union { >> >> struct { >> /* Bitfield flags for each detected error */ >> + uint32_t app_error:1; /**< Error bit for application use >> */ >> uint32_t frame_len:1; /**< Frame length error */ >> + uint32_t snap_len:1; /**< Snap length error */ >> uint32_t l2_chksum:1; /**< L2 checksum error, checks TBD >> */ >> uint32_t ip_err:1; /**< IP error, checks TBD */ >> uint32_t tcp_err:1; /**< TCP error, checks TBD */ >> @@ -78,8 +83,8 @@ typedef union { >> }; >> } error_flags_t; >> >> -ODP_STATIC_ASSERT(sizeof(error_flags_t) == sizeof(uint32_t), >> - "ERROR_FLAGS_SIZE_ERROR"); >> +_ODP_STATIC_ASSERT(sizeof(error_flags_t) == sizeof(uint32_t), >> + "ERROR_FLAGS_SIZE_ERROR"); >> >> /** >> * Packet output flags >> @@ -90,12 +95,15 @@ typedef union { >> >> struct { >> /* Bitfield flags for each output option */ >> - uint32_t l4_chksum:1; /**< Request L4 checksum >> calculation */ >> + uint32_t l3_chksum_set:1; /**< L3 chksum bit is valid */ >> + uint32_t l3_chksum:1; /**< L3 chksum override */ >> + uint32_t l4_chksum_set:1; /**< L3 chksum bit is valid */ >> + uint32_t l4_chksum:1; /**< L4 chksum override */ >> }; >> } output_flags_t; >> >> -ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), >> - "OUTPUT_FLAGS_SIZE_ERROR"); >> +_ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), >> + "OUTPUT_FLAGS_SIZE_ERROR"); >> >> /** >> * Internal Packet header >> @@ -108,28 +116,131 @@ typedef struct { >> error_flags_t error_flags; >> output_flags_t output_flags; >> >> - uint32_t frame_offset; /**< offset to start of frame, even on >> error */ >> uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */ >> uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */ >> uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also >> ICMP) */ >> + uint32_t payload_offset; /**< offset to payload */ >> >> - uint64_t user_ctx; /**< user context */ >> + uint32_t vlan_s_tag; /**< Parsed 1st VLAN header (S-TAG) */ >> + uint32_t vlan_c_tag; /**< Parsed 2nd VLAN header (C-TAG) */ >> + uint32_t l3_protocol; /**< Parsed L3 protocol */ >> + uint32_t l3_len; /**< Layer 3 length */ >> + uint32_t l4_protocol; /**< Parsed L4 protocol */ >> + uint32_t l4_len; /**< Layer 4 length */ >> + >> + uint32_t frame_len; >> + uint32_t headroom; >> + uint32_t tailroom; >> >> odp_pktio_t input; >> } odp_packet_hdr_t; >> >> +typedef struct odp_packet_hdr_stride { >> + uint8_t >> pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_packet_hdr_t))]; >> +} odp_packet_hdr_stride; >> + >> +_ODP_STATIC_ASSERT(sizeof(odp_packet_hdr_t) % sizeof(uint64_t) == 0, >> + "ODP_PACKET_HDR_T__SIZE_ERR2"); >> + >> /** >> * Return the packet header >> */ >> static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) >> { >> - return (odp_packet_hdr_t *)pkt; >> + return (odp_packet_hdr_t *)odp_buf_to_hdr((odp_buffer_t)pkt); >> } >> >> /** >> - * Parse packet and set internal metadata >> + * Initialize packet buffer >> */ >> -void odp_packet_parse(odp_packet_t pkt, size_t len, size_t l2_offset); >> +static inline void packet_init(pool_entry_t *pool, >> + odp_packet_hdr_t *pkt_hdr, >> + size_t size) >> +{ >> + /* >> + * Reset parser metadata. Note that we clear via memset to make >> + * this routine indepenent of any additional adds to packet >> metadata. >> + */ >> + struct rte_mbuf *mb; >> + uint8_t *start; >> + >> + mb = (struct rte_mbuf *)&pkt_hdr->buf_hdr; >> + mb->buf_len = size; >> + start = (uint8_t *)mb->buf_addr; >> + memset(start, 0, mb->buf_len); >> + >> + /* >> + * Packet headroom is set from the pool's headroom >> + * Packet tailroom is rounded up to fill the last >> + * segment occupied by the allocated length. >> + */ >> + pkt_hdr->frame_len = size; >> + pkt_hdr->headroom = pool->s.headroom; >> + pkt_hdr->tailroom = >> + (pool->s.seg_size * pkt_hdr->buf_hdr.segcount) - >> + (pool->s.headroom + size); >> +} >> + >> +static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, >> + uint32_t offset, uint32_t *seglen) >> +{ >> + struct rte_mbuf *mbuf = (struct rte_mbuf *)&pkt_hdr->buf_hdr; >> + if (offset > pkt_hdr->frame_len) >> + return NULL; >> + >> + if (seglen != NULL) { >> + *seglen = mbuf->pkt.data_len; >> + pkt_hdr->frame_len = mbuf->pkt.data_len; >> + } >> + return (void *)((uint8_t *)mbuf->pkt.data + offset); >> +} >> + >> +#define pull_offset(x, len) (x = x < len ? 0 : x - len) >> + >> +static inline void push_head(odp_packet_hdr_t *pkt_hdr, size_t len) >> +{ >> + pkt_hdr->headroom -= len; >> + pkt_hdr->frame_len += len; >> + pkt_hdr->l2_offset += len; >> + pkt_hdr->l3_offset += len; >> + pkt_hdr->l4_offset += len; >> +} >> + >> +static inline void pull_head(odp_packet_hdr_t *pkt_hdr, size_t len) >> +{ >> + pkt_hdr->headroom += len; >> + pkt_hdr->frame_len -= len; >> + pull_offset(pkt_hdr->l2_offset, len); >> + pull_offset(pkt_hdr->l3_offset, len); >> + pull_offset(pkt_hdr->l4_offset, len); >> +} >> + >> +static inline void push_tail(odp_packet_hdr_t *pkt_hdr, size_t len) >> +{ >> + pkt_hdr->tailroom -= len; >> + pkt_hdr->frame_len += len; >> +} >> + >> + >> +static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, size_t len) >> +{ >> + pkt_hdr->tailroom += len; >> + pkt_hdr->frame_len -= len; >> +} >> + >> +static inline void packet_set_len(odp_packet_t pkt, uint32_t len) >> +{ >> + odp_packet_hdr(pkt)->frame_len = len; >> +} >> + >> +/* Forward declarations */ >> +int _odp_packet_copy_to_packet(odp_packet_t srcpkt, uint32_t srcoffset, >> + odp_packet_t dstpkt, uint32_t dstoffset, >> + uint32_t len); >> + >> +odp_packet_t _odp_packet_alloc(odp_buffer_pool_t pool_hdl); >> + >> +int _odp_packet_parse(odp_packet_t pkt); >> >> #ifdef __cplusplus >> } >> diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h >> b/platform/linux-dpdk/include/odp_packet_io_internal.h >> index 9263349..5ece843 100644 >> --- a/platform/linux-dpdk/include/odp_packet_io_internal.h >> +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h >> @@ -20,17 +20,36 @@ extern "C" { >> >> #include <odp_spinlock.h> >> #include <odp_packet_socket.h> >> -#ifdef ODP_HAVE_NETMAP >> -#include <odp_packet_netmap.h> >> -#endif >> +#include <odp_classification_datamodel.h> >> +#include <odp_align_internal.h> >> + >> +#include <odp_config.h> >> +#include <odp_hints.h> >> +#include <linux/if.h> >> + >> +/* DPDK */ >> #include <odp_packet_dpdk.h> >> >> +/** >> + * Packet IO types >> + */ >> +typedef enum { >> + ODP_PKTIO_TYPE_SOCKET_BASIC = 0x1, >> + ODP_PKTIO_TYPE_SOCKET_MMSG, >> + ODP_PKTIO_TYPE_SOCKET_MMAP, >> +} odp_pktio_type_t; >> + >> struct pktio_entry { >> odp_spinlock_t lock; /**< entry spinlock */ >> int taken; /**< is entry taken(1) or free(0) >> */ >> odp_queue_t inq_default; /**< default input queue, if set >> */ >> odp_queue_t outq_default; /**< default out queue */ >> - odp_pktio_params_t params; /**< pktio parameters */ >> + odp_pktio_type_t type; /**< pktio type */ >> + pkt_sock_t pkt_sock; /**< using socket API for IO */ >> + pkt_sock_mmap_t pkt_sock_mmap; /**< using socket mmap API for IO >> */ >> + classifier_t cls; /**< classifier linked with this >> pktio*/ >> + char name[IFNAMSIZ]; /**< name of pktio provided to >> + pktio_open() */ >> pkt_dpdk_t pkt_dpdk; /**< using DPDK API for IO */ >> }; >> >> @@ -39,6 +58,21 @@ typedef union { >> uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct >> pktio_entry))]; >> } pktio_entry_t; >> >> +typedef struct { >> + pktio_entry_t entries[ODP_CONFIG_PKTIO_ENTRIES]; >> +} pktio_table_t; >> + >> +extern void *pktio_entry_ptr[]; >> + >> + >> +static inline pktio_entry_t *get_pktio_entry(odp_pktio_t id) >> +{ >> + if (odp_unlikely(id == ODP_PKTIO_INVALID || >> + id > ODP_CONFIG_PKTIO_ENTRIES)) >> + return NULL; >> + >> + return pktio_entry_ptr[id - 1]; >> +} >> #ifdef __cplusplus >> } >> #endif >> diff --git a/platform/linux-dpdk/odp_buffer.c >> b/platform/linux-dpdk/odp_buffer.c >> index 4914ca2..d89f2af 100644 >> --- a/platform/linux-dpdk/odp_buffer.c >> +++ b/platform/linux-dpdk/odp_buffer.c >> @@ -5,8 +5,10 @@ >> */ >> >> #include <odp_buffer.h> >> -#include <odp_buffer_internal.h> >> #include <odp_buffer_pool_internal.h> >> +#include <odp_buffer_internal.h> >> +#include <odp_buffer_inlines.h> >> +#include <odp_debug_internal.h> >> >> #include <string.h> >> #include <stdio.h> >> @@ -44,13 +46,13 @@ int odp_buffer_is_valid(odp_buffer_t buf) >> } >> >> >> -int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf) >> +int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf) >> { >> odp_buffer_hdr_t *hdr; >> int len = 0; >> >> if (!odp_buffer_is_valid(buf)) { >> - printf("Buffer is not valid.\n"); >> + ODP_PRINT("Buffer is not valid.\n"); >> return len; >> } >> >> @@ -67,7 +69,9 @@ int odp_buffer_snprint(char *str, size_t n, >> odp_buffer_t buf) >> len += snprintf(&str[len], n-len, >> " size %u\n", hdr->mb.buf_len); >> len += snprintf(&str[len], n-len, >> - " ref_count %i\n", hdr->mb.refcnt); >> + " ref_count %i\n", >> + odp_atomic_load_u32((odp_atomic_u32_t *) >> + &hdr->mb.refcnt)); >> len += snprintf(&str[len], n-len, >> " dpdk type %i\n", hdr->mb.type); >> len += snprintf(&str[len], n-len, >> @@ -86,5 +90,5 @@ void odp_buffer_print(odp_buffer_t buf) >> len = odp_buffer_snprint(str, max_len-1, buf); >> str[len] = 0; >> >> - printf("\n%s\n", str); >> + ODP_PRINT("\n%s\n", str); >> } >> diff --git a/platform/linux-dpdk/odp_buffer_pool.c >> b/platform/linux-dpdk/odp_buffer_pool.c >> index ca25ace..6636b8d 100644 >> --- a/platform/linux-dpdk/odp_buffer_pool.c >> +++ b/platform/linux-dpdk/odp_buffer_pool.c >> @@ -6,16 +6,19 @@ >> >> #include <odp_std_types.h> >> #include <odp_buffer_pool.h> >> -#include <odp_buffer_pool_internal.h> >> #include <odp_buffer_internal.h> >> +#include <odp_buffer_pool_internal.h> >> +#include <odp_buffer_inlines.h> >> #include <odp_packet_internal.h> >> #include <odp_timer_internal.h> >> +#include <odp_align_internal.h> >> #include <odp_shared_memory.h> >> #include <odp_align.h> >> #include <odp_internal.h> >> #include <odp_config.h> >> #include <odp_hints.h> >> -#include <odp_debug.h> >> +#include <odp_debug_internal.h> >> +#include <odp_atomic_internal.h> >> >> #include <string.h> >> #include <stdlib.h> >> @@ -26,44 +29,27 @@ >> #define MBUF_SIZE (2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM) >> #define NB_MBUF 32768 >> >> -#ifdef POOL_USE_TICKETLOCK >> -#include <odp_ticketlock.h> >> -#define LOCK(a) odp_ticketlock_lock(a) >> -#define UNLOCK(a) odp_ticketlock_unlock(a) >> -#define LOCK_INIT(a) odp_ticketlock_init(a) >> -#else >> -#include <odp_spinlock.h> >> -#define LOCK(a) odp_spinlock_lock(a) >> -#define UNLOCK(a) odp_spinlock_unlock(a) >> -#define LOCK_INIT(a) odp_spinlock_init(a) >> -#endif >> - >> - >> #if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >> #error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >> #endif >> >> -#define NULL_INDEX ((uint32_t)-1) >> - >> -union buffer_type_any_u { >> +typedef union buffer_type_any_u { >> odp_buffer_hdr_t buf; >> odp_packet_hdr_t pkt; >> odp_timeout_hdr_t tmo; >> -}; >> - >> -typedef union buffer_type_any_u odp_any_buffer_hdr_t; >> +} odp_anybuf_t; >> >> -typedef union pool_entry_u { >> - struct pool_entry_s s; >> - >> - uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct >> pool_entry_s))]; >> - >> -} pool_entry_t; >> +/* Any buffer type header */ >> +typedef struct { >> + odp_anybuf_t any_hdr; /* any buffer type */ >> +} odp_any_buffer_hdr_t; >> >> +typedef struct odp_any_hdr_stride { >> + uint8_t >> pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_any_buffer_hdr_t))]; >> +} odp_any_hdr_stride; >> >> typedef struct pool_table_t { >> pool_entry_t pool[ODP_CONFIG_BUFFER_POOLS]; >> - >> } pool_table_t; >> >> >> @@ -73,6 +59,8 @@ static pool_table_t *pool_tbl; >> /* Pool entry pointers (for inlining) */ >> void *pool_entry_ptr[ODP_CONFIG_BUFFER_POOLS]; >> >> +/* Local cache for buffer alloc/free acceleration */ >> +static __thread local_cache_t local_cache[ODP_CONFIG_BUFFER_POOLS]; >> >> int odp_buffer_pool_init_global(void) >> { >> @@ -94,9 +82,9 @@ int odp_buffer_pool_init_global(void) >> for (i = 0; i < ODP_CONFIG_BUFFER_POOLS; i++) { >> /* init locks */ >> pool_entry_t *pool = &pool_tbl->pool[i]; >> - LOCK_INIT(&pool->s.lock); >> - pool->s.pool = i; >> - >> + POOL_LOCK_INIT(&pool->s.lock); >> + pool->s.pool_hdl = pool_index_to_handle(i); >> + pool->s.pool_id = i; >> pool_entry_ptr[i] = pool; >> } >> >> @@ -187,10 +175,13 @@ odp_dpdk_mbuf_ctor(struct rte_mempool *mp, >> buf_hdr->index = i; >> } >> >> +/** >> + * Buffer pool creation >> + */ >> + >> odp_buffer_pool_t odp_buffer_pool_create(const char *name, >> - void *base_addr, uint64_t size, >> - size_t buf_size, size_t >> buf_align, >> - int buf_type) >> + odp_shm_t shm, >> + odp_buffer_pool_param_t *params) >> { >> struct rte_mempool *pool = NULL; >> struct mbuf_pool_ctor_arg mbp_ctor_arg; >> @@ -198,47 +189,39 @@ odp_buffer_pool_t odp_buffer_pool_create(const char >> *name, >> unsigned mb_size; >> size_t hdr_size; >> >> - /* Not used for rte_mempool; the new ODP buffer management >> introduces >> - * rte_mempool_create_from_region where base_addr makes sense */ >> - (void)base_addr; >> - >> - /* buf_align will be removed soon, no need to wory about it */ >> - (void)buf_align; >> - >> - ODP_DBG("odp_buffer_pool_create: %s, %lx, %u, %u, %u, %d\n", name, >> - (uint64_t) base_addr, (unsigned) size, >> - (unsigned) buf_size, (unsigned) buf_align, >> - buf_type); >> + ODP_DBG("odp_buffer_pool_create: %s, %lx, %u, %u, %d\n", name, >> + (uint64_t) shm, (unsigned) params->buf_size, >> + (unsigned) params->buf_align, params->buf_type); >> >> - switch (buf_type) { >> + switch (params->buf_type) { >> case ODP_BUFFER_TYPE_RAW: >> hdr_size = sizeof(odp_buffer_hdr_t); >> - mbp_ctor_arg.seg_buf_size = (uint16_t) buf_size; >> + mbp_ctor_arg.seg_buf_size = (uint16_t) params->buf_size; >> break; >> case ODP_BUFFER_TYPE_PACKET: >> hdr_size = sizeof(odp_packet_hdr_t); >> mbp_ctor_arg.seg_buf_size = >> - (uint16_t) (RTE_PKTMBUF_HEADROOM + buf_size); >> + (uint16_t) (RTE_PKTMBUF_HEADROOM + >> params->buf_size); >> break; >> case ODP_BUFFER_TYPE_TIMEOUT: >> hdr_size = sizeof(odp_timeout_hdr_t); >> - mbp_ctor_arg.seg_buf_size = (uint16_t) buf_size; >> + mbp_ctor_arg.seg_buf_size = (uint16_t) params->buf_size; >> break; >> case ODP_BUFFER_TYPE_ANY: >> hdr_size = sizeof(odp_any_buffer_hdr_t); >> mbp_ctor_arg.seg_buf_size = >> - (uint16_t) (RTE_PKTMBUF_HEADROOM + buf_size); >> + (uint16_t) (RTE_PKTMBUF_HEADROOM + >> params->buf_size); >> break; >> default: >> - ODP_ERR("odp_buffer_pool_create: Bad type %i\n", >> buf_type); >> + ODP_ERR("odp_buffer_pool_create: Bad type %i\n", >> + params->buf_type); >> return ODP_BUFFER_POOL_INVALID; >> - break; >> } >> >> mb_ctor_arg.seg_buf_offset = >> (uint16_t) ODP_CACHE_LINE_SIZE_ROUNDUP(hdr_size); >> mb_ctor_arg.seg_buf_size = mbp_ctor_arg.seg_buf_size; >> - mb_ctor_arg.buf_type = buf_type; >> + mb_ctor_arg.buf_type = params->buf_type; >> mb_size = mb_ctor_arg.seg_buf_offset + mb_ctor_arg.seg_buf_size; >> >> pool = rte_mempool_create(name, NB_MBUF, >> @@ -267,6 +250,61 @@ odp_buffer_pool_t odp_buffer_pool_lookup(const char >> *name) >> return (odp_buffer_pool_t)mp; >> } >> >> +odp_buffer_t buffer_alloc(odp_buffer_pool_t pool_hdl, size_t size) >> +{ >> + uint32_t pool_id = pool_handle_to_index(pool_hdl); >> + pool_entry_t *pool = get_pool_entry(pool_id); >> + uintmax_t totsize = pool->s.headroom + size + pool->s.tailroom; >> + odp_anybuf_t *buf; >> + >> + /* Reject oversized allocation requests */ >> + if ((pool->s.flags.unsegmented && totsize > pool->s.seg_size) || >> + (!pool->s.flags.unsegmented && >> + totsize > ODP_CONFIG_PACKET_BUF_LEN_MAX)) >> + return ODP_BUFFER_INVALID; >> + >> + /* Try to satisfy request from the local cache */ >> + buf = (odp_anybuf_t *)(void *)get_local_buf(&local_cache[pool_id], >> + &pool->s, totsize); >> + >> + /* If cache is empty, satisfy request from the pool */ >> + if (odp_unlikely(buf == NULL)) { >> + buf = (odp_anybuf_t *)(void *)get_buf(&pool->s); >> + >> + if (odp_unlikely(buf == NULL)) >> + return ODP_BUFFER_INVALID; >> + >> + /* Get blocks for this buffer, if pool uses application >> data */ >> + if (buf->buf.size < totsize) { >> + intmax_t needed = totsize - buf->buf.size; >> + do { >> + uint8_t *blk = get_blk(&pool->s); >> + if (blk == NULL) { >> + ret_buf(&pool->s, &buf->buf); >> + return ODP_BUFFER_INVALID; >> + } >> + buf->buf.addr[buf->buf.segcount++] = blk; >> + needed -= pool->s.seg_size; >> + } while (needed > 0); >> + buf->buf.size = buf->buf.segcount * >> pool->s.seg_size; >> + } >> + } >> + >> + /* By default, buffers inherit their pool's zeroization setting */ >> + buf->buf.flags.zeroized = pool->s.flags.zeroized; >> + >> + if (buf->buf.type == ODP_BUFFER_TYPE_PACKET) { >> + packet_init(pool, &buf->pkt, size); >> + >> + if (pool->s.init_params.buf_init != NULL) >> + (*pool->s.init_params.buf_init) >> + (buf->buf.handle.handle, >> + pool->s.init_params.buf_init_arg); >> + } >> + >> + return odp_hdr_to_buf(&buf->buf); >> +} >> + >> >> odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_id) >> { >> @@ -279,6 +317,15 @@ void odp_buffer_free(odp_buffer_t buf) >> rte_pktmbuf_free((struct rte_mbuf *)buf); >> } >> >> +void _odp_flush_caches(void) >> +{ >> + int i; >> + >> + for (i = 0; i < ODP_CONFIG_BUFFER_POOLS; i++) { >> + pool_entry_t *pool = get_pool_entry(i); >> + flush_cache(&local_cache[i], &pool->s); >> + } >> +} >> >> void odp_buffer_pool_print(odp_buffer_pool_t pool_id) >> { >> diff --git a/platform/linux-dpdk/odp_init.c >> b/platform/linux-dpdk/odp_init.c >> index fa10022..7c8763d 100644 >> --- a/platform/linux-dpdk/odp_init.c >> +++ b/platform/linux-dpdk/odp_init.c >> @@ -7,6 +7,8 @@ >> #include <odp_init.h> >> #include <odp_internal.h> >> #include <odp_debug.h> >> +#include <odp_debug_internal.h> >> +#include <odp_system_info.h> >> #include <odp_packet_dpdk.h> >> >> int odp_init_dpdk(void) >> @@ -45,10 +47,9 @@ int odp_init_dpdk(void) >> return 0; >> } >> >> -int odp_init_global(void) >> +int odp_init_global(odp_init_t *params ODP_UNUSED, >> + odp_platform_init_t *platform_params ODP_UNUSED) >> { >> - odp_thread_init_global(); >> - >> odp_system_info_init(); >> >> if (odp_init_dpdk()) { >> @@ -61,6 +62,11 @@ int odp_init_global(void) >> return -1; >> } >> >> + if (odp_thread_init_global()) { >> + ODP_ERR("ODP thread init failed.\n"); >> + return -1; >> + } >> + >> if (odp_buffer_pool_init_global()) { >> ODP_ERR("ODP buffer pool init failed.\n"); >> return -1; >> @@ -91,13 +97,26 @@ int odp_init_global(void) >> return -1; >> } >> >> + if (odp_classification_init_global()) { >> + ODP_ERR("ODP classification init failed.\n"); >> + return -1; >> + } >> + >> return 0; >> } >> >> +int odp_term_global(void) >> +{ >> + ODP_UNIMPLEMENTED(); >> + return 0; >> +} >> >> -int odp_init_local(int thr_id) >> +int odp_init_local(void) >> { >> - odp_thread_init_local(thr_id); >> + if (odp_thread_init_local()) { >> + ODP_ERR("ODP thread local init failed.\n"); >> + return -1; >> + } >> >> if (odp_pktio_init_local()) { >> ODP_ERR("ODP packet io local init failed.\n"); >> @@ -111,3 +130,9 @@ int odp_init_local(int thr_id) >> >> return 0; >> } >> + >> +int odp_term_local(void) >> +{ >> + ODP_UNIMPLEMENTED(); >> + return 0; >> +} >> diff --git a/platform/linux-dpdk/odp_linux.c >> b/platform/linux-dpdk/odp_linux.c >> index 96c91a5..2b68c13 100644 >> --- a/platform/linux-dpdk/odp_linux.c >> +++ b/platform/linux-dpdk/odp_linux.c >> @@ -8,6 +8,9 @@ >> #define _GNU_SOURCE >> #endif >> #include <sched.h> >> +#include <unistd.h> >> +#include <sys/types.h> >> +#include <sys/wait.h> >> >> #include <stdlib.h> >> #include <string.h> >> @@ -19,28 +22,30 @@ >> #include <odp_thread.h> >> #include <odp_init.h> >> #include <odp_system_info.h> >> +#include <odp_debug_internal.h> >> >> #include <rte_lcore.h> >> >> typedef struct { >> - int thr_id; >> void *(*start_routine) (void *); >> void *arg; >> - >> } odp_start_args_t; >> >> - >> static void *odp_run_start_routine(void *arg) >> { >> odp_start_args_t *start_args = arg; >> >> /* ODP thread local init */ >> - odp_init_local(start_args->thr_id); >> + if (odp_init_local()) { >> + ODP_ERR("Local init failed\n"); >> + return NULL; >> + } >> >> - return start_args->start_routine(start_args->arg); >> + void *ret = start_args->start_routine(start_args->arg); >> + _odp_flush_caches(); >> + return ret; >> } >> >> - >> void odph_linux_pthread_create(odph_linux_pthread_t *thread_tbl, int num, >> int first_core, void *(*start_routine) (void *), void >> *arg) >> { >> @@ -62,12 +67,12 @@ void odph_linux_pthread_create(odph_linux_pthread_t >> *thread_tbl, int num, >> cpu = (first_core + i) % core_count; >> >> start_args = malloc(sizeof(odp_start_args_t)); >> + if (start_args == NULL) >> + ODP_ABORT("Malloc failed"); >> memset(start_args, 0, sizeof(odp_start_args_t)); >> start_args->start_routine = start_routine; >> start_args->arg = arg; >> >> - odp_thread_create(cpu); >> - start_args->thr_id = cpu; >> /* If not master core */ >> if (cpu != 0) { >> rte_eal_remote_launch( >> @@ -94,3 +99,99 @@ void odph_linux_pthread_join(odph_linux_pthread_t >> *thread_tbl, int num) >> return; >> } >> } >> + >> +int odph_linux_process_fork_n(odph_linux_process_t *proc_tbl, >> + int num, int first_core) >> +{ >> + cpu_set_t cpu_set; >> + pid_t pid; >> + int core_count; >> + int cpu; >> + int i; >> + >> + memset(proc_tbl, 0, num*sizeof(odph_linux_process_t)); >> + >> + core_count = odp_sys_core_count(); >> + >> + if (first_core < 0 || first_core >= core_count) { >> + ODP_ERR("Bad first_core\n"); >> + return -1; >> + } >> + >> + if (num < 0 || num > core_count) { >> + ODP_ERR("Bad num\n"); >> + return -1; >> + } >> + >> + for (i = 0; i < num; i++) { >> + cpu = (first_core + i) % core_count; >> + pid = fork(); >> + >> + if (pid < 0) { >> + ODP_ERR("fork() failed\n"); >> + return -1; >> + } >> + >> + /* Parent continues to fork */ >> + if (pid > 0) { >> + proc_tbl[i].pid = pid; >> + proc_tbl[i].core = cpu; >> + continue; >> + } >> + >> + /* Child process */ >> + CPU_ZERO(&cpu_set); >> + CPU_SET(cpu, &cpu_set); >> + >> + if (sched_setaffinity(0, sizeof(cpu_set_t), &cpu_set)) { >> + ODP_ERR("sched_setaffinity() failed\n"); >> + return -2; >> + } >> + >> + if (odp_init_local()) { >> + ODP_ERR("Local init failed\n"); >> + return -2; >> + } >> + >> + return 0; >> + } >> + >> + return 1; >> +} >> + >> + >> +int odph_linux_process_fork(odph_linux_process_t *proc, int core) >> +{ >> + return odph_linux_process_fork_n(proc, 1, core); >> +} >> + >> + >> +int odph_linux_process_wait_n(odph_linux_process_t *proc_tbl, int num) >> +{ >> + pid_t pid; >> + int i, j; >> + int status; >> + >> + for (i = 0; i < num; i++) { >> + pid = wait(&status); >> + >> + if (pid < 0) { >> + ODP_ERR("wait() failed\n"); >> + return -1; >> + } >> + >> + for (j = 0; j < num; j++) { >> + if (proc_tbl[j].pid == pid) { >> + proc_tbl[j].status = status; >> + break; >> + } >> + } >> + >> + if (j == num) { >> + ODP_ERR("Bad pid\n"); >> + return -1; >> + } >> + } >> + >> + return 0; >> +} >> diff --git a/platform/linux-dpdk/odp_packet.c >> b/platform/linux-dpdk/odp_packet.c >> index beb69b2..754ce41 100644 >> --- a/platform/linux-dpdk/odp_packet.c >> +++ b/platform/linux-dpdk/odp_packet.c >> @@ -6,34 +6,63 @@ >> >> #include <odp_packet.h> >> #include <odp_packet_internal.h> >> +#include <odp_debug_internal.h> >> #include <odp_hints.h> >> #include <odp_byteorder.h> >> >> #include <odph_eth.h> >> #include <odph_ip.h> >> +#include <odph_tcp.h> >> +#include <odph_udp.h> >> >> #include <string.h> >> #include <stdio.h> >> >> -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >> - odph_ipv4hdr_t *ipv4, size_t >> *offset_out); >> -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >> - odph_ipv6hdr_t *ipv6, size_t >> *offset_out); >> +/* >> + * >> + * Alloc and free >> + * ******************************************************** >> + * >> + */ >> >> -void odp_packet_init(odp_packet_t pkt) >> +odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool_hdl, uint32_t len) >> { >> - odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >> - struct rte_mbuf *mb; >> - void *start; >> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >> + >> + if (pool->s.params.buf_type != ODP_BUFFER_TYPE_PACKET) >> + return ODP_PACKET_INVALID; >> + >> + /* Handle special case for zero-length packets */ >> + if (len == 0) { >> + odp_packet_t pkt = >> + (odp_packet_t)buffer_alloc(pool_hdl, >> + >> pool->s.params.buf_size); >> + if (pkt != ODP_PACKET_INVALID) >> + pull_tail(odp_packet_hdr(pkt), >> + pool->s.params.buf_size); >> + >> + return pkt; >> + } >> + >> + return (odp_packet_t)buffer_alloc(pool_hdl, len); >> +} >> >> - mb = &pkt_hdr->buf_hdr.mb; >> +void odp_packet_free(odp_packet_t pkt) >> +{ >> + odp_buffer_free((odp_buffer_t)pkt); >> +} >> + >> +int odp_packet_reset(odp_packet_t pkt, uint32_t len) >> +{ >> + odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >> + pool_entry_t *pool = odp_buf_to_pool(&pkt_hdr->buf_hdr); >> + uint32_t totsize = pool->s.headroom + len + pool->s.tailroom; >> >> - start = mb->buf_addr; >> - memset(start, 0, mb->buf_len); >> + if (totsize > pkt_hdr->buf_hdr.size) >> + return -1; >> >> - pkt_hdr->l2_offset = (uint32_t) ODP_PACKET_OFFSET_INVALID; >> - pkt_hdr->l3_offset = (uint32_t) ODP_PACKET_OFFSET_INVALID; >> - pkt_hdr->l4_offset = (uint32_t) ODP_PACKET_OFFSET_INVALID; >> + packet_init(pool, pkt_hdr, len); >> + return 0; >> } >> >> odp_packet_t odp_packet_from_buffer(odp_buffer_t buf) >> @@ -41,305 +70,478 @@ odp_packet_t odp_packet_from_buffer(odp_buffer_t >> buf) >> return (odp_packet_t)buf; >> } >> >> -odp_buffer_t odp_buffer_from_packet(odp_packet_t pkt) >> +odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt) >> { >> return (odp_buffer_t)pkt; >> } >> >> -/* Advance the pkt data pointer and set len in one call */ >> -static int odp_packet_set_offset_len(odp_packet_t pkt, size_t >> frame_offset, >> - size_t len) >> +/* >> + * >> + * Pointers and lengths >> + * ******************************************************** >> + * >> + */ >> + >> +void *odp_packet_head(odp_packet_t pkt) >> { >> - struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); >> - uint16_t offset; >> - uint16_t data_len; >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + return buffer_map(&pkt_hdr->buf_hdr, 0, NULL, 0); >> +} >> >> - /* The pkt buf may have been pulled back into the headroom >> - * so we cannot rely on finding the data right after the >> - * ODP header and HEADROOM */ >> - offset = (uint16_t)((unsigned long)mb->pkt.data - >> - (unsigned long)mb->buf_addr); >> - ODP_ASSERT(mb->buf_len >= offset, "Corrupted mbuf"); >> - data_len = mb->buf_len - offset; >> +uint32_t odp_packet_buf_len(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->buf_hdr.size; >> +} >> >> - if (data_len < frame_offset) { >> - ODP_ERR("Frame offset too big"); >> - return -1; >> - } >> - mb->pkt.data = (void *)((char *)mb->pkt.data + frame_offset); >> - data_len -= frame_offset; >> +void *odp_packet_data(odp_packet_t pkt) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + return packet_map(pkt_hdr, 0, NULL); >> +} >> >> - if (data_len < len) { >> - ODP_ERR("Packet len too big"); >> - return -1; >> - } >> - mb->pkt.pkt_len = len; >> - mb->pkt.data_len = len; >> +uint32_t odp_packet_seg_len(odp_packet_t pkt) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + uint32_t seglen; >> >> - return 0; >> + /* Call returns length of 1st data segment */ >> + packet_map(pkt_hdr, 0, &seglen); >> + return seglen; >> } >> >> -void odp_packet_set_len(odp_packet_t pkt, size_t len) >> +uint32_t odp_packet_len(odp_packet_t pkt) >> { >> - (void)odp_packet_set_offset_len(pkt, 0, len); >> + return odp_packet_hdr(pkt)->frame_len; >> } >> >> -size_t odp_packet_get_len(odp_packet_t pkt) >> +uint32_t odp_packet_headroom(odp_packet_t pkt) >> { >> - struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); >> - return mb->pkt.pkt_len; >> + return odp_packet_hdr(pkt)->headroom; >> } >> >> -uint8_t *odp_packet_buf_addr(odp_packet_t pkt) >> +uint32_t odp_packet_tailroom(odp_packet_t pkt) >> { >> - return odp_buffer_addr(odp_buffer_from_packet(pkt)); >> + return odp_packet_hdr(pkt)->tailroom; >> } >> >> -uint8_t *odp_packet_start(odp_packet_t pkt) >> +void *odp_packet_tail(odp_packet_t pkt) >> { >> - struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); >> - return mb->pkt.data; >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL); >> } >> >> - >> -uint8_t *odp_packet_l2(odp_packet_t pkt) >> +void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) >> { >> - const size_t offset = odp_packet_l2_offset(pkt); >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> >> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >> + if (len > pkt_hdr->headroom) >> return NULL; >> >> - return odp_packet_start(pkt) + offset; >> + push_head(pkt_hdr, len); >> + return packet_map(pkt_hdr, 0, NULL); >> } >> >> -size_t odp_packet_l2_offset(odp_packet_t pkt) >> +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) >> { >> - return odp_packet_hdr(pkt)->l2_offset; >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + if (len > pkt_hdr->frame_len) >> + return NULL; >> + >> + pull_head(pkt_hdr, len); >> + return packet_map(pkt_hdr, 0, NULL); >> } >> >> -void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset) >> +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) >> { >> - odp_packet_hdr(pkt)->l2_offset = offset; >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + struct rte_mbuf *mbuf = (struct rte_mbuf *)&pkt_hdr->buf_hdr; >> + uint32_t origin = pkt_hdr->frame_len; >> + >> + mbuf->pkt.pkt_len += len; >> + mbuf->pkt.data_len += len; >> + >> + if (len > pkt_hdr->tailroom) >> + return NULL; >> + >> + push_tail(pkt_hdr, len); >> + return packet_map(pkt_hdr, origin, NULL); >> } >> >> -uint8_t *odp_packet_l3(odp_packet_t pkt) >> +void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) >> { >> - const size_t offset = odp_packet_l3_offset(pkt); >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> >> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >> + if (len > pkt_hdr->frame_len) >> return NULL; >> >> - return odp_packet_start(pkt) + offset; >> + pull_tail(pkt_hdr, len); >> + return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL); >> } >> >> -size_t odp_packet_l3_offset(odp_packet_t pkt) >> +void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, >> + odp_packet_seg_t *seg) >> { >> - return odp_packet_hdr(pkt)->l3_offset; >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + void *addr = packet_map(pkt_hdr, offset, len); >> + >> + if (addr != NULL && seg != NULL) { >> + odp_buffer_bits_t seghandle; >> + seghandle.u32 = (uint32_t)pkt; >> + seghandle.seg = (pkt_hdr->headroom + offset) / >> + pkt_hdr->buf_hdr.segsize; >> + *seg = seghandle.handle; >> + } >> + >> + return addr; >> } >> >> -void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset) >> +/* >> + * >> + * Meta-data >> + * ******************************************************** >> + * >> + */ >> + >> +odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt) >> { >> - odp_packet_hdr(pkt)->l3_offset = offset; >> + return odp_packet_hdr(pkt)->buf_hdr.pool_hdl; >> } >> >> -uint8_t *odp_packet_l4(odp_packet_t pkt) >> +odp_pktio_t odp_packet_input(odp_packet_t pkt) >> { >> - const size_t offset = odp_packet_l4_offset(pkt); >> + return odp_packet_hdr(pkt)->input; >> +} >> >> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >> - return NULL; >> +void *odp_packet_user_ptr(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->buf_hdr.buf_ctx; >> +} >> + >> +void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ctx) >> +{ >> + odp_packet_hdr(pkt)->buf_hdr.buf_cctx = ctx; >> +} >> + >> +uint64_t odp_packet_user_u64(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->buf_hdr.buf_u64; >> +} >> >> - return odp_packet_start(pkt) + offset; >> +void odp_packet_user_u64_set(odp_packet_t pkt, uint64_t ctx) >> +{ >> + odp_packet_hdr(pkt)->buf_hdr.buf_u64 = ctx; >> +} >> + >> +void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + return packet_map(pkt_hdr, pkt_hdr->l2_offset, len); >> +} >> + >> +uint32_t odp_packet_l2_offset(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->l2_offset; >> } >> >> -size_t odp_packet_l4_offset(odp_packet_t pkt) >> +int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + if (offset >= pkt_hdr->frame_len) >> + return -1; >> + >> + pkt_hdr->l2_offset = offset; >> + return 0; >> +} >> + >> +void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + return packet_map(pkt_hdr, pkt_hdr->l3_offset, len); >> +} >> + >> +uint32_t odp_packet_l3_offset(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->l3_offset; >> +} >> + >> +int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + if (offset >= pkt_hdr->frame_len) >> + return -1; >> + >> + pkt_hdr->l3_offset = offset; >> + return 0; >> +} >> + >> +void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + return packet_map(pkt_hdr, pkt_hdr->l4_offset, len); >> +} >> + >> +uint32_t odp_packet_l4_offset(odp_packet_t pkt) >> { >> return odp_packet_hdr(pkt)->l4_offset; >> } >> >> -void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset) >> +int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset) >> { >> - odp_packet_hdr(pkt)->l4_offset = offset; >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + if (offset >= pkt_hdr->frame_len) >> + return -1; >> + >> + pkt_hdr->l4_offset = offset; >> + return 0; >> } >> >> -/** >> - * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP >> +int odp_packet_is_segmented(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->buf_hdr.segcount > 1; >> +} >> + >> +int odp_packet_num_segs(odp_packet_t pkt) >> +{ >> + return odp_packet_hdr(pkt)->buf_hdr.segcount; >> +} >> + >> +odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) >> +{ >> + return (odp_packet_seg_t)pkt; >> +} >> + >> +odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + odp_buffer_bits_t seghandle; >> + >> + seghandle.u32 = (uint32_t)pkt; >> + seghandle.seg = pkt_hdr->buf_hdr.segcount - 1; >> + return seghandle.handle; >> +} >> + >> +odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t >> seg) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + return segment_next(&pkt_hdr->buf_hdr, seg); >> +} >> + >> +/* >> * >> - * Internal function: caller is resposible for passing only valid packet >> handles >> - * , lengths and offsets (usually done&called in packet input). >> + * Segment level >> + * ******************************************************** >> * >> - * @param pkt Packet handle >> - * @param len Packet length in bytes >> - * @param frame_offset Byte offset to L2 header >> */ >> -void odp_packet_parse(odp_packet_t pkt, size_t len, size_t frame_offset) >> + >> +void *odp_packet_seg_buf_addr(odp_packet_t pkt, odp_packet_seg_t seg) >> { >> - odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >> - odph_ethhdr_t *eth; >> - odph_vlanhdr_t *vlan; >> - odph_ipv4hdr_t *ipv4; >> - odph_ipv6hdr_t *ipv6; >> - uint16_t ethtype; >> - size_t offset = 0; >> - uint8_t ip_proto = 0; >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> >> - /* The frame_offset is not relevant for frames from DPDK */ >> - pkt_hdr->input_flags.eth = 1; >> - (void) frame_offset; >> - pkt_hdr->frame_offset = 0; >> - if (odp_packet_set_offset_len(pkt, 0, len)) { >> - return; >> - } >> + return segment_map(&pkt_hdr->buf_hdr, seg, NULL, >> + pkt_hdr->headroom + pkt_hdr->frame_len, 0); >> +} >> >> - if (odp_unlikely(len < ODPH_ETH_LEN_MIN)) { >> - pkt_hdr->error_flags.frame_len = 1; >> - return; >> - } else if (len > ODPH_ETH_LEN_MAX) { >> - pkt_hdr->input_flags.jumbo = 1; >> - } >> +uint32_t odp_packet_seg_buf_len(odp_packet_t pkt, >> + odp_packet_seg_t seg ODP_UNUSED) >> +{ >> + return odp_packet_hdr(pkt)->buf_hdr.segsize; >> +} >> >> - /* Assume valid L2 header, no CRC/FCS check in SW */ >> - pkt_hdr->input_flags.l2 = 1; >> - pkt_hdr->l2_offset = 0; >> +void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> >> - eth = (odph_ethhdr_t *)odp_packet_start(pkt); >> - ethtype = odp_be_to_cpu_16(eth->type); >> - vlan = (odph_vlanhdr_t *)ð->type; >> + return segment_map(&pkt_hdr->buf_hdr, seg, NULL, >> + pkt_hdr->frame_len, pkt_hdr->headroom); >> +} >> >> - if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { >> - pkt_hdr->input_flags.vlan_qinq = 1; >> - ethtype = odp_be_to_cpu_16(vlan->tpid); >> - offset += sizeof(odph_vlanhdr_t); >> - vlan = &vlan[1]; >> - } >> +uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + uint32_t seglen = 0; >> >> - if (ethtype == ODPH_ETHTYPE_VLAN) { >> - pkt_hdr->input_flags.vlan = 1; >> - ethtype = odp_be_to_cpu_16(vlan->tpid); >> - offset += sizeof(odph_vlanhdr_t); >> - } >> + segment_map(&pkt_hdr->buf_hdr, seg, &seglen, >> + pkt_hdr->frame_len, pkt_hdr->headroom); >> >> - /* Set l3_offset+flag only for known ethtypes */ >> - switch (ethtype) { >> - case ODPH_ETHTYPE_IPV4: >> - pkt_hdr->input_flags.ipv4 = 1; >> - pkt_hdr->input_flags.l3 = 1; >> - pkt_hdr->l3_offset = ODPH_ETHHDR_LEN + offset; >> - ipv4 = (odph_ipv4hdr_t *)odp_packet_l3(pkt); >> - ip_proto = parse_ipv4(pkt_hdr, ipv4, &offset); >> - break; >> - case ODPH_ETHTYPE_IPV6: >> - pkt_hdr->input_flags.ipv6 = 1; >> - pkt_hdr->input_flags.l3 = 1; >> - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + >> offset; >> - ipv6 = (odph_ipv6hdr_t *)odp_packet_l3(pkt); >> - ip_proto = parse_ipv6(pkt_hdr, ipv6, &offset); >> - break; >> - case ODPH_ETHTYPE_ARP: >> - pkt_hdr->input_flags.arp = 1; >> - /* fall through */ >> - default: >> - ip_proto = 0; >> - break; >> - } >> + return seglen; >> +} >> >> - switch (ip_proto) { >> - case ODPH_IPPROTO_UDP: >> - pkt_hdr->input_flags.udp = 1; >> - pkt_hdr->input_flags.l4 = 1; >> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >> - break; >> - case ODPH_IPPROTO_TCP: >> - pkt_hdr->input_flags.tcp = 1; >> - pkt_hdr->input_flags.l4 = 1; >> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >> - break; >> - case ODPH_IPPROTO_SCTP: >> - pkt_hdr->input_flags.sctp = 1; >> - pkt_hdr->input_flags.l4 = 1; >> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >> - break; >> - case ODPH_IPPROTO_ICMP: >> - pkt_hdr->input_flags.icmp = 1; >> - pkt_hdr->input_flags.l4 = 1; >> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >> - break; >> - default: >> - /* 0 or unhandled IP protocols, don't set L4 flag+offset >> */ >> - if (pkt_hdr->input_flags.ipv6) { >> - /* IPv6 next_hdr is not L4, mark as IP-option >> instead */ >> - pkt_hdr->input_flags.ipopt = 1; >> +/* >> + * >> + * Manipulation >> + * ******************************************************** >> + * >> + */ >> + >> +odp_packet_t odp_packet_add_data(odp_packet_t pkt, uint32_t offset, >> + uint32_t len) >> +{ >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + uint32_t pktlen = pkt_hdr->frame_len; >> + odp_packet_t newpkt; >> + >> + if (offset > pktlen) >> + return ODP_PACKET_INVALID; >> + >> + newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen + >> len); >> + >> + if (newpkt != ODP_PACKET_INVALID) { >> + if (_odp_packet_copy_to_packet(pkt, 0, >> + newpkt, 0, offset) != 0 || >> + _odp_packet_copy_to_packet(pkt, offset, newpkt, >> + offset + len, >> + pktlen - offset) != 0) { >> + odp_packet_free(newpkt); >> + newpkt = ODP_PACKET_INVALID; >> + } else { >> + odp_packet_hdr_t *new_hdr = >> odp_packet_hdr(newpkt); >> + new_hdr->buf_hdr.buf_u64 = >> pkt_hdr->buf_hdr.buf_u64; >> + odp_atomic_store_u32( >> + &new_hdr->buf_hdr.ref_count, >> + odp_atomic_load_u32( >> + &pkt_hdr->buf_hdr.ref_count)); >> + _odp_packet_parse(newpkt); >> + odp_packet_free(pkt); >> } >> - break; >> } >> + >> + return newpkt; >> } >> >> -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >> - odph_ipv4hdr_t *ipv4, size_t *offset_out) >> +odp_packet_t odp_packet_rem_data(odp_packet_t pkt, uint32_t offset, >> + uint32_t len) >> { >> - uint8_t ihl; >> - uint16_t frag_offset; >> - >> - ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); >> - if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN)) { >> - pkt_hdr->error_flags.ip_err = 1; >> - return 0; >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + uint32_t pktlen = pkt_hdr->frame_len; >> + odp_packet_t newpkt; >> + >> + if (offset > pktlen || offset + len > pktlen) >> + return ODP_PACKET_INVALID; >> + >> + newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen - >> len); >> + >> + if (newpkt != ODP_PACKET_INVALID) { >> + if (_odp_packet_copy_to_packet(pkt, 0, >> + newpkt, 0, offset) != 0 || >> + _odp_packet_copy_to_packet(pkt, offset + len, >> + newpkt, offset, >> + pktlen - offset - len) != >> 0) { >> + odp_packet_free(newpkt); >> + newpkt = ODP_PACKET_INVALID; >> + } else { >> + odp_packet_hdr_t *new_hdr = >> odp_packet_hdr(newpkt); >> + new_hdr->buf_hdr.buf_u64 = >> pkt_hdr->buf_hdr.buf_u64; >> + odp_atomic_store_u32( >> + &new_hdr->buf_hdr.ref_count, >> + odp_atomic_load_u32( >> + &pkt_hdr->buf_hdr.ref_count)); >> + _odp_packet_parse(newpkt); >> + odp_packet_free(pkt); >> + } >> } >> >> - if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) { >> - pkt_hdr->input_flags.ipopt = 1; >> - return 0; >> - } >> + return newpkt; >> +} >> >> - /* A packet is a fragment if: >> - * "more fragments" flag is set (all fragments except the last) >> - * OR >> - * "fragment offset" field is nonzero (all fragments except the >> first) >> - */ >> - frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); >> - if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) { >> - pkt_hdr->input_flags.ipfrag = 1; >> - return 0; >> - } >> +/* >> + * >> + * Copy >> + * ******************************************************** >> + * >> + */ >> >> - if (ipv4->proto == ODPH_IPPROTO_ESP || >> - ipv4->proto == ODPH_IPPROTO_AH) { >> - pkt_hdr->input_flags.ipsec = 1; >> - return 0; >> +odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool) >> +{ >> + odp_packet_hdr_t *srchdr = odp_packet_hdr(pkt); >> + uint32_t pktlen = srchdr->frame_len; >> + uint32_t meta_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, >> buf_hdr); >> + odp_packet_t newpkt = odp_packet_alloc(pool, pktlen); >> + >> + if (newpkt != ODP_PACKET_INVALID) { >> + odp_packet_hdr_t *newhdr = odp_packet_hdr(newpkt); >> + uint8_t *newstart, *srcstart; >> + >> + /* Must copy meta data first, followed by packet data */ >> + newstart = (uint8_t *)newhdr + meta_offset; >> + srcstart = (uint8_t *)srchdr + meta_offset; >> + >> + memcpy(newstart, srcstart, >> + sizeof(odp_packet_hdr_t) - meta_offset); >> + >> + if (_odp_packet_copy_to_packet(pkt, 0, >> + newpkt, 0, pktlen) != 0) { >> + odp_packet_free(newpkt); >> + newpkt = ODP_PACKET_INVALID; >> + } >> } >> >> - /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after >> return */ >> - >> - *offset_out = sizeof(uint32_t) * ihl; >> - return ipv4->proto; >> + return newpkt; >> } >> >> -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >> - odph_ipv6hdr_t *ipv6, size_t *offset_out) >> +int odp_packet_copydata_out(odp_packet_t pkt, uint32_t offset, >> + uint32_t len, void *dst) >> { >> - if (ipv6->next_hdr == ODPH_IPPROTO_ESP || >> - ipv6->next_hdr == ODPH_IPPROTO_AH) { >> - pkt_hdr->input_flags.ipopt = 1; >> - pkt_hdr->input_flags.ipsec = 1; >> - return 0; >> + void *mapaddr; >> + uint32_t seglen = 0; /* GCC */ >> + uint32_t cpylen; >> + uint8_t *dstaddr = (uint8_t *)dst; >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + if (offset + len > pkt_hdr->frame_len) >> + return -1; >> + >> + while (len > 0) { >> + mapaddr = packet_map(pkt_hdr, offset, &seglen); >> + cpylen = len > seglen ? seglen : len; >> + memcpy(dstaddr, mapaddr, cpylen); >> + offset += cpylen; >> + dstaddr += cpylen; >> + len -= cpylen; >> } >> >> - if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { >> - pkt_hdr->input_flags.ipopt = 1; >> - pkt_hdr->input_flags.ipfrag = 1; >> - return 0; >> + return 0; >> +} >> + >> +int odp_packet_copydata_in(odp_packet_t pkt, uint32_t offset, >> + uint32_t len, const void *src) >> +{ >> + void *mapaddr; >> + uint32_t seglen = 0; /* GCC */ >> + uint32_t cpylen; >> + const uint8_t *srcaddr = (const uint8_t *)src; >> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >> + >> + if (offset + len > pkt_hdr->frame_len) >> + return -1; >> + >> + while (len > 0) { >> + mapaddr = packet_map(pkt_hdr, offset, &seglen); >> + cpylen = len > seglen ? seglen : len; >> + memcpy(mapaddr, srcaddr, cpylen); >> + offset += cpylen; >> + srcaddr += cpylen; >> + len -= cpylen; >> } >> >> - /* Don't step through more extensions */ >> - *offset_out = ODPH_IPV6HDR_LEN; >> - return ipv6->next_hdr; >> + return 0; >> } >> >> +/* >> + * >> + * Debugging >> + * ******************************************************** >> + * >> + */ >> + >> void odp_packet_print(odp_packet_t pkt) >> { >> int max_len = 512; >> char str[max_len]; >> - uint8_t *p; >> int len = 0; >> int n = max_len-1; >> odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >> @@ -353,77 +555,358 @@ void odp_packet_print(odp_packet_t pkt) >> len += snprintf(&str[len], n-len, >> " output_flags 0x%x\n", hdr->output_flags.all); >> len += snprintf(&str[len], n-len, >> - " frame_offset %u\n", hdr->frame_offset); >> - len += snprintf(&str[len], n-len, >> " l2_offset %u\n", hdr->l2_offset); >> len += snprintf(&str[len], n-len, >> " l3_offset %u\n", hdr->l3_offset); >> len += snprintf(&str[len], n-len, >> " l4_offset %u\n", hdr->l4_offset); >> len += snprintf(&str[len], n-len, >> - " frame_len %u\n", >> hdr->buf_hdr.mb.pkt.pkt_len); >> + " frame_len %u\n", hdr->frame_len); >> len += snprintf(&str[len], n-len, >> " input %u\n", hdr->input); >> str[len] = '\0'; >> >> - printf("\n%s\n", str); >> - rte_pktmbuf_dump(stdout, &hdr->buf_hdr.mb, 32); >> + ODP_PRINT("\n%s\n", str); >> +} >> >> - p = odp_packet_start(pkt); >> - printf("00000000: %02X %02X %02X %02X %02X %02X %02X %02X\n", >> - p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7]); >> - printf("00000008: %02X %02X %02X %02X %02X %02X %02X %02X\n", >> - p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); >> +int odp_packet_is_valid(odp_packet_t pkt) >> +{ >> + odp_buffer_hdr_t *buf = validate_buf((odp_buffer_t)pkt); >> >> + return (buf != NULL && buf->type == ODP_BUFFER_TYPE_PACKET); >> } >> >> -/* For now we can only copy between packets of the same segment size >> - * We should probably refine this API, maybe introduce a clone API */ >> -int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src) >> +/* >> + * >> + * Internal Use Routines >> + * ******************************************************** >> + * >> + */ >> + >> +int _odp_packet_copy_to_packet(odp_packet_t srcpkt, uint32_t srcoffset, >> + odp_packet_t dstpkt, uint32_t dstoffset, >> + uint32_t len) >> { >> - struct rte_mbuf *mb_dst, *mb_src; >> - uint8_t nb_segs, i; >> + odp_packet_hdr_t *srchdr = odp_packet_hdr(srcpkt); >> + odp_packet_hdr_t *dsthdr = odp_packet_hdr(dstpkt); >> + void *srcmap; >> + void *dstmap; >> + uint32_t cpylen, minseg; >> + uint32_t srcseglen = 0; /* GCC */ >> + uint32_t dstseglen = 0; /* GCC */ >> + >> + if (srcoffset + len > srchdr->frame_len || >> + dstoffset + len > dsthdr->frame_len) >> + return -1; >> >> - ODP_ASSERT(odp_buffer_type(pkt_dst) == ODP_BUFFER_TYPE_PACKET && >> - odp_buffer_type(pkt_src) == ODP_BUFFER_TYPE_PACKET, >> - "dst_pkt or src_pkt not of type >> ODP_BUFFER_TYPE_PACKET"); >> + while (len > 0) { >> + srcmap = packet_map(srchdr, srcoffset, &srcseglen); >> + dstmap = packet_map(dsthdr, dstoffset, &dstseglen); >> >> - if (pkt_dst == ODP_PACKET_INVALID || pkt_src == >> ODP_PACKET_INVALID) >> - return -1; >> + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; >> + cpylen = len > minseg ? minseg : len; >> + memcpy(dstmap, srcmap, cpylen); >> >> - mb_dst = &(odp_packet_hdr(pkt_dst)->buf_hdr.mb); >> - mb_src = &(odp_packet_hdr(pkt_src)->buf_hdr.mb); >> + srcoffset += cpylen; >> + dstoffset += cpylen; >> + len -= cpylen; >> + } >> >> - if (mb_dst->pkt.nb_segs != mb_src->pkt.nb_segs) { >> - ODP_ERR("Different nb_segs in pkt_dst and pkt_src"); >> - return -1; >> + return 0; >> +} >> + >> +odp_packet_t _odp_packet_alloc(odp_buffer_pool_t pool_hdl) >> +{ >> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >> + >> + if (pool->s.params.buf_type != ODP_BUFFER_TYPE_PACKET) >> + return ODP_PACKET_INVALID; >> + >> + return (odp_packet_t)buffer_alloc(pool_hdl, >> + pool->s.params.buf_size); >> +} >> + >> +/** >> + * Parser helper function for IPv4 >> + */ >> +static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >> + uint8_t **parseptr, uint32_t *offset) >> +{ >> + odph_ipv4hdr_t *ipv4 = (odph_ipv4hdr_t *)*parseptr; >> + uint8_t ver = ODPH_IPV4HDR_VER(ipv4->ver_ihl); >> + uint8_t ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); >> + uint16_t frag_offset; >> + >> + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv4->tot_len); >> + >> + if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN) || >> + odp_unlikely(ver != 4) || >> + (pkt_hdr->l3_len > pkt_hdr->frame_len - *offset)) { >> + pkt_hdr->error_flags.ip_err = 1; >> + return 0; >> } >> >> - nb_segs = mb_src->pkt.nb_segs; >> + *offset += ihl * 4; >> + *parseptr += ihl * 4; >> >> - if (mb_dst->buf_len < mb_src->buf_len) { >> - ODP_ERR("dst_pkt smaller than src_pkt"); >> - return -1; >> + if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) >> + pkt_hdr->input_flags.ipopt = 1; >> + >> + /* A packet is a fragment if: >> + * "more fragments" flag is set (all fragments except the last) >> + * OR >> + * "fragment offset" field is nonzero (all fragments except the >> first) >> + */ >> + frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); >> + if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) >> + pkt_hdr->input_flags.ipfrag = 1; >> + >> + return ipv4->proto; >> +} >> + >> +/** >> + * Parser helper function for IPv6 >> + */ >> +static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >> + uint8_t **parseptr, uint32_t *offset) >> +{ >> + odph_ipv6hdr_t *ipv6 = (odph_ipv6hdr_t *)*parseptr; >> + odph_ipv6hdr_ext_t *ipv6ext; >> + >> + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv6->payload_len); >> + >> + /* Basic sanity checks on IPv6 header */ >> + if ((ipv6->ver_tc_flow >> 28) != 6 || >> + pkt_hdr->l3_len > pkt_hdr->frame_len - *offset) { >> + pkt_hdr->error_flags.ip_err = 1; >> + return 0; >> } >> >> - for (i = 0; i < nb_segs; i++) { >> - if (mb_src == NULL || mb_dst == NULL) { >> - ODP_ERR("Corrupted packets"); >> - return -1; >> + /* Skip past IPv6 header */ >> + *offset += sizeof(odph_ipv6hdr_t); >> + *parseptr += sizeof(odph_ipv6hdr_t); >> + >> + >> + /* Skip past any IPv6 extension headers */ >> + if (ipv6->next_hdr == ODPH_IPPROTO_HOPOPTS || >> + ipv6->next_hdr == ODPH_IPPROTO_ROUTE) { >> + pkt_hdr->input_flags.ipopt = 1; >> + >> + do { >> + ipv6ext = (odph_ipv6hdr_ext_t *)*parseptr; >> + uint16_t extlen = 8 + ipv6ext->ext_len * 8; >> + >> + *offset += extlen; >> + *parseptr += extlen; >> + } while ((ipv6ext->next_hdr == ODPH_IPPROTO_HOPOPTS || >> + ipv6ext->next_hdr == ODPH_IPPROTO_ROUTE) && >> + *offset < pkt_hdr->frame_len); >> + >> + if (*offset >= pkt_hdr->l3_offset + ipv6->payload_len) { >> + pkt_hdr->error_flags.ip_err = 1; >> + return 0; >> } >> - memcpy(mb_dst->buf_addr, mb_src->buf_addr, >> mb_src->buf_len); >> - mb_dst = mb_dst->pkt.next; >> - mb_src = mb_src->pkt.next; >> + >> + if (ipv6ext->next_hdr == ODPH_IPPROTO_FRAG) >> + pkt_hdr->input_flags.ipfrag = 1; >> + >> + return ipv6ext->next_hdr; >> } >> - return 0; >> + >> + if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { >> + pkt_hdr->input_flags.ipopt = 1; >> + pkt_hdr->input_flags.ipfrag = 1; >> + } >> + >> + return ipv6->next_hdr; >> +} >> + >> +/** >> + * Parser helper function for TCP >> + */ >> +static inline void parse_tcp(odp_packet_hdr_t *pkt_hdr, >> + uint8_t **parseptr, uint32_t *offset) >> +{ >> + odph_tcphdr_t *tcp = (odph_tcphdr_t *)*parseptr; >> + >> + if (tcp->hl < sizeof(odph_tcphdr_t)/sizeof(uint32_t)) >> + pkt_hdr->error_flags.tcp_err = 1; >> + else if ((uint32_t)tcp->hl * 4 > sizeof(odph_tcphdr_t)) >> + pkt_hdr->input_flags.tcpopt = 1; >> + >> + pkt_hdr->l4_len = pkt_hdr->l3_len + >> + pkt_hdr->l3_offset - pkt_hdr->l4_offset; >> + >> + *offset += (uint32_t)tcp->hl * 4; >> + *parseptr += (uint32_t)tcp->hl * 4; >> } >> >> -void odp_packet_set_ctx(odp_packet_t pkt, const void *ctx) >> +/** >> + * Parser helper function for UDP >> + */ >> +static inline void parse_udp(odp_packet_hdr_t *pkt_hdr, >> + uint8_t **parseptr, uint32_t *offset) >> { >> - odp_packet_hdr(pkt)->user_ctx = (intptr_t)ctx; >> + odph_udphdr_t *udp = (odph_udphdr_t *)*parseptr; >> + uint32_t udplen = odp_be_to_cpu_16(udp->length); >> + >> + if (udplen < sizeof(odph_udphdr_t) || >> + udplen > (pkt_hdr->l3_len + >> + pkt_hdr->l3_offset - pkt_hdr->l4_offset)) { >> + pkt_hdr->error_flags.udp_err = 1; >> + } >> + >> + pkt_hdr->l4_len = udplen; >> + >> + *offset += sizeof(odph_udphdr_t); >> + *parseptr += sizeof(odph_udphdr_t); >> } >> >> -void *odp_packet_get_ctx(odp_packet_t pkt) >> +/** >> + * Simple packet parser >> + */ >> + >> +int _odp_packet_parse(odp_packet_t pkt) >> { >> - return (void *)(intptr_t)odp_packet_hdr(pkt)->user_ctx; >> + odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >> + odph_ethhdr_t *eth; >> + odph_vlanhdr_t *vlan; >> + uint16_t ethtype; >> + uint8_t *parseptr; >> + uint32_t offset, seglen; >> + uint8_t ip_proto = 0; >> + >> + /* Reset parser metadata for new parse */ >> + pkt_hdr->error_flags.all = 0; >> + pkt_hdr->input_flags.all = 0; >> + pkt_hdr->output_flags.all = 0; >> + pkt_hdr->l2_offset = 0; >> + pkt_hdr->l3_offset = 0; >> + pkt_hdr->l4_offset = 0; >> + pkt_hdr->payload_offset = 0; >> + pkt_hdr->vlan_s_tag = 0; >> + pkt_hdr->vlan_c_tag = 0; >> + pkt_hdr->l3_protocol = 0; >> + pkt_hdr->l4_protocol = 0; >> + >> + /* We only support Ethernet for now */ >> + pkt_hdr->input_flags.eth = 1; >> + >> + /* Detect jumbo frames */ >> + if (pkt_hdr->frame_len > ODPH_ETH_LEN_MAX) >> + pkt_hdr->input_flags.jumbo = 1; >> + >> + /* Assume valid L2 header, no CRC/FCS check in SW */ >> + pkt_hdr->input_flags.l2 = 1; >> + >> + eth = (odph_ethhdr_t *)packet_map(pkt_hdr, 0, &seglen); >> + offset = sizeof(odph_ethhdr_t); >> + parseptr = (uint8_t *)ð->type; >> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); >> + >> + /* Parse the VLAN header(s), if present */ >> + if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { >> + pkt_hdr->input_flags.vlan_qinq = 1; >> + pkt_hdr->input_flags.vlan = 1; >> + vlan = (odph_vlanhdr_t *)(void *)parseptr; >> + pkt_hdr->vlan_s_tag = ((ethtype << 16) | >> + odp_be_to_cpu_16(vlan->tci)); >> + offset += sizeof(odph_vlanhdr_t); >> + parseptr += sizeof(odph_vlanhdr_t); >> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >> *)parseptr)); >> + } >> + >> + if (ethtype == ODPH_ETHTYPE_VLAN) { >> + pkt_hdr->input_flags.vlan = 1; >> + vlan = (odph_vlanhdr_t *)(void *)parseptr; >> + pkt_hdr->vlan_c_tag = ((ethtype << 16) | >> + odp_be_to_cpu_16(vlan->tci)); >> + offset += sizeof(odph_vlanhdr_t); >> + parseptr += sizeof(odph_vlanhdr_t); >> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >> *)parseptr)); >> + } >> + >> + /* Check for SNAP vs. DIX */ >> + if (ethtype < ODPH_ETH_LEN_MAX) { >> + pkt_hdr->input_flags.snap = 1; >> + if (ethtype > pkt_hdr->frame_len - offset) { >> + pkt_hdr->error_flags.snap_len = 1; >> + goto parse_exit; >> + } >> + offset += 8; >> + parseptr += 8; >> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >> *)parseptr)); >> + } >> + >> + /* Consume Ethertype for Layer 3 parse */ >> + parseptr += 2; >> + >> + /* Set l3_offset+flag only for known ethtypes */ >> + pkt_hdr->input_flags.l3 = 1; >> + pkt_hdr->l3_offset = offset; >> + pkt_hdr->l3_protocol = ethtype; >> + >> + /* Parse Layer 3 headers */ >> + switch (ethtype) { >> + case ODPH_ETHTYPE_IPV4: >> + pkt_hdr->input_flags.ipv4 = 1; >> + ip_proto = parse_ipv4(pkt_hdr, &parseptr, &offset); >> + break; >> + >> + case ODPH_ETHTYPE_IPV6: >> + pkt_hdr->input_flags.ipv6 = 1; >> + ip_proto = parse_ipv6(pkt_hdr, &parseptr, &offset); >> + break; >> + >> + case ODPH_ETHTYPE_ARP: >> + pkt_hdr->input_flags.arp = 1; >> + ip_proto = 255; /* Reserved invalid by IANA */ >> + break; >> + >> + default: >> + pkt_hdr->input_flags.l3 = 0; >> + ip_proto = 255; /* Reserved invalid by IANA */ >> + } >> + >> + /* Set l4_offset+flag only for known ip_proto */ >> + pkt_hdr->input_flags.l4 = 1; >> + pkt_hdr->l4_offset = offset; >> + pkt_hdr->l4_protocol = ip_proto; >> + >> + /* Parse Layer 4 headers */ >> + switch (ip_proto) { >> + case ODPH_IPPROTO_ICMP: >> + pkt_hdr->input_flags.icmp = 1; >> + break; >> + >> + case ODPH_IPPROTO_TCP: >> + pkt_hdr->input_flags.tcp = 1; >> + parse_tcp(pkt_hdr, &parseptr, &offset); >> + break; >> + >> + case ODPH_IPPROTO_UDP: >> + pkt_hdr->input_flags.udp = 1; >> + parse_udp(pkt_hdr, &parseptr, &offset); >> + break; >> + >> + case ODPH_IPPROTO_AH: >> + case ODPH_IPPROTO_ESP: >> + pkt_hdr->input_flags.ipsec = 1; >> + break; >> + >> + default: >> + pkt_hdr->input_flags.l4 = 0; >> + break; >> + } >> + >> + /* >> + * Anything beyond what we parse here is considered payload. >> + * Note: Payload is really only relevant for TCP and UDP. For >> + * all other protocols, the payload offset will point to the >> + * final header (ARP, ICMP, AH, ESP, or IP Fragment). >> + */ >> + pkt_hdr->payload_offset = offset; >> + >> +parse_exit: >> + return pkt_hdr->error_flags.all != 0; >> } >> diff --git a/platform/linux-dpdk/odp_packet_dpdk.c >> b/platform/linux-dpdk/odp_packet_dpdk.c >> index 9fe4bbd..80ce74f 100644 >> --- a/platform/linux-dpdk/odp_packet_dpdk.c >> +++ b/platform/linux-dpdk/odp_packet_dpdk.c >> @@ -23,6 +23,8 @@ >> #include <odp_hints.h> >> #include <odp_thread.h> >> >> +#include <odp_system_info.h> >> +#include <odp_debug_internal.h> >> #include <odp_packet_dpdk.h> >> #include <net/if.h> >> >> @@ -164,11 +166,8 @@ int recv_pkt_dpdk(pkt_dpdk_t * const pkt_dpdk, >> odp_packet_t pkt_table[], >> nb_rx = rte_eth_rx_burst((uint8_t)pkt_dpdk->portid, >> (uint16_t)pkt_dpdk->queueid, >> (struct rte_mbuf **)pkt_table, >> (uint16_t)len); >> - for (i = 0; i < nb_rx; i++) { >> - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt_table[i]); >> - struct rte_mbuf *mb = &pkt_hdr->buf_hdr.mb; >> - odp_packet_parse(pkt_table[i], mb->pkt.pkt_len, 0); >> - } >> + for (i = 0; i < nb_rx; i++) >> + _odp_packet_parse(pkt_table[i]); >> return nb_rx; >> } >> >> diff --git a/platform/linux-dpdk/odp_packet_io.c >> b/platform/linux-dpdk/odp_packet_io.c >> index 79394bb..b6d1d35 100644 >> --- a/platform/linux-dpdk/odp_packet_io.c >> +++ b/platform/linux-dpdk/odp_packet_io.c >> @@ -16,34 +16,21 @@ >> #ifdef ODP_HAVE_NETMAP >> #include <odp_packet_netmap.h> >> #endif >> -#include <odp_hints.h> >> #include <odp_config.h> >> #include <odp_queue_internal.h> >> #include <odp_schedule_internal.h> >> -#include <odp_debug.h> >> - >> -#include <odp_pktio_socket.h> >> -#ifdef ODP_HAVE_NETMAP >> -#include <odp_pktio_netmap.h> >> -#endif >> +#include <odp_classification_internal.h> >> +#include <odp_debug_internal.h> >> >> #include <string.h> >> - >> -typedef struct { >> - pktio_entry_t entries[ODP_CONFIG_PKTIO_ENTRIES]; >> -} pktio_table_t; >> +#include <sys/ioctl.h> >> +#include <linux/if_arp.h> >> +#include <ifaddrs.h> >> >> static pktio_table_t *pktio_tbl; >> >> - >> -static pktio_entry_t *get_entry(odp_pktio_t id) >> -{ >> - if (odp_unlikely(id == ODP_PKTIO_INVALID || >> - id > ODP_CONFIG_PKTIO_ENTRIES)) >> - return NULL; >> - >> - return &pktio_tbl->entries[id - 1]; >> -} >> +/* pktio pointer entries ( for inlines) */ >> +void *pktio_entry_ptr[ODP_CONFIG_PKTIO_ENTRIES]; >> >> int odp_pktio_init_global(void) >> { >> @@ -65,10 +52,12 @@ int odp_pktio_init_global(void) >> memset(pktio_tbl, 0, sizeof(pktio_table_t)); >> >> for (id = 1; id <= ODP_CONFIG_PKTIO_ENTRIES; ++id) { >> - pktio_entry = get_entry(id); >> + pktio_entry = &pktio_tbl->entries[id - 1]; >> >> odp_spinlock_init(&pktio_entry->s.lock); >> + odp_spinlock_init(&pktio_entry->s.cls.lock); >> >> + pktio_entry_ptr[id - 1] = pktio_entry; >> /* Create a default output queue for each pktio resource >> */ >> snprintf(name, sizeof(name), "%i-pktio_outq_default", >> (int)id); >> name[ODP_QUEUE_NAME_LEN-1] = '\0'; >> @@ -115,16 +104,27 @@ static void unlock_entry(pktio_entry_t *entry) >> odp_spinlock_unlock(&entry->s.lock); >> } >> >> -static void init_pktio_entry(pktio_entry_t *entry, odp_pktio_params_t >> *params) >> +static void lock_entry_classifier(pktio_entry_t *entry) >> +{ >> + odp_spinlock_lock(&entry->s.lock); >> + odp_spinlock_lock(&entry->s.cls.lock); >> +} >> + >> +static void unlock_entry_classifier(pktio_entry_t *entry) >> +{ >> + odp_spinlock_unlock(&entry->s.cls.lock); >> + odp_spinlock_unlock(&entry->s.lock); >> +} >> + >> +static void init_pktio_entry(pktio_entry_t *entry) >> { >> set_taken(entry); >> entry->s.inq_default = ODP_QUEUE_INVALID; >> memset(&entry->s.pkt_dpdk, 0, sizeof(entry->s.pkt_dpdk)); >> - /* Save pktio parameters, type is the most useful */ >> - memcpy(&entry->s.params, params, sizeof(*params)); >> + pktio_classifier_init(entry); >> } >> >> -static odp_pktio_t alloc_lock_pktio_entry(odp_pktio_params_t *params) >> +static odp_pktio_t alloc_lock_pktio_entry(void) >> { >> odp_pktio_t id; >> pktio_entry_t *entry; >> @@ -133,13 +133,13 @@ static odp_pktio_t >> alloc_lock_pktio_entry(odp_pktio_params_t *params) >> for (i = 0; i < ODP_CONFIG_PKTIO_ENTRIES; ++i) { >> entry = &pktio_tbl->entries[i]; >> if (is_free(entry)) { >> - lock_entry(entry); >> + lock_entry_classifier(entry); >> if (is_free(entry)) { >> - init_pktio_entry(entry, params); >> + init_pktio_entry(entry); >> id = i + 1; >> return id; /* return with entry locked! */ >> } >> - unlock_entry(entry); >> + unlock_entry_classifier(entry); >> } >> } >> >> @@ -148,7 +148,7 @@ static odp_pktio_t >> alloc_lock_pktio_entry(odp_pktio_params_t *params) >> >> static int free_pktio_entry(odp_pktio_t id) >> { >> - pktio_entry_t *entry = get_entry(id); >> + pktio_entry_t *entry = get_pktio_entry(id); >> >> if (entry == NULL) >> return -1; >> @@ -158,28 +158,22 @@ static int free_pktio_entry(odp_pktio_t id) >> return 0; >> } >> >> -odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool, >> - odp_pktio_params_t *params) >> +odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool) >> { >> odp_pktio_t id; >> pktio_entry_t *pktio_entry; >> int res; >> >> - if (params == NULL) { >> - ODP_ERR("Invalid pktio params\n"); >> - return ODP_PKTIO_INVALID; >> - } >> - >> ODP_DBG("Allocating dpdk pktio\n"); >> >> - id = alloc_lock_pktio_entry(params); >> + id = alloc_lock_pktio_entry(); >> if (id == ODP_PKTIO_INVALID) { >> ODP_ERR("No resources available.\n"); >> return ODP_PKTIO_INVALID; >> } >> /* if successful, alloc_pktio_entry() returns with the entry >> locked */ >> >> - pktio_entry = get_entry(id); >> + pktio_entry = get_pktio_entry(id); >> >> res = setup_pkt_dpdk(&pktio_entry->s.pkt_dpdk, dev, pool); >> if (res == -1) { >> @@ -188,7 +182,7 @@ odp_pktio_t odp_pktio_open(const char *dev, >> odp_buffer_pool_t pool, >> id = ODP_PKTIO_INVALID; >> } >> >> - unlock_entry(pktio_entry); >> + unlock_entry_classifier(pktio_entry); >> return id; >> } >> >> @@ -197,7 +191,7 @@ int odp_pktio_close(odp_pktio_t id) >> pktio_entry_t *entry; >> int res = -1; >> >> - entry = get_entry(id); >> + entry = get_pktio_entry(id); >> if (entry == NULL) >> return -1; >> >> @@ -226,7 +220,7 @@ odp_pktio_t odp_pktio_get_input(odp_packet_t pkt) >> >> int odp_pktio_recv(odp_pktio_t id, odp_packet_t pkt_table[], unsigned >> len) >> { >> - pktio_entry_t *pktio_entry = get_entry(id); >> + pktio_entry_t *pktio_entry = get_pktio_entry(id); >> int pkts; >> int i; >> >> @@ -249,7 +243,7 @@ int odp_pktio_recv(odp_pktio_t id, odp_packet_t >> pkt_table[], unsigned len) >> >> int odp_pktio_send(odp_pktio_t id, odp_packet_t pkt_table[], unsigned >> len) >> { >> - pktio_entry_t *pktio_entry = get_entry(id); >> + pktio_entry_t *pktio_entry = get_pktio_entry(id); >> int pkts; >> >> if (pktio_entry == NULL) >> @@ -265,15 +259,17 @@ int odp_pktio_send(odp_pktio_t id, odp_packet_t >> pkt_table[], unsigned len) >> >> int odp_pktio_inq_setdef(odp_pktio_t id, odp_queue_t queue) >> { >> - pktio_entry_t *pktio_entry = get_entry(id); >> + pktio_entry_t *pktio_entry = get_pktio_entry(id); >> queue_entry_t *qentry = queue_to_qentry(queue); >> >> - if (pktio_entry == NULL || qentry == NULL) >> + if (pktio_entry == NULL || qentry == ODP_QUEUE_INVALID) >> return -1; >> >> if (qentry->s.type != ODP_QUEUE_TYPE_PKTIN) >> return -1; >> >> + qentry = queue_to_qentry(queue); >> + >> lock_entry(pktio_entry); >> pktio_entry->s.inq_default = queue; >> unlock_entry(pktio_entry); >> @@ -295,7 +291,7 @@ int odp_pktio_inq_remdef(odp_pktio_t id) >> >> odp_queue_t odp_pktio_inq_getdef(odp_pktio_t id) >> { >> - pktio_entry_t *pktio_entry = get_entry(id); >> + pktio_entry_t *pktio_entry = get_pktio_entry(id); >> >> if (pktio_entry == NULL) >> return ODP_QUEUE_INVALID; >> @@ -305,7 +301,7 @@ odp_queue_t odp_pktio_inq_getdef(odp_pktio_t id) >> >> odp_queue_t odp_pktio_outq_getdef(odp_pktio_t id) >> { >> - pktio_entry_t *pktio_entry = get_entry(id); >> + pktio_entry_t *pktio_entry = get_pktio_entry(id); >> >> if (pktio_entry == NULL) >> return ODP_QUEUE_INVALID; >> @@ -362,32 +358,32 @@ int pktin_enqueue(queue_entry_t *qentry, >> odp_buffer_hdr_t *buf_hdr) >> odp_buffer_hdr_t *pktin_dequeue(queue_entry_t *qentry) >> { >> odp_buffer_hdr_t *buf_hdr; >> + odp_buffer_t buf; >> + odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; >> + odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX]; >> + int pkts, i, j; >> >> buf_hdr = queue_deq(qentry); >> + if (buf_hdr != NULL) >> + return buf_hdr; >> >> - if (buf_hdr == NULL) { >> - odp_packet_t pkt; >> - odp_buffer_t buf; >> - odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; >> - odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX]; >> - int pkts, i, j; >> - >> - pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl, >> - QUEUE_MULTI_MAX); >> - >> - if (pkts > 0) { >> - pkt = pkt_tbl[0]; >> - buf = odp_buffer_from_packet(pkt); >> - buf_hdr = odp_buf_to_hdr(buf); >> + pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl, QUEUE_MULTI_MAX); >> + if (pkts <= 0) >> + return NULL; >> >> - for (i = 1, j = 0; i < pkts; ++i) { >> - buf = odp_buffer_from_packet(pkt_tbl[i]); >> - tmp_hdr_tbl[j++] = odp_buf_to_hdr(buf); >> - } >> - queue_enq_multi(qentry, tmp_hdr_tbl, j); >> - } >> + for (i = 0, j = 0; i < pkts; ++i) { >> + buf = odp_packet_to_buffer(pkt_tbl[i]); >> + buf_hdr = odp_buf_to_hdr(buf); >> + if (0 > packet_classifier(qentry->s.pktin, pkt_tbl[i])) >> + tmp_hdr_tbl[j++] = buf_hdr; >> } >> >> + if (0 == j) >> + return NULL; >> + >> + if (j > 1) >> + queue_enq_multi(qentry, &tmp_hdr_tbl[1], j-1); >> + buf_hdr = tmp_hdr_tbl[0]; >> return buf_hdr; >> } >> >> @@ -400,32 +396,179 @@ int pktin_enq_multi(queue_entry_t *qentry, >> odp_buffer_hdr_t *buf_hdr[], int num) >> int pktin_deq_multi(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr[], >> int num) >> { >> int nbr; >> + odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; >> + odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX]; >> + odp_buffer_hdr_t *tmp_hdr; >> + odp_buffer_t buf; >> + int pkts, i, j; >> >> nbr = queue_deq_multi(qentry, buf_hdr, num); >> - >> - if (nbr < num) { >> - odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; >> - odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX]; >> - odp_buffer_t buf; >> - int pkts, i; >> - >> - pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl, >> - QUEUE_MULTI_MAX); >> - if (pkts > 0) { >> - for (i = 0; i < pkts; ++i) { >> - buf = odp_buffer_from_packet(pkt_tbl[i]); >> - tmp_hdr_tbl[i] = odp_buf_to_hdr(buf); >> - } >> - queue_enq_multi(qentry, tmp_hdr_tbl, pkts); >> - } >> + if (odp_unlikely(nbr > num)) >> + ODP_ABORT("queue_deq_multi req: %d, returned %d\n", >> + num, nbr); >> + >> + /** queue already has number of requsted buffers, >> + * do not do receive in that case. >> + */ >> + if (nbr == num) >> + return nbr; >> + >> + pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl, QUEUE_MULTI_MAX); >> + if (pkts <= 0) >> + return nbr; >> + >> + for (i = 0, j = 0; i < pkts; ++i) { >> + buf = odp_packet_to_buffer(pkt_tbl[i]); >> + tmp_hdr = odp_buf_to_hdr(buf); >> + if (0 > packet_classifier(qentry->s.pktin, pkt_tbl[i])) >> + tmp_hdr_tbl[j++] = tmp_hdr; >> } >> >> + if (j) >> + queue_enq_multi(qentry, tmp_hdr_tbl, j); >> return nbr; >> } >> >> +int odp_pktio_set_mtu(odp_pktio_t id, int mtu) >> +{ >> + pktio_entry_t *entry; >> + int ret; >> + >> + if (mtu <= 0) { >> + ODP_DBG("illegal MTU value %d\n", mtu); >> + return -1; >> + } >> + >> + entry = get_pktio_entry(id); >> + if (entry == NULL) { >> + ODP_DBG("pktio entry %d does not exist\n", id); >> + return -1; >> + } >> + >> + lock_entry(entry); >> + >> + if (odp_unlikely(is_free(entry))) { >> + unlock_entry(entry); >> + ODP_DBG("already freed pktio\n"); >> + return -1; >> + } >> + >> + ret = rte_eth_dev_set_mtu(entry->s.pkt_dpdk.portid, >> (uint16_t)mtu); >> + >> + unlock_entry(entry); >> + return ret; >> +} >> + >> +int odp_pktio_mtu(odp_pktio_t id) >> +{ >> + pktio_entry_t *entry; >> + int mtu; >> + >> + entry = get_pktio_entry(id); >> + if (entry == NULL) { >> + ODP_DBG("pktio entry %d does not exist\n", id); >> + return -1; >> + } >> + >> + lock_entry(entry); >> + >> + if (odp_unlikely(is_free(entry))) { >> + unlock_entry(entry); >> + ODP_DBG("already freed pktio\n"); >> + return -1; >> + } >> + >> + rte_eth_dev_get_mtu(entry->s.pkt_dpdk.portid , (uint16_t *)&mtu); >> + >> + unlock_entry(entry); >> + return mtu; >> +} >> + >> +int odp_pktio_promisc_mode_set(odp_pktio_t id, odp_bool_t enable) >> +{ >> + pktio_entry_t *entry; >> + >> + entry = get_pktio_entry(id); >> + if (entry == NULL) { >> + ODP_DBG("pktio entry %d does not exist\n", id); >> + return -1; >> + } >> + >> + lock_entry(entry); >> + >> + if (odp_unlikely(is_free(entry))) { >> + unlock_entry(entry); >> + ODP_DBG("already freed pktio\n"); >> + return -1; >> + } >> + >> + if (enable) >> + rte_eth_promiscuous_enable(entry->s.pkt_dpdk.portid); >> + else >> + rte_eth_promiscuous_disable(entry->s.pkt_dpdk.portid); >> + >> + unlock_entry(entry); >> + return 0; >> +} >> + >> +int odp_pktio_promisc_mode(odp_pktio_t id) >> +{ >> + pktio_entry_t *entry; >> + int promisc; >> + >> + entry = get_pktio_entry(id); >> + if (entry == NULL) { >> + ODP_DBG("pktio entry %d does not exist\n", id); >> + return -1; >> + } >> + >> + lock_entry(entry); >> + >> + if (odp_unlikely(is_free(entry))) { >> + unlock_entry(entry); >> + ODP_DBG("already freed pktio\n"); >> + return -1; >> + } >> + >> + promisc = rte_eth_promiscuous_get(entry->s.pkt_dpdk.portid); >> + >> + unlock_entry(entry); >> + >> + return promisc; >> +} >> + >> +size_t odp_pktio_mac_addr(odp_pktio_t id, void *mac_addr, >> + size_t addr_size) >> +{ >> + pktio_entry_t *entry; >> + >> + if (addr_size < ETH_ALEN) >> + return 0; >> + >> + entry = get_pktio_entry(id); >> + if (entry == NULL) { >> + ODP_DBG("pktio entry %d does not exist\n", id); >> + return 0; >> + } >> + >> + lock_entry(entry); >> + >> + if (odp_unlikely(is_free(entry))) { >> + unlock_entry(entry); >> + ODP_DBG("already freed pktio\n"); >> + return -1; >> + } >> + >> + rte_eth_macaddr_get(entry->s.pkt_dpdk.portid, >> + (struct ether_addr *)mac_addr); >> + unlock_entry(entry); >> + >> + return ETH_ALEN; >> +} >> + >> int odp_pktio_get_mac_addr(odp_pktio_t id, unsigned char *mac_addr) >> { >> - pktio_entry_t *pktio_entry = get_entry(id); >> + pktio_entry_t *pktio_entry = get_pktio_entry(id); >> if (!pktio_entry) { >> ODP_ERR("Invalid odp_pktio_t value\n"); >> return -1; >> diff --git a/platform/linux-dpdk/odp_queue.c >> b/platform/linux-dpdk/odp_queue.c >> index 03f7c2c..9e906a3 100644 >> --- a/platform/linux-dpdk/odp_queue.c >> +++ b/platform/linux-dpdk/odp_queue.c >> @@ -11,13 +11,14 @@ >> #include <odp_buffer.h> >> #include <odp_buffer_internal.h> >> #include <odp_buffer_pool_internal.h> >> +#include <odp_buffer_inlines.h> >> #include <odp_internal.h> >> #include <odp_shared_memory.h> >> #include <odp_schedule_internal.h> >> #include <odp_config.h> >> #include <odp_packet_io_internal.h> >> #include <odp_packet_io_queue.h> >> -#include <odp_debug.h> >> +#include <odp_debug_internal.h> >> #include <odp_hints.h> >> >> #ifdef USE_TICKETLOCK >> diff --git a/platform/linux-dpdk/odp_schedule.c >> b/platform/linux-dpdk/odp_schedule.c >> deleted file mode 100644 >> index cb559e4..0000000 >> --- a/platform/linux-dpdk/odp_schedule.c >> +++ /dev/null >> @@ -1,421 +0,0 @@ >> -/* Copyright (c) 2013, Linaro Limited >> - * All rights reserved. >> - * >> - * SPDX-License-Identifier: BSD-3-Clause >> - */ >> - >> -#include <odp_schedule.h> >> -#include <odp_schedule_internal.h> >> -#include <odp_align.h> >> -#include <odp_queue.h> >> -#include <odp_shared_memory.h> >> -#include <odp_buffer.h> >> -#include <odp_buffer_pool.h> >> -#include <odp_internal.h> >> -#include <odp_config.h> >> -#include <odp_debug.h> >> -#include <odp_thread.h> >> -#include <odp_time.h> >> -#include <odp_spinlock.h> >> -#include <odp_hints.h> >> - >> -#include <odp_queue_internal.h> >> - >> - >> -/* Limits to number of scheduled queues */ >> -#define SCHED_POOL_SIZE (256*1024) >> - >> -/* Scheduler sub queues */ >> -#define QUEUES_PER_PRIO 4 >> - >> -/* TODO: random or queue based selection */ >> -#define SEL_PRI_QUEUE(x) ((QUEUES_PER_PRIO-1) & (queue_to_id(x))) >> - >> -/* Maximum number of dequeues */ >> -#define MAX_DEQ 4 >> - >> - >> -/* Mask of queues per priority */ >> -typedef uint8_t pri_mask_t; >> - >> -ODP_STATIC_ASSERT((8 * sizeof(pri_mask_t)) >= >> - QUEUES_PER_PRIO, "pri_mask_t_is_too_small"); >> - >> - >> -typedef struct { >> - odp_queue_t >> pri_queue[ODP_CONFIG_SCHED_PRIOS][QUEUES_PER_PRIO]; >> - pri_mask_t pri_mask[ODP_CONFIG_SCHED_PRIOS]; >> - odp_spinlock_t mask_lock; >> - odp_buffer_pool_t pool; >> -} sched_t; >> - >> -typedef struct { >> - odp_queue_t queue; >> - >> -} queue_desc_t; >> - >> -typedef struct { >> - odp_queue_t pri_queue; >> - odp_buffer_t desc_buf; >> - >> - odp_buffer_t buf[MAX_DEQ]; >> - int num; >> - int index; >> - odp_queue_t queue; >> - int pause; >> - >> -} sched_local_t; >> - >> -/* Global scheduler context */ >> -static sched_t *sched; >> - >> -/* Thread local scheduler context */ >> -static __thread sched_local_t sched_local; >> - >> - >> -static inline odp_queue_t select_pri_queue(odp_queue_t queue, int prio) >> -{ >> - int id = SEL_PRI_QUEUE(queue); >> - return sched->pri_queue[prio][id]; >> -} >> - >> - >> -int odp_schedule_init_global(void) >> -{ >> - odp_shm_t shm; >> - odp_buffer_pool_t pool; >> - int i, j; >> - >> - ODP_DBG("Schedule init ... "); >> - >> - shm = odp_shm_reserve("odp_scheduler", >> - sizeof(sched_t), >> - ODP_CACHE_LINE_SIZE, 0); >> - >> - sched = odp_shm_addr(shm); >> - >> - if (sched == NULL) { >> - ODP_ERR("Schedule init: Shm reserve failed.\n"); >> - return -1; >> - } >> - >> - pool = odp_buffer_pool_create("odp_sched_pool", NULL, >> - SCHED_POOL_SIZE, >> sizeof(queue_desc_t), >> - ODP_CACHE_LINE_SIZE, >> - ODP_BUFFER_TYPE_RAW); >> - >> - if (pool == ODP_BUFFER_POOL_INVALID) { >> - ODP_ERR("Schedule init: Pool create failed.\n"); >> - return -1; >> - } >> - >> - sched->pool = pool; >> - odp_spinlock_init(&sched->mask_lock); >> - >> - for (i = 0; i < ODP_CONFIG_SCHED_PRIOS; i++) { >> - odp_queue_t queue; >> - char name[] = "odp_priXX_YY"; >> - >> - name[7] = '0' + i / 10; >> - name[8] = '0' + i - 10*(i / 10); >> - >> - for (j = 0; j < QUEUES_PER_PRIO; j++) { >> - name[10] = '0' + j / 10; >> - name[11] = '0' + j - 10*(j / 10); >> - >> - queue = odp_queue_create(name, >> - ODP_QUEUE_TYPE_POLL, >> NULL); >> - >> - if (queue == ODP_QUEUE_INVALID) { >> - ODP_ERR("Sched init: Queue create >> failed.\n"); >> - return -1; >> - } >> - >> - sched->pri_queue[i][j] = queue; >> - sched->pri_mask[i] = 0; >> - } >> - } >> - >> - ODP_DBG("done\n"); >> - >> - return 0; >> -} >> - >> - >> -int odp_schedule_init_local(void) >> -{ >> - int i; >> - >> - sched_local.pri_queue = ODP_QUEUE_INVALID; >> - sched_local.desc_buf = ODP_BUFFER_INVALID; >> - >> - for (i = 0; i < MAX_DEQ; i++) >> - sched_local.buf[i] = ODP_BUFFER_INVALID; >> - >> - sched_local.num = 0; >> - sched_local.index = 0; >> - sched_local.queue = ODP_QUEUE_INVALID; >> - sched_local.pause = 0; >> - >> - return 0; >> -} >> - >> - >> -void odp_schedule_mask_set(odp_queue_t queue, int prio) >> -{ >> - int id = SEL_PRI_QUEUE(queue); >> - >> - odp_spinlock_lock(&sched->mask_lock); >> - sched->pri_mask[prio] |= 1 << id; >> - odp_spinlock_unlock(&sched->mask_lock); >> -} >> - >> - >> -odp_buffer_t odp_schedule_buffer_alloc(odp_queue_t queue) >> -{ >> - odp_buffer_t buf; >> - >> - buf = odp_buffer_alloc(sched->pool); >> - >> - if (buf != ODP_BUFFER_INVALID) { >> - queue_desc_t *desc; >> - desc = odp_buffer_addr(buf); >> - desc->queue = queue; >> - } >> - >> - return buf; >> -} >> - >> - >> -void odp_schedule_queue(odp_queue_t queue, int prio) >> -{ >> - odp_buffer_t desc_buf; >> - odp_queue_t pri_queue; >> - >> - pri_queue = select_pri_queue(queue, prio); >> - desc_buf = queue_sched_buf(queue); >> - >> - odp_queue_enq(pri_queue, desc_buf); >> -} >> - >> - >> -void odp_schedule_release_atomic(void) >> -{ >> - if (sched_local.pri_queue != ODP_QUEUE_INVALID && >> - sched_local.num == 0) { >> - /* Release current atomic queue */ >> - odp_queue_enq(sched_local.pri_queue, >> sched_local.desc_buf); >> - sched_local.pri_queue = ODP_QUEUE_INVALID; >> - } >> -} >> - >> - >> -static inline int copy_bufs(odp_buffer_t out_buf[], unsigned int max) >> -{ >> - int i = 0; >> - >> - while (sched_local.num && max) { >> - out_buf[i] = sched_local.buf[sched_local.index]; >> - sched_local.index++; >> - sched_local.num--; >> - max--; >> - i++; >> - } >> - >> - return i; >> -} >> - >> - >> -/* >> - * Schedule queues >> - * >> - * TODO: SYNC_ORDERED not implemented yet >> - */ >> -static int schedule(odp_queue_t *out_queue, odp_buffer_t out_buf[], >> - unsigned int max_num, unsigned int max_deq) >> -{ >> - int i, j; >> - int thr; >> - int ret; >> - >> - if (sched_local.num) { >> - ret = copy_bufs(out_buf, max_num); >> - >> - if (out_queue) >> - *out_queue = sched_local.queue; >> - >> - return ret; >> - } >> - >> - odp_schedule_release_atomic(); >> - >> - if (odp_unlikely(sched_local.pause)) >> - return 0; >> - >> - thr = odp_thread_id(); >> - >> - for (i = 0; i < ODP_CONFIG_SCHED_PRIOS; i++) { >> - int id; >> - >> - if (sched->pri_mask[i] == 0) >> - continue; >> - >> - id = thr & (QUEUES_PER_PRIO-1); >> - >> - for (j = 0; j < QUEUES_PER_PRIO; j++, id++) { >> - odp_queue_t pri_q; >> - odp_buffer_t desc_buf; >> - >> - if (id >= QUEUES_PER_PRIO) >> - id = 0; >> - >> - if (odp_unlikely((sched->pri_mask[i] & (1 << id)) >> == 0)) >> - continue; >> - >> - pri_q = sched->pri_queue[i][id]; >> - desc_buf = odp_queue_deq(pri_q); >> - >> - if (desc_buf != ODP_BUFFER_INVALID) { >> - queue_desc_t *desc; >> - odp_queue_t queue; >> - int num; >> - >> - desc = odp_buffer_addr(desc_buf); >> - queue = desc->queue; >> - >> - num = odp_queue_deq_multi(queue, >> - sched_local.buf, >> - max_deq); >> - >> - if (num == 0) { >> - /* Remove empty queue from >> scheduling, >> - * except packet input queues >> - */ >> - if (odp_queue_type(queue) == >> - ODP_QUEUE_TYPE_PKTIN) >> - odp_queue_enq(pri_q, >> desc_buf); >> - >> - continue; >> - } >> - >> - sched_local.num = num; >> - sched_local.index = 0; >> - ret = copy_bufs(out_buf, max_num); >> - >> - sched_local.queue = queue; >> - >> - if (queue_sched_atomic(queue)) { >> - /* Hold queue during atomic >> access */ >> - sched_local.pri_queue = pri_q; >> - sched_local.desc_buf = desc_buf; >> - } else { >> - /* Continue scheduling the queue >> */ >> - odp_queue_enq(pri_q, desc_buf); >> - } >> - >> - /* Output the source queue handle */ >> - if (out_queue) >> - *out_queue = queue; >> - >> - return ret; >> - } >> - } >> - } >> - >> - return 0; >> -} >> - >> - >> -static int schedule_loop(odp_queue_t *out_queue, uint64_t wait, >> - odp_buffer_t out_buf[], >> - unsigned int max_num, unsigned int max_deq) >> -{ >> - uint64_t start_cycle, cycle, diff; >> - int ret; >> - >> - start_cycle = 0; >> - >> - while (1) { >> - ret = schedule(out_queue, out_buf, max_num, max_deq); >> - >> - if (ret) >> - break; >> - >> - if (wait == ODP_SCHED_WAIT) >> - continue; >> - >> - if (wait == ODP_SCHED_NO_WAIT) >> - break; >> - >> - if (start_cycle == 0) { >> - start_cycle = odp_time_get_cycles(); >> - continue; >> - } >> - >> - cycle = odp_time_get_cycles(); >> - diff = odp_time_diff_cycles(start_cycle, cycle); >> - >> - if (wait < diff) >> - break; >> - } >> - >> - return ret; >> -} >> - >> - >> -odp_buffer_t odp_schedule(odp_queue_t *out_queue, uint64_t wait) >> -{ >> - odp_buffer_t buf; >> - >> - buf = ODP_BUFFER_INVALID; >> - >> - schedule_loop(out_queue, wait, &buf, 1, MAX_DEQ); >> - >> - return buf; >> -} >> - >> - >> -odp_buffer_t odp_schedule_one(odp_queue_t *out_queue, uint64_t wait) >> -{ >> - odp_buffer_t buf; >> - >> - buf = ODP_BUFFER_INVALID; >> - >> - schedule_loop(out_queue, wait, &buf, 1, 1); >> - >> - return buf; >> -} >> - >> - >> -int odp_schedule_multi(odp_queue_t *out_queue, uint64_t wait, >> - odp_buffer_t out_buf[], unsigned int num) >> -{ >> - return schedule_loop(out_queue, wait, out_buf, num, MAX_DEQ); >> -} >> - >> - >> -void odp_schedule_pause(void) >> -{ >> - sched_local.pause = 1; >> -} >> - >> - >> -void odp_schedule_resume(void) >> -{ >> - sched_local.pause = 0; >> -} >> - >> - >> -uint64_t odp_schedule_wait_time(uint64_t ns) >> -{ >> - if (ns <= ODP_SCHED_NO_WAIT) >> - ns = ODP_SCHED_NO_WAIT + 1; >> - >> - return odp_time_ns_to_cycles(ns); >> -} >> - >> - >> -int odp_schedule_num_prio(void) >> -{ >> - return ODP_CONFIG_SCHED_PRIOS; >> -} >> -- >> 1.9.1 >> >> > > _______________________________________________ > lng-odp mailing list > lng-odp@lists.linaro.org > http://lists.linaro.org/mailman/listinfo/lng-odp > >
On 4 February 2015 at 21:05, Mike Holmes <mike.holmes@linaro.org> wrote: > Venki - I think Zoltan is breaking his 0.6 patches up, andI think the > right thing to do is create patches for all the changes as I believe > ODP-DPDK will become a more important > If it is a feature, then yes we can break into small patches, but since this is a new baseline(say 0.5.0) we have no choice other than reviewing the whole patch is what I believe. Even if we try to break things into small groups, it will only help the reviewer but one cannot commit it as is, because it will not compile. > implementation to us this year. > > Maybe we do this at HKG15 to reduce the review cycle time ? > > On 4 February 2015 at 10:30, Venkatesh Vivekanandan < > venkatesh.vivekanandan@linaro.org> wrote: > >> Hi All, >> >> Inorder to make the REVIEW PROCESS SIMPLER as it is a huge patch, I have >> attached the changes that went in this patch on top of ODP 0.5.0 (diff >> between platform/linux-generic and platform/linux-dpdk). Hope this helps. >> >> Thanks, >> Venky >> >> On 4 February 2015 at 20:48, <venkatesh.vivekanandan@linaro.org> wrote: >> >>> From: Venkatesh Vivekanandan <venkatesh.vivekanandan@linaro.org> >>> >>> Signed-off-by: Venkatesh Vivekanandan <venkatesh.vivekanandan@linaro.org >>> > >>> --- >>> -> This patch must be applied on odp v0.5.0 ontop of odp-dpdk >>> -> odp_l2fwd, odp_pktio were tested both in burst and queue mode which >>> worked >>> fine. >>> -> ipsec app was tested and working fine. For this to work, >>> USE_MAC_ADDR_HACK has to be set to 0 for now. Will send out a patch >>> for >>> MAC address fix once this patch gets in. >>> >>> Checkpatch warning for camel case could not be avoided. >>> WARNING: Avoid CamelCase: <_Static_assert> >>> #373: FILE: platform/linux-dpdk/include/api/odp_debug.h:34: >>> +#define _Static_assert(e, s) extern int (*static_assert_checker(void)) \ >>> >>> WARNING: storage class should be at the beginning of the declaration >>> #373: FILE: platform/linux-dpdk/include/api/odp_debug.h:34: >>> +#define _Static_assert(e, s) extern int (*static_assert_checker(void)) \ >>> >>> v2: Fixed odp_flush_caches compilation error >>> v3: Removed debug code that slipped in >>> >>> platform/linux-dpdk/Makefile.am | 13 +- >>> platform/linux-dpdk/include/api/odp_buffer.h | 30 +- >>> platform/linux-dpdk/include/api/odp_buffer_pool.h | 124 ++- >>> platform/linux-dpdk/include/api/odp_debug.h | 77 +- >>> platform/linux-dpdk/include/api/odp_packet.h | 772 >>> +++++++++++++-- >>> platform/linux-dpdk/include/api/odp_packet_io.h | 92 +- >>> platform/linux-dpdk/include/api/odp_pktio_types.h | 45 - >>> .../linux-dpdk/include/api/odp_platform_types.h | 78 ++ >>> platform/linux-dpdk/include/odp_buffer_inlines.h | 179 ++++ >>> platform/linux-dpdk/include/odp_buffer_internal.h | 134 ++- >>> .../linux-dpdk/include/odp_buffer_pool_internal.h | 333 ++++++- >>> platform/linux-dpdk/include/odp_packet_dpdk.h | 2 - >>> platform/linux-dpdk/include/odp_packet_internal.h | 135 ++- >>> .../linux-dpdk/include/odp_packet_io_internal.h | 42 +- >>> platform/linux-dpdk/odp_buffer.c | 14 +- >>> platform/linux-dpdk/odp_buffer_pool.c | 153 +-- >>> platform/linux-dpdk/odp_init.c | 35 +- >>> platform/linux-dpdk/odp_linux.c | 117 ++- >>> platform/linux-dpdk/odp_packet.c | 1001 >>> +++++++++++++++----- >>> platform/linux-dpdk/odp_packet_dpdk.c | 9 +- >>> platform/linux-dpdk/odp_packet_io.c | 309 ++++-- >>> platform/linux-dpdk/odp_queue.c | 3 +- >>> platform/linux-dpdk/odp_schedule.c | 421 -------- >>> 23 files changed, 2993 insertions(+), 1125 deletions(-) >>> delete mode 100644 platform/linux-dpdk/include/api/odp_pktio_types.h >>> create mode 100644 platform/linux-dpdk/include/api/odp_platform_types.h >>> create mode 100644 platform/linux-dpdk/include/odp_buffer_inlines.h >>> delete mode 100644 platform/linux-dpdk/odp_schedule.c >>> >>> diff --git a/platform/linux-dpdk/Makefile.am >>> b/platform/linux-dpdk/Makefile.am >>> index db11e0a..6491d3a 100644 >>> --- a/platform/linux-dpdk/Makefile.am >>> +++ b/platform/linux-dpdk/Makefile.am >>> @@ -29,6 +29,7 @@ include_HEADERS = \ >>> >>> $(top_srcdir)/platform/linux-generic/include/api/odp_barrier.h \ >>> $(srcdir)/include/api/odp_buffer_pool.h \ >>> >>> $(top_srcdir)/platform/linux-generic/include/api/odp_byteorder.h \ >>> + >>> $(top_srcdir)/platform/linux-generic/include/api/odp_classification.h \ >>> >>> $(top_srcdir)/platform/linux-generic/include/api/odp_compiler.h \ >>> >>> $(top_srcdir)/platform/linux-generic/include/api/odp_config.h \ >>> >>> $(top_srcdir)/platform/linux-generic/include/api/odp_coremask.h \ >>> @@ -52,15 +53,16 @@ include_HEADERS = \ >>> >>> $(top_srcdir)/platform/linux-generic/include/api/odp_time.h \ >>> >>> $(top_srcdir)/platform/linux-generic/include/api/odp_timer.h \ >>> >>> $(top_srcdir)/platform/linux-generic/include/api/odp_version.h \ >>> - $(srcdir)/include/api/odp_pktio_types.h >>> + $(srcdir)/include/api/odp_platform_types.h >>> >>> subdirheadersdir = $(includedir)/helper >>> subdirheaders_HEADERS = \ >>> $(top_srcdir)/helper/include/odph_chksum.h \ >>> $(top_srcdir)/helper/include/odph_eth.h \ >>> + $(top_srcdir)/helper/include/odph_icmp.h \ >>> $(top_srcdir)/helper/include/odph_ip.h \ >>> + $(top_srcdir)/helper/include/odph_ipsec.h \ >>> $(top_srcdir)/helper/include/odph_linux.h \ >>> - $(top_srcdir)/helper/include/odph_packet.h \ >>> $(top_srcdir)/helper/include/odph_ring.h \ >>> $(top_srcdir)/helper/include/odph_udp.h >>> >>> @@ -68,9 +70,11 @@ __LIB__libodp_la_SOURCES = \ >>> ../linux-generic/odp_barrier.c \ >>> odp_buffer.c \ >>> odp_buffer_pool.c \ >>> + ../linux-generic/odp_classification.c \ >>> ../linux-generic/odp_coremask.c \ >>> ../linux-generic/odp_crypto.c \ >>> odp_init.c \ >>> + ../linux-generic/odp_impl.c \ >>> odp_linux.c \ >>> odp_packet.c \ >>> odp_packet_dpdk.c \ >>> @@ -80,11 +84,12 @@ __LIB__libodp_la_SOURCES = \ >>> odp_queue.c \ >>> ../linux-generic/odp_ring.c \ >>> ../linux-generic/odp_rwlock.c \ >>> - odp_schedule.c \ >>> + ../linux-generic/odp_schedule.c \ >>> ../linux-generic/odp_shared_memory.c \ >>> ../linux-generic/odp_spinlock.c \ >>> ../linux-generic/odp_system_info.c \ >>> ../linux-generic/odp_thread.c \ >>> ../linux-generic/odp_ticketlock.c \ >>> ../linux-generic/odp_time.c \ >>> - ../linux-generic/odp_timer.c >>> + ../linux-generic/odp_timer.c \ >>> + ../linux-generic/odp_weak.c >>> diff --git a/platform/linux-dpdk/include/api/odp_buffer.h >>> b/platform/linux-dpdk/include/api/odp_buffer.h >>> index b2fbc76..3c23035 100644 >>> --- a/platform/linux-dpdk/include/api/odp_buffer.h >>> +++ b/platform/linux-dpdk/include/api/odp_buffer.h >>> @@ -19,20 +19,13 @@ extern "C" { >>> #endif >>> >>> >>> - >>> #include <odp_std_types.h> >>> +#include <odp_platform_types.h> >>> >>> - >>> - >>> - >>> - >>> -/** >>> - * ODP buffer >>> +/** @defgroup odp_buffer ODP BUFFER >>> + * Operations on a buffer. >>> + * @{ >>> */ >>> -typedef unsigned long odp_buffer_t; >>> - >>> - >>> -#define ODP_BUFFER_INVALID (unsigned long)(-1L) /**< Invalid buffer */ >>> >>> >>> /** >>> @@ -74,11 +67,21 @@ int odp_buffer_type(odp_buffer_t buf); >>> * >>> * @param buf Buffer handle >>> * >>> - * @return 1 if valid, otherwise 0 >>> + * @retval 1 Buffer handle represents a valid buffer. >>> + * @retval 0 Buffer handle does not represent a valid buffer. >>> */ >>> int odp_buffer_is_valid(odp_buffer_t buf); >>> >>> /** >>> + * Buffer pool of the buffer >>> + * >>> + * @param buf Buffer handle >>> + * >>> + * @return Handle of buffer pool buffer belongs to >>> + */ >>> +odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf); >>> + >>> +/** >>> * Print buffer metadata to STDOUT >>> * >>> * @param buf Buffer handle >>> @@ -86,6 +89,9 @@ int odp_buffer_is_valid(odp_buffer_t buf); >>> */ >>> void odp_buffer_print(odp_buffer_t buf); >>> >>> +/** >>> + * @} >>> + */ >>> >>> #ifdef __cplusplus >>> } >>> diff --git a/platform/linux-dpdk/include/api/odp_buffer_pool.h >>> b/platform/linux-dpdk/include/api/odp_buffer_pool.h >>> index 382f4f0..4da5f84 100644 >>> --- a/platform/linux-dpdk/include/api/odp_buffer_pool.h >>> +++ b/platform/linux-dpdk/include/api/odp_buffer_pool.h >>> @@ -21,66 +21,143 @@ extern "C" { >>> >>> >>> #include <odp_std_types.h> >>> +#include <odp_platform_types.h> >>> #include <odp_buffer.h> >>> >>> +/** @addtogroup odp_buffer >>> + * Operations on a buffer pool. >>> + * @{ >>> + */ >>> + >>> /** Maximum queue name lenght in chars */ >>> #define ODP_BUFFER_POOL_NAME_LEN 32 >>> >>> -/** Invalid buffer pool */ >>> -#define ODP_BUFFER_POOL_INVALID (unsigned long)(-1L) >>> - >>> -/** ODP buffer pool */ >>> -typedef unsigned long odp_buffer_pool_t; >>> - >>> +/** >>> + * Buffer pool parameters >>> + * Used to communicate buffer pool creation options. >>> + */ >>> +typedef struct odp_buffer_pool_param_t { >>> + uint32_t buf_size; /**< Buffer size in bytes. The maximum >>> + number of bytes application will >>> + store in each buffer. For packets, this >>> + is the maximum packet data length, and >>> + configured headroom and tailroom will be >>> + added to this number */ >>> + uint32_t buf_align; /**< Minimum buffer alignment in bytes. >>> + Valid values are powers of two. Use 0 >>> + for default alignment. Default will >>> + always be a multiple of 8. */ >>> + uint32_t num_bufs; /**< Number of buffers in the pool */ >>> + int buf_type; /**< Buffer type */ >>> +} odp_buffer_pool_param_t; >>> >>> /** >>> * Create a buffer pool >>> + * This routine is used to create a buffer pool. It take three >>> + * arguments: the optional name of the pool to be created, an optional >>> shared >>> + * memory handle, and a parameter struct that describes the pool to be >>> + * created. If a name is not specified the result is an anonymous pool >>> that >>> + * cannot be referenced by odp_buffer_pool_lookup(). >>> + * >>> + * @param name Name of the pool, max ODP_BUFFER_POOL_NAME_LEN-1 >>> chars. >>> + * May be specified as NULL for anonymous pools. >>> + * >>> + * @param shm The shared memory object in which to create the pool. >>> + * Use ODP_SHM_NULL to reserve default memory type >>> + * for the buffer type. >>> * >>> - * @param name Name of the pool (max ODP_BUFFER_POOL_NAME_LEN - 1 >>> chars) >>> - * @param base_addr Pool base address >>> - * @param size Pool size in bytes >>> - * @param buf_size Buffer size in bytes >>> - * @param buf_align Minimum buffer alignment >>> - * @param buf_type Buffer type >>> + * @param params Buffer pool parameters. >>> * >>> - * @return Buffer pool handle >>> + * @return Handle of the created buffer pool >>> + * @retval ODP_BUFFER_POOL_INVALID Buffer pool could not be created >>> */ >>> + >>> odp_buffer_pool_t odp_buffer_pool_create(const char *name, >>> - void *base_addr, uint64_t size, >>> - size_t buf_size, size_t >>> buf_align, >>> - int buf_type); >>> + odp_shm_t shm, >>> + odp_buffer_pool_param_t >>> *params); >>> >>> +/** >>> + * Destroy a buffer pool previously created by odp_buffer_pool_create() >>> + * >>> + * @param pool Handle of the buffer pool to be destroyed >>> + * >>> + * @retval 0 Success >>> + * @retval -1 Failure >>> + * >>> + * @note This routine destroys a previously created buffer pool. This >>> call >>> + * does not destroy any shared memory object passed to >>> + * odp_buffer_pool_create() used to store the buffer pool contents. The >>> caller >>> + * takes responsibility for that. If no shared memory object was passed >>> as >>> + * part of the create call, then this routine will destroy any internal >>> shared >>> + * memory objects associated with the buffer pool. Results are >>> undefined if >>> + * an attempt is made to destroy a buffer pool that contains allocated >>> or >>> + * otherwise active buffers. >>> + */ >>> +int odp_buffer_pool_destroy(odp_buffer_pool_t pool); >>> >>> /** >>> * Find a buffer pool by name >>> * >>> * @param name Name of the pool >>> * >>> - * @return Buffer pool handle, or ODP_BUFFER_POOL_INVALID if not found. >>> + * @return Handle of found buffer pool >>> + * @retval ODP_BUFFER_POOL_INVALID Buffer pool could not be found >>> + * >>> + * @note This routine cannot be used to look up an anonymous pool (one >>> created >>> + * with no name). >>> */ >>> odp_buffer_pool_t odp_buffer_pool_lookup(const char *name); >>> >>> +/** >>> + * Buffer pool information struct >>> + * Used to get information about a buffer pool. >>> + */ >>> +typedef struct odp_buffer_pool_info_t { >>> + const char *name; /**< pool name */ >>> + odp_shm_t shm; /**< handle of shared memory >>> area >>> + supplied by application to >>> + contain buffer pool, or >>> + ODP_SHM_NULL if this pool is >>> + managed by ODP */ >>> + odp_buffer_pool_param_t params; /**< pool parameters */ >>> +} odp_buffer_pool_info_t; >>> + >>> +/** >>> + * Retrieve information about a buffer pool >>> + * >>> + * @param pool Buffer pool handle >>> + * >>> + * @param[out] info Receives an odp_buffer_pool_info_t object >>> + * that describes the pool. >>> + * >>> + * @retval 0 Success >>> + * @retval -1 Failure. Info could not be retrieved. >>> + */ >>> + >>> +int odp_buffer_pool_info(odp_buffer_pool_t pool, >>> + odp_buffer_pool_info_t *info); >>> >>> /** >>> * Print buffer pool info >>> * >>> * @param pool Pool handle >>> * >>> + * @note This routine writes implementation-defined information about >>> the >>> + * specified buffer pool to the ODP log. The intended use is for >>> debugging. >>> */ >>> void odp_buffer_pool_print(odp_buffer_pool_t pool); >>> >>> - >>> - >>> /** >>> * Buffer alloc >>> * >>> + * The validity of a buffer can be cheked at any time with >>> odp_buffer_is_valid() >>> * @param pool Pool handle >>> * >>> - * @return Buffer handle or ODP_BUFFER_INVALID >>> + * @return Handle of allocated buffer >>> + * @retval ODP_BUFFER_INVALID Buffer could not be allocated >>> */ >>> odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool); >>> >>> - >>> /** >>> * Buffer free >>> * >>> @@ -89,8 +166,9 @@ odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool); >>> */ >>> void odp_buffer_free(odp_buffer_t buf); >>> >>> - >>> - >>> +/** >>> + * @} >>> + */ >>> >>> #ifdef __cplusplus >>> } >>> diff --git a/platform/linux-dpdk/include/api/odp_debug.h >>> b/platform/linux-dpdk/include/api/odp_debug.h >>> index 6b145b5..aed8a5f 100644 >>> --- a/platform/linux-dpdk/include/api/odp_debug.h >>> +++ b/platform/linux-dpdk/include/api/odp_debug.h >>> @@ -12,73 +12,74 @@ >>> #ifndef ODP_DEBUG_H_ >>> #define ODP_DEBUG_H_ >>> >>> -#include <stdio.h> >>> -#include <stdlib.h> >>> >>> #ifdef __cplusplus >>> extern "C" { >>> #endif >>> >>> +/** @addtogroup odp_ver_abt_log_dbg >>> + * Macros that allows different messages. >>> + * @{ >>> + */ >>> + >>> #ifdef __GNUC__ >>> >>> -/** >>> - * Indicate deprecated variables, functions or types >>> - */ >>> -#define ODP_DEPRECATED __attribute__((__deprecated__)) >>> + >>> +#if __GNUC__ < 4 || (__GNUC__ == 4 && (__GNUC_MINOR__ < 6)) >>> >>> /** >>> - * Intentionally unused variables ot functions >>> + * _Static_assert was only added in GCC 4.6. Provide a weak replacement >>> + * for previous versions. >>> */ >>> -#define ODP_UNUSED __attribute__((__unused__)) >>> +#define _Static_assert(e, s) extern int (*static_assert_checker(void)) \ >>> + [sizeof(struct { unsigned int error_if_negative:(e) ? 1 : -1; })] >>> >>> -#else >>> +#endif >>> >>> -#define ODP_DEPRECATED >>> -#define ODP_UNUSED >>> >>> -#endif >>> >>> -/** >>> - * Runtime assertion-macro - aborts if 'cond' is false. >>> - */ >>> -#ifndef ODP_NO_DEBUG >>> -#define ODP_ASSERT(cond, msg) \ >>> - do { if (!(cond)) {ODP_ERR("%s\n", msg); abort(); } } while (0) >>> -#else >>> -#define ODP_ASSERT(cond, msg) >>> #endif >>> >>> + >>> /** >>> * Compile time assertion-macro - fail compilation if cond is false. >>> * @note This macro has zero runtime overhead >>> */ >>> -#define ODP_STATIC_ASSERT(cond, msg) _Static_assert(1, msg) >>> +#define _ODP_STATIC_ASSERT(cond, msg) _Static_assert(1, msg) >>> >>> /** >>> - * Debug printing macro, which prints output when DEBUG flag is set. >>> + * ODP log level. >>> */ >>> -#define ODP_DBG(fmt, ...) \ >>> - do { if (ODP_DEBUG_PRINT == 1) \ >>> - printf(fmt, ##__VA_ARGS__); \ >>> - } while (0) >>> +typedef enum odp_log_level { >>> + ODP_LOG_DBG, >>> + ODP_LOG_ERR, >>> + ODP_LOG_UNIMPLEMENTED, >>> + ODP_LOG_ABORT, >>> + ODP_LOG_PRINT >>> +} odp_log_level_e; >>> >>> /** >>> - * Print output to stderr (file, line and function). >>> + * ODP log function >>> + * >>> + * Instead of direct prints to stdout/stderr all logging in ODP >>> implementation >>> + * should be done via this function or its wrappers. >>> + * ODP platform MUST provide a default *weak* implementation of this >>> function. >>> + * Application MAY override the function if needed by providing a strong >>> + * function. >>> + * >>> + * @param[in] level Log level >>> + * @param[in] fmt printf-style message format >>> + * >>> + * @return The number of characters logged if succeeded. Otherwise >>> returns >>> + * a negative number. >>> */ >>> -#define ODP_ERR(fmt, ...) \ >>> -do { fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, \ >>> - __LINE__, __func__, ##__VA_ARGS__); \ >>> -} while (0) >>> +extern int odp_override_log(odp_log_level_e level, const char *fmt, >>> ...); >>> + >>> + >>> >>> /** >>> - * Print output to stderr (file, line and function), >>> - * then abort. >>> + * @} >>> */ >>> -#define ODP_ABORT(fmt, ...) \ >>> -do { fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, \ >>> - __LINE__, __func__, ##__VA_ARGS__); \ >>> - abort(); \ >>> -} while (0) >>> >>> #ifdef __cplusplus >>> } >>> diff --git a/platform/linux-dpdk/include/api/odp_packet.h >>> b/platform/linux-dpdk/include/api/odp_packet.h >>> index 64a92e8..97c2cb6 100644 >>> --- a/platform/linux-dpdk/include/api/odp_packet.h >>> +++ b/platform/linux-dpdk/include/api/odp_packet.h >>> @@ -1,7 +1,7 @@ >>> /* Copyright (c) 2013, Linaro Limited >>> * All rights reserved. >>> * >>> - * SPDX-License-Identifier: BSD-3-Clause >>> + * SPDX-License-Identifier: BSD-3-Clause >>> */ >>> >>> >>> @@ -19,33 +19,84 @@ extern "C" { >>> #endif >>> >>> #include <odp_buffer.h> >>> +#include <odp_platform_types.h> >>> >>> +/** @defgroup odp_packet ODP PACKET >>> + * Operations on a packet. >>> + * @{ >>> + */ >>> >>> -/** >>> - * ODP packet descriptor >>> + >>> +/* >>> + * Packet API v0.5 notes >>> + * - Push/pull operations only on packet level >>> + * - Push/pull within limits of segment headroom/tailroom/data lengths >>> + * - Segment data length must be always at least one byte (i.e. there >>> are no >>> + * empty segments) >>> + * - Head/tailroom content belong to packet content (in addition to data >>> + * and meta-data) and thus is preserved over packet ownership changes. >>> + * - _addr refer to a fixed address, which operations do not modify >>> + * - _ptr refer to pointer to data, which may be modified by operations >>> */ >>> -typedef odp_buffer_t odp_packet_t; >>> >>> >>> -/** Invalid packet */ >>> -#define ODP_PACKET_INVALID ODP_BUFFER_INVALID >>> +/* >>> + * >>> + * Alloc and free >>> + * ******************************************************** >>> + * >>> + */ >>> >>> -/** Invalid offset */ >>> -#define ODP_PACKET_OFFSET_INVALID ((size_t)-1) >>> +/** >>> + * Allocate a packet from a buffer pool >>> + * >>> + * Allocates a packet of the requested length from the specified buffer >>> pool. >>> + * Pool must have been created with buffer type ODP_BUFFER_TYPE_PACKET. >>> The >>> + * packet is initialized with data pointers and lengths set according >>> to the >>> + * specified len, and the default headroom and tailroom length >>> settings. All >>> + * other packet metadata are set to their default values. >>> + * >>> + * @param pool Pool handle >>> + * @param len Packet data length >>> + * >>> + * @return Handle of allocated packet >>> + * @retval ODP_PACKET_INVALID Packet could not be allocated >>> + * >>> + * @note The default headroom and tailroom used for packets is >>> specified by >>> + * the ODP_CONFIG_PACKET_HEADROOM and ODP_CONFIG_PACKET_TAILROOM >>> defines in >>> + * odp_config.h. >>> + */ >>> +odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool, uint32_t len); >>> >>> +/** >>> + * Free packet >>> + * >>> + * Frees the packet into the buffer pool it was allocated from. >>> + * >>> + * @param pkt Packet handle >>> + */ >>> +void odp_packet_free(odp_packet_t pkt); >>> >>> /** >>> - * Initialize the packet >>> + * Reset packet >>> * >>> - * Needs to be called if the user allocates a packet buffer, i.e. the >>> packet >>> - * has not been received from I/O through ODP. >>> + * Resets all packet meta-data to their default values. Packet length >>> is used >>> + * to initialize pointers and lengths. It must be less than the total >>> buffer >>> + * length of the packet minus the default headroom length. Packet is not >>> + * modified on failure. >>> * >>> - * @param pkt Packet handle >>> + * @param pkt Packet handle >>> + * @param len Packet data length >>> + * >>> + * @retval 0 Success >>> + * @retval Non-zero Failure >>> + * >>> + * @see odp_packet_buf_len() >>> */ >>> -void odp_packet_init(odp_packet_t pkt); >>> +int odp_packet_reset(odp_packet_t pkt, uint32_t len); >>> >>> /** >>> - * Convert from packet handle to buffer handle >>> + * Convert a buffer handle to a packet handle >>> * >>> * @param buf Buffer handle >>> * >>> @@ -54,190 +105,737 @@ void odp_packet_init(odp_packet_t pkt); >>> odp_packet_t odp_packet_from_buffer(odp_buffer_t buf); >>> >>> /** >>> - * Convert from buffer handle to packet handle >>> + * Convert a packet handle to a buffer handle >>> * >>> * @param pkt Packet handle >>> * >>> * @return Buffer handle >>> */ >>> -odp_buffer_t odp_buffer_from_packet(odp_packet_t pkt); >>> +odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt); >>> + >>> + >>> +/* >>> + * >>> + * Pointers and lengths >>> + * ******************************************************** >>> + * >>> + */ >>> + >>> +/** >>> + * Packet head address >>> + * >>> + * Returns start address of the first segment. Packet level headroom >>> starts >>> + * from here. Use odp_packet_data() or odp_packet_l2_ptr() to return the >>> + * packet data start address. >>> + * >>> + * @param pkt Packet handle >>> + * >>> + * @return Pointer to the start address of the first packet segment >>> + * >>> + * @see odp_packet_data(), odp_packet_l2_ptr(), odp_packet_headroom() >>> + */ >>> +void *odp_packet_head(odp_packet_t pkt); >>> + >>> +/** >>> + * Total packet buffer length >>> + * >>> + * Returns sum of buffer lengths over all packet segments. >>> + * >>> + * @param pkt Packet handle >>> + * >>> + * @return Total packet buffer length in bytes >>> + * >>> + * @see odp_packet_reset() >>> + */ >>> +uint32_t odp_packet_buf_len(odp_packet_t pkt); >>> >>> /** >>> - * Set the packet length >>> + * Packet data pointer >>> + * >>> + * Returns the current packet data pointer. When a packet is received >>> + * from packet input, this points to the first byte of the received >>> + * packet. Packet level offsets are calculated relative to this >>> position. >>> + * >>> + * User can adjust the data pointer with head_push/head_pull (does not >>> modify >>> + * segmentation) and add_data/rem_data calls (may modify segmentation). >>> + * >>> + * @param pkt Packet handle >>> + * >>> + * @return Pointer to the packet data >>> + * >>> + * @see odp_packet_l2_ptr(), odp_packet_seg_len() >>> + */ >>> +void *odp_packet_data(odp_packet_t pkt); >>> + >>> +/** >>> + * Packet segment data length >>> + * >>> + * Returns number of data bytes following the current data pointer >>> + * (odp_packet_data()) location in the segment. >>> * >>> * @param pkt Packet handle >>> - * @param len Length of packet in bytes >>> + * >>> + * @return Segment data length in bytes (pointed by odp_packet_data()) >>> + * >>> + * @see odp_packet_data() >>> */ >>> -void odp_packet_set_len(odp_packet_t pkt, size_t len); >>> +uint32_t odp_packet_seg_len(odp_packet_t pkt); >>> >>> /** >>> - * Get the packet length >>> + * Packet data length >>> + * >>> + * Returns sum of data lengths over all packet segments. >>> * >>> * @param pkt Packet handle >>> * >>> - * @return Packet length in bytes >>> + * @return Packet data length >>> */ >>> -size_t odp_packet_get_len(odp_packet_t pkt); >>> +uint32_t odp_packet_len(odp_packet_t pkt); >>> >>> /** >>> - * Set packet user context >>> + * Packet headroom length >>> * >>> - * @param buf Packet handle >>> - * @param ctx User context >>> + * Returns the current packet level headroom length. >>> + * >>> + * @param pkt Packet handle >>> * >>> + * @return Headroom length >>> */ >>> -void odp_packet_set_ctx(odp_packet_t buf, const void *ctx); >>> +uint32_t odp_packet_headroom(odp_packet_t pkt); >>> >>> /** >>> - * Get packet user context >>> + * Packet tailroom length >>> * >>> - * @param buf Packet handle >>> + * Returns the current packet level tailroom length. >>> * >>> - * @return User context >>> + * @param pkt Packet handle >>> + * >>> + * @return Tailroom length >>> */ >>> -void *odp_packet_get_ctx(odp_packet_t buf); >>> +uint32_t odp_packet_tailroom(odp_packet_t pkt); >>> >>> /** >>> - * Get address to the start of the packet buffer >>> + * Packet tailroom pointer >>> * >>> - * The address of the packet buffer is not necessarily the same as the >>> start >>> - * address of the received frame, e.g. an eth frame may be offset by 2 >>> or 6 >>> - * bytes to ensure 32 or 64-bit alignment of the IP header. >>> - * Use odp_packet_l2(pkt) to get the start address of a received valid >>> frame >>> - * or odp_packet_start(pkt) to get the start address even if no valid >>> L2 header >>> - * could be found. >>> + * Returns pointer to the start of the current packet level tailroom. >>> + * >>> + * User can adjust the tail pointer with tail_push/tail_pull (does not >>> modify >>> + * segmentation) and add_data/rem_data calls (may modify segmentation). >>> * >>> * @param pkt Packet handle >>> * >>> - * @return Pointer to the start of the packet buffer >>> + * @return Tailroom pointer >>> * >>> - * @see odp_packet_l2(), odp_packet_start() >>> + * @see odp_packet_tailroom() >>> */ >>> -uint8_t *odp_packet_buf_addr(odp_packet_t pkt); >>> +void *odp_packet_tail(odp_packet_t pkt); >>> >>> /** >>> - * Get pointer to the start of the received frame >>> + * Push out packet head >>> + * >>> + * Increase packet data length by moving packet head into packet >>> headroom. >>> + * Packet headroom is decreased with the same amount. The packet head >>> may be >>> + * pushed out up to 'headroom' bytes. Packet is not modified if there's >>> not >>> + * enough headroom space. >>> * >>> - * The address of the packet buffer is not necessarily the same as the >>> start >>> - * address of the received frame, e.g. an eth frame may be offset by 2 >>> or 6 >>> - * bytes to ensure 32 or 64-bit alignment of the IP header. >>> - * Use odp_packet_l2(pkt) to get the start address of a received valid >>> eth frame >>> + * odp_packet_xxx: >>> + * seg_len += len >>> + * len += len >>> + * headroom -= len >>> + * data -= len >>> * >>> - * odp_packet_start() will always return a pointer to the start of the >>> frame, >>> - * even if the frame is unrecognized and no valid L2 header could be >>> found. >>> + * Operation does not modify packet segmentation or move data. Handles >>> and >>> + * pointers remain valid. User is responsible to update packet meta-data >>> + * offsets when needed. >>> * >>> * @param pkt Packet handle >>> + * @param len Number of bytes to push the head (0 ... headroom) >>> * >>> - * @return Pointer to the start of the received frame >>> + * @return The new data pointer >>> + * @retval NULL Requested offset exceeds available headroom >>> * >>> - * @see odp_packet_l2(), odp_packet_buf_addr() >>> + * @see odp_packet_headroom(), odp_packet_pull_head() >>> */ >>> -uint8_t *odp_packet_start(odp_packet_t pkt); >>> +void *odp_packet_push_head(odp_packet_t pkt, uint32_t len); >>> >>> /** >>> - * Get pointer to the start of the L2 frame >>> + * Pull in packet head >>> * >>> - * The L2 frame header address is not necessarily the same as the >>> address of the >>> - * packet buffer, see odp_packet_buf_addr() >>> + * Decrease packet data length by removing data from the head of the >>> packet. >>> + * Packet headroom is increased with the same amount. Packet head may >>> be pulled >>> + * in up to seg_len - 1 bytes (i.e. packet data pointer must stay in the >>> + * first segment). Packet is not modified if there's not enough data. >>> + * >>> + * odp_packet_xxx: >>> + * seg_len -= len >>> + * len -= len >>> + * headroom += len >>> + * data += len >>> + * >>> + * Operation does not modify packet segmentation or move data. Handles >>> and >>> + * pointers remain valid. User is responsible to update packet meta-data >>> + * offsets when needed. >>> * >>> * @param pkt Packet handle >>> + * @param len Number of bytes to pull the head (0 ... seg_len - 1) >>> * >>> - * @return Pointer to L2 header or NULL if not found >>> + * @return The new data pointer, or NULL in case of an error. >>> + * @retval NULL Requested offset exceeds packet segment length >>> * >>> - * @see odp_packet_buf_addr(), odp_packet_start() >>> + * @see odp_packet_seg_len(), odp_packet_push_head() >>> */ >>> -uint8_t *odp_packet_l2(odp_packet_t pkt); >>> +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len); >>> >>> /** >>> - * Return the byte offset from the packet buffer to the L2 frame >>> + * Push out packet tail >>> + * >>> + * Increase packet data length by moving packet tail into packet >>> tailroom. >>> + * Packet tailroom is decreased with the same amount. The packet tail >>> may be >>> + * pushed out up to 'tailroom' bytes. Packet is not modified if there's >>> not >>> + * enough tailroom. >>> + * >>> + * last_seg: >>> + * data_len += len >>> + * >>> + * odp_packet_xxx: >>> + * len += len >>> + * tail += len >>> + * tailroom -= len >>> + * >>> + * Operation does not modify packet segmentation or move data. Handles, >>> + * pointers and offsets remain valid. >>> * >>> * @param pkt Packet handle >>> + * @param len Number of bytes to push the tail (0 ... tailroom) >>> * >>> - * @return L2 byte offset or ODP_PACKET_OFFSET_INVALID if not found >>> + * @return The old tail pointer >>> + * @retval NULL Requested offset exceeds available tailroom >>> + * >>> + * @see odp_packet_tailroom(), odp_packet_pull_tail() >>> */ >>> -size_t odp_packet_l2_offset(odp_packet_t pkt); >>> +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len); >>> >>> /** >>> - * Set the byte offset to the L2 frame >>> + * Pull in packet tail >>> + * >>> + * Decrease packet data length by removing data from the tail of the >>> packet. >>> + * Packet tailroom is increased with the same amount. Packet tail may >>> be pulled >>> + * in up to last segment data_len - 1 bytes. (i.e. packet tail must >>> stay in the >>> + * last segment). Packet is not modified if there's not enough data. >>> + * >>> + * last_seg: >>> + * data_len -= len >>> + * >>> + * odp_packet_xxx: >>> + * len -= len >>> + * tail -= len >>> + * tailroom += len >>> + * >>> + * Operation does not modify packet segmentation or move data. Handles >>> and >>> + * pointers remain valid. User is responsible to update packet meta-data >>> + * offsets when needed. >>> + * >>> + * @param pkt Packet handle >>> + * @param len Number of bytes to pull the tail (0 ... >>> last_seg:data_len - 1) >>> + * >>> + * @return The new tail pointer >>> + * @retval NULL The specified offset exceeds allowable data length >>> + */ >>> +void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len); >>> + >>> +/** >>> + * Packet offset pointer >>> + * >>> + * Returns pointer to data in the packet offset. The packet level byte >>> offset is >>> + * calculated from the current odp_packet_data() position. Optionally >>> outputs >>> + * handle to the segment and number of data bytes in the segment >>> following the >>> + * pointer. >>> + * >>> + * @param pkt Packet handle >>> + * @param offset Byte offset into the packet >>> + * @param[out] len Number of data bytes remaining in the segment >>> (output). >>> + * Ignored when NULL. >>> + * @param[out] seg Handle to the segment containing the address >>> (output). >>> + * Ignored when NULL. >>> + * >>> + * @return Pointer to the offset >>> + * @retval NULL Requested offset exceeds packet length >>> + */ >>> +void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t >>> *len, >>> + odp_packet_seg_t *seg); >>> + >>> +/* >>> + * >>> + * Meta-data >>> + * ******************************************************** >>> + * >>> + */ >>> + >>> +/** >>> + * Packet pool >>> + * >>> + * Returns handle to the buffer pool where the packet was allocated >>> from. >>> + * >>> + * @param pkt Packet handle >>> + * >>> + * @return Buffer pool handle >>> + */ >>> +odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt); >>> + >>> +/** >>> + * Packet input interface >>> + * >>> + * Returns handle to the packet IO interface which received the packet >>> or >>> + * ODP_PKTIO_INVALID when the packet was allocated/reset by the >>> application. >>> + * >>> + * @param pkt Packet handle >>> + * >>> + * @return Packet interface handle >>> + * @retval ODP_PKTIO_INVALID Packet was not received >>> + */ >>> +odp_pktio_t odp_packet_input(odp_packet_t pkt); >>> + >>> +/** >>> + * User context pointer >>> + * >>> + * Return previously stored user context pointer. >>> + * >>> + * @param pkt Packet handle >>> + * >>> + * @return User context pointer >>> + */ >>> +void *odp_packet_user_ptr(odp_packet_t pkt); >>> + >>> +/** >>> + * Set user context pointer >>> + * >>> + * Each packet has room for a user defined context. The context can be >>> stored >>> + * either as a pointer OR as a uint64_t value, but not both at the same >>> time. >>> + * The latest context set operation determines which one has been >>> stored. >>> + * >>> + * @param pkt Packet handle >>> + * @param ctx User context pointer >>> + */ >>> +void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ctx); >>> + >>> +/** >>> + * User context data (uint64_t) >>> + * >>> + * Return previously stored user context uint64_t value. >>> + * >>> + * @param pkt Packet handle >>> + * >>> + * @return User context data >>> + */ >>> +uint64_t odp_packet_user_u64(odp_packet_t pkt); >>> + >>> +/** >>> + * Set user context data (uint64_t) >>> + * >>> + * Each packet has room for a user defined context. The context can be >>> stored >>> + * either as a pointer OR as a uint64_t value, but not both at the same >>> time. >>> + * The latest context set operation determines which one has been >>> stored. >>> + * >>> + * @param pkt Packet handle >>> + * @param ctx User context data >>> + */ >>> +void odp_packet_user_u64_set(odp_packet_t pkt, uint64_t ctx); >>> + >>> +/** >>> + * Layer 2 start pointer >>> + * >>> + * Returns pointer to the start of the layer 2 header. Optionally, >>> outputs >>> + * number of data bytes in the segment following the pointer. >>> + * >>> + * @param pkt Packet handle >>> + * @param[out] len Number of data bytes remaining in the segment >>> (output). >>> + * Ignored when NULL. >>> + * >>> + * @return Layer 2 start pointer, or offset 0 by default >>> + * >>> + * @see odp_packet_l2_offset(), odp_packet_l2_offset_set() >>> + */ >>> +void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len); >>> + >>> +/** >>> + * Layer 2 start offset >>> + * >>> + * Returns offset to the start of the layer 2 header. The offset is >>> calculated >>> + * from the current odp_packet_data() position in bytes. >>> + * >>> + * User is responsible to update the offset when modifying the packet >>> data >>> + * pointer position. >>> + * >>> + * @param pkt Packet handle >>> + * >>> + * @return Layer 2 start offset >>> + */ >>> +uint32_t odp_packet_l2_offset(odp_packet_t pkt); >>> + >>> +/** >>> + * Set layer 2 start offset >>> + * >>> + * Set offset to the start of the layer 2 header. The offset is >>> calculated from >>> + * the current odp_packet_data() position in bytes. Offset must not >>> exceed >>> + * packet data length. Packet is not modified on an error. >>> * >>> * @param pkt Packet handle >>> - * @param offset L2 byte offset >>> + * @param offset Layer 2 start offset (0 ... odp_packet_len()-1) >>> + * >>> + * @retval 0 Success >>> + * @retval Non-zero Failure >>> */ >>> -void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset); >>> +int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset); >>> >>> +/** >>> + * Layer 3 start pointer >>> + * >>> + * Returns pointer to the start of the layer 3 header. Optionally, >>> outputs >>> + * number of data bytes in the segment following the pointer. >>> + * >>> + * @param pkt Packet handle >>> + * @param[out] len Number of data bytes remaining in the segment >>> (output). >>> + * Ignored when NULL. >>> + * >>> + * @return Layer 3 start pointer, or NULL >>> + * >>> + * @see odp_packet_l3_offset(), odp_packet_l3_offset_set() >>> + */ >>> +void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len); >>> >>> /** >>> - * Get pointer to the start of the L3 packet >>> + * Layer 3 start offset >>> + * >>> + * Returns offset to the start of the layer 3 header. The offset is >>> calculated >>> + * from the current odp_packet_data() position in bytes. >>> + * >>> + * User is responsible to update the offset when modifying the packet >>> data >>> + * pointer position. >>> * >>> * @param pkt Packet handle >>> * >>> - * @return Pointer to L3 packet or NULL if not found >>> + * @return Layer 3 start offset or ODP_PACKET_OFFSET_INVALID if not >>> found >>> + */ >>> +uint32_t odp_packet_l3_offset(odp_packet_t pkt); >>> + >>> +/** >>> + * Set layer 3 start offset >>> + * >>> + * Set offset to the start of the layer 3 header. The offset is >>> calculated from >>> + * the current odp_packet_data() position in bytes. Offset must not >>> exceed >>> + * packet data length. Packet is not modified on an error. >>> + * >>> + * @param pkt Packet handle >>> + * @param offset Layer 3 start offset (0 ... odp_packet_len()-1) >>> * >>> + * @retval 0 Success >>> + * @retval Non-zero Failure >>> */ >>> -uint8_t *odp_packet_l3(odp_packet_t pkt); >>> +int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset); >>> >>> /** >>> - * Return the byte offset from the packet buffer to the L3 packet >>> + * Layer 4 start pointer >>> + * >>> + * Returns pointer to the start of the layer 4 header. Optionally, >>> outputs >>> + * number of data bytes in the segment following the pointer. >>> + * >>> + * @param pkt Packet handle >>> + * @param[out] len Number of data bytes remaining in the segment >>> (output). >>> + * Ignored when NULL. >>> + * >>> + * @return Layer 4 start pointer, or NULL >>> + * >>> + * @see odp_packet_l4_offset(), odp_packet_l4_offset_set() >>> + */ >>> +void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len); >>> + >>> +/** >>> + * Layer 4 start offset >>> + * >>> + * Returns offset to the start of the layer 4 header. The offset is >>> calculated >>> + * from the current odp_packet_data() position in bytes. >>> + * >>> + * User is responsible to update the offset when modifying the packet >>> data >>> + * pointer position. >>> * >>> * @param pkt Packet handle >>> * >>> - * @return L3 byte offset or ODP_PACKET_OFFSET_INVALID if not found >>> + * @return Layer 4 start offset or ODP_PACKET_OFFSET_INVALID if not >>> found >>> */ >>> -size_t odp_packet_l3_offset(odp_packet_t pkt); >>> +uint32_t odp_packet_l4_offset(odp_packet_t pkt); >>> >>> /** >>> - * Set the byte offset to the L3 packet >>> + * Set layer 4 start offset >>> + * >>> + * Set offset to the start of the layer 4 header. The offset is >>> calculated from >>> + * the current odp_packet_data() position in bytes. Offset must not >>> exceed >>> + * packet data length. Packet is not modified on an error. >>> * >>> * @param pkt Packet handle >>> - * @param offset L3 byte offset >>> + * @param offset Layer 4 start offset (0 ... odp_packet_len()-1) >>> + * >>> + * @retval 0 Success >>> + * @retval Non-zero Failure >>> + */ >>> +int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset); >>> + >>> +/** >>> + * Tests if packet is segmented >>> + * >>> + * @param pkt Packet handle >>> + * >>> + * @retval 0 Packet is not segmented >>> + * @retval 1 Packet is segmented >>> + */ >>> +int odp_packet_is_segmented(odp_packet_t pkt); >>> + >>> +/** >>> + * Number of segments >>> + * >>> + * Returns number of segments in the packet. A packet has always at >>> least one >>> + * segment. >>> + * >>> + * @param pkt Packet handle >>> + * >>> + * @return Number of segments (>0) >>> + */ >>> +int odp_packet_num_segs(odp_packet_t pkt); >>> + >>> +/** >>> + * First segment in packet >>> + * >>> + * A packet has always the first segment (has at least one segment). >>> + * >>> + * @param pkt Packet handle >>> + * >>> + * @return Handle to the first segment >>> + */ >>> +odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt); >>> + >>> +/** >>> + * Last segment in packet >>> + * >>> + * A packet has always the last segment (has at least one segment). >>> + * >>> + * @param pkt Packet handle >>> + * >>> + * @return Handle to the last segment >>> + */ >>> +odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt); >>> + >>> +/** >>> + * Next segment in packet >>> + * >>> + * Returns handle to the next segment after the current segment, or >>> + * ODP_PACKET_SEG_INVALID if there are no more segments. Use >>> + * odp_packet_first_seg() to get handle to the first segment. >>> + * >>> + * @param pkt Packet handle >>> + * @param seg Current segment handle >>> + * >>> + * @return Handle to the next segment, or ODP_PACKET_SEG_INVALID >>> + */ >>> +odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t >>> seg); >>> + >>> + >>> +/* >>> + * >>> + * Segment level >>> + * ******************************************************** >>> + * >>> */ >>> -void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset); >>> >>> +/** >>> + * Segment buffer address >>> + * >>> + * Returns start address of the segment. >>> + * >>> + * @param pkt Packet handle >>> + * @param seg Segment handle >>> + * >>> + * @return Start address of the segment, or NULL on an error >>> + * >>> + * @see odp_packet_seg_buf_len() >>> + */ >>> +void *odp_packet_seg_buf_addr(odp_packet_t pkt, odp_packet_seg_t seg); >>> >>> /** >>> - * Get pointer to the start of the L4 packet >>> + * Segment buffer length >>> + * >>> + * Returns segment buffer length in bytes. >>> * >>> * @param pkt Packet handle >>> + * @param seg Segment handle >>> * >>> - * @return Pointer to L4 packet or NULL if not found >>> + * @return Segment buffer length in bytes >>> * >>> + * @see odp_packet_seg_buf_addr() >>> */ >>> -uint8_t *odp_packet_l4(odp_packet_t pkt); >>> +uint32_t odp_packet_seg_buf_len(odp_packet_t pkt, odp_packet_seg_t seg); >>> >>> /** >>> - * Return the byte offset from the packet buffer to the L4 packet >>> + * Segment data pointer >>> + * >>> + * Returns pointer to the first byte of data in the segment. >>> + * >>> + * @param pkt Packet handle >>> + * @param seg Segment handle >>> + * >>> + * @return Pointer to the segment data, or NULL on an error >>> + * >>> + * @see odp_packet_seg_data_len() >>> + */ >>> +void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg); >>> + >>> +/** >>> + * Segment data length >>> + * >>> + * Returns segment data length in bytes. >>> * >>> * @param pkt Packet handle >>> + * @param seg Segment handle >>> * >>> - * @return L4 byte offset or ODP_PACKET_OFFSET_INVALID if not found >>> + * @return Segment data length in bytes >>> + * >>> + * @see odp_packet_seg_data() >>> */ >>> -size_t odp_packet_l4_offset(odp_packet_t pkt); >>> +uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t >>> seg); >>> + >>> + >>> +/* >>> + * >>> + * Manipulation >>> + * ******************************************************** >>> + * >>> + */ >>> + >>> >>> /** >>> - * Set the byte offset to the L4 packet >>> + * Add data into an offset >>> + * >>> + * Increases packet data length by adding new data area into the >>> specified >>> + * offset. The operation returns a new packet handle on success. It may >>> modify >>> + * packet segmentation and move data. Handles and pointers must be >>> updated >>> + * after the operation. User is responsible to update packet meta-data >>> offsets >>> + * when needed. The packet is not modified on an error. >>> * >>> * @param pkt Packet handle >>> - * @param offset L4 byte offset >>> + * @param offset Byte offset into the packet >>> + * @param len Number of bytes to add into the offset >>> + * >>> + * @return New packet handle, or ODP_PACKET_INVALID in case of an error. >>> */ >>> -void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset); >>> +odp_packet_t odp_packet_add_data(odp_packet_t pkt, uint32_t offset, >>> + uint32_t len); >>> >>> /** >>> - * Print (debug) information about the packet >>> + * Remove data from an offset >>> + * >>> + * Decreases packet data length by removing data from the specified >>> offset. >>> + * The operation returns a new packet handle on success, and may modify >>> + * packet segmentation and move data. Handles and pointers must be >>> updated >>> + * after the operation. User is responsible to update packet meta-data >>> offsets >>> + * when needed. The packet is not modified on an error. >>> + * >>> + * @param pkt Packet handle >>> + * @param offset Byte offset into the packet >>> + * @param len Number of bytes to remove from the offset >>> + * >>> + * @return New packet handle, or ODP_PACKET_INVALID in case of an error. >>> + */ >>> +odp_packet_t odp_packet_rem_data(odp_packet_t pkt, uint32_t offset, >>> + uint32_t len); >>> + >>> + >>> +/* >>> + * >>> + * Copy >>> + * ******************************************************** >>> + * >>> + */ >>> + >>> +/** >>> + * Copy packet >>> + * >>> + * Create a new copy of the packet. The new packet is exact copy of the >>> source >>> + * packet (incl. data and meta-data). The pool must have been created >>> with >>> + * buffer type ODP_BUFFER_TYPE_PACKET. >>> + * >>> + * @param pkt Packet handle >>> + * @param pool Buffer pool for allocation of the new packet. >>> + * >>> + * @return Handle to the copy of the packet, or ODP_PACKET_INVALID >>> + */ >>> +odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool); >>> + >>> +/** >>> + * Copy data from packet >>> + * >>> + * Copy 'len' bytes of data from the packet level offset to the >>> destination >>> + * address. >>> + * >>> + * @param pkt Packet handle >>> + * @param offset Byte offset into the packet >>> + * @param len Number of bytes to copy >>> + * @param dst Destination address >>> + * >>> + * @retval 0 Success >>> + * @retval Non-zero Failure >>> + */ >>> +int odp_packet_copydata_out(odp_packet_t pkt, uint32_t offset, >>> + uint32_t len, void *dst); >>> + >>> +/** >>> + * Copy data into packet >>> + * >>> + * Copy 'len' bytes of data from the source address into the packet >>> level >>> + * offset. Maximum number of bytes to copy is packet data length minus >>> the >>> + * offset. Packet is not modified on an error. >>> + * >>> + * @param pkt Packet handle >>> + * @param offset Byte offset into the packet >>> + * @param len Number of bytes to copy >>> + * @param src Source address >>> + * >>> + * @retval 0 Success >>> + * @retval Non-zero Failure >>> + */ >>> +int odp_packet_copydata_in(odp_packet_t pkt, uint32_t offset, >>> + uint32_t len, const void *src); >>> + >>> +/* >>> + * >>> + * Debugging >>> + * ******************************************************** >>> + * >>> + */ >>> + >>> +/** >>> + * Print packet to the console >>> + * >>> + * Print all packet debug information to the console. >>> * >>> * @param pkt Packet handle >>> */ >>> void odp_packet_print(odp_packet_t pkt); >>> >>> /** >>> - * Copy contents and metadata from pkt_src to pkt_dst >>> - * Useful when creating copies of packets >>> + * Perform full packet validity check >>> * >>> - * @param pkt_dst Destination packet >>> - * @param pkt_src Source packet >>> + * The operation may consume considerable number of cpu cycles >>> depending on >>> + * the check level. >>> + * >>> + * @param pkt Packet handle >>> * >>> - * @return 0 if successful >>> + * @retval 0 Packet is not valid >>> + * @retval 1 Packet is valid >>> + */ >>> +int odp_packet_is_valid(odp_packet_t pkt); >>> + >>> + >>> +/** >>> + * @} >>> */ >>> -int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src); >>> >>> #ifdef __cplusplus >>> } >>> diff --git a/platform/linux-dpdk/include/api/odp_packet_io.h >>> b/platform/linux-dpdk/include/api/odp_packet_io.h >>> index bd6868a..787df7d 100644 >>> --- a/platform/linux-dpdk/include/api/odp_packet_io.h >>> +++ b/platform/linux-dpdk/include/api/odp_packet_io.h >>> @@ -1,4 +1,4 @@ >>> -/* Copyright (c) 2014, Linaro Limited >>> +/* Copyright (c) 2013, Linaro Limited >>> * All rights reserved. >>> * >>> * SPDX-License-Identifier: BSD-3-Clause >>> @@ -19,29 +19,29 @@ extern "C" { >>> #endif >>> >>> #include <odp_std_types.h> >>> +#include <odp_platform_types.h> >>> #include <odp_buffer_pool.h> >>> #include <odp_packet.h> >>> #include <odp_queue.h> >>> >>> -#include <odp_pktio_types.h> >>> - >>> -/** ODP packet IO handle */ >>> -typedef uint32_t odp_pktio_t; >>> - >>> -/** Invalid packet IO handle */ >>> -#define ODP_PKTIO_INVALID 0 >>> +/** @defgroup odp_packet_io ODP PACKET IO >>> + * Operations on a packet. >>> + * @{ >>> + */ >>> >>> /** >>> * Open an ODP packet IO instance >>> * >>> * @param dev Packet IO device >>> * @param pool Pool to use for packet IO >>> - * @param params Set of parameters to pass to the arch dependent >>> implementation >>> * >>> * @return ODP packet IO handle or ODP_PKTIO_INVALID on error >>> + * >>> + * @note dev name loop is specially pktio reserved name for >>> + * device used for testing. Usually it's loop back >>> + * interface. >>> */ >>> -odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool, >>> - odp_pktio_params_t *params); >>> +odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool); >>> >>> /** >>> * Close an ODP packet IO instance >>> @@ -130,21 +130,77 @@ void odp_pktio_set_input(odp_packet_t pkt, >>> odp_pktio_t id); >>> odp_pktio_t odp_pktio_get_input(odp_packet_t pkt); >>> >>> /** >>> - * Defines the maximum length of mac address supported by this platform >>> + * Configure the MTU for a packet IO interface. >>> + * >>> + * @param[in] id ODP packet IO handle. >>> + * @param[in] mtu The value of MTU that the interface will be >>> configured to >>> + * use. >>> + * >>> + * @retval 0 on success. >>> + * @retval -1 if specified mtu can not be handled. >>> + * @retval -1 on any other error or illegal input parameters. >>> + */ >>> +int odp_pktio_set_mtu(odp_pktio_t id, int mtu); >>> + >>> +/** >>> + * Return the currently configured MTU value of a packet IO interface. >>> + * >>> + * @param[in] id ODP packet IO handle. >>> + * >>> + * @retval MTU value >0 on success. >>> + * @retval -1 on any error or not existance pktio id. >>> + */ >>> +int odp_pktio_mtu(odp_pktio_t id); >>> + >>> +/** >>> + * Enable/Disable promiscuous mode on a packet IO interface. >>> + * >>> + * @param[in] id ODP packet IO handle. >>> + * @param[in] enable 1 to enable, 0 to disable. >>> + * >>> + * @retval 0 on success. >>> + * @retval non-zero on any error. >>> + */ >>> +int odp_pktio_promisc_mode_set(odp_pktio_t id, odp_bool_t enable); >>> + >>> +/** >>> + * Determine if promiscuous mode is enabled for a packet IO interface. >>> + * >>> + * @param[in] id ODP packet IO handle. >>> + * >>> + * @retval 1 if promiscuous mode is enabled. >>> + * @retval 0 if promiscuous mode is disabled. >>> + * @retval -1 on any error. >>> +*/ >>> +int odp_pktio_promisc_mode(odp_pktio_t id); >>> + >>> +/** >>> + * Get the default MAC address of a packet IO interface. >>> + * >>> + * @param id ODP packet IO handle. >>> + * @param[out] mac_addr Storage for MAC address of the packet IO >>> interface. >>> + * @param addr_size Storage size for the address >>> + * >>> + * @retval Number of bytes written on success, 0 on failure. >>> */ >>> -#define ODP_MAC_ADDR_MAX_LENGTH ETH_ALEN >>> +size_t odp_pktio_mac_addr(odp_pktio_t id, void *mac_addr, >>> + size_t addr_size); >>> >>> /** >>> * Get mac address of the interface >>> * >>> - * @param id ODP packet IO handle >>> - * @param mac_addr Storage for Mac address of the packet IO >>> interface >>> - * Storage provided by the caller should be equal >>> - * to ODP_MAC_ADDR_MAX_LENGTH (filled by function) >>> + * @param id ODP packet IO handle >>> + * @param mac_addr Storage for Mac address of the packet IO >>> interface >>> + * Storage provided by the caller should be equal >>> + * to ODP_MAC_ADDR_MAX_LENGTH (filled by function) >>> * @return 0 on success or -1 on error >>> -**/ >>> + */ >>> int odp_pktio_get_mac_addr(odp_pktio_t id, unsigned char *mac_addr); >>> >>> +/** >>> + * @} >>> + */ >>> + >>> #ifdef __cplusplus >>> } >>> #endif >>> diff --git a/platform/linux-dpdk/include/api/odp_pktio_types.h >>> b/platform/linux-dpdk/include/api/odp_pktio_types.h >>> deleted file mode 100644 >>> index b23e6da..0000000 >>> --- a/platform/linux-dpdk/include/api/odp_pktio_types.h >>> +++ /dev/null >>> @@ -1,45 +0,0 @@ >>> - >>> -/* Copyright (c) 2013, Linaro Limited >>> - * All rights reserved. >>> - * >>> - * SPDX-License-Identifier: BSD-3-Clause >>> - */ >>> - >>> -#ifndef ODP_PKTIO_TYPES_H >>> -#define ODP_PKTIO_TYPES_H >>> - >>> -#ifdef __cplusplus >>> -extern "C" { >>> -#endif >>> - >>> -/* We should ensure that future enum values will never overlap, >>> otherwise >>> - * applications that want netmap suport might get in trouble if the odp >>> lib >>> - * was not built with netmap support and there are more types define >>> below >>> - */ >>> - >>> -typedef enum { >>> - ODP_PKTIO_TYPE_SOCKET_BASIC = 0x1, >>> - ODP_PKTIO_TYPE_SOCKET_MMSG, >>> - ODP_PKTIO_TYPE_SOCKET_MMAP, >>> - ODP_PKTIO_TYPE_NETMAP, >>> - ODP_PKTIO_TYPE_DPDK, >>> -} odp_pktio_type_t; >>> - >>> -#include <odp_pktio_socket.h> >>> -#ifdef ODP_HAVE_NETMAP >>> -#include <odp_pktio_netmap.h> >>> -#endif >>> - >>> -typedef union odp_pktio_params_t { >>> - odp_pktio_type_t type; >>> - socket_params_t sock_params; >>> -#ifdef ODP_HAVE_NETMAP >>> - netmap_params_t nm_params; >>> -#endif >>> -} odp_pktio_params_t; >>> - >>> -#ifdef __cplusplus >>> -} >>> -#endif >>> - >>> -#endif >>> diff --git a/platform/linux-dpdk/include/api/odp_platform_types.h >>> b/platform/linux-dpdk/include/api/odp_platform_types.h >>> new file mode 100644 >>> index 0000000..3bedb5d >>> --- /dev/null >>> +++ b/platform/linux-dpdk/include/api/odp_platform_types.h >>> @@ -0,0 +1,78 @@ >>> +/* Copyright (c) 2014, Linaro Limited >>> + * All rights reserved. >>> + * >>> + * SPDX-License-Identifier: BSD-3-Clause >>> + */ >>> + >>> + >>> +/** >>> + * @file >>> + * ODP implementation types >>> + * This file contains all of the implementation-defined types for ODP >>> abstract >>> + * definitions. Having this in one file means that other ODP API files >>> are >>> + * implementation-independent and avoids circular dependencies for >>> files that >>> + * refer to types managed by other components. Included here are >>> typedefs and >>> + * related typed constants that are referenced by other ODP API files. >>> + */ >>> + >>> +#ifndef ODP_IMPL_TYPES_H_ >>> +#define ODP_IMPL_TYPES_H_ >>> + >>> +/** @defgroup odp_platform_types ODP PLATFORM TYPES >>> + * Implementation specific definitions for ODP abstract types. >>> + * @{ >>> + */ >>> + >>> +/** ODP Buffer pool */ >>> +typedef unsigned long odp_buffer_pool_t; >>> + >>> +/** Invalid buffer pool */ >>> +#define ODP_BUFFER_POOL_INVALID (0xffffffff) >>> + >>> +/** ODP buffer */ >>> +typedef unsigned long odp_buffer_t; >>> + >>> +/** Invalid buffer */ >>> +#define ODP_BUFFER_INVALID (0xffffffff) >>> + >>> +/** ODP buffer segment */ >>> +typedef odp_buffer_t odp_buffer_seg_t; >>> + >>> +/** Invalid segment */ >>> +#define ODP_SEGMENT_INVALID ODP_BUFFER_INVALID >>> + >>> +/** ODP packet */ >>> +typedef odp_buffer_t odp_packet_t; >>> + >>> +/** Invalid packet */ >>> +#define ODP_PACKET_INVALID ODP_BUFFER_INVALID >>> + >>> +/** ODP packet segment */ >>> +typedef odp_buffer_t odp_packet_seg_t; >>> + >>> +/** Invalid packet segment */ >>> +#define ODP_PACKET_SEG_INVALID ODP_BUFFER_INVALID >>> + >>> +/** ODP packet IO handle */ >>> +typedef uint32_t odp_pktio_t; >>> + >>> +/** Invalid packet IO handle */ >>> +#define ODP_PKTIO_INVALID 0 >>> + >>> +/** odp_pktio_t value to indicate any port */ >>> +#define ODP_PKTIO_ANY ((odp_pktio_t)~0) >>> + >>> +/** >>> + * ODP shared memory block >>> + */ >>> +typedef uint32_t odp_shm_t; >>> + >>> +/** Invalid shared memory block */ >>> +#define ODP_SHM_INVALID 0 >>> +#define ODP_SHM_NULL ODP_SHM_INVALID /**< Synonym for buffer pool use */ >>> + >>> +/** >>> + * @} >>> + */ >>> + >>> +#endif >>> diff --git a/platform/linux-dpdk/include/odp_buffer_inlines.h >>> b/platform/linux-dpdk/include/odp_buffer_inlines.h >>> new file mode 100644 >>> index 0000000..ebf600b >>> --- /dev/null >>> +++ b/platform/linux-dpdk/include/odp_buffer_inlines.h >>> @@ -0,0 +1,179 @@ >>> +/* Copyright (c) 2014, Linaro Limited >>> + * All rights reserved. >>> + * >>> + * SPDX-License-Identifier: BSD-3-Clause >>> + */ >>> + >>> +/** >>> + * @file >>> + * >>> + * Inline functions for ODP buffer mgmt routines - implementation >>> internal >>> + */ >>> + >>> +#ifndef ODP_BUFFER_INLINES_H_ >>> +#define ODP_BUFFER_INLINES_H_ >>> + >>> +#ifdef __cplusplus >>> +extern "C" { >>> +#endif >>> + >>> +static inline odp_buffer_t odp_buffer_encode_handle(odp_buffer_hdr_t >>> *hdr) >>> +{ >>> + odp_buffer_bits_t handle; >>> + uint32_t pool_id = pool_handle_to_index(hdr->pool_hdl); >>> + struct pool_entry_s *pool = get_pool_entry(pool_id); >>> + >>> + handle.pool_id = pool_id; >>> + handle.index = ((uint8_t *)hdr - pool->pool_mdata_addr) / >>> + ODP_CACHE_LINE_SIZE; >>> + handle.seg = 0; >>> + >>> + return handle.u32; >>> +} >>> + >>> +static inline odp_buffer_t odp_hdr_to_buf(odp_buffer_hdr_t *hdr) >>> +{ >>> + return hdr->handle.handle; >>> +} >>> + >>> +static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf) >>> +{ >>> + return (odp_buffer_hdr_t *)buf; >>> +} >>> + >>> +static inline uint32_t odp_buffer_refcount(odp_buffer_hdr_t *buf) >>> +{ >>> + return odp_atomic_load_u32(&buf->ref_count); >>> +} >>> + >>> +static inline uint32_t odp_buffer_incr_refcount(odp_buffer_hdr_t *buf, >>> + uint32_t val) >>> +{ >>> + return odp_atomic_fetch_add_u32(&buf->ref_count, val) + val; >>> +} >>> + >>> +static inline uint32_t odp_buffer_decr_refcount(odp_buffer_hdr_t *buf, >>> + uint32_t val) >>> +{ >>> + uint32_t tmp; >>> + >>> + tmp = odp_atomic_fetch_sub_u32(&buf->ref_count, val); >>> + >>> + if (tmp < val) { >>> + odp_atomic_fetch_add_u32(&buf->ref_count, val - tmp); >>> + return 0; >>> + } else { >>> + return tmp - val; >>> + } >>> +} >>> + >>> +static inline odp_buffer_hdr_t *validate_buf(odp_buffer_t buf) >>> +{ >>> + odp_buffer_bits_t handle; >>> + odp_buffer_hdr_t *buf_hdr; >>> + handle.u32 = buf; >>> + >>> + /* For buffer handles, segment index must be 0 and pool id in >>> range */ >>> + if (handle.seg != 0 || handle.pool_id >= ODP_CONFIG_BUFFER_POOLS) >>> + return NULL; >>> + >>> + pool_entry_t *pool = odp_pool_to_entry(handle.pool_id); >>> + >>> + /* If pool not created, handle is invalid */ >>> + if (pool->s.pool_shm == ODP_SHM_INVALID) >>> + return NULL; >>> + >>> + uint32_t buf_stride = pool->s.buf_stride / ODP_CACHE_LINE_SIZE; >>> + >>> + /* A valid buffer index must be on stride, and must be in range >>> */ >>> + if ((handle.index % buf_stride != 0) || >>> + ((uint32_t)(handle.index / buf_stride) >= >>> pool->s.params.num_bufs)) >>> + return NULL; >>> + >>> + buf_hdr = (odp_buffer_hdr_t *)(void *) >>> + (pool->s.pool_mdata_addr + >>> + (handle.index * ODP_CACHE_LINE_SIZE)); >>> + >>> + /* Handle is valid, so buffer is valid if it is allocated */ >>> + return buf_hdr->allocator == ODP_FREEBUF ? NULL : buf_hdr; >>> +} >>> + >>> +int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf); >>> + >>> +static inline void *buffer_map(odp_buffer_hdr_t *buf, >>> + uint32_t offset, >>> + uint32_t *seglen, >>> + uint32_t limit) >>> +{ >>> + int seg_index = offset / buf->segsize; >>> + int seg_offset = offset % buf->segsize; >>> + >>> + if (seglen != NULL) { >>> + uint32_t buf_left = limit - offset; >>> + *seglen = buf_left < buf->segsize ? >>> + buf_left : buf->segsize - seg_offset; >>> + } >>> + >>> + return (void *)(seg_offset + (uint8_t *)buf->addr[seg_index]); >>> +} >>> + >>> +static inline odp_buffer_seg_t segment_next(odp_buffer_hdr_t *buf, >>> + odp_buffer_seg_t seg) >>> +{ >>> + odp_buffer_bits_t seghandle; >>> + seghandle.u32 = seg; >>> + >>> + if (seg == ODP_SEGMENT_INVALID || >>> + seghandle.prefix != buf->handle.prefix || >>> + seghandle.seg >= buf->segcount - 1) >>> + return ODP_SEGMENT_INVALID; >>> + else { >>> + seghandle.seg++; >>> + return (odp_buffer_seg_t)seghandle.u32; >>> + } >>> +} >>> + >>> +static inline void *segment_map(odp_buffer_hdr_t *buf, >>> + odp_buffer_seg_t seg, >>> + uint32_t *seglen, >>> + uint32_t limit, >>> + uint32_t hr) >>> +{ >>> + uint32_t seg_offset, buf_left; >>> + odp_buffer_bits_t seghandle; >>> + uint8_t *seg_addr; >>> + seghandle.u32 = seg; >>> + >>> + if (seghandle.prefix != buf->handle.prefix || >>> + seghandle.seg >= buf->segcount) >>> + return NULL; >>> + >>> + seg_addr = (uint8_t *)buf->addr[seghandle.seg]; >>> + seg_offset = seghandle.seg * buf->segsize; >>> + limit += hr; >>> + >>> + /* Can't map this segment if it's nothing but headroom or >>> tailroom */ >>> + if (hr >= seg_offset + buf->segsize || seg_offset > limit) >>> + return NULL; >>> + >>> + /* Adjust address & offset if this segment contains any headroom >>> */ >>> + if (hr > seg_offset) { >>> + seg_addr += hr % buf->segsize; >>> + seg_offset += hr % buf->segsize; >>> + } >>> + >>> + /* Set seglen if caller is asking for it */ >>> + if (seglen != NULL) { >>> + buf_left = limit - seg_offset; >>> + *seglen = buf_left < buf->segsize ? buf_left : >>> buf->segsize; >>> + } >>> + >>> + return (void *)seg_addr; >>> +} >>> + >>> + >>> +#ifdef __cplusplus >>> +} >>> +#endif >>> + >>> +#endif >>> diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h >>> b/platform/linux-dpdk/include/odp_buffer_internal.h >>> index 5406606..acc958b 100644 >>> --- a/platform/linux-dpdk/include/odp_buffer_internal.h >>> +++ b/platform/linux-dpdk/include/odp_buffer_internal.h >>> @@ -24,50 +24,144 @@ extern "C" { >>> #include <odp_buffer.h> >>> #include <odp_debug.h> >>> #include <odp_align.h> >>> -#include <rte_mbuf.h> >>> +#include <odp_align_internal.h> >>> +#include <odp_config.h> >>> +#include <odp_byteorder.h> >>> +#include <odp_thread.h> >>> >>> -/* TODO: move these to correct files */ >>> +/* DPDK */ >>> +#include <rte_mbuf.h> >>> >>> -typedef uint64_t odp_phys_addr_t; >>> +#define ODP_BITSIZE(x) \ >>> + ((x) <= 2 ? 1 : \ >>> + ((x) <= 4 ? 2 : \ >>> + ((x) <= 8 ? 3 : \ >>> + ((x) <= 16 ? 4 : \ >>> + ((x) <= 32 ? 5 : \ >>> + ((x) <= 64 ? 6 : \ >>> + ((x) <= 128 ? 7 : \ >>> + ((x) <= 256 ? 8 : \ >>> + ((x) <= 512 ? 9 : \ >>> + ((x) <= 1024 ? 10 : \ >>> + ((x) <= 2048 ? 11 : \ >>> + ((x) <= 4096 ? 12 : \ >>> + ((x) <= 8196 ? 13 : \ >>> + ((x) <= 16384 ? 14 : \ >>> + ((x) <= 32768 ? 15 : \ >>> + ((x) <= 65536 ? 16 : \ >>> + (0/0))))))))))))))))) >>> + >>> +_ODP_STATIC_ASSERT(ODP_CONFIG_PACKET_BUF_LEN_MIN >= 256, >>> + "ODP Segment size must be a minimum of 256 bytes"); >>> + >>> +_ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MIN % >>> ODP_CACHE_LINE_SIZE) == 0, >>> + "ODP Segment size must be a multiple of cache line >>> size"); >>> + >>> +_ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MAX % >>> + ODP_CONFIG_PACKET_BUF_LEN_MIN) == 0, >>> + "Packet max size must be a multiple of segment size"); >>> + >>> +#define ODP_BUFFER_MAX_SEG \ >>> + (ODP_CONFIG_PACKET_BUF_LEN_MAX / ODP_CONFIG_PACKET_BUF_LEN_MIN) >>> + >>> +/* We can optimize storage of small raw buffers within metadata area */ >>> +#define ODP_MAX_INLINE_BUF ((sizeof(void *)) * (ODP_BUFFER_MAX_SEG >>> - 1)) >>> + >>> +#define ODP_BUFFER_POOL_BITS ODP_BITSIZE(ODP_CONFIG_BUFFER_POOLS) >>> +#define ODP_BUFFER_SEG_BITS ODP_BITSIZE(ODP_BUFFER_MAX_SEG) >>> +#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS - >>> ODP_BUFFER_SEG_BITS) >>> +#define ODP_BUFFER_PREFIX_BITS (ODP_BUFFER_POOL_BITS + >>> ODP_BUFFER_INDEX_BITS) >>> +#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS) >>> +#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS) >>> >>> #define ODP_BUFFER_MAX_INDEX (ODP_BUFFER_MAX_BUFFERS - 2) >>> #define ODP_BUFFER_INVALID_INDEX (ODP_BUFFER_MAX_BUFFERS - 1) >>> >>> -#define ODP_BUFS_PER_CHUNK 16 >>> -#define ODP_BUFS_PER_SCATTER 4 >>> - >>> -#define ODP_BUFFER_TYPE_CHUNK 0xffff >>> - >>> - >>> -#define ODP_BUFFER_POOL_BITS 4 >>> -#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS) >>> -#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS) >>> -#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS) >>> - >>> typedef union odp_buffer_bits_t { >>> uint32_t u32; >>> odp_buffer_t handle; >>> >>> struct { >>> - uint32_t pool:ODP_BUFFER_POOL_BITS; >>> +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN >>> + uint32_t pool_id:ODP_BUFFER_POOL_BITS; >>> + uint32_t index:ODP_BUFFER_INDEX_BITS; >>> + uint32_t seg:ODP_BUFFER_SEG_BITS; >>> +#else >>> + uint32_t seg:ODP_BUFFER_SEG_BITS; >>> uint32_t index:ODP_BUFFER_INDEX_BITS; >>> + uint32_t pool_id:ODP_BUFFER_POOL_BITS; >>> +#endif >>> }; >>> -} odp_buffer_bits_t; >>> >>> + struct { >>> +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN >>> + uint32_t prefix:ODP_BUFFER_PREFIX_BITS; >>> + uint32_t pfxseg:ODP_BUFFER_SEG_BITS; >>> +#else >>> + uint32_t pfxseg:ODP_BUFFER_SEG_BITS; >>> + uint32_t prefix:ODP_BUFFER_PREFIX_BITS; >>> +#endif >>> + }; >>> +} odp_buffer_bits_t; >>> >>> /* forward declaration */ >>> struct odp_buffer_hdr_t; >>> >>> - >>> +/* Common buffer header */ >>> typedef struct odp_buffer_hdr_t { >>> + /* DPDK specific */ >>> struct rte_mbuf mb; /* Underlying DPDK rte_mbuf */ >>> - struct odp_buffer_hdr_t *next; /* Next buf in a list */ >>> - int type; /* ODP buffer type; not DPDK buf >>> type */ >>> uint32_t index; /* Index in the rte_mempool */ >>> + >>> + struct odp_buffer_hdr_t *next; /* next buf in a list */ >>> + int allocator; /* allocating thread id */ >>> + odp_buffer_bits_t handle; /* handle */ >>> + union { >>> + uint32_t all; >>> + struct { >>> + uint32_t zeroized:1; /* Zeroize buf data on free >>> */ >>> + uint32_t hdrdata:1; /* Data is in buffer hdr */ >>> + }; >>> + } flags; >>> + int type; /* buffer type */ >>> + size_t size; /* max data size */ >>> + odp_atomic_u32_t ref_count; /* reference count */ >>> + odp_buffer_pool_t pool_hdl; /* buffer pool handle */ >>> + union { >>> + uint64_t buf_u64; /* user u64 */ >>> + void *buf_ctx; /* user context */ >>> + const void *buf_cctx; /* const alias for ctx */ >>> + void *udata_addr; /* user metadata addr */ >>> + }; >>> + size_t udata_size; /* size of user metadata */ >>> + uint32_t segcount; /* segment count */ >>> + uint32_t segsize; /* segment size */ >>> + void *addr[ODP_BUFFER_MAX_SEG]; /* block >>> addrs */ >>> + >>> } odp_buffer_hdr_t; >>> >>> -int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf); >>> +typedef struct odp_buffer_hdr_stride { >>> + uint8_t >>> pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_buffer_hdr_t))]; >>> +} odp_buffer_hdr_stride; >>> +/* Ensure next header starts from 8 byte align */ >>> +_ODP_STATIC_ASSERT((sizeof(odp_buffer_hdr_t) % 8) == 0, >>> + "ODP_BUFFER_HDR_T__SIZE_ERROR"); >>> + >>> +typedef struct odp_buf_blk_t { >>> + struct odp_buf_blk_t *next; >>> + struct odp_buf_blk_t *prev; >>> +} odp_buf_blk_t; >>> + >>> +/* Raw buffer header */ >>> +typedef struct { >>> + odp_buffer_hdr_t buf_hdr; /* common buffer header */ >>> +} odp_raw_buffer_hdr_t; >>> + >>> +/* Free buffer marker */ >>> +#define ODP_FREEBUF -1 >>> >>> +/* Forward declarations */ >>> +odp_buffer_t buffer_alloc(odp_buffer_pool_t pool, size_t size); >>> >>> #ifdef __cplusplus >>> } >>> diff --git a/platform/linux-dpdk/include/odp_buffer_pool_internal.h >>> b/platform/linux-dpdk/include/odp_buffer_pool_internal.h >>> index 1a36655..0f19f87 100644 >>> --- a/platform/linux-dpdk/include/odp_buffer_pool_internal.h >>> +++ b/platform/linux-dpdk/include/odp_buffer_pool_internal.h >>> @@ -19,16 +19,48 @@ extern "C" { >>> #endif >>> >>> #include <odp_std_types.h> >>> +#include <odp_align.h> >>> +#include <odp_align_internal.h> >>> #include <odp_buffer_pool.h> >>> #include <odp_buffer_internal.h> >>> -#include <odp_align.h> >>> #include <odp_hints.h> >>> #include <odp_config.h> >>> #include <odp_debug.h> >>> +#include <odp_shared_memory.h> >>> +#include <odp_atomic.h> >>> +#include <odp_atomic_internal.h> >>> +#include <string.h> >>> >>> -/* for DPDK */ >>> +/* DPDK */ >>> #include <rte_mempool.h> >>> >>> +/** >>> + * Buffer initialization routine prototype >>> + * >>> + * @note Routines of this type MAY be passed as part of the >>> + * _odp_buffer_pool_init_t structure to be called whenever a >>> + * buffer is allocated to initialize the user metadata >>> + * associated with that buffer. >>> + */ >>> +typedef void (_odp_buf_init_t)(odp_buffer_t buf, void *buf_init_arg); >>> + >>> +/** >>> + * Buffer pool initialization parameters >>> + * Used to communicate buffer pool initialization options. Internal for >>> now. >>> + */ >>> +typedef struct _odp_buffer_pool_init_t { >>> + size_t udata_size; /**< Size of user metadata for each >>> buffer */ >>> + _odp_buf_init_t *buf_init; /**< Buffer initialization routine to >>> use */ >>> + void *buf_init_arg; /**< Argument to be passed to >>> buf_init() */ >>> +} _odp_buffer_pool_init_t; /**< Type of buffer initialization >>> struct */ >>> + >>> +/* Local cache for buffer alloc/free acceleration */ >>> +typedef struct local_cache_t { >>> + odp_buffer_hdr_t *buf_freelist; /* The local cache */ >>> + uint64_t bufallocs; /* Local buffer alloc count */ >>> + uint64_t buffrees; /* Local buffer free count */ >>> +} local_cache_t; >>> + >>> /* Use ticketlock instead of spinlock */ >>> #define POOL_USE_TICKETLOCK >>> >>> @@ -38,11 +70,16 @@ extern "C" { >>> >>> #ifdef POOL_USE_TICKETLOCK >>> #include <odp_ticketlock.h> >>> +#define POOL_LOCK(a) odp_ticketlock_lock(a) >>> +#define POOL_UNLOCK(a) odp_ticketlock_unlock(a) >>> +#define POOL_LOCK_INIT(a) odp_ticketlock_init(a) >>> #else >>> #include <odp_spinlock.h> >>> +#define POOL_LOCK(a) odp_spinlock_lock(a) >>> +#define POOL_UNLOCK(a) odp_spinlock_unlock(a) >>> +#define POOL_LOCK_INIT(a) odp_spinlock_init(a) >>> #endif >>> >>> - >>> struct pool_entry_s { >>> #ifdef POOL_USE_TICKETLOCK >>> odp_ticketlock_t lock ODP_ALIGNED_CACHE; >>> @@ -50,38 +87,294 @@ struct pool_entry_s { >>> odp_spinlock_t lock ODP_ALIGNED_CACHE; >>> #endif >>> >>> - uint64_t free_bufs; >>> char name[ODP_BUFFER_POOL_NAME_LEN]; >>> - >>> - >>> - odp_buffer_pool_t pool ODP_ALIGNED_CACHE; >>> - uintptr_t buf_base; >>> - size_t buf_size; >>> - size_t buf_offset; >>> - uint64_t num_bufs; >>> - void *pool_base_addr; >>> - uint64_t pool_size; >>> - size_t payload_size; >>> - size_t payload_align; >>> - int buf_type; >>> - size_t hdr_size; >>> + odp_buffer_pool_param_t params; >>> + _odp_buffer_pool_init_t init_params; >>> + odp_buffer_pool_t pool_hdl; >>> + uint32_t pool_id; >>> + odp_shm_t pool_shm; >>> + union { >>> + uint32_t all; >>> + struct { >>> + uint32_t has_name:1; >>> + uint32_t user_supplied_shm:1; >>> + uint32_t unsegmented:1; >>> + uint32_t zeroized:1; >>> + uint32_t predefined:1; >>> + }; >>> + } flags; >>> + uint32_t quiesced; >>> + uint32_t low_wm_assert; >>> + uint8_t *pool_base_addr; >>> + uint8_t *pool_mdata_addr; >>> + size_t pool_size; >>> + uint32_t buf_align; >>> + uint32_t buf_stride; >>> + _odp_atomic_ptr_t buf_freelist; >>> + _odp_atomic_ptr_t blk_freelist; >>> + odp_atomic_u32_t bufcount; >>> + odp_atomic_u32_t blkcount; >>> + odp_atomic_u64_t bufallocs; >>> + odp_atomic_u64_t buffrees; >>> + odp_atomic_u64_t blkallocs; >>> + odp_atomic_u64_t blkfrees; >>> + odp_atomic_u64_t bufempty; >>> + odp_atomic_u64_t blkempty; >>> + odp_atomic_u64_t high_wm_count; >>> + odp_atomic_u64_t low_wm_count; >>> + uint32_t seg_size; >>> + uint32_t high_wm; >>> + uint32_t low_wm; >>> + uint32_t headroom; >>> + uint32_t tailroom; >>> }; >>> >>> +typedef union pool_entry_u { >>> + struct pool_entry_s s; >>> + >>> + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct >>> pool_entry_s))]; >>> +} pool_entry_t; >>> >>> extern void *pool_entry_ptr[]; >>> >>> +#if defined(ODP_CONFIG_SECURE_POOLS) && (ODP_CONFIG_SECURE_POOLS == 1) >>> +#define buffer_is_secure(buf) (buf->flags.zeroized) >>> +#define pool_is_secure(pool) (pool->flags.zeroized) >>> +#else >>> +#define buffer_is_secure(buf) 0 >>> +#define pool_is_secure(pool) 0 >>> +#endif >>> + >>> +#define TAG_ALIGN ((size_t)16) >>> + >>> +#define odp_cs(ptr, old, new) \ >>> + _odp_atomic_ptr_cmp_xchg_strong(&ptr, (void **)&old, (void >>> *)new, \ >>> + _ODP_MEMMODEL_SC, \ >>> + _ODP_MEMMODEL_SC) >>> >>> -static inline void *get_pool_entry(odp_buffer_pool_t pool_id) >>> +/* Helper functions for pointer tagging to avoid ABA race conditions */ >>> +#define odp_tag(ptr) \ >>> + (((size_t)ptr) & (TAG_ALIGN - 1)) >>> + >>> +#define odp_detag(ptr) \ >>> + ((void *)(((size_t)ptr) & -TAG_ALIGN)) >>> + >>> +#define odp_retag(ptr, tag) \ >>> + ((void *)(((size_t)ptr) | odp_tag(tag))) >>> + >>> + >>> +static inline void *get_blk(struct pool_entry_s *pool) >>> +{ >>> + void *oldhead, *myhead, *newhead; >>> + >>> + oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, >>> _ODP_MEMMODEL_ACQ); >>> + >>> + do { >>> + size_t tag = odp_tag(oldhead); >>> + myhead = odp_detag(oldhead); >>> + if (odp_unlikely(myhead == NULL)) >>> + break; >>> + newhead = odp_retag(((odp_buf_blk_t *)myhead)->next, tag >>> + 1); >>> + } while (odp_cs(pool->blk_freelist, oldhead, newhead) == 0); >>> + >>> + if (odp_unlikely(myhead == NULL)) >>> + odp_atomic_inc_u64(&pool->blkempty); >>> + else >>> + odp_atomic_dec_u32(&pool->blkcount); >>> + >>> + return (void *)myhead; >>> +} >>> + >>> +static inline void ret_blk(struct pool_entry_s *pool, void *block) >>> +{ >>> + void *oldhead, *myhead, *myblock; >>> + >>> + oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, >>> _ODP_MEMMODEL_ACQ); >>> + >>> + do { >>> + size_t tag = odp_tag(oldhead); >>> + myhead = odp_detag(oldhead); >>> + ((odp_buf_blk_t *)block)->next = myhead; >>> + myblock = odp_retag(block, tag + 1); >>> + } while (odp_cs(pool->blk_freelist, oldhead, myblock) == 0); >>> + >>> + odp_atomic_inc_u32(&pool->blkcount); >>> + odp_atomic_inc_u64(&pool->blkfrees); >>> +} >>> + >>> +static inline odp_buffer_hdr_t *get_buf(struct pool_entry_s *pool) >>> +{ >>> + odp_buffer_hdr_t *oldhead, *myhead, *newhead; >>> + >>> + oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, >>> _ODP_MEMMODEL_ACQ); >>> + >>> + do { >>> + size_t tag = odp_tag(oldhead); >>> + myhead = odp_detag(oldhead); >>> + if (odp_unlikely(myhead == NULL)) >>> + break; >>> + newhead = odp_retag(myhead->next, tag + 1); >>> + } while (odp_cs(pool->buf_freelist, oldhead, newhead) == 0); >>> + >>> + if (odp_unlikely(myhead == NULL)) { >>> + odp_atomic_inc_u64(&pool->bufempty); >>> + } else { >>> + uint64_t bufcount = >>> + odp_atomic_fetch_sub_u32(&pool->bufcount, 1) - 1; >>> + >>> + /* Check for low watermark condition */ >>> + if (bufcount == pool->low_wm && !pool->low_wm_assert) { >>> + pool->low_wm_assert = 1; >>> + odp_atomic_inc_u64(&pool->low_wm_count); >>> + } >>> + >>> + odp_atomic_inc_u64(&pool->bufallocs); >>> + myhead->next = myhead; /* Mark buffer allocated */ >>> + myhead->allocator = odp_thread_id(); >>> + } >>> + >>> + return (void *)myhead; >>> +} >>> + >>> +static inline void ret_buf(struct pool_entry_s *pool, odp_buffer_hdr_t >>> *buf) >>> +{ >>> + odp_buffer_hdr_t *oldhead, *myhead, *mybuf; >>> + >>> + buf->allocator = ODP_FREEBUF; /* Mark buffer free */ >>> + >>> + if (!buf->flags.hdrdata && buf->type != ODP_BUFFER_TYPE_RAW) { >>> + while (buf->segcount > 0) { >>> + if (buffer_is_secure(buf) || >>> pool_is_secure(pool)) >>> + memset(buf->addr[buf->segcount - 1], >>> + 0, buf->segsize); >>> + ret_blk(pool, buf->addr[--buf->segcount]); >>> + } >>> + buf->size = 0; >>> + } >>> + >>> + oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, >>> _ODP_MEMMODEL_ACQ); >>> + >>> + do { >>> + size_t tag = odp_tag(oldhead); >>> + myhead = odp_detag(oldhead); >>> + buf->next = myhead; >>> + mybuf = odp_retag(buf, tag + 1); >>> + } while (odp_cs(pool->buf_freelist, oldhead, mybuf) == 0); >>> + >>> + uint64_t bufcount = odp_atomic_fetch_add_u32(&pool->bufcount, 1) >>> + 1; >>> + >>> + /* Check if low watermark condition should be deasserted */ >>> + if (bufcount == pool->high_wm && pool->low_wm_assert) { >>> + pool->low_wm_assert = 0; >>> + odp_atomic_inc_u64(&pool->high_wm_count); >>> + } >>> + >>> + odp_atomic_inc_u64(&pool->buffrees); >>> +} >>> + >>> +static inline void *get_local_buf(local_cache_t *buf_cache, >>> + struct pool_entry_s *pool, >>> + size_t totsize) >>> +{ >>> + odp_buffer_hdr_t *buf = buf_cache->buf_freelist; >>> + >>> + if (odp_likely(buf != NULL)) { >>> + buf_cache->buf_freelist = buf->next; >>> + >>> + if (odp_unlikely(buf->size < totsize)) { >>> + intmax_t needed = totsize - buf->size; >>> + >>> + do { >>> + void *blk = get_blk(pool); >>> + if (odp_unlikely(blk == NULL)) { >>> + ret_buf(pool, buf); >>> + buf_cache->buffrees--; >>> + return NULL; >>> + } >>> + buf->addr[buf->segcount++] = blk; >>> + needed -= pool->seg_size; >>> + } while (needed > 0); >>> + >>> + buf->size = buf->segcount * pool->seg_size; >>> + } >>> + >>> + buf_cache->bufallocs++; >>> + buf->allocator = odp_thread_id(); /* Mark buffer >>> allocated */ >>> + } >>> + >>> + return buf; >>> +} >>> + >>> +static inline void ret_local_buf(local_cache_t *buf_cache, >>> + odp_buffer_hdr_t *buf) >>> +{ >>> + buf->allocator = ODP_FREEBUF; >>> + buf->next = buf_cache->buf_freelist; >>> + buf_cache->buf_freelist = buf; >>> + >>> + buf_cache->buffrees++; >>> +} >>> + >>> +static inline void flush_cache(local_cache_t *buf_cache, >>> + struct pool_entry_s *pool) >>> +{ >>> + odp_buffer_hdr_t *buf = buf_cache->buf_freelist; >>> + uint32_t flush_count = 0; >>> + >>> + while (buf != NULL) { >>> + odp_buffer_hdr_t *next = buf->next; >>> + ret_buf(pool, buf); >>> + buf = next; >>> + flush_count++; >>> + } >>> + >>> + odp_atomic_add_u64(&pool->bufallocs, buf_cache->bufallocs); >>> + odp_atomic_add_u64(&pool->buffrees, buf_cache->buffrees - >>> flush_count); >>> + >>> + buf_cache->buf_freelist = NULL; >>> + buf_cache->bufallocs = 0; >>> + buf_cache->buffrees = 0; >>> +} >>> + >>> +static inline odp_buffer_pool_t pool_index_to_handle(uint32_t pool_id) >>> +{ >>> + return pool_id; >>> +} >>> + >>> +static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl) >>> +{ >>> + return pool_hdl; >>> +} >>> + >>> +static inline void *get_pool_entry(uint32_t pool_id) >>> { >>> return pool_entry_ptr[pool_id]; >>> } >>> >>> +static inline pool_entry_t *odp_pool_to_entry(odp_buffer_pool_t pool) >>> +{ >>> + return (pool_entry_t >>> *)get_pool_entry(pool_handle_to_index(pool)); >>> +} >>> + >>> +static inline pool_entry_t *odp_buf_to_pool(odp_buffer_hdr_t *buf) >>> +{ >>> + return odp_pool_to_entry(buf->pool_hdl); >>> +} >>> >>> -static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf) >>> +static inline uint32_t odp_buffer_pool_segment_size(odp_buffer_pool_t >>> pool) >>> { >>> - return (odp_buffer_hdr_t *)buf; >>> + return odp_pool_to_entry(pool)->s.seg_size; >>> } >>> >>> +static inline uint32_t odp_buffer_pool_headroom(odp_buffer_pool_t pool) >>> +{ >>> + return odp_pool_to_entry(pool)->s.headroom; >>> +} >>> + >>> +static inline uint32_t odp_buffer_pool_tailroom(odp_buffer_pool_t pool) >>> +{ >>> + return odp_pool_to_entry(pool)->s.tailroom; >>> +} >>> >>> #ifdef __cplusplus >>> } >>> diff --git a/platform/linux-dpdk/include/odp_packet_dpdk.h >>> b/platform/linux-dpdk/include/odp_packet_dpdk.h >>> index c3a2b70..c982d5e 100644 >>> --- a/platform/linux-dpdk/include/odp_packet_dpdk.h >>> +++ b/platform/linux-dpdk/include/odp_packet_dpdk.h >>> @@ -8,10 +8,8 @@ >>> #define ODP_PACKET_DPDK_H >>> >>> #include <stdint.h> >>> -#include <net/if.h> >>> >>> #include <odph_eth.h> >>> -#include <odph_packet.h> >>> #include <odp_align.h> >>> #include <odp_debug.h> >>> #include <odp_packet.h> >>> diff --git a/platform/linux-dpdk/include/odp_packet_internal.h >>> b/platform/linux-dpdk/include/odp_packet_internal.h >>> index 1cd44b1..24de33f 100644 >>> --- a/platform/linux-dpdk/include/odp_packet_internal.h >>> +++ b/platform/linux-dpdk/include/odp_packet_internal.h >>> @@ -22,6 +22,7 @@ extern "C" { >>> #include <odp_debug.h> >>> #include <odp_buffer_internal.h> >>> #include <odp_buffer_pool_internal.h> >>> +#include <odp_buffer_inlines.h> >>> #include <odp_packet.h> >>> #include <odp_packet_io.h> >>> >>> @@ -43,6 +44,7 @@ typedef union { >>> uint32_t vlan:1; /**< VLAN hdr found */ >>> uint32_t vlan_qinq:1; /**< Stacked VLAN found, QinQ */ >>> >>> + uint32_t snap:1; /**< SNAP */ >>> uint32_t arp:1; /**< ARP */ >>> >>> uint32_t ipv4:1; /**< IPv4 */ >>> @@ -53,13 +55,14 @@ typedef union { >>> >>> uint32_t udp:1; /**< UDP */ >>> uint32_t tcp:1; /**< TCP */ >>> + uint32_t tcpopt:1; /**< TCP options present */ >>> uint32_t sctp:1; /**< SCTP */ >>> uint32_t icmp:1; /**< ICMP */ >>> }; >>> } input_flags_t; >>> >>> -ODP_STATIC_ASSERT(sizeof(input_flags_t) == sizeof(uint32_t), >>> - "INPUT_FLAGS_SIZE_ERROR"); >>> +_ODP_STATIC_ASSERT(sizeof(input_flags_t) == sizeof(uint32_t), >>> + "INPUT_FLAGS_SIZE_ERROR"); >>> >>> /** >>> * Packet error flags >>> @@ -70,7 +73,9 @@ typedef union { >>> >>> struct { >>> /* Bitfield flags for each detected error */ >>> + uint32_t app_error:1; /**< Error bit for application use >>> */ >>> uint32_t frame_len:1; /**< Frame length error */ >>> + uint32_t snap_len:1; /**< Snap length error */ >>> uint32_t l2_chksum:1; /**< L2 checksum error, checks TBD >>> */ >>> uint32_t ip_err:1; /**< IP error, checks TBD */ >>> uint32_t tcp_err:1; /**< TCP error, checks TBD */ >>> @@ -78,8 +83,8 @@ typedef union { >>> }; >>> } error_flags_t; >>> >>> -ODP_STATIC_ASSERT(sizeof(error_flags_t) == sizeof(uint32_t), >>> - "ERROR_FLAGS_SIZE_ERROR"); >>> +_ODP_STATIC_ASSERT(sizeof(error_flags_t) == sizeof(uint32_t), >>> + "ERROR_FLAGS_SIZE_ERROR"); >>> >>> /** >>> * Packet output flags >>> @@ -90,12 +95,15 @@ typedef union { >>> >>> struct { >>> /* Bitfield flags for each output option */ >>> - uint32_t l4_chksum:1; /**< Request L4 checksum >>> calculation */ >>> + uint32_t l3_chksum_set:1; /**< L3 chksum bit is valid */ >>> + uint32_t l3_chksum:1; /**< L3 chksum override */ >>> + uint32_t l4_chksum_set:1; /**< L3 chksum bit is valid */ >>> + uint32_t l4_chksum:1; /**< L4 chksum override */ >>> }; >>> } output_flags_t; >>> >>> -ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), >>> - "OUTPUT_FLAGS_SIZE_ERROR"); >>> +_ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), >>> + "OUTPUT_FLAGS_SIZE_ERROR"); >>> >>> /** >>> * Internal Packet header >>> @@ -108,28 +116,131 @@ typedef struct { >>> error_flags_t error_flags; >>> output_flags_t output_flags; >>> >>> - uint32_t frame_offset; /**< offset to start of frame, even on >>> error */ >>> uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */ >>> uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */ >>> uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also >>> ICMP) */ >>> + uint32_t payload_offset; /**< offset to payload */ >>> >>> - uint64_t user_ctx; /**< user context */ >>> + uint32_t vlan_s_tag; /**< Parsed 1st VLAN header (S-TAG) */ >>> + uint32_t vlan_c_tag; /**< Parsed 2nd VLAN header (C-TAG) */ >>> + uint32_t l3_protocol; /**< Parsed L3 protocol */ >>> + uint32_t l3_len; /**< Layer 3 length */ >>> + uint32_t l4_protocol; /**< Parsed L4 protocol */ >>> + uint32_t l4_len; /**< Layer 4 length */ >>> + >>> + uint32_t frame_len; >>> + uint32_t headroom; >>> + uint32_t tailroom; >>> >>> odp_pktio_t input; >>> } odp_packet_hdr_t; >>> >>> +typedef struct odp_packet_hdr_stride { >>> + uint8_t >>> pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_packet_hdr_t))]; >>> +} odp_packet_hdr_stride; >>> + >>> +_ODP_STATIC_ASSERT(sizeof(odp_packet_hdr_t) % sizeof(uint64_t) == 0, >>> + "ODP_PACKET_HDR_T__SIZE_ERR2"); >>> + >>> /** >>> * Return the packet header >>> */ >>> static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) >>> { >>> - return (odp_packet_hdr_t *)pkt; >>> + return (odp_packet_hdr_t *)odp_buf_to_hdr((odp_buffer_t)pkt); >>> } >>> >>> /** >>> - * Parse packet and set internal metadata >>> + * Initialize packet buffer >>> */ >>> -void odp_packet_parse(odp_packet_t pkt, size_t len, size_t l2_offset); >>> +static inline void packet_init(pool_entry_t *pool, >>> + odp_packet_hdr_t *pkt_hdr, >>> + size_t size) >>> +{ >>> + /* >>> + * Reset parser metadata. Note that we clear via memset to make >>> + * this routine indepenent of any additional adds to packet >>> metadata. >>> + */ >>> + struct rte_mbuf *mb; >>> + uint8_t *start; >>> + >>> + mb = (struct rte_mbuf *)&pkt_hdr->buf_hdr; >>> + mb->buf_len = size; >>> + start = (uint8_t *)mb->buf_addr; >>> + memset(start, 0, mb->buf_len); >>> + >>> + /* >>> + * Packet headroom is set from the pool's headroom >>> + * Packet tailroom is rounded up to fill the last >>> + * segment occupied by the allocated length. >>> + */ >>> + pkt_hdr->frame_len = size; >>> + pkt_hdr->headroom = pool->s.headroom; >>> + pkt_hdr->tailroom = >>> + (pool->s.seg_size * pkt_hdr->buf_hdr.segcount) - >>> + (pool->s.headroom + size); >>> +} >>> + >>> +static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, >>> + uint32_t offset, uint32_t *seglen) >>> +{ >>> + struct rte_mbuf *mbuf = (struct rte_mbuf *)&pkt_hdr->buf_hdr; >>> + if (offset > pkt_hdr->frame_len) >>> + return NULL; >>> + >>> + if (seglen != NULL) { >>> + *seglen = mbuf->pkt.data_len; >>> + pkt_hdr->frame_len = mbuf->pkt.data_len; >>> + } >>> + return (void *)((uint8_t *)mbuf->pkt.data + offset); >>> +} >>> + >>> +#define pull_offset(x, len) (x = x < len ? 0 : x - len) >>> + >>> +static inline void push_head(odp_packet_hdr_t *pkt_hdr, size_t len) >>> +{ >>> + pkt_hdr->headroom -= len; >>> + pkt_hdr->frame_len += len; >>> + pkt_hdr->l2_offset += len; >>> + pkt_hdr->l3_offset += len; >>> + pkt_hdr->l4_offset += len; >>> +} >>> + >>> +static inline void pull_head(odp_packet_hdr_t *pkt_hdr, size_t len) >>> +{ >>> + pkt_hdr->headroom += len; >>> + pkt_hdr->frame_len -= len; >>> + pull_offset(pkt_hdr->l2_offset, len); >>> + pull_offset(pkt_hdr->l3_offset, len); >>> + pull_offset(pkt_hdr->l4_offset, len); >>> +} >>> + >>> +static inline void push_tail(odp_packet_hdr_t *pkt_hdr, size_t len) >>> +{ >>> + pkt_hdr->tailroom -= len; >>> + pkt_hdr->frame_len += len; >>> +} >>> + >>> + >>> +static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, size_t len) >>> +{ >>> + pkt_hdr->tailroom += len; >>> + pkt_hdr->frame_len -= len; >>> +} >>> + >>> +static inline void packet_set_len(odp_packet_t pkt, uint32_t len) >>> +{ >>> + odp_packet_hdr(pkt)->frame_len = len; >>> +} >>> + >>> +/* Forward declarations */ >>> +int _odp_packet_copy_to_packet(odp_packet_t srcpkt, uint32_t srcoffset, >>> + odp_packet_t dstpkt, uint32_t dstoffset, >>> + uint32_t len); >>> + >>> +odp_packet_t _odp_packet_alloc(odp_buffer_pool_t pool_hdl); >>> + >>> +int _odp_packet_parse(odp_packet_t pkt); >>> >>> #ifdef __cplusplus >>> } >>> diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h >>> b/platform/linux-dpdk/include/odp_packet_io_internal.h >>> index 9263349..5ece843 100644 >>> --- a/platform/linux-dpdk/include/odp_packet_io_internal.h >>> +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h >>> @@ -20,17 +20,36 @@ extern "C" { >>> >>> #include <odp_spinlock.h> >>> #include <odp_packet_socket.h> >>> -#ifdef ODP_HAVE_NETMAP >>> -#include <odp_packet_netmap.h> >>> -#endif >>> +#include <odp_classification_datamodel.h> >>> +#include <odp_align_internal.h> >>> + >>> +#include <odp_config.h> >>> +#include <odp_hints.h> >>> +#include <linux/if.h> >>> + >>> +/* DPDK */ >>> #include <odp_packet_dpdk.h> >>> >>> +/** >>> + * Packet IO types >>> + */ >>> +typedef enum { >>> + ODP_PKTIO_TYPE_SOCKET_BASIC = 0x1, >>> + ODP_PKTIO_TYPE_SOCKET_MMSG, >>> + ODP_PKTIO_TYPE_SOCKET_MMAP, >>> +} odp_pktio_type_t; >>> + >>> struct pktio_entry { >>> odp_spinlock_t lock; /**< entry spinlock */ >>> int taken; /**< is entry taken(1) or >>> free(0) */ >>> odp_queue_t inq_default; /**< default input queue, if set >>> */ >>> odp_queue_t outq_default; /**< default out queue */ >>> - odp_pktio_params_t params; /**< pktio parameters */ >>> + odp_pktio_type_t type; /**< pktio type */ >>> + pkt_sock_t pkt_sock; /**< using socket API for IO */ >>> + pkt_sock_mmap_t pkt_sock_mmap; /**< using socket mmap API for >>> IO */ >>> + classifier_t cls; /**< classifier linked with this >>> pktio*/ >>> + char name[IFNAMSIZ]; /**< name of pktio provided to >>> + pktio_open() */ >>> pkt_dpdk_t pkt_dpdk; /**< using DPDK API for IO */ >>> }; >>> >>> @@ -39,6 +58,21 @@ typedef union { >>> uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct >>> pktio_entry))]; >>> } pktio_entry_t; >>> >>> +typedef struct { >>> + pktio_entry_t entries[ODP_CONFIG_PKTIO_ENTRIES]; >>> +} pktio_table_t; >>> + >>> +extern void *pktio_entry_ptr[]; >>> + >>> + >>> +static inline pktio_entry_t *get_pktio_entry(odp_pktio_t id) >>> +{ >>> + if (odp_unlikely(id == ODP_PKTIO_INVALID || >>> + id > ODP_CONFIG_PKTIO_ENTRIES)) >>> + return NULL; >>> + >>> + return pktio_entry_ptr[id - 1]; >>> +} >>> #ifdef __cplusplus >>> } >>> #endif >>> diff --git a/platform/linux-dpdk/odp_buffer.c >>> b/platform/linux-dpdk/odp_buffer.c >>> index 4914ca2..d89f2af 100644 >>> --- a/platform/linux-dpdk/odp_buffer.c >>> +++ b/platform/linux-dpdk/odp_buffer.c >>> @@ -5,8 +5,10 @@ >>> */ >>> >>> #include <odp_buffer.h> >>> -#include <odp_buffer_internal.h> >>> #include <odp_buffer_pool_internal.h> >>> +#include <odp_buffer_internal.h> >>> +#include <odp_buffer_inlines.h> >>> +#include <odp_debug_internal.h> >>> >>> #include <string.h> >>> #include <stdio.h> >>> @@ -44,13 +46,13 @@ int odp_buffer_is_valid(odp_buffer_t buf) >>> } >>> >>> >>> -int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf) >>> +int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf) >>> { >>> odp_buffer_hdr_t *hdr; >>> int len = 0; >>> >>> if (!odp_buffer_is_valid(buf)) { >>> - printf("Buffer is not valid.\n"); >>> + ODP_PRINT("Buffer is not valid.\n"); >>> return len; >>> } >>> >>> @@ -67,7 +69,9 @@ int odp_buffer_snprint(char *str, size_t n, >>> odp_buffer_t buf) >>> len += snprintf(&str[len], n-len, >>> " size %u\n", hdr->mb.buf_len); >>> len += snprintf(&str[len], n-len, >>> - " ref_count %i\n", hdr->mb.refcnt); >>> + " ref_count %i\n", >>> + odp_atomic_load_u32((odp_atomic_u32_t *) >>> + &hdr->mb.refcnt)); >>> len += snprintf(&str[len], n-len, >>> " dpdk type %i\n", hdr->mb.type); >>> len += snprintf(&str[len], n-len, >>> @@ -86,5 +90,5 @@ void odp_buffer_print(odp_buffer_t buf) >>> len = odp_buffer_snprint(str, max_len-1, buf); >>> str[len] = 0; >>> >>> - printf("\n%s\n", str); >>> + ODP_PRINT("\n%s\n", str); >>> } >>> diff --git a/platform/linux-dpdk/odp_buffer_pool.c >>> b/platform/linux-dpdk/odp_buffer_pool.c >>> index ca25ace..6636b8d 100644 >>> --- a/platform/linux-dpdk/odp_buffer_pool.c >>> +++ b/platform/linux-dpdk/odp_buffer_pool.c >>> @@ -6,16 +6,19 @@ >>> >>> #include <odp_std_types.h> >>> #include <odp_buffer_pool.h> >>> -#include <odp_buffer_pool_internal.h> >>> #include <odp_buffer_internal.h> >>> +#include <odp_buffer_pool_internal.h> >>> +#include <odp_buffer_inlines.h> >>> #include <odp_packet_internal.h> >>> #include <odp_timer_internal.h> >>> +#include <odp_align_internal.h> >>> #include <odp_shared_memory.h> >>> #include <odp_align.h> >>> #include <odp_internal.h> >>> #include <odp_config.h> >>> #include <odp_hints.h> >>> -#include <odp_debug.h> >>> +#include <odp_debug_internal.h> >>> +#include <odp_atomic_internal.h> >>> >>> #include <string.h> >>> #include <stdlib.h> >>> @@ -26,44 +29,27 @@ >>> #define MBUF_SIZE (2048 + sizeof(struct rte_mbuf) + >>> RTE_PKTMBUF_HEADROOM) >>> #define NB_MBUF 32768 >>> >>> -#ifdef POOL_USE_TICKETLOCK >>> -#include <odp_ticketlock.h> >>> -#define LOCK(a) odp_ticketlock_lock(a) >>> -#define UNLOCK(a) odp_ticketlock_unlock(a) >>> -#define LOCK_INIT(a) odp_ticketlock_init(a) >>> -#else >>> -#include <odp_spinlock.h> >>> -#define LOCK(a) odp_spinlock_lock(a) >>> -#define UNLOCK(a) odp_spinlock_unlock(a) >>> -#define LOCK_INIT(a) odp_spinlock_init(a) >>> -#endif >>> - >>> - >>> #if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >>> #error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS >>> #endif >>> >>> -#define NULL_INDEX ((uint32_t)-1) >>> - >>> -union buffer_type_any_u { >>> +typedef union buffer_type_any_u { >>> odp_buffer_hdr_t buf; >>> odp_packet_hdr_t pkt; >>> odp_timeout_hdr_t tmo; >>> -}; >>> - >>> -typedef union buffer_type_any_u odp_any_buffer_hdr_t; >>> +} odp_anybuf_t; >>> >>> -typedef union pool_entry_u { >>> - struct pool_entry_s s; >>> - >>> - uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct >>> pool_entry_s))]; >>> - >>> -} pool_entry_t; >>> +/* Any buffer type header */ >>> +typedef struct { >>> + odp_anybuf_t any_hdr; /* any buffer type */ >>> +} odp_any_buffer_hdr_t; >>> >>> +typedef struct odp_any_hdr_stride { >>> + uint8_t >>> pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_any_buffer_hdr_t))]; >>> +} odp_any_hdr_stride; >>> >>> typedef struct pool_table_t { >>> pool_entry_t pool[ODP_CONFIG_BUFFER_POOLS]; >>> - >>> } pool_table_t; >>> >>> >>> @@ -73,6 +59,8 @@ static pool_table_t *pool_tbl; >>> /* Pool entry pointers (for inlining) */ >>> void *pool_entry_ptr[ODP_CONFIG_BUFFER_POOLS]; >>> >>> +/* Local cache for buffer alloc/free acceleration */ >>> +static __thread local_cache_t local_cache[ODP_CONFIG_BUFFER_POOLS]; >>> >>> int odp_buffer_pool_init_global(void) >>> { >>> @@ -94,9 +82,9 @@ int odp_buffer_pool_init_global(void) >>> for (i = 0; i < ODP_CONFIG_BUFFER_POOLS; i++) { >>> /* init locks */ >>> pool_entry_t *pool = &pool_tbl->pool[i]; >>> - LOCK_INIT(&pool->s.lock); >>> - pool->s.pool = i; >>> - >>> + POOL_LOCK_INIT(&pool->s.lock); >>> + pool->s.pool_hdl = pool_index_to_handle(i); >>> + pool->s.pool_id = i; >>> pool_entry_ptr[i] = pool; >>> } >>> >>> @@ -187,10 +175,13 @@ odp_dpdk_mbuf_ctor(struct rte_mempool *mp, >>> buf_hdr->index = i; >>> } >>> >>> +/** >>> + * Buffer pool creation >>> + */ >>> + >>> odp_buffer_pool_t odp_buffer_pool_create(const char *name, >>> - void *base_addr, uint64_t size, >>> - size_t buf_size, size_t >>> buf_align, >>> - int buf_type) >>> + odp_shm_t shm, >>> + odp_buffer_pool_param_t *params) >>> { >>> struct rte_mempool *pool = NULL; >>> struct mbuf_pool_ctor_arg mbp_ctor_arg; >>> @@ -198,47 +189,39 @@ odp_buffer_pool_t odp_buffer_pool_create(const >>> char *name, >>> unsigned mb_size; >>> size_t hdr_size; >>> >>> - /* Not used for rte_mempool; the new ODP buffer management >>> introduces >>> - * rte_mempool_create_from_region where base_addr makes sense */ >>> - (void)base_addr; >>> - >>> - /* buf_align will be removed soon, no need to wory about it */ >>> - (void)buf_align; >>> - >>> - ODP_DBG("odp_buffer_pool_create: %s, %lx, %u, %u, %u, %d\n", >>> name, >>> - (uint64_t) base_addr, (unsigned) size, >>> - (unsigned) buf_size, (unsigned) buf_align, >>> - buf_type); >>> + ODP_DBG("odp_buffer_pool_create: %s, %lx, %u, %u, %d\n", name, >>> + (uint64_t) shm, (unsigned) params->buf_size, >>> + (unsigned) params->buf_align, params->buf_type); >>> >>> - switch (buf_type) { >>> + switch (params->buf_type) { >>> case ODP_BUFFER_TYPE_RAW: >>> hdr_size = sizeof(odp_buffer_hdr_t); >>> - mbp_ctor_arg.seg_buf_size = (uint16_t) buf_size; >>> + mbp_ctor_arg.seg_buf_size = (uint16_t) params->buf_size; >>> break; >>> case ODP_BUFFER_TYPE_PACKET: >>> hdr_size = sizeof(odp_packet_hdr_t); >>> mbp_ctor_arg.seg_buf_size = >>> - (uint16_t) (RTE_PKTMBUF_HEADROOM + buf_size); >>> + (uint16_t) (RTE_PKTMBUF_HEADROOM + >>> params->buf_size); >>> break; >>> case ODP_BUFFER_TYPE_TIMEOUT: >>> hdr_size = sizeof(odp_timeout_hdr_t); >>> - mbp_ctor_arg.seg_buf_size = (uint16_t) buf_size; >>> + mbp_ctor_arg.seg_buf_size = (uint16_t) params->buf_size; >>> break; >>> case ODP_BUFFER_TYPE_ANY: >>> hdr_size = sizeof(odp_any_buffer_hdr_t); >>> mbp_ctor_arg.seg_buf_size = >>> - (uint16_t) (RTE_PKTMBUF_HEADROOM + buf_size); >>> + (uint16_t) (RTE_PKTMBUF_HEADROOM + >>> params->buf_size); >>> break; >>> default: >>> - ODP_ERR("odp_buffer_pool_create: Bad type %i\n", >>> buf_type); >>> + ODP_ERR("odp_buffer_pool_create: Bad type %i\n", >>> + params->buf_type); >>> return ODP_BUFFER_POOL_INVALID; >>> - break; >>> } >>> >>> mb_ctor_arg.seg_buf_offset = >>> (uint16_t) ODP_CACHE_LINE_SIZE_ROUNDUP(hdr_size); >>> mb_ctor_arg.seg_buf_size = mbp_ctor_arg.seg_buf_size; >>> - mb_ctor_arg.buf_type = buf_type; >>> + mb_ctor_arg.buf_type = params->buf_type; >>> mb_size = mb_ctor_arg.seg_buf_offset + mb_ctor_arg.seg_buf_size; >>> >>> pool = rte_mempool_create(name, NB_MBUF, >>> @@ -267,6 +250,61 @@ odp_buffer_pool_t odp_buffer_pool_lookup(const char >>> *name) >>> return (odp_buffer_pool_t)mp; >>> } >>> >>> +odp_buffer_t buffer_alloc(odp_buffer_pool_t pool_hdl, size_t size) >>> +{ >>> + uint32_t pool_id = pool_handle_to_index(pool_hdl); >>> + pool_entry_t *pool = get_pool_entry(pool_id); >>> + uintmax_t totsize = pool->s.headroom + size + pool->s.tailroom; >>> + odp_anybuf_t *buf; >>> + >>> + /* Reject oversized allocation requests */ >>> + if ((pool->s.flags.unsegmented && totsize > pool->s.seg_size) || >>> + (!pool->s.flags.unsegmented && >>> + totsize > ODP_CONFIG_PACKET_BUF_LEN_MAX)) >>> + return ODP_BUFFER_INVALID; >>> + >>> + /* Try to satisfy request from the local cache */ >>> + buf = (odp_anybuf_t *)(void >>> *)get_local_buf(&local_cache[pool_id], >>> + &pool->s, totsize); >>> + >>> + /* If cache is empty, satisfy request from the pool */ >>> + if (odp_unlikely(buf == NULL)) { >>> + buf = (odp_anybuf_t *)(void *)get_buf(&pool->s); >>> + >>> + if (odp_unlikely(buf == NULL)) >>> + return ODP_BUFFER_INVALID; >>> + >>> + /* Get blocks for this buffer, if pool uses application >>> data */ >>> + if (buf->buf.size < totsize) { >>> + intmax_t needed = totsize - buf->buf.size; >>> + do { >>> + uint8_t *blk = get_blk(&pool->s); >>> + if (blk == NULL) { >>> + ret_buf(&pool->s, &buf->buf); >>> + return ODP_BUFFER_INVALID; >>> + } >>> + buf->buf.addr[buf->buf.segcount++] = blk; >>> + needed -= pool->s.seg_size; >>> + } while (needed > 0); >>> + buf->buf.size = buf->buf.segcount * >>> pool->s.seg_size; >>> + } >>> + } >>> + >>> + /* By default, buffers inherit their pool's zeroization setting >>> */ >>> + buf->buf.flags.zeroized = pool->s.flags.zeroized; >>> + >>> + if (buf->buf.type == ODP_BUFFER_TYPE_PACKET) { >>> + packet_init(pool, &buf->pkt, size); >>> + >>> + if (pool->s.init_params.buf_init != NULL) >>> + (*pool->s.init_params.buf_init) >>> + (buf->buf.handle.handle, >>> + pool->s.init_params.buf_init_arg); >>> + } >>> + >>> + return odp_hdr_to_buf(&buf->buf); >>> +} >>> + >>> >>> odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_id) >>> { >>> @@ -279,6 +317,15 @@ void odp_buffer_free(odp_buffer_t buf) >>> rte_pktmbuf_free((struct rte_mbuf *)buf); >>> } >>> >>> +void _odp_flush_caches(void) >>> +{ >>> + int i; >>> + >>> + for (i = 0; i < ODP_CONFIG_BUFFER_POOLS; i++) { >>> + pool_entry_t *pool = get_pool_entry(i); >>> + flush_cache(&local_cache[i], &pool->s); >>> + } >>> +} >>> >>> void odp_buffer_pool_print(odp_buffer_pool_t pool_id) >>> { >>> diff --git a/platform/linux-dpdk/odp_init.c >>> b/platform/linux-dpdk/odp_init.c >>> index fa10022..7c8763d 100644 >>> --- a/platform/linux-dpdk/odp_init.c >>> +++ b/platform/linux-dpdk/odp_init.c >>> @@ -7,6 +7,8 @@ >>> #include <odp_init.h> >>> #include <odp_internal.h> >>> #include <odp_debug.h> >>> +#include <odp_debug_internal.h> >>> +#include <odp_system_info.h> >>> #include <odp_packet_dpdk.h> >>> >>> int odp_init_dpdk(void) >>> @@ -45,10 +47,9 @@ int odp_init_dpdk(void) >>> return 0; >>> } >>> >>> -int odp_init_global(void) >>> +int odp_init_global(odp_init_t *params ODP_UNUSED, >>> + odp_platform_init_t *platform_params ODP_UNUSED) >>> { >>> - odp_thread_init_global(); >>> - >>> odp_system_info_init(); >>> >>> if (odp_init_dpdk()) { >>> @@ -61,6 +62,11 @@ int odp_init_global(void) >>> return -1; >>> } >>> >>> + if (odp_thread_init_global()) { >>> + ODP_ERR("ODP thread init failed.\n"); >>> + return -1; >>> + } >>> + >>> if (odp_buffer_pool_init_global()) { >>> ODP_ERR("ODP buffer pool init failed.\n"); >>> return -1; >>> @@ -91,13 +97,26 @@ int odp_init_global(void) >>> return -1; >>> } >>> >>> + if (odp_classification_init_global()) { >>> + ODP_ERR("ODP classification init failed.\n"); >>> + return -1; >>> + } >>> + >>> return 0; >>> } >>> >>> +int odp_term_global(void) >>> +{ >>> + ODP_UNIMPLEMENTED(); >>> + return 0; >>> +} >>> >>> -int odp_init_local(int thr_id) >>> +int odp_init_local(void) >>> { >>> - odp_thread_init_local(thr_id); >>> + if (odp_thread_init_local()) { >>> + ODP_ERR("ODP thread local init failed.\n"); >>> + return -1; >>> + } >>> >>> if (odp_pktio_init_local()) { >>> ODP_ERR("ODP packet io local init failed.\n"); >>> @@ -111,3 +130,9 @@ int odp_init_local(int thr_id) >>> >>> return 0; >>> } >>> + >>> +int odp_term_local(void) >>> +{ >>> + ODP_UNIMPLEMENTED(); >>> + return 0; >>> +} >>> diff --git a/platform/linux-dpdk/odp_linux.c >>> b/platform/linux-dpdk/odp_linux.c >>> index 96c91a5..2b68c13 100644 >>> --- a/platform/linux-dpdk/odp_linux.c >>> +++ b/platform/linux-dpdk/odp_linux.c >>> @@ -8,6 +8,9 @@ >>> #define _GNU_SOURCE >>> #endif >>> #include <sched.h> >>> +#include <unistd.h> >>> +#include <sys/types.h> >>> +#include <sys/wait.h> >>> >>> #include <stdlib.h> >>> #include <string.h> >>> @@ -19,28 +22,30 @@ >>> #include <odp_thread.h> >>> #include <odp_init.h> >>> #include <odp_system_info.h> >>> +#include <odp_debug_internal.h> >>> >>> #include <rte_lcore.h> >>> >>> typedef struct { >>> - int thr_id; >>> void *(*start_routine) (void *); >>> void *arg; >>> - >>> } odp_start_args_t; >>> >>> - >>> static void *odp_run_start_routine(void *arg) >>> { >>> odp_start_args_t *start_args = arg; >>> >>> /* ODP thread local init */ >>> - odp_init_local(start_args->thr_id); >>> + if (odp_init_local()) { >>> + ODP_ERR("Local init failed\n"); >>> + return NULL; >>> + } >>> >>> - return start_args->start_routine(start_args->arg); >>> + void *ret = start_args->start_routine(start_args->arg); >>> + _odp_flush_caches(); >>> + return ret; >>> } >>> >>> - >>> void odph_linux_pthread_create(odph_linux_pthread_t *thread_tbl, int >>> num, >>> int first_core, void *(*start_routine) (void *), void >>> *arg) >>> { >>> @@ -62,12 +67,12 @@ void odph_linux_pthread_create(odph_linux_pthread_t >>> *thread_tbl, int num, >>> cpu = (first_core + i) % core_count; >>> >>> start_args = malloc(sizeof(odp_start_args_t)); >>> + if (start_args == NULL) >>> + ODP_ABORT("Malloc failed"); >>> memset(start_args, 0, sizeof(odp_start_args_t)); >>> start_args->start_routine = start_routine; >>> start_args->arg = arg; >>> >>> - odp_thread_create(cpu); >>> - start_args->thr_id = cpu; >>> /* If not master core */ >>> if (cpu != 0) { >>> rte_eal_remote_launch( >>> @@ -94,3 +99,99 @@ void odph_linux_pthread_join(odph_linux_pthread_t >>> *thread_tbl, int num) >>> return; >>> } >>> } >>> + >>> +int odph_linux_process_fork_n(odph_linux_process_t *proc_tbl, >>> + int num, int first_core) >>> +{ >>> + cpu_set_t cpu_set; >>> + pid_t pid; >>> + int core_count; >>> + int cpu; >>> + int i; >>> + >>> + memset(proc_tbl, 0, num*sizeof(odph_linux_process_t)); >>> + >>> + core_count = odp_sys_core_count(); >>> + >>> + if (first_core < 0 || first_core >= core_count) { >>> + ODP_ERR("Bad first_core\n"); >>> + return -1; >>> + } >>> + >>> + if (num < 0 || num > core_count) { >>> + ODP_ERR("Bad num\n"); >>> + return -1; >>> + } >>> + >>> + for (i = 0; i < num; i++) { >>> + cpu = (first_core + i) % core_count; >>> + pid = fork(); >>> + >>> + if (pid < 0) { >>> + ODP_ERR("fork() failed\n"); >>> + return -1; >>> + } >>> + >>> + /* Parent continues to fork */ >>> + if (pid > 0) { >>> + proc_tbl[i].pid = pid; >>> + proc_tbl[i].core = cpu; >>> + continue; >>> + } >>> + >>> + /* Child process */ >>> + CPU_ZERO(&cpu_set); >>> + CPU_SET(cpu, &cpu_set); >>> + >>> + if (sched_setaffinity(0, sizeof(cpu_set_t), &cpu_set)) { >>> + ODP_ERR("sched_setaffinity() failed\n"); >>> + return -2; >>> + } >>> + >>> + if (odp_init_local()) { >>> + ODP_ERR("Local init failed\n"); >>> + return -2; >>> + } >>> + >>> + return 0; >>> + } >>> + >>> + return 1; >>> +} >>> + >>> + >>> +int odph_linux_process_fork(odph_linux_process_t *proc, int core) >>> +{ >>> + return odph_linux_process_fork_n(proc, 1, core); >>> +} >>> + >>> + >>> +int odph_linux_process_wait_n(odph_linux_process_t *proc_tbl, int num) >>> +{ >>> + pid_t pid; >>> + int i, j; >>> + int status; >>> + >>> + for (i = 0; i < num; i++) { >>> + pid = wait(&status); >>> + >>> + if (pid < 0) { >>> + ODP_ERR("wait() failed\n"); >>> + return -1; >>> + } >>> + >>> + for (j = 0; j < num; j++) { >>> + if (proc_tbl[j].pid == pid) { >>> + proc_tbl[j].status = status; >>> + break; >>> + } >>> + } >>> + >>> + if (j == num) { >>> + ODP_ERR("Bad pid\n"); >>> + return -1; >>> + } >>> + } >>> + >>> + return 0; >>> +} >>> diff --git a/platform/linux-dpdk/odp_packet.c >>> b/platform/linux-dpdk/odp_packet.c >>> index beb69b2..754ce41 100644 >>> --- a/platform/linux-dpdk/odp_packet.c >>> +++ b/platform/linux-dpdk/odp_packet.c >>> @@ -6,34 +6,63 @@ >>> >>> #include <odp_packet.h> >>> #include <odp_packet_internal.h> >>> +#include <odp_debug_internal.h> >>> #include <odp_hints.h> >>> #include <odp_byteorder.h> >>> >>> #include <odph_eth.h> >>> #include <odph_ip.h> >>> +#include <odph_tcp.h> >>> +#include <odph_udp.h> >>> >>> #include <string.h> >>> #include <stdio.h> >>> >>> -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >>> - odph_ipv4hdr_t *ipv4, size_t >>> *offset_out); >>> -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >>> - odph_ipv6hdr_t *ipv6, size_t >>> *offset_out); >>> +/* >>> + * >>> + * Alloc and free >>> + * ******************************************************** >>> + * >>> + */ >>> >>> -void odp_packet_init(odp_packet_t pkt) >>> +odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool_hdl, uint32_t len) >>> { >>> - odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >>> - struct rte_mbuf *mb; >>> - void *start; >>> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >>> + >>> + if (pool->s.params.buf_type != ODP_BUFFER_TYPE_PACKET) >>> + return ODP_PACKET_INVALID; >>> + >>> + /* Handle special case for zero-length packets */ >>> + if (len == 0) { >>> + odp_packet_t pkt = >>> + (odp_packet_t)buffer_alloc(pool_hdl, >>> + >>> pool->s.params.buf_size); >>> + if (pkt != ODP_PACKET_INVALID) >>> + pull_tail(odp_packet_hdr(pkt), >>> + pool->s.params.buf_size); >>> + >>> + return pkt; >>> + } >>> + >>> + return (odp_packet_t)buffer_alloc(pool_hdl, len); >>> +} >>> >>> - mb = &pkt_hdr->buf_hdr.mb; >>> +void odp_packet_free(odp_packet_t pkt) >>> +{ >>> + odp_buffer_free((odp_buffer_t)pkt); >>> +} >>> + >>> +int odp_packet_reset(odp_packet_t pkt, uint32_t len) >>> +{ >>> + odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >>> + pool_entry_t *pool = odp_buf_to_pool(&pkt_hdr->buf_hdr); >>> + uint32_t totsize = pool->s.headroom + len + pool->s.tailroom; >>> >>> - start = mb->buf_addr; >>> - memset(start, 0, mb->buf_len); >>> + if (totsize > pkt_hdr->buf_hdr.size) >>> + return -1; >>> >>> - pkt_hdr->l2_offset = (uint32_t) ODP_PACKET_OFFSET_INVALID; >>> - pkt_hdr->l3_offset = (uint32_t) ODP_PACKET_OFFSET_INVALID; >>> - pkt_hdr->l4_offset = (uint32_t) ODP_PACKET_OFFSET_INVALID; >>> + packet_init(pool, pkt_hdr, len); >>> + return 0; >>> } >>> >>> odp_packet_t odp_packet_from_buffer(odp_buffer_t buf) >>> @@ -41,305 +70,478 @@ odp_packet_t odp_packet_from_buffer(odp_buffer_t >>> buf) >>> return (odp_packet_t)buf; >>> } >>> >>> -odp_buffer_t odp_buffer_from_packet(odp_packet_t pkt) >>> +odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt) >>> { >>> return (odp_buffer_t)pkt; >>> } >>> >>> -/* Advance the pkt data pointer and set len in one call */ >>> -static int odp_packet_set_offset_len(odp_packet_t pkt, size_t >>> frame_offset, >>> - size_t len) >>> +/* >>> + * >>> + * Pointers and lengths >>> + * ******************************************************** >>> + * >>> + */ >>> + >>> +void *odp_packet_head(odp_packet_t pkt) >>> { >>> - struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); >>> - uint16_t offset; >>> - uint16_t data_len; >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + return buffer_map(&pkt_hdr->buf_hdr, 0, NULL, 0); >>> +} >>> >>> - /* The pkt buf may have been pulled back into the headroom >>> - * so we cannot rely on finding the data right after the >>> - * ODP header and HEADROOM */ >>> - offset = (uint16_t)((unsigned long)mb->pkt.data - >>> - (unsigned long)mb->buf_addr); >>> - ODP_ASSERT(mb->buf_len >= offset, "Corrupted mbuf"); >>> - data_len = mb->buf_len - offset; >>> +uint32_t odp_packet_buf_len(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->buf_hdr.size; >>> +} >>> >>> - if (data_len < frame_offset) { >>> - ODP_ERR("Frame offset too big"); >>> - return -1; >>> - } >>> - mb->pkt.data = (void *)((char *)mb->pkt.data + frame_offset); >>> - data_len -= frame_offset; >>> +void *odp_packet_data(odp_packet_t pkt) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + return packet_map(pkt_hdr, 0, NULL); >>> +} >>> >>> - if (data_len < len) { >>> - ODP_ERR("Packet len too big"); >>> - return -1; >>> - } >>> - mb->pkt.pkt_len = len; >>> - mb->pkt.data_len = len; >>> +uint32_t odp_packet_seg_len(odp_packet_t pkt) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + uint32_t seglen; >>> >>> - return 0; >>> + /* Call returns length of 1st data segment */ >>> + packet_map(pkt_hdr, 0, &seglen); >>> + return seglen; >>> } >>> >>> -void odp_packet_set_len(odp_packet_t pkt, size_t len) >>> +uint32_t odp_packet_len(odp_packet_t pkt) >>> { >>> - (void)odp_packet_set_offset_len(pkt, 0, len); >>> + return odp_packet_hdr(pkt)->frame_len; >>> } >>> >>> -size_t odp_packet_get_len(odp_packet_t pkt) >>> +uint32_t odp_packet_headroom(odp_packet_t pkt) >>> { >>> - struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); >>> - return mb->pkt.pkt_len; >>> + return odp_packet_hdr(pkt)->headroom; >>> } >>> >>> -uint8_t *odp_packet_buf_addr(odp_packet_t pkt) >>> +uint32_t odp_packet_tailroom(odp_packet_t pkt) >>> { >>> - return odp_buffer_addr(odp_buffer_from_packet(pkt)); >>> + return odp_packet_hdr(pkt)->tailroom; >>> } >>> >>> -uint8_t *odp_packet_start(odp_packet_t pkt) >>> +void *odp_packet_tail(odp_packet_t pkt) >>> { >>> - struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); >>> - return mb->pkt.data; >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL); >>> } >>> >>> - >>> -uint8_t *odp_packet_l2(odp_packet_t pkt) >>> +void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) >>> { >>> - const size_t offset = odp_packet_l2_offset(pkt); >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> >>> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >>> + if (len > pkt_hdr->headroom) >>> return NULL; >>> >>> - return odp_packet_start(pkt) + offset; >>> + push_head(pkt_hdr, len); >>> + return packet_map(pkt_hdr, 0, NULL); >>> } >>> >>> -size_t odp_packet_l2_offset(odp_packet_t pkt) >>> +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) >>> { >>> - return odp_packet_hdr(pkt)->l2_offset; >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + if (len > pkt_hdr->frame_len) >>> + return NULL; >>> + >>> + pull_head(pkt_hdr, len); >>> + return packet_map(pkt_hdr, 0, NULL); >>> } >>> >>> -void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset) >>> +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) >>> { >>> - odp_packet_hdr(pkt)->l2_offset = offset; >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + struct rte_mbuf *mbuf = (struct rte_mbuf *)&pkt_hdr->buf_hdr; >>> + uint32_t origin = pkt_hdr->frame_len; >>> + >>> + mbuf->pkt.pkt_len += len; >>> + mbuf->pkt.data_len += len; >>> + >>> + if (len > pkt_hdr->tailroom) >>> + return NULL; >>> + >>> + push_tail(pkt_hdr, len); >>> + return packet_map(pkt_hdr, origin, NULL); >>> } >>> >>> -uint8_t *odp_packet_l3(odp_packet_t pkt) >>> +void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) >>> { >>> - const size_t offset = odp_packet_l3_offset(pkt); >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> >>> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >>> + if (len > pkt_hdr->frame_len) >>> return NULL; >>> >>> - return odp_packet_start(pkt) + offset; >>> + pull_tail(pkt_hdr, len); >>> + return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL); >>> } >>> >>> -size_t odp_packet_l3_offset(odp_packet_t pkt) >>> +void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t >>> *len, >>> + odp_packet_seg_t *seg) >>> { >>> - return odp_packet_hdr(pkt)->l3_offset; >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + void *addr = packet_map(pkt_hdr, offset, len); >>> + >>> + if (addr != NULL && seg != NULL) { >>> + odp_buffer_bits_t seghandle; >>> + seghandle.u32 = (uint32_t)pkt; >>> + seghandle.seg = (pkt_hdr->headroom + offset) / >>> + pkt_hdr->buf_hdr.segsize; >>> + *seg = seghandle.handle; >>> + } >>> + >>> + return addr; >>> } >>> >>> -void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset) >>> +/* >>> + * >>> + * Meta-data >>> + * ******************************************************** >>> + * >>> + */ >>> + >>> +odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt) >>> { >>> - odp_packet_hdr(pkt)->l3_offset = offset; >>> + return odp_packet_hdr(pkt)->buf_hdr.pool_hdl; >>> } >>> >>> -uint8_t *odp_packet_l4(odp_packet_t pkt) >>> +odp_pktio_t odp_packet_input(odp_packet_t pkt) >>> { >>> - const size_t offset = odp_packet_l4_offset(pkt); >>> + return odp_packet_hdr(pkt)->input; >>> +} >>> >>> - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) >>> - return NULL; >>> +void *odp_packet_user_ptr(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->buf_hdr.buf_ctx; >>> +} >>> + >>> +void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ctx) >>> +{ >>> + odp_packet_hdr(pkt)->buf_hdr.buf_cctx = ctx; >>> +} >>> + >>> +uint64_t odp_packet_user_u64(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->buf_hdr.buf_u64; >>> +} >>> >>> - return odp_packet_start(pkt) + offset; >>> +void odp_packet_user_u64_set(odp_packet_t pkt, uint64_t ctx) >>> +{ >>> + odp_packet_hdr(pkt)->buf_hdr.buf_u64 = ctx; >>> +} >>> + >>> +void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + return packet_map(pkt_hdr, pkt_hdr->l2_offset, len); >>> +} >>> + >>> +uint32_t odp_packet_l2_offset(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->l2_offset; >>> } >>> >>> -size_t odp_packet_l4_offset(odp_packet_t pkt) >>> +int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + if (offset >= pkt_hdr->frame_len) >>> + return -1; >>> + >>> + pkt_hdr->l2_offset = offset; >>> + return 0; >>> +} >>> + >>> +void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + return packet_map(pkt_hdr, pkt_hdr->l3_offset, len); >>> +} >>> + >>> +uint32_t odp_packet_l3_offset(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->l3_offset; >>> +} >>> + >>> +int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + if (offset >= pkt_hdr->frame_len) >>> + return -1; >>> + >>> + pkt_hdr->l3_offset = offset; >>> + return 0; >>> +} >>> + >>> +void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + return packet_map(pkt_hdr, pkt_hdr->l4_offset, len); >>> +} >>> + >>> +uint32_t odp_packet_l4_offset(odp_packet_t pkt) >>> { >>> return odp_packet_hdr(pkt)->l4_offset; >>> } >>> >>> -void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset) >>> +int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset) >>> { >>> - odp_packet_hdr(pkt)->l4_offset = offset; >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + if (offset >= pkt_hdr->frame_len) >>> + return -1; >>> + >>> + pkt_hdr->l4_offset = offset; >>> + return 0; >>> } >>> >>> -/** >>> - * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP >>> +int odp_packet_is_segmented(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->buf_hdr.segcount > 1; >>> +} >>> + >>> +int odp_packet_num_segs(odp_packet_t pkt) >>> +{ >>> + return odp_packet_hdr(pkt)->buf_hdr.segcount; >>> +} >>> + >>> +odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) >>> +{ >>> + return (odp_packet_seg_t)pkt; >>> +} >>> + >>> +odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + odp_buffer_bits_t seghandle; >>> + >>> + seghandle.u32 = (uint32_t)pkt; >>> + seghandle.seg = pkt_hdr->buf_hdr.segcount - 1; >>> + return seghandle.handle; >>> +} >>> + >>> +odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t >>> seg) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + return segment_next(&pkt_hdr->buf_hdr, seg); >>> +} >>> + >>> +/* >>> * >>> - * Internal function: caller is resposible for passing only valid >>> packet handles >>> - * , lengths and offsets (usually done&called in packet input). >>> + * Segment level >>> + * ******************************************************** >>> * >>> - * @param pkt Packet handle >>> - * @param len Packet length in bytes >>> - * @param frame_offset Byte offset to L2 header >>> */ >>> -void odp_packet_parse(odp_packet_t pkt, size_t len, size_t frame_offset) >>> + >>> +void *odp_packet_seg_buf_addr(odp_packet_t pkt, odp_packet_seg_t seg) >>> { >>> - odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >>> - odph_ethhdr_t *eth; >>> - odph_vlanhdr_t *vlan; >>> - odph_ipv4hdr_t *ipv4; >>> - odph_ipv6hdr_t *ipv6; >>> - uint16_t ethtype; >>> - size_t offset = 0; >>> - uint8_t ip_proto = 0; >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> >>> - /* The frame_offset is not relevant for frames from DPDK */ >>> - pkt_hdr->input_flags.eth = 1; >>> - (void) frame_offset; >>> - pkt_hdr->frame_offset = 0; >>> - if (odp_packet_set_offset_len(pkt, 0, len)) { >>> - return; >>> - } >>> + return segment_map(&pkt_hdr->buf_hdr, seg, NULL, >>> + pkt_hdr->headroom + pkt_hdr->frame_len, 0); >>> +} >>> >>> - if (odp_unlikely(len < ODPH_ETH_LEN_MIN)) { >>> - pkt_hdr->error_flags.frame_len = 1; >>> - return; >>> - } else if (len > ODPH_ETH_LEN_MAX) { >>> - pkt_hdr->input_flags.jumbo = 1; >>> - } >>> +uint32_t odp_packet_seg_buf_len(odp_packet_t pkt, >>> + odp_packet_seg_t seg ODP_UNUSED) >>> +{ >>> + return odp_packet_hdr(pkt)->buf_hdr.segsize; >>> +} >>> >>> - /* Assume valid L2 header, no CRC/FCS check in SW */ >>> - pkt_hdr->input_flags.l2 = 1; >>> - pkt_hdr->l2_offset = 0; >>> +void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> >>> - eth = (odph_ethhdr_t *)odp_packet_start(pkt); >>> - ethtype = odp_be_to_cpu_16(eth->type); >>> - vlan = (odph_vlanhdr_t *)ð->type; >>> + return segment_map(&pkt_hdr->buf_hdr, seg, NULL, >>> + pkt_hdr->frame_len, pkt_hdr->headroom); >>> +} >>> >>> - if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { >>> - pkt_hdr->input_flags.vlan_qinq = 1; >>> - ethtype = odp_be_to_cpu_16(vlan->tpid); >>> - offset += sizeof(odph_vlanhdr_t); >>> - vlan = &vlan[1]; >>> - } >>> +uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + uint32_t seglen = 0; >>> >>> - if (ethtype == ODPH_ETHTYPE_VLAN) { >>> - pkt_hdr->input_flags.vlan = 1; >>> - ethtype = odp_be_to_cpu_16(vlan->tpid); >>> - offset += sizeof(odph_vlanhdr_t); >>> - } >>> + segment_map(&pkt_hdr->buf_hdr, seg, &seglen, >>> + pkt_hdr->frame_len, pkt_hdr->headroom); >>> >>> - /* Set l3_offset+flag only for known ethtypes */ >>> - switch (ethtype) { >>> - case ODPH_ETHTYPE_IPV4: >>> - pkt_hdr->input_flags.ipv4 = 1; >>> - pkt_hdr->input_flags.l3 = 1; >>> - pkt_hdr->l3_offset = ODPH_ETHHDR_LEN + offset; >>> - ipv4 = (odph_ipv4hdr_t *)odp_packet_l3(pkt); >>> - ip_proto = parse_ipv4(pkt_hdr, ipv4, &offset); >>> - break; >>> - case ODPH_ETHTYPE_IPV6: >>> - pkt_hdr->input_flags.ipv6 = 1; >>> - pkt_hdr->input_flags.l3 = 1; >>> - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + >>> offset; >>> - ipv6 = (odph_ipv6hdr_t *)odp_packet_l3(pkt); >>> - ip_proto = parse_ipv6(pkt_hdr, ipv6, &offset); >>> - break; >>> - case ODPH_ETHTYPE_ARP: >>> - pkt_hdr->input_flags.arp = 1; >>> - /* fall through */ >>> - default: >>> - ip_proto = 0; >>> - break; >>> - } >>> + return seglen; >>> +} >>> >>> - switch (ip_proto) { >>> - case ODPH_IPPROTO_UDP: >>> - pkt_hdr->input_flags.udp = 1; >>> - pkt_hdr->input_flags.l4 = 1; >>> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >>> - break; >>> - case ODPH_IPPROTO_TCP: >>> - pkt_hdr->input_flags.tcp = 1; >>> - pkt_hdr->input_flags.l4 = 1; >>> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >>> - break; >>> - case ODPH_IPPROTO_SCTP: >>> - pkt_hdr->input_flags.sctp = 1; >>> - pkt_hdr->input_flags.l4 = 1; >>> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >>> - break; >>> - case ODPH_IPPROTO_ICMP: >>> - pkt_hdr->input_flags.icmp = 1; >>> - pkt_hdr->input_flags.l4 = 1; >>> - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; >>> - break; >>> - default: >>> - /* 0 or unhandled IP protocols, don't set L4 flag+offset >>> */ >>> - if (pkt_hdr->input_flags.ipv6) { >>> - /* IPv6 next_hdr is not L4, mark as IP-option >>> instead */ >>> - pkt_hdr->input_flags.ipopt = 1; >>> +/* >>> + * >>> + * Manipulation >>> + * ******************************************************** >>> + * >>> + */ >>> + >>> +odp_packet_t odp_packet_add_data(odp_packet_t pkt, uint32_t offset, >>> + uint32_t len) >>> +{ >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + uint32_t pktlen = pkt_hdr->frame_len; >>> + odp_packet_t newpkt; >>> + >>> + if (offset > pktlen) >>> + return ODP_PACKET_INVALID; >>> + >>> + newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen + >>> len); >>> + >>> + if (newpkt != ODP_PACKET_INVALID) { >>> + if (_odp_packet_copy_to_packet(pkt, 0, >>> + newpkt, 0, offset) != 0 || >>> + _odp_packet_copy_to_packet(pkt, offset, newpkt, >>> + offset + len, >>> + pktlen - offset) != 0) { >>> + odp_packet_free(newpkt); >>> + newpkt = ODP_PACKET_INVALID; >>> + } else { >>> + odp_packet_hdr_t *new_hdr = >>> odp_packet_hdr(newpkt); >>> + new_hdr->buf_hdr.buf_u64 = >>> pkt_hdr->buf_hdr.buf_u64; >>> + odp_atomic_store_u32( >>> + &new_hdr->buf_hdr.ref_count, >>> + odp_atomic_load_u32( >>> + &pkt_hdr->buf_hdr.ref_count)); >>> + _odp_packet_parse(newpkt); >>> + odp_packet_free(pkt); >>> } >>> - break; >>> } >>> + >>> + return newpkt; >>> } >>> >>> -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >>> - odph_ipv4hdr_t *ipv4, size_t >>> *offset_out) >>> +odp_packet_t odp_packet_rem_data(odp_packet_t pkt, uint32_t offset, >>> + uint32_t len) >>> { >>> - uint8_t ihl; >>> - uint16_t frag_offset; >>> - >>> - ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); >>> - if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN)) { >>> - pkt_hdr->error_flags.ip_err = 1; >>> - return 0; >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + uint32_t pktlen = pkt_hdr->frame_len; >>> + odp_packet_t newpkt; >>> + >>> + if (offset > pktlen || offset + len > pktlen) >>> + return ODP_PACKET_INVALID; >>> + >>> + newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen - >>> len); >>> + >>> + if (newpkt != ODP_PACKET_INVALID) { >>> + if (_odp_packet_copy_to_packet(pkt, 0, >>> + newpkt, 0, offset) != 0 || >>> + _odp_packet_copy_to_packet(pkt, offset + len, >>> + newpkt, offset, >>> + pktlen - offset - len) != >>> 0) { >>> + odp_packet_free(newpkt); >>> + newpkt = ODP_PACKET_INVALID; >>> + } else { >>> + odp_packet_hdr_t *new_hdr = >>> odp_packet_hdr(newpkt); >>> + new_hdr->buf_hdr.buf_u64 = >>> pkt_hdr->buf_hdr.buf_u64; >>> + odp_atomic_store_u32( >>> + &new_hdr->buf_hdr.ref_count, >>> + odp_atomic_load_u32( >>> + &pkt_hdr->buf_hdr.ref_count)); >>> + _odp_packet_parse(newpkt); >>> + odp_packet_free(pkt); >>> + } >>> } >>> >>> - if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) { >>> - pkt_hdr->input_flags.ipopt = 1; >>> - return 0; >>> - } >>> + return newpkt; >>> +} >>> >>> - /* A packet is a fragment if: >>> - * "more fragments" flag is set (all fragments except the last) >>> - * OR >>> - * "fragment offset" field is nonzero (all fragments except the >>> first) >>> - */ >>> - frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); >>> - if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) { >>> - pkt_hdr->input_flags.ipfrag = 1; >>> - return 0; >>> - } >>> +/* >>> + * >>> + * Copy >>> + * ******************************************************** >>> + * >>> + */ >>> >>> - if (ipv4->proto == ODPH_IPPROTO_ESP || >>> - ipv4->proto == ODPH_IPPROTO_AH) { >>> - pkt_hdr->input_flags.ipsec = 1; >>> - return 0; >>> +odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool) >>> +{ >>> + odp_packet_hdr_t *srchdr = odp_packet_hdr(pkt); >>> + uint32_t pktlen = srchdr->frame_len; >>> + uint32_t meta_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, >>> buf_hdr); >>> + odp_packet_t newpkt = odp_packet_alloc(pool, pktlen); >>> + >>> + if (newpkt != ODP_PACKET_INVALID) { >>> + odp_packet_hdr_t *newhdr = odp_packet_hdr(newpkt); >>> + uint8_t *newstart, *srcstart; >>> + >>> + /* Must copy meta data first, followed by packet data */ >>> + newstart = (uint8_t *)newhdr + meta_offset; >>> + srcstart = (uint8_t *)srchdr + meta_offset; >>> + >>> + memcpy(newstart, srcstart, >>> + sizeof(odp_packet_hdr_t) - meta_offset); >>> + >>> + if (_odp_packet_copy_to_packet(pkt, 0, >>> + newpkt, 0, pktlen) != 0) { >>> + odp_packet_free(newpkt); >>> + newpkt = ODP_PACKET_INVALID; >>> + } >>> } >>> >>> - /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after >>> return */ >>> - >>> - *offset_out = sizeof(uint32_t) * ihl; >>> - return ipv4->proto; >>> + return newpkt; >>> } >>> >>> -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >>> - odph_ipv6hdr_t *ipv6, size_t >>> *offset_out) >>> +int odp_packet_copydata_out(odp_packet_t pkt, uint32_t offset, >>> + uint32_t len, void *dst) >>> { >>> - if (ipv6->next_hdr == ODPH_IPPROTO_ESP || >>> - ipv6->next_hdr == ODPH_IPPROTO_AH) { >>> - pkt_hdr->input_flags.ipopt = 1; >>> - pkt_hdr->input_flags.ipsec = 1; >>> - return 0; >>> + void *mapaddr; >>> + uint32_t seglen = 0; /* GCC */ >>> + uint32_t cpylen; >>> + uint8_t *dstaddr = (uint8_t *)dst; >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + if (offset + len > pkt_hdr->frame_len) >>> + return -1; >>> + >>> + while (len > 0) { >>> + mapaddr = packet_map(pkt_hdr, offset, &seglen); >>> + cpylen = len > seglen ? seglen : len; >>> + memcpy(dstaddr, mapaddr, cpylen); >>> + offset += cpylen; >>> + dstaddr += cpylen; >>> + len -= cpylen; >>> } >>> >>> - if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { >>> - pkt_hdr->input_flags.ipopt = 1; >>> - pkt_hdr->input_flags.ipfrag = 1; >>> - return 0; >>> + return 0; >>> +} >>> + >>> +int odp_packet_copydata_in(odp_packet_t pkt, uint32_t offset, >>> + uint32_t len, const void *src) >>> +{ >>> + void *mapaddr; >>> + uint32_t seglen = 0; /* GCC */ >>> + uint32_t cpylen; >>> + const uint8_t *srcaddr = (const uint8_t *)src; >>> + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); >>> + >>> + if (offset + len > pkt_hdr->frame_len) >>> + return -1; >>> + >>> + while (len > 0) { >>> + mapaddr = packet_map(pkt_hdr, offset, &seglen); >>> + cpylen = len > seglen ? seglen : len; >>> + memcpy(mapaddr, srcaddr, cpylen); >>> + offset += cpylen; >>> + srcaddr += cpylen; >>> + len -= cpylen; >>> } >>> >>> - /* Don't step through more extensions */ >>> - *offset_out = ODPH_IPV6HDR_LEN; >>> - return ipv6->next_hdr; >>> + return 0; >>> } >>> >>> +/* >>> + * >>> + * Debugging >>> + * ******************************************************** >>> + * >>> + */ >>> + >>> void odp_packet_print(odp_packet_t pkt) >>> { >>> int max_len = 512; >>> char str[max_len]; >>> - uint8_t *p; >>> int len = 0; >>> int n = max_len-1; >>> odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); >>> @@ -353,77 +555,358 @@ void odp_packet_print(odp_packet_t pkt) >>> len += snprintf(&str[len], n-len, >>> " output_flags 0x%x\n", hdr->output_flags.all); >>> len += snprintf(&str[len], n-len, >>> - " frame_offset %u\n", hdr->frame_offset); >>> - len += snprintf(&str[len], n-len, >>> " l2_offset %u\n", hdr->l2_offset); >>> len += snprintf(&str[len], n-len, >>> " l3_offset %u\n", hdr->l3_offset); >>> len += snprintf(&str[len], n-len, >>> " l4_offset %u\n", hdr->l4_offset); >>> len += snprintf(&str[len], n-len, >>> - " frame_len %u\n", >>> hdr->buf_hdr.mb.pkt.pkt_len); >>> + " frame_len %u\n", hdr->frame_len); >>> len += snprintf(&str[len], n-len, >>> " input %u\n", hdr->input); >>> str[len] = '\0'; >>> >>> - printf("\n%s\n", str); >>> - rte_pktmbuf_dump(stdout, &hdr->buf_hdr.mb, 32); >>> + ODP_PRINT("\n%s\n", str); >>> +} >>> >>> - p = odp_packet_start(pkt); >>> - printf("00000000: %02X %02X %02X %02X %02X %02X %02X %02X\n", >>> - p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7]); >>> - printf("00000008: %02X %02X %02X %02X %02X %02X %02X %02X\n", >>> - p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); >>> +int odp_packet_is_valid(odp_packet_t pkt) >>> +{ >>> + odp_buffer_hdr_t *buf = validate_buf((odp_buffer_t)pkt); >>> >>> + return (buf != NULL && buf->type == ODP_BUFFER_TYPE_PACKET); >>> } >>> >>> -/* For now we can only copy between packets of the same segment size >>> - * We should probably refine this API, maybe introduce a clone API */ >>> -int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src) >>> +/* >>> + * >>> + * Internal Use Routines >>> + * ******************************************************** >>> + * >>> + */ >>> + >>> +int _odp_packet_copy_to_packet(odp_packet_t srcpkt, uint32_t srcoffset, >>> + odp_packet_t dstpkt, uint32_t dstoffset, >>> + uint32_t len) >>> { >>> - struct rte_mbuf *mb_dst, *mb_src; >>> - uint8_t nb_segs, i; >>> + odp_packet_hdr_t *srchdr = odp_packet_hdr(srcpkt); >>> + odp_packet_hdr_t *dsthdr = odp_packet_hdr(dstpkt); >>> + void *srcmap; >>> + void *dstmap; >>> + uint32_t cpylen, minseg; >>> + uint32_t srcseglen = 0; /* GCC */ >>> + uint32_t dstseglen = 0; /* GCC */ >>> + >>> + if (srcoffset + len > srchdr->frame_len || >>> + dstoffset + len > dsthdr->frame_len) >>> + return -1; >>> >>> - ODP_ASSERT(odp_buffer_type(pkt_dst) == ODP_BUFFER_TYPE_PACKET && >>> - odp_buffer_type(pkt_src) == ODP_BUFFER_TYPE_PACKET, >>> - "dst_pkt or src_pkt not of type >>> ODP_BUFFER_TYPE_PACKET"); >>> + while (len > 0) { >>> + srcmap = packet_map(srchdr, srcoffset, &srcseglen); >>> + dstmap = packet_map(dsthdr, dstoffset, &dstseglen); >>> >>> - if (pkt_dst == ODP_PACKET_INVALID || pkt_src == >>> ODP_PACKET_INVALID) >>> - return -1; >>> + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; >>> + cpylen = len > minseg ? minseg : len; >>> + memcpy(dstmap, srcmap, cpylen); >>> >>> - mb_dst = &(odp_packet_hdr(pkt_dst)->buf_hdr.mb); >>> - mb_src = &(odp_packet_hdr(pkt_src)->buf_hdr.mb); >>> + srcoffset += cpylen; >>> + dstoffset += cpylen; >>> + len -= cpylen; >>> + } >>> >>> - if (mb_dst->pkt.nb_segs != mb_src->pkt.nb_segs) { >>> - ODP_ERR("Different nb_segs in pkt_dst and pkt_src"); >>> - return -1; >>> + return 0; >>> +} >>> + >>> +odp_packet_t _odp_packet_alloc(odp_buffer_pool_t pool_hdl) >>> +{ >>> + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); >>> + >>> + if (pool->s.params.buf_type != ODP_BUFFER_TYPE_PACKET) >>> + return ODP_PACKET_INVALID; >>> + >>> + return (odp_packet_t)buffer_alloc(pool_hdl, >>> + pool->s.params.buf_size); >>> +} >>> + >>> +/** >>> + * Parser helper function for IPv4 >>> + */ >>> +static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, >>> + uint8_t **parseptr, uint32_t *offset) >>> +{ >>> + odph_ipv4hdr_t *ipv4 = (odph_ipv4hdr_t *)*parseptr; >>> + uint8_t ver = ODPH_IPV4HDR_VER(ipv4->ver_ihl); >>> + uint8_t ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); >>> + uint16_t frag_offset; >>> + >>> + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv4->tot_len); >>> + >>> + if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN) || >>> + odp_unlikely(ver != 4) || >>> + (pkt_hdr->l3_len > pkt_hdr->frame_len - *offset)) { >>> + pkt_hdr->error_flags.ip_err = 1; >>> + return 0; >>> } >>> >>> - nb_segs = mb_src->pkt.nb_segs; >>> + *offset += ihl * 4; >>> + *parseptr += ihl * 4; >>> >>> - if (mb_dst->buf_len < mb_src->buf_len) { >>> - ODP_ERR("dst_pkt smaller than src_pkt"); >>> - return -1; >>> + if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) >>> + pkt_hdr->input_flags.ipopt = 1; >>> + >>> + /* A packet is a fragment if: >>> + * "more fragments" flag is set (all fragments except the last) >>> + * OR >>> + * "fragment offset" field is nonzero (all fragments except the >>> first) >>> + */ >>> + frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); >>> + if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) >>> + pkt_hdr->input_flags.ipfrag = 1; >>> + >>> + return ipv4->proto; >>> +} >>> + >>> +/** >>> + * Parser helper function for IPv6 >>> + */ >>> +static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, >>> + uint8_t **parseptr, uint32_t *offset) >>> +{ >>> + odph_ipv6hdr_t *ipv6 = (odph_ipv6hdr_t *)*parseptr; >>> + odph_ipv6hdr_ext_t *ipv6ext; >>> + >>> + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv6->payload_len); >>> + >>> + /* Basic sanity checks on IPv6 header */ >>> + if ((ipv6->ver_tc_flow >> 28) != 6 || >>> + pkt_hdr->l3_len > pkt_hdr->frame_len - *offset) { >>> + pkt_hdr->error_flags.ip_err = 1; >>> + return 0; >>> } >>> >>> - for (i = 0; i < nb_segs; i++) { >>> - if (mb_src == NULL || mb_dst == NULL) { >>> - ODP_ERR("Corrupted packets"); >>> - return -1; >>> + /* Skip past IPv6 header */ >>> + *offset += sizeof(odph_ipv6hdr_t); >>> + *parseptr += sizeof(odph_ipv6hdr_t); >>> + >>> + >>> + /* Skip past any IPv6 extension headers */ >>> + if (ipv6->next_hdr == ODPH_IPPROTO_HOPOPTS || >>> + ipv6->next_hdr == ODPH_IPPROTO_ROUTE) { >>> + pkt_hdr->input_flags.ipopt = 1; >>> + >>> + do { >>> + ipv6ext = (odph_ipv6hdr_ext_t *)*parseptr; >>> + uint16_t extlen = 8 + ipv6ext->ext_len * 8; >>> + >>> + *offset += extlen; >>> + *parseptr += extlen; >>> + } while ((ipv6ext->next_hdr == ODPH_IPPROTO_HOPOPTS || >>> + ipv6ext->next_hdr == ODPH_IPPROTO_ROUTE) && >>> + *offset < pkt_hdr->frame_len); >>> + >>> + if (*offset >= pkt_hdr->l3_offset + ipv6->payload_len) { >>> + pkt_hdr->error_flags.ip_err = 1; >>> + return 0; >>> } >>> - memcpy(mb_dst->buf_addr, mb_src->buf_addr, >>> mb_src->buf_len); >>> - mb_dst = mb_dst->pkt.next; >>> - mb_src = mb_src->pkt.next; >>> + >>> + if (ipv6ext->next_hdr == ODPH_IPPROTO_FRAG) >>> + pkt_hdr->input_flags.ipfrag = 1; >>> + >>> + return ipv6ext->next_hdr; >>> } >>> - return 0; >>> + >>> + if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { >>> + pkt_hdr->input_flags.ipopt = 1; >>> + pkt_hdr->input_flags.ipfrag = 1; >>> + } >>> + >>> + return ipv6->next_hdr; >>> +} >>> + >>> +/** >>> + * Parser helper function for TCP >>> + */ >>> +static inline void parse_tcp(odp_packet_hdr_t *pkt_hdr, >>> + uint8_t **parseptr, uint32_t *offset) >>> +{ >>> + odph_tcphdr_t *tcp = (odph_tcphdr_t *)*parseptr; >>> + >>> + if (tcp->hl < sizeof(odph_tcphdr_t)/sizeof(uint32_t)) >>> + pkt_hdr->error_flags.tcp_err = 1; >>> + else if ((uint32_t)tcp->hl * 4 > sizeof(odph_tcphdr_t)) >>> + pkt_hdr->input_flags.tcpopt = 1; >>> + >>> + pkt_hdr->l4_len = pkt_hdr->l3_len + >>> + pkt_hdr->l3_offset - pkt_hdr->l4_offset; >>> + >>> + *offset += (uint32_t)tcp->hl * 4; >>> + *parseptr += (uint32_t)tcp->hl * 4; >>> } >>> >>> -void odp_packet_set_ctx(odp_packet_t pkt, const void *ctx) >>> +/** >>> + * Parser helper function for UDP >>> + */ >>> +static inline void parse_udp(odp_packet_hdr_t *pkt_hdr, >>> + uint8_t **parseptr, uint32_t *offset) >>> { >>> - odp_packet_hdr(pkt)->user_ctx = (intptr_t)ctx; >>> + odph_udphdr_t *udp = (odph_udphdr_t *)*parseptr; >>> + uint32_t udplen = odp_be_to_cpu_16(udp->length); >>> + >>> + if (udplen < sizeof(odph_udphdr_t) || >>> + udplen > (pkt_hdr->l3_len + >>> + pkt_hdr->l3_offset - pkt_hdr->l4_offset)) { >>> + pkt_hdr->error_flags.udp_err = 1; >>> + } >>> + >>> + pkt_hdr->l4_len = udplen; >>> + >>> + *offset += sizeof(odph_udphdr_t); >>> + *parseptr += sizeof(odph_udphdr_t); >>> } >>> >>> -void *odp_packet_get_ctx(odp_packet_t pkt) >>> +/** >>> + * Simple packet parser >>> + */ >>> + >>> +int _odp_packet_parse(odp_packet_t pkt) >>> { >>> - return (void *)(intptr_t)odp_packet_hdr(pkt)->user_ctx; >>> + odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); >>> + odph_ethhdr_t *eth; >>> + odph_vlanhdr_t *vlan; >>> + uint16_t ethtype; >>> + uint8_t *parseptr; >>> + uint32_t offset, seglen; >>> + uint8_t ip_proto = 0; >>> + >>> + /* Reset parser metadata for new parse */ >>> + pkt_hdr->error_flags.all = 0; >>> + pkt_hdr->input_flags.all = 0; >>> + pkt_hdr->output_flags.all = 0; >>> + pkt_hdr->l2_offset = 0; >>> + pkt_hdr->l3_offset = 0; >>> + pkt_hdr->l4_offset = 0; >>> + pkt_hdr->payload_offset = 0; >>> + pkt_hdr->vlan_s_tag = 0; >>> + pkt_hdr->vlan_c_tag = 0; >>> + pkt_hdr->l3_protocol = 0; >>> + pkt_hdr->l4_protocol = 0; >>> + >>> + /* We only support Ethernet for now */ >>> + pkt_hdr->input_flags.eth = 1; >>> + >>> + /* Detect jumbo frames */ >>> + if (pkt_hdr->frame_len > ODPH_ETH_LEN_MAX) >>> + pkt_hdr->input_flags.jumbo = 1; >>> + >>> + /* Assume valid L2 header, no CRC/FCS check in SW */ >>> + pkt_hdr->input_flags.l2 = 1; >>> + >>> + eth = (odph_ethhdr_t *)packet_map(pkt_hdr, 0, &seglen); >>> + offset = sizeof(odph_ethhdr_t); >>> + parseptr = (uint8_t *)ð->type; >>> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); >>> + >>> + /* Parse the VLAN header(s), if present */ >>> + if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { >>> + pkt_hdr->input_flags.vlan_qinq = 1; >>> + pkt_hdr->input_flags.vlan = 1; >>> + vlan = (odph_vlanhdr_t *)(void *)parseptr; >>> + pkt_hdr->vlan_s_tag = ((ethtype << 16) | >>> + odp_be_to_cpu_16(vlan->tci)); >>> + offset += sizeof(odph_vlanhdr_t); >>> + parseptr += sizeof(odph_vlanhdr_t); >>> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >>> *)parseptr)); >>> + } >>> + >>> + if (ethtype == ODPH_ETHTYPE_VLAN) { >>> + pkt_hdr->input_flags.vlan = 1; >>> + vlan = (odph_vlanhdr_t *)(void *)parseptr; >>> + pkt_hdr->vlan_c_tag = ((ethtype << 16) | >>> + odp_be_to_cpu_16(vlan->tci)); >>> + offset += sizeof(odph_vlanhdr_t); >>> + parseptr += sizeof(odph_vlanhdr_t); >>> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >>> *)parseptr)); >>> + } >>> + >>> + /* Check for SNAP vs. DIX */ >>> + if (ethtype < ODPH_ETH_LEN_MAX) { >>> + pkt_hdr->input_flags.snap = 1; >>> + if (ethtype > pkt_hdr->frame_len - offset) { >>> + pkt_hdr->error_flags.snap_len = 1; >>> + goto parse_exit; >>> + } >>> + offset += 8; >>> + parseptr += 8; >>> + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void >>> *)parseptr)); >>> + } >>> + >>> + /* Consume Ethertype for Layer 3 parse */ >>> + parseptr += 2; >>> + >>> + /* Set l3_offset+flag only for known ethtypes */ >>> + pkt_hdr->input_flags.l3 = 1; >>> + pkt_hdr->l3_offset = offset; >>> + pkt_hdr->l3_protocol = ethtype; >>> + >>> + /* Parse Layer 3 headers */ >>> + switch (ethtype) { >>> + case ODPH_ETHTYPE_IPV4: >>> + pkt_hdr->input_flags.ipv4 = 1; >>> + ip_proto = parse_ipv4(pkt_hdr, &parseptr, &offset); >>> + break; >>> + >>> + case ODPH_ETHTYPE_IPV6: >>> + pkt_hdr->input_flags.ipv6 = 1; >>> + ip_proto = parse_ipv6(pkt_hdr, &parseptr, &offset); >>> + break; >>> + >>> + case ODPH_ETHTYPE_ARP: >>> + pkt_hdr->input_flags.arp = 1; >>> + ip_proto = 255; /* Reserved invalid by IANA */ >>> + break; >>> + >>> + default: >>> + pkt_hdr->input_flags.l3 = 0; >>> + ip_proto = 255; /* Reserved invalid by IANA */ >>> + } >>> + >>> + /* Set l4_offset+flag only for known ip_proto */ >>> + pkt_hdr->input_flags.l4 = 1; >>> + pkt_hdr->l4_offset = offset; >>> + pkt_hdr->l4_protocol = ip_proto; >>> + >>> + /* Parse Layer 4 headers */ >>> + switch (ip_proto) { >>> + case ODPH_IPPROTO_ICMP: >>> + pkt_hdr->input_flags.icmp = 1; >>> + break; >>> + >>> + case ODPH_IPPROTO_TCP: >>> + pkt_hdr->input_flags.tcp = 1; >>> + parse_tcp(pkt_hdr, &parseptr, &offset); >>> + break; >>> + >>> + case ODPH_IPPROTO_UDP: >>> + pkt_hdr->input_flags.udp = 1; >>> + parse_udp(pkt_hdr, &parseptr, &offset); >>> + break; >>> + >>> + case ODPH_IPPROTO_AH: >>> + case ODPH_IPPROTO_ESP: >>> + pkt_hdr->input_flags.ipsec = 1; >>> + break; >>> + >>> + default: >>> + pkt_hdr->input_flags.l4 = 0; >>> + break; >>> + } >>> + >>> + /* >>> + * Anything beyond what we parse here is considered payload. >>> + * Note: Payload is really only relevant for TCP and UDP. For >>> + * all other protocols, the payload offset will point to the >>> + * final header (ARP, ICMP, AH, ESP, or IP Fragment). >>> + */ >>> + pkt_hdr->payload_offset = offset; >>> + >>> +parse_exit: >>> + return pkt_hdr->error_flags.all != 0; >>> } >>> diff --git a/platform/linux-dpdk/odp_packet_dpdk.c >>> b/platform/linux-dpdk/odp_packet_dpdk.c >>> index 9fe4bbd..80ce74f 100644 >>> --- a/platform/linux-dpdk/odp_packet_dpdk.c >>> +++ b/platform/linux-dpdk/odp_packet_dpdk.c >>> @@ -23,6 +23,8 @@ >>> #include <odp_hints.h> >>> #include <odp_thread.h> >>> >>> +#include <odp_system_info.h> >>> +#include <odp_debug_internal.h> >>> #include <odp_packet_dpdk.h> >>> #include <net/if.h> >>> >>> @@ -164,11 +166,8 @@ int recv_pkt_dpdk(pkt_dpdk_t * const pkt_dpdk, >>> odp_packet_t pkt_table[], >>> nb_rx = rte_eth_rx_burst((uint8_t)pkt_dpdk->portid, >>> (uint16_t)pkt_dpdk->queueid, >>> (struct rte_mbuf **)pkt_table, >>> (uint16_t)len); >>> - for (i = 0; i < nb_rx; i++) { >>> - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt_table[i]); >>> - struct rte_mbuf *mb = &pkt_hdr->buf_hdr.mb; >>> - odp_packet_parse(pkt_table[i], mb->pkt.pkt_len, 0); >>> - } >>> + for (i = 0; i < nb_rx; i++) >>> + _odp_packet_parse(pkt_table[i]); >>> return nb_rx; >>> } >>> >>> diff --git a/platform/linux-dpdk/odp_packet_io.c >>> b/platform/linux-dpdk/odp_packet_io.c >>> index 79394bb..b6d1d35 100644 >>> --- a/platform/linux-dpdk/odp_packet_io.c >>> +++ b/platform/linux-dpdk/odp_packet_io.c >>> @@ -16,34 +16,21 @@ >>> #ifdef ODP_HAVE_NETMAP >>> #include <odp_packet_netmap.h> >>> #endif >>> -#include <odp_hints.h> >>> #include <odp_config.h> >>> #include <odp_queue_internal.h> >>> #include <odp_schedule_internal.h> >>> -#include <odp_debug.h> >>> - >>> -#include <odp_pktio_socket.h> >>> -#ifdef ODP_HAVE_NETMAP >>> -#include <odp_pktio_netmap.h> >>> -#endif >>> +#include <odp_classification_internal.h> >>> +#include <odp_debug_internal.h> >>> >>> #include <string.h> >>> - >>> -typedef struct { >>> - pktio_entry_t entries[ODP_CONFIG_PKTIO_ENTRIES]; >>> -} pktio_table_t; >>> +#include <sys/ioctl.h> >>> +#include <linux/if_arp.h> >>> +#include <ifaddrs.h> >>> >>> static pktio_table_t *pktio_tbl; >>> >>> - >>> -static pktio_entry_t *get_entry(odp_pktio_t id) >>> -{ >>> - if (odp_unlikely(id == ODP_PKTIO_INVALID || >>> - id > ODP_CONFIG_PKTIO_ENTRIES)) >>> - return NULL; >>> - >>> - return &pktio_tbl->entries[id - 1]; >>> -} >>> +/* pktio pointer entries ( for inlines) */ >>> +void *pktio_entry_ptr[ODP_CONFIG_PKTIO_ENTRIES]; >>> >>> int odp_pktio_init_global(void) >>> { >>> @@ -65,10 +52,12 @@ int odp_pktio_init_global(void) >>> memset(pktio_tbl, 0, sizeof(pktio_table_t)); >>> >>> for (id = 1; id <= ODP_CONFIG_PKTIO_ENTRIES; ++id) { >>> - pktio_entry = get_entry(id); >>> + pktio_entry = &pktio_tbl->entries[id - 1]; >>> >>> odp_spinlock_init(&pktio_entry->s.lock); >>> + odp_spinlock_init(&pktio_entry->s.cls.lock); >>> >>> + pktio_entry_ptr[id - 1] = pktio_entry; >>> /* Create a default output queue for each pktio resource >>> */ >>> snprintf(name, sizeof(name), "%i-pktio_outq_default", >>> (int)id); >>> name[ODP_QUEUE_NAME_LEN-1] = '\0'; >>> @@ -115,16 +104,27 @@ static void unlock_entry(pktio_entry_t *entry) >>> odp_spinlock_unlock(&entry->s.lock); >>> } >>> >>> -static void init_pktio_entry(pktio_entry_t *entry, odp_pktio_params_t >>> *params) >>> +static void lock_entry_classifier(pktio_entry_t *entry) >>> +{ >>> + odp_spinlock_lock(&entry->s.lock); >>> + odp_spinlock_lock(&entry->s.cls.lock); >>> +} >>> + >>> +static void unlock_entry_classifier(pktio_entry_t *entry) >>> +{ >>> + odp_spinlock_unlock(&entry->s.cls.lock); >>> + odp_spinlock_unlock(&entry->s.lock); >>> +} >>> + >>> +static void init_pktio_entry(pktio_entry_t *entry) >>> { >>> set_taken(entry); >>> entry->s.inq_default = ODP_QUEUE_INVALID; >>> memset(&entry->s.pkt_dpdk, 0, sizeof(entry->s.pkt_dpdk)); >>> - /* Save pktio parameters, type is the most useful */ >>> - memcpy(&entry->s.params, params, sizeof(*params)); >>> + pktio_classifier_init(entry); >>> } >>> >>> -static odp_pktio_t alloc_lock_pktio_entry(odp_pktio_params_t *params) >>> +static odp_pktio_t alloc_lock_pktio_entry(void) >>> { >>> odp_pktio_t id; >>> pktio_entry_t *entry; >>> @@ -133,13 +133,13 @@ static odp_pktio_t >>> alloc_lock_pktio_entry(odp_pktio_params_t *params) >>> for (i = 0; i < ODP_CONFIG_PKTIO_ENTRIES; ++i) { >>> entry = &pktio_tbl->entries[i]; >>> if (is_free(entry)) { >>> - lock_entry(entry); >>> + lock_entry_classifier(entry); >>> if (is_free(entry)) { >>> - init_pktio_entry(entry, params); >>> + init_pktio_entry(entry); >>> id = i + 1; >>> return id; /* return with entry locked! >>> */ >>> } >>> - unlock_entry(entry); >>> + unlock_entry_classifier(entry); >>> } >>> } >>> >>> @@ -148,7 +148,7 @@ static odp_pktio_t >>> alloc_lock_pktio_entry(odp_pktio_params_t *params) >>> >>> static int free_pktio_entry(odp_pktio_t id) >>> { >>> - pktio_entry_t *entry = get_entry(id); >>> + pktio_entry_t *entry = get_pktio_entry(id); >>> >>> if (entry == NULL) >>> return -1; >>> @@ -158,28 +158,22 @@ static int free_pktio_entry(odp_pktio_t id) >>> return 0; >>> } >>> >>> -odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool, >>> - odp_pktio_params_t *params) >>> +odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool) >>> { >>> odp_pktio_t id; >>> pktio_entry_t *pktio_entry; >>> int res; >>> >>> - if (params == NULL) { >>> - ODP_ERR("Invalid pktio params\n"); >>> - return ODP_PKTIO_INVALID; >>> - } >>> - >>> ODP_DBG("Allocating dpdk pktio\n"); >>> >>> - id = alloc_lock_pktio_entry(params); >>> + id = alloc_lock_pktio_entry(); >>> if (id == ODP_PKTIO_INVALID) { >>> ODP_ERR("No resources available.\n"); >>> return ODP_PKTIO_INVALID; >>> } >>> /* if successful, alloc_pktio_entry() returns with the entry >>> locked */ >>> >>> - pktio_entry = get_entry(id); >>> + pktio_entry = get_pktio_entry(id); >>> >>> res = setup_pkt_dpdk(&pktio_entry->s.pkt_dpdk, dev, pool); >>> if (res == -1) { >>> @@ -188,7 +182,7 @@ odp_pktio_t odp_pktio_open(const char *dev, >>> odp_buffer_pool_t pool, >>> id = ODP_PKTIO_INVALID; >>> } >>> >>> - unlock_entry(pktio_entry); >>> + unlock_entry_classifier(pktio_entry); >>> return id; >>> } >>> >>> @@ -197,7 +191,7 @@ int odp_pktio_close(odp_pktio_t id) >>> pktio_entry_t *entry; >>> int res = -1; >>> >>> - entry = get_entry(id); >>> + entry = get_pktio_entry(id); >>> if (entry == NULL) >>> return -1; >>> >>> @@ -226,7 +220,7 @@ odp_pktio_t odp_pktio_get_input(odp_packet_t pkt) >>> >>> int odp_pktio_recv(odp_pktio_t id, odp_packet_t pkt_table[], unsigned >>> len) >>> { >>> - pktio_entry_t *pktio_entry = get_entry(id); >>> + pktio_entry_t *pktio_entry = get_pktio_entry(id); >>> int pkts; >>> int i; >>> >>> @@ -249,7 +243,7 @@ int odp_pktio_recv(odp_pktio_t id, odp_packet_t >>> pkt_table[], unsigned len) >>> >>> int odp_pktio_send(odp_pktio_t id, odp_packet_t pkt_table[], unsigned >>> len) >>> { >>> - pktio_entry_t *pktio_entry = get_entry(id); >>> + pktio_entry_t *pktio_entry = get_pktio_entry(id); >>> int pkts; >>> >>> if (pktio_entry == NULL) >>> @@ -265,15 +259,17 @@ int odp_pktio_send(odp_pktio_t id, odp_packet_t >>> pkt_table[], unsigned len) >>> >>> int odp_pktio_inq_setdef(odp_pktio_t id, odp_queue_t queue) >>> { >>> - pktio_entry_t *pktio_entry = get_entry(id); >>> + pktio_entry_t *pktio_entry = get_pktio_entry(id); >>> queue_entry_t *qentry = queue_to_qentry(queue); >>> >>> - if (pktio_entry == NULL || qentry == NULL) >>> + if (pktio_entry == NULL || qentry == ODP_QUEUE_INVALID) >>> return -1; >>> >>> if (qentry->s.type != ODP_QUEUE_TYPE_PKTIN) >>> return -1; >>> >>> + qentry = queue_to_qentry(queue); >>> + >>> lock_entry(pktio_entry); >>> pktio_entry->s.inq_default = queue; >>> unlock_entry(pktio_entry); >>> @@ -295,7 +291,7 @@ int odp_pktio_inq_remdef(odp_pktio_t id) >>> >>> odp_queue_t odp_pktio_inq_getdef(odp_pktio_t id) >>> { >>> - pktio_entry_t *pktio_entry = get_entry(id); >>> + pktio_entry_t *pktio_entry = get_pktio_entry(id); >>> >>> if (pktio_entry == NULL) >>> return ODP_QUEUE_INVALID; >>> @@ -305,7 +301,7 @@ odp_queue_t odp_pktio_inq_getdef(odp_pktio_t id) >>> >>> odp_queue_t odp_pktio_outq_getdef(odp_pktio_t id) >>> { >>> - pktio_entry_t *pktio_entry = get_entry(id); >>> + pktio_entry_t *pktio_entry = get_pktio_entry(id); >>> >>> if (pktio_entry == NULL) >>> return ODP_QUEUE_INVALID; >>> @@ -362,32 +358,32 @@ int pktin_enqueue(queue_entry_t *qentry, >>> odp_buffer_hdr_t *buf_hdr) >>> odp_buffer_hdr_t *pktin_dequeue(queue_entry_t *qentry) >>> { >>> odp_buffer_hdr_t *buf_hdr; >>> + odp_buffer_t buf; >>> + odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; >>> + odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX]; >>> + int pkts, i, j; >>> >>> buf_hdr = queue_deq(qentry); >>> + if (buf_hdr != NULL) >>> + return buf_hdr; >>> >>> - if (buf_hdr == NULL) { >>> - odp_packet_t pkt; >>> - odp_buffer_t buf; >>> - odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; >>> - odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX]; >>> - int pkts, i, j; >>> - >>> - pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl, >>> - QUEUE_MULTI_MAX); >>> - >>> - if (pkts > 0) { >>> - pkt = pkt_tbl[0]; >>> - buf = odp_buffer_from_packet(pkt); >>> - buf_hdr = odp_buf_to_hdr(buf); >>> + pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl, QUEUE_MULTI_MAX); >>> + if (pkts <= 0) >>> + return NULL; >>> >>> - for (i = 1, j = 0; i < pkts; ++i) { >>> - buf = odp_buffer_from_packet(pkt_tbl[i]); >>> - tmp_hdr_tbl[j++] = odp_buf_to_hdr(buf); >>> - } >>> - queue_enq_multi(qentry, tmp_hdr_tbl, j); >>> - } >>> + for (i = 0, j = 0; i < pkts; ++i) { >>> + buf = odp_packet_to_buffer(pkt_tbl[i]); >>> + buf_hdr = odp_buf_to_hdr(buf); >>> + if (0 > packet_classifier(qentry->s.pktin, pkt_tbl[i])) >>> + tmp_hdr_tbl[j++] = buf_hdr; >>> } >>> >>> + if (0 == j) >>> + return NULL; >>> + >>> + if (j > 1) >>> + queue_enq_multi(qentry, &tmp_hdr_tbl[1], j-1); >>> + buf_hdr = tmp_hdr_tbl[0]; >>> return buf_hdr; >>> } >>> >>> @@ -400,32 +396,179 @@ int pktin_enq_multi(queue_entry_t *qentry, >>> odp_buffer_hdr_t *buf_hdr[], int num) >>> int pktin_deq_multi(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr[], >>> int num) >>> { >>> int nbr; >>> + odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; >>> + odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX]; >>> + odp_buffer_hdr_t *tmp_hdr; >>> + odp_buffer_t buf; >>> + int pkts, i, j; >>> >>> nbr = queue_deq_multi(qentry, buf_hdr, num); >>> - >>> - if (nbr < num) { >>> - odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; >>> - odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX]; >>> - odp_buffer_t buf; >>> - int pkts, i; >>> - >>> - pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl, >>> - QUEUE_MULTI_MAX); >>> - if (pkts > 0) { >>> - for (i = 0; i < pkts; ++i) { >>> - buf = odp_buffer_from_packet(pkt_tbl[i]); >>> - tmp_hdr_tbl[i] = odp_buf_to_hdr(buf); >>> - } >>> - queue_enq_multi(qentry, tmp_hdr_tbl, pkts); >>> - } >>> + if (odp_unlikely(nbr > num)) >>> + ODP_ABORT("queue_deq_multi req: %d, returned %d\n", >>> + num, nbr); >>> + >>> + /** queue already has number of requsted buffers, >>> + * do not do receive in that case. >>> + */ >>> + if (nbr == num) >>> + return nbr; >>> + >>> + pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl, QUEUE_MULTI_MAX); >>> + if (pkts <= 0) >>> + return nbr; >>> + >>> + for (i = 0, j = 0; i < pkts; ++i) { >>> + buf = odp_packet_to_buffer(pkt_tbl[i]); >>> + tmp_hdr = odp_buf_to_hdr(buf); >>> + if (0 > packet_classifier(qentry->s.pktin, pkt_tbl[i])) >>> + tmp_hdr_tbl[j++] = tmp_hdr; >>> } >>> >>> + if (j) >>> + queue_enq_multi(qentry, tmp_hdr_tbl, j); >>> return nbr; >>> } >>> >>> +int odp_pktio_set_mtu(odp_pktio_t id, int mtu) >>> +{ >>> + pktio_entry_t *entry; >>> + int ret; >>> + >>> + if (mtu <= 0) { >>> + ODP_DBG("illegal MTU value %d\n", mtu); >>> + return -1; >>> + } >>> + >>> + entry = get_pktio_entry(id); >>> + if (entry == NULL) { >>> + ODP_DBG("pktio entry %d does not exist\n", id); >>> + return -1; >>> + } >>> + >>> + lock_entry(entry); >>> + >>> + if (odp_unlikely(is_free(entry))) { >>> + unlock_entry(entry); >>> + ODP_DBG("already freed pktio\n"); >>> + return -1; >>> + } >>> + >>> + ret = rte_eth_dev_set_mtu(entry->s.pkt_dpdk.portid, >>> (uint16_t)mtu); >>> + >>> + unlock_entry(entry); >>> + return ret; >>> +} >>> + >>> +int odp_pktio_mtu(odp_pktio_t id) >>> +{ >>> + pktio_entry_t *entry; >>> + int mtu; >>> + >>> + entry = get_pktio_entry(id); >>> + if (entry == NULL) { >>> + ODP_DBG("pktio entry %d does not exist\n", id); >>> + return -1; >>> + } >>> + >>> + lock_entry(entry); >>> + >>> + if (odp_unlikely(is_free(entry))) { >>> + unlock_entry(entry); >>> + ODP_DBG("already freed pktio\n"); >>> + return -1; >>> + } >>> + >>> + rte_eth_dev_get_mtu(entry->s.pkt_dpdk.portid , (uint16_t *)&mtu); >>> + >>> + unlock_entry(entry); >>> + return mtu; >>> +} >>> + >>> +int odp_pktio_promisc_mode_set(odp_pktio_t id, odp_bool_t enable) >>> +{ >>> + pktio_entry_t *entry; >>> + >>> + entry = get_pktio_entry(id); >>> + if (entry == NULL) { >>> + ODP_DBG("pktio entry %d does not exist\n", id); >>> + return -1; >>> + } >>> + >>> + lock_entry(entry); >>> + >>> + if (odp_unlikely(is_free(entry))) { >>> + unlock_entry(entry); >>> + ODP_DBG("already freed pktio\n"); >>> + return -1; >>> + } >>> + >>> + if (enable) >>> + rte_eth_promiscuous_enable(entry->s.pkt_dpdk.portid); >>> + else >>> + rte_eth_promiscuous_disable(entry->s.pkt_dpdk.portid); >>> + >>> + unlock_entry(entry); >>> + return 0; >>> +} >>> + >>> +int odp_pktio_promisc_mode(odp_pktio_t id) >>> +{ >>> + pktio_entry_t *entry; >>> + int promisc; >>> + >>> + entry = get_pktio_entry(id); >>> + if (entry == NULL) { >>> + ODP_DBG("pktio entry %d does not exist\n", id); >>> + return -1; >>> + } >>> + >>> + lock_entry(entry); >>> + >>> + if (odp_unlikely(is_free(entry))) { >>> + unlock_entry(entry); >>> + ODP_DBG("already freed pktio\n"); >>> + return -1; >>> + } >>> + >>> + promisc = rte_eth_promiscuous_get(entry->s.pkt_dpdk.portid); >>> + >>> + unlock_entry(entry); >>> + >>> + return promisc; >>> +} >>> + >>> +size_t odp_pktio_mac_addr(odp_pktio_t id, void *mac_addr, >>> + size_t addr_size) >>> +{ >>> + pktio_entry_t *entry; >>> + >>> + if (addr_size < ETH_ALEN) >>> + return 0; >>> + >>> + entry = get_pktio_entry(id); >>> + if (entry == NULL) { >>> + ODP_DBG("pktio entry %d does not exist\n", id); >>> + return 0; >>> + } >>> + >>> + lock_entry(entry); >>> + >>> + if (odp_unlikely(is_free(entry))) { >>> + unlock_entry(entry); >>> + ODP_DBG("already freed pktio\n"); >>> + return -1; >>> + } >>> + >>> + rte_eth_macaddr_get(entry->s.pkt_dpdk.portid, >>> + (struct ether_addr *)mac_addr); >>> + unlock_entry(entry); >>> + >>> + return ETH_ALEN; >>> +} >>> + >>> int odp_pktio_get_mac_addr(odp_pktio_t id, unsigned char *mac_addr) >>> { >>> - pktio_entry_t *pktio_entry = get_entry(id); >>> + pktio_entry_t *pktio_entry = get_pktio_entry(id); >>> if (!pktio_entry) { >>> ODP_ERR("Invalid odp_pktio_t value\n"); >>> return -1; >>> diff --git a/platform/linux-dpdk/odp_queue.c >>> b/platform/linux-dpdk/odp_queue.c >>> index 03f7c2c..9e906a3 100644 >>> --- a/platform/linux-dpdk/odp_queue.c >>> +++ b/platform/linux-dpdk/odp_queue.c >>> @@ -11,13 +11,14 @@ >>> #include <odp_buffer.h> >>> #include <odp_buffer_internal.h> >>> #include <odp_buffer_pool_internal.h> >>> +#include <odp_buffer_inlines.h> >>> #include <odp_internal.h> >>> #include <odp_shared_memory.h> >>> #include <odp_schedule_internal.h> >>> #include <odp_config.h> >>> #include <odp_packet_io_internal.h> >>> #include <odp_packet_io_queue.h> >>> -#include <odp_debug.h> >>> +#include <odp_debug_internal.h> >>> #include <odp_hints.h> >>> >>> #ifdef USE_TICKETLOCK >>> diff --git a/platform/linux-dpdk/odp_schedule.c >>> b/platform/linux-dpdk/odp_schedule.c >>> deleted file mode 100644 >>> index cb559e4..0000000 >>> --- a/platform/linux-dpdk/odp_schedule.c >>> +++ /dev/null >>> @@ -1,421 +0,0 @@ >>> -/* Copyright (c) 2013, Linaro Limited >>> - * All rights reserved. >>> - * >>> - * SPDX-License-Identifier: BSD-3-Clause >>> - */ >>> - >>> -#include <odp_schedule.h> >>> -#include <odp_schedule_internal.h> >>> -#include <odp_align.h> >>> -#include <odp_queue.h> >>> -#include <odp_shared_memory.h> >>> -#include <odp_buffer.h> >>> -#include <odp_buffer_pool.h> >>> -#include <odp_internal.h> >>> -#include <odp_config.h> >>> -#include <odp_debug.h> >>> -#include <odp_thread.h> >>> -#include <odp_time.h> >>> -#include <odp_spinlock.h> >>> -#include <odp_hints.h> >>> - >>> -#include <odp_queue_internal.h> >>> - >>> - >>> -/* Limits to number of scheduled queues */ >>> -#define SCHED_POOL_SIZE (256*1024) >>> - >>> -/* Scheduler sub queues */ >>> -#define QUEUES_PER_PRIO 4 >>> - >>> -/* TODO: random or queue based selection */ >>> -#define SEL_PRI_QUEUE(x) ((QUEUES_PER_PRIO-1) & (queue_to_id(x))) >>> - >>> -/* Maximum number of dequeues */ >>> -#define MAX_DEQ 4 >>> - >>> - >>> -/* Mask of queues per priority */ >>> -typedef uint8_t pri_mask_t; >>> - >>> -ODP_STATIC_ASSERT((8 * sizeof(pri_mask_t)) >= >>> - QUEUES_PER_PRIO, "pri_mask_t_is_too_small"); >>> - >>> - >>> -typedef struct { >>> - odp_queue_t >>> pri_queue[ODP_CONFIG_SCHED_PRIOS][QUEUES_PER_PRIO]; >>> - pri_mask_t pri_mask[ODP_CONFIG_SCHED_PRIOS]; >>> - odp_spinlock_t mask_lock; >>> - odp_buffer_pool_t pool; >>> -} sched_t; >>> - >>> -typedef struct { >>> - odp_queue_t queue; >>> - >>> -} queue_desc_t; >>> - >>> -typedef struct { >>> - odp_queue_t pri_queue; >>> - odp_buffer_t desc_buf; >>> - >>> - odp_buffer_t buf[MAX_DEQ]; >>> - int num; >>> - int index; >>> - odp_queue_t queue; >>> - int pause; >>> - >>> -} sched_local_t; >>> - >>> -/* Global scheduler context */ >>> -static sched_t *sched; >>> - >>> -/* Thread local scheduler context */ >>> -static __thread sched_local_t sched_local; >>> - >>> - >>> -static inline odp_queue_t select_pri_queue(odp_queue_t queue, int prio) >>> -{ >>> - int id = SEL_PRI_QUEUE(queue); >>> - return sched->pri_queue[prio][id]; >>> -} >>> - >>> - >>> -int odp_schedule_init_global(void) >>> -{ >>> - odp_shm_t shm; >>> - odp_buffer_pool_t pool; >>> - int i, j; >>> - >>> - ODP_DBG("Schedule init ... "); >>> - >>> - shm = odp_shm_reserve("odp_scheduler", >>> - sizeof(sched_t), >>> - ODP_CACHE_LINE_SIZE, 0); >>> - >>> - sched = odp_shm_addr(shm); >>> - >>> - if (sched == NULL) { >>> - ODP_ERR("Schedule init: Shm reserve failed.\n"); >>> - return -1; >>> - } >>> - >>> - pool = odp_buffer_pool_create("odp_sched_pool", NULL, >>> - SCHED_POOL_SIZE, >>> sizeof(queue_desc_t), >>> - ODP_CACHE_LINE_SIZE, >>> - ODP_BUFFER_TYPE_RAW); >>> - >>> - if (pool == ODP_BUFFER_POOL_INVALID) { >>> - ODP_ERR("Schedule init: Pool create failed.\n"); >>> - return -1; >>> - } >>> - >>> - sched->pool = pool; >>> - odp_spinlock_init(&sched->mask_lock); >>> - >>> - for (i = 0; i < ODP_CONFIG_SCHED_PRIOS; i++) { >>> - odp_queue_t queue; >>> - char name[] = "odp_priXX_YY"; >>> - >>> - name[7] = '0' + i / 10; >>> - name[8] = '0' + i - 10*(i / 10); >>> - >>> - for (j = 0; j < QUEUES_PER_PRIO; j++) { >>> - name[10] = '0' + j / 10; >>> - name[11] = '0' + j - 10*(j / 10); >>> - >>> - queue = odp_queue_create(name, >>> - ODP_QUEUE_TYPE_POLL, >>> NULL); >>> - >>> - if (queue == ODP_QUEUE_INVALID) { >>> - ODP_ERR("Sched init: Queue create >>> failed.\n"); >>> - return -1; >>> - } >>> - >>> - sched->pri_queue[i][j] = queue; >>> - sched->pri_mask[i] = 0; >>> - } >>> - } >>> - >>> - ODP_DBG("done\n"); >>> - >>> - return 0; >>> -} >>> - >>> - >>> -int odp_schedule_init_local(void) >>> -{ >>> - int i; >>> - >>> - sched_local.pri_queue = ODP_QUEUE_INVALID; >>> - sched_local.desc_buf = ODP_BUFFER_INVALID; >>> - >>> - for (i = 0; i < MAX_DEQ; i++) >>> - sched_local.buf[i] = ODP_BUFFER_INVALID; >>> - >>> - sched_local.num = 0; >>> - sched_local.index = 0; >>> - sched_local.queue = ODP_QUEUE_INVALID; >>> - sched_local.pause = 0; >>> - >>> - return 0; >>> -} >>> - >>> - >>> -void odp_schedule_mask_set(odp_queue_t queue, int prio) >>> -{ >>> - int id = SEL_PRI_QUEUE(queue); >>> - >>> - odp_spinlock_lock(&sched->mask_lock); >>> - sched->pri_mask[prio] |= 1 << id; >>> - odp_spinlock_unlock(&sched->mask_lock); >>> -} >>> - >>> - >>> -odp_buffer_t odp_schedule_buffer_alloc(odp_queue_t queue) >>> -{ >>> - odp_buffer_t buf; >>> - >>> - buf = odp_buffer_alloc(sched->pool); >>> - >>> - if (buf != ODP_BUFFER_INVALID) { >>> - queue_desc_t *desc; >>> - desc = odp_buffer_addr(buf); >>> - desc->queue = queue; >>> - } >>> - >>> - return buf; >>> -} >>> - >>> - >>> -void odp_schedule_queue(odp_queue_t queue, int prio) >>> -{ >>> - odp_buffer_t desc_buf; >>> - odp_queue_t pri_queue; >>> - >>> - pri_queue = select_pri_queue(queue, prio); >>> - desc_buf = queue_sched_buf(queue); >>> - >>> - odp_queue_enq(pri_queue, desc_buf); >>> -} >>> - >>> - >>> -void odp_schedule_release_atomic(void) >>> -{ >>> - if (sched_local.pri_queue != ODP_QUEUE_INVALID && >>> - sched_local.num == 0) { >>> - /* Release current atomic queue */ >>> - odp_queue_enq(sched_local.pri_queue, >>> sched_local.desc_buf); >>> - sched_local.pri_queue = ODP_QUEUE_INVALID; >>> - } >>> -} >>> - >>> - >>> -static inline int copy_bufs(odp_buffer_t out_buf[], unsigned int max) >>> -{ >>> - int i = 0; >>> - >>> - while (sched_local.num && max) { >>> - out_buf[i] = sched_local.buf[sched_local.index]; >>> - sched_local.index++; >>> - sched_local.num--; >>> - max--; >>> - i++; >>> - } >>> - >>> - return i; >>> -} >>> - >>> - >>> -/* >>> - * Schedule queues >>> - * >>> - * TODO: SYNC_ORDERED not implemented yet >>> - */ >>> -static int schedule(odp_queue_t *out_queue, odp_buffer_t out_buf[], >>> - unsigned int max_num, unsigned int max_deq) >>> -{ >>> - int i, j; >>> - int thr; >>> - int ret; >>> - >>> - if (sched_local.num) { >>> - ret = copy_bufs(out_buf, max_num); >>> - >>> - if (out_queue) >>> - *out_queue = sched_local.queue; >>> - >>> - return ret; >>> - } >>> - >>> - odp_schedule_release_atomic(); >>> - >>> - if (odp_unlikely(sched_local.pause)) >>> - return 0; >>> - >>> - thr = odp_thread_id(); >>> - >>> - for (i = 0; i < ODP_CONFIG_SCHED_PRIOS; i++) { >>> - int id; >>> - >>> - if (sched->pri_mask[i] == 0) >>> - continue; >>> - >>> - id = thr & (QUEUES_PER_PRIO-1); >>> - >>> - for (j = 0; j < QUEUES_PER_PRIO; j++, id++) { >>> - odp_queue_t pri_q; >>> - odp_buffer_t desc_buf; >>> - >>> - if (id >= QUEUES_PER_PRIO) >>> - id = 0; >>> - >>> - if (odp_unlikely((sched->pri_mask[i] & (1 << >>> id)) == 0)) >>> - continue; >>> - >>> - pri_q = sched->pri_queue[i][id]; >>> - desc_buf = odp_queue_deq(pri_q); >>> - >>> - if (desc_buf != ODP_BUFFER_INVALID) { >>> - queue_desc_t *desc; >>> - odp_queue_t queue; >>> - int num; >>> - >>> - desc = odp_buffer_addr(desc_buf); >>> - queue = desc->queue; >>> - >>> - num = odp_queue_deq_multi(queue, >>> - >>> sched_local.buf, >>> - max_deq); >>> - >>> - if (num == 0) { >>> - /* Remove empty queue from >>> scheduling, >>> - * except packet input queues >>> - */ >>> - if (odp_queue_type(queue) == >>> - ODP_QUEUE_TYPE_PKTIN) >>> - odp_queue_enq(pri_q, >>> desc_buf); >>> - >>> - continue; >>> - } >>> - >>> - sched_local.num = num; >>> - sched_local.index = 0; >>> - ret = copy_bufs(out_buf, max_num); >>> - >>> - sched_local.queue = queue; >>> - >>> - if (queue_sched_atomic(queue)) { >>> - /* Hold queue during atomic >>> access */ >>> - sched_local.pri_queue = pri_q; >>> - sched_local.desc_buf = desc_buf; >>> - } else { >>> - /* Continue scheduling the queue >>> */ >>> - odp_queue_enq(pri_q, desc_buf); >>> - } >>> - >>> - /* Output the source queue handle */ >>> - if (out_queue) >>> - *out_queue = queue; >>> - >>> - return ret; >>> - } >>> - } >>> - } >>> - >>> - return 0; >>> -} >>> - >>> - >>> -static int schedule_loop(odp_queue_t *out_queue, uint64_t wait, >>> - odp_buffer_t out_buf[], >>> - unsigned int max_num, unsigned int max_deq) >>> -{ >>> - uint64_t start_cycle, cycle, diff; >>> - int ret; >>> - >>> - start_cycle = 0; >>> - >>> - while (1) { >>> - ret = schedule(out_queue, out_buf, max_num, max_deq); >>> - >>> - if (ret) >>> - break; >>> - >>> - if (wait == ODP_SCHED_WAIT) >>> - continue; >>> - >>> - if (wait == ODP_SCHED_NO_WAIT) >>> - break; >>> - >>> - if (start_cycle == 0) { >>> - start_cycle = odp_time_get_cycles(); >>> - continue; >>> - } >>> - >>> - cycle = odp_time_get_cycles(); >>> - diff = odp_time_diff_cycles(start_cycle, cycle); >>> - >>> - if (wait < diff) >>> - break; >>> - } >>> - >>> - return ret; >>> -} >>> - >>> - >>> -odp_buffer_t odp_schedule(odp_queue_t *out_queue, uint64_t wait) >>> -{ >>> - odp_buffer_t buf; >>> - >>> - buf = ODP_BUFFER_INVALID; >>> - >>> - schedule_loop(out_queue, wait, &buf, 1, MAX_DEQ); >>> - >>> - return buf; >>> -} >>> - >>> - >>> -odp_buffer_t odp_schedule_one(odp_queue_t *out_queue, uint64_t wait) >>> -{ >>> - odp_buffer_t buf; >>> - >>> - buf = ODP_BUFFER_INVALID; >>> - >>> - schedule_loop(out_queue, wait, &buf, 1, 1); >>> - >>> - return buf; >>> -} >>> - >>> - >>> -int odp_schedule_multi(odp_queue_t *out_queue, uint64_t wait, >>> - odp_buffer_t out_buf[], unsigned int num) >>> -{ >>> - return schedule_loop(out_queue, wait, out_buf, num, MAX_DEQ); >>> -} >>> - >>> - >>> -void odp_schedule_pause(void) >>> -{ >>> - sched_local.pause = 1; >>> -} >>> - >>> - >>> -void odp_schedule_resume(void) >>> -{ >>> - sched_local.pause = 0; >>> -} >>> - >>> - >>> -uint64_t odp_schedule_wait_time(uint64_t ns) >>> -{ >>> - if (ns <= ODP_SCHED_NO_WAIT) >>> - ns = ODP_SCHED_NO_WAIT + 1; >>> - >>> - return odp_time_ns_to_cycles(ns); >>> -} >>> - >>> - >>> -int odp_schedule_num_prio(void) >>> -{ >>> - return ODP_CONFIG_SCHED_PRIOS; >>> -} >>> -- >>> 1.9.1 >>> >>> >> >> _______________________________________________ >> lng-odp mailing list >> lng-odp@lists.linaro.org >> http://lists.linaro.org/mailman/listinfo/lng-odp >> >> > > > -- > *Mike Holmes* > Linaro Sr Technical Manager > LNG - ODP >
My idea is that when I finish cleaning up my changes we can make a diff between your 0.5 repo and my 0.6,and that hopefully will be much shorter as my stuff supposed to have most of your changes,but it is broken up to reasonable patches. How does that sound? I will try to finish that now on the plane,but otherwise we can do that on Connect,or afterwards. Zoli On 4 Feb 2015 16:07, "Venkatesh Vivekanandan" < venkatesh.vivekanandan@linaro.org> wrote: > > > On 4 February 2015 at 21:05, Mike Holmes <mike.holmes@linaro.org> wrote: > >> Venki - I think Zoltan is breaking his 0.6 patches up, andI think the >> right thing to do is create patches for all the changes as I believe >> ODP-DPDK will become a more important >> > > If it is a feature, then yes we can break into small patches, but since > this is a new baseline(say 0.5.0) we have no choice other than reviewing > the whole patch is what I believe. Even if we try to break things into > small groups, it will only help the reviewer but one cannot commit it as > is, because it will not compile. > > >> implementation to us this year. >> >> Maybe we do this at HKG15 to reduce the review cycle time ? >> >> On 4 February 2015 at 10:30, Venkatesh Vivekanandan < >> venkatesh.vivekanandan@linaro.org> wrote: >> >>> Hi All, >>> >>> Inorder to make the REVIEW PROCESS SIMPLER as it is a huge patch, I have >>> attached the changes that went in this patch on top of ODP 0.5.0 (diff >>> between platform/linux-generic and platform/linux-dpdk). Hope this helps. >>> >>> Thanks, >>> Venky >>> >>> On 4 February 2015 at 20:48, <venkatesh.vivekanandan@linaro.org> wrote: >>> >>>> From: Venkatesh Vivekanandan <venkatesh.vivekanandan@linaro.org> >>>> >>>> Signed-off-by: Venkatesh Vivekanandan < >>>> venkatesh.vivekanandan@linaro.org> >>>> --- >>>> -> This patch must be applied on odp v0.5.0 ontop of odp-dpdk >>>> -> odp_l2fwd, odp_pktio were tested both in burst and queue mode which >>>> worked >>>> fine. >>>> -> ipsec app was tested and working fine. For this to work, >>>> USE_MAC_ADDR_HACK has to be set to 0 for now. Will send out a patch >>>> for >>>> MAC address fix once this patch gets in. >>>> >>>> Checkpatch warning for camel case could not be avoided. >>>> WARNING: Avoid CamelCase: <_Static_assert> >>>> #373: FILE: platform/linux-dpdk/include/api/odp_debug.h:34: >>>> +#define _Static_assert(e, s) extern int (*static_assert_checker(void)) >>>> \ >>>> >>>> WARNING: storage class should be at the beginning of the declaration >>>> #373: FILE: platform/linux-dpdk/include/api/odp_debug.h:34: >>>> +#define _Static_assert(e, s) extern int (*static_assert_checker(void)) >>>> \ >>>> >>>> v2: Fixed odp_flush_caches compilation error >>>> v3: Removed debug code that slipped in >>>> >>>> platform/linux-dpdk/Makefile.am | 13 +- >>>> platform/linux-dpdk/include/api/odp_buffer.h | 30 +- >>>> platform/linux-dpdk/include/api/odp_buffer_pool.h | 124 ++- >>>> platform/linux-dpdk/include/api/odp_debug.h | 77 +- >>>> platform/linux-dpdk/include/api/odp_packet.h | 772 >>>> +++++++++++++-- >>>> platform/linux-dpdk/include/api/odp_packet_io.h | 92 +- >>>> platform/linux-dpdk/include/api/odp_pktio_types.h | 45 - >>>> .../linux-dpdk/include/api/odp_platform_types.h | 78 ++ >>>> platform/linux-dpdk/include/odp_buffer_inlines.h | 179 ++++ >>>> platform/linux-dpdk/include/odp_buffer_internal.h | 134 ++- >>>> .../linux-dpdk/include/odp_buffer_pool_internal.h | 333 ++++++- >>>> platform/linux-dpdk/include/odp_packet_dpdk.h | 2 - >>>> platform/linux-dpdk/include/odp_packet_internal.h | 135 ++- >>>> .../linux-dpdk/include/odp_packet_io_internal.h | 42 +- >>>> platform/linux-dpdk/odp_buffer.c | 14 +- >>>> platform/linux-dpdk/odp_buffer_pool.c | 153 +-- >>>> platform/linux-dpdk/odp_init.c | 35 +- >>>> platform/linux-dpdk/odp_linux.c | 117 ++- >>>> platform/linux-dpdk/odp_packet.c | 1001 >>>> +++++++++++++++----- >>>> platform/linux-dpdk/odp_packet_dpdk.c | 9 +- >>>> platform/linux-dpdk/odp_packet_io.c | 309 ++++-- >>>> platform/linux-dpdk/odp_queue.c | 3 +- >>>> platform/linux-dpdk/odp_schedule.c | 421 -------- >>>> 23 files changed, 2993 insertions(+), 1125 deletions(-) >>>> delete mode 100644 platform/linux-dpdk/include/api/odp_pktio_types.h >>>> create mode 100644 platform/linux-dpdk/include/api/odp_platform_types.h >>>> create mode 100644 platform/linux-dpdk/include/odp_buffer_inlines.h >>>> delete mode 100644 platform/linux-dpdk/odp_schedule.c >>>> >>>> diff --git a/platform/linux-dpdk/Makefile.am >>>> b/platform/linux-dpdk/Makefile.am >>>> index db11e0a..6491d3a 100644 >>>> --- a/platform/linux-dpdk/Makefile.am >>>> +++ b/platform/linux-dpdk/Makefile.am >>>> @@ -29,6 +29,7 @@ include_HEADERS = \ >>>> >>>> $(top_srcdir)/platform/linux-generic/include/api/odp_barrier.h \ >>>> $(srcdir)/include/api/odp_buffer_pool.h \ >>>> >>>> $(top_srcdir)/platform/linux-generic/include/api/odp_byteorder.h \ >>>> + >>>> $(top_srcdir)/platform/linux-generic/include/api/odp_classification.h \ >>>> >>>> $(top_srcdir)/platform/linux-generic/include/api/odp_compiler.h \ >>>> >>>> $(top_srcdir)/platform/linux-generic/include/api/odp_config.h \ >>>> >>>> $(top_srcdir)/platform/linux-generic/include/api/odp_coremask.h \ >>>> @@ -52,15 +53,16 @@ include_HEADERS = \ >>>> >>>> $(top_srcdir)/platform/linux-generic/include/api/odp_time.h \ >>>> >>>> $(top_srcdir)/platform/linux-generic/include/api/odp_timer.h \ >>>> >>>> $(top_srcdir)/platform/linux-generic/include/api/odp_version.h \ >>>> - $(srcdir)/include/api/odp_pktio_types.h >>>> + $(srcdir)/include/api/odp_platform_types.h >>>> >>>> subdirheadersdir = $(includedir)/helper >>>> subdirheaders_HEADERS = \ >>>> $(top_srcdir)/helper/include/odph_chksum.h \ >>>> $(top_srcdir)/helper/include/odph_eth.h \ >>>> + $(top_srcdir)/helper/include/odph_icmp.h \ >>>> $(top_srcdir)/helper/include/odph_ip.h \ >>>> + $(top_srcdir)/helper/include/odph_ipsec.h \ >>>> $(top_srcdir)/helper/include/odph_linux.h \ >>>> - $(top_srcdir)/helper/include/odph_packet.h \ >>>> $(top_srcdir)/helper/include/odph_ring.h \ >>>> $(top_srcdir)/helper/include/odph_udp.h >>>> >>>> @@ -68,9 +70,11 @@ __LIB__libodp_la_SOURCES = \ >>>> ../linux-generic/odp_barrier.c \ >>>> odp_buffer.c \ >>>> odp_buffer_pool.c \ >>>> + ../linux-generic/odp_classification.c \ >>>> ../linux-generic/odp_coremask.c \ >>>> ../linux-generic/odp_crypto.c \ >>>> odp_init.c \ >>>> + ../linux-generic/odp_impl.c \ >>>> odp_linux.c \ >>>> odp_packet.c \ >>>> odp_packet_dpdk.c \ >>>> @@ -80,11 +84,12 @@ __LIB__libodp_la_SOURCES = \ >>>> odp_queue.c \ >>>> ../linux-generic/odp_ring.c \ >>>> ../linux-generic/odp_rwlock.c \ >>>> - odp_schedule.c \ >>>> + ../linux-generic/odp_schedule.c \ >>>> ../linux-generic/odp_shared_memory.c \ >>>> ../linux-generic/odp_spinlock.c \ >>>> ../linux-generic/odp_system_info.c \ >>>> ../linux-generic/odp_thread.c \ >>>> ../linux-generic/odp_ticketlock.c \ >>>> ../linux-generic/odp_time.c \ >>>> - ../linux-generic/odp_timer.c >>>> + ../linux-generic/odp_timer.c \ >>>> + ../linux-generic/odp_weak.c >>>> diff --git a/platform/linux-dpdk/include/api/odp_buffer.h >>>> b/platform/linux-dpdk/include/api/odp_buffer.h >>>> index b2fbc76..3c23035 100644 >>>> --- a/platform/linux-dpdk/include/api/odp_buffer.h >>>> +++ b/platform/linux-dpdk/include/api/odp_buffer.h >>>> @@ -19,20 +19,13 @@ extern "C" { >>>> #endif >>>> >>>> >>>> - >>>> #include <odp_std_types.h> >>>> +#include <odp_platform_types.h> >>>> >>>> - >>>> - >>>> - >>>> - >>>> -/** >>>> - * ODP buffer >>>> +/** @defgroup odp_buffer ODP BUFFER >>>> + * Operations on a buffer. >>>> + * @{ >>>> */ >>>> -typedef unsigned long odp_buffer_t; >>>> - >>>> - >>>> -#define ODP_BUFFER_INVALID (unsigned long)(-1L) /**< Invalid buffer */ >>>> >>>> >>>> /** >>>> @@ -74,11 +67,21 @@ int odp_buffer_type(odp_buffer_t buf); >>>> * >>>> * @param buf Buffer handle >>>> * >>>> - * @return 1 if valid, otherwise 0 >>>> + * @retval 1 Buffer handle represents a valid buffer. >>>> + * @retval 0 Buffer handle does not represent a valid buffer. >>>> */ >>>> int odp_buffer_is_valid(odp_buffer_t buf); >>>> >>>> /** >>>> + * Buffer pool of the buffer >>>> + * >>>> + * @param buf Buffer handle >>>> + * >>>> + * @return Handle of buffer pool buffer belongs to >>>> + */ >>>> +odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf); >>>> + >>>> +/** >>>> * Print buffer metadata to STDOUT >>>> * >>>> * @param buf Buffer handle >>>> @@ -86,6 +89,9 @@ int odp_buffer_is_valid(odp_buffer_t buf); >>>> */ >>>> void odp_buffer_print(odp_buffer_t buf); >>>> >>>> +/** >>>> + * @} >>>> + */ >>>> >>>> #ifdef __cplusplus >>>> } >>>> diff --git a/platform/linux-dpdk/include/api/odp_buffer_pool.h >>>> b/platform/linux-dpdk/include/api/odp_buffer_pool.h >>>> index 382f4f0..4da5f84 100644 >>>> --- a/platform/linux-dpdk/include/api/odp_buffer_pool.h >>>> +++ b/platform/linux-dpdk/include/api/odp_buffer_pool.h >>>> @@ -21,66 +21,143 @@ extern "C" { >>>> >>>> >>>> #include <odp_std_types.h> >>>> +#include <odp_platform_types.h> >>>> #include <odp_buffer.h> >>>> >>>> +/** @addtogroup odp_buffer >>>> + * Operations on a buffer pool. >>>> + * @{ >>>> + */ >>>> + >>>> /** Maximum queue name lenght in chars */ >>>> #define ODP_BUFFER_POOL_NAME_LEN 32 >>>> >>>> -/** Invalid buffer pool */ >>>> -#define ODP_BUFFER_POOL_INVALID (unsigned long)(-1L) >>>> - >>>> -/** ODP buffer pool */ >>>> -typedef unsigned long odp_buffer_pool_t; >>>> - >>>> +/** >>>> + * Buffer pool parameters >>>> + * Used to communicate buffer pool creation options. >>>> + */ >>>> +typedef struct odp_buffer_pool_param_t { >>>> + uint32_t buf_size; /**< Buffer size in bytes. The maximum >>>> + number of bytes application will >>>> + store in each buffer. For packets, this >>>> + is the maximum packet data length, and >>>> + configured headroom and tailroom will be >>>> + added to this number */ >>>> + uint32_t buf_align; /**< Minimum buffer alignment in bytes. >>>> + Valid values are powers of two. Use 0 >>>> + for default alignment. Default will >>>> + always be a multiple of 8. */ >>>> + uint32_t num_bufs; /**< Number of buffers in the pool */ >>>> + int buf_type; /**< Buffer type */ >>>> +} odp_buffer_pool_param_t; >>>> >>>> /** >>>> * Create a buffer pool >>>> + * This routine is used to create a buffer pool. It take three >>>> + * arguments: the optional name of the pool to be created, an optional >>>> shared >>>> + * memory handle, and a parameter struct that describes the pool to be >>>> + * created. If a name is not specified the result is an anonymous pool >>>> that >>>> + * cannot be referenced by odp_buffer_pool_lookup(). >>>> + * >>>> + * @param name Name of the pool, max ODP_BUFFER_POOL_NAME_LEN-1 >>>> chars. >>>> + * May be specified as NULL for anonymous pools. >>>> + * >>>> + * @param shm The shared memory object in which to create the >>>> pool. >>>> + * Use ODP_SHM_NULL to reserve default memory type >>>> + * for the buffer type. >>>> * >>>> - * @param name Name of the pool (max ODP_BUFFER_POOL_NAME_LEN - 1 >>>> chars) >>>> - * @param base_addr Pool base address >>>> - * @param size Pool size in bytes >>>> - * @param buf_size Buffer size in bytes >>>> - * @param buf_align Minimum buffer alignment >>>> - * @param buf_type Buffer type >>>> + * @param params Buffer pool parameters. >>>> * >>>> - * @return Buffer pool handle >>>> + * @return Handle of the created buffer pool >>>> + * @retval ODP_BUFFER_POOL_INVALID Buffer pool could not be created >>>> */ >>>> + >>>> odp_buffer_pool_t odp_buffer_pool_create(const char *name, >>>> - void *base_addr, uint64_t size, >>>> - size_t buf_size, size_t >>>> buf_align, >>>> - int buf_type); >>>> + odp_shm_t shm, >>>> + odp_buffer_pool_param_t >>>> *params); >>>> >>>> +/** >>>> + * Destroy a buffer pool previously created by odp_buffer_pool_create() >>>> + * >>>> + * @param pool Handle of the buffer pool to be destroyed >>>> + * >>>> + * @retval 0 Success >>>> + * @retval -1 Failure >>>> + * >>>> + * @note This routine destroys a previously created buffer pool. This >>>> call >>>> + * does not destroy any shared memory object passed to >>>> + * odp_buffer_pool_create() used to store the buffer pool contents. >>>> The caller >>>> + * takes responsibility for that. If no shared memory object was >>>> passed as >>>> + * part of the create call, then this routine will destroy any >>>> internal shared >>>> + * memory objects associated with the buffer pool. Results are >>>> undefined if >>>> + * an attempt is made to destroy a buffer pool that contains allocated >>>> or >>>> + * otherwise active buffers. >>>> + */ >>>> +int odp_buffer_pool_destroy(odp_buffer_pool_t pool); >>>> >>>> /** >>>> * Find a buffer pool by name >>>> * >>>> * @param name Name of the pool >>>> * >>>> - * @return Buffer pool handle, or ODP_BUFFER_POOL_INVALID if not found. >>>> + * @return Handle of found buffer pool >>>> + * @retval ODP_BUFFER_POOL_INVALID Buffer pool could not be found >>>> + * >>>> + * @note This routine cannot be used to look up an anonymous pool (one >>>> created >>>> + * with no name). >>>> */ >>>> odp_buffer_pool_t odp_buffer_pool_lookup(const char *name); >>>> >>>> +/** >>>> + * Buffer pool information struct >>>> + * Used to get information about a buffer pool. >>>> + */ >>>> +typedef struct odp_buffer_pool_info_t { >>>> + const char *name; /**< pool name */ >>>> + odp_shm_t shm; /**< handle of shared memory >>>> area >>>> + supplied by application to >>>> + contain buffer pool, or >>>> + ODP_SHM_NULL if this pool >>>> is >>>> + managed by ODP */ >>>> + odp_buffer_pool_param_t params; /**< pool parameters */ >>>> +} odp_buffer_pool_info_t; >>>> + >>>> +/** >>>> + * Retrieve information about a buffer pool >>>> + * >>>> + * @param pool Buffer pool handle >>>> + * >>>> + * @param[out] info Receives an odp_buffer_pool_info_t object >>>> + * that describes the pool. >>>> + * >>>> + * @retval 0 Success >>>> + * @retval -1 Failure. Info could not be retrieved. >>>> + */ >>>> + >>>> +int odp_buffer_pool_info(odp_buffer_pool_t pool, >>>> + odp_buffer_pool_info_t *info); >>>> >>>> /** >>>> * Print buffer pool info >>>> * >>>> * @param pool Pool handle >>>> * >>>> + * @note This routine writes implementation-defined information about >>>> the >>>> + * specified buffer pool to the ODP log. The intended use is for >>>> debugging. >>>> */ >>>> void odp_buffer_pool_print(odp_buffer_pool_t pool); >>>> >>>> - >>>> - >>>> /** >>>> * Buffer alloc >>>> * >>>> + * The validity of a buffer can be cheked at any time with >>>> odp_buffer_is_valid() >>>> * @param pool Pool handle >>>> * >>>> - * @return Buffer handle or ODP_BUFFER_INVALID >>>> + * @return Handle of allocated buffer >>>> + * @retval ODP_BUFFER_INVALID Buffer could not be allocated >>>> */ >>>> odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool); >>>> >>>> - >>>> /** >>>> * Buffer free >>>> * >>>> @@ -89,8 +166,9 @@ odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t >>>> pool); >>>> */ >>>> void odp_buffer_free(odp_buffer_t buf); >>>> >>>> - >>>> - >>>> +/** >>>> + * @} >>>> + */ >>>> >>>> #ifdef __cplusplus >>>> } >>>> diff --git a/platform/linux-dpdk/include/api/odp_debug.h >>>> b/platform/linux-dpdk/include/api/odp_debug.h >>>> index 6b145b5..aed8a5f 100644 >>>> --- a/platform/linux-dpdk/include/api/odp_debug.h >>>> +++ b/platform/linux-dpdk/include/api/odp_debug.h >>>> @@ -12,73 +12,74 @@ >>>> #ifndef ODP_DEBUG_H_ >>>> #define ODP_DEBUG_H_ >>>> >>>> -#include <stdio.h> >>>> -#include <stdlib.h> >>>> >>>> #ifdef __cplusplus >>>> extern "C" { >>>> #endif >>>> >>>> +/** @addtogroup odp_ver_abt_log_dbg >>>> + * Macros that allows different messages. >>>> + * @{ >>>> + */ >>>> + >>>> #ifdef __GNUC__ >>>> >>>> -/** >>>> - * Indicate deprecated variables, functions or types >>>> - */ >>>> -#define ODP_DEPRECATED __attribute__((__deprecated__)) >>>> + >>>> +#if __GNUC__ < 4 || (__GNUC__ == 4 && (__GNUC_MINOR__ < 6)) >>>> >>>> /** >>>> - * Intentionally unused variables ot functions >>>> + * _Static_assert was only added in GCC 4.6. Provide a weak replacement >>>> + * for previous versions. >>>> */ >>>> -#define ODP_UNUSED __attribute__((__unused__)) >>>> +#define _Static_assert(e, s) extern int (*static_assert_checker(void)) >>>> \ >>>> + [sizeof(struct { unsigned int error_if_negative:(e) ? 1 : -1; >>>> })] >>>> >>>> -#else >>>> +#endif >>>> >>>> -#define ODP_DEPRECATED >>>> -#define ODP_UNUSED >>>> >>>> -#endif >>>> >>>> -/** >>>> - * Runtime assertion-macro - aborts if 'cond' is false. >>>> - */ >>>> -#ifndef ODP_NO_DEBUG >>>> -#define ODP_ASSERT(cond, msg) \ >>>> - do { if (!(cond)) {ODP_ERR("%s\n", msg); abort(); } } while (0) >>>> -#else >>>> -#define ODP_ASSERT(cond, msg) >>>> #endif >>>> >>>> + >>>> /** >>>> * Compile time assertion-macro - fail compilation if cond is false. >>>> * @note This macro has zero runtime overhead >>>> */ >>>> -#define ODP_STATIC_ASSERT(cond, msg) _Static_assert(1, msg) >>>> +#define _ODP_STATIC_ASSERT(cond, msg) _Static_assert(1, msg) >>>> >>>> /** >>>> - * Debug printing macro, which prints output when DEBUG flag is set. >>>> + * ODP log level. >>>> */ >>>> -#define ODP_DBG(fmt, ...) \ >>>> - do { if (ODP_DEBUG_PRINT == 1) \ >>>> - printf(fmt, ##__VA_ARGS__); \ >>>> - } while (0) >>>> +typedef enum odp_log_level { >>>> + ODP_LOG_DBG, >>>> + ODP_LOG_ERR, >>>> + ODP_LOG_UNIMPLEMENTED, >>>> + ODP_LOG_ABORT, >>>> + ODP_LOG_PRINT >>>> +} odp_log_level_e; >>>> >>>> /** >>>> - * Print output to stderr (file, line and function). >>>> + * ODP log function >>>> + * >>>> + * Instead of direct prints to stdout/stderr all logging in ODP >>>> implementation >>>> + * should be done via this function or its wrappers. >>>> + * ODP platform MUST provide a default *weak* implementation of this >>>> function. >>>> + * Application MAY override the function if needed by providing a >>>> strong >>>> + * function. >>>> + * >>>> + * @param[in] level Log level >>>> + * @param[in] fmt printf-style message format >>>> + * >>>> + * @return The number of characters logged if succeeded. Otherwise >>>> returns >>>> + * a negative number. >>>> */ >>>> -#define ODP_ERR(fmt, ...) \ >>>> -do { fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, \ >>>> - __LINE__, __func__, ##__VA_ARGS__); \ >>>> -} while (0) >>>> +extern int odp_override_log(odp_log_level_e level, const char *fmt, >>>> ...); >>>> + >>>> + >>>> >>>> /** >>>> - * Print output to stderr (file, line and function), >>>> - * then abort. >>>> + * @} >>>> */ >>>> -#define ODP_ABORT(fmt, ...) \ >>>> -do { fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, \ >>>> - __LINE__, __func__, ##__VA_ARGS__); \ >>>> - abort(); \ >>>> -} while (0) >>>> >>>> #ifdef __cplusplus >>>> } >>>> diff --git a/platform/linux-dpdk/include/api/odp_packet.h >>>> b/platform/linux-dpdk/include/api/odp_packet.h >>>> index 64a92e8..97c2cb6 100644 >>>> --- a/platform/linux-dpdk/include/api/odp_packet.h >>>> +++ b/platform/linux-dpdk/include/api/odp_packet.h >>>> @@ -1,7 +1,7 @@ >>>> /* Copyright (c) 2013, Linaro Limited >>>> * All rights reserved. >>>> * >>>> - * SPDX-License-Identifier: BSD-3-Clause >>>> + * SPDX-License-Identifier: BSD-3-Clause >>>> */ >>>> >>>> >>>> @@ -19,33 +19,84 @@ extern "C" { >>>> #endif >>>> >>>> #include <odp_buffer.h> >>>> +#include <odp_platform_types.h> >>>> >>>> +/** @defgroup odp_packet ODP PACKET >>>> + * Operations on a packet. >>>> + * @{ >>>> + */ >>>> >>>> -/** >>>> - * ODP packet descriptor >>>> + >>>> +/* >>>> + * Packet API v0.5 notes >>>> + * - Push/pull operations only on packet level >>>> + * - Push/pull within limits of segment headroom/tailroom/data lengths >>>> + * - Segment data length must be always at least one byte (i.e. there >>>> are no >>>> + * empty segments) >>>> + * - Head/tailroom content belong to packet content (in addition to >>>> data >>>> + * and meta-data) and thus is preserved over packet ownership >>>> changes. >>>> + * - _addr refer to a fixed address, which operations do not modify >>>> + * - _ptr refer to pointer to data, which may be modified by operations >>>> */ >>>> -typedef odp_buffer_t odp_packet_t; >>>> >>>> >>>> -/** Invalid packet */ >>>> -#define ODP_PACKET_INVALID ODP_BUFFER_INVALID >>>> +/* >>>> + * >>>> + * Alloc and free >>>> + * ******************************************************** >>>> + * >>>> + */ >>>> >>>> -/** Invalid offset */ >>>> -#define ODP_PACKET_OFFSET_INVALID ((size_t)-1) >>>> +/** >>>> + * Allocate a packet from a buffer pool >>>> + * >>>> + * Allocates a packet of the requested length from the specified >>>> buffer pool. >>>> + * Pool must have been created with buffer type >>>> ODP_BUFFER_TYPE_PACKET. The >>>> + * packet is initialized with data pointers and lengths set according >>>> to the >>>> + * specified len, and the default headroom and tailroom length >>>> settings. All >>>> + * other packet metadata are set to their default values. >>>> + * >>>> + * @param pool Pool handle >>>> + * @param len Packet data length >>>> + * >>>> + * @return Handle of allocated packet >>>> + * @retval ODP_PACKET_INVALID Packet could not be allocated >>>> + * >>>> + * @note The default headroom and tailroom used for packets is >>>> specified by >>>> + * the ODP_CONFIG_PACKET_HEADROOM and ODP_CONFIG_PACKET_TAILROOM >>>> defines in >>>> + * odp_config.h. >>>> + */ >>>> +odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool, uint32_t len); >>>> >>>> +/** >>>> + * Free packet >>>> + * >>>> + * Frees the packet into the buffer pool it was allocated from. >>>> + * >>>> + * @param pkt Packet handle >>>> + */ >>>> +void odp_packet_free(odp_packet_t pkt); >>>> >>>> /** >>>> - * Initialize the packet >>>> + * Reset packet >>>> * >>>> - * Needs to be called if the user allocates a packet buffer, i.e. the >>>> packet >>>> - * has not been received from I/O through ODP. >>>> + * Resets all packet meta-data to their default values. Packet length >>>> is used >>>> + * to initialize pointers and lengths. It must be less than the total >>>> buffer >>>> + * length of the packet minus the default headroom length. Packet is >>>> not >>>> + * modified on failure. >>>> * >>>> - * @param pkt Packet handle >>>> + * @param pkt Packet handle >>>> + * @param len Packet data length >>>> + * >>>> + * @retval 0 Success >>>> + * @retval Non-zero Failure >>>> + * >>>> + * @see odp_packet_buf_len() >>>> */ >>>> -void odp_packet_init(odp_packet_t pkt); >>>> +int odp_packet_reset(odp_packet_t pkt, uint32_t len); >>>> >>>> /** >>>> - * Convert from packet handle to buffer handle >>>> + * Convert a buffer handle to a packet handle >>>> * >>>> * @param buf Buffer handle >>>> * >>>> @@ -54,190 +105,737 @@ void odp_packet_init(odp_packet_t pkt); >>>> odp_packet_t odp_packet_from_buffer(odp_buffer_t buf); >>>> >>>> /** >>>> - * Convert from buffer handle to packet handle >>>> + * Convert a packet handle to a buffer handle >>>> * >>>> * @param pkt Packet handle >>>> * >>>> * @return Buffer handle >>>> */ >>>> -odp_buffer_t odp_buffer_from_packet(odp_packet_t pkt); >>>> +odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt); >>>> + >>>> + >>>> +/* >>>> + * >>>> + * Pointers and lengths >>>> + * ******************************************************** >>>> + * >>>> + */ >>>> + >>>> +/** >>>> + * Packet head address >>>> + * >>>> + * Returns start address of the first segment. Packet level headroom >>>> starts >>>> + * from here. Use odp_packet_data() or odp_packet_l2_ptr() to return >>>> the >>>> + * packet data start address. >>>> + * >>>> + * @param pkt Packet handle >>>> + * >>>> + * @return Pointer to the start address of the first packet segment >>>> + * >>>> + * @see odp_packet_data(), odp_packet_l2_ptr(), odp_packet_headroom() >>>> + */ >>>> +void *odp_packet_head(odp_packet_t pkt); >>>> + >>>> +/** >>>> + * Total packet buffer length >>>> + * >>>> + * Returns sum of buffer lengths over all packet segments. >>>> + * >>>> + * @param pkt Packet handle >>>> + * >>>> + * @return Total packet buffer length in bytes >>>> + * >>>> + * @see odp_packet_reset() >>>> + */ >>>> +uint32_t odp_packet_buf_len(odp_packet_t pkt); >>>> >>>> /** >>>> - * Set the packet length >>>> + * Packet data pointer >>>> + * >>>> + * Returns the current packet data pointer. When a packet is received >>>> + * from packet input, this points to the first byte of the received >>>> + * packet. Packet level offsets are calculated relative to this >>>> position. >>>> + * >>>> + * User can adjust the data pointer with head_push/head_pull (does not >>>> modify >>>> + * segmentation) and add_data/rem_data calls (may modify segmentation). >>>> + * >>>> + * @param pkt Packet handle >>>> + * >>>> + * @return Pointer to the packet data >>>> + * >>>> + * @see odp_packet_l2_ptr(), odp_packet_seg_len() >>>> + */ >>>> +void *odp_packet_data(odp_packet_t pkt); >>>> + >>>> +/** >>>> + * Packet segment data length >>>> + * >>>> + * Returns number of data bytes following the current data pointer >>>> + * (odp_packet_data()) location in the segment. >>>> * >>>> * @param pkt Packet handle >>>> - * @param len Length of packet in bytes >>>> + * >>>> + * @return Segment data length in bytes (pointed by odp_packet_data()) >>>> + * >>>> + * @see odp_packet_data() >>>> */ >>>> -void odp_packet_set_len(odp_packet_t pkt, size_t len); >>>> +uint32_t odp_packet_seg_len(odp_packet_t pkt); >>>> >>>> /** >>>> - * Get the packet length >>>> + * Packet data length >>>> + * >>>> + * Returns sum of data lengths over all packet segments. >>>> * >>>> * @param pkt Packet handle >>>> * >>>> - * @return Packet length in bytes >>>> + * @return Packet data length >>>> */ >>>> -size_t odp_packet_get_len(odp_packet_t pkt); >>>> +uint32_t odp_packet_len(odp_packet_t pkt); >>>> >>>> /** >>>> - * Set packet user context >>>> + * Packet headroom length >>>> * >>>> - * @param buf Packet handle >>>> - * @param ctx User context >>>> + * Returns the current packet level headroom length. >>>> + * >>>> + * @param pkt Packet handle >>>> * >>>> + * @return Headroom length >>>> */ >>>> -void odp_packet_set_ctx(odp_packet_t buf, const void *ctx); >>>> +uint32_t odp_packet_headroom(odp_packet_t pkt); >>>> >>>> /** >>>> - * Get packet user context >>>> + * Packet tailroom length >>>> * >>>> - * @param buf Packet handle >>>> + * Returns the current packet level tailroom length. >>>> * >>>> - * @return User context >>>> + * @param pkt Packet handle >>>> + * >>>> + * @return Tailroom length >>>> */ >>>> -void *odp_packet_get_ctx(odp_packet_t buf); >>>> +uint32_t odp_packet_tailroom(odp_packet_t pkt); >>>> >>>> /** >>>> - * Get address to the start of the packet buffer >>>> + * Packet tailroom pointer >>>> * >>>> - * The address of the packet buffer is not necessarily the same as the >>>> start >>>> - * address of the received frame, e.g. an eth frame may be offset by 2 >>>> or 6 >>>> - * bytes to ensure 32 or 64-bit alignment of the IP header. >>>> - * Use odp_packet_l2(pkt) to get the start address of a received valid >>>> frame >>>> - * or odp_packet_start(pkt) to get the start address even if no valid >>>> L2 header >>>> - * could be found. >>>> + * Returns pointer to the start of the current packet level tailroom. >>>> + * >>>> + * User can adjust the tail pointer with tail_push/tail_pull (does not >>>> modify >>>> + * segmentation) and add_data/rem_data calls (may modify segmentation). >>>> * >>>> * @param pkt Packet handle >>>> * >>>> - * @return Pointer to the start of the packet buffer >>>> + * @return Tailroom pointer >>>> * >>>> - * @see odp_packet_l2(), odp_packet_start() >>>> + * @see odp_packet_tailroom() >>>> */ >>>> -uint8_t *odp_packet_buf_addr(odp_packet_t pkt); >>>> +void *odp_packet_tail(odp_packet_t pkt); >>>> >>>> /** >>>> - * Get pointer to the start of the received frame >>>> + * Push out packet head >>>> + * >>>> + * Increase packet data length by moving packet head into packet >>>> headroom. >>>> + * Packet headroom is decreased with the same amount. The packet head >>>> may be >>>> + * pushed out up to 'headroom' bytes. Packet is not modified if >>>> there's not >>>> + * enough headroom space. >>>> * >>>> - * The address of the packet buffer is not necessarily the same as the >>>> start >>>> - * address of the received frame, e.g. an eth frame may be offset by 2 >>>> or 6 >>>> - * bytes to ensure 32 or 64-bit alignment of the IP header. >>>> - * Use odp_packet_l2(pkt) to get the start address of a received valid >>>> eth frame >>>> + * odp_packet_xxx: >>>> + * seg_len += len >>>> + * len += len >>>> + * headroom -= len >>>> + * data -= len >>>> * >>>> - * odp_packet_start() will always return a pointer to the start of the >>>> frame, >>>> - * even if the frame is unrecognized and no valid L2 header could be >>>> found. >>>> + * Operation does not modify packet segmentation or move data. Handles >>>> and >>>> + * pointers remain valid. User is responsible to update packet >>>> meta-data >>>> + * offsets when needed. >>>> * >>>> * @param pkt Packet handle >>>> + * @param len Number of bytes to push the head (0 ... headroom) >>>> * >>>> - * @return Pointer to the start of the received frame >>>> + * @return The new data pointer >>>> + * @retval NULL Requested offset exceeds available headroom >>>> * >>>> - * @see odp_packet_l2(), odp_packet_buf_addr() >>>> + * @see odp_packet_headroom(), odp_packet_pull_head() >>>> */ >>>> -uint8_t *odp_packet_start(odp_packet_t pkt); >>>> +void *odp_packet_push_head(odp_packet_t pkt, uint32_t len); >>>> >>>> /** >>>> - * Get pointer to the start of the L2 frame >>>> + * Pull in packet head >>>> * >>>> - * The L2 frame header address is not necessarily the same as the >>>> address of the >>>> - * packet buffer, see odp_packet_buf_addr() >>>> + * Decrease packet data length by removing data from the head of the >>>> packet. >>>> + * Packet headroom is increased with the same amount. Packet head may >>>> be pulled >>>> + * in up to seg_len - 1 bytes (i.e. packet data pointer must stay in >>>> the >>>> + * first segment). Packet is not modified if there's not enough data. >>>> + * >>>> + * odp_packet_xxx: >>>> + * seg_len -= len >>>> + * len -= len >>>> + * headroom += len >>>> + * data += len >>>> + * >>>> + * Operation does not modify packet segmentation or move data. Handles >>>> and >>>> + * pointers remain valid. User is responsible to update packet >>>> meta-data >>>> + * offsets when needed. >>>> * >>>> * @param pkt Packet handle >>>> + * @param len Number of bytes to pull the head (0 ... seg_len - 1) >>>> * >>>> - * @return Pointer to L2 header or NULL if not found >>>> + * @return The new data pointer, or NULL in case of an error. >>>> + * @retval NULL Requested offset exceeds packet segment length >>>> * >>>> - * @see odp_packet_buf_addr(), odp_packet_start() >>>> + * @see odp_packet_seg_len(), odp_packet_push_head() >>>> */ >>>> -uint8_t *odp_packet_l2(odp_packet_t pkt); >>>> +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len); >>>> >>>> /** >>>> - * Return the byte offset from the packet buffer to the L2 frame >>>> + * Push out packet tail >>>> + * >>>> + * Increase packet data length by moving packet tail into packet >>>> tailroom. >>>> + * Packet tailroom is decreased with the same amount. The packet tail >>>> may be >>>> + * pushed out up to 'tailroom' bytes. Packet is not modified if >>>> there's not >>>> + * enough tailroom. >>>> + * >>>> + * last_seg: >>>> + * data_len += len >>>> + * >>>> + * odp_packet_xxx: >>>> + * len += len >>>> + * tail += len >>>> + * tailroom -= len >>>> + * >>>> + * Operation does not modify packet segmentation or move data. Handles, >>>> + * pointers and offsets remain valid. >>>> * >>>> * @param pkt Packet handle >>>> + * @param len Number of bytes to push the tail (0 ... tailroom) >>>> * >>>> - * @return L2 byte offset or ODP_PACKET_OFFSET_INVALID if not found >>>> + * @return The old tail pointer >>>> + * @retval NULL Requested offset exceeds available tailroom >>>> + * >>>> + * @see odp_packet_tailroom(), odp_packet_pull_tail() >>>> */ >>>> -size_t odp_packet_l2_offset(odp_packet_t pkt); >>>> +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len); >>>> >>>> /** >>>> - * Set the byte offset to the L2 frame >>>> + * Pull in packet tail >>>> + * >>>> + * Decrease packet data length by removing data from the tail of the >>>> packet. >>>> + * Packet tailroom is increased with the same amount. Packet tail may >>>> be pulled >>>> + * in up to last segment data_len - 1 bytes. (i.e. packet tail must >>>> stay in the >>>> + * last segment). Packet is not modified if there's not enough data. >>>> + * >>>> + * last_seg: >>>> + * data_len -= len >>>> + * >>>> + * odp_packet_xxx: >>>> + * len -= len >>>> + * tail -= len >>>> + * tailroom += len >>>> + * >>>> + * Operation does not modify packet segmentation or move data. Handles >>>> and >>>> + * pointers remain valid. User is responsible to update packet >>>> meta-data >>>> + * offsets when needed. >>>> + * >>>> + * @param pkt Packet handle >>>> + * @param len Number of bytes to pull the tail (0 ... >>>> last_seg:data_len - 1) >>>> + * >>>> + * @return The new tail pointer >>>> + * @retval NULL The specified offset exceeds allowable data length >>>> + */ >>>> +void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len); >>>> + >>>> +/** >>>> + * Packet offset pointer >>>> + * >>>> + * Returns pointer to data in the packet offset. The packet level byte >>>> offset is >>>> + * calculated from the current odp_packet_data() position. Optionally >>>> outputs >>>> + * handle to the segment and number of data bytes in the segment >>>> following the >>>> + * pointer. >>>> + * >>>> + * @param pkt Packet handle >>>> + * @param offset Byte offset into the packet >>>> + * @param[out] len Number of data bytes remaining in the segment >>>> (output). >>>> + * Ignored when NULL. >>>> + * @param[out] seg Handle to the segment containing the address >>>> (output). >>>> + * Ignored when NULL. >>>> + * >>>> + * @return Pointer to the offset >>>> + * @retval NULL Requested offset exceeds packet length >>>> + */ >>>> +void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t >>>> *len, >>>> + odp_packet_seg_t *seg); >>>> + >>>> +/* >>>> + * >>>> + * Meta-data >>>> + * ******************************************************** >>>> + * >>>> + */ >>>> + >>>> +/** >>>> + * Packet pool >>>> + * >>>> + * Returns handle to the buffer pool where the packet was allocated >>>> from. >>>> + * >>>> + * @param pkt Packet handle >>>> + * >>>> + * @return Buffer pool handle >>>> + */ >>>> +odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt); >>>> + >>>> +/** >>>> + * Packet input interface >>>> + * >>>> + * Returns handle to the packet IO interface which received the packet >>>> or >>>> + * ODP_PKTIO_INVALID when the packet was allocated/reset by the >>>> application. >>>> + * >>>> + * @param pkt Packet handle >>>> + * >>>> + * @return Packet interface handle >>>> + * @retval ODP_PKTIO_INVALID Packet was not received >>>> + */ >>>> +odp_pktio_t odp_packet_input(odp_packet_t pkt); >>>> + >>>> +/** >>>> + * User context pointer >>>> + * >>>> + * Return previously stored user context pointer. >>>> + * >>>> + * @param pkt Packet handle >>>> + * >>>> + * @return User context pointer >>>> + */ >>>> +void *odp_packet_user_ptr(odp_packet_t pkt); >>>> + >>>> +/** >>>> + * Set user context pointer >>>> + * >>>> + * Each packet has room for a user defined context. The context can be >>>> stored >>>> + * either as a pointer OR as a uint64_t value, but not both at the >>>> same time. >>>> + * The latest context set operation determines which one has been >>>> stored. >>>> + * >>>> + * @param pkt Packet handle >>>> + * @param ctx User context pointer >>>> + */ >>>> +void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ctx); >>>> + >>>> +/** >>>> + * User context data (uint64_t) >>>> + * >>>> + * Return previously stored user context uint64_t value. >>>> + * >>>> + * @param pkt Packet handle >>>> + * >>>> + * @return User context data >>>> + */ >>>> +uint64_t odp_packet_user_u64(odp_packet_t pkt); >>>> + >>>> +/** >>>> + * Set user context data (uint64_t) >>>> + * >>>> + * Each packet has room for a user defined context. The context can be >>>> stored >>>> + * either as a pointer OR as a uint64_t value, but not both at the >>>> same time. >>>> + * The latest context set operation determines which one has been >>>> stored. >>>> + * >>>> + * @param pkt Packet handle >>>> + * @param ctx User context data >>>> + */ >>>> +void odp_packet_user_u64_set(odp_packet_t pkt, uint64_t ctx); >>>> + >>>> +/** >>>> + * Layer 2 start pointer >>>> + * >>>> + * Returns pointer to the start of the layer 2 header. Optionally, >>>> outputs >>>> + * number of data bytes in the segment following the pointer. >>>> + * >>>> + * @param pkt Packet handle >>>> + * @param[out] len Number of data bytes remaining in the segment >>>> (output). >>>> + * Ignored when NULL. >>>> + * >>>> + * @return Layer 2 start pointer, or offset 0 by default >>>> + * >>>> + * @see odp_packet_l2_offset(), odp_packet_l2_offset_set() >>>> + */ >>>> +void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len); >>>> + >>>> +/** >>>> + * Layer 2 start offset >>>> + * >>>> + * Returns offset to the start of the layer 2 header. The offset is >>>> calculated >>>> + * from the current odp_packet_data() position in bytes. >>>> + * >>>> + * User is responsible to update the offset when modifying the packet >>>> data >>>> + * pointer position. >>>> + * >>>> + * @param pkt Packet handle >>>> + * >>>> + * @return Layer 2 start offset >>>> + */ >>>> +uint32_t odp_packet_l2_offset(odp_packet_t pkt); >>>> + >>>> +/** >>>> + * Set layer 2 start offset >>>> + * >>>> + * Set offset to the start of the layer 2 header. The offset is >>>> calculated from >>>> + * the current odp_packet_data() position in bytes. Offset must not >>>> exceed >>>> + * packet data length. Packet is not modified on an error. >>>> * >>>> * @param pkt Packet handle >>>> - * @param offset L2 byte offset >>>> + * @param offset Layer 2 start offset (0 ... odp_packet_len()-1) >>>> + * >>>> + * @retval 0 Success >>>> + * @retval Non-zero Failure >>>> */ >>>> -void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset); >>>> +int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset); >>>> >>>> +/** >>>> + * Layer 3 start pointer >>>> + * >>>> + * Returns pointer to the start of the layer 3 header. Optionally, >>>> outputs >>>> + * number of data bytes in the segment following the pointer. >>>> + * >>>> + * @param pkt Packet handle >>>> + * @param[out] len Number of data bytes remaining in the segment >>>> (output). >>>> + * Ignored when NULL. >>>> + * >>>> + * @return Layer 3 start pointer, or NULL >>>> + * >>>> + * @see odp_packet_l3_offset(), odp_packet_l3_offset_set() >>>> + */ >>>> +void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len); >>>> >>>> /** >>>> - * Get pointer to the start of the L3 packet >>>> + * Layer 3 start offset >>>> + * >>>> + * Returns offset to the start of the layer 3 header. The offset is >>>> calculated >>>> + * from the current odp_packet_data() position in bytes. >>>> + * >>>> + * User is responsible to update the offset when modifying the packet >>>> data >>>> + * pointer position. >>>> * >>>> * @param pkt Packet handle >>>> * >>>> - * @return Pointer to L3 packet or NULL if not found >>>> + * @return Layer 3 start offset or ODP_PACKET_OFFSET_INVALID if not >>>> found >>>> + */ >>>> +uint32_t odp_packet_l3_offset(odp_packet_t pkt); >>>> + >>>> +/** >>>> + * Set layer 3 start offset >>>> + * >>>> + * Set offset to the start of the layer 3 header. The offset is >>>> calculated from >>>> + * the current odp_packet_data() position in bytes. Offset must not >>>> exceed >>>> + * packet data length. Packet is not modified on an error. >>>> + * >>>> + * @param pkt Packet handle >>>> + * @param offset Layer 3 start offset (0 ... odp_packet_len()-1) >>>> * >>>> + * @retval 0 Success >>>> + * @retval Non-zero Failure >>>> */ >>>> -uint8_t *odp_packet_l3(odp_packet_t pkt); >>>> +int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset); >>>> >>>> /** >>>> - * Return the byte offset from the packet buffer to the L3 packet >>>> + * Layer 4 start pointer >>>> + * >>>> + * Returns pointer to the start of the layer 4 header. Optionally, >>>> outputs >>>> + * number of data bytes in the segment following the pointer. >>>> + * >>>> + * @param pkt Packet handle >>>> + * @param[out] len Number of data bytes remaining in the segment >>>> (output). >>>> + * Ignored when NULL. >>>> + * >>>> + * @return Layer 4 start pointer, or NULL >>>> + * >>>> + * @see odp_packet_l4_offset(), odp_packet_l4_offset_set() >>>> + */ >>>> +void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len); >>>> + >>>> +/** >>>> + * Layer 4 start offset >>>> + * >>>> + * Returns offset to the start of the layer 4 header. The offset is >>>> calculated >>>> + * from the current odp_packet_data() position in bytes. >>>> + * >>>> + * User is responsible to update the offset when modifying the packet >>>> data >>>> + * pointer position. >>>> * >>>> * @param pkt Packet handle >>>> * >>>> - * @return L3 byte offset or ODP_PACKET_OFFSET_INVALID if not found >>>> + * @return Layer 4 start offset or ODP_PACKET_OFFSET_INVALID if not >>>> found >>>> */ >>>> -size_t odp_packet_l3_offset(odp_packet_t pkt); >>>> +uint32_t odp_packet_l4_offset(odp_packet_t pkt); >>>> >>>> /** >>>> - * Set the byte offset to the L3 packet >>>> + * Set layer 4 start offset >>>> + * >>>> + * Set offset to the start of the layer 4 header. The offset is >>>> calculated from >>>> + * the current odp_packet_data() position in bytes. Offset must not >>>> exceed >>>> + * packet data length. Packet is not modified on an error. >>>> * >>>> * @param pkt Packet handle >>>> - * @param offset L3 byte offset >>>> + * @param offset Layer 4 start offset (0 ... odp_packet_len()-1) >>>> + * >>>> + * @retval 0 Success >>>> + * @retval Non-zero Failure >>>> + */ >>>> +int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset); >>>> + >>>> +/** >>>> + * Tests if packet is segmented >>>> + * >>>> + * @param pkt Packet handle >>>> + * >>>> + * @retval 0 Packet is not segmented >>>> + * @retval 1 Packet is segmented >>>> + */ >>>> +int odp_packet_is_segmented(odp_packet_t pkt); >>>> + >>>> +/** >>>> + * Number of segments >>>> + * >>>> + * Returns number of segments in the packet. A packet has always at >>>> least one >>>> + * segment. >>>> + * >>>> + * @param pkt Packet handle >>>> + * >>>> + * @return Number of segments (>0) >>>> + */ >>>> +int odp_packet_num_segs(odp_packet_t pkt); >>>> + >>>> +/** >>>> + * First segment in packet >>>> + * >>>> + * A packet has always the first segment (has at least one segment). >>>> + * >>>> + * @param pkt Packet handle >>>> + * >>>> + * @return Handle to the first segment >>>> + */ >>>> +odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt); >>>> + >>>> +/** >>>> + * Last segment in packet >>>> + * >>>> + * A packet has always the last segment (has at least one segment). >>>> + * >>>> + * @param pkt Packet handle >>>> + * >>>> + * @return Handle to the last segment >>>> + */ >>>> +odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt); >>>> + >>>> +/** >>>> + * Next segment in packet >>>> + * >>>> + * Returns handle to the next segment after the current segment, or >>>> + * ODP_PACKET_SEG_INVALID if there are no more segments. Use >>>> + * odp_packet_first_seg() to get handle to the first segment. >>>> + * >>>> + * @param pkt Packet handle >>>> + * @param seg Current segment handle >>>> + * >>>> + * @return Handle to the next segment, or ODP_PACKET_SEG_INVALID >>>> + */ >>>> +odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, >>>> odp_packet_seg_t seg); >>>> + >>>> + >>>> +/* >>>> + * >>>> + * Segment level >>>> + * ******************************************************** >>>> + * >>>> */ >>>> -void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset); >>>> >>>> +/** >>>> + * Segment buffer address >>>> + * >>>> + * Returns start address of the segment. >>>> + * >>>> + * @param pkt Packet handle >>>> + * @param seg Segment handle >>>> + * >>>> + * @return Start address of the segment, or NULL on an error >>>> + * >>>> + * @see odp_packet_seg_buf_len() >>>> + */ >>>> +void *odp_packet_seg_buf_addr(odp_packet_t pkt, odp_packet_seg_t seg); >>>> >>>> /** >>>> - * Get pointer to the start of the L4 packet >>>> + * Segment buffer length >>>> + * >>>> + * Returns segment buffer length in bytes. >>>> * >>>> * @param pkt Packet handle >>>> + * @param seg Segment handle >>>> * >>>> - * @return Pointer to L4 packet or NULL if not found >>>> + * @return Segment buffer length in bytes >>>> * >>>> + * @see odp_packet_seg_buf_addr() >>>> */ >>>> -uint8_t *odp_packet_l4(odp_packet_t pkt); >>>> +uint32_t odp_packet_seg_buf_len(odp_packet_t pkt, odp_packet_seg_t >>>> seg); >>>> >>>> /** >>>> - * Return the byte offset from the packet buffer to the L4 packet >>>> + * Segment data pointer >>>> + * >>>> + * Returns pointer to the first byte of data in the segment. >>>> + * >>>> + * @param pkt Packet handle >>>> + * @param seg Segment handle >>>> + * >>>> + * @return Pointer to the segment data, or NULL on an error >>>> + * >>>> + * @see odp_packet_seg_data_len() >>>> + */ >>>> +void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg); >>>> + >>>> +/** >>>> + * Segment data length >>>> + * >>>> + * Returns segment data length in bytes. >>>> * >>>> * @param pkt Packet handle >>>> + * @param seg Segment handle >>>> * >>>> - * @return L4 byte offset or ODP_PACKET_OFFSET_INVALID if not found >>>> + * @return Segment data length in bytes >>>> + * >>>> + * @see odp_packet_seg_data() >>>> */ >>>> -size_t odp_packet_l4_offset(odp_packet_t pkt); >>>> +uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t >>>> seg); >>>> + >>>> + >>>> +/* >>>> + * >>>> + * Manipulation >>>> + * ******************************************************** >>>> + * >>>> + */ >>>> + >>>> >>>> /** >>>> - * Set the byte offset to the L4 packet >>>> + * Add data into an offset >>>> + * >>>> + * Increases packet data length by adding new data area into the >>>> specified >>>> + * offset. The operation returns a new packet handle on success. It >>>> may modify >>>> + * packet segmentation and move data. Handles and pointers must be >>>> updated >>>> + * after the operation. User is responsible to update packet meta-data >>>> offsets >>>> + * when needed. The packet is not modified on an error. >>>> * >>>> * @param pkt Packet handle >>>> - * @param offset L4 byte offset >>>> + * @param offset Byte offset into the packet >>>> + * @param len Number of bytes to add into the offset >>>> + * >>>> + * @return New packet handle, or ODP_PACKET_INVALID in case of an >>>> error. >>>> */ >>>> -void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset); >>>> +odp_packet_t odp_packet_add_data(odp_packet_t pkt, uint32_t offset, >>>> + uint32_t len); >>>> >>>> /** >>>> - * Print (debug) information about the packet >>>> + * Remove data from an offset >>>> + * >>>> + * Decreases packet data length by removing data from the specified >>>> offset. >>>> + * The operation returns a new packet handle on success, and may modify >>>> + * packet segmentation and move data. Handles and pointers must be >>>> updated >>>> + * after the operation. User is responsible to update packet meta-data >>>> offsets >>>> + * when needed. The packet is not modified on an error. >>>> + * >>>> + * @param pkt Packet handle >>>> + * @param offset Byte offset into the packet >>>> + * @param len Number of bytes to remove from the offset >>>> + * >>>> + * @return New packet handle, or ODP_PACKET_INVALID in case of an >>>> error. >>>> + */ >>>> +odp_packet_t odp_packet_rem_data(odp_packet_t pkt, uint32_t offset, >>>> + uint32_t len); >>>> + >>>> + >>>> +/* >>>> + * >>>> + * Copy >>>> + * ******************************************************** >>>> + * >>>> + */ >>>> + >>>> +/** >>>> + * Copy packet >>>> + * >>>> + * Create a new copy of the packet. The new packet is exact copy of >>>> the source >>>> + * packet (incl. data and meta-data). The pool must have been created >>>> with >>>> + * buffer type ODP_BUFFER_TYPE_PACKET. >>>> + * >>>> + * @param pkt Packet handle >>>> + * @param pool Buffer pool for allocation of the new packet. >>>> + * >>>> + * @return Handle to the copy of the packet, or ODP_PACKET_INVALID >>>> + */ >>>> +odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool); >>>> + >>>> +/** >>>> + * Copy data from packet >>>> + * >>>> + * Copy 'len' bytes of data from the packet level offset to the >>>> destination >>>> + * address. >>>> + * >>>> + * @param pkt Packet handle >>>> + * @param offset Byte offset into the packet >>>> + * @param len Number of bytes to copy >>>> + * @param dst Destination address >>>> + * >>>> + * @retval 0 Success >>>> + * @retval Non-zero Failure >>>> + */ >>>> +int odp_packet_copydata_out(odp_packet_t pkt, uint32_t offset, >>>> + uint32_t len, void *dst); >>>> + >>>> +/** >>>> + * Copy data into packet >>>> + * >>>> + * Copy 'len' bytes of data from the source address into the packet >>>> level >>>> + * offset. Maximum number of bytes to copy is packet data length minus >>>> the >>>> + * offset. Packet is not modified on an error. >>>> + * >>>> + * @param pkt Packet handle >>>> + * @param offset Byte offset into the packet >>>> + * @param len Number of bytes to copy >>>> + * @param src Source address >>>> + * >>>> + * @retval 0 Success >>>> + * @retval Non-zero Failure >>>> + */ >>>> +int odp_packet_copydata_in(odp_packet_t pkt, uint32_t offset, >>>> + uint32_t len, const void *src); >>>> + >>>> +/* >>>> + * >>>> + * Debugging >>>> + * ******************************************************** >>>> + * >>>> + */ >>>> + >>>> +/** >>>> + * Print packet to the console >>>> + * >>>> + * Print all packet debug information to the console. >>>> * >>>> * @param pkt Packet handle >>>> */ >>>> void odp_packet_print(odp_packet_t pkt); >>>> >>>> /** >>>> - * Copy contents and metadata from pkt_src to pkt_dst >>>> - * Useful when creating copies of packets >>>> + * Perform full packet validity check >>>> * >>>> - * @param pkt_dst Destination packet >>>> - * @param pkt_src Source packet >>>> + * The operation may consume considerable number of cpu cycles >>>> depending on >>>> + * the check level. >>>> + * >>>> + * @param pkt Packet handle >>>> * >>>> - * @return 0 if successful >>>> + * @retval 0 Packet is not valid >>>> + * @retval 1 Packet is valid >>>> + */ >>>> +int odp_packet_is_valid(odp_packet_t pkt); >>>> + >>>> + >>>> +/** >>>> + * @} >>>> */ >>>> -int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src); >>>> >>>> #ifdef __cplusplus >>>> } >>>> diff --git a/platform/linux-dpdk/include/api/odp_packet_io.h >>>> b/platform/linux-dpdk/include/api/odp_packet_io.h >>>> index bd6868a..787df7d 100644 >>>> --- a/platform/linux-dpdk/include/api/odp_packet_io.h >>>> +++ b/platform/linux-dpdk/include/api/odp_packet_io.h >>>> @@ -1,4 +1,4 @@ >>>> -/* Copyright (c) 2014, Linaro Limited >>>> +/* Copyright (c) 2013, Linaro Limited >>>> * All rights reserved. >>>> * >>>> * SPDX-License-Identifier: BSD-3-Clause >>>> @@ -19,29 +19,29 @@ extern "C" { >>>> #endif >>>> >>>> #include <odp_std_types.h> >>>> +#include <odp_platform_types.h> >>>> #include <odp_buffer_pool.h> >>>> #include <odp_packet.h> >>>> #include <odp_queue.h> >>>> >>>> -#include <odp_pktio_types.h> >>>> - >>>> -/** ODP packet IO handle */ >>>> -typedef uint32_t odp_pktio_t; >>>> - >>>> -/** Invalid packet IO handle */ >>>> -#define ODP_PKTIO_INVALID 0 >>>> +/** @defgroup odp_packet_io ODP PACKET IO >>>> + * Operations on a packet. >>>> + * @{ >>>> + */ >>>> >>>> /** >>>> * Open an ODP packet IO instance >>>> * >>>> * @param dev Packet IO device >>>> * @param pool Pool to use for packet IO >>>> - * @param params Set of parameters to pass to the arch dependent >>>> implementation >>>> * >>>> * @return ODP packet IO handle or ODP_PKTIO_INVALID on error >>>> + * >>>> + * @note dev name loop is specially pktio reserved name for >>>> + * device used for testing. Usually it's loop back >>>> + * interface. >>>> */ >>>> -odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool, >>>> - odp_pktio_params_t *params); >>>> +odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool); >>>> >>>> /** >>>> * Close an ODP packet IO instance >>>> @@ -130,21 +130,77 @@ void odp_pktio_set_input(odp_packet_t pkt, >>>> odp_pktio_t id); >>>> odp_pktio_t odp_pktio_get_input(odp_packet_t pkt); >>>> >>>> /** >>>> - * Defines the maximum length of mac address supported by this platform >>>> + * Configure the MTU for a packet IO interface. >>>> + * >>>> + * @param[in] id ODP packet IO handle. >>>> + * @param[in] mtu The value of MTU that the interface will be >>>> configured to >>>> + * use. >>>> + * >>>> + * @retval 0 on success. >>>> + * @retval -1 if specified mtu can not be handled. >>>> + * @retval -1 on any other error or illegal input parameters. >>>> + */ >>>> +int odp_pktio_set_mtu(odp_pktio_t id, int mtu); >>>> + >>>> +/** >>>> + * Return the currently configured MTU value of a packet IO interface. >>>> + * >>>> + * @param[in] id ODP packet IO handle. >>>> + * >>>> + * @retval MTU value >0 on success. >>>> + * @retval -1 on any error or not existance pktio id. >>>> + */ >>>> +int odp_pktio_mtu(odp_pktio_t id); >>>> + >>>> +/** >>>> + * Enable/Disable promiscuous mode on a packet IO interface. >>>> + * >>>> + * @param[in] id ODP packet IO handle. >>>> + * @param[in] enable 1 to enable, 0 to disable. >>>> + * >>>> + * @retval 0 on success. >>>> + * @retval non-zero on any error. >>>> + */ >>>> +int odp_pktio_promisc_mode_set(odp_pktio_t id, odp_bool_t enable); >>>> + >>>> +/** >>>> + * Determine if promiscuous mode is enabled for a packet IO interface. >>>> + * >>>> + * @param[in] id ODP packet IO handle. >>>> + * >>>> + * @retval 1 if promiscuous mode is enabled. >>>> + * @retval 0 if promiscuous mode is disabled. >>>> + * @retval -1 on any error. >>>> +*/ >>>> +int odp_pktio_promisc_mode(odp_pktio_t id); >>>> + >>>> +/** >>>> + * Get the default MAC address of a packet IO interface. >>>> + * >>>> + * @param id ODP packet IO handle. >>>> + * @param[out] mac_addr Storage for MAC address of the packet IO >>>> interface. >>>> + * @param addr_size Storage size for the address >>>> + * >>>> + * @retval Number of bytes written on success, 0 on failure. >>>> */ >>>> -#define ODP_MAC_ADDR_MAX_LENGTH ETH_ALEN >>>> +size_t odp_pktio_mac_addr(odp_pktio_t id, void *mac_addr, >>>> + size_t addr_size); >>>> >>>> /** >>>> * Get mac address of the interface >>>> * >>>> - * @param id ODP packet IO handle >>>> - * @param mac_addr Storage for Mac address of the packet IO >>>> interface >>>> - * Storage provided by the caller should be equal >>>> - * to ODP_MAC_ADDR_MAX_LENGTH (filled by function) >>>> + * @param id ODP packet IO handle >>>> + * @param mac_addr Storage for Mac address of the packet IO >>>> interface >>>> + * Storage provided by the caller should be equal >>>> + * to ODP_MAC_ADDR_MAX_LENGTH (filled by function) >>>> * @return 0 on success or -1 on error >>>> -**/ >>>> + */ >>>> int odp_pktio_get_mac_addr(odp_pktio_t id, unsigned char *mac_addr); >>>> >>>> +/** >>>> + * @} >>>> + */ >>>> + >>>> #ifdef __cplusplus >>>> } >>>> #endif >>>> diff --git a/platform/linux-dpdk/include/api/odp_pktio_types.h >>>> b/platform/linux-dpdk/include/api/odp_pktio_types.h >>>> deleted file mode 100644 >>>> index b23e6da..0000000 >>>> --- a/platform/linux-dpdk/include/api/odp_pktio_types.h >>>> +++ /dev/null >>>> @@ -1,45 +0,0 @@ >>>> - >>>> -/* Copyright (c) 2013, Linaro Limited >>>> - * All rights reserved. >>>> - * >>>> - * SPDX-License-Identifier: BSD-3-Clause >>>> - */ >>>> - >>>> -#ifndef ODP_PKTIO_TYPES_H >>>> -#define ODP_PKTIO_TYPES_H >>>> - >>>> -#ifdef __cplusplus >>>> -extern "C" { >>>> -#endif >>>> - >>>> -/* We should ensure that future enum values will never overlap, >>>> otherwise >>>> - * applications that want netmap suport might get in trouble if the >>>> odp lib >>>> - * was not built with netmap support and there are more types define >>>> below >>>> - */ >>>> - >>>> -typedef enum { >>>> - ODP_PKTIO_TYPE_SOCKET_BASIC = 0x1, >>>> - ODP_PKTIO_TYPE_SOCKET_MMSG, >>>> - ODP_PKTIO_TYPE_SOCKET_MMAP, >>>> - ODP_PKTIO_TYPE_NETMAP, >>>> - ODP_PKTIO_TYPE_DPDK, >>>> -} odp_pktio_type_t; >>>> - >>>> -#include <odp_pktio_socket.h> >>>> -#ifdef ODP_HAVE_NETMAP >>>> -#include <odp_pktio_netmap.h> >>>> -#endif >>>> - >>>> -typedef union odp_pktio_params_t { >>>> - odp_pktio_type_t type; >>>> - socket_params_t sock_params; >>>> -#ifdef ODP_HAVE_NETMAP >>>> - netmap_params_t nm_params; >>>> -#endif >>>> -} odp_pktio_params_t; >>>> - >>>> -#ifdef __cplusplus >>>> -} >>>> -#endif >>>> - >>>> -#endif >>>> diff --git a/platform/linux-dpdk/include/api/odp_platform_types.h >>>> b/platform/linux-dpdk/include/api/odp_platform_types.h >>>> new file mode 100644 >>>> index 0000000..3bedb5d >>>> --- /dev/null >>>> +++ b/platform/linux-dpdk/include/api/odp_platform_types.h >>>> @@ -0,0 +1,78 @@ >>>> +/* Copyright (c) 2014, Linaro Limited >>>> + * All rights reserved. >>>> + * >>>> + * SPDX-License-Identifier: BSD-3-Clause >>>> + */ >>>> + >>>> + >>>> +/** >>>> + * @file >>>> + * ODP implementation types >>>> + * This file contains all of the implementation-defined types for ODP >>>> abstract >>>> + * definitions. Having this in one file means that other ODP API files >>>> are >>>> + * implementation-independent and avoids circular dependencies for >>>> files that >>>> + * refer to types managed by other components. Included here are >>>> typedefs and >>>> + * related typed constants that are referenced by other ODP API files. >>>> + */ >>>> + >>>> +#ifndef ODP_IMPL_TYPES_H_ >>>> +#define ODP_IMPL_TYPES_H_ >>>> + >>>> +/** @defgroup odp_platform_types ODP PLATFORM TYPES >>>> + * Implementation specific definitions for ODP abstract types. >>>> + * @{ >>>> + */ >>>> + >>>> +/** ODP Buffer pool */ >>>> +typedef unsigned long odp_buffer_pool_t; >>>> + >>>> +/** Invalid buffer pool */ >>>> +#define ODP_BUFFER_POOL_INVALID (0xffffffff) >>>> + >>>> +/** ODP buffer */ >>>> +typedef unsigned long odp_buffer_t; >>>> + >>>> +/** Invalid buffer */ >>>> +#define ODP_BUFFER_INVALID (0xffffffff) >>>> + >>>> +/** ODP buffer segment */ >>>> +typedef odp_buffer_t odp_buffer_seg_t; >>>> + >>>> +/** Invalid segment */ >>>> +#define ODP_SEGMENT_INVALID ODP_BUFFER_INVALID >>>> + >>>> +/** ODP packet */ >>>> +typedef odp_buffer_t odp_packet_t; >>>> + >>>> +/** Invalid packet */ >>>> +#define ODP_PACKET_INVALID ODP_BUFFER_INVALID >>>> + >>>> +/** ODP packet segment */ >>>> +typedef odp_buffer_t odp_packet_seg_t; >>>> + >>>> +/** Invalid packet segment */ >>>> +#define ODP_PACKET_SEG_INVALID ODP_BUFFER_INVALID >>>> + >>>> +/** ODP packet IO handle */ >>>> +typedef uint32_t odp_pktio_t; >>>> + >>>> +/** Invalid packet IO handle */ >>>> +#define ODP_PKTIO_INVALID 0 >>>> + >>>> +/** odp_pktio_t value to indicate any port */ >>>> +#define ODP_PKTIO_ANY ((odp_pktio_t)~0) >>>> + >>>> +/** >>>> + * ODP shared memory block >>>> + */ >>>> +typedef uint32_t odp_shm_t; >>>> + >>>> +/** Invalid shared memory block */ >>>> +#define ODP_SHM_INVALID 0 >>>> +#define ODP_SHM_NULL ODP_SHM_INVALID /**< Synonym for buffer pool use >>>> */ >>>> + >>>> +/** >>>> + * @} >>>> + */ >>>> + >>>> +#endif >>>> diff --git a/platform/linux-dpdk/include/odp_buffer_inlines.h >>>> b/platform/linux-dpdk/include/odp_buffer_inlines.h >>>> new file mode 100644 >>>> index 0000000..ebf600b >>>> --- /dev/null >>>> +++ b/platform/linux-dpdk/include/odp_buffer_inlines.h >>>> @@ -0,0 +1,179 @@ >>>> +/* Copyright (c) 2014, Linaro Limited >>>> + * All rights reserved. >>>> + * >>>> + * SPDX-License-Identifier: BSD-3-Clause >>>> + */ >>>> + >>>> +/** >>>> + * @file >>>> + * >>>> + * Inline functions for ODP buffer mgmt routines - implementation >>>> internal >>>> + */ >>>> + >>>> +#ifndef ODP_BUFFER_INLINES_H_ >>>> +#define ODP_BUFFER_INLINES_H_ >>>> + >>>> +#ifdef __cplusplus >>>> +extern "C" { >>>> +#endif >>>> + >>>> +static inline odp_buffer_t odp_buffer_encode_handle(odp_buffer_hdr_t >>>> *hdr) >>>> +{ >>>> + odp_buffer_bits_t handle; >>>> + uint32_t pool_id = pool_handle_to_index(hdr->pool_hdl); >>>> + struct pool_entry_s *pool = get_pool_entry(pool_id); >>>> + >>>> + handle.pool_id = pool_id; >>>> + handle.index = ((uint8_t *)hdr - pool->pool_mdata_addr) / >>>> + ODP_CACHE_LINE_SIZE; >>>> + handle.seg = 0; >>>> + >>>> + return handle.u32; >>>> +} >>>> + >>>> +static inline odp_buffer_t odp_hdr_to_buf(odp_buffer_hdr_t *hdr) >>>> +{ >>>> + return hdr->handle.handle; >>>> +} >>>> + >>>> +static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf) >>>> +{ >>>> + return (odp_buffer_hdr_t *)buf; >>>> +} >>>> + >>>> +static inline uint32_t odp_buffer_refcount(odp_buffer_hdr_t *buf) >>>> +{ >>>> + return odp_atomic_load_u32(&buf->ref_count); >>>> +} >>>> + >>>> +static inline uint32_t odp_buffer_incr_refcount(odp_buffer_hdr_t *buf, >>>> + uint32_t val) >>>> +{ >>>> + return odp_atomic_fetch_add_u32(&buf->ref_count, val) + val; >>>> +} >>>> + >>>> +static inline uint32_t odp_buffer_decr_refcount(odp_buffer_hdr_t *buf, >>>> + uint32_t val) >>>> +{ >>>> + uint32_t tmp; >>>> + >>>> + tmp = odp_atomic_fetch_sub_u32(&buf->ref_count, val); >>>> + >>>> + if (tmp < val) { >>>> + odp_atomic_fetch_add_u32(&buf->ref_count, val - tmp); >>>> + return 0; >>>> + } else { >>>> + return tmp - val; >>>> + } >>>> +} >>>> + >>>> +static inline odp_buffer_hdr_t *validate_buf(odp_buffer_t buf) >>>> +{ >>>> + odp_buffer_bits_t handle; >>>> + odp_buffer_hdr_t *buf_hdr; >>>> + handle.u32 = buf; >>>> + >>>> + /* For buffer handles, segment index must be 0 and pool id in >>>> range */ >>>> + if (handle.seg != 0 || handle.pool_id >= >>>> ODP_CONFIG_BUFFER_POOLS) >>>> + return NULL; >>>> + >>>> + pool_entry_t *pool = odp_pool_to_entry(handle.pool_id); >>>> + >>>> + /* If pool not created, handle is invalid */ >>>> + if (pool->s.pool_shm == ODP_SHM_INVALID) >>>> + return NULL; >>>> + >>>> + uint32_t buf_stride = pool->s.buf_stride / ODP_CACHE_LINE_SIZE; >>>> + >>>> + /* A valid buffer index must be on stride, and must be in range >>>> */ >>>> + if ((handle.index % buf_stride != 0) || >>>> + ((uint32_t)(handle.index / buf_stride) >= >>>> pool->s.params.num_bufs)) >>>> + return NULL; >>>> + >>>> + buf_hdr = (odp_buffer_hdr_t *)(void *) >>>> + (pool->s.pool_mdata_addr + >>>> + (handle.index * ODP_CACHE_LINE_SIZE)); >>>> + >>>> + /* Handle is valid, so buffer is valid if it is allocated */ >>>> + return buf_hdr->allocator == ODP_FREEBUF ? NULL : buf_hdr; >>>> +} >>>> + >>>> +int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf); >>>> + >>>> +static inline void *buffer_map(odp_buffer_hdr_t *buf, >>>> + uint32_t offset, >>>> + uint32_t *seglen, >>>> + uint32_t limit) >>>> +{ >>>> + int seg_index = offset / buf->segsize; >>>> + int seg_offset = offset % buf->segsize; >>>> + >>>> + if (seglen != NULL) { >>>> + uint32_t buf_left = limit - offset; >>>> + *seglen = buf_left < buf->segsize ? >>>> + buf_left : buf->segsize - seg_offset; >>>> + } >>>> + >>>> + return (void *)(seg_offset + (uint8_t *)buf->addr[seg_index]); >>>> +} >>>> + >>>> +static inline odp_buffer_seg_t segment_next(odp_buffer_hdr_t *buf, >>>> + odp_buffer_seg_t seg) >>>> +{ >>>> + odp_buffer_bits_t seghandle; >>>> + seghandle.u32 = seg; >>>> + >>>> + if (seg == ODP_SEGMENT_INVALID || >>>> + seghandle.prefix != buf->handle.prefix || >>>> + seghandle.seg >= buf->segcount - 1) >>>> + return ODP_SEGMENT_INVALID; >>>> + else { >>>> + seghandle.seg++; >>>> + return (odp_buffer_seg_t)seghandle.u32; >>>> + } >>>> +} >>>> + >>>> +static inline void *segment_map(odp_buffer_hdr_t *buf, >>>> + odp_buffer_seg_t seg, >>>> + uint32_t *seglen, >>>> + uint32_t limit, >>>> + uint32_t hr) >>>> +{ >>>> + uint32_t seg_offset, buf_left; >>>> + odp_buffer_bits_t seghandle; >>>> + uint8_t *seg_addr; >>>> + seghandle.u32 = seg; >>>> + >>>> + if (seghandle.prefix != buf->handle.prefix || >>>> + seghandle.seg >= buf->segcount) >>>> + return NULL; >>>> + >>>> + seg_addr = (uint8_t *)buf->addr[seghandle.seg]; >>>> + seg_offset = seghandle.seg * buf->segsize; >>>> + limit += hr; >>>> + >>>> + /* Can't map this segment if it's nothing but headroom or >>>> tailroom */ >>>> + if (hr >= seg_offset + buf->segsize || seg_offset > limit) >>>> + return NULL; >>>> + >>>> + /* Adjust address & offset if this segment contains any >>>> headroom */ >>>> + if (hr > seg_offset) { >>>> + seg_addr += hr % buf->segsize; >>>> + seg_offset += hr % buf->segsize; >>>> + } >>>> + >>>> + /* Set seglen if caller is asking for it */ >>>> + if (seglen != NULL) { >>>> + buf_left = limit - seg_offset; >>>> + *seglen = buf_left < buf->segsize ? buf_left : >>>> buf->segsize; >>>> + } >>>> + >>>> + return (void *)seg_addr; >>>> +} >>>> + >>>> + >>>> +#ifdef __cplusplus >>>> +} >>>> +#endif >>>> + >>>> +#endif >>>> diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h >>>> b/platform/linux-dpdk/include/odp_buffer_internal.h >>>> index 5406606..acc958b 100644 >>>> --- a/platform/linux-dpdk/include/odp_buffer_internal.h >>>> +++ b/platform/linux-dpdk/include/odp_buffer_internal.h >>>> @@ -24,50 +24,144 @@ extern "C" { >>>> #include <odp_buffer.h> >>>> #include <odp_debug.h> >>>> #include <odp_align.h> >>>> -#include <rte_mbuf.h> >>>> +#include <odp_align_internal.h> >>>> +#include <odp_config.h> >>>> +#include <odp_byteorder.h> >>>> +#include <odp_thread.h> >>>> >>>> -/* TODO: move these to correct files */ >>>> +/* DPDK */ >>>> +#include <rte_mbuf.h> >>>> >>>> -typedef uint64_t odp_phys_addr_t; >>>> +#define ODP_BITSIZE(x) \ >>>> + ((x) <= 2 ? 1 : \ >>>> + ((x) <= 4 ? 2 : \ >>>> + ((x) <= 8 ? 3 : \ >>>> + ((x) <= 16 ? 4 : \ >>>> + ((x) <= 32 ? 5 : \ >>>> + ((x) <= 64 ? 6 : \ >>>> + ((x) <= 128 ? 7 : \ >>>> + ((x) <= 256 ? 8 : \ >>>> + ((x) <= 512 ? 9 : \ >>>> + ((x) <= 1024 ? 10 : \ >>>> + ((x) <= 2048 ? 11 : \ >>>> + ((x) <= 4096 ? 12 : \ >>>> + ((x) <= 8196 ? 13 : \ >>>> + ((x) <= 16384 ? 14 : \ >>>> + ((x) <= 32768 ? 15 : \ >>>> + ((x) <= 65536 ? 16 : \ >>>> + (0/0))))))))))))))))) >>>> + >>>> +_ODP_STATIC_ASSERT(ODP_CONFIG_PACKET_BUF_LEN_MIN >= 256, >>>> + "ODP Segment size must be a minimum of 256 bytes"); >>>> + >>>> +_ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MIN % >>>> ODP_CACHE_LINE_SIZE) == 0, >>>> + "ODP Segment size must be a multiple of cache line >>>> size"); >>>> + >>>> +_ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MAX % >>>> + ODP_CONFIG_PACKET_BUF_LEN_MIN) == 0, >>>> + "Packet max size must be a multiple of segment >>>> size"); >>>> + >>>> +#define ODP_BUFFER_MAX_SEG \ >>>> + (ODP_CONFIG_PACKET_BUF_LEN_MAX / ODP_CONFIG_PACKET_BUF_LEN_MIN) >>>> + >>>> +/* We can optimize storage of small raw buffers within metadata area */ >>>> +#define ODP_MAX_INLINE_BUF ((sizeof(void *)) * (ODP_BUFFER_MAX_SEG >>>> - 1)) >>>> + >>>> +#define ODP_BUFFER_POOL_BITS ODP_BITSIZE(ODP_CONFIG_BUFFER_POOLS) >>>> +#define ODP_BUFFER_SEG_BITS ODP_BITSIZE(ODP_BUFFER_MAX_SEG) >>>> +#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS - >>>> ODP_BUFFER_SEG_BITS) >>>> +#define ODP_BUFFER_PREFIX_BITS (ODP_BUFFER_POOL_BITS + >>>> ODP_BUFFER_INDEX_BITS) >>>> +#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS) >>>> +#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS) >>>> >>>> #define ODP_BUFFER_MAX_INDEX (ODP_BUFFER_MAX_BUFFERS - 2) >>>> #define ODP_BUFFER_INVALID_INDEX (ODP_BUFFER_MAX_BUFFERS - 1) >>>> >>>> -#define ODP_BUFS_PER_CHUNK 16 >>>> -#define ODP_BUFS_PER_SCATTER 4 >>>> - >>>> -#define ODP_BUFFER_TYPE_CHUNK 0xffff >>>> - >>>> - >>>> -#define ODP_BUFFER_POOL_BITS 4 >>>> -#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS) >>>> -#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS) >>>> -#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS) >>>> - >>>> typedef union odp_buffer_bits_t { >>>> uint32_t u32; >>>> odp_buffer_t handle; >>>> >>>> struct { >>>> - uint32_t pool:ODP_BUFFER_POOL_BITS; >>>> +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN >>>> + uint32_t pool_id:ODP_BUFFER_POOL_BITS; >>>> + uint32_t index:ODP_BUFFER_INDEX_BITS; >>>> + uint32_t seg:ODP_BUFFER_SEG_BITS; >>>> +#else >>>> + uint32_t seg:ODP_BUFFER_SEG_BITS; >>>> uint32_t index:ODP_BUFFER_INDEX_BITS; >>>> + uint32_t pool_id:ODP_BUFFER_POOL_BITS; >>>> +#endif >>>> }; >>>> -} odp_buffer_bits_t; >>>> >>>> + struct { >>>> +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN >>>> + uint32_t prefix:ODP_BUFFER_PREFIX_BITS; >>>> + uint32_t pfxseg:ODP_BUFFER_SEG_BITS; >>>> +#else >>>> + uint32_t pfxseg:ODP_BUFFER_SEG_BITS; >>>> + uint32_t prefix:ODP_BUFFER_PREFIX_BITS; >>>> +#endif >>>> + }; >>>> +} odp_buffer_bits_t; >>>> >>>> /* forward declaration */ >>>> struct odp_buffer_hdr_t; >>>> >>>> - >>>> +/* Common buffer header */ >>>> typedef struct odp_buffer_hdr_t { >>>> + /* DPDK specific */ >>>> struct rte_mbuf mb; /* Underlying DPDK rte_mbuf */ >>>> - struct odp_buffer_hdr_t *next; /* Next buf in a list */ >>>> - int type; /* ODP buffer type; not DPDK buf >>>> type */ >>>> uint32_t index; /* Index in the rte_mempool */ >>>> + >>>> + struct odp_buffer_hdr_t *next; /* next buf in a list */ >>>> + int allocator; /* allocating thread id */ >>>> + odp_buffer_bits_t handle; /* handle */ >>>> + union { >>>> + uint32_t all; >>>> + struct { >>>> + uint32_t zeroized:1; /* Zeroize buf data on >>>> free */ >>>> + uint32_t hdrdata:1; /* Data is in buffer hdr */ >>>> + }; >>>> + } flags; >>>> + int type; /* buffer type */ >>>> + size_t size; /* max data size */ >>>> + odp_atomic_u32_t ref_count; /* reference count */ >>>> + odp_buffer_pool_t pool_hdl; /* buffer pool handle */ >>>> + union { >>>> + uint64_t buf_u64; /* user u64 */ >>>> + void *buf_ctx; /* user context */ >>>> + const void *buf_cctx; /* const alias for ctx */ >>>> + void *udata_addr; /* user metadata addr */ >>>> + }; >>>> + size_t udata_size; /* size of user metadata */ >>>> + uint32_t segcount; /* segment count */ >>>> + uint32_t segsize; /* segment size */ >>>> + void *addr[ODP_BUFFER_MAX_SEG]; /* block >>>> addrs */ >>>> + >>>> } odp_buffer_hdr_t; >>>> >>>> -int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf); >>>> +typedef struct odp_buffer_hdr_stride { >>>> + uint8_t >>>> pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_buffer_hdr_t))]; >>>> +} odp_buffer_hdr_stride; >>>> +/* Ensure next header starts from 8 byte align */ >>>> +_ODP_STATIC_ASSERT((sizeof(odp_buffer_hdr_t) % 8) == 0, >>>> + "ODP_BUFFER_HDR_T__SIZE_ERROR"); >>>> + >>>> +typedef struct odp_buf_blk_t { >>>> + struct odp_buf_blk_t *next; >>>> + struct odp_buf_blk_t *prev; >>>> +} odp_buf_blk_t; >>>> + >>>> +/* Raw buffer header */ >>>> +typedef struct { >>>> + odp_buffer_hdr_t buf_hdr; /* common buffer header */ >>>> +} odp_raw_buffer_hdr_t; >>>> + >>>> +/* Free buffer marker */ >>>> +#define ODP_FREEBUF -1 >>>> >>>> +/* Forward declarations */ >>>> +odp_buffer_t buffer_alloc(odp_buffer_pool_t pool, size_t size); >>>> >>>> #ifdef __cplusplus >>>> } >>>> diff --git a/platform/linux-dpdk/include/odp_buffer_pool_internal.h >>>> b/platform/linux-dpdk/include/odp_buffer_pool_internal.h >>>> index 1a36655..0f19f87 100644 >>>> --- a/platform/linux-dpdk/include/odp_buffer_pool_internal.h >>>> +++ b/platform/linux-dpdk/include/odp_buffer_pool_internal.h >>>> @@ -19,16 +19,48 @@ extern "C" { >>>> #endif >>>> >>>> #include <odp_std_types.h> >>>> +#include <odp_align.h> >>>> +#include <odp_align_internal.h> >>>> #include <odp_buffer_pool.h> >>>> #include <odp_buffer_internal.h> >>>> -#include <odp_align.h> >>>> #include <odp_hints.h> >>>> #include <odp_config.h> >>>> #include <odp_debug.h> >>>> +#include <odp_shared_memory.h> >>>> +#include <odp_atomic.h> >>>> +#include <odp_atomic_internal.h> >>>> +#include <string.h> >>>> >>>> -/* for DPDK */ >>>> +/* DPDK */ >>>> #include <rte_mempool.h> >>>> >>>> +/** >>>> + * Buffer initialization routine prototype >>>> + * >>>> + * @note Routines of this type MAY be passed as part of the >>>> + * _odp_buffer_pool_init_t structure to be called whenever a >>>> + * buffer is allocated to initialize the user metadata >>>> + * associated with that buffer. >>>> + */ >>>> +typedef void (_odp_buf_init_t)(odp_buffer_t buf, void *buf_init_arg); >>>> + >>>> +/** >>>> + * Buffer pool initialization parameters >>>> + * Used to communicate buffer pool initialization options. Internal >>>> for now. >>>> + */ >>>> +typedef struct _odp_buffer_pool_init_t { >>>> + size_t udata_size; /**< Size of user metadata for each >>>> buffer */ >>>> + _odp_buf_init_t *buf_init; /**< Buffer initialization routine >>>> to use */ >>>> + void *buf_init_arg; /**< Argument to be passed to >>>> buf_init() */ >>>> +} _odp_buffer_pool_init_t; /**< Type of buffer initialization >>>> struct */ >>>> + >>>> +/* Local cache for buffer alloc/free acceleration */ >>>> +typedef struct local_cache_t { >>>> + odp_buffer_hdr_t *buf_freelist; /* The local cache */ >>>> + uint64_t bufallocs; /* Local buffer alloc count */ >>>> + uint64_t buffrees; /* Local buffer free count */ >>>> +} local_cache_t; >>>> + >>>> /* Use ticketlock instead of spinlock */ >>>> #define POOL_USE_TICKETLOCK >>>> >>>> @@ -38,11 +70,16 @@ extern "C" { >>>> >>>> #ifdef POOL_USE_TICKETLOCK >>>> #include <odp_ticketlock.h> >>>> +#define POOL_LOCK(a) odp_ticketlock_lock(a) >>>> +#define POOL_UNLOCK(a) odp_ticketlock_unlock(a) >>>> +#define POOL_LOCK_INIT(a) odp_ticketlock_init(a) >>>> #else >>>> #include <odp_spinlock.h> >>>> +#define POOL_LOCK(a) odp_spinlock_lock(a) >>>> +#define POOL_UNLOCK(a) odp_spinlock_unlock(a) >>>> +#define POOL_LOCK_INIT(a) odp_spinlock_init(a) >>>> #endif >>>> >>>> - >>>> struct pool_entry_s { >>>> #ifdef POOL_USE_TICKETLOCK >>>> odp_ticketlock_t lock ODP_ALIGNED_CACHE; >>>> @@ -50,38 +87,294 @@ struct pool_entry_s { >>>> odp_spinlock_t lock ODP_ALIGNED_CACHE; >>>> #endif >>>> >>>> - uint64_t free_bufs; >>>> char name[ODP_BUFFER_POOL_NAME_LEN]; >>>> - >>>> - >>>> - odp_buffer_pool_t pool ODP_ALIGNED_CACHE; >>>> - uintptr_t buf_base; >>>> - size_t buf_size; >>>> - size_t buf_offset; >>>> - uint64_t num_bufs; >>>> - void *pool_base_addr; >>>> - uint64_t pool_size; >>>> - size_t payload_size; >>>> - size_t payload_align; >>>> - int buf_type; >>>> - size_t hdr_size; >>>> + odp_buffer_pool_param_t params; >>>> + _odp_buffer_pool_init_t init_params; >>>> + odp_buffer_pool_t pool_hdl; >>>> + uint32_t pool_id; >>>> + odp_shm_t pool_shm; >>>> + union { >>>> + uint32_t all; >>>> + struct { >>>> + uint32_t has_name:1; >>>> + uint32_t user_supplied_shm:1; >>>> + uint32_t unsegmented:1; >>>> + uint32_t zeroized:1; >>>> + uint32_t predefined:1; >>>> + }; >>>> + } flags; >>>> + uint32_t quiesced; >>>> + uint32_t low_wm_assert; >>>> + uint8_t *pool_base_addr; >>>> + uint8_t *pool_mdata_addr; >>>> + size_t pool_size; >>>> + uint32_t buf_align; >>>> + uint32_t buf_stride; >>>> + _odp_atomic_ptr_t buf_freelist; >>>> + _odp_atomic_ptr_t blk_freelist; >>>> + odp_atomic_u32_t bufcount; >>>> + odp_atomic_u32_t blkcount; >>>> + odp_atomic_u64_t bufallocs; >>>> + odp_atomic_u64_t buffrees; >>>> + odp_atomic_u64_t blkallocs; >>>> + odp_atomic_u64_t blkfrees; >>>> + odp_atomic_u64_t bufempty; >>>> + odp_atomic_u64_t blkempty; >>>> + odp_atomic_u64_t high_wm_count; >>>> + odp_atomic_u64_t low_wm_count; >>>> + uint32_t seg_size; >>>> + uint32_t high_wm; >>>> + uint32_t low_wm; >>>> + uint32_t headroom; >>>> + uint32_t tailroom; >>>> }; >>>> >>>> +typedef union pool_entry_u { >>>> + struct pool_entry_s s; >>>> + >>>> + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct >>>> pool_entry_s))]; >>>> +} pool_entry_t; >>>> >>>> extern void *pool_entry_ptr[]; >>>> >>>> +#if defined(ODP_CONFIG_SECURE_POOLS) && (ODP_CONFIG_SECURE_POOLS == 1) >>>> +#define buffer_is_secure(buf) (buf->flags.zeroized) >>>> +#define pool_is_secure(pool) (pool->flags.zeroized) >>>> +#else >>>> +#define buffer_is_secure(buf) 0 >>>> +#define pool_is_secure(pool) 0 >>>> +#endif >>>> + >>>> +#define TAG_ALIGN ((size_t)16) >>>> + >>>> +#define odp_cs(ptr, old, new) \ >>>> + _odp_atomic_ptr_cmp_xchg_strong(&ptr, (void **)&old, (void >>>> *)new, \ >>>> + _ODP_MEMMODEL_SC, \ >>>> + _ODP_MEMMODEL_SC) >>>> >>>> -static inline void *get_pool_entry(odp_buffer_pool_t pool_id) >>>> +/* Helper functions for pointer tagging to avoid ABA race conditions */ >>>> +#define odp_tag(ptr) \ >>>> + (((size_t)ptr) & (TAG_ALIGN - 1)) >>>> + >>>> +#define odp_detag(ptr) \ >>>> + ((void *)(((size_t)ptr) & -TAG_ALIGN)) >>>> + >>>> +#define odp_retag(ptr, tag) \ >>>> + ((void *)(((size_t)ptr) | odp_tag(tag))) >>>> + >>>> + >>>> +static inline void *get_blk(struct pool_entry_s *pool) >>>> +{ >>>> + void *oldhead, *myhead, *newhead; >>>> + >>>> + oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, >>>> _ODP_MEMMODEL_ACQ); >>>> + >>>> + do { >>>> + size_t tag = odp_tag(oldhead); >>>> + myhead = odp_detag(oldhead); >>>> + if (odp_unlikely(myhead == NULL)) >>>> + break; >>>> + newhead = odp_retag(((odp_buf_blk_t *)myhead)->next, >>>> tag + 1); >>>> + } while (odp_cs(pool->blk_freelist, oldhead, newhead) == 0); >>>> + >>>> + if (odp_unlikely(myhead == NULL)) >>>> + odp_atomic_inc_u64(&pool->blkempty); >>>> + else >>>> + odp_atomic_dec_u32(&pool->blkcount); >>>> + >>>> + return (void *)myhead; >>>> +} >>>> + >>>> +static inline void ret_blk(struct pool_entry_s *pool, void *block) >>>> +{ >>>> + void *oldhead, *myhead, *myblock; >>>> + >>>> + oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, >>>> _ODP_MEMMODEL_ACQ); >>>> + >>>> + do { >>>> + size_t tag = odp_tag(oldhead); >>>> + myhead = odp_detag(oldhead); >>>> + ((odp_buf_blk_t *)block)->next = myhead; >>>> + myblock = odp_retag(block, tag + 1); >>>> + } while (odp_cs(pool->blk_freelist, oldhead, myblock) == 0); >>>> + >>>> + odp_atomic_inc_u32(&pool->blkcount); >>>> + odp_atomic_inc_u64(&pool->blkfrees); >>>> +} >>>> + >>>> +static inline odp_buffer_hdr_t *get_buf(struct pool_entry_s *pool) >>>> +{ >>>> + odp_buffer_hdr_t *oldhead, *myhead, *newhead; >>>> + >>>> + oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, >>>> _ODP_MEMMODEL_ACQ); >>>> + >>>> + do { >>>> + size_t tag = odp_tag(oldhead); >>>> + myhead = odp_detag(oldhead); >>>> + if (odp_unlikely(myhead == NULL)) >>>> + break; >>>> + newhead = odp_retag(myhead->next, tag + 1); >>>> + } while (odp_cs(pool->buf_freelist, oldhead, newhead) == 0); >>>> + >>>> + if (odp_unlikely(myhead == NULL)) { >>>> + odp_atomic_inc_u64(&pool->bufempty); >>>> + } else { >>>> + uint64_t bufcount = >>>> + odp_atomic_fetch_sub_u32(&pool->bufcount, 1) - >>>> 1; >>>> + >>>> + /* Check for low watermark condition */ >>>> + if (bufcount == pool->low_wm && !pool->low_wm_assert) { >>>> + pool->low_wm_assert = 1; >>>> + odp_atomic_inc_u64(&pool->low_wm_count); >>>> + } >>>> + >>>> + odp_atomic_inc_u64(&pool->bufallocs); >>>> + myhead->next = myhead; /* Mark buffer allocated */ >>>> + myhead->allocator = odp_thread_id(); >>>> + } >>>> + >>>> + return (void *)myhead; >>>> +} >>>> + >>>> +static inline void ret_buf(struct pool_entry_s *pool, odp_buffer_hdr_t >>>> *buf) >>>> +{ >>>> + odp_buffer_hdr_t *oldhead, *myhead, *mybuf; >>>> + >>>> + buf->allocator = ODP_FREEBUF; /* Mark buffer free */ >>>> + >>>> + if (!buf->flags.hdrdata && buf->type != ODP_BUFFER_TYPE_RAW) { >>>> + while (buf->segcount > 0) { >>>> + if (buffer_is_secure(buf) || >>>> pool_is_secure(pool)) >>>> + memset(buf->addr[buf->segcount - 1], >>>> + 0, buf->segsize); >>>> + ret_blk(pool, buf->addr[--buf->segcount]); >>>> + } >>>> + buf->size = 0; >>>> + } >>>> + >>>> + oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, >>>> _ODP_MEMMODEL_ACQ); >>>> + >>>> + do { >>>> + size_t tag = odp_tag(oldhead); >>>> + myhead = odp_detag(oldhead); >>>> + buf->next = myhead; >>>> + mybuf = odp_retag(buf, tag + 1); >>>> + } while (odp_cs(pool->buf_freelist, oldhead, mybuf) == 0); >>>> + >>>> + uint64_t bufcount = odp_atomic_fetch_add_u32(&pool->bufcount, >>>> 1) + 1; >>>> + >>>> + /* Check if low watermark condition should be deasserted */ >>>> + if (bufcount == pool->high_wm && pool->low_wm_assert) { >>>> + pool->low_wm_assert = 0; >>>> + odp_atomic_inc_u64(&pool->high_wm_count); >>>> + } >>>> + >>>> + odp_atomic_inc_u64(&pool->buffrees); >>>> +} >>>> + >>>> +static inline void *get_local_buf(local_cache_t *buf_cache, >>>> + struct pool_entry_s *pool, >>>> + size_t totsize) >>>> +{ >>>> + odp_buffer_hdr_t *buf = buf_cache->buf_freelist; >>>> + >>>> + if (odp_likely(buf != NULL)) { >>>> + buf_cache->buf_freelist = buf->next; >>>> + >>>> + if (odp_unlikely(buf->size < totsize)) { >>>> + intmax_t needed = totsize - buf->size; >>>> + >>>> + do { >>>> + void *blk = get_blk(pool); >>>> + if (odp_unlikely(blk == NULL)) { >>>> + ret_buf(pool, buf); >>>> + buf_cache->buffrees--; >>>> + return NULL; >>>> + } >>>> + buf->addr[buf->segcount++] = blk; >>>> + needed -= pool->seg_size; >>>> + } while (needed > 0); >>>> + >>>> + buf->size = buf->segcount * pool->seg_size; >>>> + } >>>> + >>>> + buf_cache->bufallocs++; >>>> + buf->allocator = odp_thread_id(); /* Mark buffer >>>> allocated */ >>>> + } >>>> + >>>> + return buf; >>>> +} >>>> + >>>> +static inline void ret_local_buf(local_cache_t *buf_cache, >>>> + odp_buffer_hdr_t *buf) >>>> +{ >>>> + buf->allocator = ODP_FREEBUF; >>>> + buf->next = buf_cache->buf_freelist; >>>> + buf_cache->buf_freelist = buf; >>>> + >>>> + buf_cache->buffrees++; >>>> +} >>>> + >>>> +static inline void flush_cache(local_cache_t *buf_cache, >>>> + struct pool_entry_s *pool) >>>> +{ >>>> + odp_buffer_hdr_t *buf = buf_cache->buf_freelist; >>>> + uint32_t flush_count = 0; >>>> + >>>> + while (buf != NULL) { >>>> + odp_buffer_hdr_t *next = buf->next; >>>> + ret_buf(pool, buf); >>>> + buf = next; >>>> + flush_count++; >>>> + } >>>> + >>>> + odp_atomic_add_u64(&pool->bufallocs, buf_cache->bufallocs); >>>> + odp_atomic_add_u64(&pool->buffrees, buf_cache->buffrees - >>>> flush_count); >>>> + >>>> + buf_cache->buf_freelist = NULL; >>>> + buf_cache->bufallocs = 0; >>>> + buf_cache->buffrees = 0; >>>> +} >>>> + >>>> +static inline odp_buffer_pool_t pool_index_to_handle(uint32_t pool_id) >>>> +{ >>>> + return pool_id; >>>> +} >>>> + >>>> +static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl) >>>> +{ >>>> + return pool_hdl; >>>> +} >>>> + >>>> +static inline void *get_pool_entry(uint32_t pool_id) >>>> { >>>> return pool_entry_ptr[pool_id]; >>>> } >>>> >>>> +static inline pool_entry_t *odp_pool_to_entry(odp_buffer_pool_t pool) >>>> +{ >>>> + return (pool_entry_t >>>> *)get_pool_entry(pool_handle_to_index(pool)); >>>> +} >>>> + >>>> +static inline pool_entry_t *odp_buf_to_pool(odp_buffer_hdr_t *buf) >>>> +{ >>>> + return odp_pool_to_entry(buf->pool_hdl); >>>> +} >>>> >>>> -static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf) >>>> +static inline uint32_t odp_buffer_pool_segment_size(odp_buffer_pool_t >>>> pool) >>>> { >>>> - return (odp_buffer_hdr_t *)buf; >>>> + return odp_pool_to_entry(pool)->s.seg_size; >>>> } >>>> >>>> +static inline uint32_t odp_buffer_pool_headroom(odp_buffer_pool_t pool) >>>> +{ >>>> + return odp_pool_to_entry(pool)->s.headroom; >>>> +} >>>> + >>>> +static inline uint32_t odp_buffer_pool_tailroom(odp_buffer_pool_t pool) >>>> +{ >>>> + return odp_pool_to_entry(pool)->s.tailroom; >>>> +} >>>> >>>> #ifdef __cplusplus >>>> } >>>> diff --git a/platform/linux-dpdk/include/odp_packet_dpdk.h >>>> b/platform/linux-dpdk/include/odp_packet_dpdk.h >>>> index c3a2b70..c982d5e 100644 >>>> --- a/platform/linux-dpdk/include/odp_packet_dpdk.h >>>> +++ b/platform/linux-dpdk/include/odp_packet_dpdk.h >>>> @@ -8,10 +8,8 @@ >>>> #define ODP_PACKET_DPDK_H >>>> >>>> #include <stdint.h> >>>> -#include <net/if.h> >>>> >>>> #include <odph_eth.h> >>>> -#include <odph_packet.h> >>>> #include <odp_align.h> >>>> #include <odp_debug.h> >>>> #include <odp_packet.h> >>>> diff --git a/platform/linux-dpdk/include/odp_packet_internal.h >>>> b/platform/linux-dpdk/include/odp_packet_internal.h >>>> index 1cd44b1..24de33f 100644 >>>> --- a/platform/linux-dpdk/include/odp_packet_internal.h >>>> +++ b/platform/linux-dpdk/include/odp_packet_internal.h >>>> @@ -22,6 +22,7 @@ extern "C" { >>>> #include <odp_debug.h> >>>> #include <odp_buffer_internal.h> >>>> #include <odp_buffer_pool_internal.h> >>>> +#include <odp_buffer_inlines.h> >>>> #include <odp_packet.h> >>>> #include <odp_packet_io.h> >>>> >>>> @@ -43,6 +44,7 @@ typedef union { >>>> uint32_t vlan:1; /**< VLAN hdr found */ >>>> uint32_t vlan_qinq:1; /**< Stacked VLAN found, QinQ */ >>>> >>>> + uint32_t snap:1; /**< SNAP */ >>>> uint32_t arp:1; /**< ARP */ >>>> >>>> uint32_t ipv4:1; /**< IPv4 */ >>>> @@ -53,13 +55,14 @@ typedef union { >>>> >>>> uint32_t udp:1; /**< UDP */ >>>> uint32_t tcp:1; /**< TCP */ >>>> + uint32_t tcpopt:1; /**< TCP options present */ >>>> uint32_t sctp:1; /**< SCTP */ >>>> uint32_t icmp:1; /**< ICMP */ >>>> }; >>>> } input_flags_t; >>>> >>>> -ODP_STATIC_ASSERT(sizeof(input_flags_t) == sizeof(uint32_t), >>>> - "INPUT_FLAGS_SIZE_ERROR"); >>>> +_ODP_STATIC_ASSERT(sizeof(input_flags_t) == sizeof(uint32_t), >>>> + "INPUT_FLAGS_SIZE_ERROR"); >>>> >>>> /** >>>> * Packet error flags >>>> @@ -70,7 +73,9 @@ typedef union { >>>> >>>> struct { >>>> /* Bitfield flags for each detected error */ >>>> + uint32_t app_error:1; /**< Error bit for application >>>> use */ >>>> uint32_t frame_len:1; /**< Frame length error */ >>>> + uint32_t snap_len:1; /**< Snap length error */ >>>> uint32_t l2_chksum:1; /**< L2 checksum error, checks >>>> TBD */ >>>> uint32_t ip_err:1; /**< IP error, checks TBD */ >>>> uint32_t tcp_err:1; /**< TCP error, checks TBD */ >>>> @@ -78,8 +83,8 @@ typedef union { >>>> }; >>>> } error_flags_t; >>>> >>>> -ODP_STATIC_ASSERT(sizeof(error_flags_t) == sizeof(uint32_t), >>>> - "ERROR_FLAGS_SIZE_ERROR"); >>>> +_ODP_STATIC_ASSERT(sizeof(error_flags_t) == sizeof(uint32_t), >>>> + "ERROR_FLAGS_SIZE_ERROR"); >>>> >>>> /** >>>> * Packet output flags >>>> @@ -90,12 +95,15 @@ typedef union { >>>> >>>> struct { >>>> /* Bitfield flags for each output option */ >>>> - uint32_t l4_chksum:1; /**< Request L4 checksum >>>> calculation */ >>>> + uint32_t l3_chksum_set:1; /**< L3 chksum bit is valid */ >>>> + uint32_t l3_chksum:1; /**< L3 chksum override */ >>>> + uint32_t l4_chksum_set:1; /**< L3 chksum bit is valid */ >>>> + uint32_t l4_chksum:1; /**< L4 chksum override */ >>>> }; >>>> } output_flags_t; >>>> >>>> -ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), >>>> - "OUTPUT_FLAGS_SIZE_ERROR"); >>>> +_ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), >>>> + "OUTPUT_FLAGS_SIZE_ERROR"); >>>> >>>> /** >>>> * Internal Packet header >>>> @@ -108,28 +116,131 @@ typedef struct { >>>> error_flags_t error_flags; >>>> output_flags_t output_flags; >>>> >>>> - uint32_t frame_offset; /**< offset to start of frame, even on >>>> error */ >>>> uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */ >>>> uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */ >>>> uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also >>>> ICMP) */ >>>> + uint32_t payload_offset; /**< offset to payload */ >>>> >>>> - uint64_t user_ctx; /**< user context */ >>>> + uint32_t vlan_s_tag; /**< Parsed 1st VLAN header (S-TAG) */ >>>> + uint32_t vlan_c_tag; /**< Parsed 2nd VLAN header (C-TAG) */ >>>> + uint32_t l3_protocol; /**< Parsed L3 protocol */ >>>> + uint32_t l3_len; /**< Layer 3 length */ >>>> + uint32_t l4_protocol; /**< Parsed L4 protocol */ >>>> + uint32_t l4_len; /**< Layer 4 length */ >>>> + >>>> + uint32_t frame_len; >>>> + uint32_t headroom; >>>> + uint32_t tailroom; >>>> >>>> odp_pktio_t input; >>>> } odp_packet_hdr_t; >>>> >>>> +typedef struct odp_packet_hdr_stride { >>>> + uint8_t >>>> pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_packet_hdr_t))]; >>>> +} odp_packet_hdr_stride; >>>> + >>>> +_ODP_STATIC_ASSERT(sizeof(odp_packet_hdr_t) % sizeof(uint64_t) == 0, >>>> + "ODP_PACKET_HDR_T__SIZE_ERR2"); >>>> + >>>> /** >>>> * Return the packet header >>>> */ >>>> static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) >>>> { >>>> - return (odp_packet_hdr_t *)pkt; >>>> + return (odp_packet_hdr_t *)odp_buf_to_hdr((odp_buffer_t)pkt); >>>> } >>>> >>>> /** >>>> - * Parse packet and set internal metadata >>>> + * Initialize packet buffer >>>> */ >>>> -void odp_packet_parse(odp_packet_t pkt, size_t len, size_t l2_offset); >>>> +static inline void packet_init(pool_entry_t *pool, >>>> + odp_packet_hdr_t *pkt_hdr, >>>> + size_t size) >>>> +{ >>>> + /* >>>> + * Reset parser metadata. Note that we clear via memset to make >>>> + * this routine indepenent of any additional adds to packet >>>> metadata. >>>> + */ >>>> + struct rte_mbuf *mb; >>>> + uint8_t *start; >>>> + >>>> + mb = (struct rte_mbuf *)&pkt_hdr->buf_hdr; >>>> + mb->buf_len = size; >>>> + start = (uint8_t *)mb->buf_addr; >>>> + memset(start, 0, mb->buf_len); >>>> + >>>> + /* >>>> + * Packet headroom is set from the pool's headroom >>>> + * Packet tailroom is rounded up to fill the last >>>> + * segment occupied by the allocated length. >>>> + */ >>>> + pkt_hdr->frame_len = size; >>>> + pkt_hdr->headroom = pool->s.headroom; >>>> + pkt_hdr->tailroom = >>>> + (pool->s.seg_size * pkt_hdr->buf_hdr.segcount) - >>>> + (pool->s.headroom + size); >>>> +} >>>> + >>>> +static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, >>>> + uint32_t offset, uint32_t *seglen) >>>> +{ >>>> + struct rte_mbuf *mbuf = (struct rte_mbuf *)&pkt_hdr->buf_hdr; >>>> + if (offset > pkt_hdr->frame_len) >>>> + return NULL; >>>> + >>>> + if (seglen != NULL) { >>>> + *seglen = mbuf->pkt.data_len; >>>> + pkt_hdr->frame_len = mbuf->pkt.data_len; >>>> + } >>>> + return (void *)((uint8_t *)mbuf->pkt.data + offset); >>>> +} >>>> + >>>> +#define pull_offset(x, len) (x = x < len ? 0 : x - len) >>>> + >>>> +static inline void push_head(odp_packet_hdr_t *pkt_hdr, size_t len) >>>> +{ >>>> + pkt_hdr->headroom -= len; >>>> + pkt_hdr->frame_len += len; >>>> + pkt_hdr->l2_offset += len; >>>> + pkt_hdr->l3_offset += len; >>>> + pkt_hdr->l4_offset += len; >>>> +} >>>> + >>>> +static inline void pull_head(odp_packet_hdr_t *pkt_hdr, size_t len) >>>> +{ >>>> + pkt_hdr->headroom += len; >>>> + pkt_hdr->frame_len -= len; >>>> + pull_offset(pkt_hdr->l2_offset, len); >>>> + pull_offset(pkt_hdr->l3_offset, len); >>>> + pull_offset(pkt_hdr->l4_offset, len); >>>> +} >>>> + >>>> +static inline void push_tail(odp_packet_hdr_t *pkt_hdr, size_t len) >>>> +{ >>>> + pkt_hdr->tailroom -= len; >>>> + pkt_hdr->frame_len += len; >>>> +} >>>> + >>>> + >>>> +static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, size_t len) >>>> +{ >>>> + pkt_hdr->tailroom += len; >>>> + pkt_hdr->frame_len -= len; >>>> +} >>>> + >>>> +static inline void packet_set_len(odp_packet_t pkt, uint32_t len) >>>> +{ >>>> + odp_packet_hdr(pkt)->frame_len = len; >>>> +} >>>> + >>>> +/* Forward declarations */ >>>> +int _odp_packet_copy_to_packet(odp_packet_t srcpkt, uint32_t srcoffset, >>>> + odp_packet_t dstpkt, uint32_t dstoffset, >>>> + uint32_t len); >>>> + >>>> +odp_packet_t _odp_packet_alloc(odp_buffer_pool_t pool_hdl); >>>> + >>>> +int _odp_packet_parse(odp_packet_t pkt); >>>> >>>> #ifdef __cplusplus >>>> } >>>> diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h >>>> b/platform/linux-dpdk/include/odp_packet_io_internal.h >>>> index 9263349..5ece843 100644 >>>> --- a/platform/linux-dpdk/include/odp_packet_io_internal.h >>>> +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h >>>> @@ -20,17 +20,36 @@ extern "C" { >>>> >>>> #include <odp_spinlock.h> >>>> #include <odp_packet_socket.h> >>>> -#ifdef ODP >>> >>> > _______________________________________________ > lng-odp mailing list > lng-odp@lists.linaro.org > http://lists.linaro.org/mailman/listinfo/lng-odp > > ...
On 4 Feb 2015 22:57, "Zoltan Kiss" <zoltan.kiss@linaro.org> wrote: > > My idea is that when I finish cleaning up my changes we can make a diff between your 0.5 repo and my 0.6,and that hopefully will be much shorter as my stuff supposed to have most of your changes,but it is broken up to reasonable patches. How does that sound? Sounds good, we will sync up at connect. I was under impression that you need odp-dpdk to be at 0.6 before connect. > I will try to finish that now on the plane,but otherwise we can do that on Connect,or afterwards. > > Zoli > > On 4 Feb 2015 16:07, "Venkatesh Vivekanandan" < venkatesh.vivekanandan@linaro.org> wrote: >> >> >> >> On 4 February 2015 at 21:05, Mike Holmes <mike.holmes@linaro.org> wrote: >>> >>> Venki - I think Zoltan is breaking his 0.6 patches up, andI think the right thing to do is create patches for all the changes as I believe ODP-DPDK will become a more important >> >> >> If it is a feature, then yes we can break into small patches, but since this is a new baseline(say 0.5.0) we have no choice other than reviewing the whole patch is what I believe. Even if we try to break things into small groups, it will only help the reviewer but one cannot commit it as is, because it will not compile. >> >>> >>> implementation to us this year. >>> >>> Maybe we do this at HKG15 to reduce the review cycle time ? >>> >>> On 4 February 2015 at 10:30, Venkatesh Vivekanandan < venkatesh.vivekanandan@linaro.org> wrote: >>>> >>>> Hi All, >>>> >>>> Inorder to make the REVIEW PROCESS SIMPLER as it is a huge patch, I have attached the changes that went in this patch on top of ODP 0.5.0 (diff between platform/linux-generic and platform/linux-dpdk). Hope this helps. >>>> >>>> Thanks, >>>> Venky >>>> >>>> On 4 February 2015 at 20:48, <venkatesh.vivekanandan@linaro.org> wrote: >>>>> >>>>> From: Venkatesh Vivekanandan <venkatesh.vivekanandan@linaro.org> >>>>> >>>>> Signed-off-by: Venkatesh Vivekanandan < venkatesh.vivekanandan@linaro.org> >>>>> --- >>>>> -> This patch must be applied on odp v0.5.0 ontop of odp-dpdk >>>>> -> odp_l2fwd, odp_pktio were tested both in burst and queue mode which worked >>>>> fine. >>>>> -> ipsec app was tested and working fine. For this to work, >>>>> USE_MAC_ADDR_HACK has to be set to 0 for now. Will send out a patch for >>>>> MAC address fix once this patch gets in. >>>>> >>>>> Checkpatch warning for camel case could not be avoided. >>>>> WARNING: Avoid CamelCase: <_Static_assert> >>>>> #373: FILE: platform/linux-dpdk/include/api/odp_debug.h:34: >>>>> +#define _Static_assert(e, s) extern int (*static_assert_checker(void)) \ >>>>> >>>>> WARNING: storage class should be at the beginning of the declaration >>>>> #373: FILE: platform/linux-dpdk/include/api/odp_debug.h:34: >>>>> +#define _Static_assert(e, s) extern int (*static_assert_checker(void)) \ >>>>> >>>>> v2: Fixed odp_flush_caches compilation error >>>>> v3: Removed debug code that slipped in >>>>> >>>>> platform/linux-dpdk/Makefile.am | 13 +- >>>>> platform/linux-dpdk/include/api/odp_buffer.h | 30 +- >>>>> platform/linux-dpdk/include/api/odp_buffer_pool.h | 124 ++- >>>>> platform/linux-dpdk/include/api/odp_debug.h | 77 +- >>>>> platform/linux-dpdk/include/api/odp_packet.h | 772 +++++++++++++-- >>>>> platform/linux-dpdk/include/api/odp_packet_io.h | 92 +- >>>>> platform/linux-dpdk/include/api/odp_pktio_types.h | 45 - >>>>> .../linux-dpdk/include/api/odp_platform_types.h | 78 ++ >>>>> platform/linux-dpdk/include/odp_buffer_inlines.h | 179 ++++ >>>>> platform/linux-dpdk/include/odp_buffer_internal.h | 134 ++- >>>>> .../linux-dpdk/include/odp_buffer_pool_internal.h | 333 ++++++- >>>>> platform/linux-dpdk/include/odp_packet_dpdk.h | 2 - >>>>> platform/linux-dpdk/include/odp_packet_internal.h | 135 ++- >>>>> .../linux-dpdk/include/odp_packet_io_internal.h | 42 +- >>>>> platform/linux-dpdk/odp_buffer.c | 14 +- >>>>> platform/linux-dpdk/odp_buffer_pool.c | 153 +-- >>>>> platform/linux-dpdk/odp_init.c | 35 +- >>>>> platform/linux-dpdk/odp_linux.c | 117 ++- >>>>> platform/linux-dpdk/odp_packet.c | 1001 +++++++++++++++----- >>>>> platform/linux-dpdk/odp_packet_dpdk.c | 9 +- >>>>> platform/linux-dpdk/odp_packet_io.c | 309 ++++-- >>>>> platform/linux-dpdk/odp_queue.c | 3 +- >>>>> platform/linux-dpdk/odp_schedule.c | 421 -------- >>>>> 23 files changed, 2993 insertions(+), 1125 deletions(-) >>>>> delete mode 100644 platform/linux-dpdk/include/api/odp_pktio_types.h >>>>> create mode 100644 platform/linux-dpdk/include/api/odp_platform_types.h >>>>> create mode 100644 platform/linux-dpdk/include/odp_buffer_inlines.h >>>>> delete mode 100644 platform/linux-dpdk/odp_schedule.c >>>>> >>>>> diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am >>>>> index db11e0a..6491d3a 100644 >>>>> --- a/platform/linux-dpdk/Makefile.am >>>>> +++ b/platform/linux-dpdk/Makefile.am >>>>> @@ -29,6 +29,7 @@ include_HEADERS = \ >>>>> $(top_srcdir)/platform/linux-generic/include/api/odp_barrier.h \ >>>>> $(srcdir)/include/api/odp_buffer_pool.h \ >>>>> $(top_srcdir)/platform/linux-generic/include/api/odp_byteorder.h \ >>>>> + $(top_srcdir)/platform/linux-generic/include/api/odp_classification.h \ >>>>> $(top_srcdir)/platform/linux-generic/include/api/odp_compiler.h \ >>>>> $(top_srcdir)/platform/linux-generic/include/api/odp_config.h \ >>>>> $(top_srcdir)/platform/linux-generic/include/api/odp_coremask.h \ >>>>> @@ -52,15 +53,16 @@ include_HEADERS = \ >>>>> $(top_srcdir)/platform/linux-generic/include/api/odp_time.h \ >>>>> $(top_srcdir)/platform/linux-generic/include/api/odp_timer.h \ >>>>> $(top_srcdir)/platform/linux-generic/include/api/odp_version.h \ >>>>> - $(srcdir)/include/api/odp_pktio_types.h >>>>> + $(srcdir)/include/api/odp_platform_types.h >>>>> >>>>> subdirheadersdir = $(includedir)/helper >>>>> subdirheaders_HEADERS = \ >>>>> $(top_srcdir)/helper/include/odph_chksum.h \ >>>>> $(top_srcdir)/helper/include/odph_eth.h \ >>>>> + $(top_srcdir)/helper/include/odph_icmp.h \ >>>>> $(top_srcdir)/helper/include/odph_ip.h \ >>>>> + $(top_srcdir)/helper/include/odph_ipsec.h \ >>>>> $(top_srcdir)/helper/include/odph_linux.h \ >>>>> - $(top_srcdir)/helper/include/odph_packet.h \ >>>>> $(top_srcdir)/helper/include/odph_ring.h \ >>>>> $(top_srcdir)/helper/include/odph_udp.h >>>>> >>>>> @@ -68,9 +70,11 @@ __LIB__libodp_la_SOURCES = \ >>>>> ../linux-generic/odp_barrier.c \ >>>>> odp_buffer.c \ >>>>> odp_buffer_pool.c \ >>>>> + ../linux-generic/odp_classification.c \ >>>>> ../linux-generic/odp_coremask.c \ >>>>> ../linux-generic/odp_crypto.c \ >>>>> odp_init.c \ >>>>> + ../linux-generic/odp_impl.c \ >>>>> odp_linux.c \ >>>>> odp_packet.c \ >>>>> odp_packet_dpdk.c \ >>>>> @@ -80,11 +84,12 @@ __LIB__libodp_la_SOURCES = \ >>>>> odp_queue.c \ >>>>> ../linux-generic/odp_ring.c \ >>>>> ../linux-generic/odp_rwlock.c \ >>>>> - odp_schedule.c \ >>>>> + ../linux-generic/odp_schedule.c \ >>>>> ../linux-generic/odp_shared_memory.c \ >>>>> ../linux-generic/odp_spinlock.c \ >>>>> ../linux-generic/odp_system_info.c \ >>>>> ../linux-generic/odp_thread.c \ >>>>> ../linux-generic/odp_ticketlock.c \ >>>>> ../linux-generic/odp_time.c \ >>>>> - ../linux-generic/odp_timer.c >>>>> + ../linux-generic/odp_timer.c \ >>>>> + ../linux-generic/odp_weak.c >>>>> diff --git a/platform/linux-dpdk/include/api/odp_buffer.h b/platform/linux-dpdk/include/api/odp_buffer.h >>>>> index b2fbc76..3c23035 100644 >>>>> --- a/platform/linux-dpdk/include/api/odp_buffer.h >>>>> +++ b/platform/linux-dpdk/include/api/odp_buffer.h >>>>> @@ -19,20 +19,13 @@ extern "C" { >>>>> #endif >>>>> >>>>> >>>>> - >>>>> #include <odp_std_types.h> >>>>> +#include <odp_platform_types.h> >>>>> >>>>> - >>>>> - >>>>> - >>>>> - >>>>> -/** >>>>> - * ODP buffer >>>>> +/** @defgroup odp_buffer ODP BUFFER >>>>> + * Operations on a buffer. >>>>> + * @{ >>>>> */ >>>>> -typedef unsigned long odp_buffer_t; >>>>> - >>>>> - >>>>> -#define ODP_BUFFER_INVALID (unsigned long)(-1L) /**< Invalid buffer */ >>>>> >>>>> >>>>> /** >>>>> @@ -74,11 +67,21 @@ int odp_buffer_type(odp_buffer_t buf); >>>>> * >>>>> * @param buf Buffer handle >>>>> * >>>>> - * @return 1 if valid, otherwise 0 >>>>> + * @retval 1 Buffer handle represents a valid buffer. >>>>> + * @retval 0 Buffer handle does not represent a valid buffer. >>>>> */ >>>>> int odp_buffer_is_valid(odp_buffer_t buf); >>>>> >>>>> /** >>>>> + * Buffer pool of the buffer >>>>> + * >>>>> + * @param buf Buffer handle >>>>> + * >>>>> + * @return Handle of buffer pool buffer belongs to >>>>> + */ >>>>> +odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf); >>>>> + >>>>> +/** >>>>> * Print buffer metadata to STDOUT >>>>> * >>>>> * @param buf Buffer handle >>>>> @@ -86,6 +89,9 @@ int odp_buffer_is_valid(odp_buffer_t buf); >>>>> */ >>>>> void odp_buffer_print(odp_buffer_t buf); >>>>> >>>>> +/** >>>>> + * @} >>>>> + */ >>>>> >>>>> #ifdef __cplusplus >>>>> } >>>>> diff --git a/platform/linux-dpdk/include/api/odp_buffer_pool.h b/platform/linux-dpdk/include/api/odp_buffer_pool.h >>>>> index 382f4f0..4da5f84 100644 >>>>> --- a/platform/linux-dpdk/include/api/odp_buffer_pool.h >>>>> +++ b/platform/linux-dpdk/include/api/odp_buffer_pool.h >>>>> @@ -21,66 +21,143 @@ extern "C" { >>>>> >>>>> >>>>> #include <odp_std_types.h> >>>>> +#include <odp_platform_types.h> >>>>> #include <odp_buffer.h> >>>>> >>>>> +/** @addtogroup odp_buffer >>>>> + * Operations on a buffer pool. >>>>> + * @{ >>>>> + */ >>>>> + >>>>> /** Maximum queue name lenght in chars */ >>>>> #define ODP_BUFFER_POOL_NAME_LEN 32 >>>>> >>>>> -/** Invalid buffer pool */ >>>>> -#define ODP_BUFFER_POOL_INVALID (unsigned long)(-1L) >>>>> - >>>>> -/** ODP buffer pool */ >>>>> -typedef unsigned long odp_buffer_pool_t; >>>>> - >>>>> +/** >>>>> + * Buffer pool parameters >>>>> + * Used to communicate buffer pool creation options. >>>>> + */ >>>>> +typedef struct odp_buffer_pool_param_t { >>>>> + uint32_t buf_size; /**< Buffer size in bytes. The maximum >>>>> + number of bytes application will >>>>> + store in each buffer. For packets, this >>>>> + is the maximum packet data length, and >>>>> + configured headroom and tailroom will be >>>>> + added to this number */ >>>>> + uint32_t buf_align; /**< Minimum buffer alignment in bytes. >>>>> + Valid values are powers of two. Use 0 >>>>> + for default alignment. Default will >>>>> + always be a multiple of 8. */ >>>>> + uint32_t num_bufs; /**< Number of buffers in the pool */ >>>>> + int buf_type; /**< Buffer type */ >>>>> +} odp_buffer_pool_param_t; >>>>> >>>>> /** >>>>> * Create a buffer pool >>>>> + * This routine is used to create a buffer pool. It take three >>>>> + * arguments: the optional name of the pool to be created, an optional shared >>>>> + * memory handle, and a parameter struct that describes the pool to be >>>>> + * created. If a name is not specified the result is an anonymous pool that >>>>> + * cannot be referenced by odp_buffer_pool_lookup(). >>>>> + * >>>>> + * @param name Name of the pool, max ODP_BUFFER_POOL_NAME_LEN-1 chars. >>>>> + * May be specified as NULL for anonymous pools. >>>>> + * >>>>> + * @param shm The shared memory object in which to create the pool. >>>>> + * Use ODP_SHM_NULL to reserve default memory type >>>>> + * for the buffer type. >>>>> * >>>>> - * @param name Name of the pool (max ODP_BUFFER_POOL_NAME_LEN - 1 chars) >>>>> - * @param base_addr Pool base address >>>>> - * @param size Pool size in bytes >>>>> - * @param buf_size Buffer size in bytes >>>>> - * @param buf_align Minimum buffer alignment >>>>> - * @param buf_type Buffer type >>>>> + * @param params Buffer pool parameters. >>>>> * >>>>> - * @return Buffer pool handle >>>>> + * @return Handle of the created buffer pool >>>>> + * @retval ODP_BUFFER_POOL_INVALID Buffer pool could not be created >>>>> */ >>>>> + >>>>> odp_buffer_pool_t odp_buffer_pool_create(const char *name, >>>>> - void *base_addr, uint64_t size, >>>>> - size_t buf_size, size_t buf_align, >>>>> - int buf_type); >>>>> + odp_shm_t shm, >>>>> + odp_buffer_pool_param_t *params); >>>>> >>>>> +/** >>>>> + * Destroy a buffer pool previously created by odp_buffer_pool_create() >>>>> + * >>>>> + * @param pool Handle of the buffer pool to be destroyed >>>>> + * >>>>> + * @retval 0 Success >>>>> + * @retval -1 Failure >>>>> + * >>>>> + * @note This routine destroys a previously created buffer pool. This call >>>>> + * does not destroy any shared memory object passed to >>>>> + * odp_buffer_pool_create() used to store the buffer pool contents. The caller >>>>> + * takes responsibility for that. If no shared memory object was passed as >>>>> + * part of the create call, then this routine will destroy any internal shared >>>>> + * memory objects associated with the buffer pool. Results are undefined if >>>>> + * an attempt is made to destroy a buffer pool that contains allocated or >>>>> + * otherwise active buffers. >>>>> + */ >>>>> +int odp_buffer_pool_destroy(odp_buffer_pool_t pool); >>>>> >>>>> /** >>>>> * Find a buffer pool by name >>>>> * >>>>> * @param name Name of the pool >>>>> * >>>>> - * @return Buffer pool handle, or ODP_BUFFER_POOL_INVALID if not found. >>>>> + * @return Handle of found buffer pool >>>>> + * @retval ODP_BUFFER_POOL_INVALID Buffer pool could not be found >>>>> + * >>>>> + * @note This routine cannot be used to look up an anonymous pool (one created >>>>> + * with no name). >>>>> */ >>>>> odp_buffer_pool_t odp_buffer_pool_lookup(const char *name); >>>>> >>>>> +/** >>>>> + * Buffer pool information struct >>>>> + * Used to get information about a buffer pool. >>>>> + */ >>>>> +typedef struct odp_buffer_pool_info_t { >>>>> + const char *name; /**< pool name */ >>>>> + odp_shm_t shm; /**< handle of shared memory area >>>>> + supplied by application to >>>>> + contain buffer pool, or >>>>> + ODP_SHM_NULL if this pool is >>>>> + managed by ODP */ >>>>> + odp_buffer_pool_param_t params; /**< pool parameters */ >>>>> +} odp_buffer_pool_info_t; >>>>> + >>>>> +/** >>>>> + * Retrieve information about a buffer pool >>>>> + * >>>>> + * @param pool Buffer pool handle >>>>> + * >>>>> + * @param[out] info Receives an odp_buffer_pool_info_t object >>>>> + * that describes the pool. >>>>> + * >>>>> + * @retval 0 Success >>>>> + * @retval -1 Failure. Info could not be retrieved. >>>>> + */ >>>>> + >>>>> +int odp_buffer_pool_info(odp_buffer_pool_t pool, >>>>> + odp_buffer_pool_info_t *info); >>>>> >>>>> /** >>>>> * Print buffer pool info >>>>> * >>>>> * @param pool Pool handle >>>>> * >>>>> + * @note This routine writes implementation-defined information about the >>>>> + * specified buffer pool to the ODP log. The intended use is for debugging. >>>>> */ >>>>> void odp_buffer_pool_print(odp_buffer_pool_t pool); >>>>> >>>>> - >>>>> - >>>>> /** >>>>> * Buffer alloc >>>>> * >>>>> + * The validity of a buffer can be cheked at any time with odp_buffer_is_valid() >>>>> * @param pool Pool handle >>>>> * >>>>> - * @return Buffer handle or ODP_BUFFER_INVALID >>>>> + * @return Handle of allocated buffer >>>>> + * @retval ODP_BUFFER_INVALID Buffer could not be allocated >>>>> */ >>>>> odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool); >>>>> >>>>> - >>>>> /** >>>>> * Buffer free >>>>> * >>>>> @@ -89,8 +166,9 @@ odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool); >>>>> */ >>>>> void odp_buffer_free(odp_buffer_t buf); >>>>> >>>>> - >>>>> - >>>>> +/** >>>>> + * @} >>>>> + */ >>>>> >>>>> #ifdef __cplusplus >>>>> } >>>>> diff --git a/platform/linux-dpdk/include/api/odp_debug.h b/platform/linux-dpdk/include/api/odp_debug.h >>>>> index 6b145b5..aed8a5f 100644 >>>>> --- a/platform/linux-dpdk/include/api/odp_debug.h >>>>> +++ b/platform/linux-dpdk/include/api/odp_debug.h >>>>> @@ -12,73 +12,74 @@ >>>>> #ifndef ODP_DEBUG_H_ >>>>> #define ODP_DEBUG_H_ >>>>> >>>>> -#include <stdio.h> >>>>> -#include <stdlib.h> >>>>> >>>>> #ifdef __cplusplus >>>>> extern "C" { >>>>> #endif >>>>> >>>>> +/** @addtogroup odp_ver_abt_log_dbg >>>>> + * Macros that allows different messages. >>>>> + * @{ >>>>> + */ >>>>> + >>>>> #ifdef __GNUC__ >>>>> >>>>> -/** >>>>> - * Indicate deprecated variables, functions or types >>>>> - */ >>>>> -#define ODP_DEPRECATED __attribute__((__deprecated__)) >>>>> + >>>>> +#if __GNUC__ < 4 || (__GNUC__ == 4 && (__GNUC_MINOR__ < 6)) >>>>> >>>>> /** >>>>> - * Intentionally unused variables ot functions >>>>> + * _Static_assert was only added in GCC 4.6. Provide a weak replacement >>>>> + * for previous versions. >>>>> */ >>>>> -#define ODP_UNUSED __attribute__((__unused__)) >>>>> +#define _Static_assert(e, s) extern int (*static_assert_checker(void)) \ >>>>> + [sizeof(struct { unsigned int error_if_negative:(e) ? 1 : -1; })] >>>>> >>>>> -#else >>>>> +#endif >>>>> >>>>> -#define ODP_DEPRECATED >>>>> -#define ODP_UNUSED >>>>> >>>>> -#endif >>>>> >>>>> -/** >>>>> - * Runtime assertion-macro - aborts if 'cond' is false. >>>>> - */ >>>>> -#ifndef ODP_NO_DEBUG >>>>> -#define ODP_ASSERT(cond, msg) \ >>>>> - do { if (!(cond)) {ODP_ERR("%s\n", msg); abort(); } } while (0) >>>>> -#else >>>>> -#define ODP_ASSERT(cond, msg) >>>>> #endif >>>>> >>>>> + >>>>> /** >>>>> * Compile time assertion-macro - fail compilation if cond is false. >>>>> * @note This macro has zero runtime overhead >>>>> */ >>>>> -#define ODP_STATIC_ASSERT(cond, msg) _Static_assert(1, msg) >>>>> +#define _ODP_STATIC_ASSERT(cond, msg) _Static_assert(1, msg) >>>>> >>>>> /** >>>>> - * Debug printing macro, which prints output when DEBUG flag is set. >>>>> + * ODP log level. >>>>> */ >>>>> -#define ODP_DBG(fmt, ...) \ >>>>> - do { if (ODP_DEBUG_PRINT == 1) \ >>>>> - printf(fmt, ##__VA_ARGS__); \ >>>>> - } while (0) >>>>> +typedef enum odp_log_level { >>>>> + ODP_LOG_DBG, >>>>> + ODP_LOG_ERR, >>>>> + ODP_LOG_UNIMPLEMENTED, >>>>> + ODP_LOG_ABORT, >>>>> + ODP_LOG_PRINT >>>>> +} odp_log_level_e; >>>>> >>>>> /** >>>>> - * Print output to stderr (file, line and function). >>>>> + * ODP log function >>>>> + * >>>>> + * Instead of direct prints to stdout/stderr all logging in ODP implementation >>>>> + * should be done via this function or its wrappers. >>>>> + * ODP platform MUST provide a default *weak* implementation of this function. >>>>> + * Application MAY override the function if needed by providing a strong >>>>> + * function. >>>>> + * >>>>> + * @param[in] level Log level >>>>> + * @param[in] fmt printf-style message format >>>>> + * >>>>> + * @return The number of characters logged if succeeded. Otherwise returns >>>>> + * a negative number. >>>>> */ >>>>> -#define ODP_ERR(fmt, ...) \ >>>>> -do { fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, \ >>>>> - __LINE__, __func__, ##__VA_ARGS__); \ >>>>> -} while (0) >>>>> +extern int odp_override_log(odp_log_level_e level, const char *fmt, ...); >>>>> + >>>>> + >>>>> >>>>> /** >>>>> - * Print output to stderr (file, line and function), >>>>> - * then abort. >>>>> + * @} >>>>> */ >>>>> -#define ODP_ABORT(fmt, ...) \ >>>>> -do { fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, \ >>>>> - __LINE__, __func__, ##__VA_ARGS__); \ >>>>> - abort(); \ >>>>> -} while (0) >>>>> >>>>> #ifdef __cplusplus >>>>> } >>>>> diff --git a/platform/linux-dpdk/include/api/odp_packet.h b/platform/linux-dpdk/include/api/odp_packet.h >>>>> index 64a92e8..97c2cb6 100644 >>>>> --- a/platform/linux-dpdk/include/api/odp_packet.h >>>>> +++ b/platform/linux-dpdk/include/api/odp_packet.h >>>>> @@ -1,7 +1,7 @@ >>>>> /* Copyright (c) 2013, Linaro Limited >>>>> * All rights reserved. >>>>> * >>>>> - * SPDX-License-Identifier: BSD-3-Clause >>>>> + * SPDX-License-Identifier: BSD-3-Clause >>>>> */ >>>>> >>>>> >>>>> @@ -19,33 +19,84 @@ extern "C" { >>>>> #endif >>>>> >>>>> #include <odp_buffer.h> >>>>> +#include <odp_platform_types.h> >>>>> >>>>> +/** @defgroup odp_packet ODP PACKET >>>>> + * Operations on a packet. >>>>> + * @{ >>>>> + */ >>>>> >>>>> -/** >>>>> - * ODP packet descriptor >>>>> + >>>>> +/* >>>>> + * Packet API v0.5 notes >>>>> + * - Push/pull operations only on packet level >>>>> + * - Push/pull within limits of segment headroom/tailroom/data lengths >>>>> + * - Segment data length must be always at least one byte (i.e. there are no >>>>> + * empty segments) >>>>> + * - Head/tailroom content belong to packet content (in addition to data >>>>> + * and meta-data) and thus is preserved over packet ownership changes. >>>>> + * - _addr refer to a fixed address, which operations do not modify >>>>> + * - _ptr refer to pointer to data, which may be modified by operations >>>>> */ >>>>> -typedef odp_buffer_t odp_packet_t; >>>>> >>>>> >>>>> -/** Invalid packet */ >>>>> -#define ODP_PACKET_INVALID ODP_BUFFER_INVALID >>>>> +/* >>>>> + * >>>>> + * Alloc and free >>>>> + * ******************************************************** >>>>> + * >>>>> + */ >>>>> >>>>> -/** Invalid offset */ >>>>> -#define ODP_PACKET_OFFSET_INVALID ((size_t)-1) >>>>> +/** >>>>> + * Allocate a packet from a buffer pool >>>>> + * >>>>> + * Allocates a packet of the requested length from the specified buffer pool. >>>>> + * Pool must have been created with buffer type ODP_BUFFER_TYPE_PACKET. The >>>>> + * packet is initialized with data pointers and lengths set according to the >>>>> + * specified len, and the default headroom and tailroom length settings. All >>>>> + * other packet metadata are set to their default values. >>>>> + * >>>>> + * @param pool Pool handle >>>>> + * @param len Packet data length >>>>> + * >>>>> + * @return Handle of allocated packet >>>>> + * @retval ODP_PACKET_INVALID Packet could not be allocated >>>>> + * >>>>> + * @note The default headroom and tailroom used for packets is specified by >>>>> + * the ODP_CONFIG_PACKET_HEADROOM and ODP_CONFIG_PACKET_TAILROOM defines in >>>>> + * odp_config.h. >>>>> + */ >>>>> +odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool, uint32_t len); >>>>> >>>>> +/** >>>>> + * Free packet >>>>> + * >>>>> + * Frees the packet into the buffer pool it was allocated from. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + */ >>>>> +void odp_packet_free(odp_packet_t pkt); >>>>> >>>>> /** >>>>> - * Initialize the packet >>>>> + * Reset packet >>>>> * >>>>> - * Needs to be called if the user allocates a packet buffer, i.e. the packet >>>>> - * has not been received from I/O through ODP. >>>>> + * Resets all packet meta-data to their default values. Packet length is used >>>>> + * to initialize pointers and lengths. It must be less than the total buffer >>>>> + * length of the packet minus the default headroom length. Packet is not >>>>> + * modified on failure. >>>>> * >>>>> - * @param pkt Packet handle >>>>> + * @param pkt Packet handle >>>>> + * @param len Packet data length >>>>> + * >>>>> + * @retval 0 Success >>>>> + * @retval Non-zero Failure >>>>> + * >>>>> + * @see odp_packet_buf_len() >>>>> */ >>>>> -void odp_packet_init(odp_packet_t pkt); >>>>> +int odp_packet_reset(odp_packet_t pkt, uint32_t len); >>>>> >>>>> /** >>>>> - * Convert from packet handle to buffer handle >>>>> + * Convert a buffer handle to a packet handle >>>>> * >>>>> * @param buf Buffer handle >>>>> * >>>>> @@ -54,190 +105,737 @@ void odp_packet_init(odp_packet_t pkt); >>>>> odp_packet_t odp_packet_from_buffer(odp_buffer_t buf); >>>>> >>>>> /** >>>>> - * Convert from buffer handle to packet handle >>>>> + * Convert a packet handle to a buffer handle >>>>> * >>>>> * @param pkt Packet handle >>>>> * >>>>> * @return Buffer handle >>>>> */ >>>>> -odp_buffer_t odp_buffer_from_packet(odp_packet_t pkt); >>>>> +odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt); >>>>> + >>>>> + >>>>> +/* >>>>> + * >>>>> + * Pointers and lengths >>>>> + * ******************************************************** >>>>> + * >>>>> + */ >>>>> + >>>>> +/** >>>>> + * Packet head address >>>>> + * >>>>> + * Returns start address of the first segment. Packet level headroom starts >>>>> + * from here. Use odp_packet_data() or odp_packet_l2_ptr() to return the >>>>> + * packet data start address. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * >>>>> + * @return Pointer to the start address of the first packet segment >>>>> + * >>>>> + * @see odp_packet_data(), odp_packet_l2_ptr(), odp_packet_headroom() >>>>> + */ >>>>> +void *odp_packet_head(odp_packet_t pkt); >>>>> + >>>>> +/** >>>>> + * Total packet buffer length >>>>> + * >>>>> + * Returns sum of buffer lengths over all packet segments. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * >>>>> + * @return Total packet buffer length in bytes >>>>> + * >>>>> + * @see odp_packet_reset() >>>>> + */ >>>>> +uint32_t odp_packet_buf_len(odp_packet_t pkt); >>>>> >>>>> /** >>>>> - * Set the packet length >>>>> + * Packet data pointer >>>>> + * >>>>> + * Returns the current packet data pointer. When a packet is received >>>>> + * from packet input, this points to the first byte of the received >>>>> + * packet. Packet level offsets are calculated relative to this position. >>>>> + * >>>>> + * User can adjust the data pointer with head_push/head_pull (does not modify >>>>> + * segmentation) and add_data/rem_data calls (may modify segmentation). >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * >>>>> + * @return Pointer to the packet data >>>>> + * >>>>> + * @see odp_packet_l2_ptr(), odp_packet_seg_len() >>>>> + */ >>>>> +void *odp_packet_data(odp_packet_t pkt); >>>>> + >>>>> +/** >>>>> + * Packet segment data length >>>>> + * >>>>> + * Returns number of data bytes following the current data pointer >>>>> + * (odp_packet_data()) location in the segment. >>>>> * >>>>> * @param pkt Packet handle >>>>> - * @param len Length of packet in bytes >>>>> + * >>>>> + * @return Segment data length in bytes (pointed by odp_packet_data()) >>>>> + * >>>>> + * @see odp_packet_data() >>>>> */ >>>>> -void odp_packet_set_len(odp_packet_t pkt, size_t len); >>>>> +uint32_t odp_packet_seg_len(odp_packet_t pkt); >>>>> >>>>> /** >>>>> - * Get the packet length >>>>> + * Packet data length >>>>> + * >>>>> + * Returns sum of data lengths over all packet segments. >>>>> * >>>>> * @param pkt Packet handle >>>>> * >>>>> - * @return Packet length in bytes >>>>> + * @return Packet data length >>>>> */ >>>>> -size_t odp_packet_get_len(odp_packet_t pkt); >>>>> +uint32_t odp_packet_len(odp_packet_t pkt); >>>>> >>>>> /** >>>>> - * Set packet user context >>>>> + * Packet headroom length >>>>> * >>>>> - * @param buf Packet handle >>>>> - * @param ctx User context >>>>> + * Returns the current packet level headroom length. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> * >>>>> + * @return Headroom length >>>>> */ >>>>> -void odp_packet_set_ctx(odp_packet_t buf, const void *ctx); >>>>> +uint32_t odp_packet_headroom(odp_packet_t pkt); >>>>> >>>>> /** >>>>> - * Get packet user context >>>>> + * Packet tailroom length >>>>> * >>>>> - * @param buf Packet handle >>>>> + * Returns the current packet level tailroom length. >>>>> * >>>>> - * @return User context >>>>> + * @param pkt Packet handle >>>>> + * >>>>> + * @return Tailroom length >>>>> */ >>>>> -void *odp_packet_get_ctx(odp_packet_t buf); >>>>> +uint32_t odp_packet_tailroom(odp_packet_t pkt); >>>>> >>>>> /** >>>>> - * Get address to the start of the packet buffer >>>>> + * Packet tailroom pointer >>>>> * >>>>> - * The address of the packet buffer is not necessarily the same as the start >>>>> - * address of the received frame, e.g. an eth frame may be offset by 2 or 6 >>>>> - * bytes to ensure 32 or 64-bit alignment of the IP header. >>>>> - * Use odp_packet_l2(pkt) to get the start address of a received valid frame >>>>> - * or odp_packet_start(pkt) to get the start address even if no valid L2 header >>>>> - * could be found. >>>>> + * Returns pointer to the start of the current packet level tailroom. >>>>> + * >>>>> + * User can adjust the tail pointer with tail_push/tail_pull (does not modify >>>>> + * segmentation) and add_data/rem_data calls (may modify segmentation). >>>>> * >>>>> * @param pkt Packet handle >>>>> * >>>>> - * @return Pointer to the start of the packet buffer >>>>> + * @return Tailroom pointer >>>>> * >>>>> - * @see odp_packet_l2(), odp_packet_start() >>>>> + * @see odp_packet_tailroom() >>>>> */ >>>>> -uint8_t *odp_packet_buf_addr(odp_packet_t pkt); >>>>> +void *odp_packet_tail(odp_packet_t pkt); >>>>> >>>>> /** >>>>> - * Get pointer to the start of the received frame >>>>> + * Push out packet head >>>>> + * >>>>> + * Increase packet data length by moving packet head into packet headroom. >>>>> + * Packet headroom is decreased with the same amount. The packet head may be >>>>> + * pushed out up to 'headroom' bytes. Packet is not modified if there's not >>>>> + * enough headroom space. >>>>> * >>>>> - * The address of the packet buffer is not necessarily the same as the start >>>>> - * address of the received frame, e.g. an eth frame may be offset by 2 or 6 >>>>> - * bytes to ensure 32 or 64-bit alignment of the IP header. >>>>> - * Use odp_packet_l2(pkt) to get the start address of a received valid eth frame >>>>> + * odp_packet_xxx: >>>>> + * seg_len += len >>>>> + * len += len >>>>> + * headroom -= len >>>>> + * data -= len >>>>> * >>>>> - * odp_packet_start() will always return a pointer to the start of the frame, >>>>> - * even if the frame is unrecognized and no valid L2 header could be found. >>>>> + * Operation does not modify packet segmentation or move data. Handles and >>>>> + * pointers remain valid. User is responsible to update packet meta-data >>>>> + * offsets when needed. >>>>> * >>>>> * @param pkt Packet handle >>>>> + * @param len Number of bytes to push the head (0 ... headroom) >>>>> * >>>>> - * @return Pointer to the start of the received frame >>>>> + * @return The new data pointer >>>>> + * @retval NULL Requested offset exceeds available headroom >>>>> * >>>>> - * @see odp_packet_l2(), odp_packet_buf_addr() >>>>> + * @see odp_packet_headroom(), odp_packet_pull_head() >>>>> */ >>>>> -uint8_t *odp_packet_start(odp_packet_t pkt); >>>>> +void *odp_packet_push_head(odp_packet_t pkt, uint32_t len); >>>>> >>>>> /** >>>>> - * Get pointer to the start of the L2 frame >>>>> + * Pull in packet head >>>>> * >>>>> - * The L2 frame header address is not necessarily the same as the address of the >>>>> - * packet buffer, see odp_packet_buf_addr() >>>>> + * Decrease packet data length by removing data from the head of the packet. >>>>> + * Packet headroom is increased with the same amount. Packet head may be pulled >>>>> + * in up to seg_len - 1 bytes (i.e. packet data pointer must stay in the >>>>> + * first segment). Packet is not modified if there's not enough data. >>>>> + * >>>>> + * odp_packet_xxx: >>>>> + * seg_len -= len >>>>> + * len -= len >>>>> + * headroom += len >>>>> + * data += len >>>>> + * >>>>> + * Operation does not modify packet segmentation or move data. Handles and >>>>> + * pointers remain valid. User is responsible to update packet meta-data >>>>> + * offsets when needed. >>>>> * >>>>> * @param pkt Packet handle >>>>> + * @param len Number of bytes to pull the head (0 ... seg_len - 1) >>>>> * >>>>> - * @return Pointer to L2 header or NULL if not found >>>>> + * @return The new data pointer, or NULL in case of an error. >>>>> + * @retval NULL Requested offset exceeds packet segment length >>>>> * >>>>> - * @see odp_packet_buf_addr(), odp_packet_start() >>>>> + * @see odp_packet_seg_len(), odp_packet_push_head() >>>>> */ >>>>> -uint8_t *odp_packet_l2(odp_packet_t pkt); >>>>> +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len); >>>>> >>>>> /** >>>>> - * Return the byte offset from the packet buffer to the L2 frame >>>>> + * Push out packet tail >>>>> + * >>>>> + * Increase packet data length by moving packet tail into packet tailroom. >>>>> + * Packet tailroom is decreased with the same amount. The packet tail may be >>>>> + * pushed out up to 'tailroom' bytes. Packet is not modified if there's not >>>>> + * enough tailroom. >>>>> + * >>>>> + * last_seg: >>>>> + * data_len += len >>>>> + * >>>>> + * odp_packet_xxx: >>>>> + * len += len >>>>> + * tail += len >>>>> + * tailroom -= len >>>>> + * >>>>> + * Operation does not modify packet segmentation or move data. Handles, >>>>> + * pointers and offsets remain valid. >>>>> * >>>>> * @param pkt Packet handle >>>>> + * @param len Number of bytes to push the tail (0 ... tailroom) >>>>> * >>>>> - * @return L2 byte offset or ODP_PACKET_OFFSET_INVALID if not found >>>>> + * @return The old tail pointer >>>>> + * @retval NULL Requested offset exceeds available tailroom >>>>> + * >>>>> + * @see odp_packet_tailroom(), odp_packet_pull_tail() >>>>> */ >>>>> -size_t odp_packet_l2_offset(odp_packet_t pkt); >>>>> +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len); >>>>> >>>>> /** >>>>> - * Set the byte offset to the L2 frame >>>>> + * Pull in packet tail >>>>> + * >>>>> + * Decrease packet data length by removing data from the tail of the packet. >>>>> + * Packet tailroom is increased with the same amount. Packet tail may be pulled >>>>> + * in up to last segment data_len - 1 bytes. (i.e. packet tail must stay in the >>>>> + * last segment). Packet is not modified if there's not enough data. >>>>> + * >>>>> + * last_seg: >>>>> + * data_len -= len >>>>> + * >>>>> + * odp_packet_xxx: >>>>> + * len -= len >>>>> + * tail -= len >>>>> + * tailroom += len >>>>> + * >>>>> + * Operation does not modify packet segmentation or move data. Handles and >>>>> + * pointers remain valid. User is responsible to update packet meta-data >>>>> + * offsets when needed. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * @param len Number of bytes to pull the tail (0 ... last_seg:data_len - 1) >>>>> + * >>>>> + * @return The new tail pointer >>>>> + * @retval NULL The specified offset exceeds allowable data length >>>>> + */ >>>>> +void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len); >>>>> + >>>>> +/** >>>>> + * Packet offset pointer >>>>> + * >>>>> + * Returns pointer to data in the packet offset. The packet level byte offset is >>>>> + * calculated from the current odp_packet_data() position. Optionally outputs >>>>> + * handle to the segment and number of data bytes in the segment following the >>>>> + * pointer. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * @param offset Byte offset into the packet >>>>> + * @param[out] len Number of data bytes remaining in the segment (output). >>>>> + * Ignored when NULL. >>>>> + * @param[out] seg Handle to the segment containing the address (output). >>>>> + * Ignored when NULL. >>>>> + * >>>>> + * @return Pointer to the offset >>>>> + * @retval NULL Requested offset exceeds packet length >>>>> + */ >>>>> +void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, >>>>> + odp_packet_seg_t *seg); >>>>> + >>>>> +/* >>>>> + * >>>>> + * Meta-data >>>>> + * ******************************************************** >>>>> + * >>>>> + */ >>>>> + >>>>> +/** >>>>> + * Packet pool >>>>> + * >>>>> + * Returns handle to the buffer pool where the packet was allocated from. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * >>>>> + * @return Buffer pool handle >>>>> + */ >>>>> +odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt); >>>>> + >>>>> +/** >>>>> + * Packet input interface >>>>> + * >>>>> + * Returns handle to the packet IO interface which received the packet or >>>>> + * ODP_PKTIO_INVALID when the packet was allocated/reset by the application. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * >>>>> + * @return Packet interface handle >>>>> + * @retval ODP_PKTIO_INVALID Packet was not received >>>>> + */ >>>>> +odp_pktio_t odp_packet_input(odp_packet_t pkt); >>>>> + >>>>> +/** >>>>> + * User context pointer >>>>> + * >>>>> + * Return previously stored user context pointer. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * >>>>> + * @return User context pointer >>>>> + */ >>>>> +void *odp_packet_user_ptr(odp_packet_t pkt); >>>>> + >>>>> +/** >>>>> + * Set user context pointer >>>>> + * >>>>> + * Each packet has room for a user defined context. The context can be stored >>>>> + * either as a pointer OR as a uint64_t value, but not both at the same time. >>>>> + * The latest context set operation determines which one has been stored. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * @param ctx User context pointer >>>>> + */ >>>>> +void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ctx); >>>>> + >>>>> +/** >>>>> + * User context data (uint64_t) >>>>> + * >>>>> + * Return previously stored user context uint64_t value. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * >>>>> + * @return User context data >>>>> + */ >>>>> +uint64_t odp_packet_user_u64(odp_packet_t pkt); >>>>> + >>>>> +/** >>>>> + * Set user context data (uint64_t) >>>>> + * >>>>> + * Each packet has room for a user defined context. The context can be stored >>>>> + * either as a pointer OR as a uint64_t value, but not both at the same time. >>>>> + * The latest context set operation determines which one has been stored. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * @param ctx User context data >>>>> + */ >>>>> +void odp_packet_user_u64_set(odp_packet_t pkt, uint64_t ctx); >>>>> + >>>>> +/** >>>>> + * Layer 2 start pointer >>>>> + * >>>>> + * Returns pointer to the start of the layer 2 header. Optionally, outputs >>>>> + * number of data bytes in the segment following the pointer. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * @param[out] len Number of data bytes remaining in the segment (output). >>>>> + * Ignored when NULL. >>>>> + * >>>>> + * @return Layer 2 start pointer, or offset 0 by default >>>>> + * >>>>> + * @see odp_packet_l2_offset(), odp_packet_l2_offset_set() >>>>> + */ >>>>> +void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len); >>>>> + >>>>> +/** >>>>> + * Layer 2 start offset >>>>> + * >>>>> + * Returns offset to the start of the layer 2 header. The offset is calculated >>>>> + * from the current odp_packet_data() position in bytes. >>>>> + * >>>>> + * User is responsible to update the offset when modifying the packet data >>>>> + * pointer position. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * >>>>> + * @return Layer 2 start offset >>>>> + */ >>>>> +uint32_t odp_packet_l2_offset(odp_packet_t pkt); >>>>> + >>>>> +/** >>>>> + * Set layer 2 start offset >>>>> + * >>>>> + * Set offset to the start of the layer 2 header. The offset is calculated from >>>>> + * the current odp_packet_data() position in bytes. Offset must not exceed >>>>> + * packet data length. Packet is not modified on an error. >>>>> * >>>>> * @param pkt Packet handle >>>>> - * @param offset L2 byte offset >>>>> + * @param offset Layer 2 start offset (0 ... odp_packet_len()-1) >>>>> + * >>>>> + * @retval 0 Success >>>>> + * @retval Non-zero Failure >>>>> */ >>>>> -void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset); >>>>> +int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset); >>>>> >>>>> +/** >>>>> + * Layer 3 start pointer >>>>> + * >>>>> + * Returns pointer to the start of the layer 3 header. Optionally, outputs >>>>> + * number of data bytes in the segment following the pointer. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * @param[out] len Number of data bytes remaining in the segment (output). >>>>> + * Ignored when NULL. >>>>> + * >>>>> + * @return Layer 3 start pointer, or NULL >>>>> + * >>>>> + * @see odp_packet_l3_offset(), odp_packet_l3_offset_set() >>>>> + */ >>>>> +void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len); >>>>> >>>>> /** >>>>> - * Get pointer to the start of the L3 packet >>>>> + * Layer 3 start offset >>>>> + * >>>>> + * Returns offset to the start of the layer 3 header. The offset is calculated >>>>> + * from the current odp_packet_data() position in bytes. >>>>> + * >>>>> + * User is responsible to update the offset when modifying the packet data >>>>> + * pointer position. >>>>> * >>>>> * @param pkt Packet handle >>>>> * >>>>> - * @return Pointer to L3 packet or NULL if not found >>>>> + * @return Layer 3 start offset or ODP_PACKET_OFFSET_INVALID if not found >>>>> + */ >>>>> +uint32_t odp_packet_l3_offset(odp_packet_t pkt); >>>>> + >>>>> +/** >>>>> + * Set layer 3 start offset >>>>> + * >>>>> + * Set offset to the start of the layer 3 header. The offset is calculated from >>>>> + * the current odp_packet_data() position in bytes. Offset must not exceed >>>>> + * packet data length. Packet is not modified on an error. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * @param offset Layer 3 start offset (0 ... odp_packet_len()-1) >>>>> * >>>>> + * @retval 0 Success >>>>> + * @retval Non-zero Failure >>>>> */ >>>>> -uint8_t *odp_packet_l3(odp_packet_t pkt); >>>>> +int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset); >>>>> >>>>> /** >>>>> - * Return the byte offset from the packet buffer to the L3 packet >>>>> + * Layer 4 start pointer >>>>> + * >>>>> + * Returns pointer to the start of the layer 4 header. Optionally, outputs >>>>> + * number of data bytes in the segment following the pointer. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * @param[out] len Number of data bytes remaining in the segment (output). >>>>> + * Ignored when NULL. >>>>> + * >>>>> + * @return Layer 4 start pointer, or NULL >>>>> + * >>>>> + * @see odp_packet_l4_offset(), odp_packet_l4_offset_set() >>>>> + */ >>>>> +void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len); >>>>> + >>>>> +/** >>>>> + * Layer 4 start offset >>>>> + * >>>>> + * Returns offset to the start of the layer 4 header. The offset is calculated >>>>> + * from the current odp_packet_data() position in bytes. >>>>> + * >>>>> + * User is responsible to update the offset when modifying the packet data >>>>> + * pointer position. >>>>> * >>>>> * @param pkt Packet handle >>>>> * >>>>> - * @return L3 byte offset or ODP_PACKET_OFFSET_INVALID if not found >>>>> + * @return Layer 4 start offset or ODP_PACKET_OFFSET_INVALID if not found >>>>> */ >>>>> -size_t odp_packet_l3_offset(odp_packet_t pkt); >>>>> +uint32_t odp_packet_l4_offset(odp_packet_t pkt); >>>>> >>>>> /** >>>>> - * Set the byte offset to the L3 packet >>>>> + * Set layer 4 start offset >>>>> + * >>>>> + * Set offset to the start of the layer 4 header. The offset is calculated from >>>>> + * the current odp_packet_data() position in bytes. Offset must not exceed >>>>> + * packet data length. Packet is not modified on an error. >>>>> * >>>>> * @param pkt Packet handle >>>>> - * @param offset L3 byte offset >>>>> + * @param offset Layer 4 start offset (0 ... odp_packet_len()-1) >>>>> + * >>>>> + * @retval 0 Success >>>>> + * @retval Non-zero Failure >>>>> + */ >>>>> +int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset); >>>>> + >>>>> +/** >>>>> + * Tests if packet is segmented >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * >>>>> + * @retval 0 Packet is not segmented >>>>> + * @retval 1 Packet is segmented >>>>> + */ >>>>> +int odp_packet_is_segmented(odp_packet_t pkt); >>>>> + >>>>> +/** >>>>> + * Number of segments >>>>> + * >>>>> + * Returns number of segments in the packet. A packet has always at least one >>>>> + * segment. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * >>>>> + * @return Number of segments (>0) >>>>> + */ >>>>> +int odp_packet_num_segs(odp_packet_t pkt); >>>>> + >>>>> +/** >>>>> + * First segment in packet >>>>> + * >>>>> + * A packet has always the first segment (has at least one segment). >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * >>>>> + * @return Handle to the first segment >>>>> + */ >>>>> +odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt); >>>>> + >>>>> +/** >>>>> + * Last segment in packet >>>>> + * >>>>> + * A packet has always the last segment (has at least one segment). >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * >>>>> + * @return Handle to the last segment >>>>> + */ >>>>> +odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt); >>>>> + >>>>> +/** >>>>> + * Next segment in packet >>>>> + * >>>>> + * Returns handle to the next segment after the current segment, or >>>>> + * ODP_PACKET_SEG_INVALID if there are no more segments. Use >>>>> + * odp_packet_first_seg() to get handle to the first segment. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * @param seg Current segment handle >>>>> + * >>>>> + * @return Handle to the next segment, or ODP_PACKET_SEG_INVALID >>>>> + */ >>>>> +odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg); >>>>> + >>>>> + >>>>> +/* >>>>> + * >>>>> + * Segment level >>>>> + * ******************************************************** >>>>> + * >>>>> */ >>>>> -void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset); >>>>> >>>>> +/** >>>>> + * Segment buffer address >>>>> + * >>>>> + * Returns start address of the segment. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * @param seg Segment handle >>>>> + * >>>>> + * @return Start address of the segment, or NULL on an error >>>>> + * >>>>> + * @see odp_packet_seg_buf_len() >>>>> + */ >>>>> +void *odp_packet_seg_buf_addr(odp_packet_t pkt, odp_packet_seg_t seg); >>>>> >>>>> /** >>>>> - * Get pointer to the start of the L4 packet >>>>> + * Segment buffer length >>>>> + * >>>>> + * Returns segment buffer length in bytes. >>>>> * >>>>> * @param pkt Packet handle >>>>> + * @param seg Segment handle >>>>> * >>>>> - * @return Pointer to L4 packet or NULL if not found >>>>> + * @return Segment buffer length in bytes >>>>> * >>>>> + * @see odp_packet_seg_buf_addr() >>>>> */ >>>>> -uint8_t *odp_packet_l4(odp_packet_t pkt); >>>>> +uint32_t odp_packet_seg_buf_len(odp_packet_t pkt, odp_packet_seg_t seg); >>>>> >>>>> /** >>>>> - * Return the byte offset from the packet buffer to the L4 packet >>>>> + * Segment data pointer >>>>> + * >>>>> + * Returns pointer to the first byte of data in the segment. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * @param seg Segment handle >>>>> + * >>>>> + * @return Pointer to the segment data, or NULL on an error >>>>> + * >>>>> + * @see odp_packet_seg_data_len() >>>>> + */ >>>>> +void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg); >>>>> + >>>>> +/** >>>>> + * Segment data length >>>>> + * >>>>> + * Returns segment data length in bytes. >>>>> * >>>>> * @param pkt Packet handle >>>>> + * @param seg Segment handle >>>>> * >>>>> - * @return L4 byte offset or ODP_PACKET_OFFSET_INVALID if not found >>>>> + * @return Segment data length in bytes >>>>> + * >>>>> + * @see odp_packet_seg_data() >>>>> */ >>>>> -size_t odp_packet_l4_offset(odp_packet_t pkt); >>>>> +uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg); >>>>> + >>>>> + >>>>> +/* >>>>> + * >>>>> + * Manipulation >>>>> + * ******************************************************** >>>>> + * >>>>> + */ >>>>> + >>>>> >>>>> /** >>>>> - * Set the byte offset to the L4 packet >>>>> + * Add data into an offset >>>>> + * >>>>> + * Increases packet data length by adding new data area into the specified >>>>> + * offset. The operation returns a new packet handle on success. It may modify >>>>> + * packet segmentation and move data. Handles and pointers must be updated >>>>> + * after the operation. User is responsible to update packet meta-data offsets >>>>> + * when needed. The packet is not modified on an error. >>>>> * >>>>> * @param pkt Packet handle >>>>> - * @param offset L4 byte offset >>>>> + * @param offset Byte offset into the packet >>>>> + * @param len Number of bytes to add into the offset >>>>> + * >>>>> + * @return New packet handle, or ODP_PACKET_INVALID in case of an error. >>>>> */ >>>>> -void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset); >>>>> +odp_packet_t odp_packet_add_data(odp_packet_t pkt, uint32_t offset, >>>>> + uint32_t len); >>>>> >>>>> /** >>>>> - * Print (debug) information about the packet >>>>> + * Remove data from an offset >>>>> + * >>>>> + * Decreases packet data length by removing data from the specified offset. >>>>> + * The operation returns a new packet handle on success, and may modify >>>>> + * packet segmentation and move data. Handles and pointers must be updated >>>>> + * after the operation. User is responsible to update packet meta-data offsets >>>>> + * when needed. The packet is not modified on an error. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * @param offset Byte offset into the packet >>>>> + * @param len Number of bytes to remove from the offset >>>>> + * >>>>> + * @return New packet handle, or ODP_PACKET_INVALID in case of an error. >>>>> + */ >>>>> +odp_packet_t odp_packet_rem_data(odp_packet_t pkt, uint32_t offset, >>>>> + uint32_t len); >>>>> + >>>>> + >>>>> +/* >>>>> + * >>>>> + * Copy >>>>> + * ******************************************************** >>>>> + * >>>>> + */ >>>>> + >>>>> +/** >>>>> + * Copy packet >>>>> + * >>>>> + * Create a new copy of the packet. The new packet is exact copy of the source >>>>> + * packet (incl. data and meta-data). The pool must have been created with >>>>> + * buffer type ODP_BUFFER_TYPE_PACKET. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * @param pool Buffer pool for allocation of the new packet. >>>>> + * >>>>> + * @return Handle to the copy of the packet, or ODP_PACKET_INVALID >>>>> + */ >>>>> +odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool); >>>>> + >>>>> +/** >>>>> + * Copy data from packet >>>>> + * >>>>> + * Copy 'len' bytes of data from the packet level offset to the destination >>>>> + * address. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * @param offset Byte offset into the packet >>>>> + * @param len Number of bytes to copy >>>>> + * @param dst Destination address >>>>> + * >>>>> + * @retval 0 Success >>>>> + * @retval Non-zero Failure >>>>> + */ >>>>> +int odp_packet_copydata_out(odp_packet_t pkt, uint32_t offset, >>>>> + uint32_t len, void *dst); >>>>> + >>>>> +/** >>>>> + * Copy data into packet >>>>> + * >>>>> + * Copy 'len' bytes of data from the source address into the packet level >>>>> + * offset. Maximum number of bytes to copy is packet data length minus the >>>>> + * offset. Packet is not modified on an error. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> + * @param offset Byte offset into the packet >>>>> + * @param len Number of bytes to copy >>>>> + * @param src Source address >>>>> + * >>>>> + * @retval 0 Success >>>>> + * @retval Non-zero Failure >>>>> + */ >>>>> +int odp_packet_copydata_in(odp_packet_t pkt, uint32_t offset, >>>>> + uint32_t len, const void *src); >>>>> + >>>>> +/* >>>>> + * >>>>> + * Debugging >>>>> + * ******************************************************** >>>>> + * >>>>> + */ >>>>> + >>>>> +/** >>>>> + * Print packet to the console >>>>> + * >>>>> + * Print all packet debug information to the console. >>>>> * >>>>> * @param pkt Packet handle >>>>> */ >>>>> void odp_packet_print(odp_packet_t pkt); >>>>> >>>>> /** >>>>> - * Copy contents and metadata from pkt_src to pkt_dst >>>>> - * Useful when creating copies of packets >>>>> + * Perform full packet validity check >>>>> * >>>>> - * @param pkt_dst Destination packet >>>>> - * @param pkt_src Source packet >>>>> + * The operation may consume considerable number of cpu cycles depending on >>>>> + * the check level. >>>>> + * >>>>> + * @param pkt Packet handle >>>>> * >>>>> - * @return 0 if successful >>>>> + * @retval 0 Packet is not valid >>>>> + * @retval 1 Packet is valid >>>>> + */ >>>>> +int odp_packet_is_valid(odp_packet_t pkt); >>>>> + >>>>> + >>>>> +/** >>>>> + * @} >>>>> */ >>>>> -int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src); >>>>> >>>>> #ifdef __cplusplus >>>>> } >>>>> diff --git a/platform/linux-dpdk/include/api/odp_packet_io.h b/platform/linux-dpdk/include/api/odp_packet_io.h >>>>> index bd6868a..787df7d 100644 >>>>> --- a/platform/linux-dpdk/include/api/odp_packet_io.h >>>>> +++ b/platform/linux-dpdk/include/api/odp_packet_io.h >>>>> @@ -1,4 +1,4 @@ >>>>> -/* Copyright (c) 2014, Linaro Limited >>>>> +/* Copyright (c) 2013, Linaro Limited >>>>> * All rights reserved. >>>>> * >>>>> * SPDX-License-Identifier: BSD-3-Clause >>>>> @@ -19,29 +19,29 @@ extern "C" { >>>>> #endif >>>>> >>>>> #include <odp_std_types.h> >>>>> +#include <odp_platform_types.h> >>>>> #include <odp_buffer_pool.h> >>>>> #include <odp_packet.h> >>>>> #include <odp_queue.h> >>>>> >>>>> -#include <odp_pktio_types.h> >>>>> - >>>>> -/** ODP packet IO handle */ >>>>> -typedef uint32_t odp_pktio_t; >>>>> - >>>>> -/** Invalid packet IO handle */ >>>>> -#define ODP_PKTIO_INVALID 0 >>>>> +/** @defgroup odp_packet_io ODP PACKET IO >>>>> + * Operations on a packet. >>>>> + * @{ >>>>> + */ >>>>> >>>>> /** >>>>> * Open an ODP packet IO instance >>>>> * >>>>> * @param dev Packet IO device >>>>> * @param pool Pool to use for packet IO >>>>> - * @param params Set of parameters to pass to the arch dependent implementation >>>>> * >>>>> * @return ODP packet IO handle or ODP_PKTIO_INVALID on error >>>>> + * >>>>> + * @note dev name loop is specially pktio reserved name for >>>>> + * device used for testing. Usually it's loop back >>>>> + * interface. >>>>> */ >>>>> -odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool, >>>>> - odp_pktio_params_t *params); >>>>> +odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool); >>>>> >>>>> /** >>>>> * Close an ODP packet IO instance >>>>> @@ -130,21 +130,77 @@ void odp_pktio_set_input(odp_packet_t pkt, odp_pktio_t id); >>>>> odp_pktio_t odp_pktio_get_input(odp_packet_t pkt); >>>>> >>>>> /** >>>>> - * Defines the maximum length of mac address supported by this platform >>>>> + * Configure the MTU for a packet IO interface. >>>>> + * >>>>> + * @param[in] id ODP packet IO handle. >>>>> + * @param[in] mtu The value of MTU that the interface will be configured to >>>>> + * use. >>>>> + * >>>>> + * @retval 0 on success. >>>>> + * @retval -1 if specified mtu can not be handled. >>>>> + * @retval -1 on any other error or illegal input parameters. >>>>> + */ >>>>> +int odp_pktio_set_mtu(odp_pktio_t id, int mtu); >>>>> + >>>>> +/** >>>>> + * Return the currently configured MTU value of a packet IO interface. >>>>> + * >>>>> + * @param[in] id ODP packet IO handle. >>>>> + * >>>>> + * @retval MTU value >0 on success. >>>>> + * @retval -1 on any error or not existance pktio id. >>>>> + */ >>>>> +int odp_pktio_mtu(odp_pktio_t id); >>>>> + >>>>> +/** >>>>> + * Enable/Disable promiscuous mode on a packet IO interface. >>>>> + * >>>>> + * @param[in] id ODP packet IO handle. >>>>> + * @param[in] enable 1 to enable, 0 to disable. >>>>> + * >>>>> + * @retval 0 on success. >>>>> + * @retval non-zero on any error. >>>>> + */ >>>>> +int odp_pktio_promisc_mode_set(odp_pktio_t id, odp_bool_t enable); >>>>> + >>>>> +/** >>>>> + * Determine if promiscuous mode is enabled for a packet IO interface. >>>>> + * >>>>> + * @param[in] id ODP packet IO handle. >>>>> + * >>>>> + * @retval 1 if promiscuous mode is enabled. >>>>> + * @retval 0 if promiscuous mode is disabled. >>>>> + * @retval -1 on any error. >>>>> +*/ >>>>> +int odp_pktio_promisc_mode(odp_pktio_t id); >>>>> + >>>>> +/** >>>>> + * Get the default MAC address of a packet IO interface. >>>>> + * >>>>> + * @param id ODP packet IO handle. >>>>> + * @param[out] mac_addr Storage for MAC address of the packet IO interface. >>>>> + * @param addr_size Storage size for the address >>>>> + * >>>>> + * @retval Number of bytes written on success, 0 on failure. >>>>> */ >>>>> -#define ODP_MAC_ADDR_MAX_LENGTH ETH_ALEN >>>>> +size_t odp_pktio_mac_addr(odp_pktio_t id, void *mac_addr, >>>>> + size_t addr_size); >>>>> >>>>> /** >>>>> * Get mac address of the interface >>>>> * >>>>> - * @param id ODP packet IO handle >>>>> - * @param mac_addr Storage for Mac address of the packet IO interface >>>>> - * Storage provided by the caller should be equal >>>>> - * to ODP_MAC_ADDR_MAX_LENGTH (filled by function) >>>>> + * @param id ODP packet IO handle >>>>> + * @param mac_addr Storage for Mac address of the packet IO interface >>>>> + * Storage provided by the caller should be equal >>>>> + * to ODP_MAC_ADDR_MAX_LENGTH (filled by function) >>>>> * @return 0 on success or -1 on error >>>>> -**/ >>>>> + */ >>>>> int odp_pktio_get_mac_addr(odp_pktio_t id, unsigned char *mac_addr); >>>>> >>>>> +/** >>>>> + * @} >>>>> + */ >>>>> + >>>>> #ifdef __cplusplus >>>>> } >>>>> #endif >>>>> diff --git a/platform/linux-dpdk/include/api/odp_pktio_types.h b/platform/linux-dpdk/include/api/odp_pktio_types.h >>>>> deleted file mode 100644 >>>>> index b23e6da..0000000 >>>>> --- a/platform/linux-dpdk/include/api/odp_pktio_types.h >>>>> +++ /dev/null >>>>> @@ -1,45 +0,0 @@ >>>>> - >>>>> -/* Copyright (c) 2013, Linaro Limited >>>>> - * All rights reserved. >>>>> - * >>>>> - * SPDX-License-Identifier: BSD-3-Clause >>>>> - */ >>>>> - >>>>> -#ifndef ODP_PKTIO_TYPES_H >>>>> -#define ODP_PKTIO_TYPES_H >>>>> - >>>>> -#ifdef __cplusplus >>>>> -extern "C" { >>>>> -#endif >>>>> - >>>>> -/* We should ensure that future enum values will never overlap, otherwise >>>>> - * applications that want netmap suport might get in trouble if the odp lib >>>>> - * was not built with netmap support and there are more types define below >>>>> - */ >>>>> - >>>>> -typedef enum { >>>>> - ODP_PKTIO_TYPE_SOCKET_BASIC = 0x1, >>>>> - ODP_PKTIO_TYPE_SOCKET_MMSG, >>>>> - ODP_PKTIO_TYPE_SOCKET_MMAP, >>>>> - ODP_PKTIO_TYPE_NETMAP, >>>>> - ODP_PKTIO_TYPE_DPDK, >>>>> -} odp_pktio_type_t; >>>>> - >>>>> -#include <odp_pktio_socket.h> >>>>> -#ifdef ODP_HAVE_NETMAP >>>>> -#include <odp_pktio_netmap.h> >>>>> -#endif >>>>> - >>>>> -typedef union odp_pktio_params_t { >>>>> - odp_pktio_type_t type; >>>>> - socket_params_t sock_params; >>>>> -#ifdef ODP_HAVE_NETMAP >>>>> - netmap_params_t nm_params; >>>>> -#endif >>>>> -} odp_pktio_params_t; >>>>> - >>>>> -#ifdef __cplusplus >>>>> -} >>>>> -#endif >>>>> - >>>>> -#endif >>>>> diff --git a/platform/linux-dpdk/include/api/odp_platform_types.h b/platform/linux-dpdk/include/api/odp_platform_types.h >>>>> new file mode 100644 >>>>> index 0000000..3bedb5d >>>>> --- /dev/null >>>>> +++ b/platform/linux-dpdk/include/api/odp_platform_types.h >>>>> @@ -0,0 +1,78 @@ >>>>> +/* Copyright (c) 2014, Linaro Limited >>>>> + * All rights reserved. >>>>> + * >>>>> + * SPDX-License-Identifier: BSD-3-Clause >>>>> + */ >>>>> + >>>>> + >>>>> +/** >>>>> + * @file >>>>> + * ODP implementation types >>>>> + * This file contains all of the implementation-defined types for ODP abstract >>>>> + * definitions. Having this in one file means that other ODP API files are >>>>> + * implementation-independent and avoids circular dependencies for files that >>>>> + * refer to types managed by other components. Included here are typedefs and >>>>> + * related typed constants that are referenced by other ODP API files. >>>>> + */ >>>>> + >>>>> +#ifndef ODP_IMPL_TYPES_H_ >>>>> +#define ODP_IMPL_TYPES_H_ >>>>> + >>>>> +/** @defgroup odp_platform_types ODP PLATFORM TYPES >>>>> + * Implementation specific definitions for ODP abstract types. >>>>> + * @{ >>>>> + */ >>>>> + >>>>> +/** ODP Buffer pool */ >>>>> +typedef unsigned long odp_buffer_pool_t; >>>>> + >>>>> +/** Invalid buffer pool */ >>>>> +#define ODP_BUFFER_POOL_INVALID (0xffffffff) >>>>> + >>>>> +/** ODP buffer */ >>>>> +typedef unsigned long odp_buffer_t; >>>>> + >>>>> +/** Invalid buffer */ >>>>> +#define ODP_BUFFER_INVALID (0xffffffff) >>>>> + >>>>> +/** ODP buffer segment */ >>>>> +typedef odp_buffer_t odp_buffer_seg_t; >>>>> + >>>>> +/** Invalid segment */ >>>>> +#define ODP_SEGMENT_INVALID ODP_BUFFER_INVALID >>>>> + >>>>> +/** ODP packet */ >>>>> +typedef odp_buffer_t odp_packet_t; >>>>> + >>>>> +/** Invalid packet */ >>>>> +#define ODP_PACKET_INVALID ODP_BUFFER_INVALID >>>>> + >>>>> +/** ODP packet segment */ >>>>> +typedef odp_buffer_t odp_packet_seg_t; >>>>> + >>>>> +/** Invalid packet segment */ >>>>> +#define ODP_PACKET_SEG_INVALID ODP_BUFFER_INVALID >>>>> + >>>>> +/** ODP packet IO handle */ >>>>> +typedef uint32_t odp_pktio_t; >>>>> + >>>>> +/** Invalid packet IO handle */ >>>>> +#define ODP_PKTIO_INVALID 0 >>>>> + >>>>> +/** odp_pktio_t value to indicate any port */ >>>>> +#define ODP_PKTIO_ANY ((odp_pktio_t)~0) >>>>> + >>>>> +/** >>>>> + * ODP shared memory block >>>>> + */ >>>>> +typedef uint32_t odp_shm_t; >>>>> + >>>>> +/** Invalid shared memory block */ >>>>> +#define ODP_SHM_INVALID 0 >>>>> +#define ODP_SHM_NULL ODP_SHM_INVALID /**< Synonym for buffer pool use */ >>>>> + >>>>> +/** >>>>> + * @} >>>>> + */ >>>>> + >>>>> +#endif >>>>> diff --git a/platform/linux-dpdk/include/odp_buffer_inlines.h b/platform/linux-dpdk/include/odp_buffer_inlines.h >>>>> new file mode 100644 >>>>> index 0000000..ebf600b >>>>> --- /dev/null >>>>> +++ b/platform/linux-dpdk/include/odp_buffer_inlines.h >>>>> @@ -0,0 +1,179 @@ >>>>> +/* Copyright (c) 2014, Linaro Limited >>>>> + * All rights reserved. >>>>> + * >>>>> + * SPDX-License-Identifier: BSD-3-Clause >>>>> + */ >>>>> + >>>>> +/** >>>>> + * @file >>>>> + * >>>>> + * Inline functions for ODP buffer mgmt routines - implementation internal >>>>> + */ >>>>> + >>>>> +#ifndef ODP_BUFFER_INLINES_H_ >>>>> +#define ODP_BUFFER_INLINES_H_ >>>>> + >>>>> +#ifdef __cplusplus >>>>> +extern "C" { >>>>> +#endif >>>>> + >>>>> +static inline odp_buffer_t odp_buffer_encode_handle(odp_buffer_hdr_t *hdr) >>>>> +{ >>>>> + odp_buffer_bits_t handle; >>>>> + uint32_t pool_id = pool_handle_to_index(hdr->pool_hdl); >>>>> + struct pool_entry_s *pool = get_pool_entry(pool_id); >>>>> + >>>>> + handle.pool_id = pool_id; >>>>> + handle.index = ((uint8_t *)hdr - pool->pool_mdata_addr) / >>>>> + ODP_CACHE_LINE_SIZE; >>>>> + handle.seg = 0; >>>>> + >>>>> + return handle.u32; >>>>> +} >>>>> + >>>>> +static inline odp_buffer_t odp_hdr_to_buf(odp_buffer_hdr_t *hdr) >>>>> +{ >>>>> + return hdr->handle.handle; >>>>> +} >>>>> + >>>>> +static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf) >>>>> +{ >>>>> + return (odp_buffer_hdr_t *)buf; >>>>> +} >>>>> + >>>>> +static inline uint32_t odp_buffer_refcount(odp_buffer_hdr_t *buf) >>>>> +{ >>>>> + return odp_atomic_load_u32(&buf->ref_count); >>>>> +} >>>>> + >>>>> +static inline uint32_t odp_buffer_incr_refcount(odp_buffer_hdr_t *buf, >>>>> + uint32_t val) >>>>> +{ >>>>> + return odp_atomic_fetch_add_u32(&buf->ref_count, val) + val; >>>>> +} >>>>> + >>>>> +static inline uint32_t odp_buffer_decr_refcount(odp_buffer_hdr_t *buf, >>>>> + uint32_t val) >>>>> +{ >>>>> + uint32_t tmp; >>>>> + >>>>> + tmp = odp_atomic_fetch_sub_u32(&buf->ref_count, val); >>>>> + >>>>> + if (tmp < val) { >>>>> + odp_atomic_fetch_add_u32(&buf->ref_count, val - tmp); >>>>> + return 0; >>>>> + } else { >>>>> + return tmp - val; >>>>> + } >>>>> +} >>>>> + >>>>> +static inline odp_buffer_hdr_t *validate_buf(odp_buffer_t buf) >>>>> +{ >>>>> + odp_buffer_bits_t handle; >>>>> + odp_buffer_hdr_t *buf_hdr; >>>>> + handle.u32 = buf; >>>>> + >>>>> + /* For buffer handles, segment index must be 0 and pool id in range */ >>>>> + if (handle.seg != 0 || handle.pool_id >= ODP_CONFIG_BUFFER_POOLS) >>>>> + return NULL; >>>>> + >>>>> + pool_entry_t *pool = odp_pool_to_entry(handle.pool_id); >>>>> + >>>>> + /* If pool not created, handle is invalid */ >>>>> + if (pool->s.pool_shm == ODP_SHM_INVALID) >>>>> + return NULL; >>>>> + >>>>> + uint32_t buf_stride = pool->s.buf_stride / ODP_CACHE_LINE_SIZE; >>>>> + >>>>> + /* A valid buffer index must be on stride, and must be in range */ >>>>> + if ((handle.index % buf_stride != 0) || >>>>> + ((uint32_t)(handle.index / buf_stride) >= pool->s.params.num_bufs)) >>>>> + return NULL; >>>>> + >>>>> + buf_hdr = (odp_buffer_hdr_t *)(void *) >>>>> + (pool->s.pool_mdata_addr + >>>>> + (handle.index * ODP_CACHE_LINE_SIZE)); >>>>> + >>>>> + /* Handle is valid, so buffer is valid if it is allocated */ >>>>> + return buf_hdr->allocator == ODP_FREEBUF ? NULL : buf_hdr; >>>>> +} >>>>> + >>>>> +int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf); >>>>> + >>>>> +static inline void *buffer_map(odp_buffer_hdr_t *buf, >>>>> + uint32_t offset, >>>>> + uint32_t *seglen, >>>>> + uint32_t limit) >>>>> +{ >>>>> + int seg_index = offset / buf->segsize; >>>>> + int seg_offset = offset % buf->segsize; >>>>> + >>>>> + if (seglen != NULL) { >>>>> + uint32_t buf_left = limit - offset; >>>>> + *seglen = buf_left < buf->segsize ? >>>>> + buf_left : buf->segsize - seg_offset; >>>>> + } >>>>> + >>>>> + return (void *)(seg_offset + (uint8_t *)buf->addr[seg_index]); >>>>> +} >>>>> + >>>>> +static inline odp_buffer_seg_t segment_next(odp_buffer_hdr_t *buf, >>>>> + odp_buffer_seg_t seg) >>>>> +{ >>>>> + odp_buffer_bits_t seghandle; >>>>> + seghandle.u32 = seg; >>>>> + >>>>> + if (seg == ODP_SEGMENT_INVALID || >>>>> + seghandle.prefix != buf->handle.prefix || >>>>> + seghandle.seg >= buf->segcount - 1) >>>>> + return ODP_SEGMENT_INVALID; >>>>> + else { >>>>> + seghandle.seg++; >>>>> + return (odp_buffer_seg_t)seghandle.u32; >>>>> + } >>>>> +} >>>>> + >>>>> +static inline void *segment_map(odp_buffer_hdr_t *buf, >>>>> + odp_buffer_seg_t seg, >>>>> + uint32_t *seglen, >>>>> + uint32_t limit, >>>>> + uint32_t hr) >>>>> +{ >>>>> + uint32_t seg_offset, buf_left; >>>>> + odp_buffer_bits_t seghandle; >>>>> + uint8_t *seg_addr; >>>>> + seghandle.u32 = seg; >>>>> + >>>>> + if (seghandle.prefix != buf->handle.prefix || >>>>> + seghandle.seg >= buf->segcount) >>>>> + return NULL; >>>>> + >>>>> + seg_addr = (uint8_t *)buf->addr[seghandle.seg]; >>>>> + seg_offset = seghandle.seg * buf->segsize; >>>>> + limit += hr; >>>>> + >>>>> + /* Can't map this segment if it's nothing but headroom or tailroom */ >>>>> + if (hr >= seg_offset + buf->segsize || seg_offset > limit) >>>>> + return NULL; >>>>> + >>>>> + /* Adjust address & offset if this segment contains any headroom */ >>>>> + if (hr > seg_offset) { >>>>> + seg_addr += hr % buf->segsize; >>>>> + seg_offset += hr % buf->segsize; >>>>> + } >>>>> + >>>>> + /* Set seglen if caller is asking for it */ >>>>> + if (seglen != NULL) { >>>>> + buf_left = limit - seg_offset; >>>>> + *seglen = buf_left < buf->segsize ? buf_left : buf->segsize; >>>>> + } >>>>> + >>>>> + return (void *)seg_addr; >>>>> +} >>>>> + >>>>> + >>>>> +#ifdef __cplusplus >>>>> +} >>>>> +#endif >>>>> + >>>>> +#endif >>>>> diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h b/platform/linux-dpdk/include/odp_buffer_internal.h >>>>> index 5406606..acc958b 100644 >>>>> --- a/platform/linux-dpdk/include/odp_buffer_internal.h >>>>> +++ b/platform/linux-dpdk/include/odp_buffer_internal.h >>>>> @@ -24,50 +24,144 @@ extern "C" { >>>>> #include <odp_buffer.h> >>>>> #include <odp_debug.h> >>>>> #include <odp_align.h> >>>>> -#include <rte_mbuf.h> >>>>> +#include <odp_align_internal.h> >>>>> +#include <odp_config.h> >>>>> +#include <odp_byteorder.h> >>>>> +#include <odp_thread.h> >>>>> >>>>> -/* TODO: move these to correct files */ >>>>> +/* DPDK */ >>>>> +#include <rte_mbuf.h> >>>>> >>>>> -typedef uint64_t odp_phys_addr_t; >>>>> +#define ODP_BITSIZE(x) \ >>>>> + ((x) <= 2 ? 1 : \ >>>>> + ((x) <= 4 ? 2 : \ >>>>> + ((x) <= 8 ? 3 : \ >>>>> + ((x) <= 16 ? 4 : \ >>>>> + ((x) <= 32 ? 5 : \ >>>>> + ((x) <= 64 ? 6 : \ >>>>> + ((x) <= 128 ? 7 : \ >>>>> + ((x) <= 256 ? 8 : \ >>>>> + ((x) <= 512 ? 9 : \ >>>>> + ((x) <= 1024 ? 10 : \ >>>>> + ((x) <= 2048 ? 11 : \ >>>>> + ((x) <= 4096 ? 12 : \ >>>>> + ((x) <= 8196 ? 13 : \ >>>>> + ((x) <= 16384 ? 14 : \ >>>>> + ((x) <= 32768 ? 15 : \ >>>>> + ((x) <= 65536 ? 16 : \ >>>>> + (0/0))))))))))))))))) >>>>> + >>>>> +_ODP_STATIC_ASSERT(ODP_CONFIG_PACKET_BUF_LEN_MIN >= 256, >>>>> + "ODP Segment size must be a minimum of 256 bytes"); >>>>> + >>>>> +_ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MIN % ODP_CACHE_LINE_SIZE) == 0, >>>>> + "ODP Segment size must be a multiple of cache line size"); >>>>> + >>>>> +_ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MAX % >>>>> + ODP_CONFIG_PACKET_BUF_LEN_MIN) == 0, >>>>> + "Packet max size must be a multiple of segment size"); >>>>> + >>>>> +#define ODP_BUFFER_MAX_SEG \ >>>>> + (ODP_CONFIG_PACKET_BUF_LEN_MAX / ODP_CONFIG_PACKET_BUF_LEN_MIN) >>>>> + >>>>> +/* We can optimize storage of small raw buffers within metadata area */ >>>>> +#define ODP_MAX_INLINE_BUF ((sizeof(void *)) * (ODP_BUFFER_MAX_SEG - 1)) >>>>> + >>>>> +#define ODP_BUFFER_POOL_BITS ODP_BITSIZE(ODP_CONFIG_BUFFER_POOLS) >>>>> +#define ODP_BUFFER_SEG_BITS ODP_BITSIZE(ODP_BUFFER_MAX_SEG) >>>>> +#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS - ODP_BUFFER_SEG_BITS) >>>>> +#define ODP_BUFFER_PREFIX_BITS (ODP_BUFFER_POOL_BITS + ODP_BUFFER_INDEX_BITS) >>>>> +#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS) >>>>> +#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS) >>>>> >>>>> #define ODP_BUFFER_MAX_INDEX (ODP_BUFFER_MAX_BUFFERS - 2) >>>>> #define ODP_BUFFER_INVALID_INDEX (ODP_BUFFER_MAX_BUFFERS - 1) >>>>> >>>>> -#define ODP_BUFS_PER_CHUNK 16 >>>>> -#define ODP_BUFS_PER_SCATTER 4 >>>>> - >>>>> -#define ODP_BUFFER_TYPE_CHUNK 0xffff >>>>> - >>>>> - >>>>> -#define ODP_BUFFER_POOL_BITS 4 >>>>> -#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS) >>>>> -#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS) >>>>> -#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS) >>>>> - >>>>> typedef union odp_buffer_bits_t { >>>>> uint32_t u32; >>>>> odp_buffer_t handle; >>>>> >>>>> struct { >>>>> - uint32_t pool:ODP_BUFFER_POOL_BITS; >>>>> +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN >>>>> + uint32_t pool_id:ODP_BUFFER_POOL_BITS; >>>>> + uint32_t index:ODP_BUFFER_INDEX_BITS; >>>>> + uint32_t seg:ODP_BUFFER_SEG_BITS; >>>>> +#else >>>>> + uint32_t seg:ODP_BUFFER_SEG_BITS; >>>>> uint32_t index:ODP_BUFFER_INDEX_BITS; >>>>> + uint32_t pool_id:ODP_BUFFER_POOL_BITS; >>>>> +#endif >>>>> }; >>>>> -} odp_buffer_bits_t; >>>>> >>>>> + struct { >>>>> +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN >>>>> + uint32_t prefix:ODP_BUFFER_PREFIX_BITS; >>>>> + uint32_t pfxseg:ODP_BUFFER_SEG_BITS; >>>>> +#else >>>>> + uint32_t pfxseg:ODP_BUFFER_SEG_BITS; >>>>> + uint32_t prefix:ODP_BUFFER_PREFIX_BITS; >>>>> +#endif >>>>> + }; >>>>> +} odp_buffer_bits_t; >>>>> >>>>> /* forward declaration */ >>>>> struct odp_buffer_hdr_t; >>>>> >>>>> - >>>>> +/* Common buffer header */ >>>>> typedef struct odp_buffer_hdr_t { >>>>> + /* DPDK specific */ >>>>> struct rte_mbuf mb; /* Underlying DPDK rte_mbuf */ >>>>> - struct odp_buffer_hdr_t *next; /* Next buf in a list */ >>>>> - int type; /* ODP buffer type; not DPDK buf type */ >>>>> uint32_t index; /* Index in the rte_mempool */ >>>>> + >>>>> + struct odp_buffer_hdr_t *next; /* next buf in a list */ >>>>> + int allocator; /* allocating thread id */ >>>>> + odp_buffer_bits_t handle; /* handle */ >>>>> + union { >>>>> + uint32_t all; >>>>> + struct { >>>>> + uint32_t zeroized:1; /* Zeroize buf data on free */ >>>>> + uint32_t hdrdata:1; /* Data is in buffer hdr */ >>>>> + }; >>>>> + } flags; >>>>> + int type; /* buffer type */ >>>>> + size_t size; /* max data size */ >>>>> + odp_atomic_u32_t ref_count; /* reference count */ >>>>> + odp_buffer_pool_t pool_hdl; /* buffer pool handle */ >>>>> + union { >>>>> + uint64_t buf_u64; /* user u64 */ >>>>> + void *buf_ctx; /* user context */ >>>>> + const void *buf_cctx; /* const alias for ctx */ >>>>> + void *udata_addr; /* user metadata addr */ >>>>> + }; >>>>> + size_t udata_size; /* size of user metadata */ >>>>> + uint32_t segcount; /* segment count */ >>>>> + uint32_t segsize; /* segment size */ >>>>> + void *addr[ODP_BUFFER_MAX_SEG]; /* block addrs */ >>>>> + >>>>> } odp_buffer_hdr_t; >>>>> >>>>> -int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf); >>>>> +typedef struct odp_buffer_hdr_stride { >>>>> + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_buffer_hdr_t))]; >>>>> +} odp_buffer_hdr_stride; >>>>> +/* Ensure next header starts from 8 byte align */ >>>>> +_ODP_STATIC_ASSERT((sizeof(odp_buffer_hdr_t) % 8) == 0, >>>>> + "ODP_BUFFER_HDR_T__SIZE_ERROR"); >>>>> + >>>>> +typedef struct odp_buf_blk_t { >>>>> + struct odp_buf_blk_t *next; >>>>> + struct odp_buf_blk_t *prev; >>>>> +} odp_buf_blk_t; >>>>> + >>>>> +/* Raw buffer header */ >>>>> +typedef struct { >>>>> + odp_buffer_hdr_t buf_hdr; /* common buffer header */ >>>>> +} odp_raw_buffer_hdr_t; >>>>> + >>>>> +/* Free buffer marker */ >>>>> +#define ODP_FREEBUF -1 >>>>> >>>>> +/* Forward declarations */ >>>>> +odp_buffer_t buffer_alloc(odp_buffer_pool_t pool, size_t size); >>>>> >>>>> #ifdef __cplusplus >>>>> } >>>>> diff --git a/platform/linux-dpdk/include/odp_buffer_pool_internal.h b/platform/linux-dpdk/include/odp_buffer_pool_internal.h >>>>> index 1a36655..0f19f87 100644 >>>>> --- a/platform/linux-dpdk/include/odp_buffer_pool_internal.h >>>>> +++ b/platform/linux-dpdk/include/odp_buffer_pool_internal.h >>>>> @@ -19,16 +19,48 @@ extern "C" { >>>>> #endif >>>>> >>>>> #include <odp_std_types.h> >>>>> +#include <odp_align.h> >>>>> +#include <odp_align_internal.h> >>>>> #include <odp_buffer_pool.h> >>>>> #include <odp_buffer_internal.h> >>>>> -#include <odp_align.h> >>>>> #include <odp_hints.h> >>>>> #include <odp_config.h> >>>>> #include <odp_debug.h> >>>>> +#include <odp_shared_memory.h> >>>>> +#include <odp_atomic.h> >>>>> +#include <odp_atomic_internal.h> >>>>> +#include <string.h> >>>>> >>>>> -/* for DPDK */ >>>>> +/* DPDK */ >>>>> #include <rte_mempool.h> >>>>> >>>>> +/** >>>>> + * Buffer initialization routine prototype >>>>> + * >>>>> + * @note Routines of this type MAY be passed as part of the >>>>> + * _odp_buffer_pool_init_t structure to be called whenever a >>>>> + * buffer is allocated to initialize the user metadata >>>>> + * associated with that buffer. >>>>> + */ >>>>> +typedef void (_odp_buf_init_t)(odp_buffer_t buf, void *buf_init_arg); >>>>> + >>>>> +/** >>>>> + * Buffer pool initialization parameters >>>>> + * Used to communicate buffer pool initialization options. Internal for now. >>>>> + */ >>>>> +typedef struct _odp_buffer_pool_init_t { >>>>> + size_t udata_size; /**< Size of user metadata for each buffer */ >>>>> + _odp_buf_init_t *buf_init; /**< Buffer initialization routine to use */ >>>>> + void *buf_init_arg; /**< Argument to be passed to buf_init() */ >>>>> +} _odp_buffer_pool_init_t; /**< Type of buffer initialization struct */ >>>>> + >>>>> +/* Local cache for buffer alloc/free acceleration */ >>>>> +typedef struct local_cache_t { >>>>> + odp_buffer_hdr_t *buf_freelist; /* The local cache */ >>>>> + uint64_t bufallocs; /* Local buffer alloc count */ >>>>> + uint64_t buffrees; /* Local buffer free count */ >>>>> +} local_cache_t; >>>>> + >>>>> /* Use ticketlock instead of spinlock */ >>>>> #define POOL_USE_TICKETLOCK >>>>> >>>>> @@ -38,11 +70,16 @@ extern "C" { >>>>> >>>>> #ifdef POOL_USE_TICKETLOCK >>>>> #include <odp_ticketlock.h> >>>>> +#define POOL_LOCK(a) odp_ticketlock_lock(a) >>>>> +#define POOL_UNLOCK(a) odp_ticketlock_unlock(a) >>>>> +#define POOL_LOCK_INIT(a) odp_ticketlock_init(a) >>>>> #else >>>>> #include <odp_spinlock.h> >>>>> +#define POOL_LOCK(a) odp_spinlock_lock(a) >>>>> +#define POOL_UNLOCK(a) odp_spinlock_unlock(a) >>>>> +#define POOL_LOCK_INIT(a) odp_spinlock_init(a) >>>>> #endif >>>>> >>>>> - >>>>> struct pool_entry_s { >>>>> #ifdef POOL_USE_TICKETLOCK >>>>> odp_ticketlock_t lock ODP_ALIGNED_CACHE; >>>>> @@ -50,38 +87,294 @@ struct pool_entry_s { >>>>> odp_spinlock_t lock ODP_ALIGNED_CACHE; >>>>> #endif >>>>> >>>>> - uint64_t free_bufs; >>>>> char name[ODP_BUFFER_POOL_NAME_LEN]; >>>>> - >>>>> - >>>>> - odp_buffer_pool_t pool ODP_ALIGNED_CACHE; >>>>> - uintptr_t buf_base; >>>>> - size_t buf_size; >>>>> - size_t buf_offset; >>>>> - uint64_t num_bufs; >>>>> - void *pool_base_addr; >>>>> - uint64_t pool_size; >>>>> - size_t payload_size; >>>>> - size_t payload_align; >>>>> - int buf_type; >>>>> - size_t hdr_size; >>>>> + odp_buffer_pool_param_t params; >>>>> + _odp_buffer_pool_init_t init_params; >>>>> + odp_buffer_pool_t pool_hdl; >>>>> + uint32_t pool_id; >>>>> + odp_shm_t pool_shm; >>>>> + union { >>>>> + uint32_t all; >>>>> + struct { >>>>> + uint32_t has_name:1; >>>>> + uint32_t user_supplied_shm:1; >>>>> + uint32_t unsegmented:1; >>>>> + uint32_t zeroized:1; >>>>> + uint32_t predefined:1; >>>>> + }; >>>>> + } flags; >>>>> + uint32_t quiesced; >>>>> + uint32_t low_wm_assert; >>>>> + uint8_t *pool_base_addr; >>>>> + uint8_t *pool_mdata_addr; >>>>> + size_t pool_size; >>>>> + uint32_t buf_align; >>>>> + uint32_t buf_stride; >>>>> + _odp_atomic_ptr_t buf_freelist; >>>>> + _odp_atomic_ptr_t blk_freelist; >>>>> + odp_atomic_u32_t bufcount; >>>>> + odp_atomic_u32_t blkcount; >>>>> + odp_atomic_u64_t bufallocs; >>>>> + odp_atomic_u64_t buffrees; >>>>> + odp_atomic_u64_t blkallocs; >>>>> + odp_atomic_u64_t blkfrees; >>>>> + odp_atomic_u64_t bufempty; >>>>> + odp_atomic_u64_t blkempty; >>>>> + odp_atomic_u64_t high_wm_count; >>>>> + odp_atomic_u64_t low_wm_count; >>>>> + uint32_t seg_size; >>>>> + uint32_t high_wm; >>>>> + uint32_t low_wm; >>>>> + uint32_t headroom; >>>>> + uint32_t tailroom; >>>>> }; >>>>> >>>>> +typedef union pool_entry_u { >>>>> + struct pool_entry_s s; >>>>> + >>>>> + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct pool_entry_s))]; >>>>> +} pool_entry_t; >>>>> >>>>> extern void *pool_entry_ptr[]; >>>>> >>>>> +#if defined(ODP_CONFIG_SECURE_POOLS) && (ODP_CONFIG_SECURE_POOLS == 1) >>>>> +#define buffer_is_secure(buf) (buf->flags.zeroized) >>>>> +#define pool_is_secure(pool) (pool->flags.zeroized) >>>>> +#else >>>>> +#define buffer_is_secure(buf) 0 >>>>> +#define pool_is_secure(pool) 0 >>>>> +#endif >>>>> + >>>>> +#define TAG_ALIGN ((size_t)16) >>>>> + >>>>> +#define odp_cs(ptr, old, new) \ >>>>> + _odp_atomic_ptr_cmp_xchg_strong(&ptr, (void **)&old, (void *)new, \ >>>>> + _ODP_MEMMODEL_SC, \ >>>>> + _ODP_MEMMODEL_SC) >>>>> >>>>> -static inline void *get_pool_entry(odp_buffer_pool_t pool_id) >>>>> +/* Helper functions for pointer tagging to avoid ABA race conditions */ >>>>> +#define odp_tag(ptr) \ >>>>> + (((size_t)ptr) & (TAG_ALIGN - 1)) >>>>> + >>>>> +#define odp_detag(ptr) \ >>>>> + ((void *)(((size_t)ptr) & -TAG_ALIGN)) >>>>> + >>>>> +#define odp_retag(ptr, tag) \ >>>>> + ((void *)(((size_t)ptr) | odp_tag(tag))) >>>>> + >>>>> + >>>>> +static inline void *get_blk(struct pool_entry_s *pool) >>>>> +{ >>>>> + void *oldhead, *myhead, *newhead; >>>>> + >>>>> + oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, _ODP_MEMMODEL_ACQ); >>>>> + >>>>> + do { >>>>> + size_t tag = odp_tag(oldhead); >>>>> + myhead = odp_detag(oldhead); >>>>> + if (odp_unlikely(myhead == NULL)) >>>>> + break; >>>>> + newhead = odp_retag(((odp_buf_blk_t *)myhead)->next, tag + 1); >>>>> + } while (odp_cs(pool->blk_freelist, oldhead, newhead) == 0); >>>>> + >>>>> + if (odp_unlikely(myhead == NULL)) >>>>> + odp_atomic_inc_u64(&pool->blkempty); >>>>> + else >>>>> + odp_atomic_dec_u32(&pool->blkcount); >>>>> + >>>>> + return (void *)myhead; >>>>> +} >>>>> + >>>>> +static inline void ret_blk(struct pool_entry_s *pool, void *block) >>>>> +{ >>>>> + void *oldhead, *myhead, *myblock; >>>>> + >>>>> + oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, _ODP_MEMMODEL_ACQ); >>>>> + >>>>> + do { >>>>> + size_t tag = odp_tag(oldhead); >>>>> + myhead = odp_detag(oldhead); >>>>> + ((odp_buf_blk_t *)block)->next = myhead; >>>>> + myblock = odp_retag(block, tag + 1); >>>>> + } while (odp_cs(pool->blk_freelist, oldhead, myblock) == 0); >>>>> + >>>>> + odp_atomic_inc_u32(&pool->blkcount); >>>>> + odp_atomic_inc_u64(&pool->blkfrees); >>>>> +} >>>>> + >>>>> +static inline odp_buffer_hdr_t *get_buf(struct pool_entry_s *pool) >>>>> +{ >>>>> + odp_buffer_hdr_t *oldhead, *myhead, *newhead; >>>>> + >>>>> + oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, _ODP_MEMMODEL_ACQ); >>>>> + >>>>> + do { >>>>> + size_t tag = odp_tag(oldhead); >>>>> + myhead = odp_detag(oldhead); >>>>> + if (odp_unlikely(myhead == NULL)) >>>>> + break; >>>>> + newhead = odp_retag(myhead->next, tag + 1); >>>>> + } while (odp_cs(pool->buf_freelist, oldhead, newhead) == 0); >>>>> + >>>>> + if (odp_unlikely(myhead == NULL)) { >>>>> + odp_atomic_inc_u64(&pool->bufempty); >>>>> + } else { >>>>> + uint64_t bufcount = >>>>> + odp_atomic_fetch_sub_u32(&pool->bufcount, 1) - 1; >>>>> + >>>>> + /* Check for low watermark condition */ >>>>> + if (bufcount == pool->low_wm && !pool->low_wm_assert) { >>>>> + pool->low_wm_assert = 1; >>>>> + odp_atomic_inc_u64(&pool->low_wm_count); >>>>> + } >>>>> + >>>>> + odp_atomic_inc_u64(&pool->bufallocs); >>>>> + myhead->next = myhead; /* Mark buffer allocated */ >>>>> + myhead->allocator = odp_thread_id(); >>>>> + } >>>>> + >>>>> + return (void *)myhead; >>>>> +} >>>>> + >>>>> +static inline void ret_buf(struct pool_entry_s *pool, odp_buffer_hdr_t *buf) >>>>> +{ >>>>> + odp_buffer_hdr_t *oldhead, *myhead, *mybuf; >>>>> + >>>>> + buf->allocator = ODP_FREEBUF; /* Mark buffer free */ >>>>> + >>>>> + if (!buf->flags.hdrdata && buf->type != ODP_BUFFER_TYPE_RAW) { >>>>> + while (buf->segcount > 0) { >>>>> + if (buffer_is_secure(buf) || pool_is_secure(pool)) >>>>> + memset(buf->addr[buf->segcount - 1], >>>>> + 0, buf->segsize); >>>>> + ret_blk(pool, buf->addr[--buf->segcount]); >>>>> + } >>>>> + buf->size = 0; >>>>> + } >>>>> + >>>>> + oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, _ODP_MEMMODEL_ACQ); >>>>> + >>>>> + do { >>>>> + size_t tag = odp_tag(oldhead); >>>>> + myhead = odp_detag(oldhead); >>>>> + buf->next = myhead; >>>>> + mybuf = odp_retag(buf, tag + 1); >>>>> + } while (odp_cs(pool->buf_freelist, oldhead, mybuf) == 0); >>>>> + >>>>> + uint64_t bufcount = odp_atomic_fetch_add_u32(&pool->bufcount, 1) + 1; >>>>> + >>>>> + /* Check if low watermark condition should be deasserted */ >>>>> + if (bufcount == pool->high_wm && pool->low_wm_assert) { >>>>> + pool->low_wm_assert = 0; >>>>> + odp_atomic_inc_u64(&pool->high_wm_count); >>>>> + } >>>>> + >>>>> + odp_atomic_inc_u64(&pool->buffrees); >>>>> +} >>>>> + >>>>> +static inline void *get_local_buf(local_cache_t *buf_cache, >>>>> + struct pool_entry_s *pool, >>>>> + size_t totsize) >>>>> +{ >>>>> + odp_buffer_hdr_t *buf = buf_cache->buf_freelist; >>>>> + >>>>> + if (odp_likely(buf != NULL)) { >>>>> + buf_cache->buf_freelist = buf->next; >>>>> + >>>>> + if (odp_unlikely(buf->size < totsize)) { >>>>> + intmax_t needed = totsize - buf->size; >>>>> + >>>>> + do { >>>>> + void *blk = get_blk(pool); >>>>> + if (odp_unlikely(blk == NULL)) { >>>>> + ret_buf(pool, buf); >>>>> + buf_cache->buffrees--; >>>>> + return NULL; >>>>> + } >>>>> + buf->addr[buf->segcount++] = blk; >>>>> + needed -= pool->seg_size; >>>>> + } while (needed > 0); >>>>> + >>>>> + buf->size = buf->segcount * pool->seg_size; >>>>> + } >>>>> + >>>>> + buf_cache->bufallocs++; >>>>> + buf->allocator = odp_thread_id(); /* Mark buffer allocated */ >>>>> + } >>>>> + >>>>> + return buf; >>>>> +} >>>>> + >>>>> +static inline void ret_local_buf(local_cache_t *buf_cache, >>>>> + odp_buffer_hdr_t *buf) >>>>> +{ >>>>> + buf->allocator = ODP_FREEBUF; >>>>> + buf->next = buf_cache->buf_freelist; >>>>> + buf_cache->buf_freelist = buf; >>>>> + >>>>> + buf_cache->buffrees++; >>>>> +} >>>>> + >>>>> +static inline void flush_cache(local_cache_t *buf_cache, >>>>> + struct pool_entry_s *pool) >>>>> +{ >>>>> + odp_buffer_hdr_t *buf = buf_cache->buf_freelist; >>>>> + uint32_t flush_count = 0; >>>>> + >>>>> + while (buf != NULL) { >>>>> + odp_buffer_hdr_t *next = buf->next; >>>>> + ret_buf(pool, buf); >>>>> + buf = next; >>>>> + flush_count++; >>>>> + } >>>>> + >>>>> + odp_atomic_add_u64(&pool->bufallocs, buf_cache->bufallocs); >>>>> + odp_atomic_add_u64(&pool->buffrees, buf_cache->buffrees - flush_count); >>>>> + >>>>> + buf_cache->buf_freelist = NULL; >>>>> + buf_cache->bufallocs = 0; >>>>> + buf_cache->buffrees = 0; >>>>> +} >>>>> + >>>>> +static inline odp_buffer_pool_t pool_index_to_handle(uint32_t pool_id) >>>>> +{ >>>>> + return pool_id; >>>>> +} >>>>> + >>>>> +static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl) >>>>> +{ >>>>> + return pool_hdl; >>>>> +} >>>>> + >>>>> +static inline void *get_pool_entry(uint32_t pool_id) >>>>> { >>>>> return pool_entry_ptr[pool_id]; >>>>> } >>>>> >>>>> +static inline pool_entry_t *odp_pool_to_entry(odp_buffer_pool_t pool) >>>>> +{ >>>>> + return (pool_entry_t *)get_pool_entry(pool_handle_to_index(pool)); >>>>> +} >>>>> + >>>>> +static inline pool_entry_t *odp_buf_to_pool(odp_buffer_hdr_t *buf) >>>>> +{ >>>>> + return odp_pool_to_entry(buf->pool_hdl); >>>>> +} >>>>> >>>>> -static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf) >>>>> +static inline uint32_t odp_buffer_pool_segment_size(odp_buffer_pool_t pool) >>>>> { >>>>> - return (odp_buffer_hdr_t *)buf; >>>>> + return odp_pool_to_entry(pool)->s.seg_size; >>>>> } >>>>> >>>>> +static inline uint32_t odp_buffer_pool_headroom(odp_buffer_pool_t pool) >>>>> +{ >>>>> + return odp_pool_to_entry(pool)->s.headroom; >>>>> +} >>>>> + >>>>> +static inline uint32_t odp_buffer_pool_tailroom(odp_buffer_pool_t pool) >>>>> +{ >>>>> + return odp_pool_to_entry(pool)->s.tailroom; >>>>> +} >>>>> >>>>> #ifdef __cplusplus >>>>> } >>>>> diff --git a/platform/linux-dpdk/include/odp_packet_dpdk.h b/platform/linux-dpdk/include/odp_packet_dpdk.h >>>>> index c3a2b70..c982d5e 100644 >>>>> --- a/platform/linux-dpdk/include/odp_packet_dpdk.h >>>>> +++ b/platform/linux-dpdk/include/odp_packet_dpdk.h >>>>> @@ -8,10 +8,8 @@ >>>>> #define ODP_PACKET_DPDK_H >>>>> >>>>> #include <stdint.h> >>>>> -#include <net/if.h> >>>>> >>>>> #include <odph_eth.h> >>>>> -#include <odph_packet.h> >>>>> #include <odp_align.h> >>>>> #include <odp_debug.h> >>>>> #include <odp_packet.h> >>>>> diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h >>>>> index 1cd44b1..24de33f 100644 >>>>> --- a/platform/linux-dpdk/include/odp_packet_internal.h >>>>> +++ b/platform/linux-dpdk/include/odp_packet_internal.h >>>>> @@ -22,6 +22,7 @@ extern "C" { >>>>> #include <odp_debug.h> >>>>> #include <odp_buffer_internal.h> >>>>> #include <odp_buffer_pool_internal.h> >>>>> +#include <odp_buffer_inlines.h> >>>>> #include <odp_packet.h> >>>>> #include <odp_packet_io.h> >>>>> >>>>> @@ -43,6 +44,7 @@ typedef union { >>>>> uint32_t vlan:1; /**< VLAN hdr found */ >>>>> uint32_t vlan_qinq:1; /**< Stacked VLAN found, QinQ */ >>>>> >>>>> + uint32_t snap:1; /**< SNAP */ >>>>> uint32_t arp:1; /**< ARP */ >>>>> >>>>> uint32_t ipv4:1; /**< IPv4 */ >>>>> @@ -53,13 +55,14 @@ typedef union { >>>>> >>>>> uint32_t udp:1; /**< UDP */ >>>>> uint32_t tcp:1; /**< TCP */ >>>>> + uint32_t tcpopt:1; /**< TCP options present */ >>>>> uint32_t sctp:1; /**< SCTP */ >>>>> uint32_t icmp:1; /**< ICMP */ >>>>> }; >>>>> } input_flags_t; >>>>> >>>>> -ODP_STATIC_ASSERT(sizeof(input_flags_t) == sizeof(uint32_t), >>>>> - "INPUT_FLAGS_SIZE_ERROR"); >>>>> +_ODP_STATIC_ASSERT(sizeof(input_flags_t) == sizeof(uint32_t), >>>>> + "INPUT_FLAGS_SIZE_ERROR"); >>>>> >>>>> /** >>>>> * Packet error flags >>>>> @@ -70,7 +73,9 @@ typedef union { >>>>> >>>>> struct { >>>>> /* Bitfield flags for each detected error */ >>>>> + uint32_t app_error:1; /**< Error bit for application use */ >>>>> uint32_t frame_len:1; /**< Frame length error */ >>>>> + uint32_t snap_len:1; /**< Snap length error */ >>>>> uint32_t l2_chksum:1; /**< L2 checksum error, checks TBD */ >>>>> uint32_t ip_err:1; /**< IP error, checks TBD */ >>>>> uint32_t tcp_err:1; /**< TCP error, checks TBD */ >>>>> @@ -78,8 +83,8 @@ typedef union { >>>>> }; >>>>> } error_flags_t; >>>>> >>>>> -ODP_STATIC_ASSERT(sizeof(error_flags_t) == sizeof(uint32_t), >>>>> - "ERROR_FLAGS_SIZE_ERROR"); >>>>> +_ODP_STATIC_ASSERT(sizeof(error_flags_t) == sizeof(uint32_t), >>>>> + "ERROR_FLAGS_SIZE_ERROR"); >>>>> >>>>> /** >>>>> * Packet output flags >>>>> @@ -90,12 +95,15 @@ typedef union { >>>>> >>>>> struct { >>>>> /* Bitfield flags for each output option */ >>>>> - uint32_t l4_chksum:1; /**< Request L4 checksum calculation */ >>>>> + uint32_t l3_chksum_set:1; /**< L3 chksum bit is valid */ >>>>> + uint32_t l3_chksum:1; /**< L3 chksum override */ >>>>> + uint32_t l4_chksum_set:1; /**< L3 chksum bit is valid */ >>>>> + uint32_t l4_chksum:1; /**< L4 chksum override */ >>>>> }; >>>>> } output_flags_t; >>>>> >>>>> -ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), >>>>> - "OUTPUT_FLAGS_SIZE_ERROR"); >>>>> +_ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), >>>>> + "OUTPUT_FLAGS_SIZE_ERROR"); >>>>> >>>>> /** >>>>> * Internal Packet header >>>>> @@ -108,28 +116,131 @@ typedef struct { >>>>> error_flags_t error_flags; >>>>> output_flags_t output_flags; >>>>> >>>>> - uint32_t frame_offset; /**< offset to start of frame, even on error */ >>>>> uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */ >>>>> uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */ >>>>> uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ >>>>> + uint32_t payload_offset; /**< offset to payload */ >>>>> >>>>> - uint64_t user_ctx; /**< user context */ >>>>> + uint32_t vlan_s_tag; /**< Parsed 1st VLAN header (S-TAG) */ >>>>> + uint32_t vlan_c_tag; /**< Parsed 2nd VLAN header (C-TAG) */ >>>>> + uint32_t l3_protocol; /**< Parsed L3 protocol */ >>>>> + uint32_t l3_len; /**< Layer 3 length */ >>>>> + uint32_t l4_protocol; /**< Parsed L4 protocol */ >>>>> + uint32_t l4_len; /**< Layer 4 length */ >>>>> + >>>>> + uint32_t frame_len; >>>>> + uint32_t headroom; >>>>> + uint32_t tailroom; >>>>> >>>>> odp_pktio_t input; >>>>> } odp_packet_hdr_t; >>>>> >>>>> +typedef struct odp_packet_hdr_stride { >>>>> + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_packet_hdr_t))]; >>>>> +} odp_packet_hdr_stride; >>>>> + >>>>> +_ODP_STATIC_ASSERT(sizeof(odp_packet_hdr_t) % sizeof(uint64_t) == 0, >>>>> + "ODP_PACKET_HDR_T__SIZE_ERR2"); >>>>> + >>>>> /** >>>>> * Return the packet header >>>>> */ >>>>> static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) >>>>> { >>>>> - return (odp_packet_hdr_t *)pkt; >>>>> + return (odp_packet_hdr_t *)odp_buf_to_hdr((odp_buffer_t)pkt); >>>>> } >>>>> >>>>> /** >>>>> - * Parse packet and set internal metadata >>>>> + * Initialize packet buffer >>>>> */ >>>>> -void odp_packet_parse(odp_packet_t pkt, size_t len, size_t l2_offset); >>>>> +static inline void packet_init(pool_entry_t *pool, >>>>> + odp_packet_hdr_t *pkt_hdr, >>>>> + size_t size) >>>>> +{ >>>>> + /* >>>>> + * Reset parser metadata. Note that we clear via memset to make >>>>> + * this routine indepenent of any additional adds to packet metadata. >>>>> + */ >>>>> + struct rte_mbuf *mb; >>>>> + uint8_t *start; >>>>> + >>>>> + mb = (struct rte_mbuf *)&pkt_hdr->buf_hdr; >>>>> + mb->buf_len = size; >>>>> + start = (uint8_t *)mb->buf_addr; >>>>> + memset(start, 0, mb->buf_len); >>>>> + >>>>> + /* >>>>> + * Packet headroom is set from the pool's headroom >>>>> + * Packet tailroom is rounded up to fill the last >>>>> + * segment occupied by the allocated length. >>>>> + */ >>>>> + pkt_hdr->frame_len = size; >>>>> + pkt_hdr->headroom = pool->s.headroom; >>>>> + pkt_hdr->tailroom = >>>>> + (pool->s.seg_size * pkt_hdr->buf_hdr.segcount) - >>>>> + (pool->s.headroom + size); >>>>> +} >>>>> + >>>>> +static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, >>>>> + uint32_t offset, uint32_t *seglen) >>>>> +{ >>>>> + struct rte_mbuf *mbuf = (struct rte_mbuf *)&pkt_hdr->buf_hdr; >>>>> + if (offset > pkt_hdr->frame_len) >>>>> + return NULL; >>>>> + >>>>> + if (seglen != NULL) { >>>>> + *seglen = mbuf->pkt.data_len; >>>>> + pkt_hdr->frame_len = mbuf->pkt.data_len; >>>>> + } >>>>> + return (void *)((uint8_t *)mbuf->pkt.data + offset); >>>>> +} >>>>> + >>>>> +#define pull_offset(x, len) (x = x < len ? 0 : x - len) >>>>> + >>>>> +static inline void push_head(odp_packet_hdr_t *pkt_hdr, size_t len) >>>>> +{ >>>>> + pkt_hdr->headroom -= len; >>>>> + pkt_hdr->frame_len += len; >>>>> + pkt_hdr->l2_offset += len; >>>>> + pkt_hdr->l3_offset += len; >>>>> + pkt_hdr->l4_offset += len; >>>>> +} >>>>> + >>>>> +static inline void pull_head(odp_packet_hdr_t *pkt_hdr, size_t len) >>>>> +{ >>>>> + pkt_hdr->headroom += len; >>>>> + pkt_hdr->frame_len -= len; >>>>> + pull_offset(pkt_hdr->l2_offset, len); >>>>> + pull_offset(pkt_hdr->l3_offset, len); >>>>> + pull_offset(pkt_hdr->l4_offset, len); >>>>> +} >>>>> + >>>>> +static inline void push_tail(odp_packet_hdr_t *pkt_hdr, size_t len) >>>>> +{ >>>>> + pkt_hdr->tailroom -= len; >>>>> + pkt_hdr->frame_len += len; >>>>> +} >>>>> + >>>>> + >>>>> +static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, size_t len) >>>>> +{ >>>>> + pkt_hdr->tailroom += len; >>>>> + pkt_hdr->frame_len -= len; >>>>> +} >>>>> + >>>>> +static inline void packet_set_len(odp_packet_t pkt, uint32_t len) >>>>> +{ >>>>> + odp_packet_hdr(pkt)->frame_len = len; >>>>> +} >>>>> + >>>>> +/* Forward declarations */ >>>>> +int _odp_packet_copy_to_packet(odp_packet_t srcpkt, uint32_t srcoffset, >>>>> + > > ...
diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index db11e0a..6491d3a 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -29,6 +29,7 @@ include_HEADERS = \ $(top_srcdir)/platform/linux-generic/include/api/odp_barrier.h \ $(srcdir)/include/api/odp_buffer_pool.h \ $(top_srcdir)/platform/linux-generic/include/api/odp_byteorder.h \ + $(top_srcdir)/platform/linux-generic/include/api/odp_classification.h \ $(top_srcdir)/platform/linux-generic/include/api/odp_compiler.h \ $(top_srcdir)/platform/linux-generic/include/api/odp_config.h \ $(top_srcdir)/platform/linux-generic/include/api/odp_coremask.h \ @@ -52,15 +53,16 @@ include_HEADERS = \ $(top_srcdir)/platform/linux-generic/include/api/odp_time.h \ $(top_srcdir)/platform/linux-generic/include/api/odp_timer.h \ $(top_srcdir)/platform/linux-generic/include/api/odp_version.h \ - $(srcdir)/include/api/odp_pktio_types.h + $(srcdir)/include/api/odp_platform_types.h subdirheadersdir = $(includedir)/helper subdirheaders_HEADERS = \ $(top_srcdir)/helper/include/odph_chksum.h \ $(top_srcdir)/helper/include/odph_eth.h \ + $(top_srcdir)/helper/include/odph_icmp.h \ $(top_srcdir)/helper/include/odph_ip.h \ + $(top_srcdir)/helper/include/odph_ipsec.h \ $(top_srcdir)/helper/include/odph_linux.h \ - $(top_srcdir)/helper/include/odph_packet.h \ $(top_srcdir)/helper/include/odph_ring.h \ $(top_srcdir)/helper/include/odph_udp.h @@ -68,9 +70,11 @@ __LIB__libodp_la_SOURCES = \ ../linux-generic/odp_barrier.c \ odp_buffer.c \ odp_buffer_pool.c \ + ../linux-generic/odp_classification.c \ ../linux-generic/odp_coremask.c \ ../linux-generic/odp_crypto.c \ odp_init.c \ + ../linux-generic/odp_impl.c \ odp_linux.c \ odp_packet.c \ odp_packet_dpdk.c \ @@ -80,11 +84,12 @@ __LIB__libodp_la_SOURCES = \ odp_queue.c \ ../linux-generic/odp_ring.c \ ../linux-generic/odp_rwlock.c \ - odp_schedule.c \ + ../linux-generic/odp_schedule.c \ ../linux-generic/odp_shared_memory.c \ ../linux-generic/odp_spinlock.c \ ../linux-generic/odp_system_info.c \ ../linux-generic/odp_thread.c \ ../linux-generic/odp_ticketlock.c \ ../linux-generic/odp_time.c \ - ../linux-generic/odp_timer.c + ../linux-generic/odp_timer.c \ + ../linux-generic/odp_weak.c diff --git a/platform/linux-dpdk/include/api/odp_buffer.h b/platform/linux-dpdk/include/api/odp_buffer.h index b2fbc76..3c23035 100644 --- a/platform/linux-dpdk/include/api/odp_buffer.h +++ b/platform/linux-dpdk/include/api/odp_buffer.h @@ -19,20 +19,13 @@ extern "C" { #endif - #include <odp_std_types.h> +#include <odp_platform_types.h> - - - - -/** - * ODP buffer +/** @defgroup odp_buffer ODP BUFFER + * Operations on a buffer. + * @{ */ -typedef unsigned long odp_buffer_t; - - -#define ODP_BUFFER_INVALID (unsigned long)(-1L) /**< Invalid buffer */ /** @@ -74,11 +67,21 @@ int odp_buffer_type(odp_buffer_t buf); * * @param buf Buffer handle * - * @return 1 if valid, otherwise 0 + * @retval 1 Buffer handle represents a valid buffer. + * @retval 0 Buffer handle does not represent a valid buffer. */ int odp_buffer_is_valid(odp_buffer_t buf); /** + * Buffer pool of the buffer + * + * @param buf Buffer handle + * + * @return Handle of buffer pool buffer belongs to + */ +odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf); + +/** * Print buffer metadata to STDOUT * * @param buf Buffer handle @@ -86,6 +89,9 @@ int odp_buffer_is_valid(odp_buffer_t buf); */ void odp_buffer_print(odp_buffer_t buf); +/** + * @} + */ #ifdef __cplusplus } diff --git a/platform/linux-dpdk/include/api/odp_buffer_pool.h b/platform/linux-dpdk/include/api/odp_buffer_pool.h index 382f4f0..4da5f84 100644 --- a/platform/linux-dpdk/include/api/odp_buffer_pool.h +++ b/platform/linux-dpdk/include/api/odp_buffer_pool.h @@ -21,66 +21,143 @@ extern "C" { #include <odp_std_types.h> +#include <odp_platform_types.h> #include <odp_buffer.h> +/** @addtogroup odp_buffer + * Operations on a buffer pool. + * @{ + */ + /** Maximum queue name lenght in chars */ #define ODP_BUFFER_POOL_NAME_LEN 32 -/** Invalid buffer pool */ -#define ODP_BUFFER_POOL_INVALID (unsigned long)(-1L) - -/** ODP buffer pool */ -typedef unsigned long odp_buffer_pool_t; - +/** + * Buffer pool parameters + * Used to communicate buffer pool creation options. + */ +typedef struct odp_buffer_pool_param_t { + uint32_t buf_size; /**< Buffer size in bytes. The maximum + number of bytes application will + store in each buffer. For packets, this + is the maximum packet data length, and + configured headroom and tailroom will be + added to this number */ + uint32_t buf_align; /**< Minimum buffer alignment in bytes. + Valid values are powers of two. Use 0 + for default alignment. Default will + always be a multiple of 8. */ + uint32_t num_bufs; /**< Number of buffers in the pool */ + int buf_type; /**< Buffer type */ +} odp_buffer_pool_param_t; /** * Create a buffer pool + * This routine is used to create a buffer pool. It take three + * arguments: the optional name of the pool to be created, an optional shared + * memory handle, and a parameter struct that describes the pool to be + * created. If a name is not specified the result is an anonymous pool that + * cannot be referenced by odp_buffer_pool_lookup(). + * + * @param name Name of the pool, max ODP_BUFFER_POOL_NAME_LEN-1 chars. + * May be specified as NULL for anonymous pools. + * + * @param shm The shared memory object in which to create the pool. + * Use ODP_SHM_NULL to reserve default memory type + * for the buffer type. * - * @param name Name of the pool (max ODP_BUFFER_POOL_NAME_LEN - 1 chars) - * @param base_addr Pool base address - * @param size Pool size in bytes - * @param buf_size Buffer size in bytes - * @param buf_align Minimum buffer alignment - * @param buf_type Buffer type + * @param params Buffer pool parameters. * - * @return Buffer pool handle + * @return Handle of the created buffer pool + * @retval ODP_BUFFER_POOL_INVALID Buffer pool could not be created */ + odp_buffer_pool_t odp_buffer_pool_create(const char *name, - void *base_addr, uint64_t size, - size_t buf_size, size_t buf_align, - int buf_type); + odp_shm_t shm, + odp_buffer_pool_param_t *params); +/** + * Destroy a buffer pool previously created by odp_buffer_pool_create() + * + * @param pool Handle of the buffer pool to be destroyed + * + * @retval 0 Success + * @retval -1 Failure + * + * @note This routine destroys a previously created buffer pool. This call + * does not destroy any shared memory object passed to + * odp_buffer_pool_create() used to store the buffer pool contents. The caller + * takes responsibility for that. If no shared memory object was passed as + * part of the create call, then this routine will destroy any internal shared + * memory objects associated with the buffer pool. Results are undefined if + * an attempt is made to destroy a buffer pool that contains allocated or + * otherwise active buffers. + */ +int odp_buffer_pool_destroy(odp_buffer_pool_t pool); /** * Find a buffer pool by name * * @param name Name of the pool * - * @return Buffer pool handle, or ODP_BUFFER_POOL_INVALID if not found. + * @return Handle of found buffer pool + * @retval ODP_BUFFER_POOL_INVALID Buffer pool could not be found + * + * @note This routine cannot be used to look up an anonymous pool (one created + * with no name). */ odp_buffer_pool_t odp_buffer_pool_lookup(const char *name); +/** + * Buffer pool information struct + * Used to get information about a buffer pool. + */ +typedef struct odp_buffer_pool_info_t { + const char *name; /**< pool name */ + odp_shm_t shm; /**< handle of shared memory area + supplied by application to + contain buffer pool, or + ODP_SHM_NULL if this pool is + managed by ODP */ + odp_buffer_pool_param_t params; /**< pool parameters */ +} odp_buffer_pool_info_t; + +/** + * Retrieve information about a buffer pool + * + * @param pool Buffer pool handle + * + * @param[out] info Receives an odp_buffer_pool_info_t object + * that describes the pool. + * + * @retval 0 Success + * @retval -1 Failure. Info could not be retrieved. + */ + +int odp_buffer_pool_info(odp_buffer_pool_t pool, + odp_buffer_pool_info_t *info); /** * Print buffer pool info * * @param pool Pool handle * + * @note This routine writes implementation-defined information about the + * specified buffer pool to the ODP log. The intended use is for debugging. */ void odp_buffer_pool_print(odp_buffer_pool_t pool); - - /** * Buffer alloc * + * The validity of a buffer can be cheked at any time with odp_buffer_is_valid() * @param pool Pool handle * - * @return Buffer handle or ODP_BUFFER_INVALID + * @return Handle of allocated buffer + * @retval ODP_BUFFER_INVALID Buffer could not be allocated */ odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool); - /** * Buffer free * @@ -89,8 +166,9 @@ odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool); */ void odp_buffer_free(odp_buffer_t buf); - - +/** + * @} + */ #ifdef __cplusplus } diff --git a/platform/linux-dpdk/include/api/odp_debug.h b/platform/linux-dpdk/include/api/odp_debug.h index 6b145b5..aed8a5f 100644 --- a/platform/linux-dpdk/include/api/odp_debug.h +++ b/platform/linux-dpdk/include/api/odp_debug.h @@ -12,73 +12,74 @@ #ifndef ODP_DEBUG_H_ #define ODP_DEBUG_H_ -#include <stdio.h> -#include <stdlib.h> #ifdef __cplusplus extern "C" { #endif +/** @addtogroup odp_ver_abt_log_dbg + * Macros that allows different messages. + * @{ + */ + #ifdef __GNUC__ -/** - * Indicate deprecated variables, functions or types - */ -#define ODP_DEPRECATED __attribute__((__deprecated__)) + +#if __GNUC__ < 4 || (__GNUC__ == 4 && (__GNUC_MINOR__ < 6)) /** - * Intentionally unused variables ot functions + * _Static_assert was only added in GCC 4.6. Provide a weak replacement + * for previous versions. */ -#define ODP_UNUSED __attribute__((__unused__)) +#define _Static_assert(e, s) extern int (*static_assert_checker(void)) \ + [sizeof(struct { unsigned int error_if_negative:(e) ? 1 : -1; })] -#else +#endif -#define ODP_DEPRECATED -#define ODP_UNUSED -#endif -/** - * Runtime assertion-macro - aborts if 'cond' is false. - */ -#ifndef ODP_NO_DEBUG -#define ODP_ASSERT(cond, msg) \ - do { if (!(cond)) {ODP_ERR("%s\n", msg); abort(); } } while (0) -#else -#define ODP_ASSERT(cond, msg) #endif + /** * Compile time assertion-macro - fail compilation if cond is false. * @note This macro has zero runtime overhead */ -#define ODP_STATIC_ASSERT(cond, msg) _Static_assert(1, msg) +#define _ODP_STATIC_ASSERT(cond, msg) _Static_assert(1, msg) /** - * Debug printing macro, which prints output when DEBUG flag is set. + * ODP log level. */ -#define ODP_DBG(fmt, ...) \ - do { if (ODP_DEBUG_PRINT == 1) \ - printf(fmt, ##__VA_ARGS__); \ - } while (0) +typedef enum odp_log_level { + ODP_LOG_DBG, + ODP_LOG_ERR, + ODP_LOG_UNIMPLEMENTED, + ODP_LOG_ABORT, + ODP_LOG_PRINT +} odp_log_level_e; /** - * Print output to stderr (file, line and function). + * ODP log function + * + * Instead of direct prints to stdout/stderr all logging in ODP implementation + * should be done via this function or its wrappers. + * ODP platform MUST provide a default *weak* implementation of this function. + * Application MAY override the function if needed by providing a strong + * function. + * + * @param[in] level Log level + * @param[in] fmt printf-style message format + * + * @return The number of characters logged if succeeded. Otherwise returns + * a negative number. */ -#define ODP_ERR(fmt, ...) \ -do { fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, \ - __LINE__, __func__, ##__VA_ARGS__); \ -} while (0) +extern int odp_override_log(odp_log_level_e level, const char *fmt, ...); + + /** - * Print output to stderr (file, line and function), - * then abort. + * @} */ -#define ODP_ABORT(fmt, ...) \ -do { fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, \ - __LINE__, __func__, ##__VA_ARGS__); \ - abort(); \ -} while (0) #ifdef __cplusplus } diff --git a/platform/linux-dpdk/include/api/odp_packet.h b/platform/linux-dpdk/include/api/odp_packet.h index 64a92e8..97c2cb6 100644 --- a/platform/linux-dpdk/include/api/odp_packet.h +++ b/platform/linux-dpdk/include/api/odp_packet.h @@ -1,7 +1,7 @@ /* Copyright (c) 2013, Linaro Limited * All rights reserved. * - * SPDX-License-Identifier: BSD-3-Clause + * SPDX-License-Identifier: BSD-3-Clause */ @@ -19,33 +19,84 @@ extern "C" { #endif #include <odp_buffer.h> +#include <odp_platform_types.h> +/** @defgroup odp_packet ODP PACKET + * Operations on a packet. + * @{ + */ -/** - * ODP packet descriptor + +/* + * Packet API v0.5 notes + * - Push/pull operations only on packet level + * - Push/pull within limits of segment headroom/tailroom/data lengths + * - Segment data length must be always at least one byte (i.e. there are no + * empty segments) + * - Head/tailroom content belong to packet content (in addition to data + * and meta-data) and thus is preserved over packet ownership changes. + * - _addr refer to a fixed address, which operations do not modify + * - _ptr refer to pointer to data, which may be modified by operations */ -typedef odp_buffer_t odp_packet_t; -/** Invalid packet */ -#define ODP_PACKET_INVALID ODP_BUFFER_INVALID +/* + * + * Alloc and free + * ******************************************************** + * + */ -/** Invalid offset */ -#define ODP_PACKET_OFFSET_INVALID ((size_t)-1) +/** + * Allocate a packet from a buffer pool + * + * Allocates a packet of the requested length from the specified buffer pool. + * Pool must have been created with buffer type ODP_BUFFER_TYPE_PACKET. The + * packet is initialized with data pointers and lengths set according to the + * specified len, and the default headroom and tailroom length settings. All + * other packet metadata are set to their default values. + * + * @param pool Pool handle + * @param len Packet data length + * + * @return Handle of allocated packet + * @retval ODP_PACKET_INVALID Packet could not be allocated + * + * @note The default headroom and tailroom used for packets is specified by + * the ODP_CONFIG_PACKET_HEADROOM and ODP_CONFIG_PACKET_TAILROOM defines in + * odp_config.h. + */ +odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool, uint32_t len); +/** + * Free packet + * + * Frees the packet into the buffer pool it was allocated from. + * + * @param pkt Packet handle + */ +void odp_packet_free(odp_packet_t pkt); /** - * Initialize the packet + * Reset packet * - * Needs to be called if the user allocates a packet buffer, i.e. the packet - * has not been received from I/O through ODP. + * Resets all packet meta-data to their default values. Packet length is used + * to initialize pointers and lengths. It must be less than the total buffer + * length of the packet minus the default headroom length. Packet is not + * modified on failure. * - * @param pkt Packet handle + * @param pkt Packet handle + * @param len Packet data length + * + * @retval 0 Success + * @retval Non-zero Failure + * + * @see odp_packet_buf_len() */ -void odp_packet_init(odp_packet_t pkt); +int odp_packet_reset(odp_packet_t pkt, uint32_t len); /** - * Convert from packet handle to buffer handle + * Convert a buffer handle to a packet handle * * @param buf Buffer handle * @@ -54,190 +105,737 @@ void odp_packet_init(odp_packet_t pkt); odp_packet_t odp_packet_from_buffer(odp_buffer_t buf); /** - * Convert from buffer handle to packet handle + * Convert a packet handle to a buffer handle * * @param pkt Packet handle * * @return Buffer handle */ -odp_buffer_t odp_buffer_from_packet(odp_packet_t pkt); +odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt); + + +/* + * + * Pointers and lengths + * ******************************************************** + * + */ + +/** + * Packet head address + * + * Returns start address of the first segment. Packet level headroom starts + * from here. Use odp_packet_data() or odp_packet_l2_ptr() to return the + * packet data start address. + * + * @param pkt Packet handle + * + * @return Pointer to the start address of the first packet segment + * + * @see odp_packet_data(), odp_packet_l2_ptr(), odp_packet_headroom() + */ +void *odp_packet_head(odp_packet_t pkt); + +/** + * Total packet buffer length + * + * Returns sum of buffer lengths over all packet segments. + * + * @param pkt Packet handle + * + * @return Total packet buffer length in bytes + * + * @see odp_packet_reset() + */ +uint32_t odp_packet_buf_len(odp_packet_t pkt); /** - * Set the packet length + * Packet data pointer + * + * Returns the current packet data pointer. When a packet is received + * from packet input, this points to the first byte of the received + * packet. Packet level offsets are calculated relative to this position. + * + * User can adjust the data pointer with head_push/head_pull (does not modify + * segmentation) and add_data/rem_data calls (may modify segmentation). + * + * @param pkt Packet handle + * + * @return Pointer to the packet data + * + * @see odp_packet_l2_ptr(), odp_packet_seg_len() + */ +void *odp_packet_data(odp_packet_t pkt); + +/** + * Packet segment data length + * + * Returns number of data bytes following the current data pointer + * (odp_packet_data()) location in the segment. * * @param pkt Packet handle - * @param len Length of packet in bytes + * + * @return Segment data length in bytes (pointed by odp_packet_data()) + * + * @see odp_packet_data() */ -void odp_packet_set_len(odp_packet_t pkt, size_t len); +uint32_t odp_packet_seg_len(odp_packet_t pkt); /** - * Get the packet length + * Packet data length + * + * Returns sum of data lengths over all packet segments. * * @param pkt Packet handle * - * @return Packet length in bytes + * @return Packet data length */ -size_t odp_packet_get_len(odp_packet_t pkt); +uint32_t odp_packet_len(odp_packet_t pkt); /** - * Set packet user context + * Packet headroom length * - * @param buf Packet handle - * @param ctx User context + * Returns the current packet level headroom length. + * + * @param pkt Packet handle * + * @return Headroom length */ -void odp_packet_set_ctx(odp_packet_t buf, const void *ctx); +uint32_t odp_packet_headroom(odp_packet_t pkt); /** - * Get packet user context + * Packet tailroom length * - * @param buf Packet handle + * Returns the current packet level tailroom length. * - * @return User context + * @param pkt Packet handle + * + * @return Tailroom length */ -void *odp_packet_get_ctx(odp_packet_t buf); +uint32_t odp_packet_tailroom(odp_packet_t pkt); /** - * Get address to the start of the packet buffer + * Packet tailroom pointer * - * The address of the packet buffer is not necessarily the same as the start - * address of the received frame, e.g. an eth frame may be offset by 2 or 6 - * bytes to ensure 32 or 64-bit alignment of the IP header. - * Use odp_packet_l2(pkt) to get the start address of a received valid frame - * or odp_packet_start(pkt) to get the start address even if no valid L2 header - * could be found. + * Returns pointer to the start of the current packet level tailroom. + * + * User can adjust the tail pointer with tail_push/tail_pull (does not modify + * segmentation) and add_data/rem_data calls (may modify segmentation). * * @param pkt Packet handle * - * @return Pointer to the start of the packet buffer + * @return Tailroom pointer * - * @see odp_packet_l2(), odp_packet_start() + * @see odp_packet_tailroom() */ -uint8_t *odp_packet_buf_addr(odp_packet_t pkt); +void *odp_packet_tail(odp_packet_t pkt); /** - * Get pointer to the start of the received frame + * Push out packet head + * + * Increase packet data length by moving packet head into packet headroom. + * Packet headroom is decreased with the same amount. The packet head may be + * pushed out up to 'headroom' bytes. Packet is not modified if there's not + * enough headroom space. * - * The address of the packet buffer is not necessarily the same as the start - * address of the received frame, e.g. an eth frame may be offset by 2 or 6 - * bytes to ensure 32 or 64-bit alignment of the IP header. - * Use odp_packet_l2(pkt) to get the start address of a received valid eth frame + * odp_packet_xxx: + * seg_len += len + * len += len + * headroom -= len + * data -= len * - * odp_packet_start() will always return a pointer to the start of the frame, - * even if the frame is unrecognized and no valid L2 header could be found. + * Operation does not modify packet segmentation or move data. Handles and + * pointers remain valid. User is responsible to update packet meta-data + * offsets when needed. * * @param pkt Packet handle + * @param len Number of bytes to push the head (0 ... headroom) * - * @return Pointer to the start of the received frame + * @return The new data pointer + * @retval NULL Requested offset exceeds available headroom * - * @see odp_packet_l2(), odp_packet_buf_addr() + * @see odp_packet_headroom(), odp_packet_pull_head() */ -uint8_t *odp_packet_start(odp_packet_t pkt); +void *odp_packet_push_head(odp_packet_t pkt, uint32_t len); /** - * Get pointer to the start of the L2 frame + * Pull in packet head * - * The L2 frame header address is not necessarily the same as the address of the - * packet buffer, see odp_packet_buf_addr() + * Decrease packet data length by removing data from the head of the packet. + * Packet headroom is increased with the same amount. Packet head may be pulled + * in up to seg_len - 1 bytes (i.e. packet data pointer must stay in the + * first segment). Packet is not modified if there's not enough data. + * + * odp_packet_xxx: + * seg_len -= len + * len -= len + * headroom += len + * data += len + * + * Operation does not modify packet segmentation or move data. Handles and + * pointers remain valid. User is responsible to update packet meta-data + * offsets when needed. * * @param pkt Packet handle + * @param len Number of bytes to pull the head (0 ... seg_len - 1) * - * @return Pointer to L2 header or NULL if not found + * @return The new data pointer, or NULL in case of an error. + * @retval NULL Requested offset exceeds packet segment length * - * @see odp_packet_buf_addr(), odp_packet_start() + * @see odp_packet_seg_len(), odp_packet_push_head() */ -uint8_t *odp_packet_l2(odp_packet_t pkt); +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len); /** - * Return the byte offset from the packet buffer to the L2 frame + * Push out packet tail + * + * Increase packet data length by moving packet tail into packet tailroom. + * Packet tailroom is decreased with the same amount. The packet tail may be + * pushed out up to 'tailroom' bytes. Packet is not modified if there's not + * enough tailroom. + * + * last_seg: + * data_len += len + * + * odp_packet_xxx: + * len += len + * tail += len + * tailroom -= len + * + * Operation does not modify packet segmentation or move data. Handles, + * pointers and offsets remain valid. * * @param pkt Packet handle + * @param len Number of bytes to push the tail (0 ... tailroom) * - * @return L2 byte offset or ODP_PACKET_OFFSET_INVALID if not found + * @return The old tail pointer + * @retval NULL Requested offset exceeds available tailroom + * + * @see odp_packet_tailroom(), odp_packet_pull_tail() */ -size_t odp_packet_l2_offset(odp_packet_t pkt); +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len); /** - * Set the byte offset to the L2 frame + * Pull in packet tail + * + * Decrease packet data length by removing data from the tail of the packet. + * Packet tailroom is increased with the same amount. Packet tail may be pulled + * in up to last segment data_len - 1 bytes. (i.e. packet tail must stay in the + * last segment). Packet is not modified if there's not enough data. + * + * last_seg: + * data_len -= len + * + * odp_packet_xxx: + * len -= len + * tail -= len + * tailroom += len + * + * Operation does not modify packet segmentation or move data. Handles and + * pointers remain valid. User is responsible to update packet meta-data + * offsets when needed. + * + * @param pkt Packet handle + * @param len Number of bytes to pull the tail (0 ... last_seg:data_len - 1) + * + * @return The new tail pointer + * @retval NULL The specified offset exceeds allowable data length + */ +void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len); + +/** + * Packet offset pointer + * + * Returns pointer to data in the packet offset. The packet level byte offset is + * calculated from the current odp_packet_data() position. Optionally outputs + * handle to the segment and number of data bytes in the segment following the + * pointer. + * + * @param pkt Packet handle + * @param offset Byte offset into the packet + * @param[out] len Number of data bytes remaining in the segment (output). + * Ignored when NULL. + * @param[out] seg Handle to the segment containing the address (output). + * Ignored when NULL. + * + * @return Pointer to the offset + * @retval NULL Requested offset exceeds packet length + */ +void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, + odp_packet_seg_t *seg); + +/* + * + * Meta-data + * ******************************************************** + * + */ + +/** + * Packet pool + * + * Returns handle to the buffer pool where the packet was allocated from. + * + * @param pkt Packet handle + * + * @return Buffer pool handle + */ +odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt); + +/** + * Packet input interface + * + * Returns handle to the packet IO interface which received the packet or + * ODP_PKTIO_INVALID when the packet was allocated/reset by the application. + * + * @param pkt Packet handle + * + * @return Packet interface handle + * @retval ODP_PKTIO_INVALID Packet was not received + */ +odp_pktio_t odp_packet_input(odp_packet_t pkt); + +/** + * User context pointer + * + * Return previously stored user context pointer. + * + * @param pkt Packet handle + * + * @return User context pointer + */ +void *odp_packet_user_ptr(odp_packet_t pkt); + +/** + * Set user context pointer + * + * Each packet has room for a user defined context. The context can be stored + * either as a pointer OR as a uint64_t value, but not both at the same time. + * The latest context set operation determines which one has been stored. + * + * @param pkt Packet handle + * @param ctx User context pointer + */ +void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ctx); + +/** + * User context data (uint64_t) + * + * Return previously stored user context uint64_t value. + * + * @param pkt Packet handle + * + * @return User context data + */ +uint64_t odp_packet_user_u64(odp_packet_t pkt); + +/** + * Set user context data (uint64_t) + * + * Each packet has room for a user defined context. The context can be stored + * either as a pointer OR as a uint64_t value, but not both at the same time. + * The latest context set operation determines which one has been stored. + * + * @param pkt Packet handle + * @param ctx User context data + */ +void odp_packet_user_u64_set(odp_packet_t pkt, uint64_t ctx); + +/** + * Layer 2 start pointer + * + * Returns pointer to the start of the layer 2 header. Optionally, outputs + * number of data bytes in the segment following the pointer. + * + * @param pkt Packet handle + * @param[out] len Number of data bytes remaining in the segment (output). + * Ignored when NULL. + * + * @return Layer 2 start pointer, or offset 0 by default + * + * @see odp_packet_l2_offset(), odp_packet_l2_offset_set() + */ +void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len); + +/** + * Layer 2 start offset + * + * Returns offset to the start of the layer 2 header. The offset is calculated + * from the current odp_packet_data() position in bytes. + * + * User is responsible to update the offset when modifying the packet data + * pointer position. + * + * @param pkt Packet handle + * + * @return Layer 2 start offset + */ +uint32_t odp_packet_l2_offset(odp_packet_t pkt); + +/** + * Set layer 2 start offset + * + * Set offset to the start of the layer 2 header. The offset is calculated from + * the current odp_packet_data() position in bytes. Offset must not exceed + * packet data length. Packet is not modified on an error. * * @param pkt Packet handle - * @param offset L2 byte offset + * @param offset Layer 2 start offset (0 ... odp_packet_len()-1) + * + * @retval 0 Success + * @retval Non-zero Failure */ -void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset); +int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset); +/** + * Layer 3 start pointer + * + * Returns pointer to the start of the layer 3 header. Optionally, outputs + * number of data bytes in the segment following the pointer. + * + * @param pkt Packet handle + * @param[out] len Number of data bytes remaining in the segment (output). + * Ignored when NULL. + * + * @return Layer 3 start pointer, or NULL + * + * @see odp_packet_l3_offset(), odp_packet_l3_offset_set() + */ +void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len); /** - * Get pointer to the start of the L3 packet + * Layer 3 start offset + * + * Returns offset to the start of the layer 3 header. The offset is calculated + * from the current odp_packet_data() position in bytes. + * + * User is responsible to update the offset when modifying the packet data + * pointer position. * * @param pkt Packet handle * - * @return Pointer to L3 packet or NULL if not found + * @return Layer 3 start offset or ODP_PACKET_OFFSET_INVALID if not found + */ +uint32_t odp_packet_l3_offset(odp_packet_t pkt); + +/** + * Set layer 3 start offset + * + * Set offset to the start of the layer 3 header. The offset is calculated from + * the current odp_packet_data() position in bytes. Offset must not exceed + * packet data length. Packet is not modified on an error. + * + * @param pkt Packet handle + * @param offset Layer 3 start offset (0 ... odp_packet_len()-1) * + * @retval 0 Success + * @retval Non-zero Failure */ -uint8_t *odp_packet_l3(odp_packet_t pkt); +int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset); /** - * Return the byte offset from the packet buffer to the L3 packet + * Layer 4 start pointer + * + * Returns pointer to the start of the layer 4 header. Optionally, outputs + * number of data bytes in the segment following the pointer. + * + * @param pkt Packet handle + * @param[out] len Number of data bytes remaining in the segment (output). + * Ignored when NULL. + * + * @return Layer 4 start pointer, or NULL + * + * @see odp_packet_l4_offset(), odp_packet_l4_offset_set() + */ +void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len); + +/** + * Layer 4 start offset + * + * Returns offset to the start of the layer 4 header. The offset is calculated + * from the current odp_packet_data() position in bytes. + * + * User is responsible to update the offset when modifying the packet data + * pointer position. * * @param pkt Packet handle * - * @return L3 byte offset or ODP_PACKET_OFFSET_INVALID if not found + * @return Layer 4 start offset or ODP_PACKET_OFFSET_INVALID if not found */ -size_t odp_packet_l3_offset(odp_packet_t pkt); +uint32_t odp_packet_l4_offset(odp_packet_t pkt); /** - * Set the byte offset to the L3 packet + * Set layer 4 start offset + * + * Set offset to the start of the layer 4 header. The offset is calculated from + * the current odp_packet_data() position in bytes. Offset must not exceed + * packet data length. Packet is not modified on an error. * * @param pkt Packet handle - * @param offset L3 byte offset + * @param offset Layer 4 start offset (0 ... odp_packet_len()-1) + * + * @retval 0 Success + * @retval Non-zero Failure + */ +int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset); + +/** + * Tests if packet is segmented + * + * @param pkt Packet handle + * + * @retval 0 Packet is not segmented + * @retval 1 Packet is segmented + */ +int odp_packet_is_segmented(odp_packet_t pkt); + +/** + * Number of segments + * + * Returns number of segments in the packet. A packet has always at least one + * segment. + * + * @param pkt Packet handle + * + * @return Number of segments (>0) + */ +int odp_packet_num_segs(odp_packet_t pkt); + +/** + * First segment in packet + * + * A packet has always the first segment (has at least one segment). + * + * @param pkt Packet handle + * + * @return Handle to the first segment + */ +odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt); + +/** + * Last segment in packet + * + * A packet has always the last segment (has at least one segment). + * + * @param pkt Packet handle + * + * @return Handle to the last segment + */ +odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt); + +/** + * Next segment in packet + * + * Returns handle to the next segment after the current segment, or + * ODP_PACKET_SEG_INVALID if there are no more segments. Use + * odp_packet_first_seg() to get handle to the first segment. + * + * @param pkt Packet handle + * @param seg Current segment handle + * + * @return Handle to the next segment, or ODP_PACKET_SEG_INVALID + */ +odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg); + + +/* + * + * Segment level + * ******************************************************** + * */ -void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset); +/** + * Segment buffer address + * + * Returns start address of the segment. + * + * @param pkt Packet handle + * @param seg Segment handle + * + * @return Start address of the segment, or NULL on an error + * + * @see odp_packet_seg_buf_len() + */ +void *odp_packet_seg_buf_addr(odp_packet_t pkt, odp_packet_seg_t seg); /** - * Get pointer to the start of the L4 packet + * Segment buffer length + * + * Returns segment buffer length in bytes. * * @param pkt Packet handle + * @param seg Segment handle * - * @return Pointer to L4 packet or NULL if not found + * @return Segment buffer length in bytes * + * @see odp_packet_seg_buf_addr() */ -uint8_t *odp_packet_l4(odp_packet_t pkt); +uint32_t odp_packet_seg_buf_len(odp_packet_t pkt, odp_packet_seg_t seg); /** - * Return the byte offset from the packet buffer to the L4 packet + * Segment data pointer + * + * Returns pointer to the first byte of data in the segment. + * + * @param pkt Packet handle + * @param seg Segment handle + * + * @return Pointer to the segment data, or NULL on an error + * + * @see odp_packet_seg_data_len() + */ +void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg); + +/** + * Segment data length + * + * Returns segment data length in bytes. * * @param pkt Packet handle + * @param seg Segment handle * - * @return L4 byte offset or ODP_PACKET_OFFSET_INVALID if not found + * @return Segment data length in bytes + * + * @see odp_packet_seg_data() */ -size_t odp_packet_l4_offset(odp_packet_t pkt); +uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg); + + +/* + * + * Manipulation + * ******************************************************** + * + */ + /** - * Set the byte offset to the L4 packet + * Add data into an offset + * + * Increases packet data length by adding new data area into the specified + * offset. The operation returns a new packet handle on success. It may modify + * packet segmentation and move data. Handles and pointers must be updated + * after the operation. User is responsible to update packet meta-data offsets + * when needed. The packet is not modified on an error. * * @param pkt Packet handle - * @param offset L4 byte offset + * @param offset Byte offset into the packet + * @param len Number of bytes to add into the offset + * + * @return New packet handle, or ODP_PACKET_INVALID in case of an error. */ -void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset); +odp_packet_t odp_packet_add_data(odp_packet_t pkt, uint32_t offset, + uint32_t len); /** - * Print (debug) information about the packet + * Remove data from an offset + * + * Decreases packet data length by removing data from the specified offset. + * The operation returns a new packet handle on success, and may modify + * packet segmentation and move data. Handles and pointers must be updated + * after the operation. User is responsible to update packet meta-data offsets + * when needed. The packet is not modified on an error. + * + * @param pkt Packet handle + * @param offset Byte offset into the packet + * @param len Number of bytes to remove from the offset + * + * @return New packet handle, or ODP_PACKET_INVALID in case of an error. + */ +odp_packet_t odp_packet_rem_data(odp_packet_t pkt, uint32_t offset, + uint32_t len); + + +/* + * + * Copy + * ******************************************************** + * + */ + +/** + * Copy packet + * + * Create a new copy of the packet. The new packet is exact copy of the source + * packet (incl. data and meta-data). The pool must have been created with + * buffer type ODP_BUFFER_TYPE_PACKET. + * + * @param pkt Packet handle + * @param pool Buffer pool for allocation of the new packet. + * + * @return Handle to the copy of the packet, or ODP_PACKET_INVALID + */ +odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool); + +/** + * Copy data from packet + * + * Copy 'len' bytes of data from the packet level offset to the destination + * address. + * + * @param pkt Packet handle + * @param offset Byte offset into the packet + * @param len Number of bytes to copy + * @param dst Destination address + * + * @retval 0 Success + * @retval Non-zero Failure + */ +int odp_packet_copydata_out(odp_packet_t pkt, uint32_t offset, + uint32_t len, void *dst); + +/** + * Copy data into packet + * + * Copy 'len' bytes of data from the source address into the packet level + * offset. Maximum number of bytes to copy is packet data length minus the + * offset. Packet is not modified on an error. + * + * @param pkt Packet handle + * @param offset Byte offset into the packet + * @param len Number of bytes to copy + * @param src Source address + * + * @retval 0 Success + * @retval Non-zero Failure + */ +int odp_packet_copydata_in(odp_packet_t pkt, uint32_t offset, + uint32_t len, const void *src); + +/* + * + * Debugging + * ******************************************************** + * + */ + +/** + * Print packet to the console + * + * Print all packet debug information to the console. * * @param pkt Packet handle */ void odp_packet_print(odp_packet_t pkt); /** - * Copy contents and metadata from pkt_src to pkt_dst - * Useful when creating copies of packets + * Perform full packet validity check * - * @param pkt_dst Destination packet - * @param pkt_src Source packet + * The operation may consume considerable number of cpu cycles depending on + * the check level. + * + * @param pkt Packet handle * - * @return 0 if successful + * @retval 0 Packet is not valid + * @retval 1 Packet is valid + */ +int odp_packet_is_valid(odp_packet_t pkt); + + +/** + * @} */ -int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src); #ifdef __cplusplus } diff --git a/platform/linux-dpdk/include/api/odp_packet_io.h b/platform/linux-dpdk/include/api/odp_packet_io.h index bd6868a..787df7d 100644 --- a/platform/linux-dpdk/include/api/odp_packet_io.h +++ b/platform/linux-dpdk/include/api/odp_packet_io.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014, Linaro Limited +/* Copyright (c) 2013, Linaro Limited * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -19,29 +19,29 @@ extern "C" { #endif #include <odp_std_types.h> +#include <odp_platform_types.h> #include <odp_buffer_pool.h> #include <odp_packet.h> #include <odp_queue.h> -#include <odp_pktio_types.h> - -/** ODP packet IO handle */ -typedef uint32_t odp_pktio_t; - -/** Invalid packet IO handle */ -#define ODP_PKTIO_INVALID 0 +/** @defgroup odp_packet_io ODP PACKET IO + * Operations on a packet. + * @{ + */ /** * Open an ODP packet IO instance * * @param dev Packet IO device * @param pool Pool to use for packet IO - * @param params Set of parameters to pass to the arch dependent implementation * * @return ODP packet IO handle or ODP_PKTIO_INVALID on error + * + * @note dev name loop is specially pktio reserved name for + * device used for testing. Usually it's loop back + * interface. */ -odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool, - odp_pktio_params_t *params); +odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool); /** * Close an ODP packet IO instance @@ -130,21 +130,77 @@ void odp_pktio_set_input(odp_packet_t pkt, odp_pktio_t id); odp_pktio_t odp_pktio_get_input(odp_packet_t pkt); /** - * Defines the maximum length of mac address supported by this platform + * Configure the MTU for a packet IO interface. + * + * @param[in] id ODP packet IO handle. + * @param[in] mtu The value of MTU that the interface will be configured to + * use. + * + * @retval 0 on success. + * @retval -1 if specified mtu can not be handled. + * @retval -1 on any other error or illegal input parameters. + */ +int odp_pktio_set_mtu(odp_pktio_t id, int mtu); + +/** + * Return the currently configured MTU value of a packet IO interface. + * + * @param[in] id ODP packet IO handle. + * + * @retval MTU value >0 on success. + * @retval -1 on any error or not existance pktio id. + */ +int odp_pktio_mtu(odp_pktio_t id); + +/** + * Enable/Disable promiscuous mode on a packet IO interface. + * + * @param[in] id ODP packet IO handle. + * @param[in] enable 1 to enable, 0 to disable. + * + * @retval 0 on success. + * @retval non-zero on any error. + */ +int odp_pktio_promisc_mode_set(odp_pktio_t id, odp_bool_t enable); + +/** + * Determine if promiscuous mode is enabled for a packet IO interface. + * + * @param[in] id ODP packet IO handle. + * + * @retval 1 if promiscuous mode is enabled. + * @retval 0 if promiscuous mode is disabled. + * @retval -1 on any error. +*/ +int odp_pktio_promisc_mode(odp_pktio_t id); + +/** + * Get the default MAC address of a packet IO interface. + * + * @param id ODP packet IO handle. + * @param[out] mac_addr Storage for MAC address of the packet IO interface. + * @param addr_size Storage size for the address + * + * @retval Number of bytes written on success, 0 on failure. */ -#define ODP_MAC_ADDR_MAX_LENGTH ETH_ALEN +size_t odp_pktio_mac_addr(odp_pktio_t id, void *mac_addr, + size_t addr_size); /** * Get mac address of the interface * - * @param id ODP packet IO handle - * @param mac_addr Storage for Mac address of the packet IO interface - * Storage provided by the caller should be equal - * to ODP_MAC_ADDR_MAX_LENGTH (filled by function) + * @param id ODP packet IO handle + * @param mac_addr Storage for Mac address of the packet IO interface + * Storage provided by the caller should be equal + * to ODP_MAC_ADDR_MAX_LENGTH (filled by function) * @return 0 on success or -1 on error -**/ + */ int odp_pktio_get_mac_addr(odp_pktio_t id, unsigned char *mac_addr); +/** + * @} + */ + #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/include/api/odp_pktio_types.h b/platform/linux-dpdk/include/api/odp_pktio_types.h deleted file mode 100644 index b23e6da..0000000 --- a/platform/linux-dpdk/include/api/odp_pktio_types.h +++ /dev/null @@ -1,45 +0,0 @@ - -/* Copyright (c) 2013, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_PKTIO_TYPES_H -#define ODP_PKTIO_TYPES_H - -#ifdef __cplusplus -extern "C" { -#endif - -/* We should ensure that future enum values will never overlap, otherwise - * applications that want netmap suport might get in trouble if the odp lib - * was not built with netmap support and there are more types define below - */ - -typedef enum { - ODP_PKTIO_TYPE_SOCKET_BASIC = 0x1, - ODP_PKTIO_TYPE_SOCKET_MMSG, - ODP_PKTIO_TYPE_SOCKET_MMAP, - ODP_PKTIO_TYPE_NETMAP, - ODP_PKTIO_TYPE_DPDK, -} odp_pktio_type_t; - -#include <odp_pktio_socket.h> -#ifdef ODP_HAVE_NETMAP -#include <odp_pktio_netmap.h> -#endif - -typedef union odp_pktio_params_t { - odp_pktio_type_t type; - socket_params_t sock_params; -#ifdef ODP_HAVE_NETMAP - netmap_params_t nm_params; -#endif -} odp_pktio_params_t; - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-dpdk/include/api/odp_platform_types.h b/platform/linux-dpdk/include/api/odp_platform_types.h new file mode 100644 index 0000000..3bedb5d --- /dev/null +++ b/platform/linux-dpdk/include/api/odp_platform_types.h @@ -0,0 +1,78 @@ +/* Copyright (c) 2014, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + + +/** + * @file + * ODP implementation types + * This file contains all of the implementation-defined types for ODP abstract + * definitions. Having this in one file means that other ODP API files are + * implementation-independent and avoids circular dependencies for files that + * refer to types managed by other components. Included here are typedefs and + * related typed constants that are referenced by other ODP API files. + */ + +#ifndef ODP_IMPL_TYPES_H_ +#define ODP_IMPL_TYPES_H_ + +/** @defgroup odp_platform_types ODP PLATFORM TYPES + * Implementation specific definitions for ODP abstract types. + * @{ + */ + +/** ODP Buffer pool */ +typedef unsigned long odp_buffer_pool_t; + +/** Invalid buffer pool */ +#define ODP_BUFFER_POOL_INVALID (0xffffffff) + +/** ODP buffer */ +typedef unsigned long odp_buffer_t; + +/** Invalid buffer */ +#define ODP_BUFFER_INVALID (0xffffffff) + +/** ODP buffer segment */ +typedef odp_buffer_t odp_buffer_seg_t; + +/** Invalid segment */ +#define ODP_SEGMENT_INVALID ODP_BUFFER_INVALID + +/** ODP packet */ +typedef odp_buffer_t odp_packet_t; + +/** Invalid packet */ +#define ODP_PACKET_INVALID ODP_BUFFER_INVALID + +/** ODP packet segment */ +typedef odp_buffer_t odp_packet_seg_t; + +/** Invalid packet segment */ +#define ODP_PACKET_SEG_INVALID ODP_BUFFER_INVALID + +/** ODP packet IO handle */ +typedef uint32_t odp_pktio_t; + +/** Invalid packet IO handle */ +#define ODP_PKTIO_INVALID 0 + +/** odp_pktio_t value to indicate any port */ +#define ODP_PKTIO_ANY ((odp_pktio_t)~0) + +/** + * ODP shared memory block + */ +typedef uint32_t odp_shm_t; + +/** Invalid shared memory block */ +#define ODP_SHM_INVALID 0 +#define ODP_SHM_NULL ODP_SHM_INVALID /**< Synonym for buffer pool use */ + +/** + * @} + */ + +#endif diff --git a/platform/linux-dpdk/include/odp_buffer_inlines.h b/platform/linux-dpdk/include/odp_buffer_inlines.h new file mode 100644 index 0000000..ebf600b --- /dev/null +++ b/platform/linux-dpdk/include/odp_buffer_inlines.h @@ -0,0 +1,179 @@ +/* Copyright (c) 2014, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * Inline functions for ODP buffer mgmt routines - implementation internal + */ + +#ifndef ODP_BUFFER_INLINES_H_ +#define ODP_BUFFER_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline odp_buffer_t odp_buffer_encode_handle(odp_buffer_hdr_t *hdr) +{ + odp_buffer_bits_t handle; + uint32_t pool_id = pool_handle_to_index(hdr->pool_hdl); + struct pool_entry_s *pool = get_pool_entry(pool_id); + + handle.pool_id = pool_id; + handle.index = ((uint8_t *)hdr - pool->pool_mdata_addr) / + ODP_CACHE_LINE_SIZE; + handle.seg = 0; + + return handle.u32; +} + +static inline odp_buffer_t odp_hdr_to_buf(odp_buffer_hdr_t *hdr) +{ + return hdr->handle.handle; +} + +static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf) +{ + return (odp_buffer_hdr_t *)buf; +} + +static inline uint32_t odp_buffer_refcount(odp_buffer_hdr_t *buf) +{ + return odp_atomic_load_u32(&buf->ref_count); +} + +static inline uint32_t odp_buffer_incr_refcount(odp_buffer_hdr_t *buf, + uint32_t val) +{ + return odp_atomic_fetch_add_u32(&buf->ref_count, val) + val; +} + +static inline uint32_t odp_buffer_decr_refcount(odp_buffer_hdr_t *buf, + uint32_t val) +{ + uint32_t tmp; + + tmp = odp_atomic_fetch_sub_u32(&buf->ref_count, val); + + if (tmp < val) { + odp_atomic_fetch_add_u32(&buf->ref_count, val - tmp); + return 0; + } else { + return tmp - val; + } +} + +static inline odp_buffer_hdr_t *validate_buf(odp_buffer_t buf) +{ + odp_buffer_bits_t handle; + odp_buffer_hdr_t *buf_hdr; + handle.u32 = buf; + + /* For buffer handles, segment index must be 0 and pool id in range */ + if (handle.seg != 0 || handle.pool_id >= ODP_CONFIG_BUFFER_POOLS) + return NULL; + + pool_entry_t *pool = odp_pool_to_entry(handle.pool_id); + + /* If pool not created, handle is invalid */ + if (pool->s.pool_shm == ODP_SHM_INVALID) + return NULL; + + uint32_t buf_stride = pool->s.buf_stride / ODP_CACHE_LINE_SIZE; + + /* A valid buffer index must be on stride, and must be in range */ + if ((handle.index % buf_stride != 0) || + ((uint32_t)(handle.index / buf_stride) >= pool->s.params.num_bufs)) + return NULL; + + buf_hdr = (odp_buffer_hdr_t *)(void *) + (pool->s.pool_mdata_addr + + (handle.index * ODP_CACHE_LINE_SIZE)); + + /* Handle is valid, so buffer is valid if it is allocated */ + return buf_hdr->allocator == ODP_FREEBUF ? NULL : buf_hdr; +} + +int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf); + +static inline void *buffer_map(odp_buffer_hdr_t *buf, + uint32_t offset, + uint32_t *seglen, + uint32_t limit) +{ + int seg_index = offset / buf->segsize; + int seg_offset = offset % buf->segsize; + + if (seglen != NULL) { + uint32_t buf_left = limit - offset; + *seglen = buf_left < buf->segsize ? + buf_left : buf->segsize - seg_offset; + } + + return (void *)(seg_offset + (uint8_t *)buf->addr[seg_index]); +} + +static inline odp_buffer_seg_t segment_next(odp_buffer_hdr_t *buf, + odp_buffer_seg_t seg) +{ + odp_buffer_bits_t seghandle; + seghandle.u32 = seg; + + if (seg == ODP_SEGMENT_INVALID || + seghandle.prefix != buf->handle.prefix || + seghandle.seg >= buf->segcount - 1) + return ODP_SEGMENT_INVALID; + else { + seghandle.seg++; + return (odp_buffer_seg_t)seghandle.u32; + } +} + +static inline void *segment_map(odp_buffer_hdr_t *buf, + odp_buffer_seg_t seg, + uint32_t *seglen, + uint32_t limit, + uint32_t hr) +{ + uint32_t seg_offset, buf_left; + odp_buffer_bits_t seghandle; + uint8_t *seg_addr; + seghandle.u32 = seg; + + if (seghandle.prefix != buf->handle.prefix || + seghandle.seg >= buf->segcount) + return NULL; + + seg_addr = (uint8_t *)buf->addr[seghandle.seg]; + seg_offset = seghandle.seg * buf->segsize; + limit += hr; + + /* Can't map this segment if it's nothing but headroom or tailroom */ + if (hr >= seg_offset + buf->segsize || seg_offset > limit) + return NULL; + + /* Adjust address & offset if this segment contains any headroom */ + if (hr > seg_offset) { + seg_addr += hr % buf->segsize; + seg_offset += hr % buf->segsize; + } + + /* Set seglen if caller is asking for it */ + if (seglen != NULL) { + buf_left = limit - seg_offset; + *seglen = buf_left < buf->segsize ? buf_left : buf->segsize; + } + + return (void *)seg_addr; +} + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h b/platform/linux-dpdk/include/odp_buffer_internal.h index 5406606..acc958b 100644 --- a/platform/linux-dpdk/include/odp_buffer_internal.h +++ b/platform/linux-dpdk/include/odp_buffer_internal.h @@ -24,50 +24,144 @@ extern "C" { #include <odp_buffer.h> #include <odp_debug.h> #include <odp_align.h> -#include <rte_mbuf.h> +#include <odp_align_internal.h> +#include <odp_config.h> +#include <odp_byteorder.h> +#include <odp_thread.h> -/* TODO: move these to correct files */ +/* DPDK */ +#include <rte_mbuf.h> -typedef uint64_t odp_phys_addr_t; +#define ODP_BITSIZE(x) \ + ((x) <= 2 ? 1 : \ + ((x) <= 4 ? 2 : \ + ((x) <= 8 ? 3 : \ + ((x) <= 16 ? 4 : \ + ((x) <= 32 ? 5 : \ + ((x) <= 64 ? 6 : \ + ((x) <= 128 ? 7 : \ + ((x) <= 256 ? 8 : \ + ((x) <= 512 ? 9 : \ + ((x) <= 1024 ? 10 : \ + ((x) <= 2048 ? 11 : \ + ((x) <= 4096 ? 12 : \ + ((x) <= 8196 ? 13 : \ + ((x) <= 16384 ? 14 : \ + ((x) <= 32768 ? 15 : \ + ((x) <= 65536 ? 16 : \ + (0/0))))))))))))))))) + +_ODP_STATIC_ASSERT(ODP_CONFIG_PACKET_BUF_LEN_MIN >= 256, + "ODP Segment size must be a minimum of 256 bytes"); + +_ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MIN % ODP_CACHE_LINE_SIZE) == 0, + "ODP Segment size must be a multiple of cache line size"); + +_ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MAX % + ODP_CONFIG_PACKET_BUF_LEN_MIN) == 0, + "Packet max size must be a multiple of segment size"); + +#define ODP_BUFFER_MAX_SEG \ + (ODP_CONFIG_PACKET_BUF_LEN_MAX / ODP_CONFIG_PACKET_BUF_LEN_MIN) + +/* We can optimize storage of small raw buffers within metadata area */ +#define ODP_MAX_INLINE_BUF ((sizeof(void *)) * (ODP_BUFFER_MAX_SEG - 1)) + +#define ODP_BUFFER_POOL_BITS ODP_BITSIZE(ODP_CONFIG_BUFFER_POOLS) +#define ODP_BUFFER_SEG_BITS ODP_BITSIZE(ODP_BUFFER_MAX_SEG) +#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS - ODP_BUFFER_SEG_BITS) +#define ODP_BUFFER_PREFIX_BITS (ODP_BUFFER_POOL_BITS + ODP_BUFFER_INDEX_BITS) +#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS) +#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS) #define ODP_BUFFER_MAX_INDEX (ODP_BUFFER_MAX_BUFFERS - 2) #define ODP_BUFFER_INVALID_INDEX (ODP_BUFFER_MAX_BUFFERS - 1) -#define ODP_BUFS_PER_CHUNK 16 -#define ODP_BUFS_PER_SCATTER 4 - -#define ODP_BUFFER_TYPE_CHUNK 0xffff - - -#define ODP_BUFFER_POOL_BITS 4 -#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS) -#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS) -#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS) - typedef union odp_buffer_bits_t { uint32_t u32; odp_buffer_t handle; struct { - uint32_t pool:ODP_BUFFER_POOL_BITS; +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN + uint32_t pool_id:ODP_BUFFER_POOL_BITS; + uint32_t index:ODP_BUFFER_INDEX_BITS; + uint32_t seg:ODP_BUFFER_SEG_BITS; +#else + uint32_t seg:ODP_BUFFER_SEG_BITS; uint32_t index:ODP_BUFFER_INDEX_BITS; + uint32_t pool_id:ODP_BUFFER_POOL_BITS; +#endif }; -} odp_buffer_bits_t; + struct { +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN + uint32_t prefix:ODP_BUFFER_PREFIX_BITS; + uint32_t pfxseg:ODP_BUFFER_SEG_BITS; +#else + uint32_t pfxseg:ODP_BUFFER_SEG_BITS; + uint32_t prefix:ODP_BUFFER_PREFIX_BITS; +#endif + }; +} odp_buffer_bits_t; /* forward declaration */ struct odp_buffer_hdr_t; - +/* Common buffer header */ typedef struct odp_buffer_hdr_t { + /* DPDK specific */ struct rte_mbuf mb; /* Underlying DPDK rte_mbuf */ - struct odp_buffer_hdr_t *next; /* Next buf in a list */ - int type; /* ODP buffer type; not DPDK buf type */ uint32_t index; /* Index in the rte_mempool */ + + struct odp_buffer_hdr_t *next; /* next buf in a list */ + int allocator; /* allocating thread id */ + odp_buffer_bits_t handle; /* handle */ + union { + uint32_t all; + struct { + uint32_t zeroized:1; /* Zeroize buf data on free */ + uint32_t hdrdata:1; /* Data is in buffer hdr */ + }; + } flags; + int type; /* buffer type */ + size_t size; /* max data size */ + odp_atomic_u32_t ref_count; /* reference count */ + odp_buffer_pool_t pool_hdl; /* buffer pool handle */ + union { + uint64_t buf_u64; /* user u64 */ + void *buf_ctx; /* user context */ + const void *buf_cctx; /* const alias for ctx */ + void *udata_addr; /* user metadata addr */ + }; + size_t udata_size; /* size of user metadata */ + uint32_t segcount; /* segment count */ + uint32_t segsize; /* segment size */ + void *addr[ODP_BUFFER_MAX_SEG]; /* block addrs */ + } odp_buffer_hdr_t; -int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf); +typedef struct odp_buffer_hdr_stride { + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_buffer_hdr_t))]; +} odp_buffer_hdr_stride; +/* Ensure next header starts from 8 byte align */ +_ODP_STATIC_ASSERT((sizeof(odp_buffer_hdr_t) % 8) == 0, + "ODP_BUFFER_HDR_T__SIZE_ERROR"); + +typedef struct odp_buf_blk_t { + struct odp_buf_blk_t *next; + struct odp_buf_blk_t *prev; +} odp_buf_blk_t; + +/* Raw buffer header */ +typedef struct { + odp_buffer_hdr_t buf_hdr; /* common buffer header */ +} odp_raw_buffer_hdr_t; + +/* Free buffer marker */ +#define ODP_FREEBUF -1 +/* Forward declarations */ +odp_buffer_t buffer_alloc(odp_buffer_pool_t pool, size_t size); #ifdef __cplusplus } diff --git a/platform/linux-dpdk/include/odp_buffer_pool_internal.h b/platform/linux-dpdk/include/odp_buffer_pool_internal.h index 1a36655..0f19f87 100644 --- a/platform/linux-dpdk/include/odp_buffer_pool_internal.h +++ b/platform/linux-dpdk/include/odp_buffer_pool_internal.h @@ -19,16 +19,48 @@ extern "C" { #endif #include <odp_std_types.h> +#include <odp_align.h> +#include <odp_align_internal.h> #include <odp_buffer_pool.h> #include <odp_buffer_internal.h> -#include <odp_align.h> #include <odp_hints.h> #include <odp_config.h> #include <odp_debug.h> +#include <odp_shared_memory.h> +#include <odp_atomic.h> +#include <odp_atomic_internal.h> +#include <string.h> -/* for DPDK */ +/* DPDK */ #include <rte_mempool.h> +/** + * Buffer initialization routine prototype + * + * @note Routines of this type MAY be passed as part of the + * _odp_buffer_pool_init_t structure to be called whenever a + * buffer is allocated to initialize the user metadata + * associated with that buffer. + */ +typedef void (_odp_buf_init_t)(odp_buffer_t buf, void *buf_init_arg); + +/** + * Buffer pool initialization parameters + * Used to communicate buffer pool initialization options. Internal for now. + */ +typedef struct _odp_buffer_pool_init_t { + size_t udata_size; /**< Size of user metadata for each buffer */ + _odp_buf_init_t *buf_init; /**< Buffer initialization routine to use */ + void *buf_init_arg; /**< Argument to be passed to buf_init() */ +} _odp_buffer_pool_init_t; /**< Type of buffer initialization struct */ + +/* Local cache for buffer alloc/free acceleration */ +typedef struct local_cache_t { + odp_buffer_hdr_t *buf_freelist; /* The local cache */ + uint64_t bufallocs; /* Local buffer alloc count */ + uint64_t buffrees; /* Local buffer free count */ +} local_cache_t; + /* Use ticketlock instead of spinlock */ #define POOL_USE_TICKETLOCK @@ -38,11 +70,16 @@ extern "C" { #ifdef POOL_USE_TICKETLOCK #include <odp_ticketlock.h> +#define POOL_LOCK(a) odp_ticketlock_lock(a) +#define POOL_UNLOCK(a) odp_ticketlock_unlock(a) +#define POOL_LOCK_INIT(a) odp_ticketlock_init(a) #else #include <odp_spinlock.h> +#define POOL_LOCK(a) odp_spinlock_lock(a) +#define POOL_UNLOCK(a) odp_spinlock_unlock(a) +#define POOL_LOCK_INIT(a) odp_spinlock_init(a) #endif - struct pool_entry_s { #ifdef POOL_USE_TICKETLOCK odp_ticketlock_t lock ODP_ALIGNED_CACHE; @@ -50,38 +87,294 @@ struct pool_entry_s { odp_spinlock_t lock ODP_ALIGNED_CACHE; #endif - uint64_t free_bufs; char name[ODP_BUFFER_POOL_NAME_LEN]; - - - odp_buffer_pool_t pool ODP_ALIGNED_CACHE; - uintptr_t buf_base; - size_t buf_size; - size_t buf_offset; - uint64_t num_bufs; - void *pool_base_addr; - uint64_t pool_size; - size_t payload_size; - size_t payload_align; - int buf_type; - size_t hdr_size; + odp_buffer_pool_param_t params; + _odp_buffer_pool_init_t init_params; + odp_buffer_pool_t pool_hdl; + uint32_t pool_id; + odp_shm_t pool_shm; + union { + uint32_t all; + struct { + uint32_t has_name:1; + uint32_t user_supplied_shm:1; + uint32_t unsegmented:1; + uint32_t zeroized:1; + uint32_t predefined:1; + }; + } flags; + uint32_t quiesced; + uint32_t low_wm_assert; + uint8_t *pool_base_addr; + uint8_t *pool_mdata_addr; + size_t pool_size; + uint32_t buf_align; + uint32_t buf_stride; + _odp_atomic_ptr_t buf_freelist; + _odp_atomic_ptr_t blk_freelist; + odp_atomic_u32_t bufcount; + odp_atomic_u32_t blkcount; + odp_atomic_u64_t bufallocs; + odp_atomic_u64_t buffrees; + odp_atomic_u64_t blkallocs; + odp_atomic_u64_t blkfrees; + odp_atomic_u64_t bufempty; + odp_atomic_u64_t blkempty; + odp_atomic_u64_t high_wm_count; + odp_atomic_u64_t low_wm_count; + uint32_t seg_size; + uint32_t high_wm; + uint32_t low_wm; + uint32_t headroom; + uint32_t tailroom; }; +typedef union pool_entry_u { + struct pool_entry_s s; + + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct pool_entry_s))]; +} pool_entry_t; extern void *pool_entry_ptr[]; +#if defined(ODP_CONFIG_SECURE_POOLS) && (ODP_CONFIG_SECURE_POOLS == 1) +#define buffer_is_secure(buf) (buf->flags.zeroized) +#define pool_is_secure(pool) (pool->flags.zeroized) +#else +#define buffer_is_secure(buf) 0 +#define pool_is_secure(pool) 0 +#endif + +#define TAG_ALIGN ((size_t)16) + +#define odp_cs(ptr, old, new) \ + _odp_atomic_ptr_cmp_xchg_strong(&ptr, (void **)&old, (void *)new, \ + _ODP_MEMMODEL_SC, \ + _ODP_MEMMODEL_SC) -static inline void *get_pool_entry(odp_buffer_pool_t pool_id) +/* Helper functions for pointer tagging to avoid ABA race conditions */ +#define odp_tag(ptr) \ + (((size_t)ptr) & (TAG_ALIGN - 1)) + +#define odp_detag(ptr) \ + ((void *)(((size_t)ptr) & -TAG_ALIGN)) + +#define odp_retag(ptr, tag) \ + ((void *)(((size_t)ptr) | odp_tag(tag))) + + +static inline void *get_blk(struct pool_entry_s *pool) +{ + void *oldhead, *myhead, *newhead; + + oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, _ODP_MEMMODEL_ACQ); + + do { + size_t tag = odp_tag(oldhead); + myhead = odp_detag(oldhead); + if (odp_unlikely(myhead == NULL)) + break; + newhead = odp_retag(((odp_buf_blk_t *)myhead)->next, tag + 1); + } while (odp_cs(pool->blk_freelist, oldhead, newhead) == 0); + + if (odp_unlikely(myhead == NULL)) + odp_atomic_inc_u64(&pool->blkempty); + else + odp_atomic_dec_u32(&pool->blkcount); + + return (void *)myhead; +} + +static inline void ret_blk(struct pool_entry_s *pool, void *block) +{ + void *oldhead, *myhead, *myblock; + + oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, _ODP_MEMMODEL_ACQ); + + do { + size_t tag = odp_tag(oldhead); + myhead = odp_detag(oldhead); + ((odp_buf_blk_t *)block)->next = myhead; + myblock = odp_retag(block, tag + 1); + } while (odp_cs(pool->blk_freelist, oldhead, myblock) == 0); + + odp_atomic_inc_u32(&pool->blkcount); + odp_atomic_inc_u64(&pool->blkfrees); +} + +static inline odp_buffer_hdr_t *get_buf(struct pool_entry_s *pool) +{ + odp_buffer_hdr_t *oldhead, *myhead, *newhead; + + oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, _ODP_MEMMODEL_ACQ); + + do { + size_t tag = odp_tag(oldhead); + myhead = odp_detag(oldhead); + if (odp_unlikely(myhead == NULL)) + break; + newhead = odp_retag(myhead->next, tag + 1); + } while (odp_cs(pool->buf_freelist, oldhead, newhead) == 0); + + if (odp_unlikely(myhead == NULL)) { + odp_atomic_inc_u64(&pool->bufempty); + } else { + uint64_t bufcount = + odp_atomic_fetch_sub_u32(&pool->bufcount, 1) - 1; + + /* Check for low watermark condition */ + if (bufcount == pool->low_wm && !pool->low_wm_assert) { + pool->low_wm_assert = 1; + odp_atomic_inc_u64(&pool->low_wm_count); + } + + odp_atomic_inc_u64(&pool->bufallocs); + myhead->next = myhead; /* Mark buffer allocated */ + myhead->allocator = odp_thread_id(); + } + + return (void *)myhead; +} + +static inline void ret_buf(struct pool_entry_s *pool, odp_buffer_hdr_t *buf) +{ + odp_buffer_hdr_t *oldhead, *myhead, *mybuf; + + buf->allocator = ODP_FREEBUF; /* Mark buffer free */ + + if (!buf->flags.hdrdata && buf->type != ODP_BUFFER_TYPE_RAW) { + while (buf->segcount > 0) { + if (buffer_is_secure(buf) || pool_is_secure(pool)) + memset(buf->addr[buf->segcount - 1], + 0, buf->segsize); + ret_blk(pool, buf->addr[--buf->segcount]); + } + buf->size = 0; + } + + oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, _ODP_MEMMODEL_ACQ); + + do { + size_t tag = odp_tag(oldhead); + myhead = odp_detag(oldhead); + buf->next = myhead; + mybuf = odp_retag(buf, tag + 1); + } while (odp_cs(pool->buf_freelist, oldhead, mybuf) == 0); + + uint64_t bufcount = odp_atomic_fetch_add_u32(&pool->bufcount, 1) + 1; + + /* Check if low watermark condition should be deasserted */ + if (bufcount == pool->high_wm && pool->low_wm_assert) { + pool->low_wm_assert = 0; + odp_atomic_inc_u64(&pool->high_wm_count); + } + + odp_atomic_inc_u64(&pool->buffrees); +} + +static inline void *get_local_buf(local_cache_t *buf_cache, + struct pool_entry_s *pool, + size_t totsize) +{ + odp_buffer_hdr_t *buf = buf_cache->buf_freelist; + + if (odp_likely(buf != NULL)) { + buf_cache->buf_freelist = buf->next; + + if (odp_unlikely(buf->size < totsize)) { + intmax_t needed = totsize - buf->size; + + do { + void *blk = get_blk(pool); + if (odp_unlikely(blk == NULL)) { + ret_buf(pool, buf); + buf_cache->buffrees--; + return NULL; + } + buf->addr[buf->segcount++] = blk; + needed -= pool->seg_size; + } while (needed > 0); + + buf->size = buf->segcount * pool->seg_size; + } + + buf_cache->bufallocs++; + buf->allocator = odp_thread_id(); /* Mark buffer allocated */ + } + + return buf; +} + +static inline void ret_local_buf(local_cache_t *buf_cache, + odp_buffer_hdr_t *buf) +{ + buf->allocator = ODP_FREEBUF; + buf->next = buf_cache->buf_freelist; + buf_cache->buf_freelist = buf; + + buf_cache->buffrees++; +} + +static inline void flush_cache(local_cache_t *buf_cache, + struct pool_entry_s *pool) +{ + odp_buffer_hdr_t *buf = buf_cache->buf_freelist; + uint32_t flush_count = 0; + + while (buf != NULL) { + odp_buffer_hdr_t *next = buf->next; + ret_buf(pool, buf); + buf = next; + flush_count++; + } + + odp_atomic_add_u64(&pool->bufallocs, buf_cache->bufallocs); + odp_atomic_add_u64(&pool->buffrees, buf_cache->buffrees - flush_count); + + buf_cache->buf_freelist = NULL; + buf_cache->bufallocs = 0; + buf_cache->buffrees = 0; +} + +static inline odp_buffer_pool_t pool_index_to_handle(uint32_t pool_id) +{ + return pool_id; +} + +static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl) +{ + return pool_hdl; +} + +static inline void *get_pool_entry(uint32_t pool_id) { return pool_entry_ptr[pool_id]; } +static inline pool_entry_t *odp_pool_to_entry(odp_buffer_pool_t pool) +{ + return (pool_entry_t *)get_pool_entry(pool_handle_to_index(pool)); +} + +static inline pool_entry_t *odp_buf_to_pool(odp_buffer_hdr_t *buf) +{ + return odp_pool_to_entry(buf->pool_hdl); +} -static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf) +static inline uint32_t odp_buffer_pool_segment_size(odp_buffer_pool_t pool) { - return (odp_buffer_hdr_t *)buf; + return odp_pool_to_entry(pool)->s.seg_size; } +static inline uint32_t odp_buffer_pool_headroom(odp_buffer_pool_t pool) +{ + return odp_pool_to_entry(pool)->s.headroom; +} + +static inline uint32_t odp_buffer_pool_tailroom(odp_buffer_pool_t pool) +{ + return odp_pool_to_entry(pool)->s.tailroom; +} #ifdef __cplusplus } diff --git a/platform/linux-dpdk/include/odp_packet_dpdk.h b/platform/linux-dpdk/include/odp_packet_dpdk.h index c3a2b70..c982d5e 100644 --- a/platform/linux-dpdk/include/odp_packet_dpdk.h +++ b/platform/linux-dpdk/include/odp_packet_dpdk.h @@ -8,10 +8,8 @@ #define ODP_PACKET_DPDK_H #include <stdint.h> -#include <net/if.h> #include <odph_eth.h> -#include <odph_packet.h> #include <odp_align.h> #include <odp_debug.h> #include <odp_packet.h> diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h index 1cd44b1..24de33f 100644 --- a/platform/linux-dpdk/include/odp_packet_internal.h +++ b/platform/linux-dpdk/include/odp_packet_internal.h @@ -22,6 +22,7 @@ extern "C" { #include <odp_debug.h> #include <odp_buffer_internal.h> #include <odp_buffer_pool_internal.h> +#include <odp_buffer_inlines.h> #include <odp_packet.h> #include <odp_packet_io.h> @@ -43,6 +44,7 @@ typedef union { uint32_t vlan:1; /**< VLAN hdr found */ uint32_t vlan_qinq:1; /**< Stacked VLAN found, QinQ */ + uint32_t snap:1; /**< SNAP */ uint32_t arp:1; /**< ARP */ uint32_t ipv4:1; /**< IPv4 */ @@ -53,13 +55,14 @@ typedef union { uint32_t udp:1; /**< UDP */ uint32_t tcp:1; /**< TCP */ + uint32_t tcpopt:1; /**< TCP options present */ uint32_t sctp:1; /**< SCTP */ uint32_t icmp:1; /**< ICMP */ }; } input_flags_t; -ODP_STATIC_ASSERT(sizeof(input_flags_t) == sizeof(uint32_t), - "INPUT_FLAGS_SIZE_ERROR"); +_ODP_STATIC_ASSERT(sizeof(input_flags_t) == sizeof(uint32_t), + "INPUT_FLAGS_SIZE_ERROR"); /** * Packet error flags @@ -70,7 +73,9 @@ typedef union { struct { /* Bitfield flags for each detected error */ + uint32_t app_error:1; /**< Error bit for application use */ uint32_t frame_len:1; /**< Frame length error */ + uint32_t snap_len:1; /**< Snap length error */ uint32_t l2_chksum:1; /**< L2 checksum error, checks TBD */ uint32_t ip_err:1; /**< IP error, checks TBD */ uint32_t tcp_err:1; /**< TCP error, checks TBD */ @@ -78,8 +83,8 @@ typedef union { }; } error_flags_t; -ODP_STATIC_ASSERT(sizeof(error_flags_t) == sizeof(uint32_t), - "ERROR_FLAGS_SIZE_ERROR"); +_ODP_STATIC_ASSERT(sizeof(error_flags_t) == sizeof(uint32_t), + "ERROR_FLAGS_SIZE_ERROR"); /** * Packet output flags @@ -90,12 +95,15 @@ typedef union { struct { /* Bitfield flags for each output option */ - uint32_t l4_chksum:1; /**< Request L4 checksum calculation */ + uint32_t l3_chksum_set:1; /**< L3 chksum bit is valid */ + uint32_t l3_chksum:1; /**< L3 chksum override */ + uint32_t l4_chksum_set:1; /**< L3 chksum bit is valid */ + uint32_t l4_chksum:1; /**< L4 chksum override */ }; } output_flags_t; -ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), - "OUTPUT_FLAGS_SIZE_ERROR"); +_ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), + "OUTPUT_FLAGS_SIZE_ERROR"); /** * Internal Packet header @@ -108,28 +116,131 @@ typedef struct { error_flags_t error_flags; output_flags_t output_flags; - uint32_t frame_offset; /**< offset to start of frame, even on error */ uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */ uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */ uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ + uint32_t payload_offset; /**< offset to payload */ - uint64_t user_ctx; /**< user context */ + uint32_t vlan_s_tag; /**< Parsed 1st VLAN header (S-TAG) */ + uint32_t vlan_c_tag; /**< Parsed 2nd VLAN header (C-TAG) */ + uint32_t l3_protocol; /**< Parsed L3 protocol */ + uint32_t l3_len; /**< Layer 3 length */ + uint32_t l4_protocol; /**< Parsed L4 protocol */ + uint32_t l4_len; /**< Layer 4 length */ + + uint32_t frame_len; + uint32_t headroom; + uint32_t tailroom; odp_pktio_t input; } odp_packet_hdr_t; +typedef struct odp_packet_hdr_stride { + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_packet_hdr_t))]; +} odp_packet_hdr_stride; + +_ODP_STATIC_ASSERT(sizeof(odp_packet_hdr_t) % sizeof(uint64_t) == 0, + "ODP_PACKET_HDR_T__SIZE_ERR2"); + /** * Return the packet header */ static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) { - return (odp_packet_hdr_t *)pkt; + return (odp_packet_hdr_t *)odp_buf_to_hdr((odp_buffer_t)pkt); } /** - * Parse packet and set internal metadata + * Initialize packet buffer */ -void odp_packet_parse(odp_packet_t pkt, size_t len, size_t l2_offset); +static inline void packet_init(pool_entry_t *pool, + odp_packet_hdr_t *pkt_hdr, + size_t size) +{ + /* + * Reset parser metadata. Note that we clear via memset to make + * this routine indepenent of any additional adds to packet metadata. + */ + struct rte_mbuf *mb; + uint8_t *start; + + mb = (struct rte_mbuf *)&pkt_hdr->buf_hdr; + mb->buf_len = size; + start = (uint8_t *)mb->buf_addr; + memset(start, 0, mb->buf_len); + + /* + * Packet headroom is set from the pool's headroom + * Packet tailroom is rounded up to fill the last + * segment occupied by the allocated length. + */ + pkt_hdr->frame_len = size; + pkt_hdr->headroom = pool->s.headroom; + pkt_hdr->tailroom = + (pool->s.seg_size * pkt_hdr->buf_hdr.segcount) - + (pool->s.headroom + size); +} + +static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, + uint32_t offset, uint32_t *seglen) +{ + struct rte_mbuf *mbuf = (struct rte_mbuf *)&pkt_hdr->buf_hdr; + if (offset > pkt_hdr->frame_len) + return NULL; + + if (seglen != NULL) { + *seglen = mbuf->pkt.data_len; + pkt_hdr->frame_len = mbuf->pkt.data_len; + } + return (void *)((uint8_t *)mbuf->pkt.data + offset); +} + +#define pull_offset(x, len) (x = x < len ? 0 : x - len) + +static inline void push_head(odp_packet_hdr_t *pkt_hdr, size_t len) +{ + pkt_hdr->headroom -= len; + pkt_hdr->frame_len += len; + pkt_hdr->l2_offset += len; + pkt_hdr->l3_offset += len; + pkt_hdr->l4_offset += len; +} + +static inline void pull_head(odp_packet_hdr_t *pkt_hdr, size_t len) +{ + pkt_hdr->headroom += len; + pkt_hdr->frame_len -= len; + pull_offset(pkt_hdr->l2_offset, len); + pull_offset(pkt_hdr->l3_offset, len); + pull_offset(pkt_hdr->l4_offset, len); +} + +static inline void push_tail(odp_packet_hdr_t *pkt_hdr, size_t len) +{ + pkt_hdr->tailroom -= len; + pkt_hdr->frame_len += len; +} + + +static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, size_t len) +{ + pkt_hdr->tailroom += len; + pkt_hdr->frame_len -= len; +} + +static inline void packet_set_len(odp_packet_t pkt, uint32_t len) +{ + odp_packet_hdr(pkt)->frame_len = len; +} + +/* Forward declarations */ +int _odp_packet_copy_to_packet(odp_packet_t srcpkt, uint32_t srcoffset, + odp_packet_t dstpkt, uint32_t dstoffset, + uint32_t len); + +odp_packet_t _odp_packet_alloc(odp_buffer_pool_t pool_hdl); + +int _odp_packet_parse(odp_packet_t pkt); #ifdef __cplusplus } diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h index 9263349..5ece843 100644 --- a/platform/linux-dpdk/include/odp_packet_io_internal.h +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h @@ -20,17 +20,36 @@ extern "C" { #include <odp_spinlock.h> #include <odp_packet_socket.h> -#ifdef ODP_HAVE_NETMAP -#include <odp_packet_netmap.h> -#endif +#include <odp_classification_datamodel.h> +#include <odp_align_internal.h> + +#include <odp_config.h> +#include <odp_hints.h> +#include <linux/if.h> + +/* DPDK */ #include <odp_packet_dpdk.h> +/** + * Packet IO types + */ +typedef enum { + ODP_PKTIO_TYPE_SOCKET_BASIC = 0x1, + ODP_PKTIO_TYPE_SOCKET_MMSG, + ODP_PKTIO_TYPE_SOCKET_MMAP, +} odp_pktio_type_t; + struct pktio_entry { odp_spinlock_t lock; /**< entry spinlock */ int taken; /**< is entry taken(1) or free(0) */ odp_queue_t inq_default; /**< default input queue, if set */ odp_queue_t outq_default; /**< default out queue */ - odp_pktio_params_t params; /**< pktio parameters */ + odp_pktio_type_t type; /**< pktio type */ + pkt_sock_t pkt_sock; /**< using socket API for IO */ + pkt_sock_mmap_t pkt_sock_mmap; /**< using socket mmap API for IO */ + classifier_t cls; /**< classifier linked with this pktio*/ + char name[IFNAMSIZ]; /**< name of pktio provided to + pktio_open() */ pkt_dpdk_t pkt_dpdk; /**< using DPDK API for IO */ }; @@ -39,6 +58,21 @@ typedef union { uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct pktio_entry))]; } pktio_entry_t; +typedef struct { + pktio_entry_t entries[ODP_CONFIG_PKTIO_ENTRIES]; +} pktio_table_t; + +extern void *pktio_entry_ptr[]; + + +static inline pktio_entry_t *get_pktio_entry(odp_pktio_t id) +{ + if (odp_unlikely(id == ODP_PKTIO_INVALID || + id > ODP_CONFIG_PKTIO_ENTRIES)) + return NULL; + + return pktio_entry_ptr[id - 1]; +} #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/odp_buffer.c b/platform/linux-dpdk/odp_buffer.c index 4914ca2..d89f2af 100644 --- a/platform/linux-dpdk/odp_buffer.c +++ b/platform/linux-dpdk/odp_buffer.c @@ -5,8 +5,10 @@ */ #include <odp_buffer.h> -#include <odp_buffer_internal.h> #include <odp_buffer_pool_internal.h> +#include <odp_buffer_internal.h> +#include <odp_buffer_inlines.h> +#include <odp_debug_internal.h> #include <string.h> #include <stdio.h> @@ -44,13 +46,13 @@ int odp_buffer_is_valid(odp_buffer_t buf) } -int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf) +int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf) { odp_buffer_hdr_t *hdr; int len = 0; if (!odp_buffer_is_valid(buf)) { - printf("Buffer is not valid.\n"); + ODP_PRINT("Buffer is not valid.\n"); return len; } @@ -67,7 +69,9 @@ int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf) len += snprintf(&str[len], n-len, " size %u\n", hdr->mb.buf_len); len += snprintf(&str[len], n-len, - " ref_count %i\n", hdr->mb.refcnt); + " ref_count %i\n", + odp_atomic_load_u32((odp_atomic_u32_t *) + &hdr->mb.refcnt)); len += snprintf(&str[len], n-len, " dpdk type %i\n", hdr->mb.type); len += snprintf(&str[len], n-len, @@ -86,5 +90,5 @@ void odp_buffer_print(odp_buffer_t buf) len = odp_buffer_snprint(str, max_len-1, buf); str[len] = 0; - printf("\n%s\n", str); + ODP_PRINT("\n%s\n", str); } diff --git a/platform/linux-dpdk/odp_buffer_pool.c b/platform/linux-dpdk/odp_buffer_pool.c index ca25ace..6636b8d 100644 --- a/platform/linux-dpdk/odp_buffer_pool.c +++ b/platform/linux-dpdk/odp_buffer_pool.c @@ -6,16 +6,19 @@ #include <odp_std_types.h> #include <odp_buffer_pool.h> -#include <odp_buffer_pool_internal.h> #include <odp_buffer_internal.h> +#include <odp_buffer_pool_internal.h> +#include <odp_buffer_inlines.h> #include <odp_packet_internal.h> #include <odp_timer_internal.h> +#include <odp_align_internal.h> #include <odp_shared_memory.h> #include <odp_align.h> #include <odp_internal.h> #include <odp_config.h> #include <odp_hints.h> -#include <odp_debug.h> +#include <odp_debug_internal.h> +#include <odp_atomic_internal.h> #include <string.h> #include <stdlib.h> @@ -26,44 +29,27 @@ #define MBUF_SIZE (2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM) #define NB_MBUF 32768 -#ifdef POOL_USE_TICKETLOCK -#include <odp_ticketlock.h> -#define LOCK(a) odp_ticketlock_lock(a) -#define UNLOCK(a) odp_ticketlock_unlock(a) -#define LOCK_INIT(a) odp_ticketlock_init(a) -#else -#include <odp_spinlock.h> -#define LOCK(a) odp_spinlock_lock(a) -#define UNLOCK(a) odp_spinlock_unlock(a) -#define LOCK_INIT(a) odp_spinlock_init(a) -#endif - - #if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS #error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS #endif -#define NULL_INDEX ((uint32_t)-1) - -union buffer_type_any_u { +typedef union buffer_type_any_u { odp_buffer_hdr_t buf; odp_packet_hdr_t pkt; odp_timeout_hdr_t tmo; -}; - -typedef union buffer_type_any_u odp_any_buffer_hdr_t; +} odp_anybuf_t; -typedef union pool_entry_u { - struct pool_entry_s s; - - uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct pool_entry_s))]; - -} pool_entry_t; +/* Any buffer type header */ +typedef struct { + odp_anybuf_t any_hdr; /* any buffer type */ +} odp_any_buffer_hdr_t; +typedef struct odp_any_hdr_stride { + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_any_buffer_hdr_t))]; +} odp_any_hdr_stride; typedef struct pool_table_t { pool_entry_t pool[ODP_CONFIG_BUFFER_POOLS]; - } pool_table_t; @@ -73,6 +59,8 @@ static pool_table_t *pool_tbl; /* Pool entry pointers (for inlining) */ void *pool_entry_ptr[ODP_CONFIG_BUFFER_POOLS]; +/* Local cache for buffer alloc/free acceleration */ +static __thread local_cache_t local_cache[ODP_CONFIG_BUFFER_POOLS]; int odp_buffer_pool_init_global(void) { @@ -94,9 +82,9 @@ int odp_buffer_pool_init_global(void) for (i = 0; i < ODP_CONFIG_BUFFER_POOLS; i++) { /* init locks */ pool_entry_t *pool = &pool_tbl->pool[i]; - LOCK_INIT(&pool->s.lock); - pool->s.pool = i; - + POOL_LOCK_INIT(&pool->s.lock); + pool->s.pool_hdl = pool_index_to_handle(i); + pool->s.pool_id = i; pool_entry_ptr[i] = pool; } @@ -187,10 +175,13 @@ odp_dpdk_mbuf_ctor(struct rte_mempool *mp, buf_hdr->index = i; } +/** + * Buffer pool creation + */ + odp_buffer_pool_t odp_buffer_pool_create(const char *name, - void *base_addr, uint64_t size, - size_t buf_size, size_t buf_align, - int buf_type) + odp_shm_t shm, + odp_buffer_pool_param_t *params) { struct rte_mempool *pool = NULL; struct mbuf_pool_ctor_arg mbp_ctor_arg; @@ -198,47 +189,39 @@ odp_buffer_pool_t odp_buffer_pool_create(const char *name, unsigned mb_size; size_t hdr_size; - /* Not used for rte_mempool; the new ODP buffer management introduces - * rte_mempool_create_from_region where base_addr makes sense */ - (void)base_addr; - - /* buf_align will be removed soon, no need to wory about it */ - (void)buf_align; - - ODP_DBG("odp_buffer_pool_create: %s, %lx, %u, %u, %u, %d\n", name, - (uint64_t) base_addr, (unsigned) size, - (unsigned) buf_size, (unsigned) buf_align, - buf_type); + ODP_DBG("odp_buffer_pool_create: %s, %lx, %u, %u, %d\n", name, + (uint64_t) shm, (unsigned) params->buf_size, + (unsigned) params->buf_align, params->buf_type); - switch (buf_type) { + switch (params->buf_type) { case ODP_BUFFER_TYPE_RAW: hdr_size = sizeof(odp_buffer_hdr_t); - mbp_ctor_arg.seg_buf_size = (uint16_t) buf_size; + mbp_ctor_arg.seg_buf_size = (uint16_t) params->buf_size; break; case ODP_BUFFER_TYPE_PACKET: hdr_size = sizeof(odp_packet_hdr_t); mbp_ctor_arg.seg_buf_size = - (uint16_t) (RTE_PKTMBUF_HEADROOM + buf_size); + (uint16_t) (RTE_PKTMBUF_HEADROOM + params->buf_size); break; case ODP_BUFFER_TYPE_TIMEOUT: hdr_size = sizeof(odp_timeout_hdr_t); - mbp_ctor_arg.seg_buf_size = (uint16_t) buf_size; + mbp_ctor_arg.seg_buf_size = (uint16_t) params->buf_size; break; case ODP_BUFFER_TYPE_ANY: hdr_size = sizeof(odp_any_buffer_hdr_t); mbp_ctor_arg.seg_buf_size = - (uint16_t) (RTE_PKTMBUF_HEADROOM + buf_size); + (uint16_t) (RTE_PKTMBUF_HEADROOM + params->buf_size); break; default: - ODP_ERR("odp_buffer_pool_create: Bad type %i\n", buf_type); + ODP_ERR("odp_buffer_pool_create: Bad type %i\n", + params->buf_type); return ODP_BUFFER_POOL_INVALID; - break; } mb_ctor_arg.seg_buf_offset = (uint16_t) ODP_CACHE_LINE_SIZE_ROUNDUP(hdr_size); mb_ctor_arg.seg_buf_size = mbp_ctor_arg.seg_buf_size; - mb_ctor_arg.buf_type = buf_type; + mb_ctor_arg.buf_type = params->buf_type; mb_size = mb_ctor_arg.seg_buf_offset + mb_ctor_arg.seg_buf_size; pool = rte_mempool_create(name, NB_MBUF, @@ -267,6 +250,61 @@ odp_buffer_pool_t odp_buffer_pool_lookup(const char *name) return (odp_buffer_pool_t)mp; } +odp_buffer_t buffer_alloc(odp_buffer_pool_t pool_hdl, size_t size) +{ + uint32_t pool_id = pool_handle_to_index(pool_hdl); + pool_entry_t *pool = get_pool_entry(pool_id); + uintmax_t totsize = pool->s.headroom + size + pool->s.tailroom; + odp_anybuf_t *buf; + + /* Reject oversized allocation requests */ + if ((pool->s.flags.unsegmented && totsize > pool->s.seg_size) || + (!pool->s.flags.unsegmented && + totsize > ODP_CONFIG_PACKET_BUF_LEN_MAX)) + return ODP_BUFFER_INVALID; + + /* Try to satisfy request from the local cache */ + buf = (odp_anybuf_t *)(void *)get_local_buf(&local_cache[pool_id], + &pool->s, totsize); + + /* If cache is empty, satisfy request from the pool */ + if (odp_unlikely(buf == NULL)) { + buf = (odp_anybuf_t *)(void *)get_buf(&pool->s); + + if (odp_unlikely(buf == NULL)) + return ODP_BUFFER_INVALID; + + /* Get blocks for this buffer, if pool uses application data */ + if (buf->buf.size < totsize) { + intmax_t needed = totsize - buf->buf.size; + do { + uint8_t *blk = get_blk(&pool->s); + if (blk == NULL) { + ret_buf(&pool->s, &buf->buf); + return ODP_BUFFER_INVALID; + } + buf->buf.addr[buf->buf.segcount++] = blk; + needed -= pool->s.seg_size; + } while (needed > 0); + buf->buf.size = buf->buf.segcount * pool->s.seg_size; + } + } + + /* By default, buffers inherit their pool's zeroization setting */ + buf->buf.flags.zeroized = pool->s.flags.zeroized; + + if (buf->buf.type == ODP_BUFFER_TYPE_PACKET) { + packet_init(pool, &buf->pkt, size); + + if (pool->s.init_params.buf_init != NULL) + (*pool->s.init_params.buf_init) + (buf->buf.handle.handle, + pool->s.init_params.buf_init_arg); + } + + return odp_hdr_to_buf(&buf->buf); +} + odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_id) { @@ -279,6 +317,15 @@ void odp_buffer_free(odp_buffer_t buf) rte_pktmbuf_free((struct rte_mbuf *)buf); } +void _odp_flush_caches(void) +{ + int i; + + for (i = 0; i < ODP_CONFIG_BUFFER_POOLS; i++) { + pool_entry_t *pool = get_pool_entry(i); + flush_cache(&local_cache[i], &pool->s); + } +} void odp_buffer_pool_print(odp_buffer_pool_t pool_id) { diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index fa10022..7c8763d 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -7,6 +7,8 @@ #include <odp_init.h> #include <odp_internal.h> #include <odp_debug.h> +#include <odp_debug_internal.h> +#include <odp_system_info.h> #include <odp_packet_dpdk.h> int odp_init_dpdk(void) @@ -45,10 +47,9 @@ int odp_init_dpdk(void) return 0; } -int odp_init_global(void) +int odp_init_global(odp_init_t *params ODP_UNUSED, + odp_platform_init_t *platform_params ODP_UNUSED) { - odp_thread_init_global(); - odp_system_info_init(); if (odp_init_dpdk()) { @@ -61,6 +62,11 @@ int odp_init_global(void) return -1; } + if (odp_thread_init_global()) { + ODP_ERR("ODP thread init failed.\n"); + return -1; + } + if (odp_buffer_pool_init_global()) { ODP_ERR("ODP buffer pool init failed.\n"); return -1; @@ -91,13 +97,26 @@ int odp_init_global(void) return -1; } + if (odp_classification_init_global()) { + ODP_ERR("ODP classification init failed.\n"); + return -1; + } + return 0; } +int odp_term_global(void) +{ + ODP_UNIMPLEMENTED(); + return 0; +} -int odp_init_local(int thr_id) +int odp_init_local(void) { - odp_thread_init_local(thr_id); + if (odp_thread_init_local()) { + ODP_ERR("ODP thread local init failed.\n"); + return -1; + } if (odp_pktio_init_local()) { ODP_ERR("ODP packet io local init failed.\n"); @@ -111,3 +130,9 @@ int odp_init_local(int thr_id) return 0; } + +int odp_term_local(void) +{ + ODP_UNIMPLEMENTED(); + return 0; +} diff --git a/platform/linux-dpdk/odp_linux.c b/platform/linux-dpdk/odp_linux.c index 96c91a5..2b68c13 100644 --- a/platform/linux-dpdk/odp_linux.c +++ b/platform/linux-dpdk/odp_linux.c @@ -8,6 +8,9 @@ #define _GNU_SOURCE #endif #include <sched.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/wait.h> #include <stdlib.h> #include <string.h> @@ -19,28 +22,30 @@ #include <odp_thread.h> #include <odp_init.h> #include <odp_system_info.h> +#include <odp_debug_internal.h> #include <rte_lcore.h> typedef struct { - int thr_id; void *(*start_routine) (void *); void *arg; - } odp_start_args_t; - static void *odp_run_start_routine(void *arg) { odp_start_args_t *start_args = arg; /* ODP thread local init */ - odp_init_local(start_args->thr_id); + if (odp_init_local()) { + ODP_ERR("Local init failed\n"); + return NULL; + } - return start_args->start_routine(start_args->arg); + void *ret = start_args->start_routine(start_args->arg); + _odp_flush_caches(); + return ret; } - void odph_linux_pthread_create(odph_linux_pthread_t *thread_tbl, int num, int first_core, void *(*start_routine) (void *), void *arg) { @@ -62,12 +67,12 @@ void odph_linux_pthread_create(odph_linux_pthread_t *thread_tbl, int num, cpu = (first_core + i) % core_count; start_args = malloc(sizeof(odp_start_args_t)); + if (start_args == NULL) + ODP_ABORT("Malloc failed"); memset(start_args, 0, sizeof(odp_start_args_t)); start_args->start_routine = start_routine; start_args->arg = arg; - odp_thread_create(cpu); - start_args->thr_id = cpu; /* If not master core */ if (cpu != 0) { rte_eal_remote_launch( @@ -94,3 +99,99 @@ void odph_linux_pthread_join(odph_linux_pthread_t *thread_tbl, int num) return; } } + +int odph_linux_process_fork_n(odph_linux_process_t *proc_tbl, + int num, int first_core) +{ + cpu_set_t cpu_set; + pid_t pid; + int core_count; + int cpu; + int i; + + memset(proc_tbl, 0, num*sizeof(odph_linux_process_t)); + + core_count = odp_sys_core_count(); + + if (first_core < 0 || first_core >= core_count) { + ODP_ERR("Bad first_core\n"); + return -1; + } + + if (num < 0 || num > core_count) { + ODP_ERR("Bad num\n"); + return -1; + } + + for (i = 0; i < num; i++) { + cpu = (first_core + i) % core_count; + pid = fork(); + + if (pid < 0) { + ODP_ERR("fork() failed\n"); + return -1; + } + + /* Parent continues to fork */ + if (pid > 0) { + proc_tbl[i].pid = pid; + proc_tbl[i].core = cpu; + continue; + } + + /* Child process */ + CPU_ZERO(&cpu_set); + CPU_SET(cpu, &cpu_set); + + if (sched_setaffinity(0, sizeof(cpu_set_t), &cpu_set)) { + ODP_ERR("sched_setaffinity() failed\n"); + return -2; + } + + if (odp_init_local()) { + ODP_ERR("Local init failed\n"); + return -2; + } + + return 0; + } + + return 1; +} + + +int odph_linux_process_fork(odph_linux_process_t *proc, int core) +{ + return odph_linux_process_fork_n(proc, 1, core); +} + + +int odph_linux_process_wait_n(odph_linux_process_t *proc_tbl, int num) +{ + pid_t pid; + int i, j; + int status; + + for (i = 0; i < num; i++) { + pid = wait(&status); + + if (pid < 0) { + ODP_ERR("wait() failed\n"); + return -1; + } + + for (j = 0; j < num; j++) { + if (proc_tbl[j].pid == pid) { + proc_tbl[j].status = status; + break; + } + } + + if (j == num) { + ODP_ERR("Bad pid\n"); + return -1; + } + } + + return 0; +} diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index beb69b2..754ce41 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -6,34 +6,63 @@ #include <odp_packet.h> #include <odp_packet_internal.h> +#include <odp_debug_internal.h> #include <odp_hints.h> #include <odp_byteorder.h> #include <odph_eth.h> #include <odph_ip.h> +#include <odph_tcp.h> +#include <odph_udp.h> #include <string.h> #include <stdio.h> -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, - odph_ipv4hdr_t *ipv4, size_t *offset_out); -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, - odph_ipv6hdr_t *ipv6, size_t *offset_out); +/* + * + * Alloc and free + * ******************************************************** + * + */ -void odp_packet_init(odp_packet_t pkt) +odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool_hdl, uint32_t len) { - odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); - struct rte_mbuf *mb; - void *start; + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); + + if (pool->s.params.buf_type != ODP_BUFFER_TYPE_PACKET) + return ODP_PACKET_INVALID; + + /* Handle special case for zero-length packets */ + if (len == 0) { + odp_packet_t pkt = + (odp_packet_t)buffer_alloc(pool_hdl, + pool->s.params.buf_size); + if (pkt != ODP_PACKET_INVALID) + pull_tail(odp_packet_hdr(pkt), + pool->s.params.buf_size); + + return pkt; + } + + return (odp_packet_t)buffer_alloc(pool_hdl, len); +} - mb = &pkt_hdr->buf_hdr.mb; +void odp_packet_free(odp_packet_t pkt) +{ + odp_buffer_free((odp_buffer_t)pkt); +} + +int odp_packet_reset(odp_packet_t pkt, uint32_t len) +{ + odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); + pool_entry_t *pool = odp_buf_to_pool(&pkt_hdr->buf_hdr); + uint32_t totsize = pool->s.headroom + len + pool->s.tailroom; - start = mb->buf_addr; - memset(start, 0, mb->buf_len); + if (totsize > pkt_hdr->buf_hdr.size) + return -1; - pkt_hdr->l2_offset = (uint32_t) ODP_PACKET_OFFSET_INVALID; - pkt_hdr->l3_offset = (uint32_t) ODP_PACKET_OFFSET_INVALID; - pkt_hdr->l4_offset = (uint32_t) ODP_PACKET_OFFSET_INVALID; + packet_init(pool, pkt_hdr, len); + return 0; } odp_packet_t odp_packet_from_buffer(odp_buffer_t buf) @@ -41,305 +70,478 @@ odp_packet_t odp_packet_from_buffer(odp_buffer_t buf) return (odp_packet_t)buf; } -odp_buffer_t odp_buffer_from_packet(odp_packet_t pkt) +odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt) { return (odp_buffer_t)pkt; } -/* Advance the pkt data pointer and set len in one call */ -static int odp_packet_set_offset_len(odp_packet_t pkt, size_t frame_offset, - size_t len) +/* + * + * Pointers and lengths + * ******************************************************** + * + */ + +void *odp_packet_head(odp_packet_t pkt) { - struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); - uint16_t offset; - uint16_t data_len; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + return buffer_map(&pkt_hdr->buf_hdr, 0, NULL, 0); +} - /* The pkt buf may have been pulled back into the headroom - * so we cannot rely on finding the data right after the - * ODP header and HEADROOM */ - offset = (uint16_t)((unsigned long)mb->pkt.data - - (unsigned long)mb->buf_addr); - ODP_ASSERT(mb->buf_len >= offset, "Corrupted mbuf"); - data_len = mb->buf_len - offset; +uint32_t odp_packet_buf_len(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->buf_hdr.size; +} - if (data_len < frame_offset) { - ODP_ERR("Frame offset too big"); - return -1; - } - mb->pkt.data = (void *)((char *)mb->pkt.data + frame_offset); - data_len -= frame_offset; +void *odp_packet_data(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + return packet_map(pkt_hdr, 0, NULL); +} - if (data_len < len) { - ODP_ERR("Packet len too big"); - return -1; - } - mb->pkt.pkt_len = len; - mb->pkt.data_len = len; +uint32_t odp_packet_seg_len(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + uint32_t seglen; - return 0; + /* Call returns length of 1st data segment */ + packet_map(pkt_hdr, 0, &seglen); + return seglen; } -void odp_packet_set_len(odp_packet_t pkt, size_t len) +uint32_t odp_packet_len(odp_packet_t pkt) { - (void)odp_packet_set_offset_len(pkt, 0, len); + return odp_packet_hdr(pkt)->frame_len; } -size_t odp_packet_get_len(odp_packet_t pkt) +uint32_t odp_packet_headroom(odp_packet_t pkt) { - struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); - return mb->pkt.pkt_len; + return odp_packet_hdr(pkt)->headroom; } -uint8_t *odp_packet_buf_addr(odp_packet_t pkt) +uint32_t odp_packet_tailroom(odp_packet_t pkt) { - return odp_buffer_addr(odp_buffer_from_packet(pkt)); + return odp_packet_hdr(pkt)->tailroom; } -uint8_t *odp_packet_start(odp_packet_t pkt) +void *odp_packet_tail(odp_packet_t pkt) { - struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); - return mb->pkt.data; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL); } - -uint8_t *odp_packet_l2(odp_packet_t pkt) +void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) { - const size_t offset = odp_packet_l2_offset(pkt); + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) + if (len > pkt_hdr->headroom) return NULL; - return odp_packet_start(pkt) + offset; + push_head(pkt_hdr, len); + return packet_map(pkt_hdr, 0, NULL); } -size_t odp_packet_l2_offset(odp_packet_t pkt) +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) { - return odp_packet_hdr(pkt)->l2_offset; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (len > pkt_hdr->frame_len) + return NULL; + + pull_head(pkt_hdr, len); + return packet_map(pkt_hdr, 0, NULL); } -void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset) +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) { - odp_packet_hdr(pkt)->l2_offset = offset; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + struct rte_mbuf *mbuf = (struct rte_mbuf *)&pkt_hdr->buf_hdr; + uint32_t origin = pkt_hdr->frame_len; + + mbuf->pkt.pkt_len += len; + mbuf->pkt.data_len += len; + + if (len > pkt_hdr->tailroom) + return NULL; + + push_tail(pkt_hdr, len); + return packet_map(pkt_hdr, origin, NULL); } -uint8_t *odp_packet_l3(odp_packet_t pkt) +void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) { - const size_t offset = odp_packet_l3_offset(pkt); + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) + if (len > pkt_hdr->frame_len) return NULL; - return odp_packet_start(pkt) + offset; + pull_tail(pkt_hdr, len); + return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL); } -size_t odp_packet_l3_offset(odp_packet_t pkt) +void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, + odp_packet_seg_t *seg) { - return odp_packet_hdr(pkt)->l3_offset; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + void *addr = packet_map(pkt_hdr, offset, len); + + if (addr != NULL && seg != NULL) { + odp_buffer_bits_t seghandle; + seghandle.u32 = (uint32_t)pkt; + seghandle.seg = (pkt_hdr->headroom + offset) / + pkt_hdr->buf_hdr.segsize; + *seg = seghandle.handle; + } + + return addr; } -void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset) +/* + * + * Meta-data + * ******************************************************** + * + */ + +odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt) { - odp_packet_hdr(pkt)->l3_offset = offset; + return odp_packet_hdr(pkt)->buf_hdr.pool_hdl; } -uint8_t *odp_packet_l4(odp_packet_t pkt) +odp_pktio_t odp_packet_input(odp_packet_t pkt) { - const size_t offset = odp_packet_l4_offset(pkt); + return odp_packet_hdr(pkt)->input; +} - if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) - return NULL; +void *odp_packet_user_ptr(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->buf_hdr.buf_ctx; +} + +void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ctx) +{ + odp_packet_hdr(pkt)->buf_hdr.buf_cctx = ctx; +} + +uint64_t odp_packet_user_u64(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->buf_hdr.buf_u64; +} - return odp_packet_start(pkt) + offset; +void odp_packet_user_u64_set(odp_packet_t pkt, uint64_t ctx) +{ + odp_packet_hdr(pkt)->buf_hdr.buf_u64 = ctx; +} + +void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + return packet_map(pkt_hdr, pkt_hdr->l2_offset, len); +} + +uint32_t odp_packet_l2_offset(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->l2_offset; } -size_t odp_packet_l4_offset(odp_packet_t pkt) +int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (offset >= pkt_hdr->frame_len) + return -1; + + pkt_hdr->l2_offset = offset; + return 0; +} + +void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + return packet_map(pkt_hdr, pkt_hdr->l3_offset, len); +} + +uint32_t odp_packet_l3_offset(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->l3_offset; +} + +int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (offset >= pkt_hdr->frame_len) + return -1; + + pkt_hdr->l3_offset = offset; + return 0; +} + +void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + return packet_map(pkt_hdr, pkt_hdr->l4_offset, len); +} + +uint32_t odp_packet_l4_offset(odp_packet_t pkt) { return odp_packet_hdr(pkt)->l4_offset; } -void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset) +int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset) { - odp_packet_hdr(pkt)->l4_offset = offset; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (offset >= pkt_hdr->frame_len) + return -1; + + pkt_hdr->l4_offset = offset; + return 0; } -/** - * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP +int odp_packet_is_segmented(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->buf_hdr.segcount > 1; +} + +int odp_packet_num_segs(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->buf_hdr.segcount; +} + +odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) +{ + return (odp_packet_seg_t)pkt; +} + +odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_buffer_bits_t seghandle; + + seghandle.u32 = (uint32_t)pkt; + seghandle.seg = pkt_hdr->buf_hdr.segcount - 1; + return seghandle.handle; +} + +odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + return segment_next(&pkt_hdr->buf_hdr, seg); +} + +/* * - * Internal function: caller is resposible for passing only valid packet handles - * , lengths and offsets (usually done&called in packet input). + * Segment level + * ******************************************************** * - * @param pkt Packet handle - * @param len Packet length in bytes - * @param frame_offset Byte offset to L2 header */ -void odp_packet_parse(odp_packet_t pkt, size_t len, size_t frame_offset) + +void *odp_packet_seg_buf_addr(odp_packet_t pkt, odp_packet_seg_t seg) { - odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); - odph_ethhdr_t *eth; - odph_vlanhdr_t *vlan; - odph_ipv4hdr_t *ipv4; - odph_ipv6hdr_t *ipv6; - uint16_t ethtype; - size_t offset = 0; - uint8_t ip_proto = 0; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - /* The frame_offset is not relevant for frames from DPDK */ - pkt_hdr->input_flags.eth = 1; - (void) frame_offset; - pkt_hdr->frame_offset = 0; - if (odp_packet_set_offset_len(pkt, 0, len)) { - return; - } + return segment_map(&pkt_hdr->buf_hdr, seg, NULL, + pkt_hdr->headroom + pkt_hdr->frame_len, 0); +} - if (odp_unlikely(len < ODPH_ETH_LEN_MIN)) { - pkt_hdr->error_flags.frame_len = 1; - return; - } else if (len > ODPH_ETH_LEN_MAX) { - pkt_hdr->input_flags.jumbo = 1; - } +uint32_t odp_packet_seg_buf_len(odp_packet_t pkt, + odp_packet_seg_t seg ODP_UNUSED) +{ + return odp_packet_hdr(pkt)->buf_hdr.segsize; +} - /* Assume valid L2 header, no CRC/FCS check in SW */ - pkt_hdr->input_flags.l2 = 1; - pkt_hdr->l2_offset = 0; +void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - eth = (odph_ethhdr_t *)odp_packet_start(pkt); - ethtype = odp_be_to_cpu_16(eth->type); - vlan = (odph_vlanhdr_t *)ð->type; + return segment_map(&pkt_hdr->buf_hdr, seg, NULL, + pkt_hdr->frame_len, pkt_hdr->headroom); +} - if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { - pkt_hdr->input_flags.vlan_qinq = 1; - ethtype = odp_be_to_cpu_16(vlan->tpid); - offset += sizeof(odph_vlanhdr_t); - vlan = &vlan[1]; - } +uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + uint32_t seglen = 0; - if (ethtype == ODPH_ETHTYPE_VLAN) { - pkt_hdr->input_flags.vlan = 1; - ethtype = odp_be_to_cpu_16(vlan->tpid); - offset += sizeof(odph_vlanhdr_t); - } + segment_map(&pkt_hdr->buf_hdr, seg, &seglen, + pkt_hdr->frame_len, pkt_hdr->headroom); - /* Set l3_offset+flag only for known ethtypes */ - switch (ethtype) { - case ODPH_ETHTYPE_IPV4: - pkt_hdr->input_flags.ipv4 = 1; - pkt_hdr->input_flags.l3 = 1; - pkt_hdr->l3_offset = ODPH_ETHHDR_LEN + offset; - ipv4 = (odph_ipv4hdr_t *)odp_packet_l3(pkt); - ip_proto = parse_ipv4(pkt_hdr, ipv4, &offset); - break; - case ODPH_ETHTYPE_IPV6: - pkt_hdr->input_flags.ipv6 = 1; - pkt_hdr->input_flags.l3 = 1; - pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + offset; - ipv6 = (odph_ipv6hdr_t *)odp_packet_l3(pkt); - ip_proto = parse_ipv6(pkt_hdr, ipv6, &offset); - break; - case ODPH_ETHTYPE_ARP: - pkt_hdr->input_flags.arp = 1; - /* fall through */ - default: - ip_proto = 0; - break; - } + return seglen; +} - switch (ip_proto) { - case ODPH_IPPROTO_UDP: - pkt_hdr->input_flags.udp = 1; - pkt_hdr->input_flags.l4 = 1; - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; - break; - case ODPH_IPPROTO_TCP: - pkt_hdr->input_flags.tcp = 1; - pkt_hdr->input_flags.l4 = 1; - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; - break; - case ODPH_IPPROTO_SCTP: - pkt_hdr->input_flags.sctp = 1; - pkt_hdr->input_flags.l4 = 1; - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; - break; - case ODPH_IPPROTO_ICMP: - pkt_hdr->input_flags.icmp = 1; - pkt_hdr->input_flags.l4 = 1; - pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset; - break; - default: - /* 0 or unhandled IP protocols, don't set L4 flag+offset */ - if (pkt_hdr->input_flags.ipv6) { - /* IPv6 next_hdr is not L4, mark as IP-option instead */ - pkt_hdr->input_flags.ipopt = 1; +/* + * + * Manipulation + * ******************************************************** + * + */ + +odp_packet_t odp_packet_add_data(odp_packet_t pkt, uint32_t offset, + uint32_t len) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + uint32_t pktlen = pkt_hdr->frame_len; + odp_packet_t newpkt; + + if (offset > pktlen) + return ODP_PACKET_INVALID; + + newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen + len); + + if (newpkt != ODP_PACKET_INVALID) { + if (_odp_packet_copy_to_packet(pkt, 0, + newpkt, 0, offset) != 0 || + _odp_packet_copy_to_packet(pkt, offset, newpkt, + offset + len, + pktlen - offset) != 0) { + odp_packet_free(newpkt); + newpkt = ODP_PACKET_INVALID; + } else { + odp_packet_hdr_t *new_hdr = odp_packet_hdr(newpkt); + new_hdr->buf_hdr.buf_u64 = pkt_hdr->buf_hdr.buf_u64; + odp_atomic_store_u32( + &new_hdr->buf_hdr.ref_count, + odp_atomic_load_u32( + &pkt_hdr->buf_hdr.ref_count)); + _odp_packet_parse(newpkt); + odp_packet_free(pkt); } - break; } + + return newpkt; } -static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, - odph_ipv4hdr_t *ipv4, size_t *offset_out) +odp_packet_t odp_packet_rem_data(odp_packet_t pkt, uint32_t offset, + uint32_t len) { - uint8_t ihl; - uint16_t frag_offset; - - ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); - if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN)) { - pkt_hdr->error_flags.ip_err = 1; - return 0; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + uint32_t pktlen = pkt_hdr->frame_len; + odp_packet_t newpkt; + + if (offset > pktlen || offset + len > pktlen) + return ODP_PACKET_INVALID; + + newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen - len); + + if (newpkt != ODP_PACKET_INVALID) { + if (_odp_packet_copy_to_packet(pkt, 0, + newpkt, 0, offset) != 0 || + _odp_packet_copy_to_packet(pkt, offset + len, + newpkt, offset, + pktlen - offset - len) != 0) { + odp_packet_free(newpkt); + newpkt = ODP_PACKET_INVALID; + } else { + odp_packet_hdr_t *new_hdr = odp_packet_hdr(newpkt); + new_hdr->buf_hdr.buf_u64 = pkt_hdr->buf_hdr.buf_u64; + odp_atomic_store_u32( + &new_hdr->buf_hdr.ref_count, + odp_atomic_load_u32( + &pkt_hdr->buf_hdr.ref_count)); + _odp_packet_parse(newpkt); + odp_packet_free(pkt); + } } - if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) { - pkt_hdr->input_flags.ipopt = 1; - return 0; - } + return newpkt; +} - /* A packet is a fragment if: - * "more fragments" flag is set (all fragments except the last) - * OR - * "fragment offset" field is nonzero (all fragments except the first) - */ - frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); - if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) { - pkt_hdr->input_flags.ipfrag = 1; - return 0; - } +/* + * + * Copy + * ******************************************************** + * + */ - if (ipv4->proto == ODPH_IPPROTO_ESP || - ipv4->proto == ODPH_IPPROTO_AH) { - pkt_hdr->input_flags.ipsec = 1; - return 0; +odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool) +{ + odp_packet_hdr_t *srchdr = odp_packet_hdr(pkt); + uint32_t pktlen = srchdr->frame_len; + uint32_t meta_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, buf_hdr); + odp_packet_t newpkt = odp_packet_alloc(pool, pktlen); + + if (newpkt != ODP_PACKET_INVALID) { + odp_packet_hdr_t *newhdr = odp_packet_hdr(newpkt); + uint8_t *newstart, *srcstart; + + /* Must copy meta data first, followed by packet data */ + newstart = (uint8_t *)newhdr + meta_offset; + srcstart = (uint8_t *)srchdr + meta_offset; + + memcpy(newstart, srcstart, + sizeof(odp_packet_hdr_t) - meta_offset); + + if (_odp_packet_copy_to_packet(pkt, 0, + newpkt, 0, pktlen) != 0) { + odp_packet_free(newpkt); + newpkt = ODP_PACKET_INVALID; + } } - /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after return */ - - *offset_out = sizeof(uint32_t) * ihl; - return ipv4->proto; + return newpkt; } -static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, - odph_ipv6hdr_t *ipv6, size_t *offset_out) +int odp_packet_copydata_out(odp_packet_t pkt, uint32_t offset, + uint32_t len, void *dst) { - if (ipv6->next_hdr == ODPH_IPPROTO_ESP || - ipv6->next_hdr == ODPH_IPPROTO_AH) { - pkt_hdr->input_flags.ipopt = 1; - pkt_hdr->input_flags.ipsec = 1; - return 0; + void *mapaddr; + uint32_t seglen = 0; /* GCC */ + uint32_t cpylen; + uint8_t *dstaddr = (uint8_t *)dst; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (offset + len > pkt_hdr->frame_len) + return -1; + + while (len > 0) { + mapaddr = packet_map(pkt_hdr, offset, &seglen); + cpylen = len > seglen ? seglen : len; + memcpy(dstaddr, mapaddr, cpylen); + offset += cpylen; + dstaddr += cpylen; + len -= cpylen; } - if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { - pkt_hdr->input_flags.ipopt = 1; - pkt_hdr->input_flags.ipfrag = 1; - return 0; + return 0; +} + +int odp_packet_copydata_in(odp_packet_t pkt, uint32_t offset, + uint32_t len, const void *src) +{ + void *mapaddr; + uint32_t seglen = 0; /* GCC */ + uint32_t cpylen; + const uint8_t *srcaddr = (const uint8_t *)src; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (offset + len > pkt_hdr->frame_len) + return -1; + + while (len > 0) { + mapaddr = packet_map(pkt_hdr, offset, &seglen); + cpylen = len > seglen ? seglen : len; + memcpy(mapaddr, srcaddr, cpylen); + offset += cpylen; + srcaddr += cpylen; + len -= cpylen; } - /* Don't step through more extensions */ - *offset_out = ODPH_IPV6HDR_LEN; - return ipv6->next_hdr; + return 0; } +/* + * + * Debugging + * ******************************************************** + * + */ + void odp_packet_print(odp_packet_t pkt) { int max_len = 512; char str[max_len]; - uint8_t *p; int len = 0; int n = max_len-1; odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); @@ -353,77 +555,358 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n-len, " output_flags 0x%x\n", hdr->output_flags.all); len += snprintf(&str[len], n-len, - " frame_offset %u\n", hdr->frame_offset); - len += snprintf(&str[len], n-len, " l2_offset %u\n", hdr->l2_offset); len += snprintf(&str[len], n-len, " l3_offset %u\n", hdr->l3_offset); len += snprintf(&str[len], n-len, " l4_offset %u\n", hdr->l4_offset); len += snprintf(&str[len], n-len, - " frame_len %u\n", hdr->buf_hdr.mb.pkt.pkt_len); + " frame_len %u\n", hdr->frame_len); len += snprintf(&str[len], n-len, " input %u\n", hdr->input); str[len] = '\0'; - printf("\n%s\n", str); - rte_pktmbuf_dump(stdout, &hdr->buf_hdr.mb, 32); + ODP_PRINT("\n%s\n", str); +} - p = odp_packet_start(pkt); - printf("00000000: %02X %02X %02X %02X %02X %02X %02X %02X\n", - p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7]); - printf("00000008: %02X %02X %02X %02X %02X %02X %02X %02X\n", - p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); +int odp_packet_is_valid(odp_packet_t pkt) +{ + odp_buffer_hdr_t *buf = validate_buf((odp_buffer_t)pkt); + return (buf != NULL && buf->type == ODP_BUFFER_TYPE_PACKET); } -/* For now we can only copy between packets of the same segment size - * We should probably refine this API, maybe introduce a clone API */ -int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src) +/* + * + * Internal Use Routines + * ******************************************************** + * + */ + +int _odp_packet_copy_to_packet(odp_packet_t srcpkt, uint32_t srcoffset, + odp_packet_t dstpkt, uint32_t dstoffset, + uint32_t len) { - struct rte_mbuf *mb_dst, *mb_src; - uint8_t nb_segs, i; + odp_packet_hdr_t *srchdr = odp_packet_hdr(srcpkt); + odp_packet_hdr_t *dsthdr = odp_packet_hdr(dstpkt); + void *srcmap; + void *dstmap; + uint32_t cpylen, minseg; + uint32_t srcseglen = 0; /* GCC */ + uint32_t dstseglen = 0; /* GCC */ + + if (srcoffset + len > srchdr->frame_len || + dstoffset + len > dsthdr->frame_len) + return -1; - ODP_ASSERT(odp_buffer_type(pkt_dst) == ODP_BUFFER_TYPE_PACKET && - odp_buffer_type(pkt_src) == ODP_BUFFER_TYPE_PACKET, - "dst_pkt or src_pkt not of type ODP_BUFFER_TYPE_PACKET"); + while (len > 0) { + srcmap = packet_map(srchdr, srcoffset, &srcseglen); + dstmap = packet_map(dsthdr, dstoffset, &dstseglen); - if (pkt_dst == ODP_PACKET_INVALID || pkt_src == ODP_PACKET_INVALID) - return -1; + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; + cpylen = len > minseg ? minseg : len; + memcpy(dstmap, srcmap, cpylen); - mb_dst = &(odp_packet_hdr(pkt_dst)->buf_hdr.mb); - mb_src = &(odp_packet_hdr(pkt_src)->buf_hdr.mb); + srcoffset += cpylen; + dstoffset += cpylen; + len -= cpylen; + } - if (mb_dst->pkt.nb_segs != mb_src->pkt.nb_segs) { - ODP_ERR("Different nb_segs in pkt_dst and pkt_src"); - return -1; + return 0; +} + +odp_packet_t _odp_packet_alloc(odp_buffer_pool_t pool_hdl) +{ + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); + + if (pool->s.params.buf_type != ODP_BUFFER_TYPE_PACKET) + return ODP_PACKET_INVALID; + + return (odp_packet_t)buffer_alloc(pool_hdl, + pool->s.params.buf_size); +} + +/** + * Parser helper function for IPv4 + */ +static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, + uint8_t **parseptr, uint32_t *offset) +{ + odph_ipv4hdr_t *ipv4 = (odph_ipv4hdr_t *)*parseptr; + uint8_t ver = ODPH_IPV4HDR_VER(ipv4->ver_ihl); + uint8_t ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); + uint16_t frag_offset; + + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv4->tot_len); + + if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN) || + odp_unlikely(ver != 4) || + (pkt_hdr->l3_len > pkt_hdr->frame_len - *offset)) { + pkt_hdr->error_flags.ip_err = 1; + return 0; } - nb_segs = mb_src->pkt.nb_segs; + *offset += ihl * 4; + *parseptr += ihl * 4; - if (mb_dst->buf_len < mb_src->buf_len) { - ODP_ERR("dst_pkt smaller than src_pkt"); - return -1; + if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) + pkt_hdr->input_flags.ipopt = 1; + + /* A packet is a fragment if: + * "more fragments" flag is set (all fragments except the last) + * OR + * "fragment offset" field is nonzero (all fragments except the first) + */ + frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); + if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) + pkt_hdr->input_flags.ipfrag = 1; + + return ipv4->proto; +} + +/** + * Parser helper function for IPv6 + */ +static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, + uint8_t **parseptr, uint32_t *offset) +{ + odph_ipv6hdr_t *ipv6 = (odph_ipv6hdr_t *)*parseptr; + odph_ipv6hdr_ext_t *ipv6ext; + + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv6->payload_len); + + /* Basic sanity checks on IPv6 header */ + if ((ipv6->ver_tc_flow >> 28) != 6 || + pkt_hdr->l3_len > pkt_hdr->frame_len - *offset) { + pkt_hdr->error_flags.ip_err = 1; + return 0; } - for (i = 0; i < nb_segs; i++) { - if (mb_src == NULL || mb_dst == NULL) { - ODP_ERR("Corrupted packets"); - return -1; + /* Skip past IPv6 header */ + *offset += sizeof(odph_ipv6hdr_t); + *parseptr += sizeof(odph_ipv6hdr_t); + + + /* Skip past any IPv6 extension headers */ + if (ipv6->next_hdr == ODPH_IPPROTO_HOPOPTS || + ipv6->next_hdr == ODPH_IPPROTO_ROUTE) { + pkt_hdr->input_flags.ipopt = 1; + + do { + ipv6ext = (odph_ipv6hdr_ext_t *)*parseptr; + uint16_t extlen = 8 + ipv6ext->ext_len * 8; + + *offset += extlen; + *parseptr += extlen; + } while ((ipv6ext->next_hdr == ODPH_IPPROTO_HOPOPTS || + ipv6ext->next_hdr == ODPH_IPPROTO_ROUTE) && + *offset < pkt_hdr->frame_len); + + if (*offset >= pkt_hdr->l3_offset + ipv6->payload_len) { + pkt_hdr->error_flags.ip_err = 1; + return 0; } - memcpy(mb_dst->buf_addr, mb_src->buf_addr, mb_src->buf_len); - mb_dst = mb_dst->pkt.next; - mb_src = mb_src->pkt.next; + + if (ipv6ext->next_hdr == ODPH_IPPROTO_FRAG) + pkt_hdr->input_flags.ipfrag = 1; + + return ipv6ext->next_hdr; } - return 0; + + if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { + pkt_hdr->input_flags.ipopt = 1; + pkt_hdr->input_flags.ipfrag = 1; + } + + return ipv6->next_hdr; +} + +/** + * Parser helper function for TCP + */ +static inline void parse_tcp(odp_packet_hdr_t *pkt_hdr, + uint8_t **parseptr, uint32_t *offset) +{ + odph_tcphdr_t *tcp = (odph_tcphdr_t *)*parseptr; + + if (tcp->hl < sizeof(odph_tcphdr_t)/sizeof(uint32_t)) + pkt_hdr->error_flags.tcp_err = 1; + else if ((uint32_t)tcp->hl * 4 > sizeof(odph_tcphdr_t)) + pkt_hdr->input_flags.tcpopt = 1; + + pkt_hdr->l4_len = pkt_hdr->l3_len + + pkt_hdr->l3_offset - pkt_hdr->l4_offset; + + *offset += (uint32_t)tcp->hl * 4; + *parseptr += (uint32_t)tcp->hl * 4; } -void odp_packet_set_ctx(odp_packet_t pkt, const void *ctx) +/** + * Parser helper function for UDP + */ +static inline void parse_udp(odp_packet_hdr_t *pkt_hdr, + uint8_t **parseptr, uint32_t *offset) { - odp_packet_hdr(pkt)->user_ctx = (intptr_t)ctx; + odph_udphdr_t *udp = (odph_udphdr_t *)*parseptr; + uint32_t udplen = odp_be_to_cpu_16(udp->length); + + if (udplen < sizeof(odph_udphdr_t) || + udplen > (pkt_hdr->l3_len + + pkt_hdr->l3_offset - pkt_hdr->l4_offset)) { + pkt_hdr->error_flags.udp_err = 1; + } + + pkt_hdr->l4_len = udplen; + + *offset += sizeof(odph_udphdr_t); + *parseptr += sizeof(odph_udphdr_t); } -void *odp_packet_get_ctx(odp_packet_t pkt) +/** + * Simple packet parser + */ + +int _odp_packet_parse(odp_packet_t pkt) { - return (void *)(intptr_t)odp_packet_hdr(pkt)->user_ctx; + odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); + odph_ethhdr_t *eth; + odph_vlanhdr_t *vlan; + uint16_t ethtype; + uint8_t *parseptr; + uint32_t offset, seglen; + uint8_t ip_proto = 0; + + /* Reset parser metadata for new parse */ + pkt_hdr->error_flags.all = 0; + pkt_hdr->input_flags.all = 0; + pkt_hdr->output_flags.all = 0; + pkt_hdr->l2_offset = 0; + pkt_hdr->l3_offset = 0; + pkt_hdr->l4_offset = 0; + pkt_hdr->payload_offset = 0; + pkt_hdr->vlan_s_tag = 0; + pkt_hdr->vlan_c_tag = 0; + pkt_hdr->l3_protocol = 0; + pkt_hdr->l4_protocol = 0; + + /* We only support Ethernet for now */ + pkt_hdr->input_flags.eth = 1; + + /* Detect jumbo frames */ + if (pkt_hdr->frame_len > ODPH_ETH_LEN_MAX) + pkt_hdr->input_flags.jumbo = 1; + + /* Assume valid L2 header, no CRC/FCS check in SW */ + pkt_hdr->input_flags.l2 = 1; + + eth = (odph_ethhdr_t *)packet_map(pkt_hdr, 0, &seglen); + offset = sizeof(odph_ethhdr_t); + parseptr = (uint8_t *)ð->type; + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + + /* Parse the VLAN header(s), if present */ + if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { + pkt_hdr->input_flags.vlan_qinq = 1; + pkt_hdr->input_flags.vlan = 1; + vlan = (odph_vlanhdr_t *)(void *)parseptr; + pkt_hdr->vlan_s_tag = ((ethtype << 16) | + odp_be_to_cpu_16(vlan->tci)); + offset += sizeof(odph_vlanhdr_t); + parseptr += sizeof(odph_vlanhdr_t); + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + } + + if (ethtype == ODPH_ETHTYPE_VLAN) { + pkt_hdr->input_flags.vlan = 1; + vlan = (odph_vlanhdr_t *)(void *)parseptr; + pkt_hdr->vlan_c_tag = ((ethtype << 16) | + odp_be_to_cpu_16(vlan->tci)); + offset += sizeof(odph_vlanhdr_t); + parseptr += sizeof(odph_vlanhdr_t); + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + } + + /* Check for SNAP vs. DIX */ + if (ethtype < ODPH_ETH_LEN_MAX) { + pkt_hdr->input_flags.snap = 1; + if (ethtype > pkt_hdr->frame_len - offset) { + pkt_hdr->error_flags.snap_len = 1; + goto parse_exit; + } + offset += 8; + parseptr += 8; + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + } + + /* Consume Ethertype for Layer 3 parse */ + parseptr += 2; + + /* Set l3_offset+flag only for known ethtypes */ + pkt_hdr->input_flags.l3 = 1; + pkt_hdr->l3_offset = offset; + pkt_hdr->l3_protocol = ethtype; + + /* Parse Layer 3 headers */ + switch (ethtype) { + case ODPH_ETHTYPE_IPV4: + pkt_hdr->input_flags.ipv4 = 1; + ip_proto = parse_ipv4(pkt_hdr, &parseptr, &offset); + break; + + case ODPH_ETHTYPE_IPV6: + pkt_hdr->input_flags.ipv6 = 1; + ip_proto = parse_ipv6(pkt_hdr, &parseptr, &offset); + break; + + case ODPH_ETHTYPE_ARP: + pkt_hdr->input_flags.arp = 1; + ip_proto = 255; /* Reserved invalid by IANA */ + break; + + default: + pkt_hdr->input_flags.l3 = 0; + ip_proto = 255; /* Reserved invalid by IANA */ + } + + /* Set l4_offset+flag only for known ip_proto */ + pkt_hdr->input_flags.l4 = 1; + pkt_hdr->l4_offset = offset; + pkt_hdr->l4_protocol = ip_proto; + + /* Parse Layer 4 headers */ + switch (ip_proto) { + case ODPH_IPPROTO_ICMP: + pkt_hdr->input_flags.icmp = 1; + break; + + case ODPH_IPPROTO_TCP: + pkt_hdr->input_flags.tcp = 1; + parse_tcp(pkt_hdr, &parseptr, &offset); + break; + + case ODPH_IPPROTO_UDP: + pkt_hdr->input_flags.udp = 1; + parse_udp(pkt_hdr, &parseptr, &offset); + break; + + case ODPH_IPPROTO_AH: + case ODPH_IPPROTO_ESP: + pkt_hdr->input_flags.ipsec = 1; + break; + + default: + pkt_hdr->input_flags.l4 = 0; + break; + } + + /* + * Anything beyond what we parse here is considered payload. + * Note: Payload is really only relevant for TCP and UDP. For + * all other protocols, the payload offset will point to the + * final header (ARP, ICMP, AH, ESP, or IP Fragment). + */ + pkt_hdr->payload_offset = offset; + +parse_exit: + return pkt_hdr->error_flags.all != 0; } diff --git a/platform/linux-dpdk/odp_packet_dpdk.c b/platform/linux-dpdk/odp_packet_dpdk.c index 9fe4bbd..80ce74f 100644 --- a/platform/linux-dpdk/odp_packet_dpdk.c +++ b/platform/linux-dpdk/odp_packet_dpdk.c @@ -23,6 +23,8 @@ #include <odp_hints.h> #include <odp_thread.h> +#include <odp_system_info.h> +#include <odp_debug_internal.h> #include <odp_packet_dpdk.h> #include <net/if.h> @@ -164,11 +166,8 @@ int recv_pkt_dpdk(pkt_dpdk_t * const pkt_dpdk, odp_packet_t pkt_table[], nb_rx = rte_eth_rx_burst((uint8_t)pkt_dpdk->portid, (uint16_t)pkt_dpdk->queueid, (struct rte_mbuf **)pkt_table, (uint16_t)len); - for (i = 0; i < nb_rx; i++) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt_table[i]); - struct rte_mbuf *mb = &pkt_hdr->buf_hdr.mb; - odp_packet_parse(pkt_table[i], mb->pkt.pkt_len, 0); - } + for (i = 0; i < nb_rx; i++) + _odp_packet_parse(pkt_table[i]); return nb_rx; } diff --git a/platform/linux-dpdk/odp_packet_io.c b/platform/linux-dpdk/odp_packet_io.c index 79394bb..b6d1d35 100644 --- a/platform/linux-dpdk/odp_packet_io.c +++ b/platform/linux-dpdk/odp_packet_io.c @@ -16,34 +16,21 @@ #ifdef ODP_HAVE_NETMAP #include <odp_packet_netmap.h> #endif -#include <odp_hints.h> #include <odp_config.h> #include <odp_queue_internal.h> #include <odp_schedule_internal.h> -#include <odp_debug.h> - -#include <odp_pktio_socket.h> -#ifdef ODP_HAVE_NETMAP -#include <odp_pktio_netmap.h> -#endif +#include <odp_classification_internal.h> +#include <odp_debug_internal.h> #include <string.h> - -typedef struct { - pktio_entry_t entries[ODP_CONFIG_PKTIO_ENTRIES]; -} pktio_table_t; +#include <sys/ioctl.h> +#include <linux/if_arp.h> +#include <ifaddrs.h> static pktio_table_t *pktio_tbl; - -static pktio_entry_t *get_entry(odp_pktio_t id) -{ - if (odp_unlikely(id == ODP_PKTIO_INVALID || - id > ODP_CONFIG_PKTIO_ENTRIES)) - return NULL; - - return &pktio_tbl->entries[id - 1]; -} +/* pktio pointer entries ( for inlines) */ +void *pktio_entry_ptr[ODP_CONFIG_PKTIO_ENTRIES]; int odp_pktio_init_global(void) { @@ -65,10 +52,12 @@ int odp_pktio_init_global(void) memset(pktio_tbl, 0, sizeof(pktio_table_t)); for (id = 1; id <= ODP_CONFIG_PKTIO_ENTRIES; ++id) { - pktio_entry = get_entry(id); + pktio_entry = &pktio_tbl->entries[id - 1]; odp_spinlock_init(&pktio_entry->s.lock); + odp_spinlock_init(&pktio_entry->s.cls.lock); + pktio_entry_ptr[id - 1] = pktio_entry; /* Create a default output queue for each pktio resource */ snprintf(name, sizeof(name), "%i-pktio_outq_default", (int)id); name[ODP_QUEUE_NAME_LEN-1] = '\0'; @@ -115,16 +104,27 @@ static void unlock_entry(pktio_entry_t *entry) odp_spinlock_unlock(&entry->s.lock); } -static void init_pktio_entry(pktio_entry_t *entry, odp_pktio_params_t *params) +static void lock_entry_classifier(pktio_entry_t *entry) +{ + odp_spinlock_lock(&entry->s.lock); + odp_spinlock_lock(&entry->s.cls.lock); +} + +static void unlock_entry_classifier(pktio_entry_t *entry) +{ + odp_spinlock_unlock(&entry->s.cls.lock); + odp_spinlock_unlock(&entry->s.lock); +} + +static void init_pktio_entry(pktio_entry_t *entry) { set_taken(entry); entry->s.inq_default = ODP_QUEUE_INVALID; memset(&entry->s.pkt_dpdk, 0, sizeof(entry->s.pkt_dpdk)); - /* Save pktio parameters, type is the most useful */ - memcpy(&entry->s.params, params, sizeof(*params)); + pktio_classifier_init(entry); } -static odp_pktio_t alloc_lock_pktio_entry(odp_pktio_params_t *params) +static odp_pktio_t alloc_lock_pktio_entry(void) { odp_pktio_t id; pktio_entry_t *entry; @@ -133,13 +133,13 @@ static odp_pktio_t alloc_lock_pktio_entry(odp_pktio_params_t *params) for (i = 0; i < ODP_CONFIG_PKTIO_ENTRIES; ++i) { entry = &pktio_tbl->entries[i]; if (is_free(entry)) { - lock_entry(entry); + lock_entry_classifier(entry); if (is_free(entry)) { - init_pktio_entry(entry, params); + init_pktio_entry(entry); id = i + 1; return id; /* return with entry locked! */ } - unlock_entry(entry); + unlock_entry_classifier(entry); } } @@ -148,7 +148,7 @@ static odp_pktio_t alloc_lock_pktio_entry(odp_pktio_params_t *params) static int free_pktio_entry(odp_pktio_t id) { - pktio_entry_t *entry = get_entry(id); + pktio_entry_t *entry = get_pktio_entry(id); if (entry == NULL) return -1; @@ -158,28 +158,22 @@ static int free_pktio_entry(odp_pktio_t id) return 0; } -odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool, - odp_pktio_params_t *params) +odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool) { odp_pktio_t id; pktio_entry_t *pktio_entry; int res; - if (params == NULL) { - ODP_ERR("Invalid pktio params\n"); - return ODP_PKTIO_INVALID; - } - ODP_DBG("Allocating dpdk pktio\n"); - id = alloc_lock_pktio_entry(params); + id = alloc_lock_pktio_entry(); if (id == ODP_PKTIO_INVALID) { ODP_ERR("No resources available.\n"); return ODP_PKTIO_INVALID; } /* if successful, alloc_pktio_entry() returns with the entry locked */ - pktio_entry = get_entry(id); + pktio_entry = get_pktio_entry(id); res = setup_pkt_dpdk(&pktio_entry->s.pkt_dpdk, dev, pool); if (res == -1) { @@ -188,7 +182,7 @@ odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool, id = ODP_PKTIO_INVALID; } - unlock_entry(pktio_entry); + unlock_entry_classifier(pktio_entry); return id; } @@ -197,7 +191,7 @@ int odp_pktio_close(odp_pktio_t id) pktio_entry_t *entry; int res = -1; - entry = get_entry(id); + entry = get_pktio_entry(id); if (entry == NULL) return -1; @@ -226,7 +220,7 @@ odp_pktio_t odp_pktio_get_input(odp_packet_t pkt) int odp_pktio_recv(odp_pktio_t id, odp_packet_t pkt_table[], unsigned len) { - pktio_entry_t *pktio_entry = get_entry(id); + pktio_entry_t *pktio_entry = get_pktio_entry(id); int pkts; int i; @@ -249,7 +243,7 @@ int odp_pktio_recv(odp_pktio_t id, odp_packet_t pkt_table[], unsigned len) int odp_pktio_send(odp_pktio_t id, odp_packet_t pkt_table[], unsigned len) { - pktio_entry_t *pktio_entry = get_entry(id); + pktio_entry_t *pktio_entry = get_pktio_entry(id); int pkts; if (pktio_entry == NULL) @@ -265,15 +259,17 @@ int odp_pktio_send(odp_pktio_t id, odp_packet_t pkt_table[], unsigned len) int odp_pktio_inq_setdef(odp_pktio_t id, odp_queue_t queue) { - pktio_entry_t *pktio_entry = get_entry(id); + pktio_entry_t *pktio_entry = get_pktio_entry(id); queue_entry_t *qentry = queue_to_qentry(queue); - if (pktio_entry == NULL || qentry == NULL) + if (pktio_entry == NULL || qentry == ODP_QUEUE_INVALID) return -1; if (qentry->s.type != ODP_QUEUE_TYPE_PKTIN) return -1; + qentry = queue_to_qentry(queue); + lock_entry(pktio_entry); pktio_entry->s.inq_default = queue; unlock_entry(pktio_entry); @@ -295,7 +291,7 @@ int odp_pktio_inq_remdef(odp_pktio_t id) odp_queue_t odp_pktio_inq_getdef(odp_pktio_t id) { - pktio_entry_t *pktio_entry = get_entry(id); + pktio_entry_t *pktio_entry = get_pktio_entry(id); if (pktio_entry == NULL) return ODP_QUEUE_INVALID; @@ -305,7 +301,7 @@ odp_queue_t odp_pktio_inq_getdef(odp_pktio_t id) odp_queue_t odp_pktio_outq_getdef(odp_pktio_t id) { - pktio_entry_t *pktio_entry = get_entry(id); + pktio_entry_t *pktio_entry = get_pktio_entry(id); if (pktio_entry == NULL) return ODP_QUEUE_INVALID; @@ -362,32 +358,32 @@ int pktin_enqueue(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr) odp_buffer_hdr_t *pktin_dequeue(queue_entry_t *qentry) { odp_buffer_hdr_t *buf_hdr; + odp_buffer_t buf; + odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; + odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX]; + int pkts, i, j; buf_hdr = queue_deq(qentry); + if (buf_hdr != NULL) + return buf_hdr; - if (buf_hdr == NULL) { - odp_packet_t pkt; - odp_buffer_t buf; - odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; - odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX]; - int pkts, i, j; - - pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl, - QUEUE_MULTI_MAX); - - if (pkts > 0) { - pkt = pkt_tbl[0]; - buf = odp_buffer_from_packet(pkt); - buf_hdr = odp_buf_to_hdr(buf); + pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl, QUEUE_MULTI_MAX); + if (pkts <= 0) + return NULL; - for (i = 1, j = 0; i < pkts; ++i) { - buf = odp_buffer_from_packet(pkt_tbl[i]); - tmp_hdr_tbl[j++] = odp_buf_to_hdr(buf); - } - queue_enq_multi(qentry, tmp_hdr_tbl, j); - } + for (i = 0, j = 0; i < pkts; ++i) { + buf = odp_packet_to_buffer(pkt_tbl[i]); + buf_hdr = odp_buf_to_hdr(buf); + if (0 > packet_classifier(qentry->s.pktin, pkt_tbl[i])) + tmp_hdr_tbl[j++] = buf_hdr; } + if (0 == j) + return NULL; + + if (j > 1) + queue_enq_multi(qentry, &tmp_hdr_tbl[1], j-1); + buf_hdr = tmp_hdr_tbl[0]; return buf_hdr; } @@ -400,32 +396,179 @@ int pktin_enq_multi(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr[], int num) int pktin_deq_multi(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr[], int num) { int nbr; + odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; + odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX]; + odp_buffer_hdr_t *tmp_hdr; + odp_buffer_t buf; + int pkts, i, j; nbr = queue_deq_multi(qentry, buf_hdr, num); - - if (nbr < num) { - odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; - odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX]; - odp_buffer_t buf; - int pkts, i; - - pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl, - QUEUE_MULTI_MAX); - if (pkts > 0) { - for (i = 0; i < pkts; ++i) { - buf = odp_buffer_from_packet(pkt_tbl[i]); - tmp_hdr_tbl[i] = odp_buf_to_hdr(buf); - } - queue_enq_multi(qentry, tmp_hdr_tbl, pkts); - } + if (odp_unlikely(nbr > num)) + ODP_ABORT("queue_deq_multi req: %d, returned %d\n", + num, nbr); + + /** queue already has number of requsted buffers, + * do not do receive in that case. + */ + if (nbr == num) + return nbr; + + pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl, QUEUE_MULTI_MAX); + if (pkts <= 0) + return nbr; + + for (i = 0, j = 0; i < pkts; ++i) { + buf = odp_packet_to_buffer(pkt_tbl[i]); + tmp_hdr = odp_buf_to_hdr(buf); + if (0 > packet_classifier(qentry->s.pktin, pkt_tbl[i])) + tmp_hdr_tbl[j++] = tmp_hdr; } + if (j) + queue_enq_multi(qentry, tmp_hdr_tbl, j); return nbr; } +int odp_pktio_set_mtu(odp_pktio_t id, int mtu) +{ + pktio_entry_t *entry; + int ret; + + if (mtu <= 0) { + ODP_DBG("illegal MTU value %d\n", mtu); + return -1; + } + + entry = get_pktio_entry(id); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", id); + return -1; + } + + lock_entry(entry); + + if (odp_unlikely(is_free(entry))) { + unlock_entry(entry); + ODP_DBG("already freed pktio\n"); + return -1; + } + + ret = rte_eth_dev_set_mtu(entry->s.pkt_dpdk.portid, (uint16_t)mtu); + + unlock_entry(entry); + return ret; +} + +int odp_pktio_mtu(odp_pktio_t id) +{ + pktio_entry_t *entry; + int mtu; + + entry = get_pktio_entry(id); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", id); + return -1; + } + + lock_entry(entry); + + if (odp_unlikely(is_free(entry))) { + unlock_entry(entry); + ODP_DBG("already freed pktio\n"); + return -1; + } + + rte_eth_dev_get_mtu(entry->s.pkt_dpdk.portid , (uint16_t *)&mtu); + + unlock_entry(entry); + return mtu; +} + +int odp_pktio_promisc_mode_set(odp_pktio_t id, odp_bool_t enable) +{ + pktio_entry_t *entry; + + entry = get_pktio_entry(id); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", id); + return -1; + } + + lock_entry(entry); + + if (odp_unlikely(is_free(entry))) { + unlock_entry(entry); + ODP_DBG("already freed pktio\n"); + return -1; + } + + if (enable) + rte_eth_promiscuous_enable(entry->s.pkt_dpdk.portid); + else + rte_eth_promiscuous_disable(entry->s.pkt_dpdk.portid); + + unlock_entry(entry); + return 0; +} + +int odp_pktio_promisc_mode(odp_pktio_t id) +{ + pktio_entry_t *entry; + int promisc; + + entry = get_pktio_entry(id); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", id); + return -1; + } + + lock_entry(entry); + + if (odp_unlikely(is_free(entry))) { + unlock_entry(entry); + ODP_DBG("already freed pktio\n"); + return -1; + } + + promisc = rte_eth_promiscuous_get(entry->s.pkt_dpdk.portid); + + unlock_entry(entry); + + return promisc; +} + +size_t odp_pktio_mac_addr(odp_pktio_t id, void *mac_addr, + size_t addr_size) +{ + pktio_entry_t *entry; + + if (addr_size < ETH_ALEN) + return 0; + + entry = get_pktio_entry(id); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", id); + return 0; + } + + lock_entry(entry); + + if (odp_unlikely(is_free(entry))) { + unlock_entry(entry); + ODP_DBG("already freed pktio\n"); + return -1; + } + + rte_eth_macaddr_get(entry->s.pkt_dpdk.portid, + (struct ether_addr *)mac_addr); + unlock_entry(entry); + + return ETH_ALEN; +} + int odp_pktio_get_mac_addr(odp_pktio_t id, unsigned char *mac_addr) { - pktio_entry_t *pktio_entry = get_entry(id); + pktio_entry_t *pktio_entry = get_pktio_entry(id); if (!pktio_entry) { ODP_ERR("Invalid odp_pktio_t value\n"); return -1; diff --git a/platform/linux-dpdk/odp_queue.c b/platform/linux-dpdk/odp_queue.c index 03f7c2c..9e906a3 100644 --- a/platform/linux-dpdk/odp_queue.c +++ b/platform/linux-dpdk/odp_queue.c @@ -11,13 +11,14 @@ #include <odp_buffer.h> #include <odp_buffer_internal.h> #include <odp_buffer_pool_internal.h> +#include <odp_buffer_inlines.h> #include <odp_internal.h> #include <odp_shared_memory.h> #include <odp_schedule_internal.h> #include <odp_config.h> #include <odp_packet_io_internal.h> #include <odp_packet_io_queue.h> -#include <odp_debug.h> +#include <odp_debug_internal.h> #include <odp_hints.h> #ifdef USE_TICKETLOCK diff --git a/platform/linux-dpdk/odp_schedule.c b/platform/linux-dpdk/odp_schedule.c deleted file mode 100644 index cb559e4..0000000 --- a/platform/linux-dpdk/odp_schedule.c +++ /dev/null @@ -1,421 +0,0 @@ -/* Copyright (c) 2013, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp_schedule.h> -#include <odp_schedule_internal.h> -#include <odp_align.h> -#include <odp_queue.h> -#include <odp_shared_memory.h> -#include <odp_buffer.h> -#include <odp_buffer_pool.h> -#include <odp_internal.h> -#include <odp_config.h> -#include <odp_debug.h> -#include <odp_thread.h> -#include <odp_time.h> -#include <odp_spinlock.h> -#include <odp_hints.h> - -#include <odp_queue_internal.h> - - -/* Limits to number of scheduled queues */ -#define SCHED_POOL_SIZE (256*1024) - -/* Scheduler sub queues */ -#define QUEUES_PER_PRIO 4 - -/* TODO: random or queue based selection */ -#define SEL_PRI_QUEUE(x) ((QUEUES_PER_PRIO-1) & (queue_to_id(x))) - -/* Maximum number of dequeues */ -#define MAX_DEQ 4 - - -/* Mask of queues per priority */ -typedef uint8_t pri_mask_t; - -ODP_STATIC_ASSERT((8 * sizeof(pri_mask_t)) >= - QUEUES_PER_PRIO, "pri_mask_t_is_too_small"); - - -typedef struct { - odp_queue_t pri_queue[ODP_CONFIG_SCHED_PRIOS][QUEUES_PER_PRIO]; - pri_mask_t pri_mask[ODP_CONFIG_SCHED_PRIOS]; - odp_spinlock_t mask_lock; - odp_buffer_pool_t pool; -} sched_t; - -typedef struct { - odp_queue_t queue; - -} queue_desc_t; - -typedef struct { - odp_queue_t pri_queue; - odp_buffer_t desc_buf; - - odp_buffer_t buf[MAX_DEQ]; - int num; - int index; - odp_queue_t queue; - int pause; - -} sched_local_t; - -/* Global scheduler context */ -static sched_t *sched; - -/* Thread local scheduler context */ -static __thread sched_local_t sched_local; - - -static inline odp_queue_t select_pri_queue(odp_queue_t queue, int prio) -{ - int id = SEL_PRI_QUEUE(queue); - return sched->pri_queue[prio][id]; -} - - -int odp_schedule_init_global(void) -{ - odp_shm_t shm; - odp_buffer_pool_t pool; - int i, j; - - ODP_DBG("Schedule init ... "); - - shm = odp_shm_reserve("odp_scheduler", - sizeof(sched_t), - ODP_CACHE_LINE_SIZE, 0); - - sched = odp_shm_addr(shm); - - if (sched == NULL) { - ODP_ERR("Schedule init: Shm reserve failed.\n"); - return -1; - } - - pool = odp_buffer_pool_create("odp_sched_pool", NULL, - SCHED_POOL_SIZE, sizeof(queue_desc_t), - ODP_CACHE_LINE_SIZE, - ODP_BUFFER_TYPE_RAW); - - if (pool == ODP_BUFFER_POOL_INVALID) { - ODP_ERR("Schedule init: Pool create failed.\n"); - return -1; - } - - sched->pool = pool; - odp_spinlock_init(&sched->mask_lock); - - for (i = 0; i < ODP_CONFIG_SCHED_PRIOS; i++) { - odp_queue_t queue; - char name[] = "odp_priXX_YY"; - - name[7] = '0' + i / 10; - name[8] = '0' + i - 10*(i / 10); - - for (j = 0; j < QUEUES_PER_PRIO; j++) { - name[10] = '0' + j / 10; - name[11] = '0' + j - 10*(j / 10); - - queue = odp_queue_create(name, - ODP_QUEUE_TYPE_POLL, NULL); - - if (queue == ODP_QUEUE_INVALID) { - ODP_ERR("Sched init: Queue create failed.\n"); - return -1; - } - - sched->pri_queue[i][j] = queue; - sched->pri_mask[i] = 0; - } - } - - ODP_DBG("done\n"); - - return 0; -} - - -int odp_schedule_init_local(void) -{ - int i; - - sched_local.pri_queue = ODP_QUEUE_INVALID; - sched_local.desc_buf = ODP_BUFFER_INVALID; - - for (i = 0; i < MAX_DEQ; i++) - sched_local.buf[i] = ODP_BUFFER_INVALID; - - sched_local.num = 0; - sched_local.index = 0; - sched_local.queue = ODP_QUEUE_INVALID; - sched_local.pause = 0; - - return 0; -} - - -void odp_schedule_mask_set(odp_queue_t queue, int prio) -{ - int id = SEL_PRI_QUEUE(queue); - - odp_spinlock_lock(&sched->mask_lock); - sched->pri_mask[prio] |= 1 << id; - odp_spinlock_unlock(&sched->mask_lock); -} - - -odp_buffer_t odp_schedule_buffer_alloc(odp_queue_t queue) -{ - odp_buffer_t buf; - - buf = odp_buffer_alloc(sched->pool); - - if (buf != ODP_BUFFER_INVALID) { - queue_desc_t *desc; - desc = odp_buffer_addr(buf); - desc->queue = queue; - } - - return buf; -} - - -void odp_schedule_queue(odp_queue_t queue, int prio) -{ - odp_buffer_t desc_buf; - odp_queue_t pri_queue; - - pri_queue = select_pri_queue(queue, prio); - desc_buf = queue_sched_buf(queue); - - odp_queue_enq(pri_queue, desc_buf); -} - - -void odp_schedule_release_atomic(void) -{ - if (sched_local.pri_queue != ODP_QUEUE_INVALID && - sched_local.num == 0) { - /* Release current atomic queue */ - odp_queue_enq(sched_local.pri_queue, sched_local.desc_buf); - sched_local.pri_queue = ODP_QUEUE_INVALID; - } -} - - -static inline int copy_bufs(odp_buffer_t out_buf[], unsigned int max) -{ - int i = 0; - - while (sched_local.num && max) { - out_buf[i] = sched_local.buf[sched_local.index]; - sched_local.index++; - sched_local.num--; - max--; - i++; - } - - return i; -} - - -/* - * Schedule queues - * - * TODO: SYNC_ORDERED not implemented yet - */ -static int schedule(odp_queue_t *out_queue, odp_buffer_t out_buf[], - unsigned int max_num, unsigned int max_deq) -{ - int i, j; - int thr; - int ret; - - if (sched_local.num) { - ret = copy_bufs(out_buf, max_num); - - if (out_queue) - *out_queue = sched_local.queue; - - return ret; - } - - odp_schedule_release_atomic(); - - if (odp_unlikely(sched_local.pause)) - return 0; - - thr = odp_thread_id(); - - for (i = 0; i < ODP_CONFIG_SCHED_PRIOS; i++) { - int id; - - if (sched->pri_mask[i] == 0) - continue; - - id = thr & (QUEUES_PER_PRIO-1); - - for (j = 0; j < QUEUES_PER_PRIO; j++, id++) { - odp_queue_t pri_q; - odp_buffer_t desc_buf; - - if (id >= QUEUES_PER_PRIO) - id = 0; - - if (odp_unlikely((sched->pri_mask[i] & (1 << id)) == 0)) - continue; - - pri_q = sched->pri_queue[i][id]; - desc_buf = odp_queue_deq(pri_q); - - if (desc_buf != ODP_BUFFER_INVALID) { - queue_desc_t *desc; - odp_queue_t queue; - int num; - - desc = odp_buffer_addr(desc_buf); - queue = desc->queue; - - num = odp_queue_deq_multi(queue, - sched_local.buf, - max_deq); - - if (num == 0) { - /* Remove empty queue from scheduling, - * except packet input queues - */ - if (odp_queue_type(queue) == - ODP_QUEUE_TYPE_PKTIN) - odp_queue_enq(pri_q, desc_buf); - - continue; - } - - sched_local.num = num; - sched_local.index = 0; - ret = copy_bufs(out_buf, max_num); - - sched_local.queue = queue; - - if (queue_sched_atomic(queue)) { - /* Hold queue during atomic access */ - sched_local.pri_queue = pri_q; - sched_local.desc_buf = desc_buf; - } else { - /* Continue scheduling the queue */ - odp_queue_enq(pri_q, desc_buf); - } - - /* Output the source queue handle */ - if (out_queue) - *out_queue = queue; - - return ret; - } - } - } - - return 0; -} - - -static int schedule_loop(odp_queue_t *out_queue, uint64_t wait, - odp_buffer_t out_buf[], - unsigned int max_num, unsigned int max_deq) -{ - uint64_t start_cycle, cycle, diff; - int ret; - - start_cycle = 0; - - while (1) { - ret = schedule(out_queue, out_buf, max_num, max_deq); - - if (ret) - break; - - if (wait == ODP_SCHED_WAIT) - continue; - - if (wait == ODP_SCHED_NO_WAIT) - break; - - if (start_cycle == 0) { - start_cycle = odp_time_get_cycles(); - continue; - } - - cycle = odp_time_get_cycles(); - diff = odp_time_diff_cycles(start_cycle, cycle); - - if (wait < diff) - break; - } - - return ret; -} - - -odp_buffer_t odp_schedule(odp_queue_t *out_queue, uint64_t wait) -{ - odp_buffer_t buf; - - buf = ODP_BUFFER_INVALID; - - schedule_loop(out_queue, wait, &buf, 1, MAX_DEQ); - - return buf; -} - - -odp_buffer_t odp_schedule_one(odp_queue_t *out_queue, uint64_t wait) -{ - odp_buffer_t buf; - - buf = ODP_BUFFER_INVALID; - - schedule_loop(out_queue, wait, &buf, 1, 1); - - return buf; -} - - -int odp_schedule_multi(odp_queue_t *out_queue, uint64_t wait, - odp_buffer_t out_buf[], unsigned int num) -{ - return schedule_loop(out_queue, wait, out_buf, num, MAX_DEQ); -} - - -void odp_schedule_pause(void) -{ - sched_local.pause = 1; -} - - -void odp_schedule_resume(void) -{ - sched_local.pause = 0; -} - - -uint64_t odp_schedule_wait_time(uint64_t ns) -{ - if (ns <= ODP_SCHED_NO_WAIT) - ns = ODP_SCHED_NO_WAIT + 1; - - return odp_time_ns_to_cycles(ns); -} - - -int odp_schedule_num_prio(void) -{ - return ODP_CONFIG_SCHED_PRIOS; -}