[08/10] Keystone2: Add initial HW buffer management

Buffer pools are managed as a hw queue filled with free buffers.
Allocating buffer -> dequeue from pool's free queue.
Freeing buffer    -> enqueue to pool's free queue.

This approach has important limitation - all free buffers are
linked to cppi descriptor, so max number of free buffers is
limited to max number of descriptors. If fast internal linking
ram is used, then max number of desctiptors is 16k.
Applications may create pools with much more buffers.
As a temporary solution max pool size is limited to 2000 buffers,
but buffer management should be made more flexible.

Signed-off-by: Taras Kondratiuk <taras.kondratiuk@linaro.org>
---
 platform/linux-keystone2/include/api/odp_buffer.h  |    3 +-
 .../include/configs/odp_config_platform.h          |    1 -
 .../linux-keystone2/include/odp_buffer_internal.h  |   81 ++--
 .../include/odp_buffer_pool_internal.h             |   47 +-
 platform/linux-keystone2/source/odp_buffer.c       |   66 +--
 platform/linux-keystone2/source/odp_buffer_pool.c  |  486 ++++++--------------
 platform/linux-keystone2/source/odp_init.c         |    4 +-
 platform/linux-keystone2/source/odp_packet.c       |   37 +-
 platform/linux-keystone2/source/odp_queue.c        |    4 +-
 9 files changed, 209 insertions(+), 520 deletions(-)

On 04/14/2014 03:55 PM, Taras Kondratiuk wrote:
> Buffer pools are managed as a hw queue filled with free buffers.
> Allocating buffer -> dequeue from pool's free queue.
> Freeing buffer    -> enqueue to pool's free queue.
>
> This approach has important limitation - all free buffers are
> linked to cppi descriptor, so max number of free buffers is
> limited to max number of descriptors. If fast internal linking
> ram is used, then max number of desctiptors is 16k.
> Applications may create pools with much more buffers.
> As a temporary solution max pool size is limited to 2000 buffers,
> but buffer management should be made more flexible.
>
> Signed-off-by: Taras Kondratiuk <taras.kondratiuk@linaro.org>
> ---
>   platform/linux-keystone2/include/api/odp_buffer.h  |    3 +-
>   .../include/configs/odp_config_platform.h          |    1 -
>   .../linux-keystone2/include/odp_buffer_internal.h  |   81 ++--
>   .../include/odp_buffer_pool_internal.h             |   47 +-
>   platform/linux-keystone2/source/odp_buffer.c       |   66 +--
>   platform/linux-keystone2/source/odp_buffer_pool.c  |  486 ++++++--------------
>   platform/linux-keystone2/source/odp_init.c         |    4 +-
>   platform/linux-keystone2/source/odp_packet.c       |   37 +-
>   platform/linux-keystone2/source/odp_queue.c        |    4 +-
>   9 files changed, 209 insertions(+), 520 deletions(-)
>
> diff --git a/platform/linux-keystone2/include/api/odp_buffer.h b/platform/linux-keystone2/include/api/odp_buffer.h
> index 2d2c25a..2edb914 100644
> --- a/platform/linux-keystone2/include/api/odp_buffer.h
> +++ b/platform/linux-keystone2/include/api/odp_buffer.h
> @@ -31,8 +31,7 @@ extern "C" {
>    */
>   typedef uint32_t odp_buffer_t;
>   
> -#define ODP_BUFFER_INVALID (0xffffffff) /**< Invalid buffer */
> -
> +#define ODP_BUFFER_INVALID (0) /**< Invalid buffer */
>   
>   /**
>    * Buffer start address
> diff --git a/platform/linux-keystone2/include/configs/odp_config_platform.h b/platform/linux-keystone2/include/configs/odp_config_platform.h
> index 743d8dd..ed023bc 100644
> --- a/platform/linux-keystone2/include/configs/odp_config_platform.h
> +++ b/platform/linux-keystone2/include/configs/odp_config_platform.h
> @@ -40,7 +40,6 @@
>   #error "platform not defined or unsupported!"
>   #endif
>   
> -#define TI_ODP_PUBLIC_DESC_SIZE		(64u)
>   #define TI_ODP_PUBLIC_DESC_NUM		(4096u)
>   #define TI_ODP_REGION_NUM		(2)  /* local regions are not used on Linux */
>   
> diff --git a/platform/linux-keystone2/include/odp_buffer_internal.h b/platform/linux-keystone2/include/odp_buffer_internal.h
> index f8ebc7c..044da4e 100644
> --- a/platform/linux-keystone2/include/odp_buffer_internal.h
> +++ b/platform/linux-keystone2/include/odp_buffer_internal.h
> @@ -22,21 +22,23 @@ extern "C" {
>   #include <odp_atomic.h>
>   #include <odp_buffer_pool.h>
>   #include <odp_buffer.h>
> +#include <odp_queue.h>
>   #include <odp_debug.h>
>   #include <odp_align.h>
>   
> -/* TODO: move these to correct files */
> -
> -typedef uint64_t odp_phys_addr_t;
> -
> -#define ODP_BUFFER_MAX_INDEX     (ODP_BUFFER_MAX_BUFFERS - 2)
> -#define ODP_BUFFER_INVALID_INDEX (ODP_BUFFER_MAX_BUFFERS - 1)
> +#include <event_machine_macros.h>
> +#include <event_machine_types.h>
> +#include <event_machine_group.h>
> +#include <event_machine_hw_macros.h>
> +#include <event_machine_hw_types.h>
> +#include <event_machine_hw_ti_macros.h>
> +#include <event_machine_hw_ti_types.h>
> +#include <ti_em_osal_cppi.h>
> +#include <src/event_machine_hwpform.h>
>   
> -#define ODP_BUFS_PER_CHUNK       16
> -#define ODP_BUFS_PER_SCATTER      4
> -
> -#define ODP_BUFFER_TYPE_CHUNK    0xffff
> +/* TODO: move these to correct files */
>   
> +typedef uintptr_t odp_phys_addr_t;
>   
>   #define ODP_BUFFER_POOL_BITS   4
>   #define ODP_BUFFER_INDEX_BITS  (32 - ODP_BUFFER_POOL_BITS)
> @@ -53,56 +55,33 @@ typedef union odp_buffer_bits_t {
>   	};
>   } odp_buffer_bits_t;
>   
> -
> -/* forward declaration */
> -struct odp_buffer_hdr_t;
> -
> -
> -/*
> - * Scatter/gather list of buffers
> - */
> -typedef struct odp_buffer_scatter_t {
> -	/* buffer pointers */
> -	struct odp_buffer_hdr_t *buf[ODP_BUFS_PER_SCATTER];
> -	int                      num_bufs;   /* num buffers */
> -	int                      pos;        /* position on the list */
> -	size_t                   total_len;  /* Total length */
> -} odp_buffer_scatter_t;
> +typedef struct odp_buffer_hdr_t {
> +	Cppi_HostDesc   desc;
> +	void		*buf_vaddr;
> +	odp_queue_t	free_queue;
> +	int type;
> +	struct odp_buffer_hdr_t *next;       /* next buf in a list */
> +	odp_buffer_bits_t        handle;     /* handle */
> +} odp_buffer_hdr_t;
>   
>   
>   /*
>    * Chunk of buffers (in single pool)
>    */
> -typedef struct odp_buffer_chunk_t {
> -	uint32_t num_bufs;                      /* num buffers */
> -	uint32_t buf_index[ODP_BUFS_PER_CHUNK]; /* buffers */
> -} odp_buffer_chunk_t;
>   
> -
> -typedef struct odp_buffer_hdr_t {
> -	struct odp_buffer_hdr_t *next;       /* next buf in a list */
> -	odp_buffer_bits_t        handle;     /* handle */
> -	odp_phys_addr_t          phys_addr;  /* physical data start address */
> -	void                    *addr;       /* virtual data start address */
> -	uint32_t                 index;	     /* buf index in the pool */
> -	size_t                   size;       /* max data size */
> -	size_t                   cur_offset; /* current offset */
> -	odp_atomic_int_t         ref_count;  /* reference count */
> -	odp_buffer_scatter_t     scatter;    /* Scatter/gather list */
> -	int                      type;       /* type of next header */
> -	odp_buffer_pool_t        pool;       /* buffer pool */
> -
> -	uint8_t                  payload[];  /* next header or data */
> -} odp_buffer_hdr_t;
> -
> -ODP_ASSERT(sizeof(odp_buffer_hdr_t) == ODP_OFFSETOF(odp_buffer_hdr_t, payload),
> +ODP_ASSERT(sizeof(odp_buffer_hdr_t) <= ODP_CACHE_LINE_SIZE,
>   	   ODP_BUFFER_HDR_T__SIZE_ERROR);
>   
> +static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf)
> +{
> +	return (odp_buffer_hdr_t *)buf;
> +}
> +static inline odp_buffer_t hdr_to_odp_buf(odp_buffer_hdr_t *hdr)
> +{
> +	return (odp_buffer_t)hdr;
> +}
>   
> -typedef struct odp_buffer_chunk_hdr_t {
> -	odp_buffer_hdr_t   buf_hdr;
> -	odp_buffer_chunk_t chunk;
> -} odp_buffer_chunk_hdr_t;
> +extern odp_buffer_pool_t odp_buf_to_pool(odp_buffer_t buf);
>   
>   
>   int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf);
> diff --git a/platform/linux-keystone2/include/odp_buffer_pool_internal.h b/platform/linux-keystone2/include/odp_buffer_pool_internal.h
> index 381482f..6ee3eb0 100644
> --- a/platform/linux-keystone2/include/odp_buffer_pool_internal.h
> +++ b/platform/linux-keystone2/include/odp_buffer_pool_internal.h
> @@ -39,7 +39,6 @@ extern "C" {
>   #include <odp_spinlock.h>
>   #endif
>   
> -
>   struct pool_entry_s {
>   #ifdef POOL_USE_TICKETLOCK
>   	odp_ticketlock_t        lock ODP_ALIGNED_CACHE;
> @@ -47,24 +46,25 @@ struct pool_entry_s {
>   	odp_spinlock_t          lock ODP_ALIGNED_CACHE;
>   #endif
>   
> -	odp_buffer_chunk_hdr_t *head;
>   	uint64_t                free_bufs;
>   	char                    name[ODP_BUFFER_POOL_NAME_LEN];
>   
>   	odp_buffer_pool_t       pool ODP_ALIGNED_CACHE;
> -	uintptr_t               buf_base;
> -	size_t                  buf_size;
> -	size_t                  buf_offset;
>   	uint64_t                num_bufs;
>   	void                   *pool_base_addr;
> +	uintptr_t               pool_base_paddr;
>   	uint64_t                pool_size;
>   	size_t                  payload_size;
>   	size_t                  payload_align;
>   	int                     buf_type;
> +	odp_queue_t             free_queue;
> +
> +	uintptr_t               buf_base;
> +	size_t                  buf_size;
> +	size_t                  buf_offset;
>   	size_t                  hdr_size;
>   };
>   
> -
>   extern void *pool_entry_ptr[];
>   
>   
> @@ -73,41 +73,6 @@ static inline void *get_pool_entry(odp_buffer_pool_t pool_id)
>   	return pool_entry_ptr[pool_id];
>   }
>   
> -
> -static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf)
> -{
> -	odp_buffer_bits_t handle;
> -	uint32_t pool_id;
> -	uint32_t index;
> -	struct pool_entry_s *pool;
> -	odp_buffer_hdr_t *hdr;
> -
> -	handle.u32 = buf;
> -	pool_id    = handle.pool;
> -	index      = handle.index;
> -
> -#ifdef POOL_ERROR_CHECK
> -	if (odp_unlikely(pool_id > ODP_CONFIG_BUFFER_POOLS)) {
> -		ODP_ERR("odp_buf_to_hdr: Bad pool id\n");
> -		return NULL;
> -	}
> -#endif
> -
> -	pool = get_pool_entry(pool_id);
> -
> -#ifdef POOL_ERROR_CHECK
> -	if (odp_unlikely(index > pool->num_bufs - 1)) {
> -		ODP_ERR("odp_buf_to_hdr: Bad buffer index\n");
> -		return NULL;
> -	}
> -#endif
> -
> -	hdr = (odp_buffer_hdr_t *)(pool->buf_base + index * pool->buf_size);
> -
> -	return hdr;
> -}
> -
> -
>   #ifdef __cplusplus
>   }
>   #endif
> diff --git a/platform/linux-keystone2/source/odp_buffer.c b/platform/linux-keystone2/source/odp_buffer.c
> index afbe96a..7a50aa2 100644
> --- a/platform/linux-keystone2/source/odp_buffer.c
> +++ b/platform/linux-keystone2/source/odp_buffer.c
> @@ -7,59 +7,38 @@
>   #include <odp_buffer.h>
>   #include <odp_buffer_internal.h>
>   #include <odp_buffer_pool_internal.h>
> -
> -#include <string.h>
> -#include <stdio.h>
> -
> +#include <ti_em_rh.h>
>   
>   void *odp_buffer_addr(odp_buffer_t buf)
>   {
> -	odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf);
> -
> -	return hdr->addr;
> +	return odp_buf_to_hdr(buf)->buf_vaddr;
>   }
>   
> -
>   size_t odp_buffer_size(odp_buffer_t buf)
>   {
> -	odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf);
> -
> -	return hdr->size;
> +	return (size_t)odp_buf_to_hdr(buf)->desc.origBufferLen;
>   }
>   
<one empty line>
> -
>   int odp_buffer_type(odp_buffer_t buf)
>   {
> -	odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf);
> -
> -	return hdr->type;
> +	return odp_buf_to_hdr(buf)->type;
>   }
>   
<one empty line>
> -
>   int odp_buffer_is_scatter(odp_buffer_t buf)
>   {
> -	odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf);
> -
> -	if (hdr->scatter.num_bufs == 0)
> -		return 0;
> -	else
> -		return 1;
> +	return (odp_buf_to_hdr(buf)->desc.nextBDPtr) ? 1 : 0;
>   }
>   
>   
<two empty lines>
>   int odp_buffer_is_valid(odp_buffer_t buf)
>   {
> -	odp_buffer_bits_t handle;
> -
> -	handle.u32 = buf;
> -
> -	return (handle.index != ODP_BUFFER_INVALID_INDEX);
> +	return (buf != ODP_BUFFER_INVALID);
>   }
>   
>   
<two empty lines>
>   int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf)
>   {
> -	odp_buffer_hdr_t *hdr;
> +	odp_buffer_hdr_t *desc;
>   	int len = 0;
>   
>   	if (!odp_buffer_is_valid(buf)) {
> @@ -67,34 +46,27 @@ int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf)
>   		return len;
>   	}
>   
> -	hdr = odp_buf_to_hdr(buf);
> +	desc = odp_buf_to_hdr(buf);
>   
>   	len += snprintf(&str[len], n-len,
>   			"Buffer\n");
>   	len += snprintf(&str[len], n-len,
> -			"  pool         %i\n",        hdr->pool);
> -	len += snprintf(&str[len], n-len,
> -			"  index        %"PRIu32"\n", hdr->index);
> -	len += snprintf(&str[len], n-len,
> -			"  phy_addr     %"PRIu64"\n", hdr->phys_addr);
> -	len += snprintf(&str[len], n-len,
> -			"  addr         %p\n",        hdr->addr);
> -	len += snprintf(&str[len], n-len,
> -			"  size         %zu\n",       hdr->size);
> +			"  desc_vaddr  %p\n",      desc);
>   	len += snprintf(&str[len], n-len,
> -			"  cur_offset   %zu\n",       hdr->cur_offset);
> +			"  buf_vaddr   %p\n",      desc->buf_vaddr);
>   	len += snprintf(&str[len], n-len,
> -			"  ref_count    %i\n",        hdr->ref_count);
> +			"  buf_paddr_o 0x%x\n",    desc->desc.origBuffPtr);
>   	len += snprintf(&str[len], n-len,
> -			"  type         %i\n",        hdr->type);
> +			"  buf_paddr   0x%x\n",    desc->desc.buffPtr);
>   	len += snprintf(&str[len], n-len,
> -			"  Scatter list\n");
> +			"  pool        %i\n",      odp_buf_to_pool(buf));
>   	len += snprintf(&str[len], n-len,
> -			"    num_bufs   %i\n",        hdr->scatter.num_bufs);
> -	len += snprintf(&str[len], n-len,
> -			"    pos        %i\n",        hdr->scatter.pos);
> -	len += snprintf(&str[len], n-len,
> -			"    total_len  %zu\n",       hdr->scatter.total_len);
> +			"  free_queue  %u\n",      desc->free_queue);
> +
> +	len += snprintf(&str[len], n-len, "\n");
> +
> +	ti_em_rh_dump_mem(desc, sizeof(*desc), "Descriptor dump");
> +	ti_em_rh_dump_mem(desc->buf_vaddr, 64, "Buffer start");
>   
>   	return len;
>   }
> diff --git a/platform/linux-keystone2/source/odp_buffer_pool.c b/platform/linux-keystone2/source/odp_buffer_pool.c
> index 90214ba..fe39fad 100644
> --- a/platform/linux-keystone2/source/odp_buffer_pool.c
> +++ b/platform/linux-keystone2/source/odp_buffer_pool.c
> @@ -10,15 +10,18 @@
>   #include <odp_buffer_internal.h>
>   #include <odp_packet_internal.h>
>   #include <odp_shared_memory.h>
> +#include <odp_shared_memory_internal.h>
>   #include <odp_align.h>
>   #include <odp_internal.h>
>   #include <odp_config.h>
> +#include <configs/odp_config_platform.h>
>   #include <odp_hints.h>
>   #include <odp_debug.h>
> +#include <odp_sync.h>
>   
>   #include <string.h>
>   #include <stdlib.h>
> -
> +#include <ti_em_rh.h>
>   
>   #ifdef POOL_USE_TICKETLOCK
>   #include <odp_ticketlock.h>
> @@ -33,13 +36,11 @@
>   #endif
>   
>   
> -#if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS
> -#error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS
> -#endif
> -
>   #define NULL_INDEX ((uint32_t)-1)
>   
>   
> +
> +
>   typedef union pool_entry_u {
>   	struct pool_entry_s s;
>   
> @@ -53,6 +54,10 @@ typedef struct pool_table_t {
>   
>   } pool_table_t;
>   
> +typedef struct {
> +	uintptr_t p;
> +	uintptr_t v;
> +} pvaddr_t;
>   
>   /* The pool table */
>   static pool_table_t *pool_tbl;
> @@ -60,25 +65,29 @@ static pool_table_t *pool_tbl;
>   /* Pool entry pointers (for inlining) */
>   void *pool_entry_ptr[ODP_CONFIG_BUFFER_POOLS];
>   
> -
> -static __thread odp_buffer_chunk_hdr_t *local_chunk[ODP_CONFIG_BUFFER_POOLS];
> -
> -
> -static inline void set_handle(odp_buffer_hdr_t *hdr,
> -			      pool_entry_t *pool, uint32_t index)
> +static uint32_t ti_odp_alloc_public_desc(uint32_t num)
>   {
> -	uint32_t pool_id = (uint32_t) pool->s.pool;
> +	static uint32_t free_desc_id;
> +	uint32_t tmp;
>   
> -	if (pool_id > ODP_CONFIG_BUFFER_POOLS)
> -		ODP_ERR("set_handle: Bad pool id\n");
> +	if (free_desc_id + num > TI_ODP_PUBLIC_DESC_NUM)
> +		return -1;
>   
> -	if (index > ODP_BUFFER_MAX_INDEX)
> -		ODP_ERR("set_handle: Bad buffer index\n");
> +	tmp = __sync_fetch_and_add(&free_desc_id, num);
>   
> -	hdr->handle.pool  = pool_id;
> -	hdr->handle.index = index;
> +	if (tmp + num > TI_ODP_PUBLIC_DESC_NUM) {
> +		__sync_fetch_and_sub(&free_desc_id, num);
> +		return -1;
> +	}
> +	return tmp;
>   }
>   
> +odp_buffer_pool_t odp_buf_to_pool(odp_buffer_t buf)
> +{
> +	odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf);
> +	pool_entry_t *pool = get_pool_entry(0);
> +	return hdr->free_queue - pool->s.free_queue;
> +}
>   
>   int odp_buffer_pool_init_global(void)
>   {
> @@ -98,231 +107,133 @@ int odp_buffer_pool_init_global(void)
>   		pool_entry_t *pool = &pool_tbl->pool[i];
>   		LOCK_INIT(&pool->s.lock);
>   		pool->s.pool = i;
> -
>   		pool_entry_ptr[i] = pool;
> +		pool->s.free_queue = TI_ODP_FREE_QUEUE_BASE_IDX + i;
>   	}
>   
>   	ODP_DBG("\nBuffer pool init global\n");
>   	ODP_DBG("  pool_entry_s size     %zu\n", sizeof(struct pool_entry_s));
>   	ODP_DBG("  pool_entry_t size     %zu\n", sizeof(pool_entry_t));
> -	ODP_DBG("  odp_buffer_hdr_t size %zu\n", sizeof(odp_buffer_hdr_t));
>   	ODP_DBG("\n");
>   	return 0;
>   }
>   
> +#define MAX_BUFS_PER_POOL	2000
>   
> -static odp_buffer_hdr_t *index_to_hdr(pool_entry_t *pool, uint32_t index)
> -{
> -	odp_buffer_hdr_t *hdr;
> -
> -	hdr = (odp_buffer_hdr_t *)(pool->s.buf_base + index * pool->s.buf_size);
> -	return hdr;
> -}
> -
> -
> -static void add_buf_index(odp_buffer_chunk_hdr_t *chunk_hdr, uint32_t index)
> -{
> -	uint32_t i = chunk_hdr->chunk.num_bufs;
> -	chunk_hdr->chunk.buf_index[i] = index;
> -	chunk_hdr->chunk.num_bufs++;
> -}
> -
> -
> -static uint32_t rem_buf_index(odp_buffer_chunk_hdr_t *chunk_hdr)
> -{
> -	uint32_t index;
> -	uint32_t i;
> -
> -	i = chunk_hdr->chunk.num_bufs - 1;
> -	index = chunk_hdr->chunk.buf_index[i];
> -	chunk_hdr->chunk.num_bufs--;
> -	return index;
> -}
> -
> -
> -static odp_buffer_chunk_hdr_t *next_chunk(pool_entry_t *pool,
> -					  odp_buffer_chunk_hdr_t *chunk_hdr)
> -{
> -	uint32_t index;
> -
> -	index = chunk_hdr->chunk.buf_index[ODP_BUFS_PER_CHUNK-1];
> -	if (index == NULL_INDEX)
> -		return NULL;
> -	else
> -		return (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, index);
> -}
> -
> -
> -static odp_buffer_chunk_hdr_t *rem_chunk(pool_entry_t *pool)
> +static int link_bufs(pool_entry_t *pool)
>   {
> -	odp_buffer_chunk_hdr_t *chunk_hdr;
> -
> -	chunk_hdr = pool->s.head;
> -	if (chunk_hdr == NULL) {
> -		/* Pool is empty */
> -		return NULL;
> +	size_t buf_size, buf_align;
> +	uint64_t pool_size;
> +	uintptr_t pool_base;
> +	pvaddr_t buf_addr, desc_addr;
> +	uint32_t desc_index;
> +	uint32_t num_bufs, i;
> +
> +	buf_align  = pool->s.payload_align;
> +	buf_size   = ODP_ALIGN_ROUNDUP(pool->s.payload_size, buf_align);
> +	pool_size  = pool->s.pool_size;
> +	pool_base  = (uintptr_t) pool->s.pool_base_addr;
> +	/* First buffer */
> +	buf_addr.v = ODP_ALIGN_ROUNDUP(pool_base, buf_align);
> +	buf_addr.p = _odp_shm_get_paddr((void *)buf_addr.v);
> +	pool->s.buf_base = buf_addr.v;
> +
> +	num_bufs   = (pool_size - (buf_addr.v - pool_base)) / buf_size;
> +	/*
> +	 * FIXME: Currently a number of HW descriptors is limited,
> +	 *        so temporary limit max number of buffers per pool
> +	 *        to be albe to run ODP example apps.
> +	 *        Descriptor management have to be made more intelligent
> +	 *        To remove this limitation.
> +	 */
> +	if (num_bufs > MAX_BUFS_PER_POOL) {
> +		ODP_DBG("Limiting number of buffer in %s from %d to %d\n",
> +			pool->s.name, num_bufs, MAX_BUFS_PER_POOL);
> +		num_bufs = MAX_BUFS_PER_POOL;
>   	}
>   
> -	pool->s.head = next_chunk(pool, chunk_hdr);
> -	pool->s.free_bufs -= ODP_BUFS_PER_CHUNK;
> -
> -	/* unlink */
> -	rem_buf_index(chunk_hdr);
> -	return chunk_hdr;
> -}
> -
> -
> -static void add_chunk(pool_entry_t *pool, odp_buffer_chunk_hdr_t *chunk_hdr)
> -{
> -	if (pool->s.head) {
> -		/* link pool head to the chunk */
> -		add_buf_index(chunk_hdr, pool->s.head->buf_hdr.index);
> -	} else
> -		add_buf_index(chunk_hdr, NULL_INDEX);
> -
> -	pool->s.head = chunk_hdr;
> -	pool->s.free_bufs += ODP_BUFS_PER_CHUNK;
> -}
> -
> -
> -static void check_align(pool_entry_t *pool, odp_buffer_hdr_t *hdr)
> -{
> -	if (!ODP_ALIGNED_CHECK_POWER_2(hdr->addr, pool->s.payload_align)) {
> -		ODP_ERR("check_align: payload align error %p, align %zu\n",
> -			hdr->addr, pool->s.payload_align);
> -		exit(0);
> +	desc_index = ti_odp_alloc_public_desc(num_bufs);
> +
> +	ODP_DBG("%s: buf_size: %zu, buf_align: %zu\n", __func__,
> +		buf_size, buf_align);
> +	ODP_DBG("%s: pool_size: %llu, pool_base: 0x%p\n", __func__,
> +		pool_size, (void *)pool_base);
> +	ODP_DBG("%s: buf_addr.v: 0x%p, buf_addr.p: 0x%p\n", __func__,
> +		(void *)buf_addr.v, (void *)buf_addr.p);
> +	ODP_DBG("%s: num_bufs: %u, desc_index: %u\n", __func__,
> +		num_bufs, desc_index);
> +	
> +	/* FIXME: Need to define error codes somewhere */
> +	if (desc_index == (uint32_t)-1) {
> +		ODP_ERR("Failed to allocate %u descriptors for pool %s\n",
> +			num_bufs, pool->s.name);
> +		return -1;
>   	}
>   
> -	if (!ODP_ALIGNED_CHECK_POWER_2(hdr, ODP_CACHE_LINE_SIZE)) {
> -		ODP_ERR("check_align: hdr align error %p, align %i\n",
> -			hdr, ODP_CACHE_LINE_SIZE);
> -		exit(0);
> +	if (ti_em_osal_hw_queue_open(pool->s.free_queue) != EM_OK) {
> +		ODP_ERR("Failed to open HW queue %u\n", pool->s.free_queue);
> +		return -1;
>   	}
> -}
> -
> -
> -static void fill_hdr(void *ptr, pool_entry_t *pool, uint32_t index,
> -		     int buf_type)
> -{
> -	odp_buffer_hdr_t *hdr = (odp_buffer_hdr_t *)ptr;
> -	size_t size = pool->s.hdr_size;
> -	uint8_t *payload = hdr->payload;
>   
> -	if (buf_type == ODP_BUFFER_TYPE_CHUNK)
> -		size = sizeof(odp_buffer_chunk_hdr_t);
> +	for (i = 0; i < num_bufs; i++) {
> +		Cppi_DescTag     tag;
> +		odp_buffer_hdr_t *hdr;
>   
> -	if (pool->s.buf_type == ODP_BUFFER_TYPE_PACKET) {
> -		odp_packet_hdr_t *packet_hdr = ptr;
> -		payload = packet_hdr->payload;
> +		/*
> +		 * TODO: Need to get descriptor size here and shift
> +		 * descriptor address, but not query it on every iteration.
> +		 */
> +		desc_addr.v = (uintptr_t)ti_em_rh_public_desc_addr(desc_index,
> +				&desc_addr.p);
> +		hdr = (odp_buffer_hdr_t *)desc_addr.v;
> +		memset((void *)hdr, 0, sizeof(*hdr));
> +
> +		hdr->free_queue = pool->s.free_queue;
> +		hdr->buf_vaddr  = (void *)buf_addr.v;
> +
> +		/* Set defaults in descriptor */
> +		hdr->desc.descInfo = (Cppi_DescType_HOST << 30) |
> +				     (Cppi_PSLoc_PS_IN_DESC << 22) |
> +				     (buf_size & 0xFFFF);
> +		hdr->desc.packetInfo =
> +			(((uint32_t) Cppi_EPIB_EPIB_PRESENT) << 31) |
> +			(0x2 << 16) |
> +			(((uint32_t) Cppi_ReturnPolicy_RETURN_BUFFER) << 15) |
> +			(pool->s.free_queue & 0x3FFF);
> +		hdr->desc.origBuffPtr   = buf_addr.p;
> +		hdr->desc.buffPtr       = buf_addr.p;
> +		hdr->desc.origBufferLen = buf_size;
> +		hdr->desc.buffLen       = buf_size;
> +
> +		/* TODO: pslen is set to 0, but should be configurable */
> +		ti_em_cppi_set_pslen(Cppi_DescType_HOST,
> +				     (Cppi_Desc *)(hdr), 0);
> +
> +		tag.srcTagHi  = 0x00;
> +		tag.srcTagLo  = 0xFF;
> +		tag.destTagHi = 0x00;
> +		tag.destTagLo = 0x00;
> +		ti_em_cppi_set_tag(Cppi_DescType_HOST,
> +				   (Cppi_Desc *)(hdr),
> +				   &tag);
> +
> +		odp_sync_stores();
> +		ti_em_osal_hw_queue_push_size(pool->s.free_queue,
> +					 (void*)hdr,
> +					 sizeof(Cppi_HostDesc),
> +					 TI_EM_MEM_PUBLIC_DESC);
> +		buf_addr.v += buf_size;
> +		buf_addr.p += buf_size;
> +		desc_index++;
>   	}
>   
> -	memset(hdr, 0, size);
> -
> -	set_handle(hdr, pool, index);
> -
> -	hdr->addr  = &payload[pool->s.buf_offset - pool->s.hdr_size];
> -	hdr->index = index;
> -	hdr->size  = pool->s.payload_size;
> -	hdr->pool  = pool->s.pool;
> -	hdr->type  = buf_type;
> -
> -	check_align(pool, hdr);
> -}
> -
> -
> -static void link_bufs(pool_entry_t *pool)
> -{
> -	odp_buffer_chunk_hdr_t *chunk_hdr;
> -	size_t hdr_size;
> -	size_t payload_size;
> -	size_t payload_align;
> -	size_t size;
> -	size_t offset;
> -	size_t min_size;
> -	uint64_t pool_size;
> -	uintptr_t buf_base;
> -	uint32_t index;
> -	uintptr_t pool_base;
> -	int buf_type;
> -
> -	buf_type      = pool->s.buf_type;
> -	payload_size  = pool->s.payload_size;
> -	payload_align = pool->s.payload_align;
> -	pool_size     = pool->s.pool_size;
> -	pool_base     = (uintptr_t) pool->s.pool_base_addr;
> -
> -	if (buf_type == ODP_BUFFER_TYPE_RAW)
> -		hdr_size = sizeof(odp_buffer_hdr_t);
> -	else if (buf_type == ODP_BUFFER_TYPE_PACKET)
> -		hdr_size = sizeof(odp_packet_hdr_t);
> -	else {
> -		ODP_ERR("odp_buffer_pool_create: Bad type %i\n",
> -			buf_type);
> -		exit(0);
> -	}
> -
> -	/* Chunk must fit into buffer payload.*/
> -	min_size = sizeof(odp_buffer_chunk_hdr_t) - hdr_size;
> -	if (payload_size < min_size)
> -		payload_size = min_size;
> -
> -	/* Roundup payload size to full cachelines */
> -	payload_size = ODP_CACHE_LINE_SIZE_ROUNDUP(payload_size);
> -
> -	/* Min cacheline alignment for buffer header and payload */
> -	payload_align = ODP_CACHE_LINE_SIZE_ROUNDUP(payload_align);
> -	offset        = ODP_CACHE_LINE_SIZE_ROUNDUP(hdr_size);
> -
> -	/* Multiples of cacheline size */
> -	if (payload_size > payload_align)
> -		size = payload_size + offset;
> -	else
> -		size = payload_align + offset;
> -
> -	/* First buffer */
> -	buf_base = ODP_ALIGN_ROUNDUP(pool_base + offset, payload_align)
> -		   - offset;
> -
> -	pool->s.hdr_size   = hdr_size;
> -	pool->s.buf_base   = buf_base;
> -	pool->s.buf_size   = size;
> -	pool->s.buf_offset = offset;
> -	index = 0;
> -
> -	chunk_hdr = (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, index);
> -	pool->s.head   = NULL;
> -	pool_size     -= buf_base - pool_base;
> -
> -	while (pool_size > ODP_BUFS_PER_CHUNK * size) {
> -		int i;
> -
> -		fill_hdr(chunk_hdr, pool, index, ODP_BUFFER_TYPE_CHUNK);
> -
> -		index++;
> -
> -		for (i = 0; i < ODP_BUFS_PER_CHUNK - 1; i++) {
> -			odp_buffer_hdr_t *hdr = index_to_hdr(pool, index);
> -
> -			fill_hdr(hdr, pool, index, buf_type);
> -
> -			add_buf_index(chunk_hdr, index);
> -			index++;
> -		}
> -
> -		add_chunk(pool, chunk_hdr);
> -
> -		chunk_hdr = (odp_buffer_chunk_hdr_t *)index_to_hdr(pool,
> -								   index);
> -		pool->s.num_bufs += ODP_BUFS_PER_CHUNK;
> -		pool_size -=  ODP_BUFS_PER_CHUNK * size;
> -	}
> +	return 0;
>   }
>   
<one empty line>
> -
>   odp_buffer_pool_t odp_buffer_pool_create(const char *name,
> -					 void *base_addr, uint64_t size,
> -					 size_t buf_size, size_t buf_align,
> -					 int buf_type)
> +		void *base_addr, uint64_t size,
> +		size_t buf_size, size_t buf_align,
> +		int buf_type)
>   {
>   	odp_buffer_pool_t i;
>   	pool_entry_t *pool;
> @@ -335,7 +246,8 @@ odp_buffer_pool_t odp_buffer_pool_create(const char *name,
>   
>   		if (pool->s.buf_base == 0) {
>   			/* found free pool */
> -
> +			ODP_DBG("%s: found free pool id: %u for %s\n", __func__,
> +				i, name);
>   			strncpy(pool->s.name, name,
>   				ODP_BUFFER_POOL_NAME_LEN - 1);
>   			pool->s.name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0;
> @@ -344,12 +256,12 @@ odp_buffer_pool_t odp_buffer_pool_create(const char *name,
>   			pool->s.payload_size   = buf_size;
>   			pool->s.payload_align  = buf_align;
>   			pool->s.buf_type       = buf_type;
> +			pool->s.buf_base = (uintptr_t)ODP_ALIGN_ROUNDUP_PTR(
> +						   base_addr, buf_align);
>   
> -			link_bufs(pool);
> -
> +			if (link_bufs(pool) != -1)
> +				pool_id = i;
>   			UNLOCK(&pool->s.lock);
> -
> -			pool_id = i;
>   			break;
>   		}
>   
> @@ -359,7 +271,6 @@ odp_buffer_pool_t odp_buffer_pool_create(const char *name,
>   	return pool_id;
>   }
>   
> -
>   odp_buffer_pool_t odp_buffer_pool_lookup(const char *name)
>   {
>   	odp_buffer_pool_t i;
> @@ -383,129 +294,22 @@ odp_buffer_pool_t odp_buffer_pool_lookup(const char *name)
>   
>   odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_id)
>   {
> -	pool_entry_t *pool;
> -	odp_buffer_chunk_hdr_t *chunk;
> -	odp_buffer_bits_t handle;
> -
> -	pool  = get_pool_entry(pool_id);
> -	chunk = local_chunk[pool_id];
> -
> -	if (chunk == NULL) {
> -		LOCK(&pool->s.lock);
> -		chunk = rem_chunk(pool);
> -		UNLOCK(&pool->s.lock);
> -
> -		if (chunk == NULL)
> -			return ODP_BUFFER_INVALID;
> -
> -		local_chunk[pool_id] = chunk;
> -	}
> -
> -	if (chunk->chunk.num_bufs == 0) {
> -		/* give the chunk buffer */
> -		local_chunk[pool_id] = NULL;
> -		chunk->buf_hdr.type = pool->s.buf_type;
> -
> -		handle = chunk->buf_hdr.handle;
> -	} else {
> -		odp_buffer_hdr_t *hdr;
> -		uint32_t index;
> -		index = rem_buf_index(chunk);
> -		hdr = index_to_hdr(pool, index);
> -
> -		handle = hdr->handle;
> -	}
> -
> -	return handle.u32;
> +	pool_entry_t *pool = get_pool_entry(pool_id);
> +	return (odp_buffer_t)ti_em_osal_hw_queue_pop(pool->s.free_queue,
> +			TI_EM_MEM_PUBLIC_DESC);
>   }
>   
<two empty lines>
>   
>   void odp_buffer_free(odp_buffer_t buf)
>   {
> -	odp_buffer_hdr_t *hdr;
> -	odp_buffer_pool_t pool_id;
> -	pool_entry_t *pool;
> -	odp_buffer_chunk_hdr_t *chunk_hdr;
> -
> -	hdr       = odp_buf_to_hdr(buf);
> -	pool_id   = hdr->pool;
> -	pool      = get_pool_entry(pool_id);
> -	chunk_hdr = local_chunk[pool_id];
> -
> -	if (chunk_hdr && chunk_hdr->chunk.num_bufs == ODP_BUFS_PER_CHUNK - 1) {
> -		/* Current chunk is full. Push back to the pool */
> -		LOCK(&pool->s.lock);
> -		add_chunk(pool, chunk_hdr);
> -		UNLOCK(&pool->s.lock);
> -		chunk_hdr = NULL;
> -	}
> -
> -	if (chunk_hdr == NULL) {
> -		/* Use this buffer */
> -		chunk_hdr = (odp_buffer_chunk_hdr_t *)hdr;
> -		local_chunk[pool_id] = chunk_hdr;
> -		chunk_hdr->chunk.num_bufs = 0;
> -	} else {
> -		/* Add to current chunk */
> -		add_buf_index(chunk_hdr, hdr->index);
> -	}
> +	odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf);
> +	ti_em_osal_hw_queue_push_size(hdr->free_queue,
> +				 (void*)hdr,
> +				 sizeof(Cppi_HostDesc),
> +				 TI_EM_MEM_PUBLIC_DESC);
>   }
>   
> -
>   void odp_buffer_pool_print(odp_buffer_pool_t pool_id)
>   {
> -	pool_entry_t *pool;
> -	odp_buffer_chunk_hdr_t *chunk_hdr;
> -	uint32_t i;
> -
> -	pool = get_pool_entry(pool_id);
> -
> -	printf("Pool info\n");
> -	printf("---------\n");
> -	printf("  pool          %i\n",           pool->s.pool);
> -	printf("  name          %s\n",           pool->s.name);
> -	printf("  pool base     %p\n",           pool->s.pool_base_addr);
> -	printf("  buf base      0x%"PRIxPTR"\n", pool->s.buf_base);
> -	printf("  pool size     0x%"PRIx64"\n",  pool->s.pool_size);
> -	printf("  buf size      %zu\n",          pool->s.payload_size);
> -	printf("  buf align     %zu\n",          pool->s.payload_align);
> -	printf("  hdr size      %zu\n",          pool->s.hdr_size);
> -	printf("  alloc size    %zu\n",          pool->s.buf_size);
> -	printf("  offset to hdr %zu\n",          pool->s.buf_offset);
> -	printf("  num bufs      %"PRIu64"\n",    pool->s.num_bufs);
> -	printf("  free bufs     %"PRIu64"\n",    pool->s.free_bufs);
> -
> -	/* first chunk */
> -	chunk_hdr = pool->s.head;
> -
> -	if (chunk_hdr == NULL) {
> -		ODP_ERR("  POOL EMPTY\n");
> -		return;
> -	}
> -
> -	printf("\n  First chunk\n");
> -
> -	for (i = 0; i < chunk_hdr->chunk.num_bufs - 1; i++) {
> -		uint32_t index;
> -		odp_buffer_hdr_t *hdr;
> -
> -		index = chunk_hdr->chunk.buf_index[i];
> -		hdr   = index_to_hdr(pool, index);
> -
> -		printf("  [%i] addr %p, id %"PRIu32"\n", i, hdr->addr, index);
> -	}
> -
> -	printf("  [%i] addr %p, id %"PRIu32"\n", i, chunk_hdr->buf_hdr.addr,
> -	       chunk_hdr->buf_hdr.index);
> -
> -	/* next chunk */
> -	chunk_hdr = next_chunk(pool, chunk_hdr);
> -
> -	if (chunk_hdr) {
> -		printf("  Next chunk\n");
> -		printf("  addr %p, id %"PRIu32"\n", chunk_hdr->buf_hdr.addr,
> -		       chunk_hdr->buf_hdr.index);
> -	}
> -
> -	printf("\n");
> +	(void)pool_id;
>   }
> diff --git a/platform/linux-keystone2/source/odp_init.c b/platform/linux-keystone2/source/odp_init.c
> index b466e67..0b36960 100644
> --- a/platform/linux-keystone2/source/odp_init.c
> +++ b/platform/linux-keystone2/source/odp_init.c
> @@ -12,6 +12,7 @@
>   #include <ti_em_osal_queue.h>
>   #include <ti_em_rh.h>
>   #include <odp_config.h>
> +#include <odp_buffer_internal.h>
>   
>   /*
>    * Make region_configs[] global, because hw_config is saved in
> @@ -47,7 +48,8 @@ static int ti_init_hw_config(void)
>   	/* Define descriptor regions */
>   	reg_config = &region_configs[TI_EM_RH_PUBLIC];
>   	reg_config->region_idx   = TI_ODP_PUBLIC_REGION_IDX;
> -	reg_config->desc_size    = TI_ODP_PUBLIC_DESC_SIZE;
> +	reg_config->desc_size    =
> +		ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_buffer_hdr_t));
>   	reg_config->desc_num     = TI_ODP_PUBLIC_DESC_NUM;
>   	reg_config->desc_base    = TI_ODP_PUBLIC_DESC_BASE;
>   	reg_config->desc_vbase   = TI_ODP_PUBLIC_DESC_VBASE;
> diff --git a/platform/linux-keystone2/source/odp_packet.c b/platform/linux-keystone2/source/odp_packet.c
> index eb7c227..f03d849 100644
> --- a/platform/linux-keystone2/source/odp_packet.c
> +++ b/platform/linux-keystone2/source/odp_packet.c
> @@ -331,38 +331,7 @@ void odp_packet_print(odp_packet_t pkt)
>   
>   int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src)
>   {
> -	odp_packet_hdr_t *const pkt_hdr_dst = odp_packet_hdr(pkt_dst);
> -	odp_packet_hdr_t *const pkt_hdr_src = odp_packet_hdr(pkt_src);
> -	const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, buf_hdr);
> -	uint8_t *start_src;
> -	uint8_t *start_dst;
> -	size_t len;
> -
> -	if (pkt_dst == ODP_PACKET_INVALID || pkt_src == ODP_PACKET_INVALID)
> -		return -1;
> -
> -	if (pkt_hdr_dst->buf_hdr.size <
> -		pkt_hdr_src->frame_len + pkt_hdr_src->frame_offset)
> -		return -1;
> -
> -	/* Copy packet header */
> -	start_dst = (uint8_t *)pkt_hdr_dst + start_offset;
> -	start_src = (uint8_t *)pkt_hdr_src + start_offset;
> -	len = ODP_OFFSETOF(odp_packet_hdr_t, payload) - start_offset;
> -	memcpy(start_dst, start_src, len);
> -
> -	/* Copy frame payload */
> -	start_dst = (uint8_t *)odp_packet_start(pkt_dst);
> -	start_src = (uint8_t *)odp_packet_start(pkt_src);
> -	len = pkt_hdr_src->frame_len;
> -	memcpy(start_dst, start_src, len);
> -
> -	/* Copy useful things from the buffer header */
> -	pkt_hdr_dst->buf_hdr.cur_offset = pkt_hdr_src->buf_hdr.cur_offset;
> -
> -	/* Create a copy of the scatter list */
> -	odp_buffer_copy_scatter(odp_buffer_from_packet(pkt_dst),
> -				odp_buffer_from_packet(pkt_src));
> -
> -	return 0;
> +	(void) pkt_dst;
> +	(void) pkt_src;
> +	return -1;
>   }
> diff --git a/platform/linux-keystone2/source/odp_queue.c b/platform/linux-keystone2/source/odp_queue.c
> index 49bc766..7c1fffe 100644
> --- a/platform/linux-keystone2/source/odp_queue.c
> +++ b/platform/linux-keystone2/source/odp_queue.c
> @@ -393,7 +393,7 @@ int odp_queue_deq_multi(odp_queue_t handle, odp_buffer_t buf[], int num)
>   	ret = queue->s.dequeue_multi(queue, buf_hdr, num);
>   
>   	for (i = 0; i < ret; i++)
> -		buf[i] = buf_hdr[i]->handle.handle;
> +		buf[i] = hdr_to_odp_buf(buf_hdr[i]);
>   
>   	return ret;
>   }
> @@ -408,7 +408,7 @@ odp_buffer_t odp_queue_deq(odp_queue_t handle)
>   	buf_hdr = queue->s.dequeue(queue);
>   
>   	if (buf_hdr)
> -		return buf_hdr->handle.handle;
> +		return hdr_to_odp_buf(buf_hdr);
>   
>   	return ODP_BUFFER_INVALID;
>   }