[API-NEXT,7/7] ipc: example app

Simple example app creates one packet i/o to external interface
and one ipc pktio to other process. Then transfer packet from
external interface to other process and back thought ipc queue.

Signed-off-by: Maxim Uvarov <maxim.uvarov@linaro.org>
---
 configure.ac            |   1 +
 example/Makefile.am     |   2 +-
 example/ipc/.gitignore  |   1 +
 example/ipc/Makefile.am |   7 +
 example/ipc/odp_ipc.c   | 441 ++++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 451 insertions(+), 1 deletion(-)
 create mode 100644 example/ipc/.gitignore
 create mode 100644 example/ipc/Makefile.am
 create mode 100644 example/ipc/odp_ipc.c

On Thu, Apr 23, 2015 at 08:47:05PM +0300, Maxim Uvarov wrote:
> Simple example app creates one packet i/o to external interface
> and one ipc pktio to other process. Then transfer packet from
> external interface to other process and back thought ipc queue.
> 
> Signed-off-by: Maxim Uvarov <maxim.uvarov@linaro.org>
> ---
>  configure.ac            |   1 +
>  example/Makefile.am     |   2 +-
>  example/ipc/.gitignore  |   1 +
>  example/ipc/Makefile.am |   7 +
>  example/ipc/odp_ipc.c   | 441 ++++++++++++++++++++++++++++++++++++++++++++++++
>  5 files changed, 451 insertions(+), 1 deletion(-)
>  create mode 100644 example/ipc/.gitignore
>  create mode 100644 example/ipc/Makefile.am
>  create mode 100644 example/ipc/odp_ipc.c
> 
> diff --git a/configure.ac b/configure.ac
> index d20bad2..1ceb922 100644
> --- a/configure.ac
> +++ b/configure.ac
> @@ -274,6 +274,7 @@ AC_CONFIG_FILES([Makefile
>  		 example/Makefile
>  		 example/classifier/Makefile
>  		 example/generator/Makefile
> +		 example/ipc/Makefile
>  		 example/ipsec/Makefile
>  		 example/packet/Makefile
>  		 example/timer/Makefile
> diff --git a/example/Makefile.am b/example/Makefile.am
> index 353f397..506963f 100644
> --- a/example/Makefile.am
> +++ b/example/Makefile.am
> @@ -1 +1 @@
> -SUBDIRS = classifier generator ipsec packet timer
> +SUBDIRS = classifier generator ipc ipsec packet timer
> diff --git a/example/ipc/.gitignore b/example/ipc/.gitignore
> new file mode 100644
> index 0000000..963d99d
> --- /dev/null
> +++ b/example/ipc/.gitignore
> @@ -0,0 +1 @@
> +odp_ipc
> diff --git a/example/ipc/Makefile.am b/example/ipc/Makefile.am
> new file mode 100644
> index 0000000..3da9549
> --- /dev/null
> +++ b/example/ipc/Makefile.am
> @@ -0,0 +1,7 @@
> +include $(top_srcdir)/example/Makefile.inc
> +
> +bin_PROGRAMS = odp_ipc
> +odp_ipc_LDFLAGS = $(AM_LDFLAGS) -static
> +odp_ipc_CFLAGS = $(AM_CFLAGS) -I${top_srcdir}/example
> +
> +dist_odp_ipc_SOURCES = odp_ipc.c
> diff --git a/example/ipc/odp_ipc.c b/example/ipc/odp_ipc.c
> new file mode 100644
> index 0000000..2d94816
> --- /dev/null
> +++ b/example/ipc/odp_ipc.c
> @@ -0,0 +1,441 @@
> +/* Copyright (c) 2015, Linaro Limited
> + * All rights reserved.
> + *
> + * SPDX-License-Identifier:     BSD-3-Clause
> + */
> +
> +/**
> + * @file
> + *
> + * @example odp_pktio.c  ODP basic packet IO loopback test application

^^ needs update

> + */
> +
> +#include <stdlib.h>
> +#include <string.h>
> +#include <getopt.h>
> +#include <unistd.h>
> +
> +#include <example_debug.h>
> +
> +#include <odp.h>
> +#include <odp/helper/linux.h>
> +#include <odp/helper/eth.h>
> +#include <odp/helper/ip.h>

Some of these headers aren't needed.

> +
> +/** @def SHM_PKT_POOL_SIZE
> + * @brief Size of the shared memory block
> + */
> +#define SHM_PKT_POOL_SIZE      (512*2048)
> +
> +/** @def SHM_PKT_POOL_BUF_SIZE
> + * @brief Buffer size of the packet pool buffer
> + */
> +#define SHM_PKT_POOL_BUF_SIZE  1856
> +
> +/** @def MAX_PKT_BURST
> + * @brief Maximum number of packet bursts
> + */
> +#define MAX_PKT_BURST          16
> +
> +/** Get rid of path in filename - only for unix-type paths using '/' */
> +#define NO_PATH(file_name) (strrchr((file_name), '/') ? \
> +			    strrchr((file_name), '/') + 1 : (file_name))
> +
> +/** Application argument */
> +static char *pktio_name;
> +
> +/* helper funcs */
> +static int drop_err_pkts(odp_packet_t pkt_tbl[], unsigned len);
> +static void parse_args(int argc, char *argv[]);
> +static void print_info(char *progname);
> +static void usage(char *progname);
> +
> +/**
> + * Create a pktio handle, optionally associating a default input queue.

It's not doing anything with the input queue.

> + *
> + * @param dev Name of device to open
> + * @param pool Pool to associate with device for packet RX/TX
> + *
> + * @return The handle of the created pktio object.
> + * @retval ODP_PKTIO_INVALID if the create fails.
> + */
> +static odp_pktio_t create_pktio(const char *dev, odp_pool_t pool)
> +{
> +	odp_pktio_t pktio;
> +	char inq_name[ODP_QUEUE_NAME_LEN];
> +	odp_pktio_t ipc_pktio;
> +
> +	/* Open a packet IO instance */
> +	pktio = odp_pktio_open(dev, pool);
> +	if (pktio == ODP_PKTIO_INVALID)
> +		EXAMPLE_ABORT("Error: pktio create failed for %s\n", dev);
> +
> +	snprintf(inq_name, sizeof(inq_name), "%" PRIu64 "-pktio_inq_def",
> +		 odp_pktio_to_u64(pktio));
> +	inq_name[ODP_QUEUE_NAME_LEN - 1] = '\0';

The inq_name isn't used so can be removed.

> +
> +	printf("pid: %d, create IPC pktio\n", getpid());
> +	ipc_pktio = odp_pktio_open("ipc_pktio", pool);
> +	if (ipc_pktio == ODP_PKTIO_INVALID)
> +		EXAMPLE_ABORT("Error: ipc pktio create failed.\n");
> +
> +	return pktio;
> +}
> +
> +/**
> + * Packet IO loopback worker thread using bursts from/to IO resources
> + *
> + * @param arg  thread arguments of type 'thread_args_t *'
> + */
> +static void *pktio_run_loop(void *arg ODP_UNUSED)
> +{
> +	int thr;
> +	odp_pktio_t pktio;
> +	int pkts, pkts_ok;
> +	odp_packet_t pkt_tbl[MAX_PKT_BURST];
> +	odp_pktio_t ipc_pktio;
> +
> +	thr = odp_thread_id();
> +
> +	pktio = odp_pktio_lookup(pktio_name);
> +	if (pktio == ODP_PKTIO_INVALID) {
> +		EXAMPLE_ERR("  [%02i] Error: lookup of pktio %s failed\n",
> +			    thr, pktio_name);
> +		return NULL;
> +	}
> +
> +	printf("  [%02i] looked up pktio:%02" PRIu64 ", burst mode\n",
> +	       thr, odp_pktio_to_u64(pktio));
> +
> +	ipc_pktio = odp_pktio_lookup("ipc_pktio");
> +	if (pktio == ODP_PKTIO_INVALID) {
> +		EXAMPLE_ERR("  [%02i] Error: lookup of pktio %s failed\n",
> +			    thr, "ipc_pktio");
> +		return NULL;
> +	}
> +	printf("  [%02i] looked up ipc_pktio:%02" PRIu64 ", burst mode\n",
> +	       thr, odp_pktio_to_u64(pktio));
> +
> +	/* Loop packets */
> +	for (;;) {
> +		int i;
> +
> +		pkts = odp_pktio_recv(pktio, pkt_tbl, MAX_PKT_BURST);
> +		if (pkts > 0) {
> +			/* Drop packets with errors */
> +			pkts_ok = drop_err_pkts(pkt_tbl, pkts);
> +			if (pkts_ok > 0) {
> +				time_t tm = time(NULL);
> +				char *tm_str = ctime(&tm)
> +;
> +				for (i = 0; i < pkts_ok; i++) {
> +					odp_packet_copydata_in(pkt_tbl[i],
> +							       0,
> +							       strlen(tm_str),
> +							       tm_str);
> +					odp_packet_copydata_in(pkt_tbl[i],
> +							       strlen(tm_str),
> +							       1,
> +							       "\0");
> +				}

I don't get it, you're receiving packets from the network then
overwriting the first few bytes with a timestamp string and forwarding
it over the ipc interface. Isn't this corrupting the packet?

> +
> +				odp_pktio_send(ipc_pktio, pkt_tbl, pkts_ok);

Need to check return value here otherwise you risk overwriting unsent
packet handles.

> +
> +				printf("---main pid %d: ipcsend %d pkts, size %d, data: %s\n",
> +				       getpid(), pkts_ok,
> +				       odp_packet_len(pkt_tbl[0]),
> +				       tm_str);
> +			}
> +		}
> +
> +		pkts = odp_pktio_recv(ipc_pktio, pkt_tbl, MAX_PKT_BURST);
> +		if (pkts > 0) {
> +			for (i = 0; i < pkts; i++) {
> +				char *b = malloc(odp_packet_len(pkt_tbl[i]));
> +				odp_packet_copydata_out(pkt_tbl[i], 0,
> +							odp_packet_len(pkt_tbl[i]),
> +							b);
> +				printf("---main pid %d: ipcsrecv: size %d, data: %s\n",
> +				       getpid(), odp_packet_len(pkt_tbl[i]),
> +				       b);
> +				free(b);
> +				odp_packet_free(pkt_tbl[i]);
> +			}
> +		}
> +	}
> +
> +/* unreachable */
> +	return NULL;
> +}
> +
> +static int ipc_second_process(void)
> +{
> +	odp_pktio_t pktio;
> +	odp_packet_t pkt_tbl[MAX_PKT_BURST];
> +	int i;
> +	int pkts;
> +
> +	/* linux shared memory can already have objects with names which
> +	 * second process can try to connect. That might be even interrupted
> +	 * current application. Might be later I will add magic numbers to
> +	 * each ipc object in linux-generic. HW platfrom shound not have that
> +	 * problem. So just wait a little while master process will create
> +	 * all ipc objects before connectioning to them.
> +	 */
> +	sleep(3);

Why is this sleep needed if you have the delay loop below?

> +
> +	/* Do lookup packet I/O in IPC shared memory,
> +	 * and link it to local pool. */

There is no local pool.

> +	while (1) {
> +		pktio = odp_pktio_open("ipc_pktio", NULL);

Should be ODP_POOL_INVALID rather than NULL

> +		if (pktio == ODP_PKTIO_INVALID) {
> +			sleep(1);
> +			printf("%s() pid %d: looking for ipc_pktio\n",
> +			       __func__, getpid());
> +			continue;
> +		}
> +		break;
> +	}

What happens if an error occurs in the other process before it's managed
to open ipc_pktio?

> +
> +	for (;;) {
> +		pkts = odp_pktio_recv(pktio, pkt_tbl, MAX_PKT_BURST);
> +		if (pkts > 0) {
> +			for (i = 0; i < pkts; i++) {

Have packets received via the ipc_pktio been parsed?.. i.e. should they
be checked for errors?

> +				char *b = malloc(odp_packet_len(pkt_tbl[i]));
> +
> +				odp_packet_copydata_out(pkt_tbl[i], 0,
> +							odp_packet_len(pkt_tbl[i]),
> +							b);

There's no need to copy the data out like this, can't you just use
odp_packet_data()?

> +
> +				printf("++++%s: pid %d, got packet %p, size %d, data: %s\n",
> +				       __func__, getpid(),
> +				       (void *)pkt_tbl[i],
> +				       odp_packet_len(pkt_tbl[i]), b);
> +				free(b);
> +
> +				odp_pktio_send(pktio, pkt_tbl, pkts);
> +			}
> +		} else {
> +			/* No need to load cpu in example app.*/
> +			sleep(1);
> +		}
> +	}
> +
> +	EXAMPLE_ERR("Unexpected close.");
> +	return 0;
> +}
> +
> +/**
> + * ODP packet example main function
> + */
> +int main(int argc, char *argv[])
> +{
> +	odp_pool_t pool;
> +	odp_pool_param_t params;
> +	int f;
> +
> +	/* Parse and store the application arguments */
> +	parse_args(argc, argv);
> +
> +	/* Use fork() before odp_init_global() to have 2 isolated
> +	 * processes which need communicate to each other with
> +	 * shared memory.
> +	 */
> +	f = fork();
> +	if (f) {
> +		printf("Process one pid: %d\n", getpid());
> +		/* Init ODP before calling anything else */
> +		if (odp_init_global(NULL, NULL)) {
> +			EXAMPLE_ERR("Error: ODP global init failed.\n");
> +			exit(EXIT_FAILURE);
> +		}
> +
> +		/* Init this thread */
> +		if (odp_init_local()) {
> +			EXAMPLE_ERR("Error: ODP local init failed.\n");
> +			exit(EXIT_FAILURE);
> +		}
> +
> +		ipc_second_process();
> +	} else {
> +		printf("Process two pid: %d\n", getpid());
> +	}
> +
> +
> +	/* Init ODP before calling anything else */
> +	if (odp_init_global(NULL, NULL)) {
> +		EXAMPLE_ERR("Error: ODP global init failed.\n");
> +		exit(EXIT_FAILURE);
> +	}
> +
> +	/* Init this thread */
> +	if (odp_init_local()) {
> +		EXAMPLE_ERR("Error: ODP local init failed.\n");
> +		exit(EXIT_FAILURE);
> +	}
> +
> +	/* Print both system and application information */
> +	print_info(NO_PATH(argv[0]));
> +
> +	/* Create packet pool */
> +	memset(&params, 0, sizeof(params));
> +	params.pkt.seg_len = SHM_PKT_POOL_BUF_SIZE;
> +	params.pkt.len     = SHM_PKT_POOL_BUF_SIZE;
> +	params.pkt.num     = SHM_PKT_POOL_SIZE/SHM_PKT_POOL_BUF_SIZE;
> +	params.type        = ODP_POOL_PACKET;
> +	params.shm_flags   = ODP_SHM_PROC;
> +
> +	pool = odp_pool_create("packet_pool", ODP_SHM_NULL, &params);
> +	if (pool == ODP_POOL_INVALID) {
> +		EXAMPLE_ERR("Error: packet pool create failed.\n");
> +		exit(EXIT_FAILURE);
> +	}
> +
> +	odp_pool_print(pool);
> +
> +	create_pktio(pktio_name, pool);
> +
> +	pktio_run_loop(NULL);
> +
> +	return 0;
> +}
> +
> +/**
> + * Drop packets which input parsing marked as containing errors.
> + *
> + * Frees packets with error and modifies pkt_tbl[] to only contain packets with
> + * no detected errors.
> + *
> + * @param pkt_tbl  Array of packet
> + * @param len      Length of pkt_tbl[]
> + *
> + * @return Number of packets with no detected error
> + */
> +static int drop_err_pkts(odp_packet_t pkt_tbl[], unsigned len)
> +{
> +	odp_packet_t pkt;
> +	unsigned pkt_cnt = len;
> +	unsigned i, j;
> +
> +	for (i = 0, j = 0; i < len; ++i) {
> +		pkt = pkt_tbl[i];
> +
> +		if (odp_unlikely(odp_packet_has_error(pkt))) {
> +			odp_packet_free(pkt); /* Drop */
> +			pkt_cnt--;
> +		} else if (odp_unlikely(i != j++)) {
> +			pkt_tbl[j-1] = pkt;
> +		}
> +	}
> +
> +	return pkt_cnt;
> +}
> +
> +/**
> + * Parse and store the command line arguments
> + *
> + * @param argc       argument count
> + * @param argv[]     argument vector
> + * @param appl_args  Store application arguments here
> + */
> +static void parse_args(int argc, char *argv[])
> +{
> +	int opt;
> +	int long_index;
> +	size_t len;
> +	static struct option longopts[] = {
> +		{"interface", required_argument, NULL, 'i'},	/* return 'i' */
> +		{"help", no_argument, NULL, 'h'},		/* return 'h' */
> +		{NULL, 0, NULL, 0}
> +	};
> +
> +	while (1) {
> +		opt = getopt_long(argc, argv, "i:h",
> +				  longopts, &long_index);
> +
> +		if (opt == -1)
> +			break;	/* No more options */
> +
> +		switch (opt) {
> +		case 'i':
> +			len = strlen(optarg);
> +			if (len == 0) {
> +				usage(argv[0]);
> +				exit(EXIT_FAILURE);
> +			}
> +			len += 1;	/* add room for '\0' */
> +
> +			pktio_name = malloc(len);
> +			if (pktio_name == NULL) {
> +				usage(argv[0]);
> +				exit(EXIT_FAILURE);
> +			}
> +			strcpy(pktio_name, optarg);
> +
> +			break;
> +		case 'h':
> +			usage(argv[0]);
> +			exit(EXIT_SUCCESS);
> +			break;
> +
> +		default:
> +			break;
> +		}
> +	}
> +
> +	if (pktio_name == NULL) {
> +		usage(argv[0]);
> +		exit(EXIT_FAILURE);
> +	}
> +
> +	optind = 1;		/* reset 'extern optind' from the getopt lib */
> +}
> +
> +/**
> + * Print system and application info
> + */
> +static void print_info(char *progname)
> +{
> +	printf("\n"
> +	       "ODP system info\n"
> +	       "---------------\n"
> +	       "ODP API version: %s\n"
> +	       "CPU model:       %s\n"
> +	       "CPU freq (hz):   %"PRIu64"\n"
> +	       "Cache line size: %i\n"
> +	       "CPU count:       %i\n"
> +	       "\n",
> +	       odp_version_api_str(), odp_sys_cpu_model_str(), odp_sys_cpu_hz(),
> +	       odp_sys_cache_line_size(), odp_cpu_count());
> +
> +	printf("Running ODP appl: \"%s\"\n"
> +	       "-----------------\n"
> +	       "Using IF:        %s\n",
> +	       progname, pktio_name);
> +	printf("\n\n");
> +	fflush(NULL);
> +}
> +
> +/**
> + * Prinf usage information
> + */
> +static void usage(char *progname)
> +{
> +	printf("\n"
> +	       "Usage: %s OPTIONS\n"
> +	       "  E.g. %s -i eth0\n"
> +	       "\n"
> +	       "OpenDataPlane example application.\n"
> +	       "\n"
> +	       "Mandatory OPTIONS:\n"
> +	       "  -i, --interface Eth interface\n"
> +	       "\n"
> +	       "Optional OPTIONS\n"
> +	       "  -h, --help           Display help and exit.\n"
> +	       " environment variables: ODP_PKTIO_DISABLE_SOCKET_MMAP\n"
> +	       "                        ODP_PKTIO_DISABLE_SOCKET_MMSG\n"
> +	       "                        ODP_PKTIO_DISABLE_SOCKET_BASIC\n"
> +	       " can be used to advanced pkt I/O selection for linux-generic\n"
> +	       "\n", NO_PATH(progname), NO_PATH(progname)
> +	    );
> +}
> -- 
> 1.9.1
>

On 04/27/15 15:50, Stuart Haslam wrote:
> I don't get it, you're receiving packets from the network then
> overwriting the first few bytes with a timestamp string and forwarding
> it over the ipc interface. Isn't this corrupting the packet?
Yes, that corrupts original packet but it was most visible test to me to 
debug it. We know that
receive from standard pktio works well. So for test I did not need 
really this packet. I send ping
each second to generate some packet flow. Then add human readable time 
stamp to packet
and send it to ipc pktio. Then receive packets from ipc pktio and print 
that time stamps. So that
I can see for each second that I send packet to other application and 
receive it back. That might be
not the best scenario for demo included to odp.git bit it's useful for 
me. Do you have some better
example?

Thanks,
Maxim.

On Mon, Apr 27, 2015 at 04:45:44PM +0300, Maxim Uvarov wrote:
> On 04/27/15 15:50, Stuart Haslam wrote:
> >I don't get it, you're receiving packets from the network then
> >overwriting the first few bytes with a timestamp string and forwarding
> >it over the ipc interface. Isn't this corrupting the packet?
> Yes, that corrupts original packet but it was most visible test to
> me to debug it. We know that
> receive from standard pktio works well. So for test I did not need
> really this packet. I send ping
> each second to generate some packet flow. Then add human readable
> time stamp to packet
> and send it to ipc pktio. Then receive packets from ipc pktio and
> print that time stamps. So that
> I can see for each second that I send packet to other application
> and receive it back. That might be
> not the best scenario for demo included to odp.git bit it's useful
> for me. Do you have some better
> example?
> 
> Thanks,
> Maxim.

Just something simple, process A sends message to process B, process B
prints it, and vice verse. Pretty much what you have except packets are
generated rather than being sourced from a non-IPC pktio.