| Message ID | 1425649536-6580-1-git-send-email-maxim.uvarov@linaro.org |
|---|---|
| State | New |
| Headers | show |
Like the idea. However I think we need to move forward making the examples simpler so that they show one concept very clearly and not be mini apps, so I am not in favor of adding code to them. The worst offender we have for an example that is too complex is the ipsec example, that is a full blown app and not amenable to being referenced from the Doxygen at all. I feel this change should be made and that we also migrate l2fwd into test/performance where users can still see it, but we then leave odp/examples open to have examples per API rather than per mini app like l2fwd. In addition then the ODP API doc, the validation tests, the bug tracker AND the examples all follow the pattern of per API module content making it easier to navigate. On 6 March 2015 at 08:45, Maxim Uvarov <maxim.uvarov@linaro.org> wrote: > Current print in l2fwd is not useful with slow links and > also it's hard to say how fast does it work. Print pps > and packets drops. > > Signed-off-by: Maxim Uvarov <maxim.uvarov@linaro.org> > --- > example/l2fwd/odp_l2fwd.c | 67 > ++++++++++++++++++++++++++++++----------------- > 1 file changed, 43 insertions(+), 24 deletions(-) > > diff --git a/example/l2fwd/odp_l2fwd.c b/example/l2fwd/odp_l2fwd.c > index d062a72..71c2fc5 100644 > --- a/example/l2fwd/odp_l2fwd.c > +++ b/example/l2fwd/odp_l2fwd.c > @@ -104,6 +104,14 @@ static void parse_args(int argc, char *argv[], > appl_args_t *appl_args); > static void print_info(char *progname, appl_args_t *appl_args); > static void usage(char *progname); > > +/* speed and stats */ > +typedef struct { > + uint64_t packets[MAX_WORKERS]; > + uint64_t drops[MAX_WORKERS]; > +} stats_t; > + > +static stats_t stats; > + > /** > * Packet IO worker thread using ODP queues > * > @@ -115,8 +123,6 @@ static void *pktio_queue_thread(void *arg) > odp_queue_t outq_def; > odp_packet_t pkt; > odp_event_t ev; > - unsigned long pkt_cnt = 0; > - unsigned long err_cnt = 0; > > (void)arg; > > @@ -132,7 +138,7 @@ static void *pktio_queue_thread(void *arg) > > /* Drop packets with errors */ > if (odp_unlikely(drop_err_pkts(&pkt, 1) == 0)) { > - EXAMPLE_ERR("Drop frame - err_cnt:%lu\n", > ++err_cnt); > + stats.drops[thr] += 1; > continue; > } > > @@ -141,11 +147,7 @@ static void *pktio_queue_thread(void *arg) > /* Enqueue the packet for output */ > odp_queue_enq(outq_def, ev); > > - /* Print packet counts every once in a while */ > - if (odp_unlikely(pkt_cnt++ % 100000 == 0)) { > - printf(" [%02i] pkt_cnt:%lu\n", thr, pkt_cnt); > - fflush(NULL); > - } > + stats.packets[thr] += 1; > } > > /* unreachable */ > @@ -186,9 +188,6 @@ static void *pktio_ifburst_thread(void *arg) > thread_args_t *thr_args; > int pkts, pkts_ok; > odp_packet_t pkt_tbl[MAX_PKT_BURST]; > - unsigned long pkt_cnt = 0; > - unsigned long err_cnt = 0; > - unsigned long tmp = 0; > int src_idx, dst_idx; > odp_pktio_t pktio_src, pktio_dst; > > @@ -218,23 +217,13 @@ static void *pktio_ifburst_thread(void *arg) > if (pkts_ok > 0) > odp_pktio_send(pktio_dst, pkt_tbl, pkts_ok); > > - if (odp_unlikely(pkts_ok != pkts)) { > - err_cnt += pkts-pkts_ok; > - EXAMPLE_ERR("Dropped frames:%u - err_cnt:%lu\n", > - pkts-pkts_ok, err_cnt); > - } > + if (odp_unlikely(pkts_ok != pkts)) > + stats.drops[thr] += pkts - pkts_ok; > > if (pkts_ok == 0) > continue; > > - /* Print packet counts every once in a while */ > - tmp += pkts_ok; > - if (odp_unlikely(tmp >= 100000 || pkt_cnt == 0)) { > - pkt_cnt += tmp; > - tmp = 0; > - printf(" [%02i] pkt_cnt:%lu\n", thr, pkt_cnt); > - fflush(NULL); > - } > + stats.packets[thr] += pkts_ok; > } > > /* unreachable */ > @@ -295,6 +284,34 @@ static odp_pktio_t create_pktio(const char *dev, > odp_pool_t pool, > return pktio; > } > > +static void print_speed_stats(int num_workers) > +{ > + stats_t old_stats; > + stats_t new_stats; > + uint64_t old; > + uint64_t new; > + int i; > + int timeout = 10; > + > + while (1) { > + memcpy(&old_stats, &stats, sizeof(stats_t)); > + sleep(timeout); > + memcpy(&new_stats, &stats, sizeof(stats_t)); > + > + for (i = old = new = 0; i < num_workers; i++) { > + old += old_stats.packets[i]; > + new += new_stats.packets[i]; > + } > + printf(" %" PRIu64 " pps, ", (new - old) / timeout); > + > + for (i = old = new = 0; i < num_workers; i++) { > + old += old_stats.drops[i]; > + new += new_stats.drops[i]; > + } > + printf(" %" PRIu64 " drops\n", (new - old) / timeout); > + } > +} > + > /** > * ODP L2 forwarding main function > */ > @@ -412,6 +429,8 @@ int main(int argc, char *argv[]) > cpu = odp_cpumask_next(&cpumask, cpu); > } > > + print_speed_stats(num_workers); > + > /* Master thread waits for other threads to exit */ > odph_linux_pthread_join(thread_tbl, num_workers); > > -- > 1.9.1 > > > _______________________________________________ > lng-odp mailing list > lng-odp@lists.linaro.org > http://lists.linaro.org/mailman/listinfo/lng-odp >
On 03/06/15 16:55, Mike Holmes wrote: > Like the idea. > However I think we need to move forward making the examples simpler so > that they show one concept very clearly and not be mini apps, so I am > not in favor of adding code to them. > The worst offender we have for an example that is too complex is the > ipsec example, that is a full blown app and not amenable to being > referenced from the Doxygen at all. > > I feel this change should be made and that we also migrate l2fwdinto > test/performance where users can still see it, but we then leave > odp/examples open to have examples per API rather than per mini app > like l2fwd. In addition then the ODP API doc, the validation tests, > the bug tracker AND the examples all follow the pattern of per API > module content making it easier to navigate. yes, I agree l2fwd is more likely ./test/performance then ./example. For packet i/o good example is ./example/packet. But I also don't like print there and I have to hack it each time. Maxim. > > On 6 March 2015 at 08:45, Maxim Uvarov <maxim.uvarov@linaro.org > <mailto:maxim.uvarov@linaro.org>> wrote: > > Current print in l2fwd is not useful with slow links and > also it's hard to say how fast does it work. Print pps > and packets drops. > > Signed-off-by: Maxim Uvarov <maxim.uvarov@linaro.org > <mailto:maxim.uvarov@linaro.org>> > --- > example/l2fwd/odp_l2fwd.c | 67 > ++++++++++++++++++++++++++++++----------------- > 1 file changed, 43 insertions(+), 24 deletions(-) > > diff --git a/example/l2fwd/odp_l2fwd.c b/example/l2fwd/odp_l2fwd.c > index d062a72..71c2fc5 100644 > --- a/example/l2fwd/odp_l2fwd.c > +++ b/example/l2fwd/odp_l2fwd.c > @@ -104,6 +104,14 @@ static void parse_args(int argc, char > *argv[], appl_args_t *appl_args); > static void print_info(char *progname, appl_args_t *appl_args); > static void usage(char *progname); > > +/* speed and stats */ > +typedef struct { > + uint64_t packets[MAX_WORKERS]; > + uint64_t drops[MAX_WORKERS]; > +} stats_t; > + > +static stats_t stats; > + > /** > * Packet IO worker thread using ODP queues > * > @@ -115,8 +123,6 @@ static void *pktio_queue_thread(void *arg) > odp_queue_t outq_def; > odp_packet_t pkt; > odp_event_t ev; > - unsigned long pkt_cnt = 0; > - unsigned long err_cnt = 0; > > (void)arg; > > @@ -132,7 +138,7 @@ static void *pktio_queue_thread(void *arg) > > /* Drop packets with errors */ > if (odp_unlikely(drop_err_pkts(&pkt, 1) == 0)) { > - EXAMPLE_ERR("Drop frame - err_cnt:%lu\n", > ++err_cnt); > + stats.drops[thr] += 1; > continue; > } > > @@ -141,11 +147,7 @@ static void *pktio_queue_thread(void *arg) > /* Enqueue the packet for output */ > odp_queue_enq(outq_def, ev); > > - /* Print packet counts every once in a while */ > - if (odp_unlikely(pkt_cnt++ % 100000 == 0)) { > - printf(" [%02i] pkt_cnt:%lu\n", thr, > pkt_cnt); > - fflush(NULL); > - } > + stats.packets[thr] += 1; > } > > /* unreachable */ > @@ -186,9 +188,6 @@ static void *pktio_ifburst_thread(void *arg) > thread_args_t *thr_args; > int pkts, pkts_ok; > odp_packet_t pkt_tbl[MAX_PKT_BURST]; > - unsigned long pkt_cnt = 0; > - unsigned long err_cnt = 0; > - unsigned long tmp = 0; > int src_idx, dst_idx; > odp_pktio_t pktio_src, pktio_dst; > > @@ -218,23 +217,13 @@ static void *pktio_ifburst_thread(void *arg) > if (pkts_ok > 0) > odp_pktio_send(pktio_dst, pkt_tbl, pkts_ok); > > - if (odp_unlikely(pkts_ok != pkts)) { > - err_cnt += pkts-pkts_ok; > - EXAMPLE_ERR("Dropped frames:%u - > err_cnt:%lu\n", > - pkts-pkts_ok, err_cnt); > - } > + if (odp_unlikely(pkts_ok != pkts)) > + stats.drops[thr] += pkts - pkts_ok; > > if (pkts_ok == 0) > continue; > > - /* Print packet counts every once in a while */ > - tmp += pkts_ok; > - if (odp_unlikely(tmp >= 100000 || pkt_cnt == 0)) { > - pkt_cnt += tmp; > - tmp = 0; > - printf(" [%02i] pkt_cnt:%lu\n", thr, > pkt_cnt); > - fflush(NULL); > - } > + stats.packets[thr] += pkts_ok; > } > > /* unreachable */ > @@ -295,6 +284,34 @@ static odp_pktio_t create_pktio(const char > *dev, odp_pool_t pool, > return pktio; > } > > +static void print_speed_stats(int num_workers) > +{ > + stats_t old_stats; > + stats_t new_stats; > + uint64_t old; > + uint64_t new; > + int i; > + int timeout = 10; > + > + while (1) { > + memcpy(&old_stats, &stats, sizeof(stats_t)); > + sleep(timeout); > + memcpy(&new_stats, &stats, sizeof(stats_t)); > + > + for (i = old = new = 0; i < num_workers; i++) { > + old += old_stats.packets[i]; > + new += new_stats.packets[i]; > + } > + printf(" %" PRIu64 " pps, ", (new - old) / timeout); > + > + for (i = old = new = 0; i < num_workers; i++) { > + old += old_stats.drops[i]; > + new += new_stats.drops[i]; > + } > + printf(" %" PRIu64 " drops\n", (new - old) / timeout); > + } > +} > + > /** > * ODP L2 forwarding main function > */ > @@ -412,6 +429,8 @@ int main(int argc, char *argv[]) > cpu = odp_cpumask_next(&cpumask, cpu); > } > > + print_speed_stats(num_workers); > + > /* Master thread waits for other threads to exit */ > odph_linux_pthread_join(thread_tbl, num_workers); > > -- > 1.9.1 > > > _______________________________________________ > lng-odp mailing list > lng-odp@lists.linaro.org <mailto:lng-odp@lists.linaro.org> > http://lists.linaro.org/mailman/listinfo/lng-odp > > > > > -- > Mike Holmes > Technical Manager - Linaro Networking Group > Linaro.org <http://www.linaro.org/>***│ *Open source software for ARM SoCs >
On Fri, Mar 6, 2015 at 4:07 PM, Maxim Uvarov <maxim.uvarov@linaro.org> wrote: > On 03/06/15 16:55, Mike Holmes wrote: >> >> Like the idea. >> However I think we need to move forward making the examples simpler so >> that they show one concept very clearly and not be mini apps, so I am not in >> favor of adding code to them. >> The worst offender we have for an example that is too complex is the ipsec >> example, that is a full blown app and not amenable to being referenced from >> the Doxygen at all. I like this too. See some comments below. >> >> I feel this change should be made and that we also migrate l2fwdinto >> test/performance where users can still see it, but we then leave >> odp/examples open to have examples per API rather than per mini app like >> l2fwd. In addition then the ODP API doc, the validation tests, the bug >> tracker AND the examples all follow the pattern of per API module content >> making it easier to navigate. +1 > > > yes, I agree l2fwd is more likely ./test/performance then ./example. For > packet i/o good example is ./example/packet. But I also don't like print > there and I have to hack it each time. > > Maxim. > >> >> On 6 March 2015 at 08:45, Maxim Uvarov <maxim.uvarov@linaro.org >> <mailto:maxim.uvarov@linaro.org>> wrote: >> >> Current print in l2fwd is not useful with slow links and >> also it's hard to say how fast does it work. Print pps >> and packets drops. >> >> Signed-off-by: Maxim Uvarov <maxim.uvarov@linaro.org >> <mailto:maxim.uvarov@linaro.org>> >> >> --- >> example/l2fwd/odp_l2fwd.c | 67 >> ++++++++++++++++++++++++++++++----------------- >> 1 file changed, 43 insertions(+), 24 deletions(-) >> >> diff --git a/example/l2fwd/odp_l2fwd.c b/example/l2fwd/odp_l2fwd.c >> index d062a72..71c2fc5 100644 >> --- a/example/l2fwd/odp_l2fwd.c >> +++ b/example/l2fwd/odp_l2fwd.c >> @@ -104,6 +104,14 @@ static void parse_args(int argc, char >> *argv[], appl_args_t *appl_args); >> static void print_info(char *progname, appl_args_t *appl_args); >> static void usage(char *progname); >> >> +/* speed and stats */ >> +typedef struct { >> + uint64_t packets[MAX_WORKERS]; >> + uint64_t drops[MAX_WORKERS]; >> +} stats_t; >> + >> +static stats_t stats; >> + >> /** >> * Packet IO worker thread using ODP queues >> * >> @@ -115,8 +123,6 @@ static void *pktio_queue_thread(void *arg) >> odp_queue_t outq_def; >> odp_packet_t pkt; >> odp_event_t ev; >> - unsigned long pkt_cnt = 0; >> - unsigned long err_cnt = 0; >> >> (void)arg; >> >> @@ -132,7 +138,7 @@ static void *pktio_queue_thread(void *arg) >> >> /* Drop packets with errors */ >> if (odp_unlikely(drop_err_pkts(&pkt, 1) == 0)) { >> - EXAMPLE_ERR("Drop frame - err_cnt:%lu\n", >> ++err_cnt); >> + stats.drops[thr] += 1; >> continue; >> } >> >> @@ -141,11 +147,7 @@ static void *pktio_queue_thread(void *arg) >> /* Enqueue the packet for output */ >> odp_queue_enq(outq_def, ev); >> >> - /* Print packet counts every once in a while */ >> - if (odp_unlikely(pkt_cnt++ % 100000 == 0)) { >> - printf(" [%02i] pkt_cnt:%lu\n", thr, >> pkt_cnt); >> - fflush(NULL); >> - } >> + stats.packets[thr] += 1; >> } >> >> /* unreachable */ >> @@ -186,9 +188,6 @@ static void *pktio_ifburst_thread(void *arg) >> thread_args_t *thr_args; >> int pkts, pkts_ok; >> odp_packet_t pkt_tbl[MAX_PKT_BURST]; >> - unsigned long pkt_cnt = 0; >> - unsigned long err_cnt = 0; >> - unsigned long tmp = 0; >> int src_idx, dst_idx; >> odp_pktio_t pktio_src, pktio_dst; >> >> @@ -218,23 +217,13 @@ static void *pktio_ifburst_thread(void *arg) >> if (pkts_ok > 0) >> odp_pktio_send(pktio_dst, pkt_tbl, pkts_ok); >> >> - if (odp_unlikely(pkts_ok != pkts)) { >> - err_cnt += pkts-pkts_ok; >> - EXAMPLE_ERR("Dropped frames:%u - >> err_cnt:%lu\n", >> - pkts-pkts_ok, err_cnt); >> - } >> + if (odp_unlikely(pkts_ok != pkts)) >> + stats.drops[thr] += pkts - pkts_ok; >> >> if (pkts_ok == 0) >> continue; >> >> - /* Print packet counts every once in a while */ >> - tmp += pkts_ok; >> - if (odp_unlikely(tmp >= 100000 || pkt_cnt == 0)) { >> - pkt_cnt += tmp; >> - tmp = 0; >> - printf(" [%02i] pkt_cnt:%lu\n", thr, >> pkt_cnt); >> - fflush(NULL); >> - } >> + stats.packets[thr] += pkts_ok; >> } >> >> /* unreachable */ >> @@ -295,6 +284,34 @@ static odp_pktio_t create_pktio(const char >> *dev, odp_pool_t pool, >> return pktio; >> } >> >> +static void print_speed_stats(int num_workers) >> +{ >> + stats_t old_stats; >> + stats_t new_stats; >> + uint64_t old; >> + uint64_t new; >> + int i; >> + int timeout = 10; >> + >> + while (1) { >> + memcpy(&old_stats, &stats, sizeof(stats_t)); >> + sleep(timeout); Could we use ODP timers instead of sleep? And possibly print stats every second, like netmap pkt-gen does, maybe make the interval configurable? >> + memcpy(&new_stats, &stats, sizeof(stats_t)); >> + >> + for (i = old = new = 0; i < num_workers; i++) { >> + old += old_stats.packets[i]; >> + new += new_stats.packets[i]; >> + } >> + printf(" %" PRIu64 " pps, ", (new - old) / timeout); >> + >> + for (i = old = new = 0; i < num_workers; i++) { >> + old += old_stats.drops[i]; >> + new += new_stats.drops[i]; >> + } >> + printf(" %" PRIu64 " drops\n", (new - old) / timeout); >> + } >> +} >> + >> /** >> * ODP L2 forwarding main function >> */ >> @@ -412,6 +429,8 @@ int main(int argc, char *argv[]) >> cpu = odp_cpumask_next(&cpumask, cpu); >> } >> >> + print_speed_stats(num_workers); >> + >> /* Master thread waits for other threads to exit */ >> odph_linux_pthread_join(thread_tbl, num_workers); >> >> -- >> 1.9.1 >> >> >> _______________________________________________ >> lng-odp mailing list >> lng-odp@lists.linaro.org <mailto:lng-odp@lists.linaro.org> >> http://lists.linaro.org/mailman/listinfo/lng-odp >> >> >> >> >> -- >> Mike Holmes >> Technical Manager - Linaro Networking Group >> Linaro.org <http://www.linaro.org/>***│ *Open source software for ARM SoCs >> > > > _______________________________________________ > lng-odp mailing list > lng-odp@lists.linaro.org > http://lists.linaro.org/mailman/listinfo/lng-odp
Hi Ciprian, I have update of that patch, added there time to run odp_fwd, maximum pps and added it to odp_pktio_run script. So that make check can provide some numbers and we can find performance regressions. Even if linux-generic is not performance target regressions might be useful information. On 03/13/15 13:57, Ciprian Barbu wrote: >>> + sleep(timeout); > Could we use ODP timers instead of sleep? And possibly print stats > every second, like netmap pkt-gen does, maybe make the interval > configurable? > Uhh, need to do something with that timers. Most likely to add some helper function with more posix style to init and use odp posix style timers instead of timer pool, timer queue, timer buffer. That is not per packet sleep. So might be sleep() is ok here. I not sure which where we should use odp timers and where system timers. Or should we always use odp timers. Maybe it's better to add parameter for time accuracy. 1 second for linux-generic and my x86 is too small for output approximation. Might be other applications involved or laptop goes to turbo mode, but fact is 10 seconds is good for me. So I will add parameter to configure that time. Thanks, Maxim.
diff --git a/example/l2fwd/odp_l2fwd.c b/example/l2fwd/odp_l2fwd.c index d062a72..71c2fc5 100644 --- a/example/l2fwd/odp_l2fwd.c +++ b/example/l2fwd/odp_l2fwd.c @@ -104,6 +104,14 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args); static void print_info(char *progname, appl_args_t *appl_args); static void usage(char *progname); +/* speed and stats */ +typedef struct { + uint64_t packets[MAX_WORKERS]; + uint64_t drops[MAX_WORKERS]; +} stats_t; + +static stats_t stats; + /** * Packet IO worker thread using ODP queues * @@ -115,8 +123,6 @@ static void *pktio_queue_thread(void *arg) odp_queue_t outq_def; odp_packet_t pkt; odp_event_t ev; - unsigned long pkt_cnt = 0; - unsigned long err_cnt = 0; (void)arg; @@ -132,7 +138,7 @@ static void *pktio_queue_thread(void *arg) /* Drop packets with errors */ if (odp_unlikely(drop_err_pkts(&pkt, 1) == 0)) { - EXAMPLE_ERR("Drop frame - err_cnt:%lu\n", ++err_cnt); + stats.drops[thr] += 1; continue; } @@ -141,11 +147,7 @@ static void *pktio_queue_thread(void *arg) /* Enqueue the packet for output */ odp_queue_enq(outq_def, ev); - /* Print packet counts every once in a while */ - if (odp_unlikely(pkt_cnt++ % 100000 == 0)) { - printf(" [%02i] pkt_cnt:%lu\n", thr, pkt_cnt); - fflush(NULL); - } + stats.packets[thr] += 1; } /* unreachable */ @@ -186,9 +188,6 @@ static void *pktio_ifburst_thread(void *arg) thread_args_t *thr_args; int pkts, pkts_ok; odp_packet_t pkt_tbl[MAX_PKT_BURST]; - unsigned long pkt_cnt = 0; - unsigned long err_cnt = 0; - unsigned long tmp = 0; int src_idx, dst_idx; odp_pktio_t pktio_src, pktio_dst; @@ -218,23 +217,13 @@ static void *pktio_ifburst_thread(void *arg) if (pkts_ok > 0) odp_pktio_send(pktio_dst, pkt_tbl, pkts_ok); - if (odp_unlikely(pkts_ok != pkts)) { - err_cnt += pkts-pkts_ok; - EXAMPLE_ERR("Dropped frames:%u - err_cnt:%lu\n", - pkts-pkts_ok, err_cnt); - } + if (odp_unlikely(pkts_ok != pkts)) + stats.drops[thr] += pkts - pkts_ok; if (pkts_ok == 0) continue; - /* Print packet counts every once in a while */ - tmp += pkts_ok; - if (odp_unlikely(tmp >= 100000 || pkt_cnt == 0)) { - pkt_cnt += tmp; - tmp = 0; - printf(" [%02i] pkt_cnt:%lu\n", thr, pkt_cnt); - fflush(NULL); - } + stats.packets[thr] += pkts_ok; } /* unreachable */ @@ -295,6 +284,34 @@ static odp_pktio_t create_pktio(const char *dev, odp_pool_t pool, return pktio; } +static void print_speed_stats(int num_workers) +{ + stats_t old_stats; + stats_t new_stats; + uint64_t old; + uint64_t new; + int i; + int timeout = 10; + + while (1) { + memcpy(&old_stats, &stats, sizeof(stats_t)); + sleep(timeout); + memcpy(&new_stats, &stats, sizeof(stats_t)); + + for (i = old = new = 0; i < num_workers; i++) { + old += old_stats.packets[i]; + new += new_stats.packets[i]; + } + printf(" %" PRIu64 " pps, ", (new - old) / timeout); + + for (i = old = new = 0; i < num_workers; i++) { + old += old_stats.drops[i]; + new += new_stats.drops[i]; + } + printf(" %" PRIu64 " drops\n", (new - old) / timeout); + } +} + /** * ODP L2 forwarding main function */ @@ -412,6 +429,8 @@ int main(int argc, char *argv[]) cpu = odp_cpumask_next(&cpumask, cpu); } + print_speed_stats(num_workers); + /* Master thread waits for other threads to exit */ odph_linux_pthread_join(thread_tbl, num_workers);
Current print in l2fwd is not useful with slow links and also it's hard to say how fast does it work. Print pps and packets drops. Signed-off-by: Maxim Uvarov <maxim.uvarov@linaro.org> --- example/l2fwd/odp_l2fwd.c | 67 ++++++++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 24 deletions(-)