@@ -131,6 +131,7 @@ __LIB__libodp_la_SOURCES = \
odp_packet_flags.c \
odp_packet_io.c \
pktio/io_ops.c \
+ pktio/pktio_common.c \
pktio/loop.c \
pktio/netmap.c \
pktio/socket.c \
@@ -72,7 +72,6 @@ struct cos_s {
uint32_t valid; /* validity Flag */
odp_drop_e drop_policy; /* Associated Drop Policy */
odp_queue_group_t queue_group; /* Associated Queue Group */
- odp_queue_t queue_id; /* Associated Queue handle */
odp_cos_flow_set_t flow_set; /* Assigned Flow Set */
char name[ODP_COS_NAME_LEN]; /* name */
size_t headroom; /* Headroom for this CoS */
@@ -59,8 +59,17 @@ Packet Classifier
Start function for Packet Classifier
This function calls Classifier module internal functions for a given packet and
enqueues the packet to specific Queue based on PMR and CoS selected.
+The packet is allocated from the pool associated with the CoS
**/
int packet_classifier(odp_pktio_t pktio, odp_packet_t pkt);
+
+/**
+@internal
+
+Same as packet classifier uses linux-generic internal pktio struct
+**/
+int _odp_packet_classifier(pktio_entry_t *entry, odp_packet_t pkt);
+
/**
Packet IO classifier init
@@ -28,6 +28,8 @@ extern "C" {
#include <odp/crypto.h>
#include <odp_crypto_internal.h>
+#define PACKET_JUMBO_LEN (9 * 1024)
+
/**
* Packet input & protocol flags
*/
@@ -248,7 +250,7 @@ void packet_parse_l2(odp_packet_hdr_t *pkt_hdr);
int packet_parse_full(odp_packet_hdr_t *pkt_hdr);
/* Reset parser metadata for a new parse */
-void packet_parse_reset(odp_packet_t pkt);
+void packet_parse_reset(odp_packet_hdr_t *pkt_hdr);
/* Convert a packet handle to a buffer handle */
odp_buffer_t _odp_packet_to_buffer(odp_packet_t pkt);
@@ -256,6 +258,10 @@ odp_buffer_t _odp_packet_to_buffer(odp_packet_t pkt);
/* Convert a buffer handle to a packet handle */
odp_packet_t _odp_packet_from_buffer(odp_buffer_t buf);
+int _odp_parse_common(odp_packet_hdr_t *pkt_hdr, const uint8_t *parseptr);
+
+int _odp_cls_parse(odp_packet_hdr_t *pkt_hdr, const uint8_t *parseptr);
+
#ifdef __cplusplus
}
#endif
@@ -86,6 +86,7 @@ struct pktio_entry {
classifier_t cls; /**< classifier linked with this pktio*/
char name[PKTIO_NAME_LEN]; /**< name of pktio provided to
pktio_open() */
+ odp_pktio_t id;
odp_pktio_param_t param;
};
@@ -118,6 +119,9 @@ typedef struct pktio_if_ops {
int (*mac_get)(pktio_entry_t *pktio_entry, void *mac_addr);
} pktio_if_ops_t;
+int _odp_packet_cls_enq(pktio_entry_t *pktio_entry, uint8_t *base,
+ uint16_t buf_len, odp_packet_t *pkt_ret);
+
extern void *pktio_entry_ptr[];
static inline int pktio_to_id(odp_pktio_t pktio)
@@ -44,6 +44,8 @@ typedef struct {
int sockfd; /**< socket descriptor */
odp_pool_t pool; /**< pool to alloc packets from */
unsigned char if_mac[ETH_ALEN]; /**< IF eth mac addr */
+ uint8_t *cache_ptr[ODP_PACKET_SOCKET_MAX_BURST_RX];
+ odp_shm_t shm;
} pkt_sock_t;
/** packet mmap ring */
@@ -274,8 +274,10 @@ int odp_cos_queue_set(odp_cos_t cos_id, odp_queue_t queue_id)
}
/* Locking is not required as intermittent stale
data during CoS modification is acceptable*/
- cos->s.queue = queue_to_qentry(queue_id);
- cos->s.queue_id = queue_id;
+ if (queue_id == ODP_QUEUE_INVALID)
+ cos->s.queue = NULL;
+ else
+ cos->s.queue = queue_to_qentry(queue_id);
return 0;
}
@@ -288,7 +290,10 @@ odp_queue_t odp_cos_queue(odp_cos_t cos_id)
return ODP_QUEUE_INVALID;
}
- return cos->s.queue_id;
+ if (!cos->s.queue)
+ return ODP_QUEUE_INVALID;
+
+ return cos->s.queue->s.handle;
}
int odp_cos_drop_set(odp_cos_t cos_id, odp_drop_e drop_policy)
@@ -661,6 +666,40 @@ int odp_pktio_pmr_match_set_cos(odp_pmr_set_t pmr_set_id, odp_pktio_t src_pktio,
return 0;
}
+int odp_cls_cos_pool_set(odp_cos_t cos_id, odp_pool_t pool_id)
+{
+ cos_t *cos;
+
+ cos = get_cos_entry(cos_id);
+ if (cos == NULL) {
+ ODP_ERR("Invalid odp_cos_t handle");
+ return -1;
+ }
+
+ if (pool_id == ODP_POOL_INVALID)
+ cos->s.pool = NULL;
+ else
+ cos->s.pool = odp_pool_to_entry(pool_id);
+
+ return 0;
+}
+
+odp_pool_t odp_cls_cos_pool(odp_cos_t cos_id)
+{
+ cos_t *cos;
+
+ cos = get_cos_entry(cos_id);
+ if (cos == NULL) {
+ ODP_ERR("Invalid odp_cos_t handle");
+ return ODP_POOL_INVALID;
+ }
+
+ if (!cos->s.pool)
+ return ODP_POOL_INVALID;
+
+ return cos->s.pool->s.pool_hdl;
+}
+
int verify_pmr(pmr_t *pmr, uint8_t *pkt_addr, odp_packet_hdr_t *pkt_hdr)
{
int pmr_failure = 0;
@@ -824,15 +863,14 @@ int pktio_classifier_init(pktio_entry_t *entry)
return 0;
}
-int packet_classifier(odp_pktio_t pktio, odp_packet_t pkt)
+int _odp_packet_classifier(pktio_entry_t *entry, odp_packet_t pkt)
{
- pktio_entry_t *entry;
queue_entry_t *queue;
cos_t *cos;
odp_packet_hdr_t *pkt_hdr;
+ odp_packet_t new_pkt;
uint8_t *pkt_addr;
- entry = get_pktio_entry(pktio);
if (entry == NULL)
return -1;
@@ -844,9 +882,38 @@ int packet_classifier(odp_pktio_t pktio, odp_packet_t pkt)
if (cos == NULL)
return -1;
+ if (cos->s.pool == NULL) {
+ odp_packet_free(pkt);
+ return -1;
+ }
+
+ if (cos->s.queue == NULL) {
+ odp_packet_free(pkt);
+ return -1;
+ }
+
+ if (odp_packet_pool(pkt) != cos->s.pool->s.pool_hdl) {
+ new_pkt = odp_packet_copy(pkt, cos->s.pool->s.pool_hdl);
+ odp_packet_free(pkt);
+ if (new_pkt == ODP_PACKET_INVALID)
+ return -1;
+ } else {
+ new_pkt = pkt;
+ }
+
/* Enqueuing the Packet based on the CoS */
queue = cos->s.queue;
- return queue_enq(queue, odp_buf_to_hdr((odp_buffer_t)pkt), 0);
+ return queue_enq(queue, odp_buf_to_hdr((odp_buffer_t)new_pkt), 0);
+}
+
+int packet_classifier(odp_pktio_t pktio, odp_packet_t pkt)
+{
+ pktio_entry_t *entry;
+
+ entry = get_pktio_entry(pktio);
+ if (entry == NULL)
+ return -1;
+ return _odp_packet_classifier(entry, pkt);
}
cos_t *pktio_select_cos(pktio_entry_t *entry, uint8_t *pkt_addr,
@@ -32,10 +32,8 @@ static inline void packet_parse_disable(odp_packet_hdr_t *pkt_hdr)
pkt_hdr->input_flags.parsed_all = 1;
}
-void packet_parse_reset(odp_packet_t pkt)
+void packet_parse_reset(odp_packet_hdr_t *pkt_hdr)
{
- odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
-
/* Reset parser metadata before new parse */
pkt_hdr->error_flags.all = 0;
pkt_hdr->input_flags.all = 0;
@@ -780,9 +778,9 @@ int _odp_packet_copy_to_packet(odp_packet_t srcpkt, uint32_t srcoffset,
* Parser helper function for IPv4
*/
static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr,
- uint8_t **parseptr, uint32_t *offset)
+ const uint8_t **parseptr, uint32_t *offset)
{
- odph_ipv4hdr_t *ipv4 = (odph_ipv4hdr_t *)*parseptr;
+ const odph_ipv4hdr_t *ipv4 = (const odph_ipv4hdr_t *)*parseptr;
uint8_t ver = ODPH_IPV4HDR_VER(ipv4->ver_ihl);
uint8_t ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl);
uint16_t frag_offset;
@@ -818,10 +816,10 @@ static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr,
* Parser helper function for IPv6
*/
static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr,
- uint8_t **parseptr, uint32_t *offset)
+ const uint8_t **parseptr, uint32_t *offset)
{
- odph_ipv6hdr_t *ipv6 = (odph_ipv6hdr_t *)*parseptr;
- odph_ipv6hdr_ext_t *ipv6ext;
+ const odph_ipv6hdr_t *ipv6 = (const odph_ipv6hdr_t *)*parseptr;
+ const odph_ipv6hdr_ext_t *ipv6ext;
pkt_hdr->l3_len = odp_be_to_cpu_16(ipv6->payload_len);
@@ -843,7 +841,7 @@ static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr,
pkt_hdr->input_flags.ipopt = 1;
do {
- ipv6ext = (odph_ipv6hdr_ext_t *)*parseptr;
+ ipv6ext = (const odph_ipv6hdr_ext_t *)*parseptr;
uint16_t extlen = 8 + ipv6ext->ext_len * 8;
*offset += extlen;
@@ -875,9 +873,9 @@ static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr,
* Parser helper function for TCP
*/
static inline void parse_tcp(odp_packet_hdr_t *pkt_hdr,
- uint8_t **parseptr, uint32_t *offset)
+ const uint8_t **parseptr, uint32_t *offset)
{
- odph_tcphdr_t *tcp = (odph_tcphdr_t *)*parseptr;
+ const odph_tcphdr_t *tcp = (const odph_tcphdr_t *)*parseptr;
if (tcp->hl < sizeof(odph_tcphdr_t)/sizeof(uint32_t))
pkt_hdr->error_flags.tcp_err = 1;
@@ -895,9 +893,9 @@ static inline void parse_tcp(odp_packet_hdr_t *pkt_hdr,
* Parser helper function for UDP
*/
static inline void parse_udp(odp_packet_hdr_t *pkt_hdr,
- uint8_t **parseptr, uint32_t *offset)
+ const uint8_t **parseptr, uint32_t *offset)
{
- odph_udphdr_t *udp = (odph_udphdr_t *)*parseptr;
+ const odph_udphdr_t *udp = (const odph_udphdr_t *)*parseptr;
uint32_t udplen = odp_be_to_cpu_16(udp->length);
if (udplen < sizeof(odph_udphdr_t) ||
@@ -932,46 +930,51 @@ void packet_parse_l2(odp_packet_hdr_t *pkt_hdr)
pkt_hdr->input_flags.parsed_l2 = 1;
}
-/**
- * Simple packet parser
- */
-int packet_parse_full(odp_packet_hdr_t *pkt_hdr)
+int _odp_parse_common(odp_packet_hdr_t *pkt_hdr, const uint8_t *ptr)
{
- odph_ethhdr_t *eth;
- odph_vlanhdr_t *vlan;
+ const odph_ethhdr_t *eth;
+ const odph_vlanhdr_t *vlan;
uint16_t ethtype;
- uint8_t *parseptr;
uint32_t offset, seglen;
uint8_t ip_proto = 0;
+ const uint8_t *parseptr;
+ offset = sizeof(odph_ethhdr_t);
if (packet_parse_l2_not_done(pkt_hdr))
packet_parse_l2(pkt_hdr);
- eth = (odph_ethhdr_t *)packet_map(pkt_hdr, 0, &seglen);
- offset = sizeof(odph_ethhdr_t);
- parseptr = (uint8_t *)ð->type;
- ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr));
+ if (ptr == NULL) {
+ eth = (odph_ethhdr_t *)packet_map(pkt_hdr, 0, &seglen);
+ parseptr = (const uint8_t *)ð->type;
+ ethtype = odp_be_to_cpu_16(*((const uint16_t *)parseptr));
+ } else {
+ eth = (const odph_ethhdr_t *)ptr;
+ parseptr = (const uint8_t *)ð->type;
+ ethtype = odp_be_to_cpu_16(*((const uint16_t *)parseptr));
+ }
+
/* Parse the VLAN header(s), if present */
if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) {
pkt_hdr->input_flags.vlan_qinq = 1;
pkt_hdr->input_flags.vlan = 1;
- vlan = (odph_vlanhdr_t *)(void *)parseptr;
+
+ vlan = (const odph_vlanhdr_t *)parseptr;
pkt_hdr->vlan_s_tag = ((ethtype << 16) |
odp_be_to_cpu_16(vlan->tci));
offset += sizeof(odph_vlanhdr_t);
parseptr += sizeof(odph_vlanhdr_t);
- ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr));
+ ethtype = odp_be_to_cpu_16(*((const uint16_t *)parseptr));
}
if (ethtype == ODPH_ETHTYPE_VLAN) {
pkt_hdr->input_flags.vlan = 1;
- vlan = (odph_vlanhdr_t *)(void *)parseptr;
+ vlan = (const odph_vlanhdr_t *)parseptr;
pkt_hdr->vlan_c_tag = ((ethtype << 16) |
odp_be_to_cpu_16(vlan->tci));
offset += sizeof(odph_vlanhdr_t);
parseptr += sizeof(odph_vlanhdr_t);
- ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr));
+ ethtype = odp_be_to_cpu_16(*((const uint16_t *)parseptr));
}
/* Check for SNAP vs. DIX */
@@ -983,7 +986,7 @@ int packet_parse_full(odp_packet_hdr_t *pkt_hdr)
}
offset += 8;
parseptr += 8;
- ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr));
+ ethtype = odp_be_to_cpu_16(*((const uint16_t *)parseptr));
}
/* Consume Ethertype for Layer 3 parse */
@@ -1061,3 +1064,16 @@ parse_exit:
pkt_hdr->input_flags.parsed_all = 1;
return pkt_hdr->error_flags.all != 0;
}
+
+int _odp_cls_parse(odp_packet_hdr_t *pkt_hdr, const uint8_t *parseptr)
+{
+ return _odp_parse_common(pkt_hdr, parseptr);
+}
+
+/**
+ * Simple packet parser
+ */
+int packet_parse_full(odp_packet_hdr_t *pkt_hdr)
+{
+ return _odp_parse_common(pkt_hdr, NULL);
+}
@@ -196,6 +196,7 @@ static odp_pktio_t setup_pktio_entry(const char *dev, odp_pool_t pool,
return ODP_PKTIO_INVALID;
memcpy(&pktio_entry->s.param, param, sizeof(odp_pktio_param_t));
+ pktio_entry->s.id = id;
for (pktio_if = 0; pktio_if_ops[pktio_if]; ++pktio_if) {
ret = pktio_if_ops[pktio_if]->open(id, pktio_entry, dev, pool);
@@ -553,7 +554,7 @@ odp_buffer_hdr_t *pktin_dequeue(queue_entry_t *qentry)
odp_buffer_t buf;
odp_packet_t pkt_tbl[QUEUE_MULTI_MAX];
odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX];
- int pkts, i, j;
+ int pkts, i;
odp_pktio_t pktio;
buf_hdr = queue_deq(qentry);
@@ -566,27 +567,13 @@ odp_buffer_hdr_t *pktin_dequeue(queue_entry_t *qentry)
if (pkts <= 0)
return NULL;
- if (pktio_cls_enabled(get_pktio_entry(pktio))) {
- for (i = 0, j = 0; i < pkts; i++) {
- if (0 > packet_classifier(pktio, pkt_tbl[i])) {
- buf = _odp_packet_to_buffer(pkt_tbl[i]);
- hdr_tbl[j++] = odp_buf_to_hdr(buf);
- }
- }
- } else {
- for (i = 0; i < pkts; i++) {
- buf = _odp_packet_to_buffer(pkt_tbl[i]);
- hdr_tbl[i] = odp_buf_to_hdr(buf);
- }
-
- j = pkts;
+ for (i = 0; i < pkts; i++) {
+ buf = _odp_packet_to_buffer(pkt_tbl[i]);
+ hdr_tbl[i] = odp_buf_to_hdr(buf);
}
- if (0 == j)
- return NULL;
-
- if (j > 1)
- queue_enq_multi(qentry, &hdr_tbl[1], j - 1, 0);
+ if (pkts > 1)
+ queue_enq_multi(qentry, &hdr_tbl[1], pkts - 1, 0);
buf_hdr = hdr_tbl[0];
return buf_hdr;
}
@@ -625,27 +612,15 @@ int pktin_deq_multi(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr[], int num)
if (pkts <= 0)
return nbr;
- if (pktio_cls_enabled(get_pktio_entry(pktio))) {
- for (i = 0, j = 0; i < pkts; i++) {
- if (0 > packet_classifier(pktio, pkt_tbl[i])) {
- buf = _odp_packet_to_buffer(pkt_tbl[i]);
- if (nbr < num)
- buf_hdr[nbr++] = odp_buf_to_hdr(buf);
- else
- hdr_tbl[j++] = odp_buf_to_hdr(buf);
- }
- }
- } else {
- /* Fill in buf_hdr first */
- for (i = 0; i < pkts && nbr < num; i++, nbr++) {
- buf = _odp_packet_to_buffer(pkt_tbl[i]);
- buf_hdr[nbr] = odp_buf_to_hdr(buf);
- }
- /* Queue the rest for later */
- for (j = 0; i < pkts; i++, j++) {
- buf = _odp_packet_to_buffer(pkt_tbl[i]);
- hdr_tbl[j++] = odp_buf_to_hdr(buf);
- }
+ /* Fill in buf_hdr first */
+ for (i = 0; i < pkts && nbr < num; i++, nbr++) {
+ buf = _odp_packet_to_buffer(pkt_tbl[i]);
+ buf_hdr[nbr] = odp_buf_to_hdr(buf);
+ }
+ /* Queue the rest for later */
+ for (j = 0; i < pkts; i++, j++) {
+ buf = _odp_packet_to_buffer(pkt_tbl[i]);
+ hdr_tbl[j++] = odp_buf_to_hdr(buf);
}
if (j)
@@ -657,7 +632,7 @@ int pktin_poll(pktio_entry_t *entry)
{
odp_packet_t pkt_tbl[QUEUE_MULTI_MAX];
odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX];
- int num, num_enq, i;
+ int num, i;
odp_buffer_t buf;
odp_pktio_t pktio;
@@ -682,26 +657,15 @@ int pktin_poll(pktio_entry_t *entry)
return -1;
}
- if (pktio_cls_enabled(entry)) {
- for (i = 0, num_enq = 0; i < num; i++) {
- if (packet_classifier(pktio, pkt_tbl[i]) < 0) {
- buf = _odp_packet_to_buffer(pkt_tbl[i]);
- hdr_tbl[num_enq++] = odp_buf_to_hdr(buf);
- }
- }
- } else {
- for (i = 0; i < num; i++) {
- buf = _odp_packet_to_buffer(pkt_tbl[i]);
- hdr_tbl[i] = odp_buf_to_hdr(buf);
- }
-
- num_enq = num;
+ for (i = 0; i < num; i++) {
+ buf = _odp_packet_to_buffer(pkt_tbl[i]);
+ hdr_tbl[i] = odp_buf_to_hdr(buf);
}
- if (num_enq) {
+ if (num) {
queue_entry_t *qentry;
qentry = queue_to_qentry(entry->s.inq_default);
- queue_enq_multi(qentry, hdr_tbl, num_enq, 0);
+ queue_enq_multi(qentry, hdr_tbl, num, 0);
}
return 0;
@@ -12,6 +12,7 @@
#include <odp.h>
#include <odp_packet_internal.h>
#include <odp_packet_io_internal.h>
+#include <odp_classification_internal.h>
#include <odp_debug_internal.h>
#include <odp/hints.h>
@@ -50,19 +51,35 @@ static int loopback_close(pktio_entry_t *pktio_entry)
static int loopback_recv(pktio_entry_t *pktio_entry, odp_packet_t pkts[],
unsigned len)
{
- int nbr, i;
+ int nbr, i, j;
odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX];
queue_entry_t *qentry;
odp_packet_hdr_t *pkt_hdr;
+ odp_packet_t pkt;
+ nbr = 0;
qentry = queue_to_qentry(pktio_entry->s.pkt_loop.loopq);
nbr = queue_deq_multi(qentry, hdr_tbl, len);
- for (i = 0; i < nbr; ++i) {
- pkts[i] = _odp_packet_from_buffer(odp_hdr_to_buf(hdr_tbl[i]));
- pkt_hdr = odp_packet_hdr(pkts[i]);
- packet_parse_reset(pkts[i]);
- packet_parse_l2(pkt_hdr);
+ if (pktio_cls_enabled(pktio_entry)) {
+ for (i = 0, j = 0; i < nbr; i++) {
+ pkt = _odp_packet_from_buffer(odp_hdr_to_buf
+ (hdr_tbl[i]));
+ pkt_hdr = odp_packet_hdr(pkt);
+ packet_parse_reset(pkt_hdr);
+ packet_parse_l2(pkt_hdr);
+ if (0 > _odp_packet_classifier(pktio_entry, pkt))
+ pkts[j++] = pkt;
+ }
+ nbr = j;
+ } else {
+ for (i = 0; i < nbr; ++i) {
+ pkts[i] = _odp_packet_from_buffer(odp_hdr_to_buf
+ (hdr_tbl[i]));
+ pkt_hdr = odp_packet_hdr(pkts[i]);
+ packet_parse_reset(pkt_hdr);
+ packet_parse_l2(pkt_hdr);
+ }
}
return nbr;
@@ -20,6 +20,9 @@
#include <poll.h>
#include <linux/ethtool.h>
#include <linux/sockios.h>
+#include <odp_classification_datamodel.h>
+#include <odp_classification_inlines.h>
+#include <odp_classification_internal.h>
#define NETMAP_WITH_LIBS
#include <net/netmap_user.h>
@@ -197,7 +200,7 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry,
uint16_t len)
{
odp_packet_t pkt;
- odp_packet_hdr_t *pkt_hdr;
+ int ret;
if (odp_unlikely(len > pktio_entry->s.pkt_nm.max_frame_len)) {
ODP_ERR("RX: frame too big %" PRIu16 " %zu!\n", len,
@@ -210,21 +213,32 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry,
return -1;
}
- pkt = packet_alloc(pktio_entry->s.pkt_nm.pool, len, 1);
- if (pkt == ODP_PACKET_INVALID)
+ if (pktio_cls_enabled(pktio_entry)) {
+ ret = _odp_packet_cls_enq(pktio_entry, (uint8_t *)buf,
+ len, pkt_out);
+ if (ret)
+ return 0;
return -1;
+ } else {
+ odp_packet_hdr_t *pkt_hdr;
- pkt_hdr = odp_packet_hdr(pkt);
+ pkt = packet_alloc(pktio_entry->s.pkt_nm.pool, len, 1);
+ if (pkt == ODP_PACKET_INVALID)
+ return -1;
- /* For now copy the data in the mbuf,
- worry about zero-copy later */
- if (odp_packet_copydata_in(pkt, 0, len, buf) != 0) {
- odp_packet_free(pkt);
- return -1;
+ pkt_hdr = odp_packet_hdr(pkt);
+
+ /* For now copy the data in the mbuf,
+ worry about zero-copy later */
+ if (odp_packet_copydata_in(pkt, 0, len, buf) != 0) {
+ odp_packet_free(pkt);
+ return -1;
+ }
+
+ packet_parse_l2(pkt_hdr);
+ *pkt_out = pkt;
}
- packet_parse_l2(pkt_hdr);
- *pkt_out = pkt;
return 0;
}
new file mode 100644
@@ -0,0 +1,56 @@
+/* Copyright (c) 2013, Linaro Limited
+ * Copyright (c) 2013, Nokia Solutions and Networks
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <odp_packet_io_internal.h>
+#include <odp_classification_internal.h>
+
+int _odp_packet_cls_enq(pktio_entry_t *pktio_entry,
+ uint8_t *base, uint16_t buf_len,
+ odp_packet_t *pkt_ret)
+{
+ cos_t *cos;
+ odp_packet_t pkt;
+ odp_packet_hdr_t pkt_hdr;
+ int ret;
+ odp_pool_t pool;
+
+ packet_parse_reset(&pkt_hdr);
+
+ _odp_cls_parse(&pkt_hdr, base);
+ cos = pktio_select_cos(pktio_entry, (uint8_t *)base, &pkt_hdr);
+ pool = cos->s.pool->s.pool_hdl;
+
+ /* if No CoS found then drop the packet */
+ if (cos == NULL || cos->s.queue == NULL)
+ return 0;
+
+ pkt = odp_packet_alloc(pool, buf_len);
+ if (odp_unlikely(pkt == ODP_PACKET_INVALID))
+ return 0;
+
+ copy_packet_parser_metadata(&pkt_hdr, odp_packet_hdr(pkt));
+ odp_packet_hdr(pkt)->input = pktio_entry->s.id;
+
+ if (odp_packet_copydata_in(pkt, 0, buf_len, base) != 0) {
+ odp_packet_free(pkt);
+ return 0;
+ }
+
+ /* Parse and set packet header data */
+ odp_packet_pull_tail(pkt, odp_packet_len(pkt) - buf_len);
+ ret = queue_enq(cos->s.queue, odp_buf_to_hdr((odp_buffer_t)pkt), 0);
+ if (ret < 0) {
+ *pkt_ret = pkt;
+ return 1;
+ }
+
+ return 0;
+}
@@ -38,6 +38,9 @@
#include <odp_packet_io_internal.h>
#include <odp_align_internal.h>
#include <odp_debug_internal.h>
+#include <odp_classification_datamodel.h>
+#include <odp_classification_inlines.h>
+#include <odp_classification_internal.h>
#include <odp/hints.h>
#include <odp/helper/eth.h>
@@ -194,6 +197,8 @@ static int sock_close(pktio_entry_t *pktio_entry)
return -1;
}
+ odp_shm_free(pkt_sock->shm);
+
return 0;
}
@@ -205,10 +210,13 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev,
{
int sockfd;
int err;
+ int i;
unsigned int if_idx;
struct ifreq ethreq;
struct sockaddr_ll sa_ll;
+ char shm_name[ODP_SHM_NAME_LEN];
pkt_sock_t *pkt_sock = &pktio_entry->s.pkt_sock;
+ uint8_t *addr;
/* Init pktio entry */
memset(pkt_sock, 0, sizeof(*pkt_sock));
@@ -218,6 +226,20 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev,
if (pool == ODP_POOL_INVALID)
return -1;
pkt_sock->pool = pool;
+ snprintf(shm_name, ODP_SHM_NAME_LEN, "%s-%s", "pktio", netdev);
+ shm_name[ODP_SHM_NAME_LEN - 1] = '\0';
+
+ pkt_sock->shm = odp_shm_reserve(shm_name, PACKET_JUMBO_LEN,
+ PACKET_JUMBO_LEN *
+ ODP_PACKET_SOCKET_MAX_BURST_RX, 0);
+ if (pkt_sock->shm == ODP_SHM_INVALID)
+ return -1;
+
+ addr = odp_shm_addr(pkt_sock->shm);
+ for (i = 0; i < ODP_PACKET_SOCKET_MAX_BURST_RX; i++) {
+ pkt_sock->cache_ptr[i] = addr;
+ addr += PACKET_JUMBO_LEN;
+ }
sockfd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL));
if (sockfd == -1) {
@@ -253,7 +275,6 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev,
ODP_ERR("bind(to IF): %s\n", strerror(errno));
goto error;
}
-
return 0;
error:
@@ -311,58 +332,95 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry,
const int sockfd = pkt_sock->sockfd;
int msgvec_len;
struct mmsghdr msgvec[ODP_PACKET_SOCKET_MAX_BURST_RX];
- struct iovec iovecs[ODP_PACKET_SOCKET_MAX_BURST_RX][ODP_BUFFER_MAX_SEG];
int nb_rx = 0;
int recv_msgs;
+ int ret;
+ uint8_t **recv_cache;
int i;
if (odp_unlikely(len > ODP_PACKET_SOCKET_MAX_BURST_RX))
return -1;
memset(msgvec, 0, sizeof(msgvec));
+ recv_cache = pkt_sock->cache_ptr;
- for (i = 0; i < (int)len; i++) {
- pkt_table[i] = packet_alloc(pkt_sock->pool, 0 /*default*/, 1);
- if (odp_unlikely(pkt_table[i] == ODP_PACKET_INVALID))
- break;
+ if (pktio_cls_enabled(pktio_entry)) {
+ struct iovec iovecs[ODP_PACKET_SOCKET_MAX_BURST_RX];
- msgvec[i].msg_hdr.msg_iovlen =
- _rx_pkt_to_iovec(pkt_table[i], iovecs[i]);
-
- msgvec[i].msg_hdr.msg_iov = iovecs[i];
- }
- msgvec_len = i; /* number of successfully allocated pkt buffers */
+ for (i = 0; i < (int)len; i++) {
+ msgvec[i].msg_hdr.msg_iovlen = 1;
+ iovecs[i].iov_base = recv_cache[i];
+ iovecs[i].iov_len = PACKET_JUMBO_LEN;
+ msgvec[i].msg_hdr.msg_iov = &iovecs[i];
+ }
+ /* number of successfully allocated pkt buffers */
+ msgvec_len = i;
+
+ recv_msgs = recvmmsg(sockfd, msgvec, msgvec_len,
+ MSG_DONTWAIT, NULL);
+ for (i = 0; i < recv_msgs; i++) {
+ void *base = msgvec[i].msg_hdr.msg_iov->iov_base;
+ struct ethhdr *eth_hdr = base;
+ uint16_t pkt_len = msgvec[i].msg_len;
+
+ /* Don't receive packets sent by ourselves */
+ if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac,
+ eth_hdr->h_source)))
+ continue;
+
+ ret = _odp_packet_cls_enq(pktio_entry, base,
+ pkt_len, &pkt_table[nb_rx]);
+ if (ret)
+ nb_rx++;
+ }
+ } else {
+ struct iovec iovecs[ODP_PACKET_SOCKET_MAX_BURST_RX]
+ [ODP_BUFFER_MAX_SEG];
- recv_msgs = recvmmsg(sockfd, msgvec, msgvec_len, MSG_DONTWAIT, NULL);
+ for (i = 0; i < (int)len; i++) {
+ pkt_table[i] = packet_alloc(pkt_sock->pool,
+ 0 /*default*/, 1);
+ if (odp_unlikely(pkt_table[i] == ODP_PACKET_INVALID))
+ break;
- for (i = 0; i < recv_msgs; i++) {
- odp_packet_hdr_t *pkt_hdr;
- void *base = msgvec[i].msg_hdr.msg_iov->iov_base;
- struct ethhdr *eth_hdr = base;
+ msgvec[i].msg_hdr.msg_iovlen =
+ _rx_pkt_to_iovec(pkt_table[i], iovecs[i]);
- /* Don't receive packets sent by ourselves */
- if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac,
- eth_hdr->h_source))) {
- odp_packet_free(pkt_table[i]);
- continue;
+ msgvec[i].msg_hdr.msg_iov = iovecs[i];
}
- pkt_hdr = odp_packet_hdr(pkt_table[i]);
-
- odp_packet_pull_tail(pkt_table[i],
- odp_packet_len(pkt_table[i]) -
- msgvec[i].msg_len);
+ /* number of successfully allocated pkt buffers */
+ msgvec_len = i;
- packet_parse_l2(pkt_hdr);
+ recv_msgs = recvmmsg(sockfd, msgvec, msgvec_len,
+ MSG_DONTWAIT, NULL);
- pkt_table[nb_rx] = pkt_table[i];
- nb_rx++;
- }
+ for (i = 0; i < recv_msgs; i++) {
+ void *base = msgvec[i].msg_hdr.msg_iov->iov_base;
+ struct ethhdr *eth_hdr = base;
+ odp_packet_hdr_t *pkt_hdr;
- /* Free unused pkt buffers */
- for (; i < msgvec_len; i++)
- odp_packet_free(pkt_table[i]);
+ /* Don't receive packets sent by ourselves */
+ if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac,
+ eth_hdr->h_source))) {
+ odp_packet_free(pkt_table[i]);
+ continue;
+ }
+ pkt_hdr = odp_packet_hdr(pkt_table[i]);
+ /* Parse and set packet header data */
+ odp_packet_pull_tail(pkt_table[i],
+ odp_packet_len(pkt_table[i]) -
+ msgvec[i].msg_len);
+
+ packet_parse_l2(pkt_hdr);
+ pkt_table[nb_rx] = pkt_table[i];
+ nb_rx++;
+ }
+ /* Free unused pkt buffers */
+ for (; i < msgvec_len; i++)
+ odp_packet_free(pkt_table[i]);
+ }
return nb_rx;
}
@@ -377,7 +435,7 @@ static uint32_t _tx_pkt_to_iovec(odp_packet_t pkt,
uint32_t seglen;
iovecs[iov_count].iov_base = odp_packet_offset(pkt, offset,
- &seglen, NULL);
+ &seglen, NULL);
iovecs[iov_count].iov_len = seglen;
iov_count++;
offset += seglen;
@@ -407,7 +465,7 @@ static int sock_mmsg_send(pktio_entry_t *pktio_entry,
for (i = 0; i < len; i++) {
msgvec[i].msg_hdr.msg_iov = iovecs[i];
msgvec[i].msg_hdr.msg_iovlen = _tx_pkt_to_iovec(pkt_table[i],
- iovecs[i]);
+ iovecs[i]);
}
for (i = 0; i < len; ) {
@@ -415,7 +473,7 @@ static int sock_mmsg_send(pktio_entry_t *pktio_entry,
if (odp_unlikely(ret <= -1)) {
if (i == 0 && SOCK_ERR_REPORT(errno)) {
__odp_errno = errno;
- ODP_ERR("sendmmsg(): %s\n", strerror(errno));
+ ODP_ERR("sendmmsg(): %s\n", strerror(errno));
return -1;
}
break;
@@ -28,6 +28,9 @@
#include <odp_packet_internal.h>
#include <odp_packet_io_internal.h>
#include <odp_debug_internal.h>
+#include <odp_classification_datamodel.h>
+#include <odp_classification_inlines.h>
+#include <odp_classification_internal.h>
#include <odp/hints.h>
#include <odp/helper/eth.h>
@@ -108,9 +111,9 @@ static inline void mmap_tx_user_ready(struct tpacket2_hdr *hdr)
__sync_synchronize();
}
-static inline unsigned pkt_mmap_v2_rx(int sock, struct ring *ring,
+static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
+ pkt_sock_mmap_t *pkt_sock,
odp_packet_t pkt_table[], unsigned len,
- odp_pool_t pool,
unsigned char if_mac[])
{
union frame_map ppd;
@@ -118,57 +121,68 @@ static inline unsigned pkt_mmap_v2_rx(int sock, struct ring *ring,
uint8_t *pkt_buf;
int pkt_len;
struct ethhdr *eth_hdr;
- odp_packet_hdr_t *pkt_hdr;
unsigned i = 0;
+ uint8_t nb_rx = 0;
+ struct ring *ring;
+ int ret;
- (void)sock;
-
+ ring = &pkt_sock->rx_ring;
frame_num = ring->frame_num;
while (i < len) {
- if (mmap_rx_kernel_ready(ring->rd[frame_num].iov_base)) {
- ppd.raw = ring->rd[frame_num].iov_base;
+ if (!mmap_rx_kernel_ready(ring->rd[frame_num].iov_base))
+ break;
+
+ ppd.raw = ring->rd[frame_num].iov_base;
+ next_frame_num = (frame_num + 1) % ring->rd_num;
+
+ pkt_buf = (uint8_t *)ppd.raw + ppd.v2->tp_h.tp_mac;
+ pkt_len = ppd.v2->tp_h.tp_snaplen;
- next_frame_num = (frame_num + 1) % ring->rd_num;
+ /* Don't receive packets sent by ourselves */
+ eth_hdr = (struct ethhdr *)pkt_buf;
+ if (odp_unlikely(ethaddrs_equal(if_mac,
+ eth_hdr->h_source))) {
+ mmap_rx_user_ready(ppd.raw); /* drop */
+ frame_num = next_frame_num;
+ continue;
+ }
- pkt_buf = (uint8_t *)ppd.raw + ppd.v2->tp_h.tp_mac;
- pkt_len = ppd.v2->tp_h.tp_snaplen;
+ if (pktio_cls_enabled(pktio_entry)) {
+ ret = _odp_packet_cls_enq(pktio_entry, pkt_buf,
+ pkt_len, &pkt_table[nb_rx]);
+ if (ret)
+ nb_rx++;
+ } else {
+ odp_packet_hdr_t *hdr;
- /* Don't receive packets sent by ourselves */
- eth_hdr = (struct ethhdr *)pkt_buf;
- if (odp_unlikely(ethaddrs_equal(if_mac,
- eth_hdr->h_source))) {
+ pkt_table[i] = packet_alloc(pkt_sock->pool, pkt_len, 1);
+ if (odp_unlikely(pkt_table[i] == ODP_PACKET_INVALID)) {
mmap_rx_user_ready(ppd.raw); /* drop */
frame_num = next_frame_num;
continue;
}
-
- pkt_table[i] = packet_alloc(pool, pkt_len, 1);
- if (odp_unlikely(pkt_table[i] == ODP_PACKET_INVALID))
- break;
-
- pkt_hdr = odp_packet_hdr(pkt_table[i]);
-
- if (odp_packet_copydata_in(pkt_table[i], 0,
- pkt_len, pkt_buf) != 0) {
+ hdr = odp_packet_hdr(pkt_table[i]);
+ ret = odp_packet_copydata_in(pkt_table[i], 0,
+ pkt_len, pkt_buf);
+ if (ret != 0) {
odp_packet_free(pkt_table[i]);
- break;
+ mmap_rx_user_ready(ppd.raw); /* drop */
+ frame_num = next_frame_num;
+ continue;
}
- packet_parse_l2(pkt_hdr);
-
- mmap_rx_user_ready(ppd.raw);
-
- frame_num = next_frame_num;
- i++;
- } else {
- break;
+ packet_parse_l2(hdr);
+ nb_rx++;
}
+
+ mmap_rx_user_ready(ppd.raw);
+ frame_num = next_frame_num;
+ i++;
}
ring->frame_num = frame_num;
-
- return i;
+ return nb_rx;
}
static inline unsigned pkt_mmap_v2_tx(int sock, struct ring *ring,
@@ -502,9 +516,8 @@ static int sock_mmap_recv(pktio_entry_t *pktio_entry,
{
pkt_sock_mmap_t *const pkt_sock = &pktio_entry->s.pkt_sock_mmap;
- return pkt_mmap_v2_rx(pkt_sock->rx_ring.sock, &pkt_sock->rx_ring,
- pkt_table, len, pkt_sock->pool,
- pkt_sock->if_mac);
+ return pkt_mmap_v2_rx(pktio_entry, pkt_sock,
+ pkt_table, len, pkt_sock->if_mac);
}
static int sock_mmap_send(pktio_entry_t *pktio_entry,
Adds support for configuring packet pool to a class-of-service. linux-generic packet parser is enhanced to parse a packet directly from a memory location rather than from odp_packet_t. packet receive code is modified to run packet classifier directly from the stream so that the packet can be allocated from the pool specified by the CoS. Signed-off-by: Balasubramanian Manoharan <bala.manoharan@linaro.org> --- platform/linux-generic/Makefile.am | 1 + .../include/odp_classification_datamodel.h | 1 - .../include/odp_classification_internal.h | 9 ++ .../linux-generic/include/odp_packet_internal.h | 8 +- .../linux-generic/include/odp_packet_io_internal.h | 4 + platform/linux-generic/include/odp_packet_socket.h | 2 + platform/linux-generic/odp_classification.c | 81 +++++++++++-- platform/linux-generic/odp_packet.c | 74 +++++++----- platform/linux-generic/odp_packet_io.c | 80 ++++--------- platform/linux-generic/pktio/loop.c | 29 ++++- platform/linux-generic/pktio/netmap.c | 36 ++++-- platform/linux-generic/pktio/pktio_common.c | 56 +++++++++ platform/linux-generic/pktio/socket.c | 132 +++++++++++++++------ platform/linux-generic/pktio/socket_mmap.c | 87 ++++++++------ 14 files changed, 413 insertions(+), 187 deletions(-) create mode 100644 platform/linux-generic/pktio/pktio_common.c