@@ -50,8 +50,10 @@ enum {
};
static bool cfg_cork = false;
+static bool cfg_nocqe = false;
static int cfg_mode = MODE_ZC_FIXED;
static int cfg_nr_reqs = 8;
+static int cfg_nr_completions = 8;
static int cfg_family = PF_UNSPEC;
static int cfg_payload_len;
static int cfg_port = 8000;
@@ -134,11 +136,21 @@ static void do_tx(int domain, int type, int protocol)
if (mode == MODE_NONZC) {
io_uring_prep_send(sqe, fd, payload,
cfg_payload_len, msg_flags);
+ if (cfg_nocqe) {
+ sqe->flags |= IOSQE_CQE_SKIP_SUCCESS;
+ cfg_nr_completions--;
+ }
sqe->user_data = NONZC_TAG;
} else {
io_uring_prep_sendzc(sqe, fd, payload,
cfg_payload_len,
msg_flags, zc_flags);
+ if (cfg_nocqe) {
+ sqe->flags |= IOSQE_CQE_SKIP_SUCCESS;
+ packets++;
+ compl_cqes++;
+ bytes += cfg_payload_len;
+ }
if (mode == MODE_ZC_FIXED) {
sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;
sqe->buf_index = buf_idx;
@@ -153,7 +165,7 @@ static void do_tx(int domain, int type, int protocol)
if (cfg_cork)
do_setsockopt(fd, IPPROTO_UDP, UDP_CORK, 0);
- for (i = 0; i < cfg_nr_reqs; i++) {
+ for (i = 0; i < cfg_nr_completions; i++) {
ret = io_uring_wait_cqe(&ring, &cqe);
if (ret)
error(1, ret, "wait cqe");
@@ -168,7 +180,9 @@ static void do_tx(int domain, int type, int protocol)
if (compl_cqes <= 0)
error(1, -EINVAL, "notification mismatch");
compl_cqes--;
- i--;
+ if (!cfg_nocqe)
+ i--;
io_uring_cqe_seen(&ring);
continue;
}
@@ -200,6 +214,17 @@ static void do_tx(int domain, int type, int protocol)
compl_cqes--;
}
+ /* The above code does not account for a send error when
+ * IOSQE_CQE_SKIP_SUCCESS is set. This is operating under the
+ * assumption that an error CQE will get put on the ring before
+ * the above code completes:
+ */
+ while (!io_uring_peek_cqe(&ring, &cqe)) {
+ if (cqe->res == -EAGAIN)
+ continue;
+ error(1, -EINVAL, "send failed");
+ }
+
fprintf(stderr, "tx=%lu (MB=%lu), tx/s=%lu (MB/s=%lu)\n",
packets, bytes >> 20,
packets / (cfg_runtime_ms / 1000),
@@ -221,7 +246,7 @@ static void do_test(int domain, int type, int protocol)
static void usage(const char *filepath)
{
error(1, 0, "Usage: %s (-4|-6) (udp|tcp) -D<dst_ip> [-s<payload size>] "
- "[-t<time s>] [-n<batch>] [-p<port>] [-m<mode>]", filepath);
+ "[-t<time s>] [-n<batch>] [-p<port>] [-m<mode>] [-a]", filepath);
}
static void parse_opts(int argc, char **argv)
@@ -239,7 +264,7 @@ static void parse_opts(int argc, char **argv)
usage(argv[0]);
cfg_payload_len = max_payload_len;
- while ((c = getopt(argc, argv, "46D:p:s:t:n:c:m:")) != -1) {
+ while ((c = getopt(argc, argv, "46aD:p:s:t:n:c:m:")) != -1) {
switch (c) {
case '4':
if (cfg_family != PF_UNSPEC)
@@ -274,6 +299,9 @@ static void parse_opts(int argc, char **argv)
case 'm':
cfg_mode = strtol(optarg, NULL, 0);
break;
+ case 'a':
+ cfg_nocqe = true;
+ break;
}
}
@@ -302,6 +330,8 @@ static void parse_opts(int argc, char **argv)
error(1, 0, "-s: payload exceeds max (%d)", max_payload_len);
if (optind != argc - 1)
usage(argv[0]);
+
+ cfg_nr_completions = cfg_nr_reqs;
}
int main(int argc, char **argv)
@@ -25,11 +25,14 @@ readonly path_sysctl_mem="net.core.optmem_max"
# No arguments: automated test
if [[ "$#" -eq "0" ]]; then
IPs=( "4" "6" )
+ SKIPCQEs=("" "-a")
for IP in "${IPs[@]}"; do
for mode in $(seq 1 3); do
- $0 "$IP" udp -m "$mode" -t 1 -n 32
- $0 "$IP" tcp -m "$mode" -t 1 -n 1
+ for cqe in "${SKIPCQEs[@]}"; do
+ $0 "$IP" udp -m "$mode" -t 1 -n 32 "$cqe"
+ $0 "$IP" tcp -m "$mode" -t 1 -n 1 "$cqe"
+ done
done
done
Add support for the IOSQE_CQE_SKIP_SUCCESS flag in the io_uring_zerocopy_tx test, using the "-a" option. Instead of incrementing when IORING_CQE_F_MORE is set, remember how many SQEs are sent and simply wait on notifs instead of regular completions. For non-zc stuff, there won't be notifs or completions, so don't wait on either of those, but check the completion queue for errors at the end to make sure none have popped up. The changes to the shell script run the tests both with and without the "-a" option. Signed-off-by: Oliver Crumrine <ozlinuxc@gmail.com> --- .../selftests/net/io_uring_zerocopy_tx.c | 38 +++++++++++++++++-- .../selftests/net/io_uring_zerocopy_tx.sh | 7 +++- 2 files changed, 39 insertions(+), 6 deletions(-)