uring: move io_uring code to new file

Factor out the io_uring code into a new file to easy allow to move the
nvme_get_log code to be moved as well.

While at it, make uring code initialization at global context creation,
which is a huge optimization as the ring is not create per nvme_get_log
call.

Signed-off-by: Daniel Wagner <[email protected]>

Author: igaw

libnvme/src/meson.build

@@ -46,6 +46,10 @@ if want_fabrics
     ]
 endif
 
+if liburing_dep.found()
+    sources += 'nvme/uring.c'
+endif
+
 if json_c_dep.found()
     sources += 'nvme/json.c'
 else

libnvme/src/nvme/ioctl.c

@@ -16,10 +16,6 @@
 #include <sys/ioctl.h>
 #include <sys/stat.h>
 
-#ifdef CONFIG_LIBURING
-#include <liburing.h>
-#endif
-
 #include <ccan/build_assert/build_assert.h>
 #include <ccan/endian/endian.h>
 #include <ccan/minmax/minmax.h>
@@ -219,95 +215,6 @@ static void nvme_init_env(void)
 		force_4k = true;
 }
 
-#ifdef CONFIG_LIBURING
-enum {
-	IO_URING_NOT_AVAILABLE,
-	IO_URING_AVAILABLE,
-} io_uring_kernel_support = IO_URING_NOT_AVAILABLE;
-
-/*
- * gcc specific attribute, call automatically on the library loading.
- * if IORING_OP_URING_CMD is not supported, fallback to ioctl interface.
- *
- * The uring API expects the command of type struct nvme_passthru_cmd64.
- */
-__attribute__((constructor))
-static void nvme_uring_cmd_probe()
-{
-	struct io_uring_probe *probe = io_uring_get_probe();
-
-	if (!probe)
-		return;
-
-	if (!io_uring_opcode_supported(probe, IORING_OP_URING_CMD))
-		return;
-
-	io_uring_kernel_support = IO_URING_AVAILABLE;
-}
-
-static int nvme_uring_cmd_setup(struct io_uring *ring)
-{
-	if (io_uring_queue_init(NVME_URING_ENTRIES, ring,
-				   IORING_SETUP_SQE128 | IORING_SETUP_CQE32))
-		return -errno;
-	return 0;
-}
-
-static void nvme_uring_cmd_exit(struct io_uring *ring)
-{
-	io_uring_queue_exit(ring);
-}
-
-static int nvme_uring_cmd_admin_passthru_async(struct nvme_transport_handle *hdl,
-		struct io_uring *ring, struct nvme_passthru_cmd *cmd)
-{
-	struct io_uring_sqe *sqe;
-	int ret;
-
-	sqe = io_uring_get_sqe(ring);
-	if (!sqe)
-		return -1;
-
-	memcpy(&sqe->cmd, cmd, sizeof(*cmd));
-
-	sqe->fd = hdl->fd;
-	sqe->opcode = IORING_OP_URING_CMD;
-	sqe->cmd_op = NVME_URING_CMD_ADMIN;
-
-	ret = io_uring_submit(ring);
-	if (ret < 0)
-		return -errno;
-
-	return 0;
-}
-
-static int nvme_uring_cmd_wait_complete(struct io_uring *ring, int n)
-{
-	struct io_uring_cqe *cqe;
-	int ret, i;
-
-	for (i = 0; i < n; i++) {
-		ret = io_uring_wait_cqe(ring, &cqe);
-		if (ret < 0)
-			return -errno;
-		io_uring_cqe_seen(ring, cqe);
-	}
-
-	return 0;
-}
-
-static bool nvme_uring_is_usable(struct nvme_transport_handle *hdl)
-{
-	struct stat st;
-
-	if (io_uring_kernel_support != IO_URING_AVAILABLE ||
-	    hdl->type != NVME_TRANSPORT_HANDLE_TYPE_DIRECT ||
-	    fstat(hdl->fd, &st) || !S_ISCHR(st.st_mode))
-		return false;
-
-	return true;
-}
-#endif /* CONFIG_LIBURING */
 
 int nvme_get_log(struct nvme_transport_handle *hdl,
 		struct nvme_passthru_cmd *cmd, bool rae,
@@ -324,17 +231,6 @@ int nvme_get_log(struct nvme_transport_handle *hdl,
 	__u32 cdw10 = cmd->cdw10 & (NVME_VAL(LOG_CDW10_LID) |
 				    NVME_VAL(LOG_CDW10_LSP));
 	__u32 cdw11 = cmd->cdw11 & NVME_VAL(LOG_CDW11_LSI);
-#ifdef CONFIG_LIBURING
-	bool use_uring = nvme_uring_is_usable(hdl);
-	struct io_uring ring;
-	int n = 0;
-
-	if (use_uring) {
-		ret = nvme_uring_cmd_setup(&ring);
-		if (ret)
-			return ret;
-	}
-#endif /* CONFIG_LIBURING */
 
 	if (force_4k)
 		xfer_len = NVME_LOG_PAGE_PDU_SIZE;
@@ -373,43 +269,22 @@ int nvme_get_log(struct nvme_transport_handle *hdl,
 		cmd->data_len = xfer;
 		cmd->addr = (__u64)(uintptr_t)ptr;
 
-#ifdef CONFIG_LIBURING
-		if (use_uring) {
-			if (n >= NVME_URING_ENTRIES) {
-				ret = nvme_uring_cmd_wait_complete(&ring, n);
-				if (ret)
-					goto uring_exit;
-				n = 0;
-			}
-			n += 1;
-			ret = nvme_uring_cmd_admin_passthru_async(hdl,
-				&ring, cmd);
-			if (ret)
-				goto uring_exit;
-		} else {
+		if (hdl->ctx->uring_enabled)
+			ret = nvme_uring_cmd_admin_passthru_async(hdl, cmd);
+		else
 			ret = nvme_submit_admin_passthru(hdl, cmd);
-			if (ret)
-				return ret;
-		}
-#else /* CONFIG_LIBURING */
-		ret = nvme_submit_admin_passthru(hdl, cmd);
-#endif /* CONFIG_LIBURING */
 		if (ret)
 			return ret;
 
 		offset += xfer;
 		ptr += xfer;
 	} while (offset < data_len);
 
-#ifdef CONFIG_LIBURING
-	if (use_uring) {
-		ret = nvme_uring_cmd_wait_complete(&ring, n);
-uring_exit:
-		nvme_uring_cmd_exit(&ring);
+	if (hdl->ctx->uring_enabled) {
+		ret = nvme_uring_cmd_wait_complete(hdl);
 		if (ret)
 			return ret;
 	}
-#endif /* CONFIG_LIBURING */
 
 	return 0;
 }

libnvme/src/nvme/lib.c

@@ -59,6 +59,9 @@ struct nvme_global_ctx *nvme_create_global_ctx(FILE *fp, int log_level)
 	ctx->ioctl_probing = true;
 	ctx->mi_probe_enabled = nvme_mi_probe_enabled_default();
 
+	if (!nvme_open_uring(ctx))
+		ctx->uring_enabled = true;
+
 	return ctx;
 }
 
@@ -80,6 +83,7 @@ void nvme_free_global_ctx(struct nvme_global_ctx *ctx)
 		nvme_mi_close(ep);
 	free(ctx->config_file);
 	free(ctx->application);
+	nvme_close_uring(ctx);
 	free(ctx);
 }
 

libnvme/src/nvme/private.h

@@ -7,6 +7,7 @@
  */
 #pragma once
 
+#include <errno.h>
 #include <ifaddrs.h>
 #include <poll.h>
 
@@ -286,10 +287,16 @@ struct nvme_global_ctx {
 	struct nvme_log log;
 	bool mi_probe_enabled;
 	bool ioctl_probing;
+	bool uring_enabled;
 	bool create_only;
 	bool dry_run;
 	struct nvme_fabric_options *options;
 	struct ifaddrs *ifaddrs_cache; /* init with nvme_getifaddrs() */
+
+#ifdef CONFIG_LIBURING
+	int ring_cmds;
+	struct io_uring *ring;
+#endif
 };
 
 struct nvmf_discovery_ctx {
@@ -780,3 +787,31 @@ void nvme_ns_release_transport_handle(nvme_ns_t n);
  */
 int nvme_mi_admin_admin_passthru(struct nvme_transport_handle *hdl,
 		struct nvme_passthru_cmd *cmd);
+
+#ifdef CONFIG_LIBURING
+int nvme_open_uring( struct nvme_global_ctx *ctx);
+void nvme_close_uring(struct nvme_global_ctx *ctx);
+int nvme_uring_cmd_admin_passthru_async(struct nvme_transport_handle *hdl,
+		struct nvme_passthru_cmd *cmd);
+int nvme_uring_cmd_wait_complete(struct nvme_transport_handle *hdl);
+#else
+static inline int nvme_open_uring( struct nvme_global_ctx *ctx)
+{
+	return -ENOTSUP;
+}
+static inline void nvme_close_uring(struct nvme_global_ctx *ctx)
+{
+}
+static inline int nvme_uring_cmd_admin_passthru_async(
+		struct nvme_transport_handle *hdl,
+		struct nvme_passthru_cmd *cmd)
+{
+	return -ENOTSUP;
+}
+static inline int nvme_uring_cmd_wait_complete(
+		struct nvme_transport_handle *hdl)
+{
+	return -ENOTSUP;
+}
+#endif
+

libnvme/src/nvme/uring.c

@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: LGPL-2.1-or-later
+/*
+ * This file is part of libnvme.
+ * Copyright (c) 2020 Western Digital Corporation or its affiliates.
+ *
+ * Authors: Keith Busch <[email protected]>
+ *	    Chaitanya Kulkarni <[email protected]>
+ *	    Daniel Wagner <[email protected]>
+ */
+#include <liburing.h>
+
+#include <libnvme.h>
+
+#include "private.h"
+
+int nvme_open_uring(struct nvme_global_ctx *ctx)
+{
+	struct io_uring_probe *probe;
+	struct io_uring *ring;
+
+	probe = io_uring_get_probe();
+	if (!probe)
+		return -ENOTSUP;
+
+	if (!io_uring_opcode_supported(probe, IORING_OP_URING_CMD))
+		return -ENOTSUP;
+
+	ring = calloc(1, sizeof(*ring));
+	if (!ring)
+		return -ENOMEM;
+
+	if (io_uring_queue_init(NVME_URING_ENTRIES, ring,
+			IORING_SETUP_SQE128 | IORING_SETUP_CQE32)) {
+		free(ring);
+		return -errno;
+	}
+
+	ctx->ring = ring;
+	return 0;
+}
+
+void nvme_close_uring(struct nvme_global_ctx *ctx)
+{
+	if (!ctx->ring)
+		return;
+
+	io_uring_queue_exit(ctx->ring);
+	free(ctx->ring);
+}
+
+static int __nvme_uring_cmd_admin_passthru_async(struct io_uring *ring, int fd,
+		struct nvme_passthru_cmd *cmd)
+{
+	struct io_uring_sqe *sqe;
+	int ret;
+
+	sqe = io_uring_get_sqe(ring);
+	if (!sqe)
+		return -1;
+
+	memcpy(&sqe->cmd, cmd, sizeof(*cmd));
+
+	sqe->fd = fd;
+	sqe->opcode = IORING_OP_URING_CMD;
+	sqe->cmd_op = NVME_URING_CMD_ADMIN;
+
+	ret = io_uring_submit(ring);
+	if (ret < 0)
+		return -errno;
+
+	return 0;
+}
+
+int nvme_uring_cmd_wait_complete(struct nvme_transport_handle *hdl)
+{
+	struct io_uring_cqe *cqe;
+	struct io_uring *ring;
+	int err;
+
+	ring = hdl->ctx->ring;
+
+	for (int i = 0; i < hdl->ctx->ring_cmds; i++) {
+		err = io_uring_wait_cqe(ring, &cqe);
+		if (err < 0)
+			return -errno;
+		io_uring_cqe_seen(ring, cqe);
+	}
+	hdl->ctx->ring_cmds = 0;
+
+	return 0;
+}
+
+int nvme_uring_cmd_admin_passthru_async(struct nvme_transport_handle *hdl,
+		 struct nvme_passthru_cmd *cmd)
+{
+	int err;
+
+	if (hdl->ctx->ring_cmds >= NVME_URING_ENTRIES) {
+		err = nvme_uring_cmd_wait_complete(hdl);
+		if (err)
+			return err;
+	}
+
+	hdl->ctx->ring_cmds += 1;
+	return __nvme_uring_cmd_admin_passthru_async(hdl->ctx->ring,
+		 hdl->fd, cmd);
+}