libnvme/ioctl: introduce ioctl_io64 nvme transport direct handle

Since currently only NVME_IOCTL_ADMIN64_CMD checked for ioctrl64.
But NVME_IOCTL_IO64_CMD not checked as same.
Also currently NVME_IOCTL_IO64_CMD not supported for the char dev.
Therefore introduce the ioctl_io64 flag to use NVME_IOCTL_IO64_CMD.

Signed-off-by: Tokunori Ikegami <[email protected]>

Author: ikegami-t

libnvme/src/nvme/ioctl.c

@@ -178,7 +178,7 @@ static int nvme_submit_passthru64(struct nvme_transport_handle *hdl,
 int nvme_submit_io_passthru(struct nvme_transport_handle *hdl,
 		struct nvme_passthru_cmd *cmd)
 {
-	if (hdl->ioctl64)
+	if (hdl->ioctl_io64)
 		return nvme_submit_passthru64(hdl, NVME_IOCTL_IO64_CMD, cmd);
 	return nvme_submit_passthru32(hdl, NVME_IOCTL_IO_CMD, cmd);
 }

libnvme/src/nvme/lib.c

@@ -160,8 +160,12 @@ static int __nvme_transport_handle_open_direct(
 
 	if (hdl->ctx->ioctl_probing) {
 		ret = ioctl(hdl->fd, NVME_IOCTL_ADMIN64_CMD, &dummy);
-		if (ret > 0)
+		if (ret > 0) {
 			hdl->ioctl64 = true;
+			ret = ioctl(hdl->fd, NVME_IOCTL_IO64_CMD, &dummy);
+			if (ret != -1 || errno != ENOTTY)
+				hdl->ioctl_io64 = true;
+		}
 	}
 
 	return 0;

libnvme/src/nvme/private.h

@@ -114,6 +114,7 @@ struct nvme_transport_handle {
 	int fd;
 	struct stat stat;
 	bool ioctl64;
+	bool ioctl_io64;
 
 	/* mi */
 	struct nvme_mi_ep *ep;