mi-mctp: Add support for More Processing Required responses

This change allows the mi-mctp transport to receive "more processing
required" responses from MI commands. When we detect a MPR response, we
keep listening for a subsequent message containing the actual response.

The NVMe-MI spec allows for only one MPR response per request.

To do this, we need to perform explicit tag control for MCTP messaging;
we cannot use the kernel's default tag handling, which would expire the
tag value once we've received the first (MPR) response. Instead, we use
the tag control ioctl()s to allocate tags, and expire once we receive
the final response.

Signed-off-by: Jeremy Kerr <[email protected]>

Author: jk-ozlabs

src/nvme/mi-mctp.c

@@ -7,6 +7,7 @@
  */
 
 #include <errno.h>
+#include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -84,10 +85,16 @@ struct nvme_mi_transport_mctp {
 	int	sd;
 };
 
+static int ioctl_tag(int sd, unsigned long req, struct mctp_ioc_tag_ctl *ctl)
+{
+	return ioctl(sd, req, ctl);
+}
+
 static struct __mi_mctp_socket_ops ops = {
 	socket,
 	sendmsg,
 	recvmsg,
+	ioctl_tag,
 };
 
 void __nvme_mi_mctp_set_ops(const struct __mi_mctp_socket_ops *newops)
@@ -96,6 +103,105 @@ void __nvme_mi_mctp_set_ops(const struct __mi_mctp_socket_ops *newops)
 }
 static const struct nvme_mi_transport nvme_mi_transport_mctp;
 
+#ifdef SIOCMCTPALLOCTAG
+static __u8 nvme_mi_mctp_tag_alloc(struct nvme_mi_ep *ep)
+{
+	struct nvme_mi_transport_mctp *mctp;
+	struct mctp_ioc_tag_ctl ctl = { 0 };
+	static bool logged;
+	int rc;
+
+	mctp = ep->transport_data;
+
+	ctl.peer_addr = mctp->eid;
+
+	errno = 0;
+	rc = ops.ioctl_tag(mctp->sd, SIOCMCTPALLOCTAG, &ctl);
+	if (rc) {
+		if (!logged) {
+			/* not necessarily fatal, just means we can't handle
+			 * "more processing required" messages */
+			nvme_msg(ep->root, LOG_INFO,
+				 "System does not support explicit tag allocation\n");
+			logged = true;
+		}
+		return MCTP_TAG_OWNER;
+	}
+
+	return ctl.tag;
+}
+
+static void nvme_mi_mctp_tag_drop(struct nvme_mi_ep *ep, __u8 tag)
+{
+	struct nvme_mi_transport_mctp *mctp;
+	struct mctp_ioc_tag_ctl ctl = { 0 };
+
+	mctp = ep->transport_data;
+
+	if (!(tag & MCTP_TAG_PREALLOC))
+		return;
+
+	ctl.peer_addr = mctp->eid;
+	ctl.tag = tag;
+
+	ops.ioctl_tag(mctp->sd, SIOCMCTPDROPTAG, &ctl);
+}
+
+#else /*  !defined SIOMCTPTAGALLOC */
+
+static __u8 nvme_mi_mctp_tag_alloc(struct nvme_mi_ep *ep)
+{
+	static bool logged;
+	if (!logged) {
+		nvme_msg(ep->root, LOG_INFO,
+			 "Build does not support explicit tag allocation\n");
+		logged = true;
+	}
+	return MCTP_TAG_OWNER;
+}
+
+static void nvme_mi_mctp_tag_drop(struct nvme_mi_ep *ep, __u8 tag)
+{
+}
+
+#endif /* !defined SIOMCTPTAGALLOC */
+
+static bool nvme_mi_mctp_resp_is_mpr(struct nvme_mi_resp *resp, size_t len)
+{
+	struct nvme_mi_msg_resp *msg;
+	__le32 mic;
+	__u32 crc;
+
+	if (len != sizeof(*msg) + sizeof(mic))
+		return false;
+
+	msg = (struct nvme_mi_msg_resp *)resp->hdr;
+
+	if (msg->status != NVME_MI_RESP_MPR)
+		return false;
+
+	/* We can't use verify_resp_mic here, as the response structure has
+	 * not been laid-out properly in resp yet (this is deferred until
+	 * we have the actual response).
+	 *
+	 * We know the data is a fixed size, and linear in the hdr buf, so
+	 * calculation is fairly simple. We do need to find the MIC data
+	 * though, which could either be in the header buf (if the original
+	 * header was larger than the minimal header message), or the start of
+	 * the data buf (otherwise).
+	 */
+	if (resp->hdr_len > sizeof(*msg))
+		mic = *(__le32 *)(msg + 1);
+	else
+		mic = *(__le32 *)(resp->data);
+
+	crc = ~nvme_mi_crc32_update(0xffffffff, msg, sizeof(*msg));
+	if (le32_to_cpu(mic) != crc)
+		return false;
+
+	return true;
+}
+
 static int nvme_mi_mctp_submit(struct nvme_mi_ep *ep,
 			       struct nvme_mi_req *req,
 			       struct nvme_mi_resp *resp)
@@ -106,19 +212,25 @@ static int nvme_mi_mctp_submit(struct nvme_mi_ep *ep,
 	struct sockaddr_mctp addr;
 	ssize_t len;
 	__le32 mic;
-	int i;
+	int i, rc;
+	__u8 tag;
 
 	if (ep->transport != &nvme_mi_transport_mctp)
 		return -EINVAL;
 
+	/* we need enough space for at least a generic (/error) response */
+	if (resp->hdr_len < sizeof(struct nvme_mi_msg_resp))
+		return -EINVAL;
+
 	mctp = ep->transport_data;
+	tag = nvme_mi_mctp_tag_alloc(ep);
 
 	memset(&addr, 0, sizeof(addr));
 	addr.smctp_family = AF_MCTP;
 	addr.smctp_network = mctp->net;
 	addr.smctp_addr.s_addr = mctp->eid;
 	addr.smctp_type = MCTP_TYPE_NVME | MCTP_TYPE_MIC;
-	addr.smctp_tag = MCTP_TAG_OWNER;
+	addr.smctp_tag = tag;
 
 	i = 0;
 	req_iov[i].iov_base = ((__u8 *)req->hdr) + 1;
@@ -146,7 +258,8 @@ static int nvme_mi_mctp_submit(struct nvme_mi_ep *ep,
 	if (len < 0) {
 		nvme_msg(ep->root, LOG_ERR,
 			 "Failure sending MCTP message: %m\n");
-		return len;
+		rc = len;
+		goto out;
 	}
 
 	resp_iov[0].iov_base = ((__u8 *)resp->hdr) + 1;
@@ -164,17 +277,20 @@ static int nvme_mi_mctp_submit(struct nvme_mi_ep *ep,
 	resp_msg.msg_iov = resp_iov;
 	resp_msg.msg_iovlen = 3;
 
+retry:
+	rc = -1;
 	len = ops.recvmsg(mctp->sd, &resp_msg, 0);
 
 	if (len < 0) {
 		nvme_msg(ep->root, LOG_ERR,
 			 "Failure receiving MCTP message: %m\n");
-		return len;
+		goto out;
 	}
 
+
 	if (len == 0) {
 		nvme_msg(ep->root, LOG_WARNING, "No data from MCTP endpoint\n");
-		return -1;
+		goto out;
 	}
 
 	/* Re-add the type byte, so we can work on aligned lengths from here */
@@ -188,15 +304,28 @@ static int nvme_mi_mctp_submit(struct nvme_mi_ep *ep,
 		nvme_msg(ep->root, LOG_ERR,
 			 "Invalid MCTP response: too short (%zd bytes, needed %zd)\n",
 			 len, 8 + sizeof(mic));
-		return -EIO;
+		goto out;
 	}
 
 	/* We can't have header/payload data that isn't a multiple of 4 bytes */
 	if (len & 0x3) {
 		nvme_msg(ep->root, LOG_WARNING,
 			 "Response message has unaligned length (%zd)!\n",
 			 len);
-		return -EIO;
+		goto out;
+	}
+
+	/* Check for a More Processing Required response. This is a slight
+	 * layering violation, as we're pre-checking the MIC and inspecting
+	 * header fields. However, we need to do this in the transport in order
+	 * to keep the tag allocated and retry the recvmsg
+	 */
+	if (nvme_mi_mctp_resp_is_mpr(resp, len)) {
+		nvme_msg(ep->root, LOG_DEBUG,
+			 "Received More Processing Required, waiting for response\n");
+		/* TODO: when we implement timeouts, inspect the MPR response
+		 * for the estimated completion time. */
+		goto retry;
 	}
 
 	/* If we have a shorter than expected response, we need to find the
@@ -226,7 +355,12 @@ static int nvme_mi_mctp_submit(struct nvme_mi_ep *ep,
 
 	resp->mic = le32_to_cpu(mic);
 
-	return 0;
+	rc = 0;
+
+out:
+	nvme_mi_mctp_tag_drop(ep, tag);
+
+	return rc;
 }
 
 static void nvme_mi_mctp_close(struct nvme_mi_ep *ep)

src/nvme/mi.h

@@ -106,6 +106,28 @@ enum nvme_mi_ror {
 	NVME_MI_ROR_RSP = 1,
 };
 
+/**
+ * enum nvme_mi_resp_status - values for the response status field
+ * @NVME_MI_RESP_SUCCESS: success
+ * @NVME_MI_RESP_MPR: More Processing Required
+ * @NVME_MI_RESP_INTERNAL_ERR: Internal Error
+ * @NVME_MI_RESP_INVALID_OPCODE: Invalid command opcode
+ * @NVME_MI_RESP_INVALID_PARAM: Invalid command parameter
+ * @NVME_MI_RESP_INVALID_CMD_SIZE: Invalid command size
+ * @NVME_MI_RESP_INVALID_INPUT_SIZE: Invalid command input data size
+ * @NVME_MI_RESP_ACCESS_DENIED: Access Denied
+ */
+enum nvme_mi_resp_status {
+	NVME_MI_RESP_SUCCESS = 0x00,
+	NVME_MI_RESP_MPR = 0x01,
+	NVME_MI_RESP_INTERNAL_ERR = 0x02,
+	NVME_MI_RESP_INVALID_OPCODE = 0x03,
+	NVME_MI_RESP_INVALID_PARAM = 0x04,
+	NVME_MI_RESP_INVALID_CMD_SIZE = 0x05,
+	NVME_MI_RESP_INVALID_INPUT_SIZE = 0x06,
+	NVME_MI_RESP_ACCESS_DENIED = 0x07,
+};
+
 /**
  * struct nvme_mi_msg_hdr - General MI message header.
  * @type: MCTP message type, will always be NVME_MI_MSGTYPE_NVME
@@ -123,6 +145,21 @@ struct nvme_mi_msg_hdr {
 	__u8	rsvd0;
 } __attribute__((packed));
 
+/**
+ * struct nvme_mi_msg_resp - Generic response type.
+ * @hdr: the general request/response message header
+ * @status: response status value (see &enum nvme_mi_resp_status)
+ * @rsvd0: reserved data, may be defined by specific response
+ *
+ * Every response will start with one of these; command-specific responses
+ * will define parts of the reserved data, and may add further fields.
+ */
+struct nvme_mi_msg_resp {
+	struct nvme_mi_msg_hdr hdr;
+	__u8	status;
+	__u8	rsvd0[3];
+};
+
 /**
  * enum nvme_mi_mi_opcode - Operation code for supported NVMe-MI commands.
  * @nvme_mi_mi_opcode_mi_data_read: Read NVMe-MI Data Structure

src/nvme/private.h

@@ -208,10 +208,12 @@ __u32 nvme_mi_crc32_update(__u32 crc, void *data, size_t len);
 /* we have a facility to mock MCTP socket operations in the mi-mctp transport,
  * using this ops type. This should only be used for test, and isn't exposed
  * in the shared lib */;
+struct mctp_ioc_tag_ctl;
 struct __mi_mctp_socket_ops {
 	int (*socket)(int, int, int);
 	ssize_t (*sendmsg)(int, const struct msghdr *, int);
 	ssize_t (*recvmsg)(int, struct msghdr *, int);
+	int (*ioctl_tag)(int, unsigned long, struct mctp_ioc_tag_ctl *);
 };
 void __nvme_mi_mctp_set_ops(const struct __mi_mctp_socket_ops *newops);