From 3d460ec444940d50f4d87d023a660df31d7963bc Mon Sep 17 00:00:00 2001 From: Shin'ichiro Kawasaki Date: Mon, 9 Jun 2025 20:58:41 +0900 Subject: [PATCH 1/9] adding ci files --- .github/workflows/kernel_build.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/kernel_build.yml diff --git a/.github/workflows/kernel_build.yml b/.github/workflows/kernel_build.yml new file mode 100644 index 0000000000000..cd0f9d145429c --- /dev/null +++ b/.github/workflows/kernel_build.yml @@ -0,0 +1,28 @@ +name: blktests-ci + +on: + pull_request: + +jobs: + build-kernel: + runs-on: ubuntu-latest + steps: + - name: Configure git + run: | + git config --global --add safe.directory '*' + - name: Checkout git + run: | + sudo apt-get install -y libelf-dev + mkdir -p linux + cd linux + git init + git remote add origin https://github.com/${{ github.repository }} + git fetch origin --depth=5 ${{ github.event.pull_request.head.sha }} + git reset --hard ${{ github.event.pull_request.head.sha }} + git log -1 + - name: Build kernel + run: | + cd linux + make defconfig + make -j 8 + From 506f6d3ef41a619af9f2e9b36190127486c83da9 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Fri, 6 Jun 2025 15:40:04 -0600 Subject: [PATCH 2/9] ublk: check cmd_op first In preparation for skipping some of the other checks for certain IO opcodes, move the cmd_op check earlier. Signed-off-by: Caleb Sander Mateos Reviewed-by: Ming Lei --- drivers/block/ublk_drv.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index c637ea010d347..e7e2163dcb3b0 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -2177,12 +2177,17 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, struct ublk_io *io; u32 cmd_op = cmd->cmd_op; unsigned tag = ub_cmd->tag; - int ret = -EINVAL; + int ret; pr_devel("%s: received: cmd op %d queue %d tag %d result %d\n", __func__, cmd->cmd_op, ub_cmd->q_id, tag, ub_cmd->result); + ret = ublk_check_cmd_op(cmd_op); + if (ret) + goto out; + + ret = -EINVAL; if (ub_cmd->q_id >= ub->dev_info.nr_hw_queues) goto out; @@ -2215,11 +2220,6 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, ^ (_IOC_NR(cmd_op) == UBLK_IO_NEED_GET_DATA)) goto out; - ret = ublk_check_cmd_op(cmd_op); - if (ret) - goto out; - - ret = -EINVAL; switch (_IOC_NR(cmd_op)) { case UBLK_IO_REGISTER_IO_BUF: return ublk_register_io_buf(cmd, ubq, tag, ub_cmd->addr, issue_flags); From cfc1eced162784177105f3a93d5059b62373d2b0 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Fri, 6 Jun 2025 15:40:05 -0600 Subject: [PATCH 3/9] ublk: handle UBLK_IO_FETCH_REQ first Check for UBLK_IO_FETCH_REQ first in __ublk_ch_uring_cmd() and return early. This allows removing the allowances for NULL io->task and UBLK_IO_FLAG_OWNED_BY_SRV unset that only apply to FETCH. Signed-off-by: Caleb Sander Mateos Reviewed-by: Ming Lei --- drivers/block/ublk_drv.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index e7e2163dcb3b0..c4598f99be717 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -2197,8 +2197,18 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, goto out; io = &ubq->ios[tag]; + /* UBLK_IO_FETCH_REQ can be handled on any task, which sets io->task */ + if (unlikely(_IOC_NR(cmd_op) == UBLK_IO_FETCH_REQ)) { + ret = ublk_fetch(cmd, ubq, io, ub_cmd->addr); + if (ret) + goto out; + + ublk_prep_cancel(cmd, issue_flags, ubq, tag); + return -EIOCBQUEUED; + } + task = READ_ONCE(io->task); - if (task && task != current) + if (task != current) goto out; /* there is pending io cmd, something must be wrong */ @@ -2207,9 +2217,7 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, goto out; } - /* only UBLK_IO_FETCH_REQ is allowed if io is not OWNED_BY_SRV */ - if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV) && - _IOC_NR(cmd_op) != UBLK_IO_FETCH_REQ) + if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)) goto out; /* @@ -2225,11 +2233,6 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, return ublk_register_io_buf(cmd, ubq, tag, ub_cmd->addr, issue_flags); case UBLK_IO_UNREGISTER_IO_BUF: return ublk_unregister_io_buf(cmd, ubq, ub_cmd->addr, issue_flags); - case UBLK_IO_FETCH_REQ: - ret = ublk_fetch(cmd, ubq, io, ub_cmd->addr); - if (ret) - goto out; - break; case UBLK_IO_COMMIT_AND_FETCH_REQ: ret = ublk_commit_and_fetch(ubq, io, cmd, ub_cmd, issue_flags); if (ret) From bb9d08e66e979b3300154da54968cd169e1145e6 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Fri, 6 Jun 2025 15:40:06 -0600 Subject: [PATCH 4/9] ublk: remove task variable from __ublk_ch_uring_cmd() The variable is computed from a simple expression and used once, so just replace it with the expression. Signed-off-by: Caleb Sander Mateos Reviewed-by: Ming Lei --- drivers/block/ublk_drv.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index c4598f99be717..76148145d8fe5 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -2172,7 +2172,6 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, const struct ublksrv_io_cmd *ub_cmd) { struct ublk_device *ub = cmd->file->private_data; - struct task_struct *task; struct ublk_queue *ubq; struct ublk_io *io; u32 cmd_op = cmd->cmd_op; @@ -2207,8 +2206,7 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, return -EIOCBQUEUED; } - task = READ_ONCE(io->task); - if (task != current) + if (READ_ONCE(io->task) != current) goto out; /* there is pending io cmd, something must be wrong */ From a9c49c6510d332eed68af7f5a29ecc3b49f9955c Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Fri, 6 Jun 2025 15:40:07 -0600 Subject: [PATCH 5/9] ublk: consolidate UBLK_IO_FLAG_{ACTIVE,OWNED_BY_SRV} checks UBLK_IO_FLAG_ACTIVE and UBLK_IO_FLAG_OWNED_BY_SRV are mutually exclusive. So just check that UBLK_IO_FLAG_OWNED_BY_SRV is set in __ublk_ch_uring_cmd(); that implies UBLK_IO_FLAG_ACTIVE is unset. Signed-off-by: Caleb Sander Mateos Reviewed-by: Ming Lei --- drivers/block/ublk_drv.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 76148145d8fe5..295beb6ab4a5d 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -2210,14 +2210,11 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, goto out; /* there is pending io cmd, something must be wrong */ - if (io->flags & UBLK_IO_FLAG_ACTIVE) { + if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)) { ret = -EBUSY; goto out; } - if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)) - goto out; - /* * ensure that the user issues UBLK_IO_NEED_GET_DATA * iff the driver have set the UBLK_IO_FLAG_NEED_GET_DATA. From 9ce5505623f87abd279d9f41b742415af9d75359 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Fri, 6 Jun 2025 15:40:08 -0600 Subject: [PATCH 6/9] ublk: move ublk_prep_cancel() to case UBLK_IO_COMMIT_AND_FETCH_REQ UBLK_IO_COMMIT_AND_FETCH_REQ is the only one of __ublk_ch_uring_cmd()'s switch cases that doesn't return or goto. Move the logic following the switch into this case so it also returns. Drop the now unneeded default case. Signed-off-by: Caleb Sander Mateos Reviewed-by: Ming Lei --- drivers/block/ublk_drv.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 295beb6ab4a5d..a8030818f74a8 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -2232,18 +2232,16 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, ret = ublk_commit_and_fetch(ubq, io, cmd, ub_cmd, issue_flags); if (ret) goto out; - break; + + ublk_prep_cancel(cmd, issue_flags, ubq, tag); + return -EIOCBQUEUED; case UBLK_IO_NEED_GET_DATA: io->addr = ub_cmd->addr; if (!ublk_get_data(ubq, io)) return -EIOCBQUEUED; return UBLK_IO_RES_OK; - default: - goto out; } - ublk_prep_cancel(cmd, issue_flags, ubq, tag); - return -EIOCBQUEUED; out: pr_devel("%s: complete: cmd op %d, tag %d ret %x io_flags %x\n", From 16367fe63383536e3b7ab57dafc889222b6988c5 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Fri, 6 Jun 2025 15:40:09 -0600 Subject: [PATCH 7/9] ublk: allow UBLK_IO_(UN)REGISTER_IO_BUF on any task Currently, UBLK_IO_REGISTER_IO_BUF and UBLK_IO_UNREGISTER_IO_BUF are only permitted on the ublk_io's daemon task. But this restriction is unnecessary. ublk_register_io_buf() calls __ublk_check_and_get_req() to look up the request from the tagset and atomically take a reference on the request without accessing the ublk_io. ublk_unregister_io_buf() doesn't use the q_id or tag at all. So allow these opcodes even on tasks other than io->task. Handle UBLK_IO_UNREGISTER_IO_BUF before obtaining the ubq and io since the buffer index being unregistered is not necessarily related to the specified q_id and tag. Add a feature flag UBLK_F_BUF_REG_OFF_DAEMON that userspace can use to determine whether the kernel supports off-daemon buffer registration. Suggested-by: Ming Lei Signed-off-by: Caleb Sander Mateos --- drivers/block/ublk_drv.c | 37 +++++++++++++++++++++-------------- include/uapi/linux/ublk_cmd.h | 8 ++++++++ 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index a8030818f74a8..2084bbdd2cbb9 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -70,7 +70,8 @@ | UBLK_F_UPDATE_SIZE \ | UBLK_F_AUTO_BUF_REG \ | UBLK_F_QUIESCE \ - | UBLK_F_PER_IO_DAEMON) + | UBLK_F_PER_IO_DAEMON \ + | UBLK_F_BUF_REG_OFF_DAEMON) #define UBLK_F_ALL_RECOVERY_FLAGS (UBLK_F_USER_RECOVERY \ | UBLK_F_USER_RECOVERY_REISSUE \ @@ -2020,16 +2021,6 @@ static int ublk_register_io_buf(struct io_uring_cmd *cmd, return 0; } -static int ublk_unregister_io_buf(struct io_uring_cmd *cmd, - const struct ublk_queue *ubq, - unsigned int index, unsigned int issue_flags) -{ - if (!ublk_support_zero_copy(ubq)) - return -EINVAL; - - return io_buffer_unregister_bvec(cmd, index, issue_flags); -} - static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_queue *ubq, struct ublk_io *io, __u64 buf_addr) { @@ -2186,6 +2177,14 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, if (ret) goto out; + /* + * io_buffer_unregister_bvec() doesn't access the ubq or io, + * so no need to validate the q_id, tag, or task + */ + if (_IOC_NR(cmd_op) == UBLK_IO_UNREGISTER_IO_BUF) + return io_buffer_unregister_bvec(cmd, ub_cmd->addr, + issue_flags); + ret = -EINVAL; if (ub_cmd->q_id >= ub->dev_info.nr_hw_queues) goto out; @@ -2206,8 +2205,17 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, return -EIOCBQUEUED; } - if (READ_ONCE(io->task) != current) + if (READ_ONCE(io->task) != current) { + /* + * ublk_register_io_buf() accesses only the request, not io, + * so can be handled on any task + */ + if (_IOC_NR(cmd_op) == UBLK_IO_REGISTER_IO_BUF) + return ublk_register_io_buf(cmd, ubq, tag, ub_cmd->addr, + issue_flags); + goto out; + } /* there is pending io cmd, something must be wrong */ if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)) { @@ -2226,8 +2234,6 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, switch (_IOC_NR(cmd_op)) { case UBLK_IO_REGISTER_IO_BUF: return ublk_register_io_buf(cmd, ubq, tag, ub_cmd->addr, issue_flags); - case UBLK_IO_UNREGISTER_IO_BUF: - return ublk_unregister_io_buf(cmd, ubq, ub_cmd->addr, issue_flags); case UBLK_IO_COMMIT_AND_FETCH_REQ: ret = ublk_commit_and_fetch(ubq, io, cmd, ub_cmd, issue_flags); if (ret) @@ -2920,7 +2926,8 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) ub->dev_info.flags |= UBLK_F_CMD_IOCTL_ENCODE | UBLK_F_URING_CMD_COMP_IN_TASK | - UBLK_F_PER_IO_DAEMON; + UBLK_F_PER_IO_DAEMON | + UBLK_F_BUF_REG_OFF_DAEMON; /* GET_DATA isn't needed any more with USER_COPY or ZERO COPY */ if (ub->dev_info.flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY | diff --git a/include/uapi/linux/ublk_cmd.h b/include/uapi/linux/ublk_cmd.h index 77d9d6af46da8..034497ec3d2a4 100644 --- a/include/uapi/linux/ublk_cmd.h +++ b/include/uapi/linux/ublk_cmd.h @@ -281,6 +281,14 @@ */ #define UBLK_F_PER_IO_DAEMON (1ULL << 13) +/* + * If this feature is set, UBLK_U_IO_REGISTER_IO_BUF/UBLK_U_IO_UNREGISTER_IO_BUF + * can be issued for an I/O on any task. + * If it is unset, zero-copy buffers can only be registered and unregistered by + * the I/O's daemon task. + */ +#define UBLK_F_BUF_REG_OFF_DAEMON (1ULL << 14) + /* device state */ #define UBLK_S_DEV_DEAD 0 #define UBLK_S_DEV_LIVE 1 From 73e4593ebfbf224292cee20f064b34c9c740e280 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Fri, 6 Jun 2025 15:40:10 -0600 Subject: [PATCH 8/9] ublk: optimize UBLK_IO_REGISTER_IO_BUF on daemon task ublk_register_io_buf() performs an expensive atomic refcount increment, as well as a lot of pointer chasing to look up the struct request. Create a separate ublk_daemon_register_io_buf() for the daemon task to call. Initialize ublk_rq_data's reference count to a large number, count the number of buffers registered on the daemon task nonatomically, and atomically subtract the large number minus the number of registered buffers in ublk_commit_and_fetch(). Also obtain the struct request directly from ublk_io's req field instead of looking it up on the tagset. Signed-off-by: Caleb Sander Mateos --- drivers/block/ublk_drv.c | 59 ++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 2084bbdd2cbb9..ec9e0fd21b0ec 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -83,8 +83,16 @@ UBLK_PARAM_TYPE_DEVT | UBLK_PARAM_TYPE_ZONED | \ UBLK_PARAM_TYPE_DMA_ALIGN | UBLK_PARAM_TYPE_SEGMENT) +/* + * Initialize refcount to a large number to include any registered buffers. + * UBLK_IO_COMMIT_AND_FETCH_REQ will release these references minus those for + * any buffers registered on the io daemon task. + */ +#define UBLK_REFCOUNT_INIT (REFCOUNT_MAX / 2) + struct ublk_rq_data { refcount_t ref; + unsigned buffers_registered; /* for auto-unregister buffer in case of UBLK_F_AUTO_BUF_REG */ u16 buf_index; @@ -679,7 +687,8 @@ static inline void ublk_init_req_ref(const struct ublk_queue *ubq, if (ublk_need_req_ref(ubq)) { struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); - refcount_set(&data->ref, 1); + refcount_set(&data->ref, UBLK_REFCOUNT_INIT); + data->buffers_registered = 0; } } @@ -708,6 +717,15 @@ static inline void ublk_put_req_ref(const struct ublk_queue *ubq, } } +static inline void ublk_sub_req_ref(struct request *req) +{ + struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); + unsigned sub_refs = UBLK_REFCOUNT_INIT - data->buffers_registered; + + if (refcount_sub_and_test(sub_refs, &data->ref)) + __ublk_complete_rq(req); +} + static inline bool ublk_need_get_data(const struct ublk_queue *ubq) { return ubq->flags & UBLK_F_NEED_GET_DATA; @@ -1186,10 +1204,8 @@ static void ublk_auto_buf_reg_fallback(struct request *req) { const struct ublk_queue *ubq = req->mq_hctx->driver_data; struct ublksrv_io_desc *iod = ublk_get_iod(ubq, req->tag); - struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); iod->op_flags |= UBLK_IO_F_NEED_REG_BUF; - refcount_set(&data->ref, 1); } static bool ublk_auto_buf_reg(struct request *req, struct ublk_io *io, @@ -1209,9 +1225,8 @@ static bool ublk_auto_buf_reg(struct request *req, struct ublk_io *io, blk_mq_end_request(req, BLK_STS_IOERR); return false; } - /* one extra reference is dropped by ublk_io_release */ - refcount_set(&data->ref, 2); + data->buffers_registered = 1; data->buf_ctx_handle = io_uring_cmd_ctx_handle(io->cmd); /* store buffer index in request payload */ data->buf_index = pdu->buf.index; @@ -1223,10 +1238,10 @@ static bool ublk_prep_auto_buf_reg(struct ublk_queue *ubq, struct request *req, struct ublk_io *io, unsigned int issue_flags) { + ublk_init_req_ref(ubq, req); if (ublk_support_auto_buf_reg(ubq) && ublk_rq_has_data(req)) return ublk_auto_buf_reg(req, io, issue_flags); - ublk_init_req_ref(ubq, req); return true; } @@ -2021,6 +2036,27 @@ static int ublk_register_io_buf(struct io_uring_cmd *cmd, return 0; } +static int ublk_daemon_register_io_buf(struct io_uring_cmd *cmd, + const struct ublk_queue *ubq, + const struct ublk_io *io, + unsigned index, unsigned issue_flags) +{ + struct request *req = io->req; + struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); + int ret; + + if (!ublk_support_zero_copy(ubq) || !ublk_rq_has_data(req)) + return -EINVAL; + + ret = io_buffer_register_bvec(cmd, req, ublk_io_release, index, + issue_flags); + if (ret) + return ret; + + data->buffers_registered++; + return 0; +} + static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_queue *ubq, struct ublk_io *io, __u64 buf_addr) { @@ -2133,9 +2169,13 @@ static int ublk_commit_and_fetch(const struct ublk_queue *ubq, if (req_op(req) == REQ_OP_ZONE_APPEND) req->__sector = ub_cmd->zone_append_lba; - if (likely(!blk_should_fake_timeout(req->q))) - ublk_put_req_ref(ubq, req); + if (unlikely(blk_should_fake_timeout(req->q))) + return 0; + if (ublk_need_req_ref(ubq)) + ublk_sub_req_ref(req); + else + __ublk_complete_rq(req); return 0; } @@ -2233,7 +2273,8 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, switch (_IOC_NR(cmd_op)) { case UBLK_IO_REGISTER_IO_BUF: - return ublk_register_io_buf(cmd, ubq, tag, ub_cmd->addr, issue_flags); + return ublk_daemon_register_io_buf(cmd, ubq, io, ub_cmd->addr, + issue_flags); case UBLK_IO_COMMIT_AND_FETCH_REQ: ret = ublk_commit_and_fetch(ubq, io, cmd, ub_cmd, issue_flags); if (ret) From a949a81b72b3ab7f59829843c9d2cc9ecad11098 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Fri, 6 Jun 2025 15:40:11 -0600 Subject: [PATCH 9/9] ublk: remove ubq checks from ublk_{get,put}_req_ref() ublk_get_req_ref() and ublk_put_req_ref() currently call ublk_need_req_ref(ubq) to check whether the ublk device features require reference counting of its requests. However, all callers already know that reference counting is required: - __ublk_check_and_get_req() is only called from ublk_check_and_get_req() if user copy is enabled, and from ublk_register_io_buf() if zero copy is enabled - ublk_io_release() is only called for requests registered by ublk_register_io_buf(), which requires zero copy - ublk_ch_read_iter() and ublk_ch_write_iter() only call ublk_put_req_ref() if ublk_check_and_get_req() succeeded, which requires user copy to be enabled So drop the ublk_need_req_ref() check and the ubq argument in ublk_get_req_ref() and ublk_put_req_ref(). Signed-off-by: Caleb Sander Mateos --- drivers/block/ublk_drv.c | 41 +++++++++++++--------------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index ec9e0fd21b0ec..f9a6b2abfd207 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -692,29 +692,19 @@ static inline void ublk_init_req_ref(const struct ublk_queue *ubq, } } -static inline bool ublk_get_req_ref(const struct ublk_queue *ubq, - struct request *req) +static inline bool ublk_get_req_ref(struct request *req) { - if (ublk_need_req_ref(ubq)) { - struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); - - return refcount_inc_not_zero(&data->ref); - } + struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); - return true; + return refcount_inc_not_zero(&data->ref); } -static inline void ublk_put_req_ref(const struct ublk_queue *ubq, - struct request *req) +static inline void ublk_put_req_ref(struct request *req) { - if (ublk_need_req_ref(ubq)) { - struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); + struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); - if (refcount_dec_and_test(&data->ref)) - __ublk_complete_rq(req); - } else { + if (refcount_dec_and_test(&data->ref)) __ublk_complete_rq(req); - } } static inline void ublk_sub_req_ref(struct request *req) @@ -2006,9 +1996,8 @@ static inline int ublk_set_auto_buf_reg(struct io_uring_cmd *cmd) static void ublk_io_release(void *priv) { struct request *rq = priv; - struct ublk_queue *ubq = rq->mq_hctx->driver_data; - ublk_put_req_ref(ubq, rq); + ublk_put_req_ref(rq); } static int ublk_register_io_buf(struct io_uring_cmd *cmd, @@ -2029,7 +2018,7 @@ static int ublk_register_io_buf(struct io_uring_cmd *cmd, ret = io_buffer_register_bvec(cmd, req, ublk_io_release, index, issue_flags); if (ret) { - ublk_put_req_ref(ubq, req); + ublk_put_req_ref(req); return ret; } @@ -2305,7 +2294,7 @@ static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub, if (!req) return NULL; - if (!ublk_get_req_ref(ubq, req)) + if (!ublk_get_req_ref(req)) return NULL; if (unlikely(!blk_mq_request_started(req) || req->tag != tag)) @@ -2319,7 +2308,7 @@ static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub, return req; fail_put: - ublk_put_req_ref(ubq, req); + ublk_put_req_ref(req); return NULL; } @@ -2433,13 +2422,12 @@ static struct request *ublk_check_and_get_req(struct kiocb *iocb, *off = buf_off; return req; fail: - ublk_put_req_ref(ubq, req); + ublk_put_req_ref(req); return ERR_PTR(-EACCES); } static ssize_t ublk_ch_read_iter(struct kiocb *iocb, struct iov_iter *to) { - struct ublk_queue *ubq; struct request *req; size_t buf_off; size_t ret; @@ -2449,15 +2437,13 @@ static ssize_t ublk_ch_read_iter(struct kiocb *iocb, struct iov_iter *to) return PTR_ERR(req); ret = ublk_copy_user_pages(req, buf_off, to, ITER_DEST); - ubq = req->mq_hctx->driver_data; - ublk_put_req_ref(ubq, req); + ublk_put_req_ref(req); return ret; } static ssize_t ublk_ch_write_iter(struct kiocb *iocb, struct iov_iter *from) { - struct ublk_queue *ubq; struct request *req; size_t buf_off; size_t ret; @@ -2467,8 +2453,7 @@ static ssize_t ublk_ch_write_iter(struct kiocb *iocb, struct iov_iter *from) return PTR_ERR(req); ret = ublk_copy_user_pages(req, buf_off, from, ITER_SOURCE); - ubq = req->mq_hctx->driver_data; - ublk_put_req_ref(ubq, req); + ublk_put_req_ref(req); return ret; }