@@ -1509,10 +1509,16 @@ enum auto_buf_reg_res {
15091509 AUTO_BUF_REG_OK ,
15101510};
15111511
1512- static void ublk_prep_auto_buf_reg_io (const struct ublk_queue * ubq ,
1513- struct request * req , struct ublk_io * io ,
1514- struct io_uring_cmd * cmd ,
1515- enum auto_buf_reg_res res )
1512+ /*
1513+ * Setup io state after auto buffer registration.
1514+ *
1515+ * Must be called after ublk_auto_buf_register() is done.
1516+ * Caller must hold io->lock in batch context.
1517+ */
1518+ static void ublk_auto_buf_io_setup (const struct ublk_queue * ubq ,
1519+ struct request * req , struct ublk_io * io ,
1520+ struct io_uring_cmd * cmd ,
1521+ enum auto_buf_reg_res res )
15161522{
15171523 if (res == AUTO_BUF_REG_OK ) {
15181524 io -> task_registered_buffers = 1 ;
@@ -1523,8 +1529,9 @@ static void ublk_prep_auto_buf_reg_io(const struct ublk_queue *ubq,
15231529 __ublk_prep_compl_io_cmd (io , req );
15241530}
15251531
1532+ /* Register request bvec to io_uring for auto buffer registration. */
15261533static enum auto_buf_reg_res
1527- __ublk_do_auto_buf_reg (const struct ublk_queue * ubq , struct request * req ,
1534+ ublk_auto_buf_register (const struct ublk_queue * ubq , struct request * req ,
15281535 struct ublk_io * io , struct io_uring_cmd * cmd ,
15291536 unsigned int issue_flags )
15301537{
@@ -1544,15 +1551,21 @@ __ublk_do_auto_buf_reg(const struct ublk_queue *ubq, struct request *req,
15441551 return AUTO_BUF_REG_OK ;
15451552}
15461553
1547- static void ublk_do_auto_buf_reg (const struct ublk_queue * ubq , struct request * req ,
1548- struct ublk_io * io , struct io_uring_cmd * cmd ,
1549- unsigned int issue_flags )
1554+ /*
1555+ * Dispatch IO to userspace with auto buffer registration.
1556+ *
1557+ * Only called in non-batch context from task work, io->lock not held.
1558+ */
1559+ static void ublk_auto_buf_dispatch (const struct ublk_queue * ubq ,
1560+ struct request * req , struct ublk_io * io ,
1561+ struct io_uring_cmd * cmd ,
1562+ unsigned int issue_flags )
15501563{
1551- enum auto_buf_reg_res res = __ublk_do_auto_buf_reg (ubq , req , io , cmd ,
1564+ enum auto_buf_reg_res res = ublk_auto_buf_register (ubq , req , io , cmd ,
15521565 issue_flags );
15531566
15541567 if (res != AUTO_BUF_REG_FAIL ) {
1555- ublk_prep_auto_buf_reg_io (ubq , req , io , cmd , res );
1568+ ublk_auto_buf_io_setup (ubq , req , io , cmd , res );
15561569 io_uring_cmd_done (cmd , UBLK_IO_RES_OK , issue_flags );
15571570 }
15581571}
@@ -1627,7 +1640,7 @@ static void ublk_dispatch_req(struct ublk_queue *ubq, struct request *req)
16271640 return ;
16281641
16291642 if (ublk_support_auto_buf_reg (ubq ) && ublk_rq_has_data (req )) {
1630- ublk_do_auto_buf_reg (ubq , req , io , io -> cmd , issue_flags );
1643+ ublk_auto_buf_dispatch (ubq , req , io , io -> cmd , issue_flags );
16311644 } else {
16321645 ublk_init_req_ref (ubq , io );
16331646 ublk_complete_io_cmd (io , req , UBLK_IO_RES_OK , issue_flags );
@@ -1648,15 +1661,15 @@ static bool __ublk_batch_prep_dispatch(struct ublk_queue *ubq,
16481661 return false;
16491662
16501663 if (ublk_support_auto_buf_reg (ubq ) && ublk_rq_has_data (req )) {
1651- res = __ublk_do_auto_buf_reg (ubq , req , io , cmd ,
1664+ res = ublk_auto_buf_register (ubq , req , io , cmd ,
16521665 data -> issue_flags );
16531666
16541667 if (res == AUTO_BUF_REG_FAIL )
16551668 return false;
16561669 }
16571670
16581671 ublk_io_lock (io );
1659- ublk_prep_auto_buf_reg_io (ubq , req , io , cmd , res );
1672+ ublk_auto_buf_io_setup (ubq , req , io , cmd , res );
16601673 ublk_io_unlock (io );
16611674
16621675 return true;
0 commit comments