Skip to content

Commit 7f50856

Browse files
axboeaxboe
committed
Merge branch 'for-6.19/block' into for-next
* for-6.19/block: nvme: remove virtual boundary for sgl capable devices block: accumulate memory segment gaps per bio
2 parents b822e8f + bc840b2 commit 7f50856

17 files changed

Lines changed: 125 additions & 13 deletions

File tree

‎block/bio.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,7 @@ void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table,
253253
bio->bi_write_hint = 0;
254254
bio->bi_write_stream = 0;
255255
bio->bi_status = 0;
256+
bio->bi_bvec_gap_bit = 0;
256257
bio->bi_iter.bi_sector = 0;
257258
bio->bi_iter.bi_size = 0;
258259
bio->bi_iter.bi_idx = 0;

‎block/blk-map.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -459,6 +459,8 @@ int blk_rq_append_bio(struct request *rq, struct bio *bio)
459459
if (rq->bio) {
460460
if (!ll_back_merge_fn(rq, bio, nr_segs))
461461
return -EINVAL;
462+
rq->phys_gap_bit = bio_seg_gap(rq->q, rq->biotail, bio,
463+
rq->phys_gap_bit);
462464
rq->biotail->bi_next = bio;
463465
rq->biotail = bio;
464466
rq->__data_len += bio->bi_iter.bi_size;
@@ -469,6 +471,7 @@ int blk_rq_append_bio(struct request *rq, struct bio *bio)
469471
rq->nr_phys_segments = nr_segs;
470472
rq->bio = rq->biotail = bio;
471473
rq->__data_len = bio->bi_iter.bi_size;
474+
rq->phys_gap_bit = bio->bi_bvec_gap_bit;
472475
return 0;
473476
}
474477
EXPORT_SYMBOL(blk_rq_append_bio);

‎block/blk-merge.c‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,12 @@ static unsigned int bio_split_alignment(struct bio *bio,
302302
return lim->logical_block_size;
303303
}
304304

305+
static inline unsigned int bvec_seg_gap(struct bio_vec *bvprv,
306+
struct bio_vec *bv)
307+
{
308+
return bv->bv_offset | (bvprv->bv_offset + bvprv->bv_len);
309+
}
310+
305311
/**
306312
* bio_split_io_at - check if and where to split a bio
307313
* @bio: [in] bio to be split
@@ -319,8 +325,8 @@ int bio_split_io_at(struct bio *bio, const struct queue_limits *lim,
319325
unsigned *segs, unsigned max_bytes, unsigned len_align_mask)
320326
{
321327
struct bio_vec bv, bvprv, *bvprvp = NULL;
328+
unsigned nsegs = 0, bytes = 0, gaps = 0;
322329
struct bvec_iter iter;
323-
unsigned nsegs = 0, bytes = 0;
324330

325331
bio_for_each_bvec(bv, bio, iter) {
326332
if (bv.bv_offset & lim->dma_alignment ||
@@ -331,8 +337,11 @@ int bio_split_io_at(struct bio *bio, const struct queue_limits *lim,
331337
* If the queue doesn't support SG gaps and adding this
332338
* offset would create a gap, disallow it.
333339
*/
334-
if (bvprvp && bvec_gap_to_prev(lim, bvprvp, bv.bv_offset))
335-
goto split;
340+
if (bvprvp) {
341+
if (bvec_gap_to_prev(lim, bvprvp, bv.bv_offset))
342+
goto split;
343+
gaps |= bvec_seg_gap(bvprvp, &bv);
344+
}
336345

337346
if (nsegs < lim->max_segments &&
338347
bytes + bv.bv_len <= max_bytes &&
@@ -350,6 +359,7 @@ int bio_split_io_at(struct bio *bio, const struct queue_limits *lim,
350359
}
351360

352361
*segs = nsegs;
362+
bio->bi_bvec_gap_bit = ffs(gaps);
353363
return 0;
354364
split:
355365
if (bio->bi_opf & REQ_ATOMIC)
@@ -385,6 +395,7 @@ int bio_split_io_at(struct bio *bio, const struct queue_limits *lim,
385395
* big IO can be trival, disable iopoll when split needed.
386396
*/
387397
bio_clear_polled(bio);
398+
bio->bi_bvec_gap_bit = ffs(gaps);
388399
return bytes >> SECTOR_SHIFT;
389400
}
390401
EXPORT_SYMBOL_GPL(bio_split_io_at);
@@ -721,6 +732,21 @@ static bool blk_atomic_write_mergeable_rqs(struct request *rq,
721732
return (rq->cmd_flags & REQ_ATOMIC) == (next->cmd_flags & REQ_ATOMIC);
722733
}
723734

735+
u8 bio_seg_gap(struct request_queue *q, struct bio *prev, struct bio *next,
736+
u8 gaps_bit)
737+
{
738+
struct bio_vec pb, nb;
739+
740+
gaps_bit = min_not_zero(gaps_bit, prev->bi_bvec_gap_bit);
741+
gaps_bit = min_not_zero(gaps_bit, next->bi_bvec_gap_bit);
742+
743+
bio_get_last_bvec(prev, &pb);
744+
bio_get_first_bvec(next, &nb);
745+
if (!biovec_phys_mergeable(q, &pb, &nb))
746+
gaps_bit = min_not_zero(gaps_bit, ffs(bvec_seg_gap(&pb, &nb)));
747+
return gaps_bit;
748+
}
749+
724750
/*
725751
* For non-mq, this has to be called with the request spinlock acquired.
726752
* For mq with scheduling, the appropriate queue wide lock should be held.
@@ -785,6 +811,9 @@ static struct request *attempt_merge(struct request_queue *q,
785811
if (next->start_time_ns < req->start_time_ns)
786812
req->start_time_ns = next->start_time_ns;
787813

814+
req->phys_gap_bit = bio_seg_gap(req->q, req->biotail, next->bio,
815+
min_not_zero(next->phys_gap_bit,
816+
req->phys_gap_bit));
788817
req->biotail->bi_next = next->bio;
789818
req->biotail = next->biotail;
790819

@@ -908,6 +937,8 @@ enum bio_merge_status bio_attempt_back_merge(struct request *req,
908937
if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING)
909938
blk_zone_write_plug_bio_merged(bio);
910939

940+
req->phys_gap_bit = bio_seg_gap(req->q, req->biotail, bio,
941+
req->phys_gap_bit);
911942
req->biotail->bi_next = bio;
912943
req->biotail = bio;
913944
req->__data_len += bio->bi_iter.bi_size;
@@ -942,6 +973,8 @@ static enum bio_merge_status bio_attempt_front_merge(struct request *req,
942973

943974
blk_update_mixed_merge(req, bio, true);
944975

976+
req->phys_gap_bit = bio_seg_gap(req->q, bio, req->bio,
977+
req->phys_gap_bit);
945978
bio->bi_next = req->bio;
946979
req->bio = bio;
947980

‎block/blk-mq-dma.c‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,7 @@ static bool blk_map_iter_next(struct request *req, struct blk_map_iter *iter,
7979
static inline bool blk_can_dma_map_iova(struct request *req,
8080
struct device *dma_dev)
8181
{
82-
return !((queue_virt_boundary(req->q) + 1) &
83-
dma_get_merge_boundary(dma_dev));
82+
return !(req_phys_gap_mask(req) & dma_get_merge_boundary(dma_dev));
8483
}
8584

8685
static bool blk_dma_map_bus(struct blk_dma_iter *iter, struct phys_vec *vec)

‎block/blk-mq.c‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,7 @@ void blk_rq_init(struct request_queue *q, struct request *rq)
376376
INIT_LIST_HEAD(&rq->queuelist);
377377
rq->q = q;
378378
rq->__sector = (sector_t) -1;
379+
rq->phys_gap_bit = 0;
379380
INIT_HLIST_NODE(&rq->hash);
380381
RB_CLEAR_NODE(&rq->rb_node);
381382
rq->tag = BLK_MQ_NO_TAG;
@@ -668,6 +669,7 @@ struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf,
668669
goto out_queue_exit;
669670
}
670671
rq->__data_len = 0;
672+
rq->phys_gap_bit = 0;
671673
rq->__sector = (sector_t) -1;
672674
rq->bio = rq->biotail = NULL;
673675
return rq;
@@ -748,6 +750,7 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
748750
rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag);
749751
blk_mq_rq_time_init(rq, alloc_time_ns);
750752
rq->__data_len = 0;
753+
rq->phys_gap_bit = 0;
751754
rq->__sector = (sector_t) -1;
752755
rq->bio = rq->biotail = NULL;
753756
return rq;
@@ -2674,6 +2677,8 @@ static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
26742677
rq->bio = rq->biotail = bio;
26752678
rq->__sector = bio->bi_iter.bi_sector;
26762679
rq->__data_len = bio->bi_iter.bi_size;
2680+
rq->phys_gap_bit = bio->bi_bvec_gap_bit;
2681+
26772682
rq->nr_phys_segments = nr_segs;
26782683
if (bio_integrity(bio))
26792684
rq->nr_integrity_segments = blk_rq_count_integrity_sg(rq->q,
@@ -3380,6 +3385,7 @@ int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
33803385
}
33813386
rq->nr_phys_segments = rq_src->nr_phys_segments;
33823387
rq->nr_integrity_segments = rq_src->nr_integrity_segments;
3388+
rq->phys_gap_bit = rq_src->phys_gap_bit;
33833389

33843390
if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0)
33853391
goto free_and_out;

‎drivers/nvme/host/apple.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1283,6 +1283,7 @@ static const struct nvme_ctrl_ops nvme_ctrl_ops = {
12831283
.reg_read64 = apple_nvme_reg_read64,
12841284
.free_ctrl = apple_nvme_free_ctrl,
12851285
.get_address = apple_nvme_get_address,
1286+
.get_virt_boundary = nvme_get_virt_boundary,
12861287
};
12871288

12881289
static void apple_nvme_async_probe(void *data, async_cookie_t cookie)

‎drivers/nvme/host/core.c‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2069,13 +2069,13 @@ static u32 nvme_max_drv_segments(struct nvme_ctrl *ctrl)
20692069
}
20702070

20712071
static void nvme_set_ctrl_limits(struct nvme_ctrl *ctrl,
2072-
struct queue_limits *lim)
2072+
struct queue_limits *lim, bool is_admin)
20732073
{
20742074
lim->max_hw_sectors = ctrl->max_hw_sectors;
20752075
lim->max_segments = min_t(u32, USHRT_MAX,
20762076
min_not_zero(nvme_max_drv_segments(ctrl), ctrl->max_segments));
20772077
lim->max_integrity_segments = ctrl->max_integrity_segments;
2078-
lim->virt_boundary_mask = NVME_CTRL_PAGE_SIZE - 1;
2078+
lim->virt_boundary_mask = ctrl->ops->get_virt_boundary(ctrl, is_admin);
20792079
lim->max_segment_size = UINT_MAX;
20802080
lim->dma_alignment = 3;
20812081
}
@@ -2177,7 +2177,7 @@ static int nvme_update_ns_info_generic(struct nvme_ns *ns,
21772177
int ret;
21782178

21792179
lim = queue_limits_start_update(ns->disk->queue);
2180-
nvme_set_ctrl_limits(ns->ctrl, &lim);
2180+
nvme_set_ctrl_limits(ns->ctrl, &lim, false);
21812181

21822182
memflags = blk_mq_freeze_queue(ns->disk->queue);
21832183
ret = queue_limits_commit_update(ns->disk->queue, &lim);
@@ -2381,7 +2381,7 @@ static int nvme_update_ns_info_block(struct nvme_ns *ns,
23812381
ns->head->lba_shift = id->lbaf[lbaf].ds;
23822382
ns->head->nuse = le64_to_cpu(id->nuse);
23832383
capacity = nvme_lba_to_sect(ns->head, le64_to_cpu(id->nsze));
2384-
nvme_set_ctrl_limits(ns->ctrl, &lim);
2384+
nvme_set_ctrl_limits(ns->ctrl, &lim, false);
23852385
nvme_configure_metadata(ns->ctrl, ns->head, id, nvm, info);
23862386
nvme_set_chunk_sectors(ns, id, &lim);
23872387
if (!nvme_update_disk_info(ns, id, &lim))
@@ -3588,7 +3588,7 @@ static int nvme_init_identify(struct nvme_ctrl *ctrl)
35883588
min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
35893589

35903590
lim = queue_limits_start_update(ctrl->admin_q);
3591-
nvme_set_ctrl_limits(ctrl, &lim);
3591+
nvme_set_ctrl_limits(ctrl, &lim, true);
35923592
ret = queue_limits_commit_update(ctrl->admin_q, &lim);
35933593
if (ret)
35943594
goto out_free;

‎drivers/nvme/host/fabrics.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,12 @@ static inline unsigned int nvmf_nr_io_queues(struct nvmf_ctrl_options *opts)
217217
min(opts->nr_poll_queues, num_online_cpus());
218218
}
219219

220+
static inline unsigned long nvmf_get_virt_boundary(struct nvme_ctrl *ctrl,
221+
bool is_admin)
222+
{
223+
return 0;
224+
}
225+
220226
int nvmf_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val);
221227
int nvmf_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val);
222228
int nvmf_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val);

‎drivers/nvme/host/fc.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3360,6 +3360,7 @@ static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = {
33603360
.submit_async_event = nvme_fc_submit_async_event,
33613361
.delete_ctrl = nvme_fc_delete_ctrl,
33623362
.get_address = nvmf_get_address,
3363+
.get_virt_boundary = nvmf_get_virt_boundary,
33633364
};
33643365

33653366
static void

‎drivers/nvme/host/nvme.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -558,6 +558,12 @@ static inline bool nvme_ns_has_pi(struct nvme_ns_head *head)
558558
return head->pi_type && head->ms == head->pi_size;
559559
}
560560

561+
static inline unsigned long nvme_get_virt_boundary(struct nvme_ctrl *ctrl,
562+
bool is_admin)
563+
{
564+
return NVME_CTRL_PAGE_SIZE - 1;
565+
}
566+
561567
struct nvme_ctrl_ops {
562568
const char *name;
563569
struct module *module;
@@ -578,6 +584,7 @@ struct nvme_ctrl_ops {
578584
int (*get_address)(struct nvme_ctrl *ctrl, char *buf, int size);
579585
void (*print_device_info)(struct nvme_ctrl *ctrl);
580586
bool (*supports_pci_p2pdma)(struct nvme_ctrl *ctrl);
587+
unsigned long (*get_virt_boundary)(struct nvme_ctrl *ctrl, bool is_admin);
581588
};
582589

583590
/*

0 commit comments

Comments
 (0)