Skip to content

Commit 69d6011

Browse files
axboeaxboe
committed
Merge branch 'io_uring-7.1' into for-next
* io_uring-7.1: io_uring/poll: ensure EPOLL_ONESHOT is propagated for EPOLL_URING_WAKE io_uring/zcrx: warn on freelist violations io_uring/zcrx: clear RQ headers on init io_uring/zcrx: fix user_struct uaf io_uring/register: fix ring resizing with mixed/large SQEs/CQEs io_uring/futex: ensure partial wakes are appropriately dequeued io_uring/rw: add defensive hardening for negative kbuf lengths io_uring/rsrc: use kvfree() for the imu cache io_uring/rsrc: unify nospec indexing for direct descriptors io_uring: fix spurious fput in registered ring path io_uring: fix iowq_limits data race in tctx node addition io_uring/tctx: mark io_wq as exiting before error path teardown io_uring/tctx: check for setup tctx->io_wq before teardown io_uring/poll: fix signed comparison in io_poll_get_ownership()
2 parents b6a5f0b + 1967f0b commit 69d6011

10 files changed

Lines changed: 66 additions & 23 deletions

File tree

‎io_uring/alloc_cache.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ static inline void *io_cache_alloc(struct io_alloc_cache *cache, gfp_t gfp)
6464
static inline void io_cache_free(struct io_alloc_cache *cache, void *obj)
6565
{
6666
if (!io_alloc_cache_put(cache, obj))
67-
kfree(obj);
67+
kvfree(obj);
6868
}
6969

7070
#endif

‎io_uring/futex.c‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,8 +159,10 @@ static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q)
159159
struct io_kiocb *req = q->wake_data;
160160
struct io_futexv_data *ifd = req->async_data;
161161

162-
if (!io_futexv_claim(ifd))
162+
if (!io_futexv_claim(ifd)) {
163+
__futex_wake_mark(q);
163164
return;
165+
}
164166
if (unlikely(!__futex_wake_mark(q)))
165167
return;
166168

‎io_uring/io_uring.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2575,7 +2575,8 @@ struct file *io_uring_ctx_get_file(unsigned int fd, bool registered)
25752575
return ERR_PTR(-EBADF);
25762576
if (io_is_uring_fops(file))
25772577
return file;
2578-
fput(file);
2578+
if (!registered)
2579+
fput(file);
25792580
return ERR_PTR(-EOPNOTSUPP);
25802581
}
25812582

‎io_uring/poll.c‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ static bool io_poll_get_ownership_slowpath(struct io_kiocb *req)
9393
*/
9494
static inline bool io_poll_get_ownership(struct io_kiocb *req)
9595
{
96-
if (unlikely(atomic_read(&req->poll_refs) >= IO_POLL_REF_BIAS))
96+
if (unlikely((unsigned int)atomic_read(&req->poll_refs) >= IO_POLL_REF_BIAS))
9797
return io_poll_get_ownership_slowpath(req);
9898
return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK);
9999
}
@@ -417,8 +417,10 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
417417
* disable multishot as there is a circular dependency between
418418
* CQ posting and triggering the event.
419419
*/
420-
if (mask & EPOLL_URING_WAKE)
420+
if (mask & EPOLL_URING_WAKE) {
421421
poll->events |= EPOLLONESHOT;
422+
req->apoll_events |= EPOLLONESHOT;
423+
}
422424

423425
/* optional, saves extra locking for removal in tw handler */
424426
if (mask && poll->events & EPOLLONESHOT) {

‎io_uring/register.c‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -599,10 +599,20 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
599599
if (tail - old_head > p->sq_entries)
600600
goto overflow;
601601
for (i = old_head; i < tail; i++) {
602-
unsigned src_head = i & (ctx->sq_entries - 1);
603-
unsigned dst_head = i & (p->sq_entries - 1);
604-
605-
n.sq_sqes[dst_head] = o.sq_sqes[src_head];
602+
unsigned index, dst_mask, src_mask;
603+
size_t sq_size;
604+
605+
index = i;
606+
sq_size = sizeof(struct io_uring_sqe);
607+
src_mask = ctx->sq_entries - 1;
608+
dst_mask = p->sq_entries - 1;
609+
if (ctx->flags & IORING_SETUP_SQE128) {
610+
index <<= 1;
611+
sq_size <<= 1;
612+
src_mask = (ctx->sq_entries << 1) - 1;
613+
dst_mask = (p->sq_entries << 1) - 1;
614+
}
615+
memcpy(&n.sq_sqes[index & dst_mask], &o.sq_sqes[index & src_mask], sq_size);
606616
}
607617
WRITE_ONCE(n.rings->sq.head, old_head);
608618
WRITE_ONCE(n.rings->sq.tail, tail);
@@ -619,10 +629,20 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
619629
goto out;
620630
}
621631
for (i = old_head; i < tail; i++) {
622-
unsigned src_head = i & (ctx->cq_entries - 1);
623-
unsigned dst_head = i & (p->cq_entries - 1);
624-
625-
n.rings->cqes[dst_head] = o.rings->cqes[src_head];
632+
unsigned index, dst_mask, src_mask;
633+
size_t cq_size;
634+
635+
index = i;
636+
cq_size = sizeof(struct io_uring_cqe);
637+
src_mask = ctx->cq_entries - 1;
638+
dst_mask = p->cq_entries - 1;
639+
if (ctx->flags & IORING_SETUP_CQE32) {
640+
index <<= 1;
641+
cq_size <<= 1;
642+
src_mask = (ctx->cq_entries << 1) - 1;
643+
dst_mask = (p->cq_entries << 1) - 1;
644+
}
645+
memcpy(&n.rings->cqes[index & dst_mask], &o.rings->cqes[index & src_mask], cq_size);
626646
}
627647
WRITE_ONCE(n.rings->cq.head, old_head);
628648
WRITE_ONCE(n.rings->cq.tail, tail);

‎io_uring/rsrc.c‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ bool io_rsrc_cache_init(struct io_ring_ctx *ctx)
168168
void io_rsrc_cache_free(struct io_ring_ctx *ctx)
169169
{
170170
io_alloc_cache_free(&ctx->node_cache, kfree);
171-
io_alloc_cache_free(&ctx->imu_cache, kfree);
171+
io_alloc_cache_free(&ctx->imu_cache, kvfree);
172172
}
173173

174174
static void io_clear_table_tags(struct io_rsrc_data *data)
@@ -238,6 +238,9 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
238238
continue;
239239

240240
i = up->offset + done;
241+
if (i >= ctx->file_table.data.nr)
242+
break;
243+
i = array_index_nospec(i, ctx->file_table.data.nr);
241244
if (io_reset_rsrc_node(ctx, &ctx->file_table.data, i))
242245
io_file_bitmap_clear(&ctx->file_table, i);
243246

‎io_uring/rsrc.h‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,10 +109,15 @@ static inline void io_put_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node
109109
}
110110

111111
static inline bool io_reset_rsrc_node(struct io_ring_ctx *ctx,
112-
struct io_rsrc_data *data, int index)
112+
struct io_rsrc_data *data,
113+
unsigned int index)
113114
{
114-
struct io_rsrc_node *node = data->nodes[index];
115+
struct io_rsrc_node *node;
115116

117+
if (index >= data->nr)
118+
return false;
119+
index = array_index_nospec(index, data->nr);
120+
node = data->nodes[index];
116121
if (!node)
117122
return false;
118123
io_put_rsrc_node(ctx, node);

‎io_uring/rw.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -580,7 +580,7 @@ void io_req_rw_complete(struct io_tw_req tw_req, io_tw_token_t tw)
580580
io_req_io_end(req);
581581

582582
if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING))
583-
req->cqe.flags |= io_put_kbuf(req, req->cqe.res, NULL);
583+
req->cqe.flags |= io_put_kbuf(req, max(req->cqe.res, 0), NULL);
584584

585585
io_req_rw_cleanup(req, 0);
586586
io_req_task_complete(tw_req, tw);
@@ -1379,7 +1379,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)
13791379
list_del(&req->iopoll_node);
13801380
wq_list_add_tail(&req->comp_list, &ctx->submit_state.compl_reqs);
13811381
nr_events++;
1382-
req->cqe.flags = io_put_kbuf(req, req->cqe.res, NULL);
1382+
req->cqe.flags = io_put_kbuf(req, max(req->cqe.res, 0), NULL);
13831383
if (!io_is_uring_cmd(req))
13841384
io_req_rw_cleanup(req, 0);
13851385
}

‎io_uring/tctx.c‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -146,9 +146,13 @@ int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
146146
if (IS_ERR(tctx))
147147
return PTR_ERR(tctx);
148148

149-
if (ctx->int_flags & IO_RING_F_IOWQ_LIMITS_SET) {
150-
unsigned int limits[2] = { ctx->iowq_limits[0],
151-
ctx->iowq_limits[1], };
149+
if (data_race(ctx->int_flags) & IO_RING_F_IOWQ_LIMITS_SET) {
150+
unsigned int limits[2];
151+
152+
mutex_lock(&ctx->uring_lock);
153+
limits[0] = ctx->iowq_limits[0];
154+
limits[1] = ctx->iowq_limits[1];
155+
mutex_unlock(&ctx->uring_lock);
152156

153157
ret = io_wq_max_workers(tctx->io_wq, limits);
154158
if (ret)
@@ -171,7 +175,10 @@ int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
171175
}
172176
if (!current->io_uring) {
173177
err_free:
174-
io_wq_put_and_exit(tctx->io_wq);
178+
if (tctx->io_wq) {
179+
io_wq_exit_start(tctx->io_wq);
180+
io_wq_put_and_exit(tctx->io_wq);
181+
}
175182
percpu_counter_destroy(&tctx->inflight);
176183
kfree(tctx);
177184
}

‎io_uring/zcrx.c‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -396,6 +396,7 @@ static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx,
396396
ifq->rq.ring = (struct io_uring *)ptr;
397397
ifq->rq.rqes = (struct io_uring_zcrx_rqe *)(ptr + off);
398398

399+
memset(ifq->rq.ring, 0, sizeof(*ifq->rq.ring));
399400
return 0;
400401
}
401402

@@ -579,13 +580,13 @@ static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq)
579580

580581
if (ifq->area)
581582
io_zcrx_free_area(ifq, ifq->area);
582-
free_uid(ifq->user);
583583
if (ifq->mm_account)
584584
mmdrop(ifq->mm_account);
585585
if (ifq->dev)
586586
put_device(ifq->dev);
587587

588588
io_free_rbuf_ring(ifq);
589+
free_uid(ifq->user);
589590
mutex_destroy(&ifq->pp_lock);
590591
kfree(ifq);
591592
}
@@ -601,6 +602,8 @@ static void io_zcrx_return_niov_freelist(struct net_iov *niov)
601602
struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
602603

603604
guard(spinlock_bh)(&area->freelist_lock);
605+
if (WARN_ON_ONCE(area->free_count >= area->nia.num_niovs))
606+
return;
604607
area->freelist[area->free_count++] = net_iov_idx(niov);
605608
}
606609

0 commit comments

Comments
 (0)