mirros/rockchip-kernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

io_uring: assign new io_identity for task if members have changed

Browse Source 
This avoids doing a copy for each new async IO, if some parts of the
io_identity has changed. We avoid reference counting for the normal
fast path of nothing ever changing.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe
2020-10-15 17:38:03 -06:00
parent 5c3462cfd1
commit 500a373d73
2 changed files with 17 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
fs/io_uring.c
View File

@@ -1066,12 +1066,18 @@ static void io_init_identity(struct io_identity *id)
*/ */
static inline void io_req_init_async(struct io_kiocb *req) static inline void io_req_init_async(struct io_kiocb *req)
{ {
struct io_uring_task *tctx = current->io_uring;
if (req->flags & REQ_F_WORK_INITIALIZED) if (req->flags & REQ_F_WORK_INITIALIZED)
return; return;
memset(&req->work, 0, sizeof(req->work)); memset(&req->work, 0, sizeof(req->work));
req->flags |= REQ_F_WORK_INITIALIZED; req->flags |= REQ_F_WORK_INITIALIZED;
req->work.identity = &current->io_uring->identity;
/* Grab a ref if this isn't our static identity */
req->work.identity = tctx->identity;
if (tctx->identity != &tctx->__identity)
refcount_inc(&req->work.identity->count);
} }
static inline bool io_async_submit(struct io_ring_ctx *ctx) static inline bool io_async_submit(struct io_ring_ctx *ctx)
@@ -1179,7 +1185,7 @@ static void __io_commit_cqring(struct io_ring_ctx *ctx)
static void io_put_identity(struct io_uring_task *tctx, struct io_kiocb *req) static void io_put_identity(struct io_uring_task *tctx, struct io_kiocb *req)
{ {
if (req->work.identity == &tctx->identity) if (req->work.identity == &tctx->__identity)
return; return;
if (refcount_dec_and_test(&req->work.identity->count)) if (refcount_dec_and_test(&req->work.identity->count))
kfree(req->work.identity); kfree(req->work.identity);
@@ -1254,11 +1260,12 @@ static bool io_identity_cow(struct io_kiocb *req)
refcount_inc(&id->count); refcount_inc(&id->count);
/* drop old identity, assign new one. one ref for req, one for tctx */ /* drop old identity, assign new one. one ref for req, one for tctx */
if (req->work.identity != &tctx->identity && if (req->work.identity != tctx->identity &&
refcount_sub_and_test(2, &req->work.identity->count)) refcount_sub_and_test(2, &req->work.identity->count))
kfree(req->work.identity); kfree(req->work.identity);
req->work.identity = id; req->work.identity = id;
tctx->identity = id;
return true; return true;
} }
@@ -7691,7 +7698,8 @@ static int io_uring_alloc_task_context(struct task_struct *task)
tctx->in_idle = 0; tctx->in_idle = 0;
atomic_long_set(&tctx->req_issue, 0); atomic_long_set(&tctx->req_issue, 0);
atomic_long_set(&tctx->req_complete, 0); atomic_long_set(&tctx->req_complete, 0);
io_init_identity(&tctx->identity); io_init_identity(&tctx->__identity);
tctx->identity = &tctx->__identity;
task->io_uring = tctx; task->io_uring = tctx;
return 0; return 0;
} }
@@ -7701,6 +7709,9 @@ void __io_uring_free(struct task_struct *tsk)
struct io_uring_task *tctx = tsk->io_uring; struct io_uring_task *tctx = tsk->io_uring;
WARN_ON_ONCE(!xa_empty(&tctx->xa)); WARN_ON_ONCE(!xa_empty(&tctx->xa));
WARN_ON_ONCE(refcount_read(&tctx->identity->count) != 1);
if (tctx->identity != &tctx->__identity)
kfree(tctx->identity);
kfree(tctx); kfree(tctx);
tsk->io_uring = NULL; tsk->io_uring = NULL;
} }

3
include/linux/io_uring.h
View File

@@ -24,7 +24,8 @@ struct io_uring_task {
struct wait_queue_head wait; struct wait_queue_head wait;
struct file *last; struct file *last;
atomic_long_t req_issue; atomic_long_t req_issue;
struct io_identity identity; struct io_identity __identity;
struct io_identity *identity;
/* completion side */ /* completion side */
bool in_idle ____cacheline_aligned_in_smp; bool in_idle ____cacheline_aligned_in_smp;