From bcff91897e40204d32fd31b39fdff529a7eff3fa Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Wed, 11 Nov 2026 12:50:45 +0000 Subject: [PATCH] io_uring/io-wq: apply the current cpuset to new workers Since commit 84eacf177faa ("io_uring/io-wq: inherit cpuset of cgroup in io worker"), io_wq_create() copies the cpuset of the task that creates the io-wq into wq->cpu_mask, and io_init_new_worker() sets the affinity of every new worker to that copy. The copy is never refreshed, but workers are started for as long as the io-wq exists: whenever a request has to block and no worker is free, while idle ones exit after WORKER_IDLE_TIMEOUT. When the cpuset of the cgroup changes, or the task moves to another cgroup, cpuset updates the affinity of the workers that exist. Workers started later get the stale mask, which set_cpus_allowed_ptr() does not check against their cpuset, and run on CPUs their cgroup no longer has. A worker that cpuset moved is also retired at its next idle timeout, even when it is the last one, as it now runs outside wq->cpu_mask, and its replacement starts on the old CPUs. With a task in a cgroup whose cpuset.cpus goes from 2-3 to 4-5 while one request is blocked in a worker, and three more blocking requests issued afterwards: worker started before the change Cpus_allowed_list: 4-5 3 workers started after the change Cpus_allowed_list: 2-3 So io-wq workers keep running on CPUs that were taken away from their cgroup, for instance to give them to latency sensitive tasks. Keep wq->cpu_mask for the affinity of the io-wq itself, all CPUs or the mask set with IORING_REGISTER_IOWQ_AFF, and intersect it with the cpuset of the worker when the worker starts, using the cpuset alone if the two have no CPU in common. That is how the scheduler treats the affinity a task requested when its cpuset changes, see __set_cpus_allowed_ptr(). Use the same mask in the idle timeout test of a worker, and when the registered affinity is removed, go back to all CPUs rather than to the cpuset of that moment. A mask being registered must still lie within the cpuset, as required since commit 0997aa5497c7 ("io_uring/io-wq: do not allow pinning outside of cpuset"). Fixes: 84eacf177faa ("io_uring/io-wq: inherit cpuset of cgroup in io worker") Cc: stable@vger.kernel.org Signed-off-by: Mikhail Malyshev --- io_uring/io-wq.c | 44 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index fd53297af26e..0d74e365d54a 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -731,6 +731,34 @@ static bool io_worker_retire(struct io_wq_acct *acct, struct io_worker *worker) return false; } +/* + * The CPUs a worker of @wq runs on: those of the io-wq affinity that the + * worker's cpuset allows, or the cpuset if they have none in common, as the + * scheduler does with a task's requested affinity when its cpuset changes. + * Read the cpuset each time: it can change while the io-wq exists. + */ +static void io_wq_worker_cpus(struct io_wq *wq, struct task_struct *tsk, + struct cpumask *cpus) +{ + cpuset_cpus_allowed(tsk, cpus); + if (cpumask_intersects(cpus, wq->cpu_mask)) + cpumask_and(cpus, cpus, wq->cpu_mask); +} + +/* Is the current worker on a CPU that a new worker would not get? */ +static bool io_wq_worker_off_cpus(struct io_wq *wq) +{ + cpumask_var_t cpus; + bool off; + + if (!alloc_cpumask_var(&cpus, GFP_KERNEL)) + return false; + io_wq_worker_cpus(wq, current, cpus); + off = !cpumask_test_cpu(raw_smp_processor_id(), cpus); + free_cpumask_var(cpus); + return off; +} + static int io_wq_worker(void *data) { struct io_worker *worker = data; @@ -809,8 +837,7 @@ static int io_wq_worker(void *data) } if (!ret) { last_timeout = true; - exit_mask = !cpumask_test_cpu(raw_smp_processor_id(), - wq->cpu_mask); + exit_mask = io_wq_worker_off_cpus(wq); } } @@ -860,9 +887,16 @@ void io_wq_worker_sleeping(struct task_struct *tsk) static void io_init_new_worker(struct io_wq *wq, struct io_wq_acct *acct, struct io_worker *worker, struct task_struct *tsk) { + cpumask_var_t cpus; + tsk->worker_private = worker; worker->task = tsk; - set_cpus_allowed_ptr(tsk, wq->cpu_mask); + /* Without a mask, the worker keeps its creator's CPUs. */ + if (alloc_cpumask_var(&cpus, GFP_KERNEL)) { + io_wq_worker_cpus(wq, tsk, cpus); + set_cpus_allowed_ptr(tsk, cpus); + free_cpumask_var(cpus); + } raw_spin_lock(&acct->workers_lock); hlist_nulls_add_head_rcu(&worker->nulls_node, &acct->free_list); @@ -1361,7 +1395,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) if (!alloc_cpumask_var(&wq->cpu_mask, GFP_KERNEL)) goto err; - cpuset_cpus_allowed(data->task, wq->cpu_mask); + cpumask_copy(wq->cpu_mask, cpu_possible_mask); wq->acct[IO_WQ_ACCT_BOUND].max_workers = bounded; wq->acct[IO_WQ_ACCT_UNBOUND].max_workers = task_rlimit(current, RLIMIT_NPROC); @@ -1545,7 +1579,7 @@ int io_wq_cpu_affinity(struct io_uring_task *tctx, cpumask_var_t mask) else ret = -EINVAL; } else { - cpumask_copy(tctx->io_wq->cpu_mask, allowed_mask); + cpumask_copy(tctx->io_wq->cpu_mask, cpu_possible_mask); } rcu_read_unlock();