diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index fd53297af26e..0d74e365d54a 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -731,6 +731,34 @@ static bool io_worker_retire(struct io_wq_acct *acct, struct io_worker *worker) return false; } +/* + * The CPUs a worker of @wq runs on: those of the io-wq affinity that the + * worker's cpuset allows, or the cpuset if they have none in common, as the + * scheduler does with a task's requested affinity when its cpuset changes. + * Read the cpuset each time: it can change while the io-wq exists. + */ +static void io_wq_worker_cpus(struct io_wq *wq, struct task_struct *tsk, + struct cpumask *cpus) +{ + cpuset_cpus_allowed(tsk, cpus); + if (cpumask_intersects(cpus, wq->cpu_mask)) + cpumask_and(cpus, cpus, wq->cpu_mask); +} + +/* Is the current worker on a CPU that a new worker would not get? */ +static bool io_wq_worker_off_cpus(struct io_wq *wq) +{ + cpumask_var_t cpus; + bool off; + + if (!alloc_cpumask_var(&cpus, GFP_KERNEL)) + return false; + io_wq_worker_cpus(wq, current, cpus); + off = !cpumask_test_cpu(raw_smp_processor_id(), cpus); + free_cpumask_var(cpus); + return off; +} + static int io_wq_worker(void *data) { struct io_worker *worker = data; @@ -809,8 +837,7 @@ static int io_wq_worker(void *data) } if (!ret) { last_timeout = true; - exit_mask = !cpumask_test_cpu(raw_smp_processor_id(), - wq->cpu_mask); + exit_mask = io_wq_worker_off_cpus(wq); } } @@ -860,9 +887,16 @@ void io_wq_worker_sleeping(struct task_struct *tsk) static void io_init_new_worker(struct io_wq *wq, struct io_wq_acct *acct, struct io_worker *worker, struct task_struct *tsk) { + cpumask_var_t cpus; + tsk->worker_private = worker; worker->task = tsk; - set_cpus_allowed_ptr(tsk, wq->cpu_mask); + /* Without a mask, the worker keeps its creator's CPUs. */ + if (alloc_cpumask_var(&cpus, GFP_KERNEL)) { + io_wq_worker_cpus(wq, tsk, cpus); + set_cpus_allowed_ptr(tsk, cpus); + free_cpumask_var(cpus); + } raw_spin_lock(&acct->workers_lock); hlist_nulls_add_head_rcu(&worker->nulls_node, &acct->free_list); @@ -1361,7 +1395,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) if (!alloc_cpumask_var(&wq->cpu_mask, GFP_KERNEL)) goto err; - cpuset_cpus_allowed(data->task, wq->cpu_mask); + cpumask_copy(wq->cpu_mask, cpu_possible_mask); wq->acct[IO_WQ_ACCT_BOUND].max_workers = bounded; wq->acct[IO_WQ_ACCT_UNBOUND].max_workers = task_rlimit(current, RLIMIT_NPROC); @@ -1545,7 +1579,7 @@ int io_wq_cpu_affinity(struct io_uring_task *tctx, cpumask_var_t mask) else ret = -EINVAL; } else { - cpumask_copy(tctx->io_wq->cpu_mask, allowed_mask); + cpumask_copy(tctx->io_wq->cpu_mask, cpu_possible_mask); } rcu_read_unlock();