Skip to content
Draft
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 39 additions & 5 deletions io_uring/io-wq.c
Original file line number Diff line number Diff line change
Expand Up @@ -731,6 +731,34 @@ static bool io_worker_retire(struct io_wq_acct *acct, struct io_worker *worker)
return false;
}

/*
* The CPUs a worker of @wq runs on: those of the io-wq affinity that the
* worker's cpuset allows, or the cpuset if they have none in common, as the
* scheduler does with a task's requested affinity when its cpuset changes.
* Read the cpuset each time: it can change while the io-wq exists.
*/
static void io_wq_worker_cpus(struct io_wq *wq, struct task_struct *tsk,
struct cpumask *cpus)
{
cpuset_cpus_allowed(tsk, cpus);
if (cpumask_intersects(cpus, wq->cpu_mask))
cpumask_and(cpus, cpus, wq->cpu_mask);
}

/* Is the current worker on a CPU that a new worker would not get? */
static bool io_wq_worker_off_cpus(struct io_wq *wq)
{
cpumask_var_t cpus;
bool off;

if (!alloc_cpumask_var(&cpus, GFP_KERNEL))
return false;
io_wq_worker_cpus(wq, current, cpus);
off = !cpumask_test_cpu(raw_smp_processor_id(), cpus);
free_cpumask_var(cpus);
return off;
}

static int io_wq_worker(void *data)
{
struct io_worker *worker = data;
Expand Down Expand Up @@ -809,8 +837,7 @@ static int io_wq_worker(void *data)
}
if (!ret) {
last_timeout = true;
exit_mask = !cpumask_test_cpu(raw_smp_processor_id(),
wq->cpu_mask);
exit_mask = io_wq_worker_off_cpus(wq);
}
}

Expand Down Expand Up @@ -860,9 +887,16 @@ void io_wq_worker_sleeping(struct task_struct *tsk)
static void io_init_new_worker(struct io_wq *wq, struct io_wq_acct *acct, struct io_worker *worker,
struct task_struct *tsk)
{
cpumask_var_t cpus;

tsk->worker_private = worker;
worker->task = tsk;
set_cpus_allowed_ptr(tsk, wq->cpu_mask);
/* Without a mask, the worker keeps its creator's CPUs. */
if (alloc_cpumask_var(&cpus, GFP_KERNEL)) {
io_wq_worker_cpus(wq, tsk, cpus);
set_cpus_allowed_ptr(tsk, cpus);
free_cpumask_var(cpus);
}

raw_spin_lock(&acct->workers_lock);
hlist_nulls_add_head_rcu(&worker->nulls_node, &acct->free_list);
Expand Down Expand Up @@ -1361,7 +1395,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)

if (!alloc_cpumask_var(&wq->cpu_mask, GFP_KERNEL))
goto err;
cpuset_cpus_allowed(data->task, wq->cpu_mask);
cpumask_copy(wq->cpu_mask, cpu_possible_mask);
wq->acct[IO_WQ_ACCT_BOUND].max_workers = bounded;
wq->acct[IO_WQ_ACCT_UNBOUND].max_workers =
task_rlimit(current, RLIMIT_NPROC);
Expand Down Expand Up @@ -1545,7 +1579,7 @@ int io_wq_cpu_affinity(struct io_uring_task *tctx, cpumask_var_t mask)
else
ret = -EINVAL;
} else {
cpumask_copy(tctx->io_wq->cpu_mask, allowed_mask);
cpumask_copy(tctx->io_wq->cpu_mask, cpu_possible_mask);
}
rcu_read_unlock();

Expand Down