Btrfs: reduce worker thread spin_lock_irq hold times
This changes the btrfs worker threads to batch work items into a local list. It allows us to pull work items in large chunks and significantly reduces the number of times we need to take the worker thread spinlock. Signed-off-by: Chris Mason <chris.mason@oracle.com>
This commit is contained in:
parent
4e3f9c5042
commit
4f878e8475
|
@ -196,31 +196,73 @@ static int try_worker_shutdown(struct btrfs_worker_thread *worker)
|
||||||
return freeit;
|
return freeit;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static struct btrfs_work *get_next_work(struct btrfs_worker_thread *worker,
|
||||||
|
struct list_head *prio_head,
|
||||||
|
struct list_head *head)
|
||||||
|
{
|
||||||
|
struct btrfs_work *work = NULL;
|
||||||
|
struct list_head *cur = NULL;
|
||||||
|
|
||||||
|
if(!list_empty(prio_head))
|
||||||
|
cur = prio_head->next;
|
||||||
|
|
||||||
|
smp_mb();
|
||||||
|
if (!list_empty(&worker->prio_pending))
|
||||||
|
goto refill;
|
||||||
|
|
||||||
|
if (!list_empty(head))
|
||||||
|
cur = head->next;
|
||||||
|
|
||||||
|
if (cur)
|
||||||
|
goto out;
|
||||||
|
|
||||||
|
refill:
|
||||||
|
spin_lock_irq(&worker->lock);
|
||||||
|
list_splice_tail_init(&worker->prio_pending, prio_head);
|
||||||
|
list_splice_tail_init(&worker->pending, head);
|
||||||
|
|
||||||
|
if (!list_empty(prio_head))
|
||||||
|
cur = prio_head->next;
|
||||||
|
else if (!list_empty(head))
|
||||||
|
cur = head->next;
|
||||||
|
spin_unlock_irq(&worker->lock);
|
||||||
|
|
||||||
|
if (!cur)
|
||||||
|
goto out_fail;
|
||||||
|
|
||||||
|
out:
|
||||||
|
work = list_entry(cur, struct btrfs_work, list);
|
||||||
|
|
||||||
|
out_fail:
|
||||||
|
return work;
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* main loop for servicing work items
|
* main loop for servicing work items
|
||||||
*/
|
*/
|
||||||
static int worker_loop(void *arg)
|
static int worker_loop(void *arg)
|
||||||
{
|
{
|
||||||
struct btrfs_worker_thread *worker = arg;
|
struct btrfs_worker_thread *worker = arg;
|
||||||
struct list_head *cur;
|
struct list_head head;
|
||||||
|
struct list_head prio_head;
|
||||||
struct btrfs_work *work;
|
struct btrfs_work *work;
|
||||||
|
|
||||||
|
INIT_LIST_HEAD(&head);
|
||||||
|
INIT_LIST_HEAD(&prio_head);
|
||||||
|
|
||||||
do {
|
do {
|
||||||
spin_lock_irq(&worker->lock);
|
again:
|
||||||
again_locked:
|
|
||||||
while (1) {
|
while (1) {
|
||||||
if (!list_empty(&worker->prio_pending))
|
|
||||||
cur = worker->prio_pending.next;
|
|
||||||
else if (!list_empty(&worker->pending))
|
work = get_next_work(worker, &prio_head, &head);
|
||||||
cur = worker->pending.next;
|
if (!work)
|
||||||
else
|
|
||||||
break;
|
break;
|
||||||
|
|
||||||
work = list_entry(cur, struct btrfs_work, list);
|
|
||||||
list_del(&work->list);
|
list_del(&work->list);
|
||||||
clear_bit(WORK_QUEUED_BIT, &work->flags);
|
clear_bit(WORK_QUEUED_BIT, &work->flags);
|
||||||
|
|
||||||
work->worker = worker;
|
work->worker = worker;
|
||||||
spin_unlock_irq(&worker->lock);
|
|
||||||
|
|
||||||
work->func(work);
|
work->func(work);
|
||||||
|
|
||||||
|
@ -233,9 +275,11 @@ again_locked:
|
||||||
|
|
||||||
check_pending_worker_creates(worker);
|
check_pending_worker_creates(worker);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
spin_lock_irq(&worker->lock);
|
spin_lock_irq(&worker->lock);
|
||||||
check_idle_worker(worker);
|
check_idle_worker(worker);
|
||||||
}
|
|
||||||
if (freezing(current)) {
|
if (freezing(current)) {
|
||||||
worker->working = 0;
|
worker->working = 0;
|
||||||
spin_unlock_irq(&worker->lock);
|
spin_unlock_irq(&worker->lock);
|
||||||
|
@ -274,8 +318,10 @@ again_locked:
|
||||||
spin_lock_irq(&worker->lock);
|
spin_lock_irq(&worker->lock);
|
||||||
set_current_state(TASK_INTERRUPTIBLE);
|
set_current_state(TASK_INTERRUPTIBLE);
|
||||||
if (!list_empty(&worker->pending) ||
|
if (!list_empty(&worker->pending) ||
|
||||||
!list_empty(&worker->prio_pending))
|
!list_empty(&worker->prio_pending)) {
|
||||||
goto again_locked;
|
spin_unlock_irq(&worker->lock);
|
||||||
|
goto again;
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* this makes sure we get a wakeup when someone
|
* this makes sure we get a wakeup when someone
|
||||||
|
|
Reference in New Issue