radix-tree: rewrite gang lookup using iterator
Rewrite radix_tree_gang_lookup_* functions using the new radix-tree iterator. Signed-off-by: Konstantin Khlebnikov <khlebnikov@openvz.org> Tested-by: Hugh Dickins <hughd@google.com> Cc: Christoph Hellwig <hch@lst.de> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Konstantin Khlebnikov
Linus Torvalds
parent
78c1d78488
commit
cebbd29e1c
291
lib/radix-tree.c
291
lib/radix-tree.c
|
|
@ -968,57 +968,6 @@ unsigned long radix_tree_prev_hole(struct radix_tree_root *root,
|
|||
}
|
||||
EXPORT_SYMBOL(radix_tree_prev_hole);
|
||||
|
||||
static unsigned int
|
||||
__lookup(struct radix_tree_node *slot, void ***results, unsigned long *indices,
|
||||
unsigned long index, unsigned int max_items, unsigned long *next_index)
|
||||
{
|
||||
unsigned int nr_found = 0;
|
||||
unsigned int shift, height;
|
||||
unsigned long i;
|
||||
|
||||
height = slot->height;
|
||||
if (height == 0)
|
||||
goto out;
|
||||
shift = (height-1) * RADIX_TREE_MAP_SHIFT;
|
||||
|
||||
for ( ; height > 1; height--) {
|
||||
i = (index >> shift) & RADIX_TREE_MAP_MASK;
|
||||
for (;;) {
|
||||
if (slot->slots[i] != NULL)
|
||||
break;
|
||||
index &= ~((1UL << shift) - 1);
|
||||
index += 1UL << shift;
|
||||
if (index == 0)
|
||||
goto out; /* 32-bit wraparound */
|
||||
i++;
|
||||
if (i == RADIX_TREE_MAP_SIZE)
|
||||
goto out;
|
||||
}
|
||||
|
||||
shift -= RADIX_TREE_MAP_SHIFT;
|
||||
slot = rcu_dereference_raw(slot->slots[i]);
|
||||
if (slot == NULL)
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* Bottom level: grab some items */
|
||||
for (i = index & RADIX_TREE_MAP_MASK; i < RADIX_TREE_MAP_SIZE; i++) {
|
||||
if (slot->slots[i]) {
|
||||
results[nr_found] = &(slot->slots[i]);
|
||||
if (indices)
|
||||
indices[nr_found] = index;
|
||||
if (++nr_found == max_items) {
|
||||
index++;
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
index++;
|
||||
}
|
||||
out:
|
||||
*next_index = index;
|
||||
return nr_found;
|
||||
}
|
||||
|
||||
/**
|
||||
* radix_tree_gang_lookup - perform multiple lookup on a radix tree
|
||||
* @root: radix tree root
|
||||
|
|
@ -1042,48 +991,19 @@ unsigned int
|
|||
radix_tree_gang_lookup(struct radix_tree_root *root, void **results,
|
||||
unsigned long first_index, unsigned int max_items)
|
||||
{
|
||||
unsigned long max_index;
|
||||
struct radix_tree_node *node;
|
||||
unsigned long cur_index = first_index;
|
||||
unsigned int ret;
|
||||
struct radix_tree_iter iter;
|
||||
void **slot;
|
||||
unsigned int ret = 0;
|
||||
|
||||
node = rcu_dereference_raw(root->rnode);
|
||||
if (!node)
|
||||
if (unlikely(!max_items))
|
||||
return 0;
|
||||
|
||||
if (!radix_tree_is_indirect_ptr(node)) {
|
||||
if (first_index > 0)
|
||||
return 0;
|
||||
results[0] = node;
|
||||
return 1;
|
||||
}
|
||||
node = indirect_to_ptr(node);
|
||||
|
||||
max_index = radix_tree_maxindex(node->height);
|
||||
|
||||
ret = 0;
|
||||
while (ret < max_items) {
|
||||
unsigned int nr_found, slots_found, i;
|
||||
unsigned long next_index; /* Index of next search */
|
||||
|
||||
if (cur_index > max_index)
|
||||
radix_tree_for_each_slot(slot, root, &iter, first_index) {
|
||||
results[ret] = indirect_to_ptr(rcu_dereference_raw(*slot));
|
||||
if (!results[ret])
|
||||
continue;
|
||||
if (++ret == max_items)
|
||||
break;
|
||||
slots_found = __lookup(node, (void ***)results + ret, NULL,
|
||||
cur_index, max_items - ret, &next_index);
|
||||
nr_found = 0;
|
||||
for (i = 0; i < slots_found; i++) {
|
||||
struct radix_tree_node *slot;
|
||||
slot = *(((void ***)results)[ret + i]);
|
||||
if (!slot)
|
||||
continue;
|
||||
results[ret + nr_found] =
|
||||
indirect_to_ptr(rcu_dereference_raw(slot));
|
||||
nr_found++;
|
||||
}
|
||||
ret += nr_found;
|
||||
if (next_index == 0)
|
||||
break;
|
||||
cur_index = next_index;
|
||||
}
|
||||
|
||||
return ret;
|
||||
|
|
@ -1113,112 +1033,25 @@ radix_tree_gang_lookup_slot(struct radix_tree_root *root,
|
|||
void ***results, unsigned long *indices,
|
||||
unsigned long first_index, unsigned int max_items)
|
||||
{
|
||||
unsigned long max_index;
|
||||
struct radix_tree_node *node;
|
||||
unsigned long cur_index = first_index;
|
||||
unsigned int ret;
|
||||
struct radix_tree_iter iter;
|
||||
void **slot;
|
||||
unsigned int ret = 0;
|
||||
|
||||
node = rcu_dereference_raw(root->rnode);
|
||||
if (!node)
|
||||
if (unlikely(!max_items))
|
||||
return 0;
|
||||
|
||||
if (!radix_tree_is_indirect_ptr(node)) {
|
||||
if (first_index > 0)
|
||||
return 0;
|
||||
results[0] = (void **)&root->rnode;
|
||||
radix_tree_for_each_slot(slot, root, &iter, first_index) {
|
||||
results[ret] = slot;
|
||||
if (indices)
|
||||
indices[0] = 0;
|
||||
return 1;
|
||||
}
|
||||
node = indirect_to_ptr(node);
|
||||
|
||||
max_index = radix_tree_maxindex(node->height);
|
||||
|
||||
ret = 0;
|
||||
while (ret < max_items) {
|
||||
unsigned int slots_found;
|
||||
unsigned long next_index; /* Index of next search */
|
||||
|
||||
if (cur_index > max_index)
|
||||
indices[ret] = iter.index;
|
||||
if (++ret == max_items)
|
||||
break;
|
||||
slots_found = __lookup(node, results + ret,
|
||||
indices ? indices + ret : NULL,
|
||||
cur_index, max_items - ret, &next_index);
|
||||
ret += slots_found;
|
||||
if (next_index == 0)
|
||||
break;
|
||||
cur_index = next_index;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(radix_tree_gang_lookup_slot);
|
||||
|
||||
/*
|
||||
* FIXME: the two tag_get()s here should use find_next_bit() instead of
|
||||
* open-coding the search.
|
||||
*/
|
||||
static unsigned int
|
||||
__lookup_tag(struct radix_tree_node *slot, void ***results, unsigned long index,
|
||||
unsigned int max_items, unsigned long *next_index, unsigned int tag)
|
||||
{
|
||||
unsigned int nr_found = 0;
|
||||
unsigned int shift, height;
|
||||
|
||||
height = slot->height;
|
||||
if (height == 0)
|
||||
goto out;
|
||||
shift = (height-1) * RADIX_TREE_MAP_SHIFT;
|
||||
|
||||
while (height > 0) {
|
||||
unsigned long i = (index >> shift) & RADIX_TREE_MAP_MASK ;
|
||||
|
||||
for (;;) {
|
||||
if (tag_get(slot, tag, i))
|
||||
break;
|
||||
index &= ~((1UL << shift) - 1);
|
||||
index += 1UL << shift;
|
||||
if (index == 0)
|
||||
goto out; /* 32-bit wraparound */
|
||||
i++;
|
||||
if (i == RADIX_TREE_MAP_SIZE)
|
||||
goto out;
|
||||
}
|
||||
height--;
|
||||
if (height == 0) { /* Bottom level: grab some items */
|
||||
unsigned long j = index & RADIX_TREE_MAP_MASK;
|
||||
|
||||
for ( ; j < RADIX_TREE_MAP_SIZE; j++) {
|
||||
index++;
|
||||
if (!tag_get(slot, tag, j))
|
||||
continue;
|
||||
/*
|
||||
* Even though the tag was found set, we need to
|
||||
* recheck that we have a non-NULL node, because
|
||||
* if this lookup is lockless, it may have been
|
||||
* subsequently deleted.
|
||||
*
|
||||
* Similar care must be taken in any place that
|
||||
* lookup ->slots[x] without a lock (ie. can't
|
||||
* rely on its value remaining the same).
|
||||
*/
|
||||
if (slot->slots[j]) {
|
||||
results[nr_found++] = &(slot->slots[j]);
|
||||
if (nr_found == max_items)
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
}
|
||||
shift -= RADIX_TREE_MAP_SHIFT;
|
||||
slot = rcu_dereference_raw(slot->slots[i]);
|
||||
if (slot == NULL)
|
||||
break;
|
||||
}
|
||||
out:
|
||||
*next_index = index;
|
||||
return nr_found;
|
||||
}
|
||||
|
||||
/**
|
||||
* radix_tree_gang_lookup_tag - perform multiple lookup on a radix tree
|
||||
* based on a tag
|
||||
|
|
@ -1237,52 +1070,19 @@ radix_tree_gang_lookup_tag(struct radix_tree_root *root, void **results,
|
|||
unsigned long first_index, unsigned int max_items,
|
||||
unsigned int tag)
|
||||
{
|
||||
struct radix_tree_node *node;
|
||||
unsigned long max_index;
|
||||
unsigned long cur_index = first_index;
|
||||
unsigned int ret;
|
||||
struct radix_tree_iter iter;
|
||||
void **slot;
|
||||
unsigned int ret = 0;
|
||||
|
||||
/* check the root's tag bit */
|
||||
if (!root_tag_get(root, tag))
|
||||
if (unlikely(!max_items))
|
||||
return 0;
|
||||
|
||||
node = rcu_dereference_raw(root->rnode);
|
||||
if (!node)
|
||||
return 0;
|
||||
|
||||
if (!radix_tree_is_indirect_ptr(node)) {
|
||||
if (first_index > 0)
|
||||
return 0;
|
||||
results[0] = node;
|
||||
return 1;
|
||||
}
|
||||
node = indirect_to_ptr(node);
|
||||
|
||||
max_index = radix_tree_maxindex(node->height);
|
||||
|
||||
ret = 0;
|
||||
while (ret < max_items) {
|
||||
unsigned int nr_found, slots_found, i;
|
||||
unsigned long next_index; /* Index of next search */
|
||||
|
||||
if (cur_index > max_index)
|
||||
radix_tree_for_each_tagged(slot, root, &iter, first_index, tag) {
|
||||
results[ret] = indirect_to_ptr(rcu_dereference_raw(*slot));
|
||||
if (!results[ret])
|
||||
continue;
|
||||
if (++ret == max_items)
|
||||
break;
|
||||
slots_found = __lookup_tag(node, (void ***)results + ret,
|
||||
cur_index, max_items - ret, &next_index, tag);
|
||||
nr_found = 0;
|
||||
for (i = 0; i < slots_found; i++) {
|
||||
struct radix_tree_node *slot;
|
||||
slot = *(((void ***)results)[ret + i]);
|
||||
if (!slot)
|
||||
continue;
|
||||
results[ret + nr_found] =
|
||||
indirect_to_ptr(rcu_dereference_raw(slot));
|
||||
nr_found++;
|
||||
}
|
||||
ret += nr_found;
|
||||
if (next_index == 0)
|
||||
break;
|
||||
cur_index = next_index;
|
||||
}
|
||||
|
||||
return ret;
|
||||
|
|
@ -1307,42 +1107,17 @@ radix_tree_gang_lookup_tag_slot(struct radix_tree_root *root, void ***results,
|
|||
unsigned long first_index, unsigned int max_items,
|
||||
unsigned int tag)
|
||||
{
|
||||
struct radix_tree_node *node;
|
||||
unsigned long max_index;
|
||||
unsigned long cur_index = first_index;
|
||||
unsigned int ret;
|
||||
struct radix_tree_iter iter;
|
||||
void **slot;
|
||||
unsigned int ret = 0;
|
||||
|
||||
/* check the root's tag bit */
|
||||
if (!root_tag_get(root, tag))
|
||||
if (unlikely(!max_items))
|
||||
return 0;
|
||||
|
||||
node = rcu_dereference_raw(root->rnode);
|
||||
if (!node)
|
||||
return 0;
|
||||
|
||||
if (!radix_tree_is_indirect_ptr(node)) {
|
||||
if (first_index > 0)
|
||||
return 0;
|
||||
results[0] = (void **)&root->rnode;
|
||||
return 1;
|
||||
}
|
||||
node = indirect_to_ptr(node);
|
||||
|
||||
max_index = radix_tree_maxindex(node->height);
|
||||
|
||||
ret = 0;
|
||||
while (ret < max_items) {
|
||||
unsigned int slots_found;
|
||||
unsigned long next_index; /* Index of next search */
|
||||
|
||||
if (cur_index > max_index)
|
||||
radix_tree_for_each_tagged(slot, root, &iter, first_index, tag) {
|
||||
results[ret] = slot;
|
||||
if (++ret == max_items)
|
||||
break;
|
||||
slots_found = __lookup_tag(node, results + ret,
|
||||
cur_index, max_items - ret, &next_index, tag);
|
||||
ret += slots_found;
|
||||
if (next_index == 0)
|
||||
break;
|
||||
cur_index = next_index;
|
||||
}
|
||||
|
||||
return ret;
|
||||
|
|
|
|||
Reference in New Issue