dect
/
linux-2.6
Archived
13
0
Fork 0
Code Releases Activity

NFS: Replace atomic_t variables in nfs_direct_req with a single spin lock 

Three atomic_t variables cause a lot of bus locking.  Because they are all
used in the same places in the code, just use a single spin lock.

Now that the atomic_t variables are gone, we can remove the request size
limitation since the code no longer depends on the limited width of atomic_t
on some platforms.

Test plan:
Compile with CONFIG_NFS and CONFIG_NFS_DIRECTIO enabled.  Millions of fsx
operations, iozone, OraSim.

Signed-off-by: Chuck Lever <cel@netapp.com>
Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>
This commit is contained in:
Chuck Lever 2006-03-20 13:44:34 -05:00 committed by Trond Myklebust
parent 88467055f7
commit 15ce4a0c1c
1 changed files with 51 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
fs/nfs/direct.c
View File

@ -58,7 +58,6 @@
#include "iostat.h"
#define NFSDBG_FACILITY NFSDBG_VFS
#define MAX_DIRECTIO_SIZE (4096UL << PAGE_SHIFT)
static void nfs_free_user_pages(struct page **pages, int npages, int do_dirty);
static kmem_cache_t *nfs_direct_cachep;
@ -68,6 +67,8 @@ static kmem_cache_t *nfs_direct_cachep;
*/
struct nfs_direct_req {
struct kref kref; /* release manager */
/* I/O parameters */
struct list_head list; /* nfs_read/write_data structs */
struct file * filp; /* file descriptor */
struct kiocb * iocb; /* controlling i/o request */
@ -75,12 +76,14 @@ struct nfs_direct_req {
struct inode * inode; /* target file of i/o */
struct page ** pages; /* pages in our buffer */
unsigned int npages; /* count of pages */
atomic_t complete, /* i/os we're waiting for */
count, /* bytes actually processed */
/* completion state */
spinlock_t lock; /* protect completion state */
int outstanding; /* i/os we're waiting for */
ssize_t count, /* bytes actually processed */
error; /* any reported error */
};
/**
* nfs_direct_IO - NFS address space operation for direct I/O
* @rw: direction (read or write)
@ -110,12 +113,6 @@ static inline int nfs_get_user_pages(int rw, unsigned long user_addr, size_t siz
unsigned long page_count;
size_t array_size;
/* set an arbitrary limit to prevent type overflow */
if (size > MAX_DIRECTIO_SIZE) {
*pages = NULL;
return -EFBIG;
}
page_count = (user_addr + size + PAGE_SIZE - 1) >> PAGE_SHIFT;
page_count -= user_addr >> PAGE_SHIFT;
@ -164,8 +161,10 @@ static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
init_waitqueue_head(&dreq->wait);
INIT_LIST_HEAD(&dreq->list);
dreq->iocb = NULL;
atomic_set(&dreq->count, 0);
atomic_set(&dreq->error, 0);
spin_lock_init(&dreq->lock);
dreq->outstanding = 0;
dreq->count = 0;
dreq->error = 0;
return dreq;
}
@ -181,19 +180,18 @@ static void nfs_direct_req_release(struct kref *kref)
*/
static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
{
int result = -EIOCBQUEUED;
ssize_t result = -EIOCBQUEUED;
/* Async requests don't wait here */
if (dreq->iocb)
goto out;
result = wait_event_interruptible(dreq->wait,
(atomic_read(&dreq->complete) == 0));
result = wait_event_interruptible(dreq->wait, (dreq->outstanding == 0));
if (!result)
result = atomic_read(&dreq->error);
result = dreq->error;
if (!result)
result = atomic_read(&dreq->count);
result = dreq->count;
out:
kref_put(&dreq->kref, nfs_direct_req_release);
@ -214,9 +212,9 @@ static void nfs_direct_complete(struct nfs_direct_req *dreq)
nfs_free_user_pages(dreq->pages, dreq->npages, 1);
if (dreq->iocb) {
long res = atomic_read(&dreq->error);
long res = (long) dreq->error;
if (!res)
res = atomic_read(&dreq->count);
res = (long) dreq->count;
aio_complete(dreq->iocb, res, 0);
} else
wake_up(&dreq->wait);
@ -233,7 +231,6 @@ static struct nfs_direct_req *nfs_direct_read_alloc(size_t nbytes, size_t rsize)
{
struct list_head *list;
struct nfs_direct_req *dreq;
unsigned int reads = 0;
unsigned int rpages = (rsize + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
dreq = nfs_direct_req_alloc();
@ -259,13 +256,12 @@ static struct nfs_direct_req *nfs_direct_read_alloc(size_t nbytes, size_t rsize)
list_add(&data->pages, list);
data->req = (struct nfs_page *) dreq;
reads++;
dreq->outstanding++;
if (nbytes <= rsize)
break;
nbytes -= rsize;
}
kref_get(&dreq->kref);
atomic_set(&dreq->complete, reads);
return dreq;
}
@ -276,13 +272,21 @@ static void nfs_direct_read_result(struct rpc_task *task, void *calldata)
if (nfs_readpage_result(task, data) != 0)
return;
if (likely(task->tk_status >= 0))
atomic_add(data->res.count, &dreq->count);
else
atomic_set(&dreq->error, task->tk_status);
if (unlikely(atomic_dec_and_test(&dreq->complete)))
nfs_direct_complete(dreq);
spin_lock(&dreq->lock);
if (likely(task->tk_status >= 0))
dreq->count += data->res.count;
else
dreq->error = task->tk_status;
if (--dreq->outstanding) {
spin_unlock(&dreq->lock);
return;
}
spin_unlock(&dreq->lock);
nfs_direct_complete(dreq);
}
static const struct rpc_call_ops nfs_read_direct_ops = {
@ -388,7 +392,6 @@ static struct nfs_direct_req *nfs_direct_write_alloc(size_t nbytes, size_t wsize
{
struct list_head *list;
struct nfs_direct_req *dreq;
unsigned int writes = 0;
unsigned int wpages = (wsize + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
dreq = nfs_direct_req_alloc();
@ -414,16 +417,19 @@ static struct nfs_direct_req *nfs_direct_write_alloc(size_t nbytes, size_t wsize
list_add(&data->pages, list);
data->req = (struct nfs_page *) dreq;
writes++;
dreq->outstanding++;
if (nbytes <= wsize)
break;
nbytes -= wsize;
}
kref_get(&dreq->kref);
atomic_set(&dreq->complete, writes);
return dreq;
}
/*
* NB: Return the value of the first error return code. Subsequent
* errors after the first one are ignored.
*/
static void nfs_direct_write_result(struct rpc_task *task, void *calldata)
{
struct nfs_write_data *data = calldata;
@ -436,15 +442,22 @@ static void nfs_direct_write_result(struct rpc_task *task, void *calldata)
if (unlikely(data->res.verf->committed != NFS_FILE_SYNC))
status = -EIO;
if (likely(status >= 0))
atomic_add(data->res.count, &dreq->count);
else
atomic_set(&dreq->error, status);
spin_lock(&dreq->lock);
if (unlikely(atomic_dec_and_test(&dreq->complete))) {
nfs_end_data_update(data->inode);
nfs_direct_complete(dreq);
if (likely(status >= 0))
dreq->count += data->res.count;
else
dreq->error = status;
if (--dreq->outstanding) {
spin_unlock(&dreq->lock);
return;
}
spin_unlock(&dreq->lock);
nfs_end_data_update(data->inode);
nfs_direct_complete(dreq);
}
static const struct rpc_call_ops nfs_write_direct_ops = {