![Greg Kroah-Hartman](/assets/img/avatar_default.png)
Cc: Hank Janssen <hjanssen@microsoft.com> Cc: Haiyang Zhang <haiyangz@microsoft.com> Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de>
628 lines
12 KiB
C
628 lines
12 KiB
C
/*
|
|
*
|
|
* Copyright (c) 2009, Microsoft Corporation.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify it
|
|
* under the terms and conditions of the GNU General Public License,
|
|
* version 2, as published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope it will be useful, but WITHOUT
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
* more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License along with
|
|
* this program; if not, write to the Free Software Foundation, Inc., 59 Temple
|
|
* Place - Suite 330, Boston, MA 02111-1307 USA.
|
|
*
|
|
* Authors:
|
|
* Haiyang Zhang <haiyangz@microsoft.com>
|
|
* Hank Janssen <hjanssen@microsoft.com>
|
|
*
|
|
*/
|
|
|
|
#include <linux/kernel.h>
|
|
#include <linux/mm.h>
|
|
#include "osd.h"
|
|
#include "logging.h"
|
|
#include "ring_buffer.h"
|
|
|
|
|
|
/* #defines */
|
|
|
|
|
|
/* Amount of space to write to */
|
|
#define BYTES_AVAIL_TO_WRITE(r, w, z) ((w) >= (r)) ? ((z) - ((w) - (r))) : ((r) - (w))
|
|
|
|
|
|
/*++
|
|
|
|
Name:
|
|
GetRingBufferAvailBytes()
|
|
|
|
Description:
|
|
Get number of bytes available to read and to write to
|
|
for the specified ring buffer
|
|
|
|
--*/
|
|
static inline void
|
|
GetRingBufferAvailBytes(RING_BUFFER_INFO *rbi, u32 *read, u32 *write)
|
|
{
|
|
u32 read_loc, write_loc;
|
|
|
|
/* Capture the read/write indices before they changed */
|
|
read_loc = rbi->RingBuffer->ReadIndex;
|
|
write_loc = rbi->RingBuffer->WriteIndex;
|
|
|
|
*write = BYTES_AVAIL_TO_WRITE(read_loc, write_loc, rbi->RingDataSize);
|
|
*read = rbi->RingDataSize - *write;
|
|
}
|
|
|
|
/*++
|
|
|
|
Name:
|
|
GetNextWriteLocation()
|
|
|
|
Description:
|
|
Get the next write location for the specified ring buffer
|
|
|
|
--*/
|
|
static inline u32
|
|
GetNextWriteLocation(RING_BUFFER_INFO *RingInfo)
|
|
{
|
|
u32 next = RingInfo->RingBuffer->WriteIndex;
|
|
|
|
/* ASSERT(next < RingInfo->RingDataSize); */
|
|
|
|
return next;
|
|
}
|
|
|
|
/*++
|
|
|
|
Name:
|
|
SetNextWriteLocation()
|
|
|
|
Description:
|
|
Set the next write location for the specified ring buffer
|
|
|
|
--*/
|
|
static inline void
|
|
SetNextWriteLocation(RING_BUFFER_INFO *RingInfo, u32 NextWriteLocation)
|
|
{
|
|
RingInfo->RingBuffer->WriteIndex = NextWriteLocation;
|
|
}
|
|
|
|
/*++
|
|
|
|
Name:
|
|
GetNextReadLocation()
|
|
|
|
Description:
|
|
Get the next read location for the specified ring buffer
|
|
|
|
--*/
|
|
static inline u32
|
|
GetNextReadLocation(RING_BUFFER_INFO *RingInfo)
|
|
{
|
|
u32 next = RingInfo->RingBuffer->ReadIndex;
|
|
|
|
/* ASSERT(next < RingInfo->RingDataSize); */
|
|
|
|
return next;
|
|
}
|
|
|
|
/*++
|
|
|
|
Name:
|
|
GetNextReadLocationWithOffset()
|
|
|
|
Description:
|
|
Get the next read location + offset for the specified ring buffer.
|
|
This allows the caller to skip
|
|
|
|
--*/
|
|
static inline u32
|
|
GetNextReadLocationWithOffset(RING_BUFFER_INFO *RingInfo, u32 Offset)
|
|
{
|
|
u32 next = RingInfo->RingBuffer->ReadIndex;
|
|
|
|
/* ASSERT(next < RingInfo->RingDataSize); */
|
|
next += Offset;
|
|
next %= RingInfo->RingDataSize;
|
|
|
|
return next;
|
|
}
|
|
|
|
/*++
|
|
|
|
Name:
|
|
SetNextReadLocation()
|
|
|
|
Description:
|
|
Set the next read location for the specified ring buffer
|
|
|
|
--*/
|
|
static inline void
|
|
SetNextReadLocation(RING_BUFFER_INFO *RingInfo, u32 NextReadLocation)
|
|
{
|
|
RingInfo->RingBuffer->ReadIndex = NextReadLocation;
|
|
}
|
|
|
|
|
|
/*++
|
|
|
|
Name:
|
|
GetRingBuffer()
|
|
|
|
Description:
|
|
Get the start of the ring buffer
|
|
|
|
--*/
|
|
static inline void *
|
|
GetRingBuffer(RING_BUFFER_INFO *RingInfo)
|
|
{
|
|
return (void *)RingInfo->RingBuffer->Buffer;
|
|
}
|
|
|
|
|
|
/*++
|
|
|
|
Name:
|
|
GetRingBufferSize()
|
|
|
|
Description:
|
|
Get the size of the ring buffer
|
|
|
|
--*/
|
|
static inline u32
|
|
GetRingBufferSize(RING_BUFFER_INFO *RingInfo)
|
|
{
|
|
return RingInfo->RingDataSize;
|
|
}
|
|
|
|
/*++
|
|
|
|
Name:
|
|
GetRingBufferIndices()
|
|
|
|
Description:
|
|
Get the read and write indices as u64 of the specified ring buffer
|
|
|
|
--*/
|
|
static inline u64
|
|
GetRingBufferIndices(RING_BUFFER_INFO *RingInfo)
|
|
{
|
|
return ((u64)RingInfo->RingBuffer->WriteIndex << 32)
|
|
|| RingInfo->RingBuffer->ReadIndex;
|
|
}
|
|
|
|
|
|
/*++
|
|
|
|
Name:
|
|
DumpRingInfo()
|
|
|
|
Description:
|
|
Dump out to console the ring buffer info
|
|
|
|
--*/
|
|
void DumpRingInfo(RING_BUFFER_INFO *RingInfo, char *Prefix)
|
|
{
|
|
u32 bytesAvailToWrite;
|
|
u32 bytesAvailToRead;
|
|
|
|
GetRingBufferAvailBytes(RingInfo,
|
|
&bytesAvailToRead,
|
|
&bytesAvailToWrite);
|
|
|
|
DPRINT(VMBUS,
|
|
DEBUG_RING_LVL,
|
|
"%s <<ringinfo %p buffer %p avail write %u "
|
|
"avail read %u read idx %u write idx %u>>",
|
|
Prefix,
|
|
RingInfo,
|
|
RingInfo->RingBuffer->Buffer,
|
|
bytesAvailToWrite,
|
|
bytesAvailToRead,
|
|
RingInfo->RingBuffer->ReadIndex,
|
|
RingInfo->RingBuffer->WriteIndex);
|
|
}
|
|
|
|
|
|
/* Internal routines */
|
|
|
|
static u32
|
|
CopyToRingBuffer(
|
|
RING_BUFFER_INFO *RingInfo,
|
|
u32 StartWriteOffset,
|
|
void *Src,
|
|
u32 SrcLen);
|
|
|
|
static u32
|
|
CopyFromRingBuffer(
|
|
RING_BUFFER_INFO *RingInfo,
|
|
void *Dest,
|
|
u32 DestLen,
|
|
u32 StartReadOffset);
|
|
|
|
|
|
|
|
/*++
|
|
|
|
Name:
|
|
RingBufferGetDebugInfo()
|
|
|
|
Description:
|
|
Get various debug metrics for the specified ring buffer
|
|
|
|
--*/
|
|
void RingBufferGetDebugInfo(RING_BUFFER_INFO *RingInfo,
|
|
RING_BUFFER_DEBUG_INFO *DebugInfo)
|
|
{
|
|
u32 bytesAvailToWrite;
|
|
u32 bytesAvailToRead;
|
|
|
|
if (RingInfo->RingBuffer) {
|
|
GetRingBufferAvailBytes(RingInfo,
|
|
&bytesAvailToRead,
|
|
&bytesAvailToWrite);
|
|
|
|
DebugInfo->BytesAvailToRead = bytesAvailToRead;
|
|
DebugInfo->BytesAvailToWrite = bytesAvailToWrite;
|
|
DebugInfo->CurrentReadIndex = RingInfo->RingBuffer->ReadIndex;
|
|
DebugInfo->CurrentWriteIndex = RingInfo->RingBuffer->WriteIndex;
|
|
DebugInfo->CurrentInterruptMask = RingInfo->RingBuffer->InterruptMask;
|
|
}
|
|
}
|
|
|
|
|
|
/*++
|
|
|
|
Name:
|
|
GetRingBufferInterruptMask()
|
|
|
|
Description:
|
|
Get the interrupt mask for the specified ring buffer
|
|
|
|
--*/
|
|
u32 GetRingBufferInterruptMask(RING_BUFFER_INFO *rbi)
|
|
{
|
|
return rbi->RingBuffer->InterruptMask;
|
|
}
|
|
|
|
/*++
|
|
|
|
Name:
|
|
RingBufferInit()
|
|
|
|
Description:
|
|
Initialize the ring buffer
|
|
|
|
--*/
|
|
int RingBufferInit(RING_BUFFER_INFO *RingInfo, void *Buffer, u32 BufferLen)
|
|
{
|
|
if (sizeof(RING_BUFFER) != PAGE_SIZE)
|
|
return -EINVAL;
|
|
|
|
memset(RingInfo, 0, sizeof(RING_BUFFER_INFO));
|
|
|
|
RingInfo->RingBuffer = (RING_BUFFER *)Buffer;
|
|
RingInfo->RingBuffer->ReadIndex = RingInfo->RingBuffer->WriteIndex = 0;
|
|
|
|
RingInfo->RingSize = BufferLen;
|
|
RingInfo->RingDataSize = BufferLen - sizeof(RING_BUFFER);
|
|
|
|
spin_lock_init(&RingInfo->ring_lock);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*++
|
|
|
|
Name:
|
|
RingBufferCleanup()
|
|
|
|
Description:
|
|
Cleanup the ring buffer
|
|
|
|
--*/
|
|
void RingBufferCleanup(RING_BUFFER_INFO *RingInfo)
|
|
{
|
|
}
|
|
|
|
/*++
|
|
|
|
Name:
|
|
RingBufferWrite()
|
|
|
|
Description:
|
|
Write to the ring buffer
|
|
|
|
--*/
|
|
int RingBufferWrite(RING_BUFFER_INFO *OutRingInfo,
|
|
struct scatterlist *sglist, u32 sgcount)
|
|
{
|
|
int i = 0;
|
|
u32 byteAvailToWrite;
|
|
u32 byteAvailToRead;
|
|
u32 totalBytesToWrite = 0;
|
|
|
|
struct scatterlist *sg;
|
|
volatile u32 nextWriteLocation;
|
|
u64 prevIndices = 0;
|
|
unsigned long flags;
|
|
|
|
DPRINT_ENTER(VMBUS);
|
|
|
|
for_each_sg(sglist, sg, sgcount, i)
|
|
{
|
|
totalBytesToWrite += sg->length;
|
|
}
|
|
|
|
totalBytesToWrite += sizeof(u64);
|
|
|
|
spin_lock_irqsave(&OutRingInfo->ring_lock, flags);
|
|
|
|
GetRingBufferAvailBytes(OutRingInfo,
|
|
&byteAvailToRead,
|
|
&byteAvailToWrite);
|
|
|
|
DPRINT_DBG(VMBUS, "Writing %u bytes...", totalBytesToWrite);
|
|
|
|
/* DumpRingInfo(OutRingInfo, "BEFORE "); */
|
|
|
|
/* If there is only room for the packet, assume it is full. */
|
|
/* Otherwise, the next time around, we think the ring buffer */
|
|
/* is empty since the read index == write index */
|
|
if (byteAvailToWrite <= totalBytesToWrite) {
|
|
DPRINT_DBG(VMBUS,
|
|
"No more space left on outbound ring buffer "
|
|
"(needed %u, avail %u)",
|
|
totalBytesToWrite,
|
|
byteAvailToWrite);
|
|
|
|
spin_unlock_irqrestore(&OutRingInfo->ring_lock, flags);
|
|
|
|
DPRINT_EXIT(VMBUS);
|
|
|
|
return -1;
|
|
}
|
|
|
|
/* Write to the ring buffer */
|
|
nextWriteLocation = GetNextWriteLocation(OutRingInfo);
|
|
|
|
for_each_sg(sglist, sg, sgcount, i)
|
|
{
|
|
nextWriteLocation = CopyToRingBuffer(OutRingInfo,
|
|
nextWriteLocation,
|
|
sg_virt(sg),
|
|
sg->length);
|
|
}
|
|
|
|
/* Set previous packet start */
|
|
prevIndices = GetRingBufferIndices(OutRingInfo);
|
|
|
|
nextWriteLocation = CopyToRingBuffer(OutRingInfo,
|
|
nextWriteLocation,
|
|
&prevIndices,
|
|
sizeof(u64));
|
|
|
|
/* Make sure we flush all writes before updating the writeIndex */
|
|
mb();
|
|
|
|
/* Now, update the write location */
|
|
SetNextWriteLocation(OutRingInfo, nextWriteLocation);
|
|
|
|
/* DumpRingInfo(OutRingInfo, "AFTER "); */
|
|
|
|
spin_unlock_irqrestore(&OutRingInfo->ring_lock, flags);
|
|
|
|
DPRINT_EXIT(VMBUS);
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
/*++
|
|
|
|
Name:
|
|
RingBufferPeek()
|
|
|
|
Description:
|
|
Read without advancing the read index
|
|
|
|
--*/
|
|
int RingBufferPeek(RING_BUFFER_INFO *InRingInfo, void *Buffer, u32 BufferLen)
|
|
{
|
|
u32 bytesAvailToWrite;
|
|
u32 bytesAvailToRead;
|
|
u32 nextReadLocation = 0;
|
|
unsigned long flags;
|
|
|
|
spin_lock_irqsave(&InRingInfo->ring_lock, flags);
|
|
|
|
GetRingBufferAvailBytes(InRingInfo,
|
|
&bytesAvailToRead,
|
|
&bytesAvailToWrite);
|
|
|
|
/* Make sure there is something to read */
|
|
if (bytesAvailToRead < BufferLen) {
|
|
/* DPRINT_DBG(VMBUS,
|
|
"got callback but not enough to read "
|
|
"<avail to read %d read size %d>!!",
|
|
bytesAvailToRead,
|
|
BufferLen); */
|
|
|
|
spin_unlock_irqrestore(&InRingInfo->ring_lock, flags);
|
|
|
|
return -1;
|
|
}
|
|
|
|
/* Convert to byte offset */
|
|
nextReadLocation = GetNextReadLocation(InRingInfo);
|
|
|
|
nextReadLocation = CopyFromRingBuffer(InRingInfo,
|
|
Buffer,
|
|
BufferLen,
|
|
nextReadLocation);
|
|
|
|
spin_unlock_irqrestore(&InRingInfo->ring_lock, flags);
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
/*++
|
|
|
|
Name:
|
|
RingBufferRead()
|
|
|
|
Description:
|
|
Read and advance the read index
|
|
|
|
--*/
|
|
int RingBufferRead(RING_BUFFER_INFO *InRingInfo, void *Buffer,
|
|
u32 BufferLen, u32 Offset)
|
|
{
|
|
u32 bytesAvailToWrite;
|
|
u32 bytesAvailToRead;
|
|
u32 nextReadLocation = 0;
|
|
u64 prevIndices = 0;
|
|
unsigned long flags;
|
|
|
|
if (BufferLen <= 0)
|
|
return -EINVAL;
|
|
|
|
spin_lock_irqsave(&InRingInfo->ring_lock, flags);
|
|
|
|
GetRingBufferAvailBytes(InRingInfo,
|
|
&bytesAvailToRead,
|
|
&bytesAvailToWrite);
|
|
|
|
DPRINT_DBG(VMBUS, "Reading %u bytes...", BufferLen);
|
|
|
|
/* DumpRingInfo(InRingInfo, "BEFORE "); */
|
|
|
|
/* Make sure there is something to read */
|
|
if (bytesAvailToRead < BufferLen) {
|
|
DPRINT_DBG(VMBUS,
|
|
"got callback but not enough to read "
|
|
"<avail to read %d read size %d>!!",
|
|
bytesAvailToRead,
|
|
BufferLen);
|
|
|
|
spin_unlock_irqrestore(&InRingInfo->ring_lock, flags);
|
|
|
|
return -1;
|
|
}
|
|
|
|
nextReadLocation = GetNextReadLocationWithOffset(InRingInfo, Offset);
|
|
|
|
nextReadLocation = CopyFromRingBuffer(InRingInfo,
|
|
Buffer,
|
|
BufferLen,
|
|
nextReadLocation);
|
|
|
|
nextReadLocation = CopyFromRingBuffer(InRingInfo,
|
|
&prevIndices,
|
|
sizeof(u64),
|
|
nextReadLocation);
|
|
|
|
/* Make sure all reads are done before we update the read index since */
|
|
/* the writer may start writing to the read area once the read index */
|
|
/*is updated */
|
|
mb();
|
|
|
|
/* Update the read index */
|
|
SetNextReadLocation(InRingInfo, nextReadLocation);
|
|
|
|
/* DumpRingInfo(InRingInfo, "AFTER "); */
|
|
|
|
spin_unlock_irqrestore(&InRingInfo->ring_lock, flags);
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
/*++
|
|
|
|
Name:
|
|
CopyToRingBuffer()
|
|
|
|
Description:
|
|
Helper routine to copy from source to ring buffer.
|
|
Assume there is enough room. Handles wrap-around in dest case only!!
|
|
|
|
--*/
|
|
static u32
|
|
CopyToRingBuffer(
|
|
RING_BUFFER_INFO *RingInfo,
|
|
u32 StartWriteOffset,
|
|
void *Src,
|
|
u32 SrcLen)
|
|
{
|
|
void *ringBuffer = GetRingBuffer(RingInfo);
|
|
u32 ringBufferSize = GetRingBufferSize(RingInfo);
|
|
u32 fragLen;
|
|
|
|
/* wrap-around detected! */
|
|
if (SrcLen > ringBufferSize - StartWriteOffset) {
|
|
DPRINT_DBG(VMBUS, "wrap-around detected!");
|
|
|
|
fragLen = ringBufferSize - StartWriteOffset;
|
|
memcpy(ringBuffer + StartWriteOffset, Src, fragLen);
|
|
memcpy(ringBuffer, Src + fragLen, SrcLen - fragLen);
|
|
} else
|
|
memcpy(ringBuffer + StartWriteOffset, Src, SrcLen);
|
|
|
|
StartWriteOffset += SrcLen;
|
|
StartWriteOffset %= ringBufferSize;
|
|
|
|
return StartWriteOffset;
|
|
}
|
|
|
|
|
|
/*++
|
|
|
|
Name:
|
|
CopyFromRingBuffer()
|
|
|
|
Description:
|
|
Helper routine to copy to source from ring buffer.
|
|
Assume there is enough room. Handles wrap-around in src case only!!
|
|
|
|
--*/
|
|
static u32
|
|
CopyFromRingBuffer(
|
|
RING_BUFFER_INFO *RingInfo,
|
|
void *Dest,
|
|
u32 DestLen,
|
|
u32 StartReadOffset)
|
|
{
|
|
void *ringBuffer = GetRingBuffer(RingInfo);
|
|
u32 ringBufferSize = GetRingBufferSize(RingInfo);
|
|
|
|
u32 fragLen;
|
|
|
|
/* wrap-around detected at the src */
|
|
if (DestLen > ringBufferSize - StartReadOffset) {
|
|
DPRINT_DBG(VMBUS, "src wrap-around detected!");
|
|
|
|
fragLen = ringBufferSize - StartReadOffset;
|
|
|
|
memcpy(Dest, ringBuffer + StartReadOffset, fragLen);
|
|
memcpy(Dest + fragLen, ringBuffer, DestLen - fragLen);
|
|
} else
|
|
|
|
memcpy(Dest, ringBuffer + StartReadOffset, DestLen);
|
|
|
|
|
|
StartReadOffset += DestLen;
|
|
StartReadOffset %= ringBufferSize;
|
|
|
|
return StartReadOffset;
|
|
}
|
|
|
|
|
|
/* eof */
|