|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [win-pv-devel] [PATCH 6/9] Implement multi-queues
Splits XENVBD_RING into multiple XENVBD_BLKIF_RINGs, one for each shared
ring. Up-to "multi-queue-max-queues" rings are used to pass
blkif_requests and blkif_responses between frontend and backend. Reworks
the ring interactions to remove the locks used by XENVBD_QUEUE,
implementing a queue system similar to XenVifs transmitter queues.
Signed-off-by: Owen Smith <owen.smith@xxxxxxxxxx>
---
src/xenvbd/frontend.c | 18 +-
src/xenvbd/ring.c | 3097 +++++++++++++++++++++++++++++--------------------
2 files changed, 1847 insertions(+), 1268 deletions(-)
diff --git a/src/xenvbd/frontend.c b/src/xenvbd/frontend.c
index 900ad2b..fd2c247 100644
--- a/src/xenvbd/frontend.c
+++ b/src/xenvbd/frontend.c
@@ -1278,15 +1278,15 @@ FrontendConnect(
if (!NT_SUCCESS(Status))
goto abort;
- //status = XENBUS_STORE(Printf,
- // &Frontend->StoreInterface,
- // Transaction,
- // Frontend->FrontendPath,
- // "multi-queue-num-queues",
- // "%u",
- // __FrontendGetNumQueues(Frontend));
- //if (!NT_SUCCESS(status))
- // goto abort;
+ Status = XENBUS_STORE(Printf,
+ &Frontend->StoreInterface,
+ Transaction,
+ Frontend->FrontendPath,
+ "multi-queue-num-queues",
+ "%u",
+ __FrontendGetNumQueues(Frontend));
+ if (!NT_SUCCESS(Status))
+ goto abort;
Status = XENBUS_STORE(TransactionEnd,
&Frontend->StoreInterface,
diff --git a/src/xenvbd/ring.c b/src/xenvbd/ring.c
index e41c1b7..cd2cc0e 100644
--- a/src/xenvbd/ring.c
+++ b/src/xenvbd/ring.c
@@ -47,7 +47,6 @@
#include "srbext.h"
#include "driver.h"
#include "granter.h"
-#include "queue.h"
#include "util.h"
#include "debug.h"
@@ -56,47 +55,72 @@
#define XENVBD_MAX_RING_PAGE_ORDER (4)
#define XENVBD_MAX_RING_PAGES (1 << XENVBD_MAX_RING_PAGE_ORDER)
-struct _XENVBD_RING {
- PXENVBD_FRONTEND Frontend;
+#define xen_mb KeMemoryBarrier
+#define xen_wmb KeMemoryBarrier
+
+typedef struct _XENVBD_SRB_STATE {
+ LIST_ENTRY List;
+ ULONG Count;
+} XENVBD_SRB_STATE, *PXENVBD_SRB_STATE;
+
+typedef struct _XENVBD_BLKIF_RING {
+ PXENVBD_RING Ring;
+ ULONG Index;
+ PCHAR Path;
+ PXENBUS_CACHE RequestCache;
+ PXENBUS_CACHE SegmentCache;
+ PXENBUS_CACHE IndirectCache;
+ PMDL Mdl;
+ blkif_sring_t *Shared;
+ blkif_front_ring_t Front;
+ PXENBUS_GNTTAB_ENTRY Grants[XENVBD_MAX_RING_PAGES];
+ PXENBUS_EVTCHN_CHANNEL Channel;
+ KDPC Dpc;
+ ULONG Dpcs;
+ ULONG Events;
BOOLEAN Connected;
BOOLEAN Enabled;
+ BOOLEAN Stopped;
+ PVOID Lock;
+ PKTHREAD LockThread;
+ XENVBD_SRB_STATE State;
+ LIST_ENTRY SrbQueue;
+ LIST_ENTRY SubmittedList;
+ LIST_ENTRY ShutdownQueue;
+ ULONG SrbsQueued;
+ ULONG SrbsCompleted;
+ ULONG SrbsFailed;
+ ULONG RequestsPosted;
+ ULONG RequestsPushed;
+ ULONG ResponsesProcessed;
+ PXENBUS_DEBUG_CALLBACK DebugCallback;
+} XENVBD_BLKIF_RING, *PXENVBD_BLKIF_RING;
+
+typedef enum _XENVBD_STAT {
+ XENVBD_STAT_BLKIF_OP_READ_DIRECT = 0,
+ XENVBD_STAT_BLKIF_OP_READ_INDIRECT,
+ XENVBD_STAT_BLKIF_OP_WRITE_DIRECT,
+ XENVBD_STAT_BLKIF_OP_WRITE_INDIRECT,
+ XENVBD_STAT_BLKIF_OP_WRITE_BARRIER,
+ XENVBD_STAT_BLKIF_OP_FLUSH_DISKCACHE,
+ XENVBD_STAT_BLKIF_OP_DISCARD,
+ XENVBD_STAT_BLKIF_OP_UNKNOWN,
+ XENVBD_STAT_SEGMENTS_GRANTED,
+ XENVBD_STAT_SEGMENTS_BOUNCED,
+
+ XENVBD_STAT__MAX
+} XENVBD_STAT, *PXENVBD_STAT;
+struct _XENVBD_RING {
+ PXENVBD_FRONTEND Frontend;
+ XENBUS_DEBUG_INTERFACE DebugInterface;
XENBUS_CACHE_INTERFACE CacheInterface;
XENBUS_STORE_INTERFACE StoreInterface;
XENBUS_EVTCHN_INTERFACE EvtchnInterface;
- XENBUS_DEBUG_INTERFACE DebugInterface;
-
PXENBUS_DEBUG_CALLBACK DebugCallback;
-
- KSPIN_LOCK Lock;
- PMDL Mdl;
- blkif_sring_t* Shared;
- blkif_front_ring_t Front;
ULONG Order;
- PVOID Grants[XENVBD_MAX_RING_PAGES];
- PXENBUS_EVTCHN_CHANNEL Channel;
- KDPC Dpc;
-
- PXENBUS_CACHE RequestCache;
- PXENBUS_CACHE SegmentCache;
- PXENBUS_CACHE IndirectCache;
- XENVBD_QUEUE PreparedReqs;
- XENVBD_QUEUE SubmittedReqs;
- XENVBD_QUEUE ShutdownSrbs;
-
- ULONG Submitted;
- ULONG Received;
- ULONG Events;
- ULONG Dpcs;
- ULONG BlkOpRead;
- ULONG BlkOpWrite;
- ULONG BlkOpIndirectRead;
- ULONG BlkOpIndirectWrite;
- ULONG BlkOpBarrier;
- ULONG BlkOpDiscard;
- ULONG BlkOpFlush;
- ULONG64 SegsGranted;
- ULONG64 SegsBounced;
+ PXENVBD_BLKIF_RING *Ring;
+ LONG Stats[XENVBD_STAT__MAX];
};
#define MAX_NAME_LEN 64
@@ -123,138 +147,119 @@ __RingFree(
RING_POOL_TAG);
}
-static FORCEINLINE VOID
-xen_mb()
+static FORCEINLINE PCHAR
+__BlkifOperationName(
+ IN UCHAR Operation
+ )
{
- KeMemoryBarrier();
- _ReadWriteBarrier();
+ switch (Operation) {
+ case BLKIF_OP_READ: return "READ";
+ case BLKIF_OP_WRITE: return "WRITE";
+ case BLKIF_OP_WRITE_BARRIER: return "WRITE_BARRIER";
+ case BLKIF_OP_FLUSH_DISKCACHE: return "FLUSH_DISKCACHE";
+ case BLKIF_OP_RESERVED_1: return "RESERVED_1";
+ case BLKIF_OP_DISCARD: return "DISCARD";
+ case BLKIF_OP_INDIRECT: return "INDIRECT";
+ default: return "<unknown>";
+ }
}
-static FORCEINLINE VOID
-xen_wmb()
+static FORCEINLINE PCHAR
+__StatName(
+ IN XENVBD_STAT Operation
+ )
{
- KeMemoryBarrier();
- _WriteBarrier();
+ switch (Operation) {
+ case XENVBD_STAT_BLKIF_OP_READ_DIRECT: return "BLKIF_OP_READ
(direct)";
+ case XENVBD_STAT_BLKIF_OP_READ_INDIRECT: return "BLKIF_OP_READ
(indirect)";
+ case XENVBD_STAT_BLKIF_OP_WRITE_DIRECT: return "BLKIF_OP_WRITE
(direct)";
+ case XENVBD_STAT_BLKIF_OP_WRITE_INDIRECT: return "BLKIF_OP_WRITE
(indirect)";
+ case XENVBD_STAT_BLKIF_OP_WRITE_BARRIER: return
"BLKIF_OP_WRITE_BARRIER";
+ case XENVBD_STAT_BLKIF_OP_FLUSH_DISKCACHE: return
"BLKIF_OP_FLUSH_DISKCACHE";
+ case XENVBD_STAT_BLKIF_OP_DISCARD: return "BLKIF_OP_DISCARD";
+ case XENVBD_STAT_SEGMENTS_GRANTED: return "SegmentsGranted";
+ case XENVBD_STAT_SEGMENTS_BOUNCED: return "SegmentsBounced";
+ default: return "UNKNOWN";
+ }
}
-static FORCEINLINE VOID
-__RingInsert(
- IN PXENVBD_RING Ring,
- IN PXENVBD_REQUEST Request,
- IN blkif_request_t* req
+static FORCEINLINE ULONG
+__SectorsPerPage(
+ IN ULONG SectorSize
)
{
- PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend);
-
- switch (Request->Operation) {
- case BLKIF_OP_READ:
- case BLKIF_OP_WRITE:
- if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) {
- // Indirect
- ULONG PageIdx;
- ULONG SegIdx;
- PLIST_ENTRY PageEntry;
- PLIST_ENTRY SegEntry;
- blkif_request_indirect_t* req_indirect;
-
- req_indirect = (blkif_request_indirect_t*)req;
- req_indirect->operation = BLKIF_OP_INDIRECT;
- req_indirect->indirect_op = Request->Operation;
- req_indirect->nr_segments = Request->NrSegments;
- req_indirect->id = (ULONG64)(ULONG_PTR)Request;
- req_indirect->sector_number = Request->FirstSector;
- req_indirect->handle =
(USHORT)FrontendGetDeviceId(Ring->Frontend);
-
- for (PageIdx = 0,
- PageEntry = Request->Indirects.Flink,
- SegEntry = Request->Segments.Flink;
- PageIdx < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST &&
- PageEntry != &Request->Indirects &&
- SegEntry != &Request->Segments;
- ++PageIdx, PageEntry = PageEntry->Flink) {
- PXENVBD_INDIRECT Page = CONTAINING_RECORD(PageEntry,
XENVBD_INDIRECT, ListEntry);
-
- req_indirect->indirect_grefs[PageIdx] =
GranterReference(Granter, Page->Grant);
-
- for (SegIdx = 0;
- SegIdx < XENVBD_MAX_SEGMENTS_PER_PAGE &&
- SegEntry != &Request->Segments;
- ++SegIdx, SegEntry = SegEntry->Flink) {
- PXENVBD_SEGMENT Segment = CONTAINING_RECORD(SegEntry,
XENVBD_SEGMENT, ListEntry);
-
- Page->Page[SegIdx].GrantRef = GranterReference(Granter,
Segment->Grant);
- Page->Page[SegIdx].First = Segment->FirstSector;
- Page->Page[SegIdx].Last = Segment->LastSector;
- }
- }
- } else {
- // Direct
- ULONG Index;
- PLIST_ENTRY Entry;
-
- req->operation = Request->Operation;
- req->nr_segments = (UCHAR)Request->NrSegments;
- req->handle =
(USHORT)FrontendGetDeviceId(Ring->Frontend);
- req->id = (ULONG64)(ULONG_PTR)Request;
- req->sector_number = Request->FirstSector;
+ ASSERT3U(SectorSize, != , 0);
+ return PAGE_SIZE / SectorSize;
+}
- for (Index = 0, Entry = Request->Segments.Flink;
- Index < BLKIF_MAX_SEGMENTS_PER_REQUEST &&
- Entry != &Request->Segments;
- ++Index, Entry = Entry->Flink) {
- PXENVBD_SEGMENT Segment = CONTAINING_RECORD(Entry,
XENVBD_SEGMENT, ListEntry);
- req->seg[Index].gref = GranterReference(Granter,
Segment->Grant);
- req->seg[Index].first_sect = Segment->FirstSector;
- req->seg[Index].last_sect = Segment->LastSector;
- }
- }
+static FORCEINLINE VOID
+__Operation(
+ IN UCHAR CdbOp,
+ OUT PUCHAR RingOp,
+ OUT PBOOLEAN ReadOnly
+ )
+{
+ switch (CdbOp) {
+ case SCSIOP_READ:
+ *RingOp = BLKIF_OP_READ;
+ *ReadOnly = FALSE;
break;
-
- case BLKIF_OP_WRITE_BARRIER:
- case BLKIF_OP_FLUSH_DISKCACHE:
- req->operation = Request->Operation;
- req->nr_segments = 0;
- req->handle =
(USHORT)FrontendGetDeviceId(Ring->Frontend);
- req->id = (ULONG64)(ULONG_PTR)Request;
- req->sector_number = Request->FirstSector;
+ case SCSIOP_WRITE:
+ *RingOp = BLKIF_OP_WRITE;
+ *ReadOnly = TRUE;
break;
-
- case BLKIF_OP_DISCARD: {
- blkif_request_discard_t* req_discard;
- req_discard = (blkif_request_discard_t*)req;
- req_discard->operation = BLKIF_OP_DISCARD;
- req_discard->flag = Request->Flags;
- req_discard->handle =
(USHORT)FrontendGetDeviceId(Ring->Frontend);
- req_discard->id = (ULONG64)(ULONG_PTR)Request;
- req_discard->sector_number = Request->FirstSector;
- req_discard->nr_sectors = Request->NrSectors;
- } break;
-
default:
ASSERT(FALSE);
- break;
}
- ++Ring->Submitted;
+}
+
+static FORCEINLINE ULONG
+__UseIndirect(
+ IN ULONG SectorsPerPage,
+ IN ULONG MaxIndirectSegs,
+ IN ULONG SectorsLeft
+ )
+{
+ if (MaxIndirectSegs <= BLKIF_MAX_SEGMENTS_PER_REQUEST)
+ return BLKIF_MAX_SEGMENTS_PER_REQUEST; // not supported
+
+ if (SectorsLeft < BLKIF_MAX_SEGMENTS_PER_REQUEST * SectorsPerPage)
+ return BLKIF_MAX_SEGMENTS_PER_REQUEST; // first into a single
BLKIF_OP_{READ/WRITE}
+
+ return MaxIndirectSegs;
+}
+
+static FORCEINLINE MM_PAGE_PRIORITY
+__Priority(
+ IN PXENVBD_CAPS Caps
+ )
+{
+ return (Caps->Paging ||
+ Caps->Hibernation ||
+ Caps->DumpFile) ? HighPagePriority :
+ NormalPagePriority;
}
static PXENVBD_INDIRECT
-RingGetIndirect(
- IN PXENVBD_RING Ring
+BlkifRingGetIndirect(
+ IN PXENVBD_BLKIF_RING BlkifRing
)
{
- PXENVBD_INDIRECT Indirect;
- NTSTATUS status;
- PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend);
+ PXENVBD_RING Ring = BlkifRing->Ring;
+ PXENVBD_FRONTEND Frontend = Ring->Frontend;
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Frontend);
+ PXENVBD_INDIRECT Indirect;
+ NTSTATUS status;
Indirect = XENBUS_CACHE(Get,
&Ring->CacheInterface,
- Ring->IndirectCache,
- FALSE);
+ BlkifRing->IndirectCache,
+ TRUE);
if (Indirect == NULL)
goto fail1;
- ASSERT3P(Indirect->Mdl, !=, NULL);
- ASSERT3P(Indirect->Page, !=, NULL);
+ ASSERT3P(Indirect->Mdl, != , NULL);
+ ASSERT3P(Indirect->Page, != , NULL);
status = GranterGet(Granter,
MmGetMdlPfnArray(Indirect->Mdl)[0],
TRUE,
@@ -267,20 +272,22 @@ RingGetIndirect(
fail2:
XENBUS_CACHE(Put,
&Ring->CacheInterface,
- Ring->IndirectCache,
+ BlkifRing->IndirectCache,
Indirect,
- FALSE);
+ TRUE);
fail1:
return NULL;
}
static VOID
-RingPutIndirect(
- IN PXENVBD_RING Ring,
+BlkifRingPutIndirect(
+ IN PXENVBD_BLKIF_RING BlkifRing,
IN PXENVBD_INDIRECT Indirect
)
{
- PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend);
+ PXENVBD_RING Ring = BlkifRing->Ring;
+ PXENVBD_FRONTEND Frontend = Ring->Frontend;
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Frontend);
if (Indirect->Grant)
GranterPut(Granter, Indirect->Grant);
@@ -290,30 +297,34 @@ RingPutIndirect(
XENBUS_CACHE(Put,
&Ring->CacheInterface,
- Ring->IndirectCache,
+ BlkifRing->IndirectCache,
Indirect,
- FALSE);
+ TRUE);
}
static PXENVBD_SEGMENT
-RingGetSegment(
- IN PXENVBD_RING Ring
+BlkifRingGetSegment(
+ IN PXENVBD_BLKIF_RING BlkifRing
)
{
+ PXENVBD_RING Ring = BlkifRing->Ring;
+
return XENBUS_CACHE(Get,
&Ring->CacheInterface,
- Ring->SegmentCache,
- FALSE);
+ BlkifRing->SegmentCache,
+ TRUE);
}
static VOID
-RingPutSegment(
- IN PXENVBD_RING Ring,
- IN PXENVBD_SEGMENT Segment
+BlkifRingPutSegment(
+ IN PXENVBD_BLKIF_RING BlkifRing,
+ IN PXENVBD_SEGMENT Segment
)
{
- PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend);
- PXENVBD_BOUNCE Bounce = Segment->Bounce;
+ PXENVBD_RING Ring = BlkifRing->Ring;
+ PXENVBD_FRONTEND Frontend = Ring->Frontend;
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Frontend);
+ PXENVBD_BOUNCE Bounce = Segment->Bounce;
if (Segment->Grant)
GranterPut(Granter, Segment->Grant);
@@ -329,7 +340,7 @@ RingPutSegment(
Bounce->SourcePfn[0] = 0;
Bounce->SourcePfn[1] = 0;
- AdapterPutBounce(TargetGetAdapter(FrontendGetTarget(Ring->Frontend)),
+ AdapterPutBounce(TargetGetAdapter(FrontendGetTarget(Frontend)),
Bounce);
}
Segment->Bounce = NULL;
@@ -340,29 +351,32 @@ RingPutSegment(
XENBUS_CACHE(Put,
&Ring->CacheInterface,
- Ring->SegmentCache,
+ BlkifRing->SegmentCache,
Segment,
- FALSE);
+ TRUE);
}
static PXENVBD_REQUEST
-RingGetRequest(
- IN PXENVBD_RING Ring
+BlkifRingGetRequest(
+ IN PXENVBD_BLKIF_RING BlkifRing
)
{
+ PXENVBD_RING Ring = BlkifRing->Ring;
+
return XENBUS_CACHE(Get,
&Ring->CacheInterface,
- Ring->RequestCache,
- FALSE);
+ BlkifRing->RequestCache,
+ TRUE);
}
static VOID
-RingPutRequest(
- IN PXENVBD_RING Ring,
- IN PXENVBD_REQUEST Request
+BlkifRingPutRequest(
+ IN PXENVBD_BLKIF_RING BlkifRing,
+ IN PXENVBD_REQUEST Request
)
{
- PLIST_ENTRY ListEntry;
+ PXENVBD_RING Ring = BlkifRing->Ring;
+ PLIST_ENTRY ListEntry;
for (;;) {
PXENVBD_SEGMENT Segment;
@@ -371,7 +385,7 @@ RingPutRequest(
if (ListEntry == &Request->Segments)
break;
Segment = CONTAINING_RECORD(ListEntry, XENVBD_SEGMENT, ListEntry);
- RingPutSegment(Ring, Segment);
+ BlkifRingPutSegment(BlkifRing, Segment);
}
for (;;) {
@@ -381,7 +395,7 @@ RingPutRequest(
if (ListEntry == &Request->Indirects)
break;
Indirect = CONTAINING_RECORD(ListEntry, XENVBD_INDIRECT, ListEntry);
- RingPutIndirect(Ring, Indirect);
+ BlkifRingPutIndirect(BlkifRing, Indirect);
}
Request->SrbExt = NULL;
@@ -394,189 +408,277 @@ RingPutRequest(
XENBUS_CACHE(Put,
&Ring->CacheInterface,
- Ring->RequestCache,
+ BlkifRing->RequestCache,
Request,
- FALSE);
+ TRUE);
}
-static FORCEINLINE PXENVBD_REQUEST
-RingFindRequest(
- IN PXENVBD_RING Ring,
- IN ULONG64 Id
+static DECLSPEC_NOINLINE NTSTATUS
+BlkifRingRequestCtor(
+ IN PVOID Argument,
+ IN PVOID Object
)
{
- KIRQL Irql;
- PLIST_ENTRY ListEntry;
- PXENVBD_REQUEST Request;
- PXENVBD_QUEUE Queue = &Ring->SubmittedReqs;
-
- KeAcquireSpinLock(&Queue->Lock, &Irql);
+ PXENVBD_REQUEST Request = Object;
- for (ListEntry = Queue->List.Flink;
- ListEntry != &Queue->List;
- ListEntry = ListEntry->Flink) {
- Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, ListEntry);
- if ((ULONG64)(ULONG_PTR)Request == Id) {
- RemoveEntryList(&Request->ListEntry);
- --Queue->Current;
- KeReleaseSpinLock(&Queue->Lock, Irql);
- return Request;
- }
- }
+ UNREFERENCED_PARAMETER(Argument);
- KeReleaseSpinLock(&Queue->Lock, Irql);
- Warning("Target[%d] : Tag %llx not found in submitted list (%u items)\n",
- FrontendGetTargetId(Ring->Frontend),
- Id,
- QueueCount(Queue));
- return NULL;
+ InitializeListHead(&Request->Segments);
+ InitializeListHead(&Request->Indirects);
+ return STATUS_SUCCESS;
}
-static FORCEINLINE VOID
-__RingIncBlkifOpCount(
- IN PXENVBD_RING Ring,
- IN PXENVBD_REQUEST Request
+static DECLSPEC_NOINLINE VOID
+BlkifRingRequestDtor(
+ IN PVOID Argument,
+ IN PVOID Object
)
{
- switch (Request->Operation) {
- case BLKIF_OP_READ:
- if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST)
- ++Ring->BlkOpIndirectRead;
- else
- ++Ring->BlkOpRead;
- break;
- case BLKIF_OP_WRITE:
- if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST)
- ++Ring->BlkOpIndirectWrite;
- else
- ++Ring->BlkOpWrite;
- break;
- case BLKIF_OP_WRITE_BARRIER:
- ++Ring->BlkOpBarrier;
- break;
- case BLKIF_OP_DISCARD:
- ++Ring->BlkOpDiscard;
- break;
- case BLKIF_OP_FLUSH_DISKCACHE:
- ++Ring->BlkOpFlush;
- break;
- default:
- ASSERT(FALSE);
- break;
- }
+ UNREFERENCED_PARAMETER(Argument);
+ UNREFERENCED_PARAMETER(Object);
}
-static FORCEINLINE ULONG
-__RingSectorsPerPage(
- IN ULONG SectorSize
+static DECLSPEC_NOINLINE NTSTATUS
+BlkifRingSegmentCtor(
+ IN PVOID Argument,
+ IN PVOID Object
)
{
- ASSERT3U(SectorSize, !=, 0);
- return PAGE_SIZE / SectorSize;
+ UNREFERENCED_PARAMETER(Argument);
+ UNREFERENCED_PARAMETER(Object);
+ return STATUS_SUCCESS;
}
-static FORCEINLINE VOID
-__RingOperation(
- IN UCHAR CdbOp,
- OUT PUCHAR RingOp,
- OUT PBOOLEAN ReadOnly
+static DECLSPEC_NOINLINE VOID
+BlkifRingSegmentDtor(
+ IN PVOID Argument,
+ IN PVOID Object
)
{
- switch (CdbOp) {
- case SCSIOP_READ:
- *RingOp = BLKIF_OP_READ;
- *ReadOnly = FALSE;
- break;
- case SCSIOP_WRITE:
- *RingOp = BLKIF_OP_WRITE;
- *ReadOnly = TRUE;
- break;
- default:
- ASSERT(FALSE);
- }
+ UNREFERENCED_PARAMETER(Argument);
+ UNREFERENCED_PARAMETER(Object);
}
-static FORCEINLINE MM_PAGE_PRIORITY
-__RingPriority(
- IN PXENVBD_RING Ring
+static DECLSPEC_NOINLINE NTSTATUS
+BlkifRingIndirectCtor(
+ IN PVOID Argument,
+ IN PVOID Object
)
{
- PXENVBD_CAPS Caps = FrontendGetCaps(Ring->Frontend);
- if (!(Caps->Paging ||
- Caps->Hibernation ||
- Caps->DumpFile))
- return NormalPagePriority;
+ PXENVBD_INDIRECT Indirect = Object;
+ NTSTATUS status;
+
+ UNREFERENCED_PARAMETER(Argument);
+
+ status = STATUS_NO_MEMORY;
+ Indirect->Mdl = __AllocatePage();
+ if (Indirect->Mdl == NULL)
+ goto fail1;
+
+ Indirect->Page = MmGetSystemAddressForMdlSafe(Indirect->Mdl,
+ NormalPagePriority);
+ ASSERT(Indirect->Page);
- return HighPagePriority;
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 %08x\n", status);
+ return status;
}
-static FORCEINLINE VOID
-RingRequestCopyOutput(
- IN PXENVBD_REQUEST Request
+static DECLSPEC_NOINLINE VOID
+BlkifRingIndirectDtor(
+ IN PVOID Argument,
+ IN PVOID Object
)
{
- PLIST_ENTRY ListEntry;
-
- if (Request->Operation != BLKIF_OP_READ)
- return;
+ PXENVBD_INDIRECT Indirect = Object;
- for (ListEntry = Request->Segments.Flink;
- ListEntry != &Request->Segments;
- ListEntry = ListEntry->Flink) {
- PXENVBD_SEGMENT Segment = CONTAINING_RECORD(ListEntry, XENVBD_SEGMENT,
ListEntry);
- PXENVBD_BOUNCE Bounce = Segment->Bounce;
+ UNREFERENCED_PARAMETER(Argument);
- if (Bounce) {
- RtlCopyMemory(Bounce->SourcePtr,
- Bounce->BouncePtr,
- MmGetMdlByteCount(&Bounce->SourceMdl));
- }
- }
+ __FreePages(Indirect->Mdl);
+ Indirect->Page = NULL;
+ Indirect->Mdl = NULL;
}
-static BOOLEAN
-RingPrepareSegment(
- IN PXENVBD_RING Ring,
- IN PXENVBD_SEGMENT Segment,
- IN PXENVBD_SRBEXT SrbExt,
- IN BOOLEAN ReadOnly,
- IN ULONG SectorsLeft,
- OUT PULONG SectorsNow
- )
-{
- PFN_NUMBER Pfn;
- ULONG Offset;
- ULONG Length;
- NTSTATUS Status;
- PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend);
- const ULONG SectorSize =
FrontendGetDiskInfo(Ring->Frontend)->SectorSize;
- const ULONG SectorsPerPage = __RingSectorsPerPage(SectorSize);
- PXENVBD_TARGET Target = FrontendGetTarget(Ring->Frontend);
- PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target);
+static VOID
+BlkifRingDebugCallback(
+ IN PVOID Argument,
+ IN BOOLEAN Crashing
+ )
+{
+ PXENVBD_BLKIF_RING BlkifRing = Argument;
+ PXENVBD_RING Ring = BlkifRing->Ring;
- Pfn = AdapterGetNextSGEntry(Adapter,
- SrbExt,
- 0,
- &Offset,
- &Length);
- if ((Offset & (SectorSize - 1)) == 0 &&
+ UNREFERENCED_PARAMETER(Crashing);
+
+ XENBUS_DEBUG(Printf,
+ &Ring->DebugInterface,
+ "0x%p [%s]\n",
+ BlkifRing,
+ (BlkifRing->Enabled) ? "ENABLED" : "DISABLED");
+
+ // Dump front ring
+ XENBUS_DEBUG(Printf,
+ &Ring->DebugInterface,
+ "FRONT: req_prod_pvt = %u rsp_cons = %u nr_ents = %u sring =
%p\n",
+ BlkifRing->Front.req_prod_pvt,
+ BlkifRing->Front.rsp_cons,
+ BlkifRing->Front.nr_ents,
+ BlkifRing->Front.sring);
+
+ // Dump shared ring
+ XENBUS_DEBUG(Printf,
+ &Ring->DebugInterface,
+ "SHARED: req_prod = %u req_event = %u rsp_prod = %u rsp_event
= %u\n",
+ BlkifRing->Shared->req_prod,
+ BlkifRing->Shared->req_event,
+ BlkifRing->Shared->rsp_prod,
+ BlkifRing->Shared->rsp_event);
+
+ XENBUS_DEBUG(Printf,
+ &Ring->DebugInterface,
+ "RequestsPosted = %u RequestsPushed = %u ResponsesProcessed =
%u\n",
+ BlkifRing->RequestsPosted,
+ BlkifRing->RequestsPushed,
+ BlkifRing->ResponsesProcessed);
+
+ XENBUS_DEBUG(Printf,
+ &Ring->DebugInterface,
+ "SrbsQueued = %u, SrbsCompleted = %u, SrbsFailed = %u\n",
+ BlkifRing->SrbsQueued,
+ BlkifRing->SrbsCompleted,
+ BlkifRing->SrbsFailed);
+
+ XENBUS_DEBUG(Printf,
+ &Ring->DebugInterface,
+ "Dpcs = %u, Events = %u\n",
+ BlkifRing->Dpcs,
+ BlkifRing->Events);
+}
+
+static DECLSPEC_NOINLINE VOID
+__BlkifRingCompleteSrb(
+ IN PXENVBD_BLKIF_RING BlkifRing,
+ IN PXENVBD_SRBEXT SrbExt
+ )
+{
+ PXENVBD_RING Ring = BlkifRing->Ring;
+ PXENVBD_FRONTEND Frontend = Ring->Frontend;
+ PXENVBD_TARGET Target = FrontendGetTarget(Frontend);
+ PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target);
+ PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
+
+ if (Srb->SrbStatus == SRB_STATUS_PENDING) {
+ // SRB has not hit a failure condition (BLKIF_RSP_ERROR |
BLKIF_RSP_EOPNOTSUPP)
+ // from any of its responses. SRB must have succeeded
+ Srb->SrbStatus = SRB_STATUS_SUCCESS;
+ Srb->ScsiStatus = 0x00; // SCSI_GOOD
+ ++BlkifRing->SrbsCompleted;
+ } else {
+ // Srb->SrbStatus has already been set by 1 or more requests with
Status != BLKIF_RSP_OKAY
+ Srb->ScsiStatus = 0x40; // SCSI_ABORTED
+ ++BlkifRing->SrbsFailed;
+ }
+
+ AdapterCompleteSrb(Adapter, SrbExt);
+}
+
+static FORCEINLINE VOID
+BlkifRingUnprepareRequest(
+ IN PXENVBD_BLKIF_RING BlkifRing,
+ IN PLIST_ENTRY List
+ )
+{
+ for (;;) {
+ PLIST_ENTRY ListEntry;
+ PXENVBD_REQUEST Request;
+
+ ListEntry = RemoveHeadList(List);
+ if (ListEntry == List)
+ break;
+
+ Request = CONTAINING_RECORD(ListEntry,
+ XENVBD_REQUEST,
+ ListEntry);
+
+ BlkifRingPutRequest(BlkifRing, Request);
+ }
+}
+
+static FORCEINLINE VOID
+BlkifRingQueueRequests(
+ IN PXENVBD_BLKIF_RING BlkifRing,
+ IN PLIST_ENTRY List
+ )
+{
+ PXENVBD_SRB_STATE State = &BlkifRing->State;
+
+ for (;;) {
+ PLIST_ENTRY ListEntry;
+ PXENVBD_REQUEST Request;
+
+ ListEntry = RemoveHeadList(List);
+ if (ListEntry == List)
+ break;
+
+ Request = CONTAINING_RECORD(ListEntry,
+ XENVBD_REQUEST,
+ ListEntry);
+
+ InsertTailList(&State->List, ListEntry);
+ State->Count++;
+ }
+}
+
+static BOOLEAN
+BlkifRingPrepareSegment(
+ IN PXENVBD_BLKIF_RING BlkifRing,
+ IN PXENVBD_SEGMENT Segment,
+ IN PXENVBD_SRBEXT SrbExt,
+ IN BOOLEAN ReadOnly,
+ IN ULONG SectorsLeft,
+ OUT PULONG SectorsNow
+ )
+{
+ PFN_NUMBER Pfn;
+ ULONG Offset;
+ ULONG Length;
+ NTSTATUS Status;
+ PXENVBD_RING Ring = BlkifRing->Ring;
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend);
+ PXENVBD_TARGET Target = FrontendGetTarget(Ring->Frontend);
+ PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target);
+
+ const ULONG SectorSize =
FrontendGetDiskInfo(Ring->Frontend)->SectorSize;
+ const ULONG SectorsPerPage = __SectorsPerPage(SectorSize);
+
+ Pfn = AdapterGetNextSGEntry(Adapter,
+ SrbExt,
+ 0,
+ &Offset,
+ &Length);
+ if ((Offset & (SectorSize - 1)) == 0 &&
(Length & (SectorSize - 1)) == 0) {
- ++Ring->SegsGranted;
+ InterlockedIncrement(&Ring->Stats[XENVBD_STAT_SEGMENTS_GRANTED]);
+
// get first sector, last sector and count
- Segment->FirstSector = (UCHAR)((Offset + SectorSize - 1) /
SectorSize);
- *SectorsNow = __min(SectorsLeft, SectorsPerPage -
Segment->FirstSector);
- Segment->LastSector = (UCHAR)(Segment->FirstSector + *SectorsNow -
1);
+ Segment->FirstSector = (UCHAR)((Offset + SectorSize - 1) / SectorSize);
+ *SectorsNow = __min(SectorsLeft, SectorsPerPage -
Segment->FirstSector);
+ Segment->LastSector = (UCHAR)(Segment->FirstSector + *SectorsNow - 1);
- ASSERT3U((Length / SectorSize), ==, *SectorsNow);
+ ASSERT3U((Length / SectorSize), == , *SectorsNow);
} else {
PXENVBD_BOUNCE Bounce;
PMDL Mdl;
+ PXENVBD_CAPS Caps = FrontendGetCaps(Ring->Frontend);
+
+ InterlockedIncrement(&Ring->Stats[XENVBD_STAT_SEGMENTS_BOUNCED]);
- ++Ring->SegsBounced;
// get first sector, last sector and count
- Segment->FirstSector = 0;
- *SectorsNow = __min(SectorsLeft, SectorsPerPage);
- Segment->LastSector = (UCHAR)(*SectorsNow - 1);
+ Segment->FirstSector = 0;
+ *SectorsNow = __min(SectorsLeft, SectorsPerPage);
+ Segment->LastSector = (UCHAR)(*SectorsNow - 1);
Bounce = AdapterGetBounce(Adapter);
if (Bounce == NULL)
@@ -586,15 +688,15 @@ RingPrepareSegment(
#pragma warning(push)
#pragma warning(disable:28145)
Mdl = &Bounce->SourceMdl;
- Mdl->Next = NULL;
- Mdl->Size = (SHORT)(sizeof(MDL) + sizeof(PFN_NUMBER));
- Mdl->MdlFlags = MDL_PAGES_LOCKED;
- Mdl->Process = NULL;
- Mdl->MappedSystemVa = NULL;
- Mdl->StartVa = NULL;
- Mdl->ByteCount = Length;
- Mdl->ByteOffset = Offset;
- Bounce->SourcePfn[0] = Pfn;
+ Mdl->Next = NULL;
+ Mdl->Size = (SHORT)(sizeof(MDL) + sizeof(PFN_NUMBER));
+ Mdl->MdlFlags = MDL_PAGES_LOCKED;
+ Mdl->Process = NULL;
+ Mdl->MappedSystemVa = NULL;
+ Mdl->StartVa = NULL;
+ Mdl->ByteCount = Length;
+ Mdl->ByteOffset = Offset;
+ Bounce->SourcePfn[0] = Pfn;
if (Length < *SectorsNow * SectorSize) {
Pfn = AdapterGetNextSGEntry(Adapter,
@@ -602,27 +704,27 @@ RingPrepareSegment(
Length,
&Offset,
&Length);
- Mdl->Size += sizeof(PFN_NUMBER);
- Mdl->ByteCount += Length;
+ Mdl->Size += sizeof(PFN_NUMBER);
+ Mdl->ByteCount += Length;
Bounce->SourcePfn[1] = Pfn;
}
#pragma warning(pop)
ASSERT((Mdl->ByteCount & (SectorSize - 1)) == 0);
- ASSERT3U(Mdl->ByteCount, <=, PAGE_SIZE);
- ASSERT3U(*SectorsNow, ==, (Mdl->ByteCount / SectorSize));
+ ASSERT3U(Mdl->ByteCount, <= , PAGE_SIZE);
+ ASSERT3U(*SectorsNow, == , (Mdl->ByteCount / SectorSize));
Bounce->SourcePtr = MmMapLockedPagesSpecifyCache(Mdl,
KernelMode,
MmCached,
NULL,
FALSE,
- __RingPriority(Ring));
+ __Priority(Caps));
if (Bounce->SourcePtr == NULL)
goto fail2;
- ASSERT3P(MmGetMdlPfnArray(Mdl)[0], ==, Bounce->SourcePfn[0]);
- ASSERT3P(MmGetMdlPfnArray(Mdl)[1], ==, Bounce->SourcePfn[1]);
+ ASSERT3P(MmGetMdlPfnArray(Mdl)[0], == , Bounce->SourcePfn[0]);
+ ASSERT3P(MmGetMdlPfnArray(Mdl)[1], == , Bounce->SourcePfn[1]);
// copy contents in
if (ReadOnly) { // Operation == BLKIF_OP_WRITE
@@ -647,523 +749,510 @@ fail1:
return FALSE;
}
-static BOOLEAN
-RingPrepareBlkifReadWrite(
- IN PXENVBD_RING Ring,
- IN PXENVBD_REQUEST Request,
- IN PXENVBD_SRBEXT SrbExt,
- IN ULONG MaxSegments,
- IN ULONG64 SectorStart,
- IN ULONG SectorsLeft,
- OUT PULONG SectorsDone
+static NTSTATUS
+BlkifRingPrepareReadWrite(
+ IN PXENVBD_BLKIF_RING BlkifRing,
+ IN PXENVBD_SRBEXT SrbExt
)
{
- PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
- UCHAR Operation;
- BOOLEAN ReadOnly;
- ULONG Index;
- __RingOperation(Cdb_OperationEx(Srb), &Operation, &ReadOnly);
-
- Request->Operation = Operation;
- Request->NrSegments = 0;
- Request->FirstSector = SectorStart;
-
- for (Index = 0;
- Index < MaxSegments &&
- SectorsLeft > 0;
- ++Index) {
- PXENVBD_SEGMENT Segment;
- ULONG SectorsNow;
-
- Segment = RingGetSegment(Ring);
- if (Segment == NULL)
- goto fail1;
-
- InsertTailList(&Request->Segments, &Segment->ListEntry);
- ++Request->NrSegments;
+ PXENVBD_RING Ring = BlkifRing->Ring;
+ PXENVBD_FRONTEND Frontend = Ring->Frontend;
+ PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
+ ULONG64 SectorStart = Cdb_LogicalBlock(Srb);
+ ULONG SectorsLeft = Cdb_TransferBlock(Srb);
+ UCHAR Operation;
+ BOOLEAN ReadOnly;
+ LIST_ENTRY List;
- if (!RingPrepareSegment(Ring,
- Segment,
- SrbExt,
- ReadOnly,
- SectorsLeft,
- &SectorsNow))
- goto fail2;
+ const ULONG SectorSize =
FrontendGetDiskInfo(Frontend)->SectorSize;
+ const ULONG SectorsPerPage = __SectorsPerPage(SectorSize);
+ const ULONG MaxIndirect =
FrontendGetFeatures(Frontend)->Indirect;
- *SectorsDone += SectorsNow;
- SectorsLeft -= SectorsNow;
- }
- ASSERT3U(Request->NrSegments, >, 0);
- ASSERT3U(Request->NrSegments, <=, MaxSegments);
+ InitializeListHead(&List);
- return TRUE;
+ __Operation(Cdb_OperationEx(Srb), &Operation, &ReadOnly);
-fail2:
-fail1:
- return FALSE;
-}
+ Srb->SrbStatus = SRB_STATUS_PENDING;
-static BOOLEAN
-RingPrepareBlkifIndirect(
- IN PXENVBD_RING Ring,
- IN PXENVBD_REQUEST Request
- )
-{
- ULONG Index;
- ULONG NrSegments = 0;
+ SrbExt->RequestCount = 0;
- for (Index = 0;
- Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST &&
- NrSegments < Request->NrSegments;
- ++Index) {
- PXENVBD_INDIRECT Indirect;
+ while (SectorsLeft > 0) {
+ ULONG Index;
+ ULONG MaxSegments;
+ PXENVBD_REQUEST Request;
- Indirect = RingGetIndirect(Ring);
- if (Indirect == NULL)
+ Request = BlkifRingGetRequest(BlkifRing);
+ if (Request == NULL)
goto fail1;
- InsertTailList(&Request->Indirects, &Indirect->ListEntry);
-
- NrSegments += XENVBD_MAX_SEGMENTS_PER_PAGE;
- }
-
- return TRUE;
-
-fail1:
- return FALSE;
-}
+ InsertTailList(&List, &Request->ListEntry);
+ SrbExt->RequestCount++;
-static FORCEINLINE ULONG
-RingUseIndirect(
- IN PXENVBD_RING Ring,
- IN ULONG SectorsLeft
- )
-{
- const ULONG SectorsPerPage =
__RingSectorsPerPage(FrontendGetDiskInfo(Ring->Frontend)->SectorSize);
- const ULONG MaxIndirectSegs =
FrontendGetFeatures(Ring->Frontend)->Indirect;
+ Request->SrbExt = SrbExt;
- if (MaxIndirectSegs <= BLKIF_MAX_SEGMENTS_PER_REQUEST)
- return BLKIF_MAX_SEGMENTS_PER_REQUEST; // not supported
+ MaxSegments = __UseIndirect(SectorsPerPage,
+ MaxIndirect,
+ SectorsLeft);
+
+ Request->Operation = Operation;
+ Request->NrSegments = 0;
+ Request->FirstSector = SectorStart;
+
+ for (Index = 0;
+ Index < MaxSegments &&
+ SectorsLeft > 0;
+ ++Index) {
+ PXENVBD_SEGMENT Segment;
+ ULONG SectorsNow;
+
+ Segment = BlkifRingGetSegment(BlkifRing);
+ if (Segment == NULL)
+ goto fail2;
+
+ InsertTailList(&Request->Segments, &Segment->ListEntry);
+ ++Request->NrSegments;
+
+ if (!BlkifRingPrepareSegment(BlkifRing,
+ Segment,
+ SrbExt,
+ ReadOnly,
+ SectorsLeft,
+ &SectorsNow))
+ goto fail3;
- if (SectorsLeft < BLKIF_MAX_SEGMENTS_PER_REQUEST * SectorsPerPage)
- return BLKIF_MAX_SEGMENTS_PER_REQUEST; // first into a single
BLKIF_OP_{READ/WRITE}
+ SectorsLeft -= SectorsNow;
+ SectorStart += SectorsNow;
+ }
+ ASSERT3U(Request->NrSegments, >, 0);
+ ASSERT3U(Request->NrSegments, <= , MaxSegments);
- return MaxIndirectSegs;
-}
+ if (MaxSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) {
+ ULONG NrSegments = 0;
-static FORCEINLINE VOID
-RingQueueRequestList(
- IN PXENVBD_RING Ring,
- IN PLIST_ENTRY List
- )
-{
- for (;;) {
- PXENVBD_REQUEST Request;
- PLIST_ENTRY ListEntry;
+ for (Index = 0;
+ Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST &&
+ NrSegments < Request->NrSegments;
+ ++Index) {
+ PXENVBD_INDIRECT Indirect;
- ListEntry = RemoveHeadList(List);
- if (ListEntry == List)
- break;
+ Indirect = BlkifRingGetIndirect(BlkifRing);
+ if (Indirect == NULL)
+ goto fail3;
+ InsertTailList(&Request->Indirects, &Indirect->ListEntry);
- Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, ListEntry);
- __RingIncBlkifOpCount(Ring, Request);
- QueueAppend(&Ring->PreparedReqs, &Request->ListEntry);
+ NrSegments += XENVBD_MAX_SEGMENTS_PER_PAGE;
+ }
+ }
}
-}
-
-static FORCEINLINE VOID
-RingCancelRequestList(
- IN PXENVBD_RING Ring,
- IN PLIST_ENTRY List
- )
-{
- for (;;) {
- PXENVBD_REQUEST Request;
- PLIST_ENTRY ListEntry;
- ListEntry = RemoveHeadList(List);
- if (ListEntry == List)
- break;
+ BlkifRingQueueRequests(BlkifRing, &List);
+ return STATUS_SUCCESS;
- Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, ListEntry);
- RingPutRequest(Ring, Request);
- }
+fail3:
+fail2:
+fail1:
+ BlkifRingUnprepareRequest(BlkifRing, &List);
+ SrbExt->RequestCount = 0;
+ Srb->SrbStatus = SRB_STATUS_ERROR;
+ return STATUS_UNSUCCESSFUL;
}
-static BOOLEAN
-RingPrepareReadWrite(
- IN PXENVBD_RING Ring,
+static NTSTATUS
+BlkifRingPrepareUnmap(
+ IN PXENVBD_BLKIF_RING BlkifRing,
IN PXENVBD_SRBEXT SrbExt
)
{
PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
- ULONG64 SectorStart = Cdb_LogicalBlock(Srb);
- ULONG SectorsLeft = Cdb_TransferBlock(Srb);
+ PUNMAP_LIST_HEADER Unmap = Srb->DataBuffer;
+ ULONG Count;
+ ULONG Index;
LIST_ENTRY List;
+ InitializeListHead(&List);
+
+ Count = _byteswap_ushort(*(PUSHORT)Unmap->BlockDescrDataLength) /
sizeof(UNMAP_BLOCK_DESCRIPTOR);
Srb->SrbStatus = SRB_STATUS_PENDING;
- InitializeListHead(&List);
SrbExt->RequestCount = 0;
- while (SectorsLeft > 0) {
- ULONG MaxSegments;
- ULONG SectorsDone = 0;
- PXENVBD_REQUEST Request;
+ for (Index = 0; Index < Count; ++Index) {
+ PUNMAP_BLOCK_DESCRIPTOR Descr = &Unmap->Descriptors[Index];
+ PXENVBD_REQUEST Request;
- Request = RingGetRequest(Ring);
+ Request = BlkifRingGetRequest(BlkifRing);
if (Request == NULL)
goto fail1;
InsertTailList(&List, &Request->ListEntry);
- InterlockedIncrement(&SrbExt->RequestCount);
+ SrbExt->RequestCount++;
Request->SrbExt = SrbExt;
- MaxSegments = RingUseIndirect(Ring, SectorsLeft);
-
- if (!RingPrepareBlkifReadWrite(Ring,
- Request,
- SrbExt,
- MaxSegments,
- SectorStart,
- SectorsLeft,
- &SectorsDone))
- goto fail2;
-
- if (MaxSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) {
- if (!RingPrepareBlkifIndirect(Ring, Request))
- goto fail3;
- }
-
- SectorsLeft -= SectorsDone;
- SectorStart += SectorsDone;
+ Request->Operation = BLKIF_OP_DISCARD;
+ Request->FirstSector = _byteswap_uint64(*(PULONG64)Descr->StartingLba);
+ Request->NrSectors = _byteswap_ulong(*(PULONG)Descr->LbaCount);
+ Request->Flags = 0;
}
- RingQueueRequestList(Ring, &List);
- return TRUE;
+ BlkifRingQueueRequests(BlkifRing, &List);
+ return STATUS_SUCCESS;
-fail3:
-fail2:
fail1:
- RingCancelRequestList(Ring, &List);
+ BlkifRingUnprepareRequest(BlkifRing, &List);
SrbExt->RequestCount = 0;
Srb->SrbStatus = SRB_STATUS_ERROR;
- return FALSE;
+ return STATUS_UNSUCCESSFUL;
}
-static BOOLEAN
-RingPrepareSyncCache(
- IN PXENVBD_RING Ring,
+static NTSTATUS
+BlkifRingPrepareSyncCache(
+ IN PXENVBD_BLKIF_RING BlkifRing,
IN PXENVBD_SRBEXT SrbExt
)
{
+ PXENVBD_RING Ring = BlkifRing->Ring;
+ PXENVBD_FRONTEND Frontend = Ring->Frontend;
PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
PXENVBD_REQUEST Request;
- LIST_ENTRY List;
UCHAR Operation;
+ LIST_ENTRY List;
+ InitializeListHead(&List);
Srb->SrbStatus = SRB_STATUS_PENDING;
- if (FrontendGetDiskInfo(Ring->Frontend)->FlushCache)
+ if (FrontendGetDiskInfo(Frontend)->FlushCache)
Operation = BLKIF_OP_FLUSH_DISKCACHE;
else
Operation = BLKIF_OP_WRITE_BARRIER;
- InitializeListHead(&List);
SrbExt->RequestCount = 0;
- Request = RingGetRequest(Ring);
+ Request = BlkifRingGetRequest(BlkifRing);
if (Request == NULL)
goto fail1;
InsertTailList(&List, &Request->ListEntry);
- InterlockedIncrement(&SrbExt->RequestCount);
+ SrbExt->RequestCount++;
- Request->SrbExt = SrbExt;
- Request->Operation = Operation;
+ Request->SrbExt = SrbExt;
+ Request->Operation = Operation;
Request->FirstSector = Cdb_LogicalBlock(Srb);
- RingQueueRequestList(Ring, &List);
- return TRUE;
+ BlkifRingQueueRequests(BlkifRing, &List);
+ return STATUS_SUCCESS;
fail1:
- RingCancelRequestList(Ring, &List);
+ BlkifRingUnprepareRequest(BlkifRing, &List);
SrbExt->RequestCount = 0;
Srb->SrbStatus = SRB_STATUS_ERROR;
- return FALSE;
+ return STATUS_UNSUCCESSFUL;
}
-static BOOLEAN
-RingPrepareUnmap(
- IN PXENVBD_RING Ring,
+static DECLSPEC_NOINLINE NTSTATUS
+__BlkifRingPrepareSrb(
+ IN PXENVBD_BLKIF_RING BlkifRing,
IN PXENVBD_SRBEXT SrbExt
)
-{
- PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
- PUNMAP_LIST_HEADER Unmap = Srb->DataBuffer;
- ULONG Count =
_byteswap_ushort(*(PUSHORT)Unmap->BlockDescrDataLength) /
sizeof(UNMAP_BLOCK_DESCRIPTOR);
- ULONG Index;
- LIST_ENTRY List;
-
- Srb->SrbStatus = SRB_STATUS_PENDING;
-
- InitializeListHead(&List);
- SrbExt->RequestCount = 0;
-
- for (Index = 0; Index < Count; ++Index) {
- PUNMAP_BLOCK_DESCRIPTOR Descr = &Unmap->Descriptors[Index];
- PXENVBD_REQUEST Request;
-
- Request = RingGetRequest(Ring);
- if (Request == NULL)
- goto fail1;
- InsertTailList(&List, &Request->ListEntry);
- InterlockedIncrement(&SrbExt->RequestCount);
-
- Request->SrbExt = SrbExt;
- Request->Operation = BLKIF_OP_DISCARD;
- Request->FirstSector =
_byteswap_uint64(*(PULONG64)Descr->StartingLba);
- Request->NrSectors = _byteswap_ulong(*(PULONG)Descr->LbaCount);
- Request->Flags = 0;
- }
-
- RingQueueRequestList(Ring, &List);
- return TRUE;
-
-fail1:
- RingCancelRequestList(Ring, &List);
- SrbExt->RequestCount = 0;
- Srb->SrbStatus = SRB_STATUS_ERROR;
- return FALSE;
-}
-
-static FORCEINLINE BOOLEAN
-RingPrepareRequest(
- IN PXENVBD_RING Ring,
- IN PXENVBD_SRBEXT SrbExt
- )
{
switch (Cdb_OperationEx(SrbExt->Srb)) {
case SCSIOP_READ:
case SCSIOP_WRITE:
- return RingPrepareReadWrite(Ring, SrbExt);
+ return BlkifRingPrepareReadWrite(BlkifRing,
+ SrbExt);
case SCSIOP_SYNCHRONIZE_CACHE:
- return RingPrepareSyncCache(Ring, SrbExt);
+ return BlkifRingPrepareSyncCache(BlkifRing,
+ SrbExt);
case SCSIOP_UNMAP:
- return RingPrepareUnmap(Ring, SrbExt);
+ return BlkifRingPrepareUnmap(BlkifRing,
+ SrbExt);
default:
ASSERT(FALSE);
- return FALSE;
+ return STATUS_NOT_SUPPORTED;
}
}
-static BOOLEAN
-RingSubmit(
- IN PXENVBD_RING Ring,
- IN PXENVBD_REQUEST Request
+static FORCEINLINE VOID
+__BlkifRingInsertRequest(
+ IN PXENVBD_BLKIF_RING BlkifRing,
+ IN PXENVBD_REQUEST Request,
+ IN blkif_request_t *req
)
{
- KIRQL Irql;
- blkif_request_t* req;
- BOOLEAN Notify;
-
- KeAcquireSpinLock(&Ring->Lock, &Irql);
- if (RING_FULL(&Ring->Front)) {
- KeReleaseSpinLock(&Ring->Lock, Irql);
- return FALSE;
- }
-
- req = RING_GET_REQUEST(&Ring->Front, Ring->Front.req_prod_pvt);
- __RingInsert(Ring, Request, req);
- KeMemoryBarrier();
- ++Ring->Front.req_prod_pvt;
-
- RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&Ring->Front, Notify);
- KeReleaseSpinLock(&Ring->Lock, Irql);
+ PXENVBD_RING Ring = BlkifRing->Ring;
+ PXENVBD_FRONTEND Frontend = Ring->Frontend;
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Frontend);
- if (Notify) {
- if (!Ring->Enabled)
- return TRUE;
-
- XENBUS_EVTCHN(Send,
- &Ring->EvtchnInterface,
- Ring->Channel);
- }
+ switch (Request->Operation) {
+ case BLKIF_OP_READ:
+ case BLKIF_OP_WRITE:
+ if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) {
+ // Indirect
+ ULONG PageIdx;
+ ULONG SegIdx;
+ PLIST_ENTRY PageEntry;
+ PLIST_ENTRY SegEntry;
+ blkif_request_indirect_t* req_indirect;
- return TRUE;
-}
+ req_indirect = (blkif_request_indirect_t*)req;
+ req_indirect->operation = BLKIF_OP_INDIRECT;
+ req_indirect->indirect_op = Request->Operation;
+ req_indirect->nr_segments = Request->NrSegments;
+ req_indirect->id = (ULONG64)(ULONG_PTR)Request;
+ req_indirect->sector_number = Request->FirstSector;
+ req_indirect->handle = (USHORT)FrontendGetDeviceId(Frontend);
-static FORCEINLINE BOOLEAN
-RingSubmitRequests(
- IN PXENVBD_RING Ring
- )
-{
- if (!Ring->Enabled) {
- if (QueueCount(&Ring->PreparedReqs))
- Warning("Target[%d] : Paused, not submitting new requests (%u)\n",
- FrontendGetTargetId(Ring->Frontend),
- QueueCount(&Ring->PreparedReqs));
- return FALSE;
- }
+ for (PageIdx = 0,
+ PageEntry = Request->Indirects.Flink,
+ SegEntry = Request->Segments.Flink;
+ PageIdx < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST &&
+ PageEntry != &Request->Indirects &&
+ SegEntry != &Request->Segments;
+ ++PageIdx, PageEntry = PageEntry->Flink) {
+ PXENVBD_INDIRECT Page = CONTAINING_RECORD(PageEntry,
XENVBD_INDIRECT, ListEntry);
- for (;;) {
- PXENVBD_REQUEST Request;
- PLIST_ENTRY ListEntry;
+ req_indirect->indirect_grefs[PageIdx] =
GranterReference(Granter, Page->Grant);
- ListEntry = QueuePop(&Ring->PreparedReqs);
- if (ListEntry == NULL)
- break;
+ for (SegIdx = 0;
+ SegIdx < XENVBD_MAX_SEGMENTS_PER_PAGE &&
+ SegEntry != &Request->Segments;
+ ++SegIdx, SegEntry = SegEntry->Flink) {
+ PXENVBD_SEGMENT Segment = CONTAINING_RECORD(SegEntry,
XENVBD_SEGMENT, ListEntry);
+
+ Page->Page[SegIdx].GrantRef = GranterReference(Granter,
Segment->Grant);
+ Page->Page[SegIdx].First = Segment->FirstSector;
+ Page->Page[SegIdx].Last = Segment->LastSector;
+ }
+ }
+ InterlockedIncrement(&Ring->Stats[(Request->Operation ==
BLKIF_OP_READ) ?
+ XENVBD_STAT_BLKIF_OP_READ_INDIRECT :
+ XENVBD_STAT_BLKIF_OP_WRITE_INDIRECT]);
+ } else {
+ // Direct
+ ULONG Index;
+ PLIST_ENTRY Entry;
- Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, ListEntry);
+ req->operation = Request->Operation;
+ req->nr_segments = (UCHAR)Request->NrSegments;
+ req->handle = (USHORT)FrontendGetDeviceId(Frontend);
+ req->id = (ULONG64)(ULONG_PTR)Request;
+ req->sector_number = Request->FirstSector;
- QueueAppend(&Ring->SubmittedReqs, &Request->ListEntry);
- KeMemoryBarrier();
+ for (Index = 0, Entry = Request->Segments.Flink;
+ Index < BLKIF_MAX_SEGMENTS_PER_REQUEST &&
+ Entry != &Request->Segments;
+ ++Index, Entry = Entry->Flink) {
+ PXENVBD_SEGMENT Segment = CONTAINING_RECORD(Entry,
XENVBD_SEGMENT, ListEntry);
+ req->seg[Index].gref = GranterReference(Granter,
Segment->Grant);
+ req->seg[Index].first_sect = Segment->FirstSector;
+ req->seg[Index].last_sect = Segment->LastSector;
+ }
+ InterlockedIncrement(&Ring->Stats[(Request->Operation ==
BLKIF_OP_READ) ?
+ XENVBD_STAT_BLKIF_OP_READ_DIRECT :
+ XENVBD_STAT_BLKIF_OP_WRITE_DIRECT]);
+ }
+ break;
- if (RingSubmit(Ring, Request))
- continue;
+ case BLKIF_OP_WRITE_BARRIER:
+ case BLKIF_OP_FLUSH_DISKCACHE:
+ req->operation = Request->Operation;
+ req->nr_segments = 0;
+ req->handle = (USHORT)FrontendGetDeviceId(Ring->Frontend);
+ req->id = (ULONG64)(ULONG_PTR)Request;
+ req->sector_number = Request->FirstSector;
+ InterlockedIncrement(&Ring->Stats[(Request->Operation ==
BLKIF_OP_WRITE_BARRIER) ?
+ XENVBD_STAT_BLKIF_OP_WRITE_BARRIER :
+ XENVBD_STAT_BLKIF_OP_FLUSH_DISKCACHE]);
+ break;
+
+ case BLKIF_OP_DISCARD:
+ {
+ blkif_request_discard_t* req_discard;
+ req_discard = (blkif_request_discard_t*)req;
+ req_discard->operation = BLKIF_OP_DISCARD;
+ req_discard->flag = Request->Flags;
+ req_discard->handle = (USHORT)FrontendGetDeviceId(Frontend);
+ req_discard->id = (ULONG64)(ULONG_PTR)Request;
+ req_discard->sector_number = Request->FirstSector;
+ req_discard->nr_sectors = Request->NrSectors;
+ InterlockedIncrement(&Ring->Stats[XENVBD_STAT_BLKIF_OP_DISCARD]);
+ } break;
- QueueRemove(&Ring->SubmittedReqs, &Request->ListEntry);
- QueueUnPop(&Ring->PreparedReqs, &Request->ListEntry);
+ default:
+ ASSERT(FALSE);
break;
}
-
- return QueueCount(&Ring->PreparedReqs) != 0;
}
-static FORCEINLINE VOID
-RingCompleteShutdown(
- IN PXENVBD_RING Ring
+static FORCEINLINE NTSTATUS
+__BlkifRingPostRequests(
+ IN PXENVBD_BLKIF_RING BlkifRing
)
{
- PXENVBD_TARGET Target;
- PXENVBD_ADAPTER Adapter;
+#define RING_SLOTS_AVAILABLE(_Front, _req_prod, _rsp_cons) \
+ (RING_SIZE(_Front) - ((_req_prod) - (_rsp_cons)))
- if (QueueCount(&Ring->ShutdownSrbs) == 0)
- return;
+ PXENVBD_SRB_STATE State;
+ RING_IDX req_prod;
+ RING_IDX rsp_cons;
+ NTSTATUS status;
- if (QueueCount(&Ring->PreparedReqs) ||
- QueueCount(&Ring->SubmittedReqs))
- return;
+ State = &BlkifRing->State;
- Target = FrontendGetTarget(Ring->Frontend);
- Adapter = TargetGetAdapter(Target);
- for (;;) {
- PXENVBD_SRBEXT SrbExt;
- PSCSI_REQUEST_BLOCK Srb;
+ req_prod = BlkifRing->Front.req_prod_pvt;
+ rsp_cons = BlkifRing->Front.rsp_cons;
+
+ status = STATUS_ALLOTTED_SPACE_EXCEEDED;
+ if (RING_SLOTS_AVAILABLE(&BlkifRing->Front, req_prod, rsp_cons) <= 1)
+ goto fail1;
+
+ while (State->Count != 0) {
+ blkif_request_t *req;
+ PXENVBD_REQUEST Request;
PLIST_ENTRY ListEntry;
- ListEntry = QueuePop(&Ring->ShutdownSrbs);
- if (ListEntry == NULL)
+ --State->Count;
+
+ ListEntry = RemoveHeadList(&State->List);
+ ASSERT3P(ListEntry, != , &State->List);
+
+ RtlZeroMemory(ListEntry, sizeof(LIST_ENTRY));
+
+ Request = CONTAINING_RECORD(ListEntry,
+ XENVBD_REQUEST,
+ ListEntry);
+
+ req = RING_GET_REQUEST(&BlkifRing->Front, req_prod);
+ req_prod++;
+ BlkifRing->RequestsPosted++;
+
+ __BlkifRingInsertRequest(BlkifRing,
+ Request,
+ req);
+
+ InsertTailList(&BlkifRing->SubmittedList, ListEntry);
+
+ if (RING_SLOTS_AVAILABLE(&BlkifRing->Front, req_prod, rsp_cons) <= 1)
break;
- SrbExt = CONTAINING_RECORD(ListEntry, XENVBD_SRBEXT, ListEntry);
- Srb = SrbExt->Srb;
-
- Srb->SrbStatus = SRB_STATUS_SUCCESS;
- AdapterCompleteSrb(Adapter, SrbExt);
}
+
+ BlkifRing->Front.req_prod_pvt = req_prod;
+
+ return STATUS_SUCCESS;
+
+fail1:
+ return status;
+
+#undef RING_SLOTS_AVAILABLE
}
-static FORCEINLINE PCHAR
-__BlkifOperationName(
- IN UCHAR Operation
+static FORCEINLINE PXENVBD_REQUEST
+__BlkifRingGetSubmittedRequest(
+ IN PXENVBD_BLKIF_RING BlkifRing,
+ IN ULONG64 Id
)
{
- switch (Operation) {
- case BLKIF_OP_READ: return "READ";
- case BLKIF_OP_WRITE: return "WRITE";
- case BLKIF_OP_WRITE_BARRIER: return "WRITE_BARRIER";
- case BLKIF_OP_FLUSH_DISKCACHE: return "FLUSH_DISKCACHE";
- case BLKIF_OP_RESERVED_1: return "RESERVED_1";
- case BLKIF_OP_DISCARD: return "DISCARD";
- case BLKIF_OP_INDIRECT: return "INDIRECT";
- default: return "<unknown>";
+ PLIST_ENTRY ListEntry;
+ PXENVBD_REQUEST Request;
+
+ for (ListEntry = BlkifRing->SubmittedList.Flink;
+ ListEntry != &BlkifRing->SubmittedList;
+ ListEntry = ListEntry->Flink) {
+ Request = CONTAINING_RECORD(ListEntry,
+ XENVBD_REQUEST,
+ ListEntry);
+ if ((ULONG64)(ULONG_PTR)Request != Id)
+ continue;
+
+ RemoveEntryList(ListEntry);
+ return Request;
}
+ return NULL;
}
-static VOID
-RingCompleteResponse(
- IN PXENVBD_RING Ring,
- IN ULONG64 Id,
- IN SHORT Status
+static FORCEINLINE VOID
+__BlkifRingCompleteResponse(
+ IN PXENVBD_BLKIF_RING BlkifRing,
+ IN PXENVBD_REQUEST Request,
+ IN SHORT Status
)
{
- PXENVBD_REQUEST Request;
- PSCSI_REQUEST_BLOCK Srb;
- PXENVBD_SRBEXT SrbExt;
+ PXENVBD_RING Ring = BlkifRing->Ring;
+ PXENVBD_FRONTEND Frontend = Ring->Frontend;
+ PSCSI_REQUEST_BLOCK Srb;
+ PXENVBD_SRBEXT SrbExt;
+ PLIST_ENTRY ListEntry;
- Request = RingFindRequest(Ring, Id);
- if (Request == NULL)
- return;
-
- SrbExt = Request->SrbExt;
- Srb = SrbExt->Srb;
+ SrbExt = Request->SrbExt;
+ Srb = SrbExt->Srb;
switch (Status) {
case BLKIF_RSP_OKAY:
- RingRequestCopyOutput(Request);
+ if (Request->Operation != BLKIF_OP_READ)
+ break;
+
+ for (ListEntry = Request->Segments.Flink;
+ ListEntry != &Request->Segments;
+ ListEntry = ListEntry->Flink) {
+ PXENVBD_SEGMENT Segment = CONTAINING_RECORD(ListEntry,
XENVBD_SEGMENT, ListEntry);
+ PXENVBD_BOUNCE Bounce = Segment->Bounce;
+
+ if (Bounce) {
+ RtlCopyMemory(Bounce->SourcePtr,
+ Bounce->BouncePtr,
+ MmGetMdlByteCount(&Bounce->SourceMdl));
+ }
+ }
break;
case BLKIF_RSP_EOPNOTSUPP:
// Remove appropriate feature support
- FrontendRemoveFeature(Ring->Frontend, Request->Operation);
+ FrontendRemoveFeature(Frontend, Request->Operation);
// Succeed this SRB, subsiquent SRBs will be succeeded instead of
being passed to the backend.
Srb->SrbStatus = SRB_STATUS_SUCCESS;
break;
case BLKIF_RSP_ERROR:
default:
- Warning("Target[%d] : %s BLKIF_RSP_ERROR (Tag %llx)\n",
- FrontendGetTargetId(Ring->Frontend),
- __BlkifOperationName(Request->Operation),
- Id);
+ Warning("Target[%u][%u] : %s BLKIF_RSP_ERROR\n",
+ FrontendGetTargetId(Frontend),
+ BlkifRing->Index,
+ __BlkifOperationName(Request->Operation));
Srb->SrbStatus = SRB_STATUS_ERROR;
break;
}
- RingPutRequest(Ring, Request);
+ BlkifRingPutRequest(BlkifRing, Request);
// complete srb
if (InterlockedDecrement(&SrbExt->RequestCount) == 0) {
- PXENVBD_TARGET Target = FrontendGetTarget(Ring->Frontend);
- PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target);
-
- if (Srb->SrbStatus == SRB_STATUS_PENDING) {
- // SRB has not hit a failure condition (BLKIF_RSP_ERROR |
BLKIF_RSP_EOPNOTSUPP)
- // from any of its responses. SRB must have succeeded
- Srb->SrbStatus = SRB_STATUS_SUCCESS;
- Srb->ScsiStatus = 0x00; // SCSI_GOOD
- } else {
- // Srb->SrbStatus has already been set by 1 or more requests with
Status != BLKIF_RSP_OKAY
- Srb->ScsiStatus = 0x40; // SCSI_ABORTED
- }
-
- AdapterCompleteSrb(Adapter, SrbExt);
+ __BlkifRingCompleteSrb(BlkifRing, SrbExt);
}
}
-static BOOLEAN
-RingPoll(
- IN PXENVBD_RING Ring
+static FORCEINLINE BOOLEAN
+BlkifRingPoll(
+ IN PXENVBD_BLKIF_RING BlkifRing
)
{
- BOOLEAN Retry = FALSE;
+#define XENVBD_BATCH(_Ring) (RING_SIZE(&(_Ring)->Front) / 4)
- ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
- KeAcquireSpinLockAtDpcLevel(&Ring->Lock);
+ PXENVBD_RING Ring;
+ BOOLEAN Retry;
- // Guard against this locked region being called after the
- // lock on FrontendSetState
- if (Ring->Enabled == FALSE)
+ Ring = BlkifRing->Ring;
+ Retry = FALSE;
+
+ if (!BlkifRing->Enabled)
goto done;
for (;;) {
- ULONG rsp_prod;
- ULONG rsp_cons;
+ RING_IDX rsp_prod;
+ RING_IDX rsp_cons;
KeMemoryBarrier();
- rsp_prod = Ring->Shared->rsp_prod;
- rsp_cons = Ring->Front.rsp_cons;
+ rsp_prod = BlkifRing->Shared->rsp_prod;
+ rsp_cons = BlkifRing->Front.rsp_cons;
KeMemoryBarrier();
@@ -1171,318 +1260,984 @@ RingPoll(
break;
while (rsp_cons != rsp_prod && !Retry) {
- blkif_response_t* rsp;
+ blkif_response_t *rsp;
+ PXENVBD_REQUEST Request;
+
+ rsp = RING_GET_RESPONSE(&BlkifRing->Front, rsp_cons);
+ rsp_cons++;
+ BlkifRing->ResponsesProcessed++;
- rsp = RING_GET_RESPONSE(&Ring->Front, rsp_cons);
- ++rsp_cons;
- ++Ring->Received;
+ BlkifRing->Stopped = FALSE;
- RingCompleteResponse(Ring, rsp->id, rsp->status);
- RtlZeroMemory(rsp, sizeof(union blkif_sring_entry));
+ Request = __BlkifRingGetSubmittedRequest(BlkifRing,
+ rsp->id);
+ ASSERT3P(Request, != , NULL);
- if (rsp_cons - Ring->Front.rsp_cons > RING_SIZE(&Ring->Front) / 4)
+ __BlkifRingCompleteResponse(BlkifRing,
+ Request,
+ rsp->status);
+
+ if (rsp_cons - BlkifRing->Front.rsp_cons > XENVBD_BATCH(BlkifRing))
Retry = TRUE;
}
KeMemoryBarrier();
- Ring->Front.rsp_cons = rsp_cons;
- Ring->Shared->rsp_event = rsp_cons + 1;
+ BlkifRing->Front.rsp_cons = rsp_cons;
}
done:
- KeReleaseSpinLockFromDpcLevel(&Ring->Lock);
-
return Retry;
+
+#undef XENVBD_BATCH
}
-__drv_requiresIRQL(DISPATCH_LEVEL)
-static BOOLEAN
-RingNotifyResponses(
- IN PXENVBD_RING Ring
+static FORCEINLINE VOID
+__BlkifRingSend(
+ IN PXENVBD_BLKIF_RING BlkifRing
)
{
- BOOLEAN Retry = FALSE;
-
- if (!Ring->Enabled)
- return FALSE;
+ PXENVBD_RING Ring = BlkifRing->Ring;
- Retry |= RingPoll(Ring);
- Retry |= RingSubmitRequests(Ring);
-
- RingCompleteShutdown(Ring);
- return Retry;
+ XENBUS_EVTCHN(Send,
+ &Ring->EvtchnInterface,
+ BlkifRing->Channel);
}
-KSERVICE_ROUTINE RingInterrupt;
-
-BOOLEAN
-RingInterrupt(
- IN PKINTERRUPT Interrupt,
- IN PVOID Context
+static FORCEINLINE VOID
+__BlkifRingPushRequests(
+ IN PXENVBD_BLKIF_RING BlkifRing
)
{
- PXENVBD_RING Ring = Context;
+ BOOLEAN Notify;
- UNREFERENCED_PARAMETER(Interrupt);
+ if (BlkifRing->RequestsPosted == BlkifRing->RequestsPushed)
+ return;
- ASSERT(Ring != NULL);
+#pragma warning (push)
+#pragma warning (disable:4244)
- ++Ring->Events;
- if (!Ring->Connected)
- return TRUE;
+ BlkifRing->Shared->rsp_event = BlkifRing->Front.req_prod_pvt;
- if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
- ++Ring->Dpcs;
+ // Make the requests visible to the backend
+ RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&BlkifRing->Front, Notify);
- return TRUE;
+#pragma warning (pop)
+
+ if (Notify)
+ __BlkifRingSend(BlkifRing);
+
+ BlkifRing->RequestsPushed = BlkifRing->RequestsPosted;
}
-KDEFERRED_ROUTINE RingDpc;
+#define XENVBD_LOCK_BIT ((ULONG_PTR)1)
-VOID
-RingDpc(
- __in PKDPC Dpc,
- __in_opt PVOID Context,
- __in_opt PVOID Arg1,
- __in_opt PVOID Arg2
+static DECLSPEC_NOINLINE VOID
+BlkifRingSwizzle(
+ IN PXENVBD_BLKIF_RING BlkifRing
)
{
- PXENVBD_RING Ring = Context;
+ ULONG_PTR Old;
+ ULONG_PTR New;
+ PLIST_ENTRY ListEntry;
+ LIST_ENTRY List;
+ ULONG Count;
- UNREFERENCED_PARAMETER(Dpc);
- UNREFERENCED_PARAMETER(Arg1);
- UNREFERENCED_PARAMETER(Arg2);
+ ASSERT3P(BlkifRing->LockThread, == , KeGetCurrentThread());
- ASSERT(Ring != NULL);
+ InitializeListHead(&List);
- for (;;) {
- KIRQL Irql;
- BOOLEAN Retry;
+ New = XENVBD_LOCK_BIT;
+ Old = (ULONG_PTR)InterlockedExchangePointer(&BlkifRing->Lock, (PVOID)New);
- KeRaiseIrql(DISPATCH_LEVEL, &Irql);
- Retry = RingNotifyResponses(Ring);
- KeLowerIrql(Irql);
+ ASSERT(Old & XENVBD_LOCK_BIT);
+ ListEntry = (PVOID)(Old & ~XENVBD_LOCK_BIT);
- if (!Retry)
- break;
+ if (ListEntry == NULL)
+ return;
+
+ // Packets are held in the atomic packet list in reverse order
+ // so that the most recent is always head of the list. This is
+ // necessary to allow addition to the list to be done atomically.
+
+ for (Count = 0; ListEntry != NULL; ++Count) {
+ PLIST_ENTRY NextEntry;
+
+ NextEntry = ListEntry->Blink;
+ ListEntry->Flink = ListEntry->Blink = ListEntry;
+
+ InsertHeadList(&List, ListEntry);
+
+ ListEntry = NextEntry;
}
- XENBUS_EVTCHN(Unmask,
- &Ring->EvtchnInterface,
- Ring->Channel,
- FALSE);
+ if (!IsListEmpty(&List)) {
+ ListEntry = List.Flink;
+
+ RemoveEntryList(&List);
+ AppendTailList(&BlkifRing->SrbQueue, ListEntry);
+
+ BlkifRing->SrbsQueued += Count;
+ }
}
static DECLSPEC_NOINLINE VOID
-RingDebugCallback(
- IN PVOID Argument,
- IN BOOLEAN Crashing
+BlkifRingSchedule(
+ IN PXENVBD_BLKIF_RING BlkifRing
)
{
- PXENVBD_RING Ring = Argument;
- PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend);
- ULONG Index;
+ PXENVBD_SRB_STATE State;
+ BOOLEAN Polled;
- UNREFERENCED_PARAMETER(Crashing);
+ if (!BlkifRing->Enabled)
+ return;
- XENBUS_DEBUG(Printf,
- &Ring->DebugInterface,
- "Submitted: %u Received: %u\n",
- Ring->Submitted,
- Ring->Received);
+ State = &BlkifRing->State;
+ Polled = FALSE;
- XENBUS_DEBUG(Printf,
- &Ring->DebugInterface,
- "Events: %u Dpcs: %u\n",
- Ring->Events,
- Ring->Dpcs);
+ while (!BlkifRing->Stopped) {
+ PLIST_ENTRY ListEntry;
+ PXENVBD_SRBEXT SrbExt;
+ NTSTATUS status;
- XENBUS_DEBUG(Printf,
- &Ring->DebugInterface,
- "Shared : 0x%p\n",
- Ring->Shared);
+ if (State->Count != 0) {
+ status = __BlkifRingPostRequests(BlkifRing);
+ if (!NT_SUCCESS(status))
+ BlkifRing->Stopped = TRUE;
+ }
- if (Ring->Shared) {
- XENBUS_DEBUG(Printf,
- &Ring->DebugInterface,
- "Shared: %d / %d - %d / %d\n",
- Ring->Shared->req_prod,
- Ring->Shared->req_event,
- Ring->Shared->rsp_prod,
- Ring->Shared->rsp_event);
- }
+ if (BlkifRing->Stopped) {
+ if (!Polled) {
+ (VOID)BlkifRingPoll(BlkifRing);
+ Polled = TRUE;
+ }
- XENBUS_DEBUG(Printf,
- &Ring->DebugInterface,
- "Front: %d / %d (%d)\n",
- Ring->Front.req_prod_pvt,
- Ring->Front.rsp_cons,
- Ring->Front.nr_ents);
+ continue;
+ }
- XENBUS_DEBUG(Printf,
- &Ring->DebugInterface,
- "Order: %d\n",
- Ring->Order);
+ if (BlkifRing->RequestsPosted - BlkifRing->RequestsPushed >=
+ RING_SIZE(&BlkifRing->Front) / 4)
+ __BlkifRingPushRequests(BlkifRing);
- for (Index = 0; Index < (1ul << Ring->Order); ++Index) {
- XENBUS_DEBUG(Printf,
- &Ring->DebugInterface,
- "Grants[%-2d]: 0x%p (%u)\n",
- Index,
- Ring->Grants[Index],
- GranterReference(Granter, Ring->Grants[Index]));
+ ASSERT3U(State->Count, == , 0);
+
+ if (IsListEmpty(&BlkifRing->SrbQueue))
+ break;
+
+ ListEntry = RemoveHeadList(&BlkifRing->SrbQueue);
+ ASSERT3P(ListEntry, != , &BlkifRing->SrbQueue);
+
+ RtlZeroMemory(ListEntry, sizeof(LIST_ENTRY));
+
+ SrbExt = CONTAINING_RECORD(ListEntry,
+ XENVBD_SRBEXT,
+ ListEntry);
+
+ status = __BlkifRingPrepareSrb(BlkifRing, SrbExt);
+ if (!NT_SUCCESS(status)) {
+ PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
+ Srb->SrbStatus = SRB_STATUS_BUSY;
+ __BlkifRingCompleteSrb(BlkifRing, SrbExt);
+ }
}
- if (Ring->Channel) {
- ULONG Port = XENBUS_EVTCHN(GetPort,
- &Ring->EvtchnInterface,
- Ring->Channel);
+ __BlkifRingPushRequests(BlkifRing);
- XENBUS_DEBUG(Printf,
- &Ring->DebugInterface,
- "Channel : %p (%d)\n",
- Ring->Channel,
- Port);
+ if (IsListEmpty(&BlkifRing->ShutdownQueue))
+ return;
+
+ if (!IsListEmpty(&BlkifRing->SrbQueue) ||
+ !IsListEmpty(&BlkifRing->SubmittedList))
+ return;
+
+ for (;;) {
+ PLIST_ENTRY ListEntry;
+ PXENVBD_SRBEXT SrbExt;
+ PSCSI_REQUEST_BLOCK Srb;
+
+ ListEntry = RemoveHeadList(&BlkifRing->ShutdownQueue);
+ if (ListEntry == &BlkifRing->ShutdownQueue)
+ break;
+
+ SrbExt = CONTAINING_RECORD(ListEntry,
+ XENVBD_SRBEXT,
+ ListEntry);
+
+ Srb = SrbExt->Srb;
+ Srb->SrbStatus = SRB_STATUS_SUCCESS;
+ __BlkifRingCompleteSrb(BlkifRing, SrbExt);
}
+}
- XENBUS_DEBUG(Printf,
- &Ring->DebugInterface,
- "BLKIF_OPs: READ=%u WRITE=%u\n",
- Ring->BlkOpRead,
- Ring->BlkOpWrite);
- XENBUS_DEBUG(Printf,
- &Ring->DebugInterface,
- "BLKIF_OPs: INDIRECT_READ=%u INDIRECT_WRITE=%u\n",
- Ring->BlkOpIndirectRead,
- Ring->BlkOpIndirectWrite);
- XENBUS_DEBUG(Printf,
- &Ring->DebugInterface,
- "BLKIF_OPs: BARRIER=%u DISCARD=%u FLUSH=%u\n",
- Ring->BlkOpBarrier,
- Ring->BlkOpDiscard,
- Ring->BlkOpFlush);
- XENBUS_DEBUG(Printf,
- &Ring->DebugInterface,
- "Segments Granted=%llu Bounced=%llu\n",
- Ring->SegsGranted,
- Ring->SegsBounced);
+static FORCEINLINE BOOLEAN
+__drv_requiresIRQL(DISPATCH_LEVEL)
+__BlkifRingTryAcquireLock(
+ IN PXENVBD_BLKIF_RING BlkifRing
+ )
+{
+ ULONG_PTR Old;
+ ULONG_PTR New;
+ BOOLEAN Acquired;
+
+ ASSERT3U(KeGetCurrentIrql(), == , DISPATCH_LEVEL);
+
+ KeMemoryBarrier();
+
+ Old = (ULONG_PTR)BlkifRing->Lock & ~XENVBD_LOCK_BIT;
+ New = Old | XENVBD_LOCK_BIT;
+
+ Acquired = ((ULONG_PTR)InterlockedCompareExchangePointer(&BlkifRing->Lock,
+ (PVOID)New,
+ (PVOID)Old) ==
Old) ? TRUE : FALSE;
- QueueDebugCallback(&Ring->PreparedReqs,
- "Prepared ",
- &Ring->DebugInterface);
- QueueDebugCallback(&Ring->SubmittedReqs,
- "Submitted",
- &Ring->DebugInterface);
- QueueDebugCallback(&Ring->ShutdownSrbs,
- "Shutdown ",
- &Ring->DebugInterface);
+ KeMemoryBarrier();
+
+ if (Acquired) {
+ ASSERT3P(BlkifRing->LockThread, == , NULL);
+ BlkifRing->LockThread = KeGetCurrentThread();
+ KeMemoryBarrier();
+ }
+
+ return Acquired;
}
-static DECLSPEC_NOINLINE VOID
-RingAcquireLock(
- IN PVOID Argument
+static FORCEINLINE VOID
+__drv_requiresIRQL(DISPATCH_LEVEL)
+__BlkifRingAcquireLock(
+ IN PXENVBD_BLKIF_RING BlkifRing
)
{
- PXENVBD_RING Ring = Argument;
- KeAcquireSpinLockAtDpcLevel(&Ring->Lock);
+ ASSERT3U(KeGetCurrentIrql(), == , DISPATCH_LEVEL);
+
+ for (;;) {
+ if (__BlkifRingTryAcquireLock(BlkifRing))
+ break;
+
+ _mm_pause();
+ }
}
-static DECLSPEC_NOINLINE VOID
-RingReleaseLock(
- IN PVOID Argument
+static VOID
+BlkifRingAcquireLock(
+ IN PXENVBD_BLKIF_RING BlkifRing
)
{
- PXENVBD_RING Ring = Argument;
- KeReleaseSpinLockFromDpcLevel(&Ring->Lock);
+ __BlkifRingAcquireLock(BlkifRing);
}
-static DECLSPEC_NOINLINE NTSTATUS
-RingRequestCtor(
- IN PVOID Argument,
- IN PVOID Object
+static FORCEINLINE BOOLEAN
+__drv_requiresIRQL(DISPATCH_LEVEL)
+__BlkifRingTryReleaseLock(
+ IN PXENVBD_BLKIF_RING BlkifRing
)
{
- PXENVBD_REQUEST Request = Object;
+ ULONG_PTR Old;
+ ULONG_PTR New;
+ BOOLEAN Released;
- UNREFERENCED_PARAMETER(Argument);
+ ASSERT3U(KeGetCurrentIrql(), == , DISPATCH_LEVEL);
+ ASSERT3P(KeGetCurrentThread(), == , BlkifRing->LockThread);
- InitializeListHead(&Request->Segments);
- InitializeListHead(&Request->Indirects);
- return STATUS_SUCCESS;
+ Old = XENVBD_LOCK_BIT;
+ New = 0;
+
+ BlkifRing->LockThread = NULL;
+
+ KeMemoryBarrier();
+
+ Released = ((ULONG_PTR)InterlockedCompareExchangePointer(&BlkifRing->Lock,
+ (PVOID)New,
+ (PVOID)Old) ==
Old) ? TRUE : FALSE;
+
+ KeMemoryBarrier();
+
+ if (!Released) {
+ ASSERT3P(BlkifRing->LockThread, == , NULL);
+ BlkifRing->LockThread = KeGetCurrentThread();
+ KeMemoryBarrier();
+ }
+
+ return Released;
}
-static DECLSPEC_NOINLINE VOID
-RingRequestDtor(
- IN PVOID Argument,
- IN PVOID Object
+static FORCEINLINE VOID
+__drv_requiresIRQL(DISPATCH_LEVEL)
+__BlkifRingReleaseLock(
+ IN PXENVBD_BLKIF_RING BlkifRing
)
{
- UNREFERENCED_PARAMETER(Argument);
- UNREFERENCED_PARAMETER(Object);
+ ASSERT3U(KeGetCurrentIrql(), == , DISPATCH_LEVEL);
+
+ // As lock holder it is our responsibility to drain the atomic
+ // packet list into the transmit queue before we actually drop the
+ // lock. This may, of course, take a few attempts as another
+ // thread could be simuntaneously adding to the list.
+
+ do {
+ BlkifRingSwizzle(BlkifRing);
+ BlkifRingSchedule(BlkifRing);
+ } while (!__BlkifRingTryReleaseLock(BlkifRing));
}
-static DECLSPEC_NOINLINE NTSTATUS
-RingSegmentCtor(
- IN PVOID Argument,
- IN PVOID Object
+static VOID
+BlkifRingReleaseLock(
+ IN PXENVBD_BLKIF_RING BlkifRing
)
{
- UNREFERENCED_PARAMETER(Argument);
- UNREFERENCED_PARAMETER(Object);
+ __BlkifRingReleaseLock(BlkifRing);
+}
+
+KSERVICE_ROUTINE BlkifRingInterrupt;
+
+BOOLEAN
+BlkifRingInterrupt(
+ IN PKINTERRUPT InterruptObject,
+ IN PVOID Argument
+ )
+{
+ PXENVBD_BLKIF_RING BlkifRing = Argument;
+
+ UNREFERENCED_PARAMETER(InterruptObject);
+
+ ASSERT(BlkifRing != NULL);
+
+ BlkifRing->Events++;
+
+ if (KeInsertQueueDpc(&BlkifRing->Dpc, NULL, NULL))
+ BlkifRing->Dpcs++;
+
+ return TRUE;
+}
+
+__drv_functionClass(KDEFERRED_ROUTINE)
+__drv_maxIRQL(DISPATCH_LEVEL)
+__drv_minIRQL(PASSIVE_LEVEL)
+__drv_sameIRQL
+static VOID
+BlkifRingDpc(
+ IN PKDPC Dpc,
+ IN PVOID Context,
+ IN PVOID Argument1,
+ IN PVOID Argument2
+ )
+{
+ PXENVBD_BLKIF_RING BlkifRing = Context;
+ PXENVBD_RING Ring;
+
+ UNREFERENCED_PARAMETER(Dpc);
+ UNREFERENCED_PARAMETER(Argument1);
+ UNREFERENCED_PARAMETER(Argument2);
+
+ ASSERT(BlkifRing != NULL);
+
+ Ring = BlkifRing->Ring;
+
+ for (;;) {
+ BOOLEAN Retry;
+ KIRQL Irql;
+
+ KeRaiseIrql(DISPATCH_LEVEL, &Irql);
+ __BlkifRingAcquireLock(BlkifRing);
+ Retry = BlkifRingPoll(BlkifRing);
+ __BlkifRingReleaseLock(BlkifRing);
+ KeLowerIrql(Irql);
+
+ if (!Retry)
+ break;
+ }
+
+ XENBUS_EVTCHN(Unmask,
+ &Ring->EvtchnInterface,
+ BlkifRing->Channel,
+ FALSE);
+}
+
+static NTSTATUS
+BlkifRingCreate(
+ IN PXENVBD_RING Ring,
+ IN ULONG Index,
+ OUT PXENVBD_BLKIF_RING* BlkifRing
+ )
+{
+ PXENVBD_FRONTEND Frontend;
+ ULONG Length;
+ PCHAR Path;
+ CHAR Name[MAX_NAME_LEN];
+ NTSTATUS status;
+
+ Frontend = Ring->Frontend;
+
+ Length = (ULONG)strlen(FrontendGetFrontendPath(Frontend)) +
+ (ULONG)strlen("/queue-xxx");
+
+ Path = __RingAllocate(Length + 1);
+
+ status = STATUS_NO_MEMORY;
+ if (Path == NULL)
+ goto fail1;
+
+ status = RtlStringCchPrintfA(Path,
+ Length,
+ "%s/queue-%u",
+ FrontendGetFrontendPath(Frontend),
+ Index);
+ if (!NT_SUCCESS(status))
+ goto fail2;
+
+ *BlkifRing = __RingAllocate(sizeof(XENVBD_BLKIF_RING));
+
+ status = STATUS_NO_MEMORY;
+ if (*BlkifRing == NULL)
+ goto fail3;
+
+ (*BlkifRing)->Ring = Ring;
+ (*BlkifRing)->Index = Index;
+ (*BlkifRing)->Path = Path;
+ Path = NULL;
+
+ InitializeListHead(&(*BlkifRing)->SrbQueue);
+ InitializeListHead(&(*BlkifRing)->ShutdownQueue);
+ InitializeListHead(&(*BlkifRing)->SubmittedList);
+ InitializeListHead(&(*BlkifRing)->State.List);
+
+ KeInitializeThreadedDpc(&(*BlkifRing)->Dpc, BlkifRingDpc, *BlkifRing);
+
+ status = RtlStringCbPrintfA(Name,
+ sizeof(Name),
+ "vbd_%u_queue_%u_request",
+ FrontendGetTargetId(Frontend),
+ Index);
+ if (!NT_SUCCESS(status))
+ goto fail4;
+
+ status = XENBUS_CACHE(Create,
+ &Ring->CacheInterface,
+ Name,
+ sizeof(XENVBD_REQUEST),
+ 0,
+ BlkifRingRequestCtor,
+ BlkifRingRequestDtor,
+ BlkifRingAcquireLock,
+ BlkifRingReleaseLock,
+ *BlkifRing,
+ &(*BlkifRing)->RequestCache);
+ if (!NT_SUCCESS(status))
+ goto fail5;
+
+ status = RtlStringCbPrintfA(Name,
+ sizeof(Name),
+ "vbd_%u_queue_%u_segment",
+ FrontendGetTargetId(Frontend),
+ Index);
+ if (!NT_SUCCESS(status))
+ goto fail6;
+
+ status = XENBUS_CACHE(Create,
+ &Ring->CacheInterface,
+ Name,
+ sizeof(XENVBD_SEGMENT),
+ 0,
+ BlkifRingSegmentCtor,
+ BlkifRingSegmentDtor,
+ BlkifRingAcquireLock,
+ BlkifRingReleaseLock,
+ *BlkifRing,
+ &(*BlkifRing)->SegmentCache);
+ if (!NT_SUCCESS(status))
+ goto fail7;
+
+ status = RtlStringCbPrintfA(Name,
+ sizeof(Name),
+ "vbd_%u_queue_%u_indirect",
+ FrontendGetTargetId(Frontend),
+ Index);
+ if (!NT_SUCCESS(status))
+ goto fail8;
+
+ status = XENBUS_CACHE(Create,
+ &Ring->CacheInterface,
+ Name,
+ sizeof(XENVBD_INDIRECT),
+ 0,
+ BlkifRingIndirectCtor,
+ BlkifRingIndirectDtor,
+ BlkifRingAcquireLock,
+ BlkifRingReleaseLock,
+ *BlkifRing,
+ &(*BlkifRing)->IndirectCache);
+ if (!NT_SUCCESS(status))
+ goto fail9;
+
return STATUS_SUCCESS;
+
+fail9:
+ Error("fail9\n");
+fail8:
+ Error("fail8\n");
+ XENBUS_CACHE(Destroy,
+ &Ring->CacheInterface,
+ (*BlkifRing)->SegmentCache);
+ (*BlkifRing)->SegmentCache = NULL;
+fail7:
+ Error("fail7\n");
+fail6:
+ Error("fail6\n");
+ XENBUS_CACHE(Destroy,
+ &Ring->CacheInterface,
+ (*BlkifRing)->RequestCache);
+ (*BlkifRing)->RequestCache = NULL;
+fail5:
+ Error("fail5\n");
+fail4:
+ Error("fail4\n");
+
+ RtlZeroMemory(&(*BlkifRing)->Dpc, sizeof(KDPC));
+
+ RtlZeroMemory(&(*BlkifRing)->State.List, sizeof(LIST_ENTRY));
+ RtlZeroMemory(&(*BlkifRing)->SubmittedList, sizeof(LIST_ENTRY));
+ RtlZeroMemory(&(*BlkifRing)->ShutdownQueue, sizeof(LIST_ENTRY));
+ RtlZeroMemory(&(*BlkifRing)->SrbQueue, sizeof(LIST_ENTRY));
+
+ __RingFree((*BlkifRing)->Path);
+ (*BlkifRing)->Path;
+ (*BlkifRing)->Index = 0;
+ (*BlkifRing)->Ring = NULL;
+
+ ASSERT(IsZeroMemory(*BlkifRing, sizeof(XENVBD_BLKIF_RING)));
+ __RingFree(*BlkifRing);
+ *BlkifRing = NULL;
+fail3:
+ Error("fail3\n");
+fail2:
+ Error("fail2\n");
+ __RingFree(Path);
+fail1:
+ Error("fail1 (%08x)\n", status);
+ return status;
}
-static DECLSPEC_NOINLINE VOID
-RingSegmentDtor(
- IN PVOID Argument,
- IN PVOID Object
+static VOID
+BlkifRingDestroy(
+ IN PXENVBD_BLKIF_RING BlkifRing
)
{
- UNREFERENCED_PARAMETER(Argument);
- UNREFERENCED_PARAMETER(Object);
+ PXENVBD_RING Ring = BlkifRing->Ring;
+
+ XENBUS_CACHE(Destroy,
+ &Ring->CacheInterface,
+ BlkifRing->IndirectCache);
+ BlkifRing->IndirectCache = NULL;
+
+ XENBUS_CACHE(Destroy,
+ &Ring->CacheInterface,
+ BlkifRing->SegmentCache);
+ BlkifRing->SegmentCache = NULL;
+
+ XENBUS_CACHE(Destroy,
+ &Ring->CacheInterface,
+ BlkifRing->RequestCache);
+ BlkifRing->RequestCache = NULL;
+
+ RtlZeroMemory(&BlkifRing->Dpc, sizeof(KDPC));
+
+ ASSERT3U(BlkifRing->State.Count, == , 0);
+ ASSERT(IsListEmpty(&BlkifRing->State.List));
+ RtlZeroMemory(&BlkifRing->State.List, sizeof(LIST_ENTRY));
+
+ RtlZeroMemory(&BlkifRing->SubmittedList, sizeof(LIST_ENTRY));
+ RtlZeroMemory(&BlkifRing->SrbQueue, sizeof(LIST_ENTRY));
+ RtlZeroMemory(&BlkifRing->ShutdownQueue, sizeof(LIST_ENTRY));
+
+ __RingFree(BlkifRing->Path);
+ BlkifRing->Path;
+ BlkifRing->Index = 0;
+ BlkifRing->Ring = NULL;
+
+ ASSERT(IsZeroMemory(BlkifRing, sizeof(XENVBD_BLKIF_RING)));
+ __RingFree(BlkifRing);
}
-static DECLSPEC_NOINLINE NTSTATUS
-RingIndirectCtor(
- IN PVOID Argument,
- IN PVOID Object
+static NTSTATUS
+BlkifRingConnect(
+ IN PXENVBD_BLKIF_RING BlkifRing
)
{
- PXENVBD_INDIRECT Indirect = Object;
- NTSTATUS status;
+ PXENVBD_RING Ring = BlkifRing->Ring;
+ PXENVBD_FRONTEND Frontend = Ring->Frontend;
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Frontend);
+ CHAR Name[MAX_NAME_LEN];
+ ULONG Index;
+ NTSTATUS status;
- UNREFERENCED_PARAMETER(Argument);
+ Trace("====> %u\n", BlkifRing->Index);
+ ASSERT(!BlkifRing->Connected);
+
+ if (FrontendGetNumQueues(Frontend) != 1) {
+ PROCESSOR_NUMBER ProcNumber;
+
+ status = KeGetProcessorNumberFromIndex(BlkifRing->Index, &ProcNumber);
+ ASSERT(NT_SUCCESS(status));
+
+ KeSetTargetProcessorDpcEx(&BlkifRing->Dpc, &ProcNumber);
+ }
+ KeSetImportanceDpc(&BlkifRing->Dpc, MediumHighImportance);
+
+ BlkifRing->Mdl = __AllocatePages(1 << Ring->Order);
status = STATUS_NO_MEMORY;
- Indirect->Mdl = __AllocatePage();
- if (Indirect->Mdl == NULL)
+ if (BlkifRing->Mdl == NULL)
goto fail1;
- Indirect->Page = MmGetSystemAddressForMdlSafe(Indirect->Mdl,
- NormalPagePriority);
- ASSERT(Indirect->Page);
+ BlkifRing->Shared = MmGetSystemAddressForMdlSafe(BlkifRing->Mdl,
+ NormalPagePriority);
+ ASSERT(BlkifRing->Shared != NULL);
+
+#pragma warning(push)
+#pragma warning(disable: 4305)
+#pragma warning(disable: 4311) // 'type cast' pointer truncation from
'blkif_sring_entry[1]' to 'long'
+ SHARED_RING_INIT(BlkifRing->Shared);
+ FRONT_RING_INIT(&BlkifRing->Front, BlkifRing->Shared, PAGE_SIZE <<
Ring->Order);
+#pragma warning(pop)
+
+ for (Index = 0; Index < (1ul << Ring->Order); ++Index) {
+ status = GranterGet(Granter,
+ MmGetMdlPfnArray(BlkifRing->Mdl)[Index],
+ FALSE,
+ &BlkifRing->Grants[Index]);
+ if (!NT_SUCCESS(status))
+ goto fail2;
+ }
+
+ BlkifRing->Channel = XENBUS_EVTCHN(Open,
+ &Ring->EvtchnInterface,
+ XENBUS_EVTCHN_TYPE_UNBOUND,
+ BlkifRingInterrupt,
+ BlkifRing,
+
FrontendGetBackendDomain(Ring->Frontend),
+ TRUE);
+ status = STATUS_NO_MEMORY;
+ if (BlkifRing->Channel == NULL)
+ goto fail3;
+
+ XENBUS_EVTCHN(Unmask,
+ &Ring->EvtchnInterface,
+ BlkifRing->Channel,
+ FALSE);
+
+ status = RtlStringCchPrintfA(Name,
+ MAX_NAME_LEN,
+ __MODULE__"|RING[%u]",
+ Index);
+ if (!NT_SUCCESS(status))
+ goto fail4;
+
+ status = XENBUS_DEBUG(Register,
+ &Ring->DebugInterface,
+ Name,
+ BlkifRingDebugCallback,
+ BlkifRing,
+ &BlkifRing->DebugCallback);
+ if (!NT_SUCCESS(status))
+ goto fail5;
+
+ BlkifRing->Connected = TRUE;
+ Trace("<==== %u\n", BlkifRing->Index);
return STATUS_SUCCESS;
+fail5:
+ Error("fail5\n");
+fail4:
+ Error("fail4\n");
+ XENBUS_EVTCHN(Close,
+ &Ring->EvtchnInterface,
+ BlkifRing->Channel);
+ BlkifRing->Channel = NULL;
+fail3:
+ Error("fail3\n");
+fail2:
+ Error("fail2\n");
+ for (Index = 0; Index < (1ul << Ring->Order); ++Index) {
+ if (BlkifRing->Grants[Index] == NULL)
+ continue;
+
+ GranterPut(Granter, BlkifRing->Grants[Index]);
+ BlkifRing->Grants[Index] = NULL;
+ }
+
+ RtlZeroMemory(&BlkifRing->Front, sizeof(blkif_front_ring_t));
+
+ __FreePages(BlkifRing->Mdl);
+ BlkifRing->Shared = NULL;
+ BlkifRing->Mdl = NULL;
fail1:
Error("fail1 %08x\n", status);
return status;
}
+static NTSTATUS
+BlkifRingStoreWrite(
+ IN PXENVBD_BLKIF_RING BlkifRing,
+ IN PVOID Transaction
+ )
+{
+ PXENVBD_RING Ring = BlkifRing->Ring;
+ PXENVBD_FRONTEND Frontend = Ring->Frontend;
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Frontend);
+ PCHAR Path;
+ NTSTATUS status;
+
+ Path = (FrontendGetNumQueues(Frontend) == 1) ?
+ FrontendGetFrontendPath(Frontend) :
+ BlkifRing->Path;
+
+ status = XENBUS_STORE(Printf,
+ &Ring->StoreInterface,
+ Transaction,
+ Path,
+ "event-channel",
+ "%u",
+ XENBUS_EVTCHN(GetPort,
+ &Ring->EvtchnInterface,
+ BlkifRing->Channel));
+ if (!NT_SUCCESS(status))
+ goto fail1;
+
+ if (Ring->Order == 0) {
+ status = XENBUS_STORE(Printf,
+ &Ring->StoreInterface,
+ Transaction,
+ Path,
+ "ring-ref",
+ "%u",
+ GranterReference(Granter, BlkifRing->Grants[0]));
+ if (!NT_SUCCESS(status))
+ goto fail2;
+ } else {
+ ULONG Index;
+
+ for (Index = 0; Index < (1ul << Ring->Order); ++Index) {
+ CHAR Name[MAX_NAME_LEN + 1];
+
+ status = RtlStringCchPrintfA(Name,
+ MAX_NAME_LEN,
+ "ring-ref%u",
+ Index);
+ if (!NT_SUCCESS(status))
+ goto fail3;
+
+ status = XENBUS_STORE(Printf,
+ &Ring->StoreInterface,
+ Transaction,
+ Path,
+ Name,
+ "%u",
+ GranterReference(Granter,
BlkifRing->Grants[Index]));
+ if (!NT_SUCCESS(status))
+ goto fail4;
+ }
+ }
+
+ return STATUS_SUCCESS;
+
+fail4:
+fail3:
+fail2:
+fail1:
+ return status;
+}
+
+static VOID
+BlkifRingEnable(
+ IN PXENVBD_BLKIF_RING BlkifRing
+ )
+{
+ Trace("====> %u\n", BlkifRing->Index);
+
+ __BlkifRingAcquireLock(BlkifRing);
+ ASSERT(!BlkifRing->Enabled);
+ BlkifRing->Enabled = TRUE;
+ __BlkifRingReleaseLock(BlkifRing);
+
+ Trace("<==== %u\n", BlkifRing->Index);
+}
+
+static VOID
+BlkifRingDisable(
+ IN PXENVBD_BLKIF_RING BlkifRing
+ )
+{
+ PXENVBD_RING Ring = BlkifRing->Ring;
+ ULONG Attempt;
+
+ Trace("====> %u\n", BlkifRing->Index);
+
+ __BlkifRingAcquireLock(BlkifRing);
+ ASSERT(BlkifRing->Enabled);
+
+ // Discard any pending requests
+ while (!IsListEmpty(&BlkifRing->State.List)) {
+ PLIST_ENTRY ListEntry;
+ PXENVBD_REQUEST Request;
+ PXENVBD_SRBEXT SrbExt;
+ PSCSI_REQUEST_BLOCK Srb;
+
+ ListEntry = RemoveHeadList(&BlkifRing->State.List);
+ ASSERT3P(ListEntry, != , &BlkifRing->State.List);
+
+ Request = CONTAINING_RECORD(ListEntry,
+ XENVBD_REQUEST,
+ ListEntry);
+ SrbExt = Request->SrbExt;
+ Srb = SrbExt->Srb;
+ Srb->SrbStatus = SRB_STATUS_ABORTED;
+ Srb->ScsiStatus = 0x40; // SCSI_ABORTED
+
+ BlkifRingPutRequest(BlkifRing, Request);
+
+ if (InterlockedDecrement(&SrbExt->RequestCount) == 0)
+ __BlkifRingCompleteSrb(BlkifRing, SrbExt);
+ }
+
+ ASSERT3U(BlkifRing->State.Count, == , 0);
+
+ Attempt = 0;
+ ASSERT3U(BlkifRing->RequestsPushed, == , BlkifRing->RequestsPosted);
+ while (BlkifRing->ResponsesProcessed != BlkifRing->RequestsPushed) {
+ Attempt++;
+ ASSERT(Attempt < 100);
+
+ // Try to move things along
+ __BlkifRingSend(BlkifRing);
+ (VOID)BlkifRingPoll(BlkifRing);
+
+ // We are waiting for a watch event at DISPATCH_LEVEL so
+ // it is our responsibility to poll the store ring.
+ XENBUS_STORE(Poll,
+ &Ring->StoreInterface);
+
+ KeStallExecutionProcessor(1000); // 1ms
+ }
+
+ BlkifRing->Enabled = FALSE;
+ __BlkifRingReleaseLock(BlkifRing);
+
+ Trace("<==== %u\n", BlkifRing->Index);
+}
+
+static VOID
+BlkifRingDisconnect(
+ IN PXENVBD_BLKIF_RING BlkifRing
+ )
+{
+ PXENVBD_RING Ring = BlkifRing->Ring;
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend);
+ ULONG Index;
+
+ Trace("====> %u\n", BlkifRing->Index);
+ ASSERT(BlkifRing->Connected);
+
+ XENBUS_DEBUG(Deregister,
+ &Ring->DebugInterface,
+ BlkifRing->DebugCallback);
+ BlkifRing->DebugCallback = NULL;
+
+ XENBUS_EVTCHN(Close,
+ &Ring->EvtchnInterface,
+ BlkifRing->Channel);
+ BlkifRing->Channel = NULL;
+
+ for (Index = 0; Index < (1ul << Ring->Order); ++Index) {
+ if (BlkifRing->Grants[Index] == NULL)
+ continue;
+
+ GranterPut(Granter, BlkifRing->Grants[Index]);
+ BlkifRing->Grants[Index] = NULL;
+ }
+
+ RtlZeroMemory(&BlkifRing->Front, sizeof(blkif_front_ring_t));
+
+ __FreePages(BlkifRing->Mdl);
+ BlkifRing->Shared = NULL;
+ BlkifRing->Mdl = NULL;
+
+ BlkifRing->Events = 0;
+ BlkifRing->Dpcs = 0;
+ BlkifRing->RequestsPosted = 0;
+ BlkifRing->RequestsPushed = 0;
+ BlkifRing->ResponsesProcessed = 0;
+
+ BlkifRing->Connected = FALSE;
+
+ Trace("<==== %u\n", BlkifRing->Index);
+}
+
+static VOID
+__BlkifRingQueueSrb(
+ IN PXENVBD_BLKIF_RING BlkifRing,
+ IN PXENVBD_SRBEXT SrbExt
+ )
+{
+ PLIST_ENTRY ListEntry;
+ ULONG_PTR Old;
+ ULONG_PTR LockBit;
+ ULONG_PTR New;
+
+ ListEntry = &SrbExt->ListEntry;
+
+ do {
+ Old = (ULONG_PTR)BlkifRing->Lock;
+ LockBit = Old & XENVBD_LOCK_BIT;
+
+ ListEntry->Blink = (PVOID)(Old & ~XENVBD_LOCK_BIT);
+ New = (ULONG_PTR)ListEntry;
+ ASSERT((New & XENVBD_LOCK_BIT) == 0);
+ New |= LockBit;
+ } while ((ULONG_PTR)InterlockedCompareExchangePointer(&BlkifRing->Lock,
(PVOID)New, (PVOID)Old) != Old);
+
+ // __BlkifRingReleaseLock() drains the atomic SRB list into the queue
therefore,
+ // after adding to the list we need to attempt to grab and release the
lock. If we can't
+ // grab it then that's ok because whichever thread is holding it will have
to call
+ // __BlkifRingReleaseLock() and will therefore drain the atomic packet
list.
+
+ if (__BlkifRingTryAcquireLock(BlkifRing))
+ __BlkifRingReleaseLock(BlkifRing);
+}
+
+static VOID
+__BlkifRingQueueShutdown(
+ IN PXENVBD_BLKIF_RING BlkifRing,
+ IN PXENVBD_SRBEXT SrbExt
+ )
+{
+ __BlkifRingAcquireLock(BlkifRing);
+ InsertTailList(&BlkifRing->ShutdownQueue, &SrbExt->ListEntry);
+ __BlkifRingReleaseLock(BlkifRing);
+}
+
static DECLSPEC_NOINLINE VOID
-RingIndirectDtor(
+RingDebugCallback(
IN PVOID Argument,
- IN PVOID Object
+ IN BOOLEAN Crashing
)
{
- PXENVBD_INDIRECT Indirect = Object;
+ PXENVBD_RING Ring = Argument;
+ XENVBD_STAT Index;
- UNREFERENCED_PARAMETER(Argument);
+ UNREFERENCED_PARAMETER(Crashing);
- __FreePages(Indirect->Mdl);
- Indirect->Page = NULL;
- Indirect->Mdl = NULL;
+ XENBUS_DEBUG(Printf,
+ &Ring->DebugInterface,
+ "Order: %d\n",
+ Ring->Order);
+
+ for (Index = 0; Index < XENVBD_STAT__MAX; ++Index) {
+ XENBUS_DEBUG(Printf,
+ &Ring->DebugInterface,
+ "%s: %u\n",
+ __StatName(Index),
+ Ring->Stats[Index]);
+ }
}
NTSTATUS
@@ -1493,7 +2248,8 @@ RingCreate(
{
PXENVBD_TARGET Target = FrontendGetTarget(Frontend);
PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target);
- CHAR Name[MAX_NAME_LEN];
+ ULONG MaxQueues;
+ ULONG Index;
NTSTATUS status;
*Ring = __RingAllocate(sizeof(XENVBD_RING));
@@ -1502,127 +2258,74 @@ RingCreate(
if (*Ring == NULL)
goto fail1;
- (*Ring)->Frontend = Frontend;
- KeInitializeSpinLock(&(*Ring)->Lock);
- KeInitializeThreadedDpc(&(*Ring)->Dpc, RingDpc, *Ring);
- KeSetImportanceDpc(&(*Ring)->Dpc, MediumHighImportance);
-
- QueueInit(&(*Ring)->PreparedReqs);
- QueueInit(&(*Ring)->SubmittedReqs);
- QueueInit(&(*Ring)->ShutdownSrbs);
-
- AdapterGetCacheInterface(Adapter, &(*Ring)->CacheInterface);
-
- status = XENBUS_CACHE(Acquire, &(*Ring)->CacheInterface);
- if (!NT_SUCCESS(status))
- goto fail2;
+ AdapterGetDebugInterface(Adapter,
+ &(*Ring)->DebugInterface);
+ AdapterGetStoreInterface(Adapter,
+ &(*Ring)->StoreInterface);
+ AdapterGetCacheInterface(Adapter,
+ &(*Ring)->CacheInterface);
+ AdapterGetEvtchnInterface(Adapter,
+ &(*Ring)->EvtchnInterface);
- status = RtlStringCbPrintfA(Name,
- sizeof(Name),
- "vbd_%u_req",
- FrontendGetTargetId(Frontend));
- if (!NT_SUCCESS(status))
- goto fail3;
+ (*Ring)->Frontend = Frontend;
- status = XENBUS_CACHE(Create,
- &(*Ring)->CacheInterface,
- Name,
- sizeof(XENVBD_REQUEST),
- 32,
- RingRequestCtor,
- RingRequestDtor,
- RingAcquireLock,
- RingReleaseLock,
- *Ring,
- &(*Ring)->RequestCache);
- if (!NT_SUCCESS(status))
- goto fail4;
+ MaxQueues = FrontendGetMaxQueues(Frontend);
+ (*Ring)->Ring = __RingAllocate(sizeof(PXENVBD_BLKIF_RING) *
+ MaxQueues);
- status = RtlStringCbPrintfA(Name,
- sizeof(Name),
- "vbd_%u_seg",
- FrontendGetTargetId(Frontend));
- if (!NT_SUCCESS(status))
- goto fail5;
+ status = STATUS_NO_MEMORY;
+ if ((*Ring)->Ring == NULL)
+ goto fail2;
- status = XENBUS_CACHE(Create,
- &(*Ring)->CacheInterface,
- Name,
- sizeof(XENVBD_SEGMENT),
- 32,
- RingSegmentCtor,
- RingSegmentDtor,
- RingAcquireLock,
- RingReleaseLock,
- *Ring,
- &(*Ring)->SegmentCache);
- if (!NT_SUCCESS(status))
- goto fail6;
+ Index = 0;
+ while (Index < MaxQueues) {
+ PXENVBD_BLKIF_RING BlkifRing;
- status = RtlStringCbPrintfA(Name,
- sizeof(Name),
- "vbd_%u_ind",
- FrontendGetTargetId(Frontend));
- if (!NT_SUCCESS(status))
- goto fail7;
+ status = BlkifRingCreate(*Ring, Index, &BlkifRing);
+ if (!NT_SUCCESS(status))
+ goto fail3;
- status = XENBUS_CACHE(Create,
- &(*Ring)->CacheInterface,
- Name,
- sizeof(XENVBD_INDIRECT),
- 1,
- RingIndirectCtor,
- RingIndirectDtor,
- RingAcquireLock,
- RingReleaseLock,
- *Ring,
- &(*Ring)->IndirectCache);
- if (!NT_SUCCESS(status))
- goto fail8;
+ (*Ring)->Ring[Index] = BlkifRing;
+ Index++;
+ }
return STATUS_SUCCESS;
-fail8:
- Error("fail8\n");
-fail7:
- Error("fail7\n");
- XENBUS_CACHE(Destroy,
- &(*Ring)->CacheInterface,
- (*Ring)->SegmentCache);
- (*Ring)->SegmentCache = NULL;
-fail6:
- Error("fail6\n");
-fail5:
- Error("fail5\n");
- XENBUS_CACHE(Destroy,
- &(*Ring)->CacheInterface,
- (*Ring)->RequestCache);
- (*Ring)->RequestCache = NULL;
-fail4:
- Error("fail4\n");
fail3:
Error("fail3\n");
- XENBUS_CACHE(Release,
- &(*Ring)->CacheInterface);
+
+ while (--Index > 0) {
+ PXENVBD_BLKIF_RING BlkifRing = (*Ring)->Ring[Index];
+
+ (*Ring)->Ring[Index] = NULL;
+ BlkifRingDestroy(BlkifRing);
+ }
+
+ __RingFree((*Ring)->Ring);
+ (*Ring)->Ring = NULL;
+
fail2:
Error("fail2\n");
+ (*Ring)->Frontend = NULL;
+
RtlZeroMemory(&(*Ring)->CacheInterface,
- sizeof (XENBUS_CACHE_INTERFACE));
+ sizeof(XENBUS_CACHE_INTERFACE));
+
+ RtlZeroMemory(&(*Ring)->EvtchnInterface,
+ sizeof(XENBUS_EVTCHN_INTERFACE));
- RtlZeroMemory(&(*Ring)->PreparedReqs, sizeof(XENVBD_QUEUE));
- RtlZeroMemory(&(*Ring)->SubmittedReqs, sizeof(XENVBD_QUEUE));
- RtlZeroMemory(&(*Ring)->ShutdownSrbs, sizeof(XENVBD_QUEUE));
+ RtlZeroMemory(&(*Ring)->StoreInterface,
+ sizeof(XENBUS_STORE_INTERFACE));
- RtlZeroMemory(&(*Ring)->Dpc, sizeof(KDPC));
- RtlZeroMemory(&(*Ring)->Lock, sizeof(KSPIN_LOCK));
- (*Ring)->Frontend = NULL;
+ RtlZeroMemory(&(*Ring)->DebugInterface,
+ sizeof(XENBUS_DEBUG_INTERFACE));
ASSERT(IsZeroMemory(*Ring, sizeof(XENVBD_RING)));
__RingFree(*Ring);
- *Ring = NULL;
+
fail1:
- Error("fail1 %08x\n", status);
+ Error("fail1 (%08x)\n", status);
return status;
}
@@ -1631,44 +2334,35 @@ RingDestroy(
IN PXENVBD_RING Ring
)
{
- XENBUS_CACHE(Destroy,
- &Ring->CacheInterface,
- Ring->IndirectCache);
- Ring->IndirectCache = NULL;
+ ULONG Index;
- XENBUS_CACHE(Destroy,
- &Ring->CacheInterface,
- Ring->SegmentCache);
- Ring->SegmentCache = NULL;
+ Index = FrontendGetMaxQueues(Ring->Frontend);
- XENBUS_CACHE(Destroy,
- &Ring->CacheInterface,
- Ring->RequestCache);
- Ring->RequestCache = NULL;
+ while (--Index > 0) {
+ PXENVBD_BLKIF_RING BlkifRing = Ring->Ring[Index];
+
+ Ring->Ring[Index] = NULL;
+ BlkifRingDestroy(BlkifRing);
+ }
+
+ __RingFree(Ring->Ring);
+ Ring->Ring = NULL;
- XENBUS_CACHE(Release,
- &Ring->CacheInterface);
+ Ring->Frontend = NULL;
RtlZeroMemory(&Ring->CacheInterface,
- sizeof (XENBUS_CACHE_INTERFACE));
+ sizeof(XENBUS_CACHE_INTERFACE));
- RtlZeroMemory(&Ring->PreparedReqs, sizeof(XENVBD_QUEUE));
- RtlZeroMemory(&Ring->SubmittedReqs, sizeof(XENVBD_QUEUE));
- RtlZeroMemory(&Ring->ShutdownSrbs, sizeof(XENVBD_QUEUE));
+ RtlZeroMemory(&Ring->EvtchnInterface,
+ sizeof(XENBUS_EVTCHN_INTERFACE));
- RtlZeroMemory(&Ring->Dpc, sizeof(KDPC));
- RtlZeroMemory(&Ring->Lock, sizeof(KSPIN_LOCK));
- Ring->Frontend = NULL;
+ RtlZeroMemory(&Ring->StoreInterface,
+ sizeof(XENBUS_STORE_INTERFACE));
+
+ RtlZeroMemory(&Ring->DebugInterface,
+ sizeof(XENBUS_DEBUG_INTERFACE));
- Ring->BlkOpRead = 0;
- Ring->BlkOpWrite = 0;
- Ring->BlkOpIndirectRead = 0;
- Ring->BlkOpIndirectWrite = 0;
- Ring->BlkOpBarrier = 0;
- Ring->BlkOpDiscard = 0;
- Ring->BlkOpFlush = 0;
- Ring->SegsGranted = 0;
- Ring->SegsBounced = 0;
+ RtlZeroMemory(Ring->Stats, sizeof(Ring->Stats));
ASSERT(IsZeroMemory(Ring, sizeof(XENVBD_RING)));
__RingFree(Ring);
@@ -1679,31 +2373,27 @@ RingConnect(
IN PXENVBD_RING Ring
)
{
- PXENVBD_TARGET Target = FrontendGetTarget(Ring->Frontend);
- PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target);
- PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend);
- PCHAR Buffer;
+ ULONG MaxQueues;
ULONG Index;
+ PCHAR Buffer;
NTSTATUS status;
- ASSERT(Ring->Connected == FALSE);
-
- AdapterGetStoreInterface(Adapter, &Ring->StoreInterface);
- AdapterGetEvtchnInterface(Adapter, &Ring->EvtchnInterface);
- AdapterGetDebugInterface(Adapter, &Ring->DebugInterface);
-
- status = XENBUS_STORE(Acquire, &Ring->StoreInterface);
+ status = XENBUS_DEBUG(Acquire, &Ring->DebugInterface);
if (!NT_SUCCESS(status))
goto fail1;
- status = XENBUS_EVTCHN(Acquire, &Ring->EvtchnInterface);
+ status = XENBUS_STORE(Acquire, &Ring->StoreInterface);
if (!NT_SUCCESS(status))
goto fail2;
- status = XENBUS_DEBUG(Acquire, &Ring->DebugInterface);
+ status = XENBUS_CACHE(Acquire, &Ring->CacheInterface);
if (!NT_SUCCESS(status))
goto fail3;
+ status = XENBUS_EVTCHN(Acquire, &Ring->EvtchnInterface);
+ if (!NT_SUCCESS(status))
+ goto fail4;
+
status = XENBUS_STORE(Read,
&Ring->StoreInterface,
NULL,
@@ -1730,47 +2420,17 @@ RingConnect(
Ring->Order = 0;
}
- Ring->Mdl = __AllocatePages(1 << Ring->Order);
-
- status = STATUS_NO_MEMORY;
- if (Ring->Mdl == NULL)
- goto fail4;
-
- Ring->Shared = MmGetSystemAddressForMdlSafe(Ring->Mdl,
- NormalPagePriority);
- ASSERT(Ring->Shared != NULL);
-
-#pragma warning(push)
-#pragma warning(disable: 4305)
-#pragma warning(disable: 4311) // 'type cast' pointer truncation from
'blkif_sring_entry[1]' to 'long'
- SHARED_RING_INIT(Ring->Shared);
- FRONT_RING_INIT(&Ring->Front, Ring->Shared, PAGE_SIZE << Ring->Order);
-#pragma warning(pop)
+ MaxQueues = FrontendGetNumQueues(Ring->Frontend);
+ Index = 0;
+ while (Index < MaxQueues) {
+ PXENVBD_BLKIF_RING BlkifRing = Ring->Ring[Index];
- for (Index = 0; Index < (1ul << Ring->Order); ++Index) {
- status = GranterGet(Granter,
- MmGetMdlPfnArray(Ring->Mdl)[Index],
- FALSE,
- &Ring->Grants[Index]);
+ status = BlkifRingConnect(BlkifRing);
if (!NT_SUCCESS(status))
goto fail5;
- }
-
- Ring->Channel = XENBUS_EVTCHN(Open,
- &Ring->EvtchnInterface,
- XENBUS_EVTCHN_TYPE_UNBOUND,
- RingInterrupt,
- Ring,
- FrontendGetBackendDomain(Ring->Frontend),
- TRUE);
- status = STATUS_NO_MEMORY;
- if (Ring->Channel == NULL)
- goto fail6;
- XENBUS_EVTCHN(Unmask,
- &Ring->EvtchnInterface,
- Ring->Channel,
- FALSE);
+ ++Index;
+ }
status = XENBUS_DEBUG(Register,
&Ring->DebugInterface,
@@ -1779,55 +2439,44 @@ RingConnect(
Ring,
&Ring->DebugCallback);
if (!NT_SUCCESS(status))
- goto fail7;
+ goto fail6;
- Ring->Connected = TRUE;
return STATUS_SUCCESS;
-fail7:
- Error("fail7\n");
- XENBUS_EVTCHN(Close,
- &Ring->EvtchnInterface,
- Ring->Channel);
- Ring->Channel = NULL;
fail6:
Error("fail6\n");
+ Index = FrontendGetNumQueues(Ring->Frontend);
fail5:
Error("fail5\n");
- for (Index = 0; Index < (1ul << Ring->Order); ++Index) {
- if (Ring->Grants[Index] == NULL)
- continue;
- GranterPut(Granter, Ring->Grants[Index]);
- Ring->Grants[Index] = NULL;
- }
+ while (Index != 0) {
+ PXENVBD_BLKIF_RING BlkifRing;
+
+ --Index;
+ BlkifRing = Ring->Ring[Index];
- RtlZeroMemory(&Ring->Front, sizeof(blkif_front_ring_t));
+ BlkifRingDisconnect(BlkifRing);
+ }
- __FreePages(Ring->Mdl);
- Ring->Shared = NULL;
- Ring->Mdl = NULL;
+ XENBUS_EVTCHN(Release, &Ring->EvtchnInterface);
- Ring->Order = 0;
fail4:
Error("fail4\n");
- XENBUS_DEBUG(Release, &Ring->DebugInterface);
+
+ XENBUS_CACHE(Release, &Ring->CacheInterface);
+
fail3:
Error("fail3\n");
- XENBUS_EVTCHN(Release, &Ring->EvtchnInterface);
+
+ XENBUS_STORE(Release, &Ring->StoreInterface);
+
fail2:
Error("fail2\n");
- XENBUS_STORE(Release, &Ring->StoreInterface);
-fail1:
- Error("fail1 %08x\n", status);
- RtlZeroMemory(&Ring->DebugInterface,
- sizeof(XENBUS_DEBUG_INTERFACE));
- RtlZeroMemory(&Ring->EvtchnInterface,
- sizeof(XENBUS_EVTCHN_INTERFACE));
- RtlZeroMemory(&Ring->StoreInterface,
- sizeof(XENBUS_STORE_INTERFACE));
+ XENBUS_DEBUG(Release, &Ring->DebugInterface);
+fail1:
+ Error("fail1 (%08x)\n", status);
return status;
}
@@ -1837,23 +2486,33 @@ RingStoreWrite(
IN PVOID Transaction
)
{
- PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend);
- ULONG Port;
+ ULONG NumQueues;
+ ULONG Index;
NTSTATUS status;
- if (Ring->Order == 0) {
- status = XENBUS_STORE(Printf,
- &Ring->StoreInterface,
- Transaction,
- FrontendGetFrontendPath(Ring->Frontend),
- "ring-ref",
- "%u",
- GranterReference(Granter, Ring->Grants[0]));
+ NumQueues = FrontendGetNumQueues(Ring->Frontend);
+ Index = 0;
+ while (Index < NumQueues) {
+ PXENVBD_BLKIF_RING BlkifRing = Ring->Ring[Index];
+
+ status = BlkifRingStoreWrite(BlkifRing, Transaction);
if (!NT_SUCCESS(status))
- return status;
- } else {
- ULONG Index;
+ goto fail1;
+
+ ++Index;
+ }
+
+ status = XENBUS_STORE(Printf,
+ &Ring->StoreInterface,
+ Transaction,
+ FrontendGetFrontendPath(Ring->Frontend),
+ "multi-queue-num-queues",
+ "%u",
+ NumQueues);
+ if (!NT_SUCCESS(status))
+ goto fail2;
+ if (Ring->Order != 0) {
status = XENBUS_STORE(Printf,
&Ring->StoreInterface,
Transaction,
@@ -1862,28 +2521,7 @@ RingStoreWrite(
"%u",
Ring->Order);
if (!NT_SUCCESS(status))
- return status;
-
- for (Index = 0; Index < (1ul << Ring->Order); ++Index) {
- CHAR Name[MAX_NAME_LEN+1];
-
- status = RtlStringCchPrintfA(Name,
- MAX_NAME_LEN,
- "ring-ref%u",
- Index);
- if (!NT_SUCCESS(status))
- return status;
-
- status = XENBUS_STORE(Printf,
- &Ring->StoreInterface,
- Transaction,
- FrontendGetFrontendPath(Ring->Frontend),
- Name,
- "%u",
- GranterReference(Granter,
Ring->Grants[Index]));
- if (!NT_SUCCESS(status))
- return status;
- }
+ goto fail3;
}
status = XENBUS_STORE(Printf,
@@ -1893,23 +2531,15 @@ RingStoreWrite(
"protocol",
XEN_IO_PROTO_ABI);
if (!NT_SUCCESS(status))
- return status;
-
- Port = XENBUS_EVTCHN(GetPort,
- &Ring->EvtchnInterface,
- Ring->Channel);
-
- status = XENBUS_STORE(Printf,
- &Ring->StoreInterface,
- Transaction,
- FrontendGetFrontendPath(Ring->Frontend),
- "event-channel",
- "%u",
- Port);
- if (!NT_SUCCESS(status))
- return status;
+ goto fail4;
return STATUS_SUCCESS;
+
+fail4:
+fail3:
+fail2:
+fail1:
+ return status;
}
VOID
@@ -1917,12 +2547,18 @@ RingEnable(
IN PXENVBD_RING Ring
)
{
- ASSERT(Ring->Enabled == FALSE);
- Ring->Enabled = TRUE;
+ ULONG NumQueues;
+ ULONG Index;
- XENBUS_EVTCHN(Trigger,
- &Ring->EvtchnInterface,
- Ring->Channel);
+ NumQueues = FrontendGetNumQueues(Ring->Frontend);
+ Index = 0;
+ while (Index < NumQueues) {
+ PXENVBD_BLKIF_RING BlkifRing = Ring->Ring[Index];
+
+ BlkifRingEnable(BlkifRing);
+
+ ++Index;
+ }
}
VOID
@@ -1930,55 +2566,15 @@ RingDisable(
IN PXENVBD_RING Ring
)
{
- ULONG Count;
- KIRQL Irql;
- PXENVBD_TARGET Target = FrontendGetTarget(Ring->Frontend);
- PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target);
-
- ASSERT(Ring->Enabled == TRUE);
- Ring->Enabled = FALSE;
-
- // poll ring and send event channel notification every 1ms (for up to 3
minutes)
- Count = 0;
- while (QueueCount(&Ring->SubmittedReqs)) {
- if (Count > 180000)
- break;
- KeRaiseIrql(DISPATCH_LEVEL, &Irql);
- RingPoll(Ring);
- KeLowerIrql(Irql);
- XENBUS_EVTCHN(Send,
- &Ring->EvtchnInterface,
- Ring->Channel);
- StorPortStallExecution(1000); // 1000 micro-seconds
- ++Count;
- }
-
- Verbose("Target[%d] : %u Submitted requests left (%u iterrations)\n",
- FrontendGetTargetId(Ring->Frontend),
- QueueCount(&Ring->SubmittedReqs),
- Count);
-
- // Fail PreparedReqs
- for (;;) {
- PXENVBD_SRBEXT SrbExt;
- PSCSI_REQUEST_BLOCK Srb;
- PXENVBD_REQUEST Request;
- PLIST_ENTRY ListEntry;
-
- ListEntry = QueuePop(&Ring->PreparedReqs);
- if (ListEntry == NULL)
- break;
- Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, ListEntry);
- SrbExt = Request->SrbExt;
- Srb = SrbExt->Srb;
-
- Srb->SrbStatus = SRB_STATUS_ABORTED;
- Srb->ScsiStatus = 0x40; // SCSI_ABORTED
+ ULONG Index;
- RingPutRequest(Ring, Request);
+ Index = FrontendGetNumQueues(Ring->Frontend);
+ while (Index != 0) {
+ PXENVBD_BLKIF_RING BlkifRing;
- if (InterlockedDecrement(&SrbExt->RequestCount) == 0)
- AdapterCompleteSrb(Adapter, SrbExt);
+ --Index;
+ BlkifRing = Ring->Ring[Index];
+ BlkifRingDisable(BlkifRing);
}
}
@@ -1987,54 +2583,49 @@ RingDisconnect(
IN PXENVBD_RING Ring
)
{
- PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend);
ULONG Index;
- ASSERT3U(Ring->Submitted, ==, Ring->Received);
- ASSERT(Ring->Connected);
- Ring->Connected = FALSE;
-
XENBUS_DEBUG(Deregister,
&Ring->DebugInterface,
Ring->DebugCallback);
Ring->DebugCallback = NULL;
- XENBUS_EVTCHN(Close,
- &Ring->EvtchnInterface,
- Ring->Channel);
- Ring->Channel = NULL;
-
- for (Index = 0; Index < (1ul << Ring->Order); ++Index) {
- if (Ring->Grants[Index] == NULL)
- continue;
+ Index = FrontendGetNumQueues(Ring->Frontend);
- GranterPut(Granter, Ring->Grants[Index]);
- Ring->Grants[Index] = NULL;
- }
+ while (Index != 0) {
+ PXENVBD_BLKIF_RING BlkifRing;
- RtlZeroMemory(&Ring->Front, sizeof(blkif_front_ring_t));
+ --Index;
+ BlkifRing = Ring->Ring[Index];
- __FreePages(Ring->Mdl);
- Ring->Shared = NULL;
- Ring->Mdl = NULL;
+ BlkifRingDisconnect(BlkifRing);
+ }
Ring->Order = 0;
- XENBUS_DEBUG(Release, &Ring->DebugInterface);
XENBUS_EVTCHN(Release, &Ring->EvtchnInterface);
+ XENBUS_CACHE(Release, &Ring->CacheInterface);
XENBUS_STORE(Release, &Ring->StoreInterface);
+ XENBUS_DEBUG(Release, &Ring->DebugInterface);
+}
- RtlZeroMemory(&Ring->DebugInterface,
- sizeof(XENBUS_DEBUG_INTERFACE));
- RtlZeroMemory(&Ring->EvtchnInterface,
- sizeof(XENBUS_EVTCHN_INTERFACE));
- RtlZeroMemory(&Ring->StoreInterface,
- sizeof(XENBUS_STORE_INTERFACE));
+static FORCEINLINE PXENVBD_BLKIF_RING
+__RingGetBlkifRing(
+ IN PXENVBD_RING Ring,
+ IN ULONG Tag
+ )
+{
+ ULONG Value;
+ ULONG Index;
+
+ if (Tag == 0)
+ Value = KeGetCurrentProcessorNumberEx(NULL);
+ else
+ Value = Tag;
+
+ Index = Value % FrontendGetNumQueues(Ring->Frontend);
- Ring->Events = 0;
- Ring->Dpcs = 0;
- Ring->Submitted = 0;
- Ring->Received = 0;
+ return Ring->Ring[Index];
}
BOOLEAN
@@ -2044,25 +2635,14 @@ RingQueueRequest(
)
{
PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
+ PXENVBD_BLKIF_RING BlkifRing;
- if (!Ring->Enabled)
- goto fail1;
-
- if (!RingPrepareRequest(Ring, SrbExt))
- goto fail2;
+ BlkifRing = __RingGetBlkifRing(Ring, Srb->QueueTag);
+ ASSERT(BlkifRing != NULL);
- if (RingSubmitRequests(Ring)) {
- // more prepared-reqs to submit
- if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
- ++Ring->Dpcs;
- }
+ __BlkifRingQueueSrb(BlkifRing, SrbExt);
return TRUE;
-
-fail2:
-fail1:
- Srb->SrbStatus = SRB_STATUS_BUSY;
- return FALSE;
}
VOID
@@ -2071,12 +2651,11 @@ RingQueueShutdown(
IN PXENVBD_SRBEXT SrbExt
)
{
- QueueAppend(&Ring->ShutdownSrbs,
- &SrbExt->ListEntry);
+ PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
+ PXENVBD_BLKIF_RING BlkifRing;
- if (!Ring->Enabled)
- return;
+ BlkifRing = __RingGetBlkifRing(Ring, Srb->QueueTag);
+ ASSERT(BlkifRing != NULL);
- if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
- ++Ring->Dpcs;
+ __BlkifRingQueueShutdown(BlkifRing, SrbExt);
}
--
2.16.2.windows.1
_______________________________________________
win-pv-devel mailing list
win-pv-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/win-pv-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |