Re: [PATCH] xennet: improve NBL batching receive path

[Tu Dinh <ngoc-tu.dinh@xxxxxxxxxx>]

On 25/09/2026 16:02, david ambu wrote:
> Raise NblBatchMax default to 32 (max 64). Split batches at EtherType
> boundaries and set NDIS_RECEIVE_FLAGS_SINGLE_ETHER_TYPE per batch.
> 
> In __ReceiverPushPackets, sample Returned before the InterlockedAdd
> so it is read prior to the implicit full memory barrier that
> InterlockedAdd provides. Use the return value of InterlockedAdd
> directly as Indicated and remove the now-redundant KeMemoryBarrier.
> 
> Return early when queue is empty.
> 
> Signed-off-by: david ambu <david.preetham@xxxxxxxxxx>
> Co-Authored-By: Claude Sonnet 4.6 <noreply@xxxxxxxxxxxxx>

Please use the tag form "Assisted-by" to indicate LLM assistance.

[...]

> @@ -156,10 +159,25 @@ __ReceiverAllocateNetBufferList(
>   
>       if (NetBufferList != NULL) {
>           PNET_BUFFER_LIST_RESERVED   ListReserved;
> +        USHORT                      EtherType = 0;
> +
> +        if (Info->EthernetHeader.Length != 0) {
> +            PUCHAR              Buffer;
> +            PETHERNET_HEADER    EthernetHeader;
> +
> +            Buffer = MmGetSystemAddressForMdlSafe(Mdl, NormalPagePriority | 
> MdlMappingNoExecute);
> +            if (Buffer != NULL) {
> +                EthernetHeader = (PETHERNET_HEADER)(Buffer + Offset + 
> Info->EthernetHeader.Offset);
> +                EtherType = ETHERNET_HEADER_IS_TAGGED(EthernetHeader) ?
> +                            EthernetHeader->Tagged.TypeOrLength :
> +                            EthernetHeader->Untagged.TypeOrLength;
> +            }
> +        }

Per xenvif!__ParseEthernetHeader, you may want to skip EtherType 
matching if TypeOrLength <= ETHERNET_MTU, because that indicates an 
LLC/SNAP frame:

[1] 
https://github.com/xcp-ng/win-xenvif/blob/9a4f184f60aa2c9e649d9d4219424d5b16cfe27c/src/xenvif/parse.c#L606

>   
>           ListReserved = 
> (PNET_BUFFER_LIST_RESERVED)NET_BUFFER_LIST_MINIPORT_RESERVED(NetBufferList);
>           ASSERT3P(ListReserved->Cookie, ==, NULL);
>           ListReserved->Cookie = Cookie;
> +        ListReserved->EtherType = EtherType;
>       }
>   
>       return NetBufferList;
> @@ -178,6 +196,7 @@ __ReceiverReleaseNetBufferList(
>       ListReserved = 
> (PNET_BUFFER_LIST_RESERVED)NET_BUFFER_LIST_MINIPORT_RESERVED(NetBufferList);
>       Cookie = ListReserved->Cookie;
>       ListReserved->Cookie = NULL;
> +    ListReserved->EtherType = 0;
>   
>       if (Cache)
>           __ReceiverPutNetBufferList(Receiver, NetBufferList);
> @@ -240,12 +259,12 @@ __ReceiverReceivePacket(
>       NDIS_TCP_IP_CHECKSUM_NET_BUFFER_LIST_INFO   csumInfo;
>   
>       UNREFERENCED_PARAMETER(MaximumSegmentSize);
> -    UNREFERENCED_PARAMETER(Info);
>   
>       NetBufferList = __ReceiverAllocateNetBufferList(Receiver,
>                                                       Mdl,
>                                                       Offset,
>                                                       Length,
> +                                                    Info,
>                                                       Cookie);
>       if (NetBufferList == NULL)
>           goto fail1;
> @@ -328,9 +347,11 @@ fail1:
>       return NULL;
>   }
>   
> -static FORCEINLINE VOID __IndicateReceiveNetBufferLists(
> +
> +static FORCEINLINE VOID
> +__IndicateReceiveNetBufferLists(
>       IN  PXENNET_RECEIVER    Receiver,
> -    IN  PNET_BUFFER_LIST    NetBufferLists,
> +    IN  PNET_BUFFER_LIST    Remaining,
>       IN  NDIS_PORT_NUMBER    PortNumber,
>       IN  ULONG               NumberOfNetBufferLists,
>       IN  ULONG               ReceiveFlags
> @@ -339,39 +360,70 @@ static FORCEINLINE VOID __IndicateReceiveNetBufferLists(
>       PXENNET_ADAPTER         Adapter = Receiver->Adapter;
>       NDIS_HANDLE             MiniportAdapterHandle = 
> AdapterGetHandle(Adapter);
>       PXENVIF_VIF_INTERFACE   VifInterface;
> -    ULONG                   Count;
> +    ULONG                   Count = 0;
> +    ULONG                   NblMaxBatch = AdapterGetNblMaxBatchSize(Adapter);
> +
> +    while (Remaining != NULL) {
> +        PNET_BUFFER_LIST    BatchTail = Remaining;
> +        PNET_BUFFER_LIST    NextNbl;
> +        ULONG               BatchCount = 1;
> +        ULONG               BatchFlags;
> +        USHORT              FirstEth;
> +
> +        FirstEth = ((PNET_BUFFER_LIST_RESERVED)
> +                    NET_BUFFER_LIST_MINIPORT_RESERVED(Remaining))->EtherType;
> +        NextNbl = NET_BUFFER_LIST_NEXT_NBL(BatchTail);
> +
> +        while (NextNbl != NULL && BatchCount < NblMaxBatch) {
> +            USHORT  CurEth;
> +            CurEth = ((PNET_BUFFER_LIST_RESERVED)
> +                      NET_BUFFER_LIST_MINIPORT_RESERVED(NextNbl))->EtherType;
> +            if (CurEth != FirstEth)
> +                break;
> +            BatchTail = NextNbl;
> +            NextNbl = NET_BUFFER_LIST_NEXT_NBL(BatchTail);
> +            BatchCount++;
> +        }
>   
> -    VifInterface = AdapterGetVifInterface(Receiver->Adapter);
> +        NET_BUFFER_LIST_NEXT_NBL(BatchTail) = NULL;
>   
> -    Count = 0;
> -    while (NetBufferLists != NULL) {
> -        PNET_BUFFER_LIST        Next;
> -
> -        Next = NET_BUFFER_LIST_NEXT_NBL(NetBufferLists);
> -        NET_BUFFER_LIST_NEXT_NBL(NetBufferLists) = NULL;
> +        BatchFlags = ReceiveFlags;
> +        if (FirstEth != 0)
> +            BatchFlags |= NDIS_RECEIVE_FLAGS_SINGLE_ETHER_TYPE;
> +        else
> +            BatchFlags &= ~NDIS_RECEIVE_FLAGS_SINGLE_ETHER_TYPE;
>   
>           NdisMIndicateReceiveNetBufferLists(MiniportAdapterHandle,
> -                                           NetBufferLists,
> +                                           Remaining,
>                                              PortNumber,
> -                                           1,
> -                                           ReceiveFlags);
> +                                           BatchCount,
> +                                           BatchFlags);
> +
> +        if (BatchFlags & NDIS_RECEIVE_FLAGS_RESOURCES) {
> +            PNET_BUFFER_LIST    Current = Remaining;
> +
> +            VifInterface = AdapterGetVifInterface(Receiver->Adapter);
> +
> +            while (Current != NULL) {
> +                PNET_BUFFER_LIST    Next = NET_BUFFER_LIST_NEXT_NBL(Current);
> +                NET_BUFFER_LIST_NEXT_NBL(Current) = NULL;
>   
> -        if (ReceiveFlags & NDIS_RECEIVE_FLAGS_RESOURCES) {
> -            PVOID   Cookie;
> +                PVOID Cookie = __ReceiverReleaseNetBufferList(Receiver,
> +                                                              Current,
> +                                                              FALSE);
>   
> -            Cookie = __ReceiverReleaseNetBufferList(Receiver,
> -                                                    NetBufferLists,
> -                                                    FALSE);
> +                XENVIF_VIF(ReceiverReturnPacket,
> +                           VifInterface,
> +                           Cookie);
>   
> -            XENVIF_VIF(ReceiverReturnPacket,
> -                       VifInterface,
> -                       Cookie);
> +                Current = Next;
> +            }
>   
> -            (VOID) InterlockedIncrement(&Receiver->Returned);
> +            (VOID)InterlockedAdd(&Receiver->Returned, BatchCount);
>           }
>   
> -        Count++;
> -        NetBufferLists = Next;
> +        Count += BatchCount;
> +        Remaining = NextNbl;
>       }
>       ASSERT3U(Count, ==, NumberOfNetBufferLists);
>   }
> @@ -402,13 +454,11 @@ __ReceiverPushPackets(
>   
>       KeReleaseSpinLockFromDpcLevel(&Queue->Lock);
>   
> -    (VOID) InterlockedAdd(&Receiver->Indicated, Count);
> +    if (Count == 0)
> +        return;
>   
>       Returned = Receiver->Returned;
> -
> -    KeMemoryBarrier();
> -
> -    Indicated = Receiver->Indicated;
> +    Indicated = InterlockedAdd(&Receiver->Indicated, Count);
>   
>       Flags = NDIS_RECEIVE_FLAGS_DISPATCH_LEVEL |
>               NDIS_RECEIVE_FLAGS_PERFECT_FILTERED;



--
Ngoc Tu Dinh | Vates XCP-ng Developer

XCP-ng & Xen Orchestra - Vates solutions

web: https://vates.tech