Add AF_XDP notification and query for NIC checksum offload state (#767)

published/external/afxdp.h

@@ -56,6 +56,7 @@ typedef enum _XSK_RING_FLAGS {
     XSK_RING_FLAG_ERROR = 0x1,
     XSK_RING_FLAG_NEED_POKE = 0x2,
     XSK_RING_FLAG_AFFINITY_CHANGED = 0x4,
+    XSK_RING_FLAG_OFFLOAD_CHANGED = 0x8,
 } XSK_RING_FLAGS;
 
 DEFINE_ENUM_FLAG_OPERATORS(XSK_RING_FLAGS);

published/external/afxdp_experimental.h

@@ -94,6 +94,15 @@ typedef enum _XSK_POLL_MODE {
 //
 #define XSK_SOCKOPT_TX_OFFLOAD_CHECKSUM 1003
 
+//
+// XSK_SOCKOPT_TX_OFFLOAD_CURRENT_CONFIG_CHECKSUM
+//
+// Supports: get
+// Optval type: UINT32
+// Description: Returns the TX queue's current checksum offload configuration.
+//
+#define XSK_SOCKOPT_TX_OFFLOAD_CURRENT_CONFIG_CHECKSUM 1004
+
 #ifdef __cplusplus
 } // extern "C"
 #endif

published/external/afxdp_helper.h

@@ -75,7 +75,7 @@ XskRingConsumerReserve(
 {
     UINT32 Consumer = *Ring->SharedConsumer;
     UINT32 Available;
-    
+
     *Index = Consumer;
     Available =  Ring->CachedProducer - Consumer;
     if (Available >= MaxCount) {
@@ -156,6 +156,15 @@ XskRingAffinityChanged(
     return !!(XskRingGetFlags(Ring) & XSK_RING_FLAG_AFFINITY_CHANGED);
 }
 
+inline
+BOOLEAN
+XskRingOffloadChanged(
+    _In_ const XSK_RING *Ring
+    )
+{
+    return !!(XskRingGetFlags(Ring) & XSK_RING_FLAG_OFFLOAD_CHANGED);
+}
+
 #ifdef __cplusplus
 } // extern "C"
 #endif

published/external/xdpapi_experimental.h

@@ -174,6 +174,19 @@ XdpInitializeRssConfiguration(
     RssConfiguration->Header.Size = XDP_SIZEOF_RSS_CONFIGURATION_REVISION_1;
 }
 
+typedef struct _XDP_CHECKSUM_CONFIGURATION {
+    XDP_OBJECT_HEADER Header;
+
+    //
+    // Indicates whether checksum offload is enabled.
+    //
+    BOOLEAN Enabled;
+} XDP_CHECKSUM_CONFIGURATION;
+
+#define XDP_CHECKSUM_CONFIGURATION_REVISION_1 1
+#define XDP_SIZEOF_CHECKSUM_CONFIGURATION_REVISION_1 \
+    RTL_SIZEOF_THROUGH_FIELD(XDP_CHECKSUM_CONFIGURATION, Enabled)
+
 typedef enum _XDP_QUIC_OPERATION {
     XDP_QUIC_OPERATION_ADD,     // Add (or modify) a QUIC connection offload
     XDP_QUIC_OPERATION_REMOVE,  // Remove a QUIC connection offload

published/private/xdpif.h

@@ -85,6 +85,7 @@ XDP_REMOVE_INTERFACE_COMPLETE(
 typedef enum {
     XdpOffloadRss,
     XdpOffloadQeo,
+    XdpOffloadChecksum,
 } XDP_INTERFACE_OFFLOAD_TYPE;
 
 typedef enum {
@@ -110,19 +111,6 @@ typedef struct _XDP_OFFLOAD_PARAMS_RSS {
     PROCESSOR_NUMBER IndirectionTable[XDP_RSS_INDIRECTION_TABLE_SIZE];
 } XDP_OFFLOAD_PARAMS_RSS;
 
-typedef struct _XDP_OFFLOAD_PARAMS_CHECKSUM {
-    XDP_OFFLOAD_STATE Ipv4Rx;
-    XDP_OFFLOAD_STATE Ipv4Tx;
-    XDP_OFFLOAD_STATE Tcpv4Rx;
-    XDP_OFFLOAD_STATE Tcpv4Tx;
-    XDP_OFFLOAD_STATE Tcpv6Rx;
-    XDP_OFFLOAD_STATE Tcpv6Tx;
-    XDP_OFFLOAD_STATE Udpv4Rx;
-    XDP_OFFLOAD_STATE Udpv4Tx;
-    XDP_OFFLOAD_STATE Udpv6Rx;
-    XDP_OFFLOAD_STATE Udpv6Tx;
-} XDP_OFFLOAD_PARAMS_CHECKSUM;
-
 typedef struct _XDP_OFFLOAD_PARAMS_LSO {
     XDP_OFFLOAD_STATE Ipv4;
     XDP_OFFLOAD_STATE Ipv6;

published/private/xdptxqueue_internal.h

@@ -75,6 +75,10 @@ typedef enum _XDP_TX_QUEUE_NOTIFY_CODE {
     // for deletion.
     //
     XDP_TX_QUEUE_NOTIFY_MAX_FRAME_SIZE,
+    //
+    // The TX queue's current offload configuration has changed.
+    //
+    XDP_TX_QUEUE_NOTIFY_OFFLOAD_CURRENT_CONFIG,
 } XDP_TX_QUEUE_NOTIFY_CODE;
 
 typedef

src/xdp/tx.c

@@ -33,7 +33,6 @@ typedef struct _XDP_TX_QUEUE {
     XDP_TX_QUEUE_KEY Key;
     XDP_TX_QUEUE_STATE State;
     XDP_BINDING_CLIENT_ENTRY BindingClientEntry;
-    LIST_ENTRY NotifyClients;
     PCW_INSTANCE *PcwInstance;
 
     XDP_TX_CAPABILITIES InterfaceTxCapabilities;
@@ -49,9 +48,15 @@ typedef struct _XDP_TX_QUEUE {
 
     XDP_IF_OFFLOAD_HANDLE InterfaceOffloadHandle;
 
-    BOOLEAN DeleteNeeded;
-    XDP_BINDING_WORKITEM DeleteWorkItem;
-    XDP_TX_QUEUE_NOTIFY_DETAILS NotifyDetails;
+    struct {
+        KSPIN_LOCK Lock;
+        BOOLEAN WorkerQueued : 1;
+        BOOLEAN DeleteNeeded : 1;
+        BOOLEAN OffloadNeeded : 1;
+        XDP_BINDING_WORKITEM WorkItem;
+        XDP_TX_QUEUE_NOTIFY_DETAILS Details;
+        LIST_ENTRY Clients;
+    } Notify;
 
     //
     // Data path fields.
@@ -597,7 +602,7 @@ XdppTxQueueGetNotifyHandle(
 {
     XDP_TX_QUEUE *TxQueue = XdpTxQueueFromConfigCreate(TxQueueConfig);
 
-    return (XDP_TX_QUEUE_NOTIFY_HANDLE)&TxQueue->NotifyDetails;
+    return (XDP_TX_QUEUE_NOTIFY_HANDLE)&TxQueue->Notify.Details;
 }
 
 static
@@ -606,23 +611,26 @@ XdpTxQueueFromNotify(
     _In_ XDP_TX_QUEUE_NOTIFY_HANDLE TxQueueNotifyHandle
     )
 {
-    return CONTAINING_RECORD(TxQueueNotifyHandle, XDP_TX_QUEUE, NotifyDetails);
+    return CONTAINING_RECORD(TxQueueNotifyHandle, XDP_TX_QUEUE, Notify.Details);
 }
 
 static
+_IRQL_requires_(PASSIVE_LEVEL)
 VOID
 XdpTxQueueNotifyClients(
     _In_ XDP_TX_QUEUE *TxQueue,
     _In_ XDP_TX_QUEUE_NOTIFICATION_TYPE NotificationType
     )
 {
-    LIST_ENTRY *Entry = TxQueue->NotifyClients.Flink;
+    LIST_ENTRY *Entry = TxQueue->Notify.Clients.Flink;
 
     TraceInfo(
         TRACE_CORE, "TxQueue=%p NotificationType=%!TX_QUEUE_NOTIFICATION_TYPE!",
         TxQueue, NotificationType);
 
-    while (Entry != &TxQueue->NotifyClients) {
+    ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL);
+
+    while (Entry != &TxQueue->Notify.Clients) {
         XDP_TX_QUEUE_NOTIFICATION_ENTRY *NotifyEntry;
 
         NotifyEntry = CONTAINING_RECORD(Entry, XDP_TX_QUEUE_NOTIFICATION_ENTRY, Link);
@@ -632,15 +640,55 @@ XdpTxQueueNotifyClients(
     }
 }
 
+static
+_Requires_lock_held_(TxQueue->Notify.Lock)
+VOID
+XdpTxQueueNotifyClientsUnderNotifyLock(
+    _In_ XDP_TX_QUEUE *TxQueue,
+    _In_ XDP_TX_QUEUE_NOTIFICATION_TYPE NotificationType,
+    _In_ _IRQL_saves_ _IRQL_restores_ KIRQL *OldIrql
+    )
+{
+    //
+    // N.B. This routine releases and re-acquires the notify lock.
+    //
+    ASSERT(*OldIrql == PASSIVE_LEVEL);
+    KeReleaseSpinLock(&TxQueue->Notify.Lock, *OldIrql);
+    XdpTxQueueNotifyClients(TxQueue, NotificationType);
+    KeAcquireSpinLock(&TxQueue->Notify.Lock, OldIrql);
+}
+
 static
 VOID
-XdpTxQueueDeleteWorker(
+XdpTxQueueNotifyWorker(
     _In_ XDP_BINDING_WORKITEM *Item
     )
 {
-    XDP_TX_QUEUE *TxQueue = CONTAINING_RECORD(Item, XDP_TX_QUEUE, DeleteWorkItem);
+    XDP_TX_QUEUE *TxQueue = CONTAINING_RECORD(Item, XDP_TX_QUEUE, Notify.WorkItem);
+    KIRQL OldIrql;
+
+    KeAcquireSpinLock(&TxQueue->Notify.Lock, &OldIrql);
+
+    ASSERT(TxQueue->Notify.WorkerQueued);
+
+    while (TRUE) {
+        if (TxQueue->Notify.DeleteNeeded) {
+            TxQueue->Notify.DeleteNeeded = FALSE;
+            XdpTxQueueNotifyClientsUnderNotifyLock(
+                TxQueue, XDP_TX_QUEUE_NOTIFICATION_DETACH, &OldIrql);
+        } else if (TxQueue->Notify.OffloadNeeded) {
+            TxQueue->Notify.OffloadNeeded = FALSE;
+            XdpTxQueueNotifyClientsUnderNotifyLock(
+                TxQueue, XDP_TX_QUEUE_NOTIFICATION_OFFLOAD_CURRENT_CONFIG, &OldIrql);
+        } else {
+            break;
+        }
+    }
+
+    TxQueue->Notify.WorkerQueued = FALSE;
+
+    KeReleaseSpinLock(&TxQueue->Notify.Lock, OldIrql);
 
-    XdpTxQueueNotifyClients(TxQueue, XDP_TX_QUEUE_NOTIFICATION_DETACH);
     XdpTxQueueInterlockedDereference(TxQueue);
 }
 
@@ -654,6 +702,15 @@ XdpTxQueueNotify(
     )
 {
     XDP_TX_QUEUE *TxQueue = XdpTxQueueFromNotify(TxQueueNotifyHandle);
+    KIRQL OldIrql;
+    BOOLEAN NeedNotification = FALSE;
+
+    //
+    // This routine can be invoked from arbitrary contexts, including passive
+    // worker threads and data paths at up to dispatch level.
+    //
+
+    KeAcquireSpinLock(&TxQueue->Notify.Lock, &OldIrql);
 
     switch (NotifyCode) {
     case XDP_TX_QUEUE_NOTIFY_MAX_FRAME_SIZE:
@@ -667,14 +724,29 @@ XdpTxQueueNotify(
         // Similar to NetAdapter, changing MTU after a queue is created causes
         // the queue to be torn down, and perhaps re-created.
         //
-        if (!InterlockedExchangeNoFence8((CHAR *)&TxQueue->DeleteNeeded, 1)) {
-            TxQueue->DeleteWorkItem.BindingHandle = TxQueue->Binding;
-            TxQueue->DeleteWorkItem.WorkRoutine = XdpTxQueueDeleteWorker;
+        TxQueue->Notify.DeleteNeeded = TRUE;
+        NeedNotification = TRUE;
+        break;
+
+    case XDP_TX_QUEUE_NOTIFY_OFFLOAD_CURRENT_CONFIG:
+        TraceVerbose(TRACE_CORE, "TxQueue=%p Offload=%p", TxQueue, NotifyBuffer);
+
+        TxQueue->Notify.OffloadNeeded = TRUE;
+        NeedNotification = TRUE;
+        break;
+    }
+
+    if (NeedNotification) {
+        if (!TxQueue->Notify.WorkerQueued) {
+            TxQueue->Notify.WorkerQueued = TRUE;
+            TxQueue->Notify.WorkItem.BindingHandle = TxQueue->Binding;
+            TxQueue->Notify.WorkItem.WorkRoutine = XdpTxQueueNotifyWorker;
             XdpTxQueueInterlockedReference(TxQueue);
-            XdpIfQueueWorkItem(&TxQueue->DeleteWorkItem);
+            XdpIfQueueWorkItem(&TxQueue->Notify.WorkItem);
         }
-        break;
     }
+
+    KeReleaseSpinLock(&TxQueue->Notify.Lock, OldIrql);
 }
 
 static const XDP_TX_QUEUE_CONFIG_RESERVED XdpTxConfigReservedDispatch = {
@@ -819,7 +891,8 @@ XdpTxQueueCreate(
     TxQueue->Key = Key;
     TxQueue->State = XdpTxQueueStateCreated;
     XdpIfInitializeClientEntry(&TxQueue->BindingClientEntry);
-    InitializeListHead(&TxQueue->NotifyClients);
+    KeInitializeSpinLock(&TxQueue->Notify.Lock);
+    InitializeListHead(&TxQueue->Notify.Clients);
     XdpInitializeQueueInfo(&TxQueue->QueueInfo, XDP_QUEUE_TYPE_DEFAULT_RSS, QueueId);
     InitializeListHead(&TxQueue->ClientList);
     TxQueue->FillEntry = &TxQueue->ClientList;
@@ -829,7 +902,7 @@ XdpTxQueueCreate(
     TxQueue->Dispatch = XdpTxDispatch;
     TxQueue->ConfigCreate.Dispatch = &XdpTxConfigCreateDispatch;
     TxQueue->ConfigActivate.Dispatch = &XdpTxConfigActivateDispatch;
-    TxQueue->NotifyDetails.Dispatch = &XdpTxNotifyDispatch;
+    TxQueue->Notify.Details.Dispatch = &XdpTxNotifyDispatch;
 
     Status =
         XdpIfRegisterClient(
@@ -1140,7 +1213,7 @@ XdpTxQueueRegisterNotifications(
     )
 {
     NotifyEntry->NotifyRoutine = NotifyRoutine;
-    InsertTailList(&TxQueue->NotifyClients, &NotifyEntry->Link);
+    InsertTailList(&TxQueue->Notify.Clients, &NotifyEntry->Link);
 }
 
 VOID
@@ -1337,6 +1410,14 @@ XdpTxQueueGetInterfacePollHandle(
     return TxQueue->InterfacePollHandle;
 }
 
+XDP_IF_OFFLOAD_HANDLE
+XdpTxQueueGetInterfaceOffloadHandle(
+    _In_ XDP_TX_QUEUE *TxQueue
+    )
+{
+    return TxQueue->InterfaceOffloadHandle;
+}
+
 XDP_TX_QUEUE_CONFIG_ACTIVATE
 XdpTxQueueGetConfig(
     _In_ XDP_TX_QUEUE *TxQueue
@@ -1402,7 +1483,7 @@ XdpTxQueueDelete(
     TraceEnter(TRACE_CORE, "TxQueue=%p", TxQueue);
     TraceInfo(TRACE_CORE, "Deleting TxQueue=%p", TxQueue);
 
-    ASSERT(IsListEmpty(&TxQueue->NotifyClients));
+    ASSERT(IsListEmpty(&TxQueue->Notify.Clients));
     ASSERT(IsListEmpty(&TxQueue->ClientList));
 
     if (TxQueue->InterfaceOffloadHandle != NULL) {

src/xdp/tx.h

@@ -47,6 +47,7 @@ typedef struct _XDP_TX_QUEUE_NOTIFY_ENTRY XDP_TX_QUEUE_NOTIFICATION_ENTRY;
 
 typedef enum _XDP_TX_QUEUE_NOTIFICATION_TYPE {
     XDP_TX_QUEUE_NOTIFICATION_DETACH,
+    XDP_TX_QUEUE_NOTIFICATION_OFFLOAD_CURRENT_CONFIG,
 } XDP_TX_QUEUE_NOTIFICATION_TYPE;
 
 typedef
@@ -122,6 +123,11 @@ XdpTxQueueGetInterfacePollHandle(
     _In_ XDP_TX_QUEUE *TxQueue
     );
 
+XDP_IF_OFFLOAD_HANDLE
+XdpTxQueueGetInterfaceOffloadHandle(
+    _In_ XDP_TX_QUEUE *TxQueue
+    );
+
 XDP_TX_QUEUE_CONFIG_ACTIVATE
 XdpTxQueueGetConfig(
     _In_ XDP_TX_QUEUE *TxQueue