Add Queue::submit() for command buffer submission

Move command buffer submission logic from each backend's Device into
Queue::submit(), which takes ownership of the command buffers. Each
backend uses its Fence abstraction for GPU synchronization:

- Metal: commit() + waitUntilCompleted()
- Vulkan: vkQueueSubmit() signaling a timeline semaphore (VulkanFence),
  then VulkanFence::waitForCompletion()
- DX12: ExecuteCommandLists() + Queue::Signal() on the queue-owned
  DXFence, then DXFence::waitForCompletion()

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

Author: MarijnS95

include/API/Device.h

@@ -58,6 +58,19 @@ class Queue {
 public:
   virtual ~Queue() = 0;
 
+  /// Submit command buffers for execution and block until completion.
+  /// Command buffers execute in array order, but dependencies between them
+  /// require appropriate barriers within the command buffers themselves.
+  virtual llvm::Error
+  submit(llvm::SmallVector<std::unique_ptr<CommandBuffer>> CBs) = 0;
+
+  /// Convenience overload for submitting a single command buffer.
+  llvm::Error submit(std::unique_ptr<CommandBuffer> CB) {
+    llvm::SmallVector<std::unique_ptr<CommandBuffer>> CBs;
+    CBs.push_back(std::move(CB));
+    return submit(std::move(CBs));
+  }
+
 protected:
   Queue() = default;
 };

lib/API/DX/Device.cpp

@@ -394,22 +394,36 @@ class DXFence : public offloadtest::Fence {
 
 class DXQueue : public offloadtest::Queue {
 public:
+  using Queue::submit;
+
   ComPtr<ID3D12CommandQueue> Queue;
+  std::unique_ptr<DXFence> SubmitFence;
+  uint64_t FenceCounter = 0;
 
-  DXQueue(ComPtr<ID3D12CommandQueue> Queue) : Queue(Queue) {}
-  virtual ~DXQueue() {}
+  DXQueue(ComPtr<ID3D12CommandQueue> Queue,
+          std::unique_ptr<DXFence> SubmitFence)
+      : Queue(Queue), SubmitFence(std::move(SubmitFence)) {}
+  DXQueue(DXQueue &&) = default;
+  ~DXQueue() override {}
 
   static llvm::Expected<DXQueue>
   createGraphicsQueue(ComPtr<ID3D12Device> Device) {
     const D3D12_COMMAND_QUEUE_DESC Desc = {D3D12_COMMAND_LIST_TYPE_DIRECT, 0,
                                            D3D12_COMMAND_QUEUE_FLAG_NONE, 0};
-    ComPtr<ID3D12CommandQueue> Queue;
-    if (auto Err =
-            HR::toError(Device->CreateCommandQueue(&Desc, IID_PPV_ARGS(&Queue)),
-                        "Failed to create command queue."))
+    ComPtr<ID3D12CommandQueue> CmdQueue;
+    if (auto Err = HR::toError(
+            Device->CreateCommandQueue(&Desc, IID_PPV_ARGS(&CmdQueue)),
+            "Failed to create command queue."))
       return Err;
-    return DXQueue(Queue);
+    auto FenceOrErr = DXFence::create(Device.Get(), "QueueSubmitFence");
+    if (!FenceOrErr)
+      return FenceOrErr.takeError();
+    return DXQueue(CmdQueue, std::move(*FenceOrErr));
   }
+
+  llvm::Error
+  submit(llvm::SmallVector<std::unique_ptr<offloadtest::CommandBuffer>> CBs)
+      override;
 };
 
 class DXCommandBuffer : public offloadtest::CommandBuffer {
@@ -444,6 +458,42 @@ class DXCommandBuffer : public offloadtest::CommandBuffer {
   DXCommandBuffer() : CommandBuffer(GPUAPI::DirectX) {}
 };
 
+llvm::Error DXQueue::submit(
+    llvm::SmallVector<std::unique_ptr<offloadtest::CommandBuffer>> CBs) {
+  llvm::SmallVector<ID3D12CommandList *> CmdLists;
+  CmdLists.reserve(CBs.size());
+
+  // Wait on the previous submit's fence value before executing this batch,
+  // so that back-to-back submits don't overlap on the GPU. Skip on first
+  // submit since Wait(fence, 0) triggers a D3D12 validation warning.
+  if (FenceCounter > 0)
+    if (auto Err =
+            HR::toError(Queue->Wait(SubmitFence->Fence.Get(), FenceCounter),
+                        "Failed to wait on previous submit."))
+      return Err;
+
+  for (auto &CB : CBs) {
+    auto &DCB = *llvm::cast<DXCommandBuffer>(CB.get());
+    if (auto Err =
+            HR::toError(DCB.CmdList->Close(), "Failed to close command list."))
+      return Err;
+    CmdLists.push_back(DCB.CmdList.Get());
+  }
+
+  Queue->ExecuteCommandLists(CmdLists.size(), CmdLists.data());
+
+  const uint64_t CurrentCounter = ++FenceCounter;
+  if (auto Err =
+          HR::toError(Queue->Signal(SubmitFence->Fence.Get(), CurrentCounter),
+                      "Failed to add signal."))
+    return Err;
+
+  // TODO: Return a Fence+value with keepalive lists instead of blocking here.
+  if (auto Err = SubmitFence->waitForCompletion(CurrentCounter))
+    return Err;
+
+  return llvm::Error::success();
+}
 class DXDevice : public offloadtest::Device {
 private:
   ComPtr<IDXCoreAdapter> Adapter;
@@ -491,10 +541,10 @@ class DXDevice : public offloadtest::Device {
 public:
   DXDevice(ComPtr<IDXCoreAdapter> A, ComPtr<ID3D12Device> D, DXQueue Q,
            std::string Desc)
-      : Adapter(A), Device(D), GraphicsQueue(Q) {
+      : Adapter(A), Device(D), GraphicsQueue(std::move(Q)) {
     Description = Desc;
   }
-  DXDevice(const DXDevice &) = default;
+  DXDevice(const DXDevice &) = delete;
 
   ~DXDevice() override = default;
 
@@ -646,9 +696,8 @@ class DXDevice : public offloadtest::Device {
     auto GraphicsQueueOrErr = DXQueue::createGraphicsQueue(Device);
     if (!GraphicsQueueOrErr)
       return GraphicsQueueOrErr.takeError();
-    const DXQueue GraphicsQueue = *GraphicsQueueOrErr;
-
-    return std::make_unique<DXDevice>(Adapter, Device, std::move(GraphicsQueue),
+    return std::make_unique<DXDevice>(Adapter, Device,
+                                      std::move(*GraphicsQueueOrErr),
                                       std::string(DescVec.data()));
   }
 
@@ -897,7 +946,20 @@ class DXDevice : public offloadtest::Device {
         Buffer.Get(), 1, &StartCoord, &RegionSize, Heap.Get(), 1, &RangeFlag,
         &HeapRangeStartOffset, &RangeTileCount, D3D12_TILE_MAPPING_FLAG_NONE);
 
-    return waitForSignal(IS);
+    // Synchronize after UpdateTileMappings, which is a queue operation (not
+    // recorded into a command list). This is a hack but it works since this
+    // is all single threaded code.
+    // TODO: Replace with a proper fence abstraction.
+    static uint64_t TileMappingFenceCounter = 0;
+    const uint64_t CurrentCounter = ++TileMappingFenceCounter;
+    auto *F = static_cast<DXFence *>(IS.Fence.get());
+
+    if (auto Err =
+            HR::toError(CommandQueue->Signal(F->Fence.Get(), CurrentCounter),
+                        "Failed to add signal."))
+      return Err;
+
+    return IS.Fence->waitForCompletion(CurrentCounter);
   }
 
   llvm::Expected<ResourceBundle> createSRV(Resource &R, InvocationState &IS) {
@@ -1336,33 +1398,8 @@ class DXDevice : public offloadtest::Device {
     IS.CB->CmdList->ResourceBarrier(1, &Barrier);
   }
 
-  llvm::Error waitForSignal(InvocationState &IS) {
-    // This is a hack but it works since this is all single threaded code.
-    static uint64_t FenceCounter = 0;
-    const uint64_t CurrentCounter = FenceCounter + 1;
-    auto *F = static_cast<DXFence *>(IS.Fence.get());
-
-    if (auto Err = HR::toError(
-            GraphicsQueue.Queue->Signal(F->Fence.Get(), CurrentCounter),
-            "Failed to add signal."))
-      return Err;
-
-    if (auto Err = IS.Fence->waitForCompletion(CurrentCounter))
-      return Err;
-
-    FenceCounter = CurrentCounter;
-    return llvm::Error::success();
-  }
-
   llvm::Error executeCommandList(InvocationState &IS) {
-    if (auto Err = HR::toError(IS.CB->CmdList->Close(),
-                               "Failed to close command list."))
-      return Err;
-
-    ID3D12CommandList *const CmdLists[] = {IS.CB->CmdList.Get()};
-    GraphicsQueue.Queue->ExecuteCommandLists(1, CmdLists);
-
-    return waitForSignal(IS);
+    return GraphicsQueue.submit(std::move(IS.CB));
   }
 
   llvm::Error createComputeCommands(Pipeline &P, InvocationState &IS) {

lib/API/MTL/MTLDevice.cpp

@@ -77,12 +77,18 @@ static MTL::VertexFormat getMTLVertexFormat(DataFormat Format, int Channels) {
 namespace {
 class MTLQueue : public offloadtest::Queue {
 public:
+  using Queue::submit;
+
   MTL::CommandQueue *Queue;
   MTLQueue(MTL::CommandQueue *Queue) : Queue(Queue) {}
-  ~MTLQueue() {
+  ~MTLQueue() override {
     if (Queue)
       Queue->release();
   }
+
+  llvm::Error
+  submit(llvm::SmallVector<std::unique_ptr<offloadtest::CommandBuffer>> CBs)
+      override;
 };
 
 class MTLFence : public offloadtest::Fence {
@@ -172,6 +178,24 @@ class MTLCommandBuffer : public offloadtest::CommandBuffer {
   MTLCommandBuffer() : CommandBuffer(GPUAPI::Metal) {}
 };
 
+llvm::Error MTLQueue::submit(
+    llvm::SmallVector<std::unique_ptr<offloadtest::CommandBuffer>> CBs) {
+  // Metal serial queues guarantee that command buffers execute in commit order,
+  // so no explicit wait on prior work is needed here.
+  for (auto &CB : CBs)
+    llvm::cast<MTLCommandBuffer>(CB.get())->CmdBuffer->commit();
+
+  // TODO: Return a Fence+value with keepalive lists instead of blocking here.
+  for (auto &CB : CBs) {
+    auto &MCB = *llvm::cast<MTLCommandBuffer>(CB.get());
+    MCB.CmdBuffer->waitUntilCompleted();
+
+    NS::Error *Err = MCB.CmdBuffer->error();
+    if (Err)
+      return toError(Err);
+  }
+  return llvm::Error::success();
+}
 class MTLDevice : public offloadtest::Device {
   Capabilities Caps;
   MTL::Device *Device;
@@ -643,24 +667,7 @@ class MTLDevice : public offloadtest::Device {
   }
 
   llvm::Error executeCommands(InvocationState &IS) {
-    // This is a hack but it works since this is all single threaded code.
-    static uint64_t FenceCounter = 0;
-    const uint64_t CurrentCounter = FenceCounter + 1;
-    auto *F = static_cast<MTLFence *>(IS.Fence.get());
-
-    IS.CB->CmdBuffer->encodeSignalEvent(F->Event, CurrentCounter);
-    IS.CB->CmdBuffer->commit();
-
-    if (auto Err = IS.Fence->waitForCompletion(CurrentCounter))
-      return Err;
-
-    // Check and surface any errors that occurred during execution.
-    NS::Error *CBErr = IS.CB->CmdBuffer->error();
-    if (CBErr)
-      return toError(CBErr);
-
-    FenceCounter = CurrentCounter;
-    return llvm::Error::success();
+    return GraphicsQueue.submit(std::move(IS.CB));
   }
 
   llvm::Error copyBack(Pipeline &P, InvocationState &IS) {