Allow blocked command stream programming for commands without Kernel

Change-Id: I691a029bd5511c8f710ef1bff8cc5a9feca644f3
Signed-off-by: Dunajski, Bartosz <[email protected]>
Related-To: NEO-3433

Author: BartoszDunajski

Jenkinsfile

@@ -1,5 +1,5 @@
 #!groovy
 dependenciesRevision='06357fd1499ba888288e517541564865ad9c136a-1292'
 strategy='EQUAL'
-allowedCD=259
+allowedCD=260
 allowedF=5

runtime/command_queue/command_queue.cpp

@@ -588,4 +588,27 @@ bool CommandQueue::blitEnqueueAllowed(bool queueBlocked, cl_command_type cmdType
 
     return commandAllowed && !queueBlocked && blitAllowed;
 }
+
+bool CommandQueue::isBlockedCommandStreamRequired(uint32_t commandType, const EventsRequest &eventsRequest, bool blockedQueue) const {
+    if (!blockedQueue) {
+        return false;
+    }
+
+    if (isCacheFlushCommand(commandType) || !isCommandWithoutKernel(commandType)) {
+        return true;
+    }
+
+    if ((CL_COMMAND_BARRIER == commandType || CL_COMMAND_MARKER == commandType) &&
+        getGpgpuCommandStreamReceiver().peekTimestampPacketWriteEnabled()) {
+
+        for (size_t i = 0; i < eventsRequest.numEventsInWaitList; i++) {
+            auto waitlistEvent = castToObjectOrAbort<Event>(eventsRequest.eventWaitList[i]);
+            if (waitlistEvent->getTimestampPacketNodes()) {
+                return true;
+            }
+        }
+    }
+
+    return false;
+}
 } // namespace NEO

runtime/command_queue/command_queue.h

@@ -314,7 +314,7 @@ class CommandQueue : public BaseObject<_cl_command_queue> {
 
     MOCKABLE_VIRTUAL void updateFromCompletionStamp(const CompletionStamp &completionStamp);
 
-    virtual bool isCacheFlushCommand(uint32_t commandType) { return false; }
+    virtual bool isCacheFlushCommand(uint32_t commandType) const { return false; }
 
     cl_int getCommandQueueInfo(cl_command_queue_info paramName,
                                size_t paramValueSize, void *paramValue,
@@ -429,6 +429,7 @@ class CommandQueue : public BaseObject<_cl_command_queue> {
     cl_int enqueueUnmapMemObject(TransferProperties &transferProperties, EventsRequest &eventsRequest);
 
     virtual void obtainTaskLevelAndBlockedStatus(unsigned int &taskLevel, cl_uint &numEventsInWaitList, const cl_event *&eventWaitList, bool &blockQueueStatus, unsigned int commandType){};
+    bool isBlockedCommandStreamRequired(uint32_t commandType, const EventsRequest &eventsRequest, bool blockedQueue) const;
 
     MOCKABLE_VIRTUAL void obtainNewTimestampPacketNodes(size_t numberOfNodes, TimestampPacketContainer &previousNodes, bool clearAllDependencies);
     void processProperties(const cl_queue_properties *properties);

runtime/command_queue/command_queue_hw.h

@@ -367,7 +367,7 @@ class CommandQueueHw : public CommandQueue {
                           LinearStream *commandStream,
                           uint64_t postSyncAddress);
 
-    bool isCacheFlushCommand(uint32_t commandType) override;
+    bool isCacheFlushCommand(uint32_t commandType) const override;
 
   protected:
     MOCKABLE_VIRTUAL void enqueueHandlerHook(const unsigned int commandType, const MultiDispatchInfo &dispatchInfo){};
@@ -389,10 +389,11 @@ class CommandQueueHw : public CommandQueue {
     LinearStream *obtainCommandStream(const CsrDependencies &csrDependencies, bool profilingRequired,
                                       bool perfCountersRequired, bool blitEnqueue, bool blockedQueue,
                                       const MultiDispatchInfo &multiDispatchInfo,
+                                      const EventsRequest &eventsRequest,
                                       std::unique_ptr<KernelOperation> &blockedCommandsData,
                                       Surface **surfaces, size_t numSurfaces) {
         LinearStream *commandStream = nullptr;
-        if (blockedQueue && !multiDispatchInfo.empty()) {
+        if (isBlockedCommandStreamRequired(commandType, eventsRequest, blockedQueue)) {
             constexpr size_t additionalAllocationSize = CSRequirements::csOverfetchSize;
             constexpr size_t allocationSize = MemoryConstants::pageSize64k - CSRequirements::csOverfetchSize;
             commandStream = new LinearStream();

runtime/command_queue/command_queue_hw_bdw_plus.inl

@@ -17,7 +17,7 @@ void CommandQueueHw<GfxFamily>::submitCacheFlush(Surface **surfaces,
 }
 
 template <typename GfxFamily>
-bool CommandQueueHw<GfxFamily>::isCacheFlushCommand(uint32_t commandType) {
+bool CommandQueueHw<GfxFamily>::isCacheFlushCommand(uint32_t commandType) const {
     return false;
 }
 

runtime/command_queue/enqueue_common.h

@@ -228,7 +228,8 @@ void CommandQueueHw<GfxFamily>::enqueueHandler(Surface **surfacesForResidency,
     }
 
     auto &commandStream = *obtainCommandStream<commandType>(csrDeps, profilingRequired, perfCountersRequired, blitEnqueue, blockQueue,
-                                                            multiDispatchInfo, blockedCommandsData, surfacesForResidency, numSurfaceForResidency);
+                                                            multiDispatchInfo, eventsRequest, blockedCommandsData, surfacesForResidency,
+                                                            numSurfaceForResidency);
     auto commandStreamStart = commandStream.getUsed();
 
     if (eventBuilder.getEvent() && getGpgpuCommandStreamReceiver().peekTimestampPacketWriteEnabled()) {
@@ -763,15 +764,16 @@ void CommandQueueHw<GfxFamily>::enqueueBlocked(
         eventBuilder = &internalEventBuilder;
         DBG_LOG(EventsDebugEnable, "enqueueBlocked", "new virtualEvent", eventBuilder->getEvent());
     }
+    auto outEvent = eventBuilder->getEvent();
 
     //update queue taskCount
-    taskCount = eventBuilder->getEvent()->getCompletionStamp();
+    taskCount = outEvent->getCompletionStamp();
 
-    if (multiDispatchInfo.empty()) {
-        DEBUG_BREAK_IF(!isCommandWithoutKernel(commandType));
-        auto cmd = std::make_unique<CommandMarker>(*this);
+    std::unique_ptr<Command> command;
+    bool storeTimestampPackets = blockedCommandsData && timestampPacketContainer;
 
-        eventBuilder->getEvent()->setCommand(std::move(cmd));
+    if (multiDispatchInfo.empty()) {
+        command = std::make_unique<CommandMarker>(*this, blockedCommandsData);
     } else {
         //store task data in event
         std::vector<Surface *> allSurfaces;
@@ -788,28 +790,26 @@ void CommandQueueHw<GfxFamily>::enqueueBlocked(
             allSurfaces.push_back(surface->duplicate());
         }
         PreemptionMode preemptionMode = PreemptionHelper::taskPreemptionMode(*device, multiDispatchInfo);
-        auto cmd = std::make_unique<CommandComputeKernel>(
-            *this,
-            std::move(blockedCommandsData),
-            allSurfaces,
-            shouldFlushDC(commandType, printfHandler.get()),
-            slmUsed,
-            commandType == CL_COMMAND_NDRANGE_KERNEL,
-            std::move(printfHandler),
-            preemptionMode,
-            multiDispatchInfo.peekMainKernel(),
-            (uint32_t)multiDispatchInfo.size());
-
-        if (timestampPacketContainer.get()) {
-            for (cl_uint i = 0; i < eventsRequest.numEventsInWaitList; i++) {
-                auto event = castToObjectOrAbort<Event>(eventsRequest.eventWaitList[i]);
-                event->incRefInternal();
-            }
-            cmd->setTimestampPacketNode(*timestampPacketContainer, *previousTimestampPacketNodes);
+        command = std::make_unique<CommandComputeKernel>(*this,
+                                                         blockedCommandsData,
+                                                         allSurfaces,
+                                                         shouldFlushDC(commandType, printfHandler.get()),
+                                                         slmUsed,
+                                                         commandType == CL_COMMAND_NDRANGE_KERNEL,
+                                                         std::move(printfHandler),
+                                                         preemptionMode,
+                                                         multiDispatchInfo.peekMainKernel(),
+                                                         (uint32_t)multiDispatchInfo.size());
+    }
+    if (storeTimestampPackets) {
+        for (cl_uint i = 0; i < eventsRequest.numEventsInWaitList; i++) {
+            auto event = castToObjectOrAbort<Event>(eventsRequest.eventWaitList[i]);
+            event->incRefInternal();
         }
-        cmd->setEventsRequest(eventsRequest);
-        eventBuilder->getEvent()->setCommand(std::move(cmd));
+        command->setTimestampPacketNode(*timestampPacketContainer, *previousTimestampPacketNodes);
+        command->setEventsRequest(eventsRequest);
     }
+    outEvent->setCommand(std::move(command));
 
     eventBuilder->addParentEvents(ArrayRef<const cl_event>(eventsRequest.eventWaitList, eventsRequest.numEventsInWaitList));
     eventBuilder->addParentEvent(this->virtualEvent);
@@ -819,7 +819,7 @@ void CommandQueueHw<GfxFamily>::enqueueBlocked(
         this->virtualEvent->decRefInternal();
     }
 
-    this->virtualEvent = eventBuilder->getEvent();
+    this->virtualEvent = outEvent;
 }
 
 template <typename GfxFamily>

runtime/helpers/task_information.cpp

@@ -82,7 +82,7 @@ CompletionStamp &CommandMapUnmap::submit(uint32_t taskLevel, bool terminated) {
     return completionStamp;
 }
 
-CommandComputeKernel::CommandComputeKernel(CommandQueue &commandQueue, std::unique_ptr<KernelOperation> kernelOperation, std::vector<Surface *> &surfaces,
+CommandComputeKernel::CommandComputeKernel(CommandQueue &commandQueue, std::unique_ptr<KernelOperation> &kernelOperation, std::vector<Surface *> &surfaces,
                                            bool flushDC, bool usesSLM, bool ndRangeKernel, std::unique_ptr<PrintfHandler> printfHandler,
                                            PreemptionMode preemptionMode, Kernel *kernel, uint32_t kernelCount)
     : Command(commandQueue, kernelOperation), flushDC(flushDC), slmUsed(usesSLM),
@@ -135,12 +135,7 @@ CompletionStamp &CommandComputeKernel::submit(uint32_t taskLevel, bool terminate
     if (printfHandler) {
         printfHandler.get()->makeResident(commandStreamReceiver);
     }
-    if (currentTimestampPacketNodes) {
-        currentTimestampPacketNodes->makeResident(commandStreamReceiver);
-    }
-    if (previousTimestampPacketNodes) {
-        previousTimestampPacketNodes->makeResident(commandStreamReceiver);
-    }
+    makeTimestampPacketsResident();
 
     if (executionModelKernel) {
         uint32_t taskCount = commandStreamReceiver.peekTaskCount() + 1;
@@ -224,9 +219,42 @@ CompletionStamp &CommandMarker::submit(uint32_t taskLevel, bool terminated) {
     }
 
     auto &commandStreamReceiver = commandQueue.getGpgpuCommandStreamReceiver();
-    completionStamp.taskCount = commandStreamReceiver.peekTaskCount();
-    completionStamp.taskLevel = commandStreamReceiver.peekTaskLevel();
-    completionStamp.flushStamp = commandStreamReceiver.obtainCurrentFlushStamp();
+
+    if (!kernelOperation) {
+        completionStamp.taskCount = commandStreamReceiver.peekTaskCount();
+        completionStamp.taskLevel = commandStreamReceiver.peekTaskLevel();
+        completionStamp.flushStamp = commandStreamReceiver.obtainCurrentFlushStamp();
+
+        return completionStamp;
+    }
+
+    auto lockCSR = commandStreamReceiver.obtainUniqueOwnership();
+
+    DispatchFlags dispatchFlags;
+    dispatchFlags.blocking = true;
+    dispatchFlags.lowPriority = commandQueue.getPriority() == QueuePriority::LOW;
+    dispatchFlags.throttle = commandQueue.getThrottle();
+    dispatchFlags.preemptionMode = commandQueue.getDevice().getPreemptionMode();
+    dispatchFlags.multiEngineQueue = commandQueue.isMultiEngineQueue();
+    dispatchFlags.guardCommandBufferWithPipeControl = true;
+    dispatchFlags.outOfOrderExecutionAllowed = commandStreamReceiver.isNTo1SubmissionModelEnabled();
+
+    UNRECOVERABLE_IF(!commandStreamReceiver.peekTimestampPacketWriteEnabled());
+
+    dispatchFlags.csrDependencies.fillFromEventsRequestAndMakeResident(eventsRequest, commandStreamReceiver, CsrDependencies::DependenciesType::OutOfCsr);
+
+    makeTimestampPacketsResident();
+
+    gtpinNotifyPreFlushTask(&commandQueue);
+
+    completionStamp = commandStreamReceiver.flushTask(*kernelOperation->commandStream,
+                                                      0,
+                                                      commandQueue.getIndirectHeap(IndirectHeap::DYNAMIC_STATE, 0u),
+                                                      commandQueue.getIndirectHeap(IndirectHeap::INDIRECT_OBJECT, 0u),
+                                                      commandQueue.getIndirectHeap(IndirectHeap::SURFACE_STATE, 0u),
+                                                      taskLevel,
+                                                      dispatchFlags,
+                                                      commandQueue.getDevice());
 
     return completionStamp;
 }
@@ -259,6 +287,17 @@ Command::~Command() {
     }
 }
 
+void Command::makeTimestampPacketsResident() {
+    auto &commandStreamReceiver = commandQueue.getGpgpuCommandStreamReceiver();
+
+    if (currentTimestampPacketNodes) {
+        currentTimestampPacketNodes->makeResident(commandStreamReceiver);
+    }
+    if (previousTimestampPacketNodes) {
+        previousTimestampPacketNodes->makeResident(commandStreamReceiver);
+    }
+}
+
 Command::Command(CommandQueue &commandQueue) : commandQueue(commandQueue) {}
 
 Command::Command(CommandQueue &commandQueue, std::unique_ptr<KernelOperation> &kernelOperation)

runtime/helpers/task_information.h

@@ -10,6 +10,7 @@
 #include "runtime/helpers/completion_stamp.h"
 #include "runtime/helpers/hw_info.h"
 #include "runtime/helpers/properties_helper.h"
+#include "runtime/helpers/timestamp_packet.h"
 #include "runtime/indirect_heap/indirect_heap.h"
 #include "runtime/utilities/iflist.h"
 
@@ -92,6 +93,7 @@ class Command : public IFNode<Command> {
     }
     void setTimestampPacketNode(TimestampPacketContainer &current, TimestampPacketContainer &previous);
     void setEventsRequest(EventsRequest &eventsRequest);
+    void makeTimestampPacketsResident();
 
     TagNode<HwTimeStamps> *timestamp = nullptr;
     CompletionStamp completionStamp = {};
@@ -122,7 +124,7 @@ class CommandMapUnmap : public Command {
 
 class CommandComputeKernel : public Command {
   public:
-    CommandComputeKernel(CommandQueue &commandQueue, std::unique_ptr<KernelOperation> kernelResources, std::vector<Surface *> &surfaces,
+    CommandComputeKernel(CommandQueue &commandQueue, std::unique_ptr<KernelOperation> &kernelOperation, std::vector<Surface *> &surfaces,
                          bool flushDC, bool usesSLM, bool ndRangeKernel, std::unique_ptr<PrintfHandler> printfHandler,
                          PreemptionMode preemptionMode, Kernel *kernel, uint32_t kernelCount);
 
@@ -146,7 +148,6 @@ class CommandComputeKernel : public Command {
 class CommandMarker : public Command {
   public:
     using Command::Command;
-
     CompletionStamp &submit(uint32_t taskLevel, bool terminated) override;
 };
 } // namespace NEO

unit_tests/command_queue/dispatch_walker_tests.cpp

@@ -745,8 +745,10 @@ HWTEST_F(DispatchWalkerTest, givenBlockedEnqueueWhenObtainingCommandStreamThenAl
     auto expectedSizeCS = MemoryConstants::pageSize64k - CSRequirements::csOverfetchSize;
 
     CsrDependencies csrDependencies;
+    EventsRequest eventsRequest(0, nullptr, nullptr);
     auto cmdStream = mockCmdQ.template obtainCommandStream<CL_COMMAND_NDRANGE_KERNEL>(csrDependencies, false, false, false, true,
-                                                                                      multiDispatchInfo, blockedKernelData, nullptr, 0u);
+                                                                                      multiDispatchInfo, eventsRequest, blockedKernelData,
+                                                                                      nullptr, 0u);
 
     EXPECT_EQ(expectedSizeCS, cmdStream->getMaxAvailableSpace());
     EXPECT_EQ(expectedSizeCSAllocation, cmdStream->getGraphicsAllocation()->getUnderlyingBufferSize());

unit_tests/command_queue/enqueue_command_without_kernel_tests.cpp

@@ -24,7 +24,7 @@ class MockCommandQueueWithCacheFlush : public MockCommandQueueHw<GfxFamily> {
     using MockCommandQueueHw<GfxFamily>::MockCommandQueueHw;
 
   public:
-    bool isCacheFlushCommand(uint32_t commandType) override {
+    bool isCacheFlushCommand(uint32_t commandType) const override {
         return commandRequireCacheFlush;
     }
     bool commandRequireCacheFlush = false;