+ init cuda before each simulation

+ catch if initializing a cuda simulation failed + unregister image resource + save gpu model info in settings
chrxh · Aug 27, 2023 · 5c4e961 · 5c4e961
1 parent 556175b
commit 5c4e961
Show file tree

Hide file tree

Showing 16 changed files with 155 additions and 143 deletions.
diff --git a/source/Base/Exceptions.h b/source/Base/Exceptions.h
@@ -4,10 +4,18 @@
 #include <stdexcept>
 #include <string>
 
-class SpecificCudaException : public std::runtime_error
+class CudaException : public std::runtime_error
 {
 public:
-    SpecificCudaException(std::string const& what)
+    CudaException(std::string const& what)
+        : std::runtime_error(what.c_str())
+    {}
+};
+
+class CudaMemoryAllocationException : public std::runtime_error
+{
+public:
+    CudaMemoryAllocationException(std::string const& what)
         : std::runtime_error(what.c_str())
     {}
 };

diff --git a/source/Base/Resources.h b/source/Base/Resources.h
@@ -2,7 +2,7 @@
 
 namespace Const
 {
-    std::string const ProgramVersion = "4.0.0.beta.41";
+    std::string const ProgramVersion = "4.0.0.beta.42";
 
     std::string const BasePath = "resources/";
 

diff --git a/source/EngineGpuKernels/CudaSimulationFacade.cu b/source/EngineGpuKernels/CudaSimulationFacade.cu
@@ -40,92 +40,9 @@
 #include "RenderingData.cuh"
 #include "TestKernelsLauncher.cuh"
 
-namespace
-{
-    class CudaInitializer
-    {
-    public:
-        static void init() { getInstance(); }
-        static std::string getGpuName() { return getInstance()._gpuName; }
-
-        CudaInitializer()
-        {
-            int deviceNumber = getDeviceNumberOfHighestComputeCapability();
-
-            auto result = cudaSetDevice(deviceNumber);
-            if (result != cudaSuccess) {
-                throw SystemRequirementNotMetException("CUDA device could not be initialized.");
-            }
-
-            std::stringstream stream;
-            stream << "device " << deviceNumber << " is set";
-            log(Priority::Important, stream.str());
-        }
-
-        ~CudaInitializer() { cudaDeviceReset(); }
-
-    private:
-        static CudaInitializer& getInstance()
-        {
-            static CudaInitializer instance;
-            return instance;
-        }
-
-        int getDeviceNumberOfHighestComputeCapability()
-        {
-            int result = 0;
-            int numberOfDevices;
-            CHECK_FOR_CUDA_ERROR(cudaGetDeviceCount(&numberOfDevices));
-            if (numberOfDevices < 1) {
-                throw SystemRequirementNotMetException("No CUDA device found.");
-            }
-            {
-                std::stringstream stream;
-                if (1 == numberOfDevices) {
-                    stream << "1 CUDA device found";
-                } else {
-                    stream << numberOfDevices << " CUDA devices found";
-                }
-                log(Priority::Important, stream.str());
-            }
-
-            int highestComputeCapability = 0;
-            for (int deviceNumber = 0; deviceNumber < numberOfDevices; ++deviceNumber) {
-                cudaDeviceProp prop;
-                CHECK_FOR_CUDA_ERROR(cudaGetDeviceProperties(&prop, deviceNumber));
-
-                std::stringstream stream;
-                stream << "device " << deviceNumber << ": " << prop.name << " with compute capability " << prop.major
-                       << "." << prop.minor;
-                log(Priority::Important, stream.str());
-
-                int computeCapability = prop.major * 100 + prop.minor;
-                if (computeCapability > highestComputeCapability) {
-                    result = deviceNumber;
-                    highestComputeCapability = computeCapability;
-                    _gpuName = prop.name;
-                }
-            }
-            if (highestComputeCapability < 600) {
-                throw SystemRequirementNotMetException(
-                    "No CUDA device with compute capability of 6.0 or higher found.");
-            }
-
-            return result;
-        }
-
-        std::string _gpuName;
-    };
-}
-
-void _CudaSimulationFacade::initCuda()
-{
-    CudaInitializer::init();
-}
-
 _CudaSimulationFacade::_CudaSimulationFacade(uint64_t timestep, Settings const& settings)
 {
-    CHECK_FOR_CUDA_ERROR(cudaGetLastError());
+    initCuda();
 
     _settings.generalSettings = settings.generalSettings;
     setSimulationParameters(settings.simulationParameters);
@@ -161,6 +78,10 @@ _CudaSimulationFacade::_CudaSimulationFacade(uint64_t timestep, Settings const&
 
 _CudaSimulationFacade::~_CudaSimulationFacade()
 {
+    if (_cudaResource) {
+        CHECK_FOR_CUDA_ERROR(cudaGraphicsUnregisterResource(_cudaResource));
+    }
+
     _cudaSimulationData->free();
     _cudaRenderingData->free();
     _simulationStatistics->free();
@@ -173,22 +94,27 @@ _CudaSimulationFacade::~_CudaSimulationFacade()
     CudaMemoryManager::getInstance().freeMemory(_cudaAccessTO->numParticles);
     CudaMemoryManager::getInstance().freeMemory(_cudaAccessTO->numAuxiliaryData);
 
+    cudaDeviceReset();
     log(Priority::Important, "close simulation");
 }
 
 void* _CudaSimulationFacade::registerImageResource(GLuint image)
 {
-    cudaGraphicsResource* cudaResource;
+    //unregister old resource
+    if (_cudaResource) {
+        CHECK_FOR_CUDA_ERROR(cudaGraphicsUnregisterResource(_cudaResource));
+    }
 
+    //register new resource
     CHECK_FOR_CUDA_ERROR(
-        cudaGraphicsGLRegisterImage(&cudaResource, image, GL_TEXTURE_2D, cudaGraphicsMapFlagsReadOnly));
+        cudaGraphicsGLRegisterImage(&_cudaResource, image, GL_TEXTURE_2D, cudaGraphicsMapFlagsReadOnly));
 
-    return reinterpret_cast<void*>(cudaResource);
+    return reinterpret_cast<void*>(_cudaResource);
 }
 
 std::string _CudaSimulationFacade::getGpuName()
 {
-    return CudaInitializer::getGpuName();
+    return _gpuInfo.gpuModelName;
 }
 
 void _CudaSimulationFacade::calcTimestep()
@@ -505,6 +431,64 @@ void _CudaSimulationFacade::testOnly_mutate(uint64_t cellId, MutationType mutati
     resizeArraysIfNecessary();
 }
 
+void _CudaSimulationFacade::initCuda()
+{
+    _gpuInfo = checkAndReturnGpuInfo();
+
+    auto result = cudaSetDevice(_gpuInfo.deviceNumber);
+    if (result != cudaSuccess) {
+        throw SystemRequirementNotMetException("CUDA device could not be initialized.");
+    }
+
+    cudaGetLastError(); //reset error code
+
+    std::stringstream stream;
+    stream << "device " << _gpuInfo.deviceNumber << " is set";
+    log(Priority::Important, stream.str());
+}
+
+auto _CudaSimulationFacade::checkAndReturnGpuInfo() -> GpuInfo
+{
+    GpuInfo result;
+
+    int numberOfDevices;
+    CHECK_FOR_CUDA_ERROR(cudaGetDeviceCount(&numberOfDevices));
+    if (numberOfDevices < 1) {
+        throw SystemRequirementNotMetException("No CUDA device found.");
+    }
+    {
+        std::stringstream stream;
+        if (1 == numberOfDevices) {
+            stream << "1 CUDA device found";
+        } else {
+            stream << numberOfDevices << " CUDA devices found";
+        }
+        log(Priority::Important, stream.str());
+    }
+
+    int highestComputeCapability = 0;
+    for (int deviceNumber = 0; deviceNumber < numberOfDevices; ++deviceNumber) {
+        cudaDeviceProp prop;
+        CHECK_FOR_CUDA_ERROR(cudaGetDeviceProperties(&prop, deviceNumber));
+
+        std::stringstream stream;
+        stream << "device " << deviceNumber << ": " << prop.name << " with compute capability " << prop.major << "." << prop.minor;
+        log(Priority::Important, stream.str());
+
+        int computeCapability = prop.major * 100 + prop.minor;
+        if (computeCapability > highestComputeCapability) {
+            result.deviceNumber = deviceNumber;
+            highestComputeCapability = computeCapability;
+            result.gpuModelName = prop.name;
+        }
+    }
+    if (highestComputeCapability < 600) {
+        throw SystemRequirementNotMetException("No CUDA device with compute capability of 6.0 or higher found.");
+    }
+
+    return result;
+}
+
 void _CudaSimulationFacade::syncAndCheck()
 {
     cudaDeviceSynchronize();

diff --git a/source/EngineGpuKernels/CudaSimulationFacade.cuh b/source/EngineGpuKernels/CudaSimulationFacade.cuh
@@ -20,10 +20,12 @@
 
 #include "Definitions.cuh"
 
+struct cudaGraphicsResource;
+
 class _CudaSimulationFacade
 {
 public:
-    static void initCuda();
+    //static void initCuda();
 
     _CudaSimulationFacade(uint64_t timestep, Settings const& settings);
     ~_CudaSimulationFacade();
@@ -80,6 +82,14 @@ public:
     void testOnly_mutate(uint64_t cellId, MutationType mutationType);
 
 private:
+    void initCuda();
+    struct GpuInfo
+    {
+        int deviceNumber = 0;
+        std::string gpuModelName;
+    };
+    GpuInfo checkAndReturnGpuInfo();
+
     void syncAndCheck();
     void copyDataTOtoDevice(DataTO const& dataTO);
     void copyDataTOtoHost(DataTO const& dataTO);
@@ -89,6 +99,9 @@ private:
 
     SimulationData getSimulationDataIntern() const;
 
+    GpuInfo _gpuInfo;
+    cudaGraphicsResource* _cudaResource = nullptr;
+
     mutable std::mutex _mutexForSimulationParameters;
     std::optional<SimulationParameters> _newSimulationParameters;
     Settings _settings;

diff --git a/source/EngineGpuKernels/Macros.cuh b/source/EngineGpuKernels/Macros.cuh
@@ -14,21 +14,23 @@ void checkAndThrowError(T result, char const *const func, const char *const file
     if (result) {
         DEVICE_RESET
         if (cudaError::cudaErrorInsufficientDriver == result) {
-            throw SpecificCudaException(
+            throw CudaException(
                 "Your graphics driver is not compatible with the required CUDA version. Please update your Nvidia graphics driver and restart.");
         } else if (cudaError::cudaErrorOperatingSystem == result) {
-            throw SpecificCudaException("An operating system call within the CUDA api failed. Please check if your "
+            throw CudaException("An operating system call within the CUDA api failed. Please check if your "
                                         "monitor is plugged to the correct graphics card.");
         } else if (cudaError::cudaErrorInitializationError == result) {
-            throw SpecificCudaException(
+            throw CudaException(
                 "CUDA could not be initialized. Please check the minimum hardware requirements. If fulfilled please update your Nvidia graphics driver and restart.");
         } else if (cudaError::cudaErrorUnsupportedPtxVersion == result) {
-            throw SpecificCudaException("A CUDA error occurred (cudaErrorUnsupportedPtxVersion). Please update your Nvidia graphics driver and restart.");
+            throw CudaException("A CUDA error occurred (cudaErrorUnsupportedPtxVersion). Please update your Nvidia graphics driver and restart.");
+        } else if (cudaError::cudaErrorMemoryAllocation == result) {
+            throw CudaMemoryAllocationException("A CUDA error occurred while allocating memory.");
         } else {
             std::stringstream stream;
             stream << "CUDA error at " << file << ":" << line << " code=" << static_cast<unsigned int>(result) << "("
                    << _cudaGetErrorEnum(result) << ") \"" << func << "\"";
-            throw SpecificCudaException(stream.str().c_str());
+            throw CudaException(stream.str().c_str());
         }
     }
 }

diff --git a/source/EngineImpl/EngineWorker.cpp b/source/EngineImpl/EngineWorker.cpp
@@ -13,11 +13,6 @@ namespace
     std::chrono::milliseconds const StatisticsUpdate(30);
 }
 
-void EngineWorker::initCuda()
-{
-    _CudaSimulationFacade::initCuda();
-}
-
 void EngineWorker::newSimulation(uint64_t timestep, GeneralSettings const& generalSettings, SimulationParameters const& parameters)
 {
     _accessState = 0;
@@ -26,9 +21,8 @@ void EngineWorker::newSimulation(uint64_t timestep, GeneralSettings const& gener
     _dataTOCache = std::make_shared<_AccessDataTOCache>();
     _cudaSimulation = std::make_shared<_CudaSimulationFacade>(timestep, _settings);
 
-    if (_imageResourceToRegister) {
-        _cudaResource = _cudaSimulation->registerImageResource(*_imageResourceToRegister);
-        _imageResourceToRegister = std::nullopt;
+    if (_imageResource) {
+        _cudaResource = _cudaSimulation->registerImageResource(*_imageResource);
     }
     updateStatistics();
 }
@@ -39,15 +33,12 @@ void EngineWorker::clear()
     return _cudaSimulation->clear();
 }
 
-void EngineWorker::registerImageResource(void* image)
+void EngineWorker::setImageResource(void* image)
 {
     GLuint imageId = reinterpret_cast<uintptr_t>(image);
-    if (!_cudaSimulation) {
-
-        //cuda is not initialized yet => register image resource later
-        _imageResourceToRegister = imageId;
-    } else {
+    _imageResource = imageId;
 
+    if (_cudaSimulation) {
         EngineWorkerGuard access(this);
         _cudaResource = _cudaSimulation->registerImageResource(imageId);
     }
@@ -66,7 +57,6 @@ void EngineWorker::tryDrawVectorGraphics(
 {
     EngineWorkerGuard access(this, FrameTimeout);
 
-
     if (!access.isTimeout()) {
         _cudaSimulation->drawVectorGraphics(
             {rectUpperLeft.x, rectUpperLeft.y},

diff --git a/source/EngineImpl/EngineWorker.h b/source/EngineImpl/EngineWorker.h
@@ -36,12 +36,10 @@ class EngineWorker
 {
     friend class EngineWorkerGuard;
 public:
-    void initCuda();
-
     void newSimulation(uint64_t timestep, GeneralSettings const& generalSettings, SimulationParameters const& parameters);
     void clear();
 
-    void registerImageResource(void* image);
+    void setImageResource(void* image);
     std::string getGpuName() const;
 
     void tryDrawVectorGraphics(RealVector2D const& rectUpperLeft, RealVector2D const& rectLowerRight, IntVector2D const& imageSize, double zoom);
@@ -137,7 +135,7 @@ class EngineWorker
     //async jobs
     mutable std::mutex _mutexForAsyncJobs;
     std::optional<GpuSettings> _updateGpuSettingsJob;
-    std::optional<GLuint> _imageResourceToRegister;
+    std::optional<GLuint> _imageResource;
 
     struct ApplyForceJob
     {

diff --git a/source/EngineImpl/SimulationControllerImpl.cpp b/source/EngineImpl/SimulationControllerImpl.cpp
@@ -2,11 +2,6 @@
 
 #include "EngineInterface/Descriptions.h"
 
-void _SimulationControllerImpl::initCuda()
-{
-    _worker.initCuda();
-}
-
 void _SimulationControllerImpl::newSimulation(uint64_t timestep, GeneralSettings const& generalSettings, SimulationParameters const& parameters)
 {
     _generalSettings = generalSettings;
@@ -26,9 +21,9 @@ void _SimulationControllerImpl::clear()
     _selectionNeedsUpdate = true;
 }
 
-void _SimulationControllerImpl::registerImageResource(void* image)
+void _SimulationControllerImpl::setImageResource(void* image)
 {
-    _worker.registerImageResource(image);
+    _worker.setImageResource(image);
 }
 
 std::string _SimulationControllerImpl::getGpuName() const