Cherry-pick read&write timeouts

Author: Matevz Morato

README.md

@@ -197,6 +197,8 @@ The following environment variables can be set to alter default behavior of the
 | DEPTHAI_SEARCH_TIMEOUT | Specifies timeout in milliseconds for device searching in blocking functions. |
 | DEPTHAI_CONNECT_TIMEOUT | Specifies timeout in milliseconds for establishing a connection to a given device. |
 | DEPTHAI_BOOTUP_TIMEOUT | Specifies timeout in milliseconds for waiting the device to boot after sending the binary. |
+| DEPTHAI_RPC_READ_TIMEOUT | Specifies timeout in milliseconds for reading RPC responses. If 0, wait indefinitely. |
+| DEPTHAI_RPC_WRITE_TIMEOUT | Specifies timeout in milliseconds for writing RPC requests. If 0, wait indefinitely. |
 | DEPTHAI_PROTOCOL | Restricts default search to the specified protocol. Options: any, usb, tcpip. |
 | DEPTHAI_DEVICE_MXID_LIST | Restricts default search to the specified MXIDs. Accepts comma separated list of MXIDs. Lists filter results in an "AND" manner and not "OR" |
 | DEPTHAI_DEVICE_ID_LIST | Alias to MXID list. Lists filter results in an "AND" manner and not "OR" |

cmake/Depthai/DepthaiDeviceSideConfig.cmake

@@ -2,7 +2,7 @@
 set(DEPTHAI_DEVICE_SIDE_MATURITY "snapshot")
 
 # "full commit hash of device side binary"
-set(DEPTHAI_DEVICE_SIDE_COMMIT "b5224fbf5cceeb810a2e1abf27a4387cb817ea26")
+set(DEPTHAI_DEVICE_SIDE_COMMIT "dce3b2b64b47f27d91d37ada301ee4ed3a00ef83")
 
 # "version if applicable"
 set(DEPTHAI_DEVICE_SIDE_VERSION "")

examples/CMakeLists.txt

@@ -432,4 +432,8 @@ target_compile_definitions(cast_diff PRIVATE BLOB_PATH="${diff_model}")
 dai_add_example(spatial_tiny_yolo_tof_v3 SpatialDetection/spatial_tiny_yolo_tof.cpp OFF OFF)
 dai_add_example(spatial_tiny_yolo_tof_v4 SpatialDetection/spatial_tiny_yolo_tof.cpp OFF OFF)
 target_compile_definitions(spatial_tiny_yolo_tof_v3 PRIVATE BLOB_PATH="${tiny_yolo_v3_blob}")
-target_compile_definitions(spatial_tiny_yolo_tof_v4 PRIVATE BLOB_PATH="${tiny_yolo_v4_blob}")
+target_compile_definitions(spatial_tiny_yolo_tof_v4 PRIVATE BLOB_PATH="${tiny_yolo_v4_blob}")
+
+# Misc
+## Crash dump
+dai_add_example(crash_dump Misc/CrashDump/crash_dump.cpp OFF OFF)

examples/Misc/CrashDump/crash_dump.cpp

@@ -0,0 +1,52 @@
+#include <iostream>
+
+#include "depthai/depthai.hpp"
+
+// Main intention of this example is to test and verify the crash dump functionality works as intended
+int main() try {
+    std::cout << "This example crashes the device on pressing 'c' key. Press 'q' to quit." << std::endl;
+    // Create pipeline
+    dai::Pipeline pipeline;
+
+    // Define source and output
+    auto camRgb = pipeline.create<dai::node::ColorCamera>();
+    auto xoutVideo = pipeline.create<dai::node::XLinkOut>();
+
+    xoutVideo->setStreamName("video");
+
+    // Properties
+    camRgb->setBoardSocket(dai::CameraBoardSocket::CAM_A);
+    camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_1080_P);
+    camRgb->setVideoSize(1920, 1080);
+
+    xoutVideo->input.setBlocking(false);
+    xoutVideo->input.setQueueSize(1);
+
+    // Linking
+    camRgb->video.link(xoutVideo->input);
+
+    // Connect to device and start pipeline
+    dai::Device device(pipeline);
+
+    auto video = device.getOutputQueue("video");
+
+    while(true) {
+        auto videoIn = video->get<dai::ImgFrame>();
+
+        // Get BGR frame from NV12 encoded video frame to show with opencv
+        // Visualizing the frame on slower hosts might have overhead
+        cv::imshow("video", videoIn->getCvFrame());
+
+        int key = cv::waitKey(1);
+        if(key == 'q' || key == 'Q') {
+            return 0;
+        }
+        if(key == 'c') {
+            device.crashDevice();
+        }
+    }
+    return 0;
+} catch(const std::exception& ex) {
+    std::cerr << "Error: " << ex.what() << std::endl;
+    return 1;
+}

include/depthai/device/DeviceBase.hpp

@@ -906,6 +906,12 @@ class DeviceBase {
      */
     bool isClosed() const;
 
+    /**
+     * Crashes the device
+     * @warning ONLY FOR TESTING PURPOSES, it causes an unrecoverable crash on the device
+     */
+    void crashDevice();
+
     /**
      * Returns underlying XLinkConnection
      */

src/device/DeviceBase.cpp

@@ -1,7 +1,9 @@
 #include "depthai/device/DeviceBase.hpp"
 
 // std
+#include <chrono>
 #include <iostream>
+#include <stdexcept>
 
 // shared
 #include "depthai-bootloader-shared/Bootloader.hpp"
@@ -49,7 +51,8 @@ const std::string MAGIC_FACTORY_FLASHING_VALUE = "413424129";
 const std::string MAGIC_FACTORY_PROTECTED_FLASHING_VALUE = "868632271";
 
 const unsigned int DEFAULT_CRASHDUMP_TIMEOUT = 9000;
-const unsigned int RPC_READ_TIMEOUT = 30000;
+const unsigned int DEFAULT_RPC_READ_TIMEOUT = 10000;
+const unsigned int DEFAULT_RPC_WRITE_TIMEOUT = 0;  // 0 means blocking
 
 // local static function
 static void getFlashingPermissions(bool& factoryPermissions, bool& protectedPermissions) {
@@ -534,13 +537,44 @@ unsigned int getCrashdumpTimeout(XLinkProtocol_t protocol) {
     return DEFAULT_CRASHDUMP_TIMEOUT + (protocol == X_LINK_TCP_IP ? device::XLINK_TCP_WATCHDOG_TIMEOUT.count() : device::XLINK_USB_WATCHDOG_TIMEOUT.count());
 }
 
+unsigned int getRPCReadTimeout() {
+    std::string timeoutStr = utility::getEnv("DEPTHAI_RPC_READ_TIMEOUT");
+    if(!timeoutStr.empty()) {
+        try {
+            return std::stoi(timeoutStr);
+        } catch(const std::invalid_argument& e) {
+            logger::warn("DEPTHAI_RPC_READ_TIMEOUT value invalid: {}", e.what());
+        }
+    }
+    return DEFAULT_RPC_READ_TIMEOUT;
+}
+
+unsigned int getRPCWriteTimeout() {
+    std::string timeoutStr = utility::getEnv("DEPTHAI_RPC_WRITE_TIMEOUT");
+    if(!timeoutStr.empty()) {
+        try {
+            return std::stoi(timeoutStr);
+        } catch(const std::invalid_argument& e) {
+            logger::warn("DEPTHAI_RPC_WRITE_TIMEOUT value invalid: {}", e.what());
+        }
+    }
+    return DEFAULT_RPC_WRITE_TIMEOUT;
+}
+
+tl::optional<std::string> saveCrashDump(dai::CrashDump& dump, std::string mxId) {
+    std::vector<uint8_t> data;
+    utility::serialize<SerializationType::JSON>(dump, data);
+    auto crashDumpPathStr = utility::getEnv("DEPTHAI_CRASHDUMP");
+    return saveFileToTemporaryDirectory(data, mxId + "-depthai_crash_dump.json", crashDumpPathStr);
+}
+
 void DeviceBase::closeImpl() {
     using namespace std::chrono;
     auto t1 = steady_clock::now();
     auto timeout = getCrashdumpTimeout(deviceInfo.protocol);
     bool shouldGetCrashDump = false;
     if(!dumpOnly && timeout > 0) {
-        pimpl->logger.debug("Device about to be closed...");
+        pimpl->logger.debug("Crash dump collection enabled - first try to extract an existing crash dump");
         try {
             if(hasCrashDump()) {
                 connection->setRebootOnDestruction(true);
@@ -556,8 +590,10 @@ void DeviceBase::closeImpl() {
             pimpl->logger.debug("shutdown call error: {}", ex.what());
             shouldGetCrashDump = true;
         }
+    } else {
+        pimpl->logger.debug("Crash dump collection disabled - skipping existing crash dump extraction");
     }
-
+    pimpl->logger.debug("Device about to be closed...");
     // Close connection first; causes Xlink internal calls to unblock semaphore waits and
     // return error codes, which then allows queues to unblock
     // always manage ownership because other threads (e.g. watchdog) are running and need to
@@ -616,6 +652,8 @@ void DeviceBase::closeImpl() {
             }
         } else if(shouldGetCrashDump) {
             pimpl->logger.warn("Device crashed. Crash dump retrieval disabled.");
+        } else if(timeout == 0) {
+            pimpl->logger.debug("Crash dump collection disabled, skip device reconnection");
         }
 
         pimpl->logger.debug("Device closed, {}", duration_cast<milliseconds>(steady_clock::now() - t1).count());
@@ -827,11 +865,23 @@ void DeviceBase::init2(Config cfg, const dai::Path& pathToMvcmd, tl::optional<co
 
         try {
             // Send request to device
-            rpcStream->write(std::move(request));
-
+            auto writeTimeout = getRPCWriteTimeout();
+            if(writeTimeout > 0) {
+                auto success = rpcStream->write(std::move(request), std::chrono::milliseconds(writeTimeout));
+                if(!success) {
+                    throw std::runtime_error("RPC write timed out in " + std::to_string(writeTimeout) + "ms");
+                }
+            } else {
+                rpcStream->write(std::move(request));
+            }
             // Receive response back
             // Send to nanorpc to parse
-            return rpcStream->read(std::chrono::milliseconds(RPC_READ_TIMEOUT));
+            auto timeout = getRPCReadTimeout();
+            if(timeout > 0) {
+                return rpcStream->read(std::chrono::milliseconds(timeout));
+            } else {
+                return rpcStream->read();
+            }
         } catch(const std::exception& e) {
             // If any exception is thrown, log it and rethrow
             pimpl->logger.debug("RPC error: {}", e.what());
@@ -1123,6 +1173,19 @@ std::string DeviceBase::getConnectedIMU() {
     return pimpl->rpcClient->call("getConnectedIMU").as<std::string>();
 }
 
+void DeviceBase::crashDevice() {
+    // Check that the protective ENV variable is set
+    if(utility::getEnv("DEPTHAI_CRASH_DEVICE") != "1") {
+        pimpl->logger.error("Crashing the device is disabled. Set DEPTHAI_CRASH_DEVICE=1 to enable.");
+        return;
+    }
+    try {
+        pimpl->rpcClient->call("crashDevice");
+    } catch(const std::system_error& ex) {
+        pimpl->logger.debug("Crash device threw an exception: {} (expected)", ex.what());
+    }
+}
+
 dai::Version DeviceBase::getIMUFirmwareVersion() {
     std::string versionStr = pimpl->rpcClient->call("getIMUFirmwareVersion").as<std::string>();
     try {