Switch to using the vendored zstd library. (#59)

* Switch to using the vendored zstd library.

We already have it vendored in the core as necessary.

Signed-off-by: Chris Lalancette <[email protected]>

* Add `can_write_mcap_with_zstd_configured_from_yaml` test

- Add unit test to check that rosbag2_mcap_storage plugin can write
mcap file with zstd compression configured via storage config yaml file

Signed-off-by: Michael Orlov <[email protected]>

* Address build failure on CI related to the missing header from zstd lib

- Replace target_link_libraries(mcap PRIVATE zstd::zstd) with
ament_target_dependencies(mcap zstd)

Signed-off-by: Michael Orlov <[email protected]>

Signed-off-by: Chris Lalancette <[email protected]>
Signed-off-by: Michael Orlov <[email protected]>
Co-authored-by: Michael Orlov <[email protected]>

Author: clalancette

mcap_vendor/CMakeLists.txt

@@ -13,6 +13,8 @@ endif()
 
 ## Dependencies
 find_package(ament_cmake REQUIRED)
+find_package(zstd_vendor REQUIRED)
+find_package(zstd REQUIRED)
 
 
 ## Define vendor macro
@@ -30,39 +32,25 @@ macro(build_mcap_vendor)
   )
   fetchcontent_makeavailable(lz4)
 
-  fetchcontent_declare(zstd
-    GIT_REPOSITORY https://github.com/facebook/zstd.git
-    GIT_TAG e47e674cd09583ff0503f0f6defd6d23d8b718d3 # v1.5.2
-  )
-  fetchcontent_makeavailable(zstd)
-
-  file(GLOB _zstd_srcs
-    ${zstd_SOURCE_DIR}/lib/common/*.c
-    ${zstd_SOURCE_DIR}/lib/compress/*.c
-    ${zstd_SOURCE_DIR}/lib/decompress/*.c
-    ${zstd_SOURCE_DIR}/lib/decompress/*.S
-  )
-
   file(GLOB _lz4_srcs
     ${lz4_SOURCE_DIR}/lib/*.c)
 
   add_library(
     mcap SHARED
     src/main.cpp
-    ${_zstd_srcs}
     ${_lz4_srcs}
   )
 
   set(_mcap_include_dir ${mcap_SOURCE_DIR}/cpp/mcap/include)
 
   target_include_directories(mcap SYSTEM PRIVATE
     ${lz4_SOURCE_DIR}/lib
-    ${zstd_SOURCE_DIR}/lib
   )
   target_include_directories(mcap SYSTEM PUBLIC
     "$<BUILD_INTERFACE:${_mcap_include_dir}>"
     "$<INSTALL_INTERFACE:include>"
   )
+  ament_target_dependencies(mcap zstd)
 
   install(
     DIRECTORY
@@ -79,6 +67,8 @@ build_mcap_vendor()
 
 ament_export_targets(export_mcap HAS_LIBRARY_TARGET)
 
+ament_export_dependencies(zstd_vendor zstd)
+
 ## Tests
 if(BUILD_TESTING)
   find_package(ament_lint_auto REQUIRED)

mcap_vendor/package.xml

@@ -9,7 +9,9 @@
 
   <buildtool_depend>ament_cmake</buildtool_depend>
   <buildtool_depend>git</buildtool_depend>
-  
+
+  <depend>zstd_vendor</depend>
+
   <test_depend>ament_cmake_clang_format</test_depend>
   <test_depend>ament_lint_auto</test_depend>
   <test_depend>ament_lint_common</test_depend>

rosbag2_storage_mcap/CMakeLists.txt

@@ -84,6 +84,8 @@ if(BUILD_TESTING)
   find_package(rosbag2_test_common REQUIRED)
   find_package(std_msgs REQUIRED)
 
+  add_definitions(-D_TEST_RESOURCES_DIR_PATH="${CMAKE_CURRENT_SOURCE_DIR}/test/${PROJECT_NAME}")
+
   set(ament_cmake_clang_format_CONFIG_FILE .clang-format)
   list(APPEND AMENT_LINT_AUTO_EXCLUDE ament_cmake_uncrustify)
   ament_lint_auto_find_test_dependencies()

rosbag2_storage_mcap/test/rosbag2_storage_mcap/mcap_writer_options_zstd.yaml

@@ -0,0 +1,8 @@
+noCRC: false
+noChunking: false
+noMessageIndex: false
+noSummary: false
+chunkSize: 786432
+compression: "Zstd"
+compressionLevel: "Fast"
+forceCompression: true

rosbag2_storage_mcap/test/rosbag2_storage_mcap/test_mcap_storage.cpp

@@ -99,3 +99,72 @@ TEST_F(TemporaryDirectoryFixture, can_write_and_read_basic_mcap_file) {
     EXPECT_EQ(msg.data, message_data);
   }
 }
+
+#ifdef ROSBAG2_STORAGE_MCAP_HAS_STORAGE_OPTIONS
+// This test disabled on Foxy since StorageOptions doesn't have storage_config_uri field on it
+TEST_F(TemporaryDirectoryFixture, can_write_mcap_with_zstd_configured_from_yaml) {
+  auto uri = rcpputils::fs::path(temporary_dir_path_) / "bag";
+  auto expected_bag = uri / "bag_0.mcap";
+  const int64_t timestamp_nanos = 100;  // arbitrary value
+  rcutils_time_point_value_t time_stamp{timestamp_nanos};
+  const std::string topic_name = "test_topic";
+  const std::string message_data = "Test Message 1";
+  const std::string storage_id = "mcap";
+  const std::string config_path = _TEST_RESOURCES_DIR_PATH;
+  // COMPATIBILITY(foxy)
+  // using verbose APIs for Foxy compatibility which did not yet provide plain-message API
+  rclcpp::Serialization<std_msgs::msg::String> serialization;
+
+  {
+    StorageOptions options;
+    options.uri = uri.string();
+    options.storage_id = storage_id;
+    options.storage_config_uri = config_path + "/mcap_writer_options_zstd.yaml";
+    rosbag2_storage::TopicMetadata topic_metadata;
+    topic_metadata.name = topic_name;
+    topic_metadata.type = "std_msgs/msg/String";
+
+    std_msgs::msg::String msg;
+    msg.data = message_data;
+
+    rosbag2_cpp::Writer writer{std::make_unique<rosbag2_cpp::writers::SequentialWriter>()};
+#ifndef ROSBAG2_STORAGE_MCAP_WRITER_CREATES_DIRECTORY
+    rcpputils::fs::create_directories(uri);
+#endif
+    writer.open(options, rosbag2_cpp::ConverterOptions{});
+    writer.create_topic(topic_metadata);
+
+    auto serialized_msg = std::make_shared<rclcpp::SerializedMessage>();
+    serialization.serialize_message(&msg, serialized_msg.get());
+
+    // This is really kludgy, it's due to a mismatch between types in the rclcpp serialization API
+    // and the historical Foxy serialized APIs. Prevents the hacked shared ptr from deleting the
+    // data that `serialized_bag_msg` should reasonably expect to continue existing.
+    // For this example it wouldn't matter, but in case anybody extends this test, it's for safety.
+    auto serialized_bag_msg = std::make_shared<rosbag2_storage::SerializedBagMessage>();
+    serialized_bag_msg->serialized_data = std::shared_ptr<rcutils_uint8_array_t>(
+      const_cast<rcutils_uint8_array_t*>(&serialized_msg->get_rcl_serialized_message()),
+      [](rcutils_uint8_array_t* /* data */) {});
+    serialized_bag_msg->time_stamp = time_stamp;
+    serialized_bag_msg->topic_name = topic_name;
+    writer.write(serialized_bag_msg);
+    writer.write(serialized_bag_msg);
+    EXPECT_TRUE(expected_bag.is_regular_file());
+  }
+  {
+    StorageOptions options;
+    options.uri = expected_bag.string();
+    options.storage_id = storage_id;
+
+    rosbag2_cpp::Reader reader{std::make_unique<rosbag2_cpp::readers::SequentialReader>()};
+    reader.open(options, rosbag2_cpp::ConverterOptions{});
+    EXPECT_TRUE(reader.has_next());
+
+    std_msgs::msg::String msg;
+    auto serialized_bag_msg = reader.read_next();
+    rclcpp::SerializedMessage extracted_serialized_msg(*serialized_bag_msg->serialized_data);
+    serialization.deserialize_message(&extracted_serialized_msg, &msg);
+    EXPECT_EQ(msg.data, message_data);
+  }
+}
+#endif  // #ifdef ROSBAG2_STORAGE_MCAP_HAS_STORAGE_OPTIONS