Add io_buffer_size to BackupEngineOptions

Author: archang19

include/rocksdb/utilities/backup_engine.h

@@ -77,6 +77,13 @@ struct BackupEngineOptions {
   // Default: true
   bool backup_log_files;
 
+  // Size of the buffer used for reading files.
+  // Enables optimally configuring the IO size based on the storage backend.
+  // If specified, takes precendence over the rate limiter burst size (if
+  // specified) as well as kDefaultCopyFileBufferSize.
+  // Default: 0
+  uint64_t io_buffer_size;
+
   // Max bytes that can be transferred in a second during backup.
   // If 0, go as fast as you can
   // This limit only applies to writes. To also limit reads,
@@ -228,8 +235,9 @@ struct BackupEngineOptions {
       const std::string& _backup_dir, Env* _backup_env = nullptr,
       bool _share_table_files = true, Logger* _info_log = nullptr,
       bool _sync = true, bool _destroy_old_data = false,
-      bool _backup_log_files = true, uint64_t _backup_rate_limit = 0,
-      uint64_t _restore_rate_limit = 0, int _max_background_operations = 1,
+      bool _backup_log_files = true, uint64_t _io_buffer_size = 0,
+      uint64_t _backup_rate_limit = 0, uint64_t _restore_rate_limit = 0,
+      int _max_background_operations = 1,
       uint64_t _callback_trigger_interval_size = 4 * 1024 * 1024,
       int _max_valid_backups_to_open = INT_MAX,
       ShareFilesNaming _share_files_with_checksum_naming =
@@ -241,6 +249,7 @@ struct BackupEngineOptions {
         sync(_sync),
         destroy_old_data(_destroy_old_data),
         backup_log_files(_backup_log_files),
+        io_buffer_size(_io_buffer_size),
         backup_rate_limit(_backup_rate_limit),
         restore_rate_limit(_restore_rate_limit),
         share_files_with_checksum(true),

utilities/backup/backup_engine.cc

@@ -581,6 +581,8 @@ class BackupEngineImpl {
                             uint64_t* bytes_toward_next_callback,
                             uint64_t* size, std::string* checksum_hex);
 
+  uint64_t CalculateIOBufferSize(RateLimiter* rate_limiter) const;
+
   IOStatus ReadFileAndComputeChecksum(const std::string& src,
                                       const std::shared_ptr<FileSystem>& src_fs,
                                       const EnvOptions& src_env_options,
@@ -2212,9 +2214,7 @@ IOStatus BackupEngineImpl::CopyOrCreateFile(
     return io_s;
   }
 
-  size_t buf_size =
-      rate_limiter ? static_cast<size_t>(rate_limiter->GetSingleBurstBytes())
-                   : kDefaultCopyFileBufferSize;
+  size_t buf_size = CalculateIOBufferSize(rate_limiter);
 
   // TODO: pass in Histograms if the destination file is sst or blob
   std::unique_ptr<WritableFileWriter> dest_writer(
@@ -2307,6 +2307,16 @@ IOStatus BackupEngineImpl::CopyOrCreateFile(
   return io_s;
 }
 
+uint64_t BackupEngineImpl::CalculateIOBufferSize(
+    RateLimiter* rate_limiter) const {
+  if (options_.io_buffer_size > 0) {
+    return options_.io_buffer_size;
+  }
+  return rate_limiter != nullptr
+             ? static_cast<size_t>(rate_limiter->GetSingleBurstBytes())
+             : kDefaultCopyFileBufferSize;
+}
+
 // fname will always start with "/"
 IOStatus BackupEngineImpl::AddBackupFileWorkItem(
     std::unordered_set<std::string>& live_dst_paths,
@@ -2588,7 +2598,7 @@ IOStatus BackupEngineImpl::ReadFileAndComputeChecksum(
     return io_s;
   }
 
-  size_t buf_size = kDefaultCopyFileBufferSize;
+  size_t buf_size = CalculateIOBufferSize(nullptr);
   std::unique_ptr<char[]> buf(new char[buf_size]);
   Slice data;