From 676e48fb9574b98d8772996f62af6df0af6d8965 Mon Sep 17 00:00:00 2001 From: Riccardo Ressi Date: Fri, 9 May 2025 14:42:10 +0200 Subject: [PATCH 1/2] ObjectStorageProvider: cache locally restored Zips --- src/vcpkg/binarycaching.cpp | 77 ++++++++++++++++++++++++++++++++----- 1 file changed, 67 insertions(+), 10 deletions(-) diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index de0f32efbc..f0144dd388 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -26,6 +26,7 @@ #include #include +#include #include #include @@ -213,6 +214,18 @@ namespace Path files_archive_parent_path(const std::string& abi) { return Path(abi.substr(0, 2)); } Path files_archive_subpath(const std::string& abi) { return files_archive_parent_path(abi) / (abi + ".zip"); } + StringView try_extract_abi_from_zip_file(const Path& zip_file) + { + StringView file_stem = zip_file.stem(); + auto reverse_it = + std::find_if(file_stem.rbegin(), file_stem.rend(), [](char ch) { return !std::isxdigit(ch); }); + if (reverse_it != file_stem.rend() && (*reverse_it) == '_') + { + return {reverse_it.base(), file_stem.end()}; + } + return {}; + } + struct FilesWriteBinaryProvider : IWriteBinaryProvider { FilesWriteBinaryProvider(const Filesystem& fs, std::vector&& dirs) : m_fs(fs), m_dirs(std::move(dirs)) { } @@ -293,7 +306,10 @@ namespace // - IReadBinaryProvider::precheck() struct ZipReadBinaryProvider : IReadBinaryProvider { - ZipReadBinaryProvider(ZipTool zip, const Filesystem& fs) : m_zip(std::move(zip)), m_fs(fs) { } + ZipReadBinaryProvider(ZipTool zip, const Filesystem& fs, Optional local_cache_folder = {}) + : m_zip(std::move(zip)), m_fs(fs), m_local_cache_folder(std::move(local_cache_folder)) + { + } void fetch(View actions, Span out_status) const override { @@ -336,8 +352,41 @@ namespace { if (r.to_remove == RemoveWhen::always) { - m_fs.remove(r.path, IgnoreErrors{}); + if (auto new_path = try_write_back_zip_file(r.path)) + { + Debug::print("Renamed ", r.path, " to ", *(new_path.get()), "\n"); + } + else + { + m_fs.remove(r.path, IgnoreErrors{}); + Debug::print("Removed ", r.path, '\n'); + } + } + } + + [[nodiscard]] Optional try_write_back_zip_file(const Path& zip_file) const + { + if (!m_local_cache_folder) + { + return {}; + } + + const StringView abi = try_extract_abi_from_zip_file(zip_file); + if (abi.empty()) + { + return {}; + } + + const Path& local_cache_folder = *m_local_cache_folder.get(); + Path new_path = local_cache_folder / files_archive_subpath(std::string(abi)); + if (m_fs.exists(new_path, IgnoreErrors{})) + { + return {}; } + + m_fs.create_directories(new_path.parent_path(), IgnoreErrors{}); + m_fs.rename(zip_file, new_path, IgnoreErrors{}); + return new_path; } // For every action denoted by actions, at corresponding indicies in out_zips, stores a ZipResource indicating @@ -350,6 +399,7 @@ namespace protected: ZipTool m_zip; const Filesystem& m_fs; + Optional m_local_cache_folder; }; struct FilesReadBinaryProvider : ZipReadBinaryProvider @@ -857,8 +907,9 @@ namespace const Filesystem& fs, const Path& buildtrees, std::string&& prefix, - const std::shared_ptr& tool) - : ZipReadBinaryProvider(std::move(zip), fs) + const std::shared_ptr& tool, + Optional local_cache_folder) + : ZipReadBinaryProvider(std::move(zip), fs, std::move(local_cache_folder)) , m_buildtrees(buildtrees) , m_prefix(std::move(prefix)) , m_tool(tool) @@ -2253,22 +2304,28 @@ namespace vcpkg std::make_unique(zip_tool, fs, buildtrees, std::move(url), s.secrets)); } + Optional local_cache_folder; + if (!s.archives_to_write.empty()) + { + local_cache_folder = s.archives_to_write.front(); + } + for (auto&& prefix : s.gcs_read_prefixes) { - m_config.read.push_back( - std::make_unique(zip_tool, fs, buildtrees, std::move(prefix), gcs_tool)); + m_config.read.push_back(std::make_unique( + zip_tool, fs, buildtrees, std::move(prefix), gcs_tool, local_cache_folder)); } for (auto&& prefix : s.aws_read_prefixes) { - m_config.read.push_back( - std::make_unique(zip_tool, fs, buildtrees, std::move(prefix), aws_tool)); + m_config.read.push_back(std::make_unique( + zip_tool, fs, buildtrees, std::move(prefix), aws_tool, local_cache_folder)); } for (auto&& prefix : s.cos_read_prefixes) { - m_config.read.push_back( - std::make_unique(zip_tool, fs, buildtrees, std::move(prefix), cos_tool)); + m_config.read.push_back(std::make_unique( + zip_tool, fs, buildtrees, std::move(prefix), cos_tool, local_cache_folder)); } for (auto&& src : s.upkg_templates_to_get) From fc685f17b214cb8164246fd22535861cdcd9810c Mon Sep 17 00:00:00 2001 From: Riccardo Ressi Date: Fri, 21 Nov 2025 06:33:45 +0100 Subject: [PATCH 2/2] binarycaching: apply try_write_back_zip_file only if the job was successful --- src/vcpkg/binarycaching.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 7b4a13cc93..b91cccd0dd 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -403,11 +403,17 @@ namespace for (auto&& job : jobs) { const auto zip_resource = job.zip_resource; - if (auto new_path = try_write_back_zip_file(zip_resource->path)) + + if (job.success) { - Debug::print("Renamed ", zip_resource->path, " to ", *(new_path.get()), "\n"); + if (auto new_path = try_write_back_zip_file(zip_resource->path)) + { + Debug::print("Renamed ", zip_resource->path, " to ", *(new_path.get()), "\n"); + continue; + } } - else if (zip_resource->to_remove == RemoveWhen::always) + + if (zip_resource->to_remove == RemoveWhen::always) { m_fs.remove(zip_resource->path, IgnoreErrors{}); Debug::print("Removed ", zip_resource->path, '\n');