Skip to content

ObjectStorageProvider: add write back support #1677

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

ObjectStorageProvider: add write back support #1677

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
676e48f
ObjectStorageProvider: cache locally restored Zips
rressi-at-globus May 9, 2025
16e1259
Merge branch 'main' into binary_caching_preserve_zips_from_cloud_storage
rressi-at-globus Jun 18, 2025
dac97e8
Merge tag '2025-11-19' into binary_caching_preserve_zips_from_cloud_s…
rressi-at-globus Nov 21, 2025
fc685f1
binarycaching: apply try_write_back_zip_file only if the job was succ…
rressi-at-globus Nov 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 76 additions & 12 deletions src/vcpkg/binarycaching.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include <vcpkg/vcpkgcmdarguments.h>
#include <vcpkg/vcpkgpaths.h>

#include <algorithm>
#include <memory>
#include <utility>

Expand Down Expand Up @@ -245,6 +246,18 @@ namespace
Path files_archive_parent_path(const std::string& abi) { return Path(abi.substr(0, 2)); }
Path files_archive_subpath(const std::string& abi) { return files_archive_parent_path(abi) / (abi + ".zip"); }

StringView try_extract_abi_from_zip_file(const Path& zip_file)
{
StringView file_stem = zip_file.stem();
auto reverse_it =
std::find_if(file_stem.rbegin(), file_stem.rend(), [](char ch) { return !std::isxdigit(ch); });
if (reverse_it != file_stem.rend() && (*reverse_it) == '_')
{
return {reverse_it.base(), file_stem.end()};
}
return {};
}

struct FilesWriteBinaryProvider : IWriteBinaryProvider
{
FilesWriteBinaryProvider(const Filesystem& fs, std::vector<Path>&& dirs) : m_fs(fs), m_dirs(std::move(dirs)) { }
Expand Down Expand Up @@ -325,7 +338,10 @@ namespace
// - IReadBinaryProvider::precheck()
struct ZipReadBinaryProvider : IReadBinaryProvider
{
ZipReadBinaryProvider(ZipTool zip, const Filesystem& fs) : m_zip(std::move(zip)), m_fs(fs) { }
ZipReadBinaryProvider(ZipTool zip, const Filesystem& fs, Optional<Path> local_cache_folder = {})
: m_zip(std::move(zip)), m_fs(fs), m_local_cache_folder(std::move(local_cache_folder))
{
}

struct UnzipJob
{
Expand Down Expand Up @@ -382,12 +398,27 @@ namespace
DiagKind::Note, job.zip_resource->path, msg::format(msgWhileExtractingThisArchive)});
}
}
});

for (auto&& job : jobs)
{
const auto zip_resource = job.zip_resource;

if (job.success)
{
if (auto new_path = try_write_back_zip_file(zip_resource->path))
{
Debug::print("Renamed ", zip_resource->path, " to ", *(new_path.get()), "\n");
continue;
}
}

if (job.zip_resource->to_remove == RemoveWhen::always)
if (zip_resource->to_remove == RemoveWhen::always)
{
m_fs.remove(job.zip_resource->path, IgnoreErrors{});
m_fs.remove(zip_resource->path, IgnoreErrors{});
Debug::print("Removed ", zip_resource->path, '\n');
}
});
}

for (auto&& job : jobs)
{
Expand All @@ -402,6 +433,31 @@ namespace
}
}

[[nodiscard]] Optional<Path> try_write_back_zip_file(const Path& zip_file) const
{
if (!m_local_cache_folder)
{
return {};
}

const StringView abi = try_extract_abi_from_zip_file(zip_file);
if (abi.empty())
{
return {};
}

const Path& local_cache_folder = *m_local_cache_folder.get();
Path new_path = local_cache_folder / files_archive_subpath(std::string(abi));
if (m_fs.exists(new_path, IgnoreErrors{}))
{
return {};
}

m_fs.create_directories(new_path.parent_path(), IgnoreErrors{});
m_fs.rename(zip_file, new_path, IgnoreErrors{});
return new_path;
}

// For every action denoted by actions, at corresponding indicies in out_zips, stores a ZipResource indicating
// the downloaded location.
//
Expand All @@ -412,6 +468,7 @@ namespace
protected:
ZipTool m_zip;
const Filesystem& m_fs;
Optional<Path> m_local_cache_folder;
};

struct FilesReadBinaryProvider : ZipReadBinaryProvider
Expand Down Expand Up @@ -989,8 +1046,9 @@ namespace
const Filesystem& fs,
const Path& buildtrees,
std::string&& prefix,
const std::shared_ptr<const IObjectStorageTool>& tool)
: ZipReadBinaryProvider(std::move(zip), fs)
const std::shared_ptr<const IObjectStorageTool>& tool,
Optional<Path> local_cache_folder)
: ZipReadBinaryProvider(std::move(zip), fs, std::move(local_cache_folder))
, m_buildtrees(buildtrees)
, m_prefix(std::move(prefix))
, m_tool(tool)
Expand Down Expand Up @@ -2683,22 +2741,28 @@ namespace vcpkg
std::make_unique<HttpGetBinaryProvider>(zip_tool, fs, buildtrees, std::move(url), s.secrets));
}

Optional<Path> local_cache_folder;
if (!s.archives_to_write.empty())
{
local_cache_folder = s.archives_to_write.front();
}

for (auto&& prefix : s.gcs_read_prefixes)
{
m_config.read.push_back(
std::make_unique<ObjectStorageProvider>(zip_tool, fs, buildtrees, std::move(prefix), gcs_tool));
m_config.read.push_back(std::make_unique<ObjectStorageProvider>(
zip_tool, fs, buildtrees, std::move(prefix), gcs_tool, local_cache_folder));
}

for (auto&& prefix : s.aws_read_prefixes)
{
m_config.read.push_back(
std::make_unique<ObjectStorageProvider>(zip_tool, fs, buildtrees, std::move(prefix), aws_tool));
m_config.read.push_back(std::make_unique<ObjectStorageProvider>(
zip_tool, fs, buildtrees, std::move(prefix), aws_tool, local_cache_folder));
}

for (auto&& prefix : s.cos_read_prefixes)
{
m_config.read.push_back(
std::make_unique<ObjectStorageProvider>(zip_tool, fs, buildtrees, std::move(prefix), cos_tool));
m_config.read.push_back(std::make_unique<ObjectStorageProvider>(
zip_tool, fs, buildtrees, std::move(prefix), cos_tool, local_cache_folder));
}

for (auto&& src : s.upkg_templates_to_get)
Expand Down