From 792902259646cc3f470033f6103fa7016513f1d6 Mon Sep 17 00:00:00 2001 From: Sam De Roeck Date: Sat, 4 May 2024 19:31:36 -0700 Subject: [PATCH] feat(binary-cache): Add write-back support to binary cache --- .../end-to-end-tests-dir/binary-caching.ps1 | 25 +- azure-pipelines/end-to-end-tests-prelude.ps1 | 2 + include/vcpkg/base/util.h | 13 + include/vcpkg/binarycaching.h | 94 ++- src/vcpkg-test/binarycaching.cpp | 5 + src/vcpkg-test/configparser.cpp | 140 ++-- src/vcpkg/binarycaching.cpp | 649 ++++++++++++------ src/vcpkg/commands.install.cpp | 6 +- 8 files changed, 640 insertions(+), 294 deletions(-) diff --git a/azure-pipelines/end-to-end-tests-dir/binary-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/binary-caching.ps1 index fd0f151c34..f0178ddf91 100644 --- a/azure-pipelines/end-to-end-tests-dir/binary-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/binary-caching.ps1 @@ -47,6 +47,27 @@ Require-FileExists "$installRoot/$Triplet/include/hello-1.h" Require-FileExists "$buildtreesRoot/vcpkg-hello-world-1/src" Require-FileNotExists "$buildtreesRoot/detect_compiler" +# Test write back to files archive +Remove-Item -Recurse -Force $installRoot +Remove-Item -Recurse -Force $buildtreesRoot +Run-Vcpkg -TestArgs ($commonArgs + @("install","vcpkg-hello-world-1", "vcpkg-cmake", "vcpkg-cmake-config", "--x-binarysource=clear;files,$ArchiveRoot;files,$ArchiveRootSecondary,write;files,$ArchiveRootUnused,read")) +Throw-IfFailed +Require-FileExists "$installRoot/$Triplet/include/hello-1.h" +Require-FileNotExists "$buildtreesRoot/vcpkg-hello-world-1/src" +if ((Get-ChildItem $ArchiveRootSecondary -Filter '*.zip' | Measure-Object).Count -ne 4) { + throw "In '$CurrentTest': did not write back exactly 4 packages to the secondary archive" +} +if ((Get-ChildItem $ArchiveRootUnused -Filter '*.zip' | Measure-Object).Count -ne 0) { + throw "In '$CurrentTest': some packages were written to the unused archive" +} + +# Test restore from secondary archive +Remove-Item -Recurse -Force $installRoot +Run-Vcpkg -TestArgs ($commonArgs + @("install","vcpkg-hello-world-1", "vcpkg-cmake", "vcpkg-cmake-config", "--x-binarysource=clear;files,$ArchiveRootSecondary,read")) +Throw-IfFailed +Require-FileExists "$installRoot/$Triplet/include/hello-1.h" +Require-FileNotExists "$buildtreesRoot/vcpkg-hello-world-1/src" + if(-Not $IsLinux -and -Not $IsMacOS) { # Test restoring from nuget Remove-Item -Recurse -Force $installRoot @@ -78,8 +99,8 @@ if(-Not $IsLinux -and -Not $IsMacOS) { Require-FileExists "$installRoot/$Triplet/include/hello-2.h" Require-FileNotExists "$buildtreesRoot/vcpkg-hello-world-1/src" Require-FileExists "$buildtreesRoot/vcpkg-hello-world-2/src" - if ((Get-ChildItem $NuGetRoot -Filter '*.nupkg' | Measure-Object).Count -ne 1) { - throw "In '$CurrentTest': did not create exactly 1 NuGet package" + if ((Get-ChildItem $NuGetRoot -Filter '*.nupkg' | Measure-Object).Count -ne 4) { + throw "In '$CurrentTest': did not create exactly 4 NuGet packages" } # Test export diff --git a/azure-pipelines/end-to-end-tests-prelude.ps1 b/azure-pipelines/end-to-end-tests-prelude.ps1 index fe7e6095f2..742afaefa1 100644 --- a/azure-pipelines/end-to-end-tests-prelude.ps1 +++ b/azure-pipelines/end-to-end-tests-prelude.ps1 @@ -5,6 +5,8 @@ $packagesRoot = Join-Path $TestingRoot 'packages' $NuGetRoot = Join-Path $TestingRoot 'nuget' $NuGetRoot2 = Join-Path $TestingRoot 'nuget2' $ArchiveRoot = Join-Path $TestingRoot 'archives' +$ArchiveRootSecondary = Join-Path $TestingRoot 'archives-secondary' +$ArchiveRootUnused = Join-Path $TestingRoot 'archives-unused' $VersionFilesRoot = Join-Path $TestingRoot 'version-test' $directoryArgs = @( "--x-buildtrees-root=$buildtreesRoot", diff --git a/include/vcpkg/base/util.h b/include/vcpkg/base/util.h index e619df7ee8..94c2af29c5 100644 --- a/include/vcpkg/base/util.h +++ b/include/vcpkg/base/util.h @@ -44,6 +44,19 @@ namespace vcpkg::Util return false; } + template + std::vector> filtered_copy(const Vec& container, const Filter&& filter) + { + std::vector> ret; + for (auto&& item : container) + { + if (filter(item)) + { + ret.push_back(item); + } + } + return ret; + } template bool contains(const Vec& container, const Key& item) { diff --git a/include/vcpkg/binarycaching.h b/include/vcpkg/binarycaching.h index 5c70cf04d1..4a409b71b1 100644 --- a/include/vcpkg/binarycaching.h +++ b/include/vcpkg/binarycaching.h @@ -25,18 +25,26 @@ namespace vcpkg { + /// Unique identifier for a provider + using ProviderId = size_t; + struct CacheStatus { + using ReaderProviders = std::vector; + bool should_attempt_precheck(const IReadBinaryProvider* sender) const noexcept; bool should_attempt_restore(const IReadBinaryProvider* sender) const noexcept; + bool should_attempt_write_back(ProviderId provider_id) const noexcept; bool is_unavailable(const IReadBinaryProvider* sender) const noexcept; const IReadBinaryProvider* get_available_provider() const noexcept; + ReaderProviders& get_unavailable_providers() noexcept; bool is_restored() const noexcept; void mark_unavailable(const IReadBinaryProvider* sender); void mark_available(const IReadBinaryProvider* sender) noexcept; void mark_restored() noexcept; + void mark_written_back(ProviderId provider_id) noexcept; private: CacheStatusState m_status = CacheStatusState::unknown; @@ -75,10 +83,13 @@ namespace vcpkg /// Called upon a successful build of `action` to store those contents in the binary cache. /// returns the number of successful uploads - virtual size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) = 0; + virtual size_t push_success(const BinaryPackageWriteInfo& request, + MessageSink& msg_sink, + CacheStatus& cache_status) = 0; virtual bool needs_nuspec_data() const = 0; virtual bool needs_zip_file() const = 0; + virtual std::vector get_provider_ids() const = 0; }; struct IReadBinaryProvider @@ -93,6 +104,10 @@ namespace vcpkg /// Prerequisites: actions[i].package_abi(), out_status.size() == actions.size() virtual void fetch(View actions, Span out_status) const = 0; + /// Flag to indicate if the provider supports an efficient check to see if a certain set of packages are + /// available. If not, packages will assumed to be present & will always be fetched. + virtual bool can_precheck() const = 0; + /// Checks whether the `actions` are present in the cache, without restoring them. /// /// Used by CI to determine missing packages. For each `i`, out_status[i] should be set to @@ -103,6 +118,11 @@ namespace vcpkg virtual LocalizedString restored_message(size_t count, std::chrono::high_resolution_clock::duration elapsed) const = 0; + + /// Unique identifier for this provider. + /// + /// Used by the cache to exclude cache providers during the write-back phase. + virtual ProviderId id() const = 0; }; struct UrlTemplate @@ -114,6 +134,10 @@ namespace vcpkg std::string instantiate_variables(const BinaryPackageReadInfo& info) const; }; + struct GithubActionsInfo + { + }; + struct NuGetRepoInfo { std::string repo; @@ -121,37 +145,51 @@ namespace vcpkg std::string commit; }; + enum class CacheType + { + Read, + Write, + ReadWrite + }; + + template + struct CacheProvider + { + ProviderId id; + T source; + CacheType cache_type; + + [[nodiscard]] constexpr bool is_read() const noexcept + { + return cache_type == CacheType::Read || cache_type == CacheType::ReadWrite; + } + + [[nodiscard]] constexpr bool is_write() const noexcept + { + return cache_type == CacheType::Write || cache_type == CacheType::ReadWrite; + } + }; + + template + using ProviderList = std::vector>; + struct BinaryConfigParserState { + ProviderId provider_count = 0; bool nuget_interactive = false; std::set binary_cache_providers; std::string nugettimeout = "100"; - std::vector archives_to_read; - std::vector archives_to_write; - - std::vector url_templates_to_get; - std::vector url_templates_to_put; - - std::vector gcs_read_prefixes; - std::vector gcs_write_prefixes; - - std::vector aws_read_prefixes; - std::vector aws_write_prefixes; + ProviderList archives; + ProviderList url_templates; + ProviderList gcs_prefixes; + ProviderList aws_prefixes; bool aws_no_sign_request = false; - - std::vector cos_read_prefixes; - std::vector cos_write_prefixes; - - bool gha_write = false; - bool gha_read = false; - - std::vector sources_to_read; - std::vector sources_to_write; - - std::vector configs_to_read; - std::vector configs_to_write; + ProviderList cos_prefixes; + Optional> gha; + ProviderList sources; + ProviderList configs; std::vector secrets; @@ -173,6 +211,8 @@ namespace vcpkg NuGetRepoInfo nuget_repo; }; + [[nodiscard]] bool HasWriteOnlyProviders(const BinaryProviders& providers); + struct ReadOnlyBinaryCache { ReadOnlyBinaryCache() = default; @@ -182,7 +222,7 @@ namespace vcpkg /// executing `actions`. void fetch(View actions); - bool is_restored(const InstallPlanAction& ipa) const; + Optional cache_status(const InstallPlanAction& ipa) const; /// Checks whether the `actions` are present in the cache, without restoring them. Used by CI to determine /// missing packages. @@ -192,6 +232,10 @@ namespace vcpkg protected: BinaryProviders m_config; + /// Flag to indicate that at least one provider is write-only. This implies that the write-back phase should + /// always take place for every Cache item. + bool m_has_write_only_providers; + std::unordered_map m_status; }; diff --git a/src/vcpkg-test/binarycaching.cpp b/src/vcpkg-test/binarycaching.cpp index 64800c976a..4d422756bf 100644 --- a/src/vcpkg-test/binarycaching.cpp +++ b/src/vcpkg-test/binarycaching.cpp @@ -23,6 +23,9 @@ struct KnowNothingBinaryProvider : IReadBinaryProvider CHECK(out_status[idx] == RestoreResult::unavailable); } } + + bool can_precheck() const override { return true; } + void precheck(View actions, Span out_status) const override { REQUIRE(actions.size() == out_status.size()); @@ -36,6 +39,8 @@ struct KnowNothingBinaryProvider : IReadBinaryProvider { return LocalizedString::from_raw("Nothing"); } + + ProviderId id() const override { return 1; } }; TEST_CASE ("CacheStatus operations", "[BinaryCache]") diff --git a/src/vcpkg-test/configparser.cpp b/src/vcpkg-test/configparser.cpp index 05cc8470ee..29f11055bf 100644 --- a/src/vcpkg-test/configparser.cpp +++ b/src/vcpkg-test/configparser.cpp @@ -14,19 +14,39 @@ using namespace vcpkg; namespace { + template + constexpr size_t count_read_providers(const ProviderList& providers) + { + return std::count_if( + providers.cbegin(), providers.cend(), [](const CacheProvider& provider) { return provider.is_read(); }); + } + + template + constexpr size_t count_write_providers(const ProviderList& providers) + { + return std::count_if( + providers.cbegin(), providers.cend(), [](const CacheProvider& provider) { return provider.is_write(); }); + } + + template + constexpr bool has_provider_source(const ProviderList& providers, F&& f) + { + return providers.cend() != std::find_if(providers.cbegin(), providers.cend(), f); + } + void validate_readonly_url(const BinaryConfigParserState& state, StringView url) { auto extended_url = url.to_string() + "/{sha}.zip?sas"; - CHECK(state.url_templates_to_put.empty()); - CHECK(state.url_templates_to_get.size() == 1); - CHECK(state.url_templates_to_get.front().url_template == extended_url); + CHECK(count_write_providers(state.url_templates) == 0); + CHECK(count_read_providers(state.url_templates) == 1); + CHECK(state.url_templates.front().source.url_template == extended_url); } void validate_readonly_sources(const BinaryConfigParserState& state, StringView sources) { - CHECK(state.sources_to_write.empty()); - CHECK(state.sources_to_read.size() == 1); - CHECK(state.sources_to_read.front() == sources); + CHECK(count_write_providers(state.sources) == 0); + CHECK(count_read_providers(state.sources) == 1); + CHECK(state.sources.front().source == sources); } } @@ -71,23 +91,25 @@ TEST_CASE ("BinaryConfigParser files provider", "[binaryconfigparser]") auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"files"}}); - REQUIRE(!state.archives_to_read.empty()); - REQUIRE(!Util::Vectors::contains(state.archives_to_write, ABSOLUTE_PATH)); + REQUIRE(count_read_providers(state.archives) > 0); + REQUIRE(!has_provider_source(state.archives, [&](const CacheProvider& provider) { + return provider.is_write() && provider.source == ABSOLUTE_PATH; + })); } { auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH ",write", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"files"}}); - REQUIRE(!state.archives_to_write.empty()); + REQUIRE(count_write_providers(state.archives) > 0); } { auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH ",readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"files"}}); - REQUIRE(!state.archives_to_write.empty()); - REQUIRE(!state.archives_to_read.empty()); + REQUIRE(count_read_providers(state.archives) > 0); + REQUIRE(count_write_providers(state.archives) > 0); } { auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH ",readwrite,extra", {}); @@ -134,13 +156,14 @@ TEST_CASE ("BinaryConfigParser nuget source provider", "[binaryconfigparser]") auto parsed = parse_binary_provider_configs("nuget," ABSOLUTE_PATH ",readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - CHECK(state.sources_to_read.size() == 1); - CHECK(state.sources_to_write.size() == 1); - CHECK(state.sources_to_read.front() == state.sources_to_write.front()); - CHECK(state.sources_to_read.front() == ABSOLUTE_PATH); + CHECK(count_read_providers(state.sources) == 1); + CHECK(count_write_providers(state.sources) == 1); + CHECK(state.sources.front().cache_type == CacheType::ReadWrite); + CHECK(state.sources.front().source == ABSOLUTE_PATH); + CHECK(state.sources.size() == 1); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"nuget"}}); - REQUIRE(!state.archives_to_write.empty()); - REQUIRE(!state.archives_to_read.empty()); + REQUIRE(count_read_providers(state.archives) > 0); + REQUIRE(count_write_providers(state.archives) > 0); } { auto parsed = parse_binary_provider_configs("nuget," ABSOLUTE_PATH ",readwrite,extra", {}); @@ -217,33 +240,33 @@ TEST_CASE ("BinaryConfigParser nuget config provider", "[binaryconfigparser]") auto parsed = parse_binary_provider_configs("nugetconfig," ABSOLUTE_PATH ",read", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - CHECK(state.configs_to_write.empty()); - CHECK(state.configs_to_read.size() == 1); - CHECK(state.configs_to_read.front() == ABSOLUTE_PATH); + CHECK(count_read_providers(state.configs) == 1); + CHECK(count_write_providers(state.configs) == 0); + CHECK(state.configs.front().source == ABSOLUTE_PATH); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"nuget"}}); - REQUIRE(!state.archives_to_read.empty()); + REQUIRE(count_read_providers(state.archives) > 0); } { auto parsed = parse_binary_provider_configs("nugetconfig," ABSOLUTE_PATH ",write", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - CHECK(state.configs_to_read.empty()); - CHECK(state.configs_to_write.size() == 1); - CHECK(state.configs_to_write.front() == ABSOLUTE_PATH); + CHECK(count_read_providers(state.configs) == 0); + CHECK(count_write_providers(state.configs) == 1); + CHECK(state.configs.front().source == ABSOLUTE_PATH); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"nuget"}}); - REQUIRE(!state.archives_to_write.empty()); + REQUIRE(count_write_providers(state.archives) > 0); } { auto parsed = parse_binary_provider_configs("nugetconfig," ABSOLUTE_PATH ",readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - CHECK(state.configs_to_read.size() == 1); - CHECK(state.configs_to_write.size() == 1); - CHECK(state.configs_to_read.front() == state.configs_to_write.front()); - CHECK(state.configs_to_read.front() == ABSOLUTE_PATH); + CHECK(count_read_providers(state.configs) == 1); + CHECK(count_write_providers(state.configs) == 1); + CHECK(state.configs.front().cache_type == CacheType::ReadWrite); + CHECK(state.configs.front().source == ABSOLUTE_PATH); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"nuget"}}); - REQUIRE(!state.archives_to_write.empty()); - REQUIRE(!state.archives_to_read.empty()); + REQUIRE(count_read_providers(state.archives) > 0); + REQUIRE(count_write_providers(state.archives) > 0); } { auto parsed = parse_binary_provider_configs("nugetconfig," ABSOLUTE_PATH ",readwrite,extra", {}); @@ -268,18 +291,18 @@ TEST_CASE ("BinaryConfigParser default provider", "[binaryconfigparser]") { auto parsed = parse_binary_provider_configs("default,read", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - REQUIRE(!state.archives_to_read.empty()); + REQUIRE(count_read_providers(state.archives) > 0); } { auto parsed = parse_binary_provider_configs("default,readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - REQUIRE(!state.archives_to_read.empty()); - REQUIRE(!state.archives_to_write.empty()); + REQUIRE(count_read_providers(state.archives) > 0); + REQUIRE(count_write_providers(state.archives) > 0); } { auto parsed = parse_binary_provider_configs("default,write", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - REQUIRE(!state.archives_to_write.empty()); + REQUIRE(count_write_providers(state.archives) > 0); } { auto parsed = parse_binary_provider_configs("default,read,extra", {}); @@ -451,31 +474,30 @@ TEST_CASE ("BinaryConfigParser azblob provider", "[binaryconfigparser]") REQUIRE(state.binary_cache_providers == std::set{{"azblob"}, {"default"}}); validate_readonly_url(state, "https://azure/container"); REQUIRE(state.secrets == std::vector{"sas"}); - REQUIRE(!state.archives_to_read.empty()); + REQUIRE(count_read_providers(state.archives) > 0); } { auto parsed = parse_binary_provider_configs("x-azblob,https://azure/container,sas,write", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"azblob"}, {"default"}}); - CHECK(state.url_templates_to_get.empty()); - CHECK(state.url_templates_to_put.size() == 1); - CHECK(state.url_templates_to_put.front().url_template == "https://azure/container/{sha}.zip?sas"); + CHECK(count_read_providers(state.url_templates) == 0); + CHECK(count_write_providers(state.url_templates) == 1); + CHECK(state.url_templates.front().source.url_template == "https://azure/container/{sha}.zip?sas"); REQUIRE(state.secrets == std::vector{"sas"}); - REQUIRE(!state.archives_to_write.empty()); + REQUIRE(count_write_providers(state.archives) > 0); } { auto parsed = parse_binary_provider_configs("x-azblob,https://azure/container,sas,readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"azblob"}, {"default"}}); - CHECK(state.url_templates_to_get.size() == 1); - CHECK(state.url_templates_to_get.front().url_template == "https://azure/container/{sha}.zip?sas"); - CHECK(state.url_templates_to_put.size() == 1); - CHECK(state.url_templates_to_put.front().url_template == "https://azure/container/{sha}.zip?sas"); + CHECK(count_read_providers(state.url_templates) == 1); + CHECK(count_write_providers(state.url_templates) == 1); + CHECK(state.url_templates.front().source.url_template == "https://azure/container/{sha}.zip?sas"); REQUIRE(state.secrets == std::vector{"sas"}); - REQUIRE(!state.archives_to_read.empty()); - REQUIRE(!state.archives_to_write.empty()); + REQUIRE(count_read_providers(state.archives) > 0); + REQUIRE(count_write_providers(state.archives) > 0); } } @@ -485,14 +507,16 @@ TEST_CASE ("BinaryConfigParser GCS provider", "[binaryconfigparser]") auto parsed = parse_binary_provider_configs("x-gcs,gs://my-bucket/", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - REQUIRE(state.gcs_read_prefixes == std::vector{"gs://my-bucket/"}); + REQUIRE(count_read_providers(state.gcs_prefixes) == 1); + REQUIRE(state.gcs_prefixes.front().source == "gs://my-bucket/"); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"gcs"}}); } { auto parsed = parse_binary_provider_configs("x-gcs,gs://my-bucket/my-folder", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - REQUIRE(state.gcs_read_prefixes == std::vector{"gs://my-bucket/my-folder/"}); + REQUIRE(count_read_providers(state.gcs_prefixes) == 1); + REQUIRE(state.gcs_prefixes.front().source == "gs://my-bucket/my-folder/"); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"gcs"}}); } { @@ -508,26 +532,30 @@ TEST_CASE ("BinaryConfigParser GCS provider", "[binaryconfigparser]") auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"gcs"}}); - REQUIRE(state.gcs_read_prefixes == std::vector{"gs://my-bucket/my-folder/"}); - REQUIRE(!state.archives_to_read.empty()); + REQUIRE(count_read_providers(state.gcs_prefixes) == 1); + REQUIRE(state.gcs_prefixes.front().source == "gs://my-bucket/my-folder/"); + REQUIRE(count_read_providers(state.archives) > 0); } { auto parsed = parse_binary_provider_configs("x-gcs,gs://my-bucket/my-folder,write", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"gcs"}}); - REQUIRE(state.gcs_write_prefixes == std::vector{"gs://my-bucket/my-folder/"}); - REQUIRE(!state.archives_to_write.empty()); + REQUIRE(count_write_providers(state.gcs_prefixes) == 1); + REQUIRE(state.gcs_prefixes.front().source == "gs://my-bucket/my-folder/"); + REQUIRE(count_write_providers(state.archives) > 0); } { auto parsed = parse_binary_provider_configs("x-gcs,gs://my-bucket/my-folder,readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"gcs"}}); - REQUIRE(state.gcs_write_prefixes == std::vector{"gs://my-bucket/my-folder/"}); - REQUIRE(state.gcs_read_prefixes == std::vector{"gs://my-bucket/my-folder/"}); - REQUIRE(!state.archives_to_write.empty()); - REQUIRE(!state.archives_to_read.empty()); + REQUIRE(count_read_providers(state.gcs_prefixes) == 1); + REQUIRE(count_write_providers(state.gcs_prefixes) == 1); + REQUIRE(state.gcs_prefixes.size() == 1); + REQUIRE(state.gcs_prefixes.front().source == "gs://my-bucket/my-folder/"); + REQUIRE(count_read_providers(state.archives) > 0); + REQUIRE(count_write_providers(state.archives) > 0); } } diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index ab1b2b2420..682c3f6c60 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -39,16 +39,14 @@ namespace void parse_segments(std::vector>& out_segments); std::vector>> parse_all_segments(); - template - void handle_readwrite(std::vector& read, - std::vector& write, - T&& t, - const std::vector>& segments, - size_t segment_idx) + template + void parse_cache_type(const std::vector>& segments, + const size_t segment_idx, + F&& f) { if (segment_idx >= segments.size()) { - read.push_back(std::move(t)); + f(CacheType::Read); return; } @@ -56,56 +54,65 @@ namespace if (mode == "read") { - read.push_back(std::move(t)); - } - else if (mode == "write") - { - write.push_back(std::move(t)); + return f(CacheType::Read); } - else if (mode == "readwrite") + if (mode == "write") { - read.push_back(t); - write.push_back(std::move(t)); + return f(CacheType::Write); } - else + if (mode == "readwrite") { - return add_error(msg::format(msgExpectedReadWriteReadWrite), segments[segment_idx].first); + return f(CacheType::ReadWrite); } + + return add_error(msg::format(msgExpectedReadWriteReadWrite), segments[segment_idx].first); } - void handle_readwrite(bool& read, - bool& write, + template + void handle_readwrite(std::vector& read, + std::vector& write, + T&& t, const std::vector>& segments, - size_t segment_idx) + const size_t segment_idx) { - if (segment_idx >= segments.size()) - { - read = true; - return; - } - - auto& mode = segments[segment_idx].second; + parse_cache_type(segments, segment_idx, [&](const CacheType cache_type) { + switch (cache_type) + { + case CacheType::Read: read.push_back(std::forward(t)); break; + case CacheType::Write: write.push_back(std::forward(t)); break; + case CacheType::ReadWrite: + read.push_back(t); + write.push_back(std::forward(t)); + break; + } + }); + } - if (mode == "read") - { - read = true; - } - else if (mode == "write") - { - write = true; - } - else if (mode == "readwrite") - { - read = true; - write = true; - } - else - { - return add_error(msg::format(msgExpectedReadWriteReadWrite), segments[segment_idx].first); - } + template + void handle_readwrite(ProviderList& providers, + ProviderId& next_provider_id, + T&& t, + const std::vector>& segments, + const size_t segment_idx) + { + parse_cache_type(segments, segment_idx, [&](const CacheType cache_type) { + providers.push_back(CacheProvider{next_provider_id++, std::forward(t), cache_type}); + }); } }; + template + constexpr bool is_read_provider(const CacheProvider& provider) + { + return provider.is_read(); + } + + template + constexpr bool is_write_provider(const CacheProvider& provider) + { + return provider.is_write(); + } + void ConfigSegmentsParser::parse_segments(std::vector>& segments) { for (;;) @@ -213,17 +220,26 @@ namespace struct FilesWriteBinaryProvider : IWriteBinaryProvider { - FilesWriteBinaryProvider(const Filesystem& fs, std::vector&& dirs) : m_fs(fs), m_dirs(std::move(dirs)) { } + FilesWriteBinaryProvider(const Filesystem& fs, ProviderList&& dirs) + : m_fs(fs), m_dirs(std::move(dirs)) { } - size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) override + size_t push_success(const BinaryPackageWriteInfo& request, + MessageSink& msg_sink, + CacheStatus& cache_status) override { const auto& zip_path = request.zip_path.value_or_exit(VCPKG_LINE_INFO); const auto archive_subpath = files_archive_subpath(request.package_abi); size_t count_stored = 0; - for (const auto& archives_root_dir : m_dirs) + for (const auto& [provider_id, source, cache_type] : m_dirs) { - const auto archive_path = archives_root_dir / archive_subpath; + if (cache_type == CacheType::ReadWrite && !cache_status.should_attempt_write_back(provider_id)) + { + /// We already have this in the cache, so skip it + continue; + } + + const auto archive_path = source / archive_subpath; std::error_code ec; m_fs.create_directories(archive_path.parent_path(), IgnoreErrors{}); m_fs.copy_file(zip_path, archive_path, CopyOptions::overwrite_existing, ec); @@ -237,6 +253,7 @@ namespace else { count_stored++; + cache_status.mark_written_back(provider_id); } } return count_stored; @@ -244,10 +261,14 @@ namespace bool needs_nuspec_data() const override { return false; } bool needs_zip_file() const override { return true; } + std::vector get_provider_ids() const override + { + return Util::fmap(m_dirs, [](const auto& p) { return p.id; }); + } private: const Filesystem& m_fs; - std::vector m_dirs; + ProviderList m_dirs; }; enum class RemoveWhen @@ -270,7 +291,10 @@ namespace // - IReadBinaryProvider::precheck() struct ZipReadBinaryProvider : IReadBinaryProvider { - ZipReadBinaryProvider(ZipTool zip, const Filesystem& fs) : m_zip(std::move(zip)), m_fs(fs) { } + ZipReadBinaryProvider(ProviderId provider_id, ZipTool zip, const Filesystem& fs) + : m_provider_id(provider_id), m_zip(std::move(zip)), m_fs(fs) + { + } void fetch(View actions, Span out_status) const override { @@ -324,15 +348,18 @@ namespace virtual void acquire_zips(View actions, Span> out_zips) const = 0; + ProviderId id() const override { return m_provider_id; } + protected: + ProviderId m_provider_id; ZipTool m_zip; const Filesystem& m_fs; }; struct FilesReadBinaryProvider : ZipReadBinaryProvider { - FilesReadBinaryProvider(ZipTool zip, const Filesystem& fs, Path&& dir) - : ZipReadBinaryProvider(std::move(zip), fs), m_dir(std::move(dir)) + FilesReadBinaryProvider(ProviderId provider_id, ZipTool zip, const Filesystem& fs, Path&& dir) + : ZipReadBinaryProvider(provider_id, std::move(zip), fs), m_dir(std::move(dir)) { } @@ -350,6 +377,8 @@ namespace } } + bool can_precheck() const override { return true; } + void precheck(View actions, Span cache_status) const override { for (size_t idx = 0; idx < actions.size(); ++idx) @@ -382,23 +411,34 @@ namespace struct HTTPPutBinaryProvider : IWriteBinaryProvider { HTTPPutBinaryProvider(const Filesystem& fs, - std::vector&& urls, + ProviderList&& urls, const std::vector& secrets) : m_fs(fs), m_urls(std::move(urls)), m_secrets(secrets) { } - size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) override + size_t push_success(const BinaryPackageWriteInfo& request, + MessageSink& msg_sink, + CacheStatus& cache_status) override { if (!request.zip_path) return 0; const auto& zip_path = *request.zip_path.get(); size_t count_stored = 0; - for (auto&& templ : m_urls) + for (auto&& [provider_id, templ, cache_type] : m_urls) { + if (cache_type == CacheType::ReadWrite && !cache_status.should_attempt_write_back(provider_id)) + { + /// We already have this in the cache, so skip it + continue; + } + auto url = templ.instantiate_variables(request); auto maybe_success = put_file(m_fs, url, m_secrets, templ.headers, zip_path); if (maybe_success) + { count_stored++; + cache_status.mark_written_back(provider_id); + } else msg_sink.println(Color::warning, maybe_success.error()); } @@ -407,21 +447,26 @@ namespace bool needs_nuspec_data() const override { return false; } bool needs_zip_file() const override { return true; } + std::vector get_provider_ids() const override + { + return Util::fmap(m_urls, [](const auto& p) { return p.id; }); + } private: const Filesystem& m_fs; - std::vector m_urls; + ProviderList m_urls; std::vector m_secrets; }; struct HttpGetBinaryProvider : ZipReadBinaryProvider { - HttpGetBinaryProvider(ZipTool zip, + HttpGetBinaryProvider(ProviderId provider_id, + ZipTool zip, const Filesystem& fs, const Path& buildtrees, UrlTemplate&& url_template, const std::vector& secrets) - : ZipReadBinaryProvider(std::move(zip), fs) + : ZipReadBinaryProvider(provider_id, std::move(zip), fs) , m_buildtrees(buildtrees) , m_url_template(std::move(url_template)) , m_secrets(secrets) @@ -450,6 +495,8 @@ namespace } } + bool can_precheck() const override { return true; } + void precheck(View actions, Span out_status) const override { std::vector urls; @@ -482,7 +529,7 @@ namespace std::string value; }; - NuGetSource nuget_sources_arg(View sources) { return {"-Source", Strings::join(";", sources)}; } + NuGetSource nuget_source_arg(const std::string& source) { return {"-Source", source}; } NuGetSource nuget_configfile_arg(const Path& config_path) { return {"-ConfigFile", config_path.native()}; } struct NuGetTool @@ -643,11 +690,12 @@ namespace struct NugetReadBinaryProvider : IReadBinaryProvider, private NugetBaseBinaryProvider { - NugetReadBinaryProvider(const NugetBaseBinaryProvider& base, NuGetSource src) - : NugetBaseBinaryProvider(base), m_src(std::move(src)) + NugetReadBinaryProvider(ProviderId provider_id, const NugetBaseBinaryProvider& base, NuGetSource src) + : NugetBaseBinaryProvider(base), m_provider_id(provider_id), m_src(std::move(src)) { } + ProviderId m_provider_id; NuGetSource m_src; static std::string generate_packages_config(View refs) @@ -669,6 +717,9 @@ namespace return std::move(xml.buf); } + // Prechecking is too expensive with NuGet, so it is not implemented + bool can_precheck() const override { return false; } + // Prechecking is too expensive with NuGet, so it is not implemented void precheck(View, Span) const override { } @@ -704,24 +755,41 @@ namespace } } } + + ProviderId id() const override { return m_provider_id; } }; struct NugetBinaryPushProvider : IWriteBinaryProvider, private NugetBaseBinaryProvider { NugetBinaryPushProvider(const NugetBaseBinaryProvider& base, - std::vector&& sources, - std::vector&& configs) + ProviderList&& sources, + ProviderList&& configs) : NugetBaseBinaryProvider(base), m_sources(std::move(sources)), m_configs(std::move(configs)) { } - std::vector m_sources; - std::vector m_configs; + ProviderList m_sources; + ProviderList m_configs; bool needs_nuspec_data() const override { return true; } bool needs_zip_file() const override { return false; } + std::vector get_provider_ids() const override + { + std::vector ret; + for (auto&& [provider_id, source, cache_type] : m_sources) + { + ret.push_back(provider_id); + } + for (auto&& [provider_id, source, cache_type] : m_configs) + { + ret.push_back(provider_id); + } + return ret; + } - size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) override + size_t push_success(const BinaryPackageWriteInfo& request, + MessageSink& msg_sink, + CacheStatus& cache_status) override { auto& spec = request.spec; @@ -744,33 +812,44 @@ namespace size_t count_stored = 0; auto nupkg_path = m_buildtrees / make_nugetref(request, m_nuget_prefix).nupkg_filename(); - for (auto&& write_src : m_sources) + for (auto&& [provider_id, source, cache_type] : m_sources) { + if (cache_type == CacheType::ReadWrite && !cache_status.should_attempt_write_back(provider_id)) + { + /// We already have this in the cache, so skip it + continue; + } + msg_sink.println( - msgUploadingBinariesToVendor, msg::spec = spec, msg::vendor = "NuGet", msg::path = write_src); - if (!m_cmd.push(msg_sink, nupkg_path, nuget_sources_arg({&write_src, 1}))) + msgUploadingBinariesToVendor, msg::spec = spec, msg::vendor = "NuGet", msg::path = source); + if (!m_cmd.push(msg_sink, nupkg_path, nuget_source_arg(source))) { - msg_sink.println( - Color::error, msgPushingVendorFailed, msg::vendor = "NuGet", msg::path = write_src); + msg_sink.println(Color::error, msgPushingVendorFailed, msg::vendor = "NuGet", msg::path = source); } else { + cache_status.mark_written_back(provider_id); count_stored++; } } - for (auto&& write_cfg : m_configs) + for (auto&& [provider_id, source, cache_type] : m_configs) { - msg_sink.println(msgUploadingBinariesToVendor, - msg::spec = spec, - msg::vendor = "NuGet config", - msg::path = write_cfg); - if (!m_cmd.push(msg_sink, nupkg_path, nuget_configfile_arg(write_cfg))) + if (cache_type == CacheType::ReadWrite && !cache_status.should_attempt_write_back(provider_id)) + { + /// We already have this in the cache, so skip it + continue; + } + + msg_sink.println( + msgUploadingBinariesToVendor, msg::spec = spec, msg::vendor = "NuGet config", msg::path = source); + if (!m_cmd.push(msg_sink, nupkg_path, nuget_configfile_arg(source))) { msg_sink.println( - Color::error, msgPushingVendorFailed, msg::vendor = "NuGet config", msg::path = write_cfg); + Color::error, msgPushingVendorFailed, msg::vendor = "NuGet config", msg::path = source); } else { + cache_status.mark_written_back(provider_id); count_stored++; } } @@ -782,9 +861,13 @@ namespace struct GHABinaryProvider : ZipReadBinaryProvider { - GHABinaryProvider( - ZipTool zip, const Filesystem& fs, const Path& buildtrees, const std::string& url, const std::string& token) - : ZipReadBinaryProvider(std::move(zip), fs) + GHABinaryProvider(ProviderId provider_id, + ZipTool zip, + const Filesystem& fs, + const Path& buildtrees, + const std::string& url, + const std::string& token) + : ZipReadBinaryProvider(provider_id, std::move(zip), fs) , m_buildtrees(buildtrees) , m_url(url + "_apis/artifactcache/cache") , m_secrets() @@ -844,6 +927,8 @@ namespace } } + bool can_precheck() const override { return true; } + void precheck(View, Span) const override { } LocalizedString restored_message(size_t count, @@ -862,8 +947,16 @@ namespace struct GHABinaryPushProvider : IWriteBinaryProvider { - GHABinaryPushProvider(const Filesystem& fs, const std::string& url, const std::string& token) - : m_fs(fs), m_url(url + "_apis/artifactcache/caches"), m_token_header("Authorization: Bearer " + token) + GHABinaryPushProvider(ProviderId provider_id, + CacheType cache_type, + const Filesystem& fs, + const std::string& url, + const std::string& token) + : m_provider_id(provider_id) + , m_cache_type(cache_type) + , m_fs(fs) + , m_url(url + "_apis/artifactcache/caches") + , m_token_header("Authorization: Bearer " + token) { } @@ -896,10 +989,16 @@ namespace return {}; } - size_t push_success(const BinaryPackageWriteInfo& request, MessageSink&) override + size_t push_success(const BinaryPackageWriteInfo& request, MessageSink&, CacheStatus& cache_status) override { if (!request.zip_path) return 0; + if (m_cache_type == CacheType::ReadWrite && !cache_status.should_attempt_write_back(m_provider_id)) + { + /// We already have this in the cache, so skip it + return 0; + } + const auto& zip_path = *request.zip_path.get(); const ElapsedTimer timer; const auto& abi = request.package_abi; @@ -931,6 +1030,7 @@ namespace if (res) { ++upload_count; + cache_status.mark_written_back(m_provider_id); } else { @@ -943,7 +1043,10 @@ namespace bool needs_nuspec_data() const override { return false; } bool needs_zip_file() const override { return true; } + std::vector get_provider_ids() const override { return {m_provider_id}; } + ProviderId m_provider_id; + CacheType m_cache_type; const Filesystem& m_fs; std::string m_url; std::string m_token_header; @@ -964,12 +1067,13 @@ namespace struct ObjectStorageProvider : ZipReadBinaryProvider { - ObjectStorageProvider(ZipTool zip, + ObjectStorageProvider(ProviderId provider_id, + ZipTool zip, const Filesystem& fs, const Path& buildtrees, std::string&& prefix, const std::shared_ptr& tool) - : ZipReadBinaryProvider(std::move(zip), fs) + : ZipReadBinaryProvider(provider_id, std::move(zip), fs) , m_buildtrees(buildtrees) , m_prefix(std::move(prefix)) , m_tool(tool) @@ -1001,6 +1105,8 @@ namespace } } + bool can_precheck() const override { return true; } + void precheck(View actions, Span cache_status) const override { for (size_t idx = 0; idx < actions.size(); ++idx) @@ -1030,7 +1136,7 @@ namespace }; struct ObjectStoragePushProvider : IWriteBinaryProvider { - ObjectStoragePushProvider(std::vector&& prefixes, std::shared_ptr tool) + ObjectStoragePushProvider(ProviderList&& prefixes, std::shared_ptr tool) : m_prefixes(std::move(prefixes)), m_tool(std::move(tool)) { } @@ -1040,17 +1146,26 @@ namespace return Strings::concat(prefix, abi, ".zip"); } - size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) override + size_t push_success(const BinaryPackageWriteInfo& request, + MessageSink& msg_sink, + CacheStatus& cache_status) override { if (!request.zip_path) return 0; const auto& zip_path = *request.zip_path.get(); size_t upload_count = 0; - for (const auto& prefix : m_prefixes) + for (const auto& [provider_id, source, cache_type] : m_prefixes) { - auto res = m_tool->upload_file(make_object_path(prefix, request.package_abi), zip_path); + if (cache_type == CacheType::ReadWrite && !cache_status.should_attempt_write_back(provider_id)) + { + /// We already have this in the cache, so skip it + continue; + } + + auto res = m_tool->upload_file(make_object_path(source, request.package_abi), zip_path); if (res) { ++upload_count; + cache_status.mark_written_back(provider_id); } else { @@ -1062,8 +1177,12 @@ namespace bool needs_nuspec_data() const override { return false; } bool needs_zip_file() const override { return true; } + std::vector get_provider_ids() const override + { + return Util::fmap(m_prefixes, [](const auto& p) { return p.id; }); + } - std::vector m_prefixes; + ProviderList m_prefixes; std::shared_ptr m_tool; }; @@ -1274,7 +1393,7 @@ namespace segments[1].first); } - handle_readwrite(state->archives_to_read, state->archives_to_write, std::move(p), segments, 2); + handle_readwrite(state->archives, state->provider_count, std::move(p), segments, 2); if (segments.size() > 3) { return add_error( @@ -1311,7 +1430,7 @@ namespace segments[1].first); } - handle_readwrite(state->configs_to_read, state->configs_to_write, std::move(p), segments, 2); + handle_readwrite(state->configs, state->provider_count, std::move(p), segments, 2); if (segments.size() > 3) { return add_error( @@ -1336,7 +1455,7 @@ namespace msg::format(msgInvalidArgumentRequiresSourceArgument, msg::binary_source = "nuget")); } - handle_readwrite(state->sources_to_read, state->sources_to_write, std::move(p), segments, 2); + handle_readwrite(state->sources, state->provider_count, std::move(p), segments, 2); if (segments.size() > 3) { return add_error( @@ -1376,9 +1495,16 @@ namespace return add_error(LocalizedString{maybe_home.error()}, segments[0].first); } - handle_readwrite( - state->archives_to_read, state->archives_to_write, Path(*maybe_home.get()), segments, 1); - state->binary_cache_providers.insert("default"); + parse_cache_type(segments, 1, [&](const CacheType cache_type) { + if (Util::Sets::contains(state->binary_cache_providers, "default")) + { + return; + } + + state->archives.push_back( + CacheProvider{state->provider_count++, Path(*maybe_home.get()), cache_type}); + state->binary_cache_providers.insert("default"); + }); } else if (segments[0].second == "x-azblob") { @@ -1426,14 +1552,14 @@ namespace p.append(segments[2].second); state->secrets.push_back(segments[2].second); UrlTemplate url_template = {p}; - bool read = false, write = false; - handle_readwrite(read, write, segments, 3); - if (read) state->url_templates_to_get.push_back(url_template); - auto headers = azure_blob_headers(); - url_template.headers.assign(headers.begin(), headers.end()); - if (write) state->url_templates_to_put.push_back(url_template); + parse_cache_type(segments, 3, [&](const CacheType url_cache_type) { + auto headers = azure_blob_headers(); + url_template.headers.assign(headers.begin(), headers.end()); + state->url_templates.push_back( + CacheProvider{state->provider_count, url_template, url_cache_type}); - state->binary_cache_providers.insert("azblob"); + state->binary_cache_providers.insert("azblob"); + }); } else if (segments[0].second == "x-gcs") { @@ -1465,7 +1591,7 @@ namespace p.push_back('/'); } - handle_readwrite(state->gcs_read_prefixes, state->gcs_write_prefixes, std::move(p), segments, 2); + handle_readwrite(state->gcs_prefixes, state->provider_count, std::move(p), segments, 2); state->binary_cache_providers.insert("gcs"); } @@ -1499,7 +1625,7 @@ namespace p.push_back('/'); } - handle_readwrite(state->aws_read_prefixes, state->aws_write_prefixes, std::move(p), segments, 2); + handle_readwrite(state->aws_prefixes, state->provider_count, std::move(p), segments, 2); state->binary_cache_providers.insert("aws"); } @@ -1554,7 +1680,7 @@ namespace p.push_back('/'); } - handle_readwrite(state->cos_read_prefixes, state->cos_write_prefixes, std::move(p), segments, 2); + handle_readwrite(state->cos_prefixes, state->provider_count, std::move(p), segments, 2); state->binary_cache_providers.insert("cos"); } else if (segments[0].second == "x-gha") @@ -1567,9 +1693,11 @@ namespace segments[2].first); } - handle_readwrite(state->gha_read, state->gha_write, segments, 1); - - state->binary_cache_providers.insert("gha"); + parse_cache_type(segments, 1, [&](const CacheType gha_cache_type) { + state->gha.emplace( + CacheProvider{state->provider_count++, GithubActionsInfo{}, gha_cache_type}); + state->binary_cache_providers.insert("gha"); + }); } else if (segments[0].second == "http") { @@ -1606,8 +1734,7 @@ namespace url_template.headers.push_back(segments[3].second); } - handle_readwrite( - state->url_templates_to_get, state->url_templates_to_put, std::move(url_template), segments, 2); + handle_readwrite(state->url_templates, state->provider_count, std::move(url_template), segments, 2); state->binary_cache_providers.insert("http"); } else @@ -1892,30 +2019,30 @@ namespace vcpkg ret.nuget_prefix = s.nuget_prefix; std::shared_ptr gcs_tool; - if (!s.gcs_read_prefixes.empty() || !s.gcs_write_prefixes.empty()) + if (!s.gcs_prefixes.empty()) { gcs_tool = std::make_shared(tools, out_sink); } std::shared_ptr aws_tool; - if (!s.aws_read_prefixes.empty() || !s.aws_write_prefixes.empty()) + if (!s.aws_prefixes.empty()) { aws_tool = std::make_shared(tools, out_sink, s.aws_no_sign_request); } std::shared_ptr cos_tool; - if (!s.cos_read_prefixes.empty() || !s.cos_write_prefixes.empty()) + if (!s.cos_prefixes.empty()) { cos_tool = std::make_shared(tools, out_sink); } - if (s.gha_read || s.gha_write) + if (s.gha.has_value()) { if (!args.actions_cache_url.has_value() || !args.actions_runtime_token.has_value()) return msg::format_error(msgGHAParametersMissing, msg::url = "https://learn.microsoft.com/vcpkg/users/binarycaching#gha"); } - if (!s.archives_to_read.empty() || !s.url_templates_to_get.empty() || !s.gcs_read_prefixes.empty() || - !s.aws_read_prefixes.empty() || !s.cos_read_prefixes.empty() || s.gha_read) + if (!s.archives.empty() || !s.url_templates.empty() || !s.gcs_prefixes.empty() || !s.aws_prefixes.empty() || + !s.cos_prefixes.empty() || s.gha.has_value()) { auto maybe_zip_tool = ZipTool::make(tools, out_sink); if (!maybe_zip_tool.has_value()) @@ -1924,134 +2051,214 @@ namespace vcpkg } const auto& zip_tool = *maybe_zip_tool.get(); - for (auto&& dir : s.archives_to_read) + for (auto dir : s.archives) { - ret.read.push_back(std::make_unique(zip_tool, fs, std::move(dir))); + if (dir.is_read()) + { + ret.read.push_back( + std::make_unique(dir.id, zip_tool, fs, std::move(dir.source))); + } + } + auto write_dirs = Util::Vectors::filtered_copy(s.archives, is_write_provider); + if (!write_dirs.empty()) + { + ret.write.push_back(std::make_unique(fs, std::move(write_dirs))); } - for (auto&& url : s.url_templates_to_get) + for (auto url : s.url_templates) { - ret.read.push_back( - std::make_unique(zip_tool, fs, buildtrees, std::move(url), s.secrets)); + if (url.cache_type == CacheType::Read || url.cache_type == CacheType::ReadWrite) + { + ret.read.push_back(std::make_unique( + url.id, zip_tool, fs, buildtrees, std::move(url.source), s.secrets)); + } + } + auto write_url_templates = + Util::Vectors::filtered_copy(s.url_templates, is_write_provider); + if (!write_url_templates.empty()) + { + ret.write.push_back( + std::make_unique(fs, std::move(write_url_templates), s.secrets)); } - for (auto&& prefix : s.gcs_read_prefixes) + for (auto prefix : s.gcs_prefixes) + { + if (prefix.cache_type == CacheType::Read || prefix.cache_type == CacheType::ReadWrite) + { + ret.read.push_back(std::make_unique( + prefix.id, zip_tool, fs, buildtrees, std::move(prefix.source), gcs_tool)); + } + } + auto gcs_write_prefixes = Util::Vectors::filtered_copy(s.gcs_prefixes, is_write_provider); + if (!gcs_write_prefixes.empty()) { - ret.read.push_back( - std::make_unique(zip_tool, fs, buildtrees, std::move(prefix), gcs_tool)); + ret.write.push_back( + std::make_unique(std::move(gcs_write_prefixes), gcs_tool)); } - for (auto&& prefix : s.aws_read_prefixes) + for (auto prefix : s.aws_prefixes) { - ret.read.push_back( - std::make_unique(zip_tool, fs, buildtrees, std::move(prefix), aws_tool)); + if (prefix.cache_type == CacheType::Read || prefix.cache_type == CacheType::ReadWrite) + { + ret.read.push_back(std::make_unique( + prefix.id, zip_tool, fs, buildtrees, std::move(prefix.source), aws_tool)); + } + } + auto aws_write_prefixes = Util::Vectors::filtered_copy(s.aws_prefixes, is_write_provider); + if (!aws_write_prefixes.empty()) + { + ret.write.push_back( + std::make_unique(std::move(aws_write_prefixes), aws_tool)); } - for (auto&& prefix : s.cos_read_prefixes) + for (auto prefix : s.cos_prefixes) + { + ret.read.push_back(std::make_unique( + prefix.id, zip_tool, fs, buildtrees, std::move(prefix.source), cos_tool)); + } + auto cos_write_prefixes = Util::Vectors::filtered_copy(s.cos_prefixes, is_write_provider); + if (!cos_write_prefixes.empty()) { - ret.read.push_back( - std::make_unique(zip_tool, fs, buildtrees, std::move(prefix), cos_tool)); + ret.write.push_back( + std::make_unique(std::move(cos_write_prefixes), cos_tool)); } - if (s.gha_read) + if (s.gha.has_value()) { - const auto& url = *args.actions_cache_url.get(); - const auto& token = *args.actions_runtime_token.get(); - ret.read.push_back(std::make_unique(zip_tool, fs, buildtrees, url, token)); + const CacheProvider& gha = *s.gha.get(); + if (gha.is_read()) + { + const auto& url = *args.actions_cache_url.get(); + const auto& token = *args.actions_runtime_token.get(); + ret.read.push_back( + std::make_unique(gha.id, zip_tool, fs, buildtrees, url, token)); + } + if (gha.is_write()) + { + const auto& url = *args.actions_cache_url.get(); + const auto& token = *args.actions_runtime_token.get(); + ret.write.push_back( + std::make_unique(gha.id, gha.cache_type, fs, url, token)); + } } } - if (!s.archives_to_write.empty()) - { - ret.write.push_back(std::make_unique(fs, std::move(s.archives_to_write))); - } - if (!s.url_templates_to_put.empty()) - { - ret.write.push_back( - std::make_unique(fs, std::move(s.url_templates_to_put), s.secrets)); - } - if (!s.gcs_write_prefixes.empty()) - { - ret.write.push_back( - std::make_unique(std::move(s.gcs_write_prefixes), gcs_tool)); - } - if (!s.aws_write_prefixes.empty()) - { - ret.write.push_back( - std::make_unique(std::move(s.aws_write_prefixes), aws_tool)); - } - if (!s.cos_write_prefixes.empty()) - { - ret.write.push_back( - std::make_unique(std::move(s.cos_write_prefixes), cos_tool)); - } - if (s.gha_write) - { - const auto& url = *args.actions_cache_url.get(); - const auto& token = *args.actions_runtime_token.get(); - ret.write.push_back(std::make_unique(fs, url, token)); - } - if (!s.sources_to_read.empty() || !s.configs_to_read.empty() || !s.sources_to_write.empty() || - !s.configs_to_write.empty()) + if (!s.sources.empty() || !s.configs.empty()) { NugetBaseBinaryProvider nuget_base( fs, NuGetTool(tools, out_sink, s), paths.packages(), buildtrees, s.nuget_prefix); - if (!s.sources_to_read.empty()) - ret.read.push_back( - std::make_unique(nuget_base, nuget_sources_arg(s.sources_to_read))); - for (auto&& config : s.configs_to_read) - ret.read.push_back( - std::make_unique(nuget_base, nuget_configfile_arg(config))); - if (!s.sources_to_write.empty() || !s.configs_to_write.empty()) + + auto sources_to_read = Util::Vectors::filtered_copy(s.sources, is_read_provider); + for (auto&& source : sources_to_read) + { + ret.read.push_back(std::make_unique( + source.id, nuget_base, nuget_source_arg(source.source))); + } + + auto configs_to_read = Util::Vectors::filtered_copy(s.configs, is_read_provider); + for (auto&& config : configs_to_read) + { + ret.read.push_back(std::make_unique( + config.id, nuget_base, nuget_configfile_arg(config.source))); + } + + auto sources_to_write = Util::Vectors::filtered_copy(s.sources, is_write_provider); + auto configs_to_write = Util::Vectors::filtered_copy(s.configs, is_write_provider); + + if (!sources_to_write.empty() || !configs_to_write.empty()) { ret.write.push_back(std::make_unique( - nuget_base, std::move(s.sources_to_write), std::move(s.configs_to_write))); + nuget_base, std::move(sources_to_write), std::move(configs_to_write))); } } } return std::move(ret); } - ReadOnlyBinaryCache::ReadOnlyBinaryCache(BinaryProviders&& providers) : m_config(std::move(providers)) { } + ReadOnlyBinaryCache::ReadOnlyBinaryCache(BinaryProviders&& providers) + : m_config(std::move(providers)), m_has_write_only_providers(HasWriteOnlyProviders(m_config)) + { + } - void ReadOnlyBinaryCache::fetch(View actions) + void ReadOnlyBinaryCache::fetch(const View actions) { std::vector action_ptrs; - std::vector restores; - std::vector statuses; + std::vector> provider_actions; + + /// Get the overall list of actions to perform. + for (const auto& action : actions) + { + if (action.has_package_abi()) + { + action_ptrs.push_back(&action); + } + } + + if (action_ptrs.empty()) return; + + /// Precheck each read provider to see if it can provide the package. for (auto&& provider : m_config.read) { - action_ptrs.clear(); - restores.clear(); - statuses.clear(); - for (size_t i = 0; i < actions.size(); ++i) + provider_actions.emplace_back(); + + if (!provider->can_precheck()) { - if (actions[i].package_abi()) + /// A `precheck` operation is unsupported for this provider, so try to fetch all packages. + provider_actions.back() = action_ptrs; + continue; + } + + std::vector availabilities(action_ptrs.size(), CacheAvailability::unknown); + provider->precheck(action_ptrs, availabilities); + + for (size_t i = 0; i < action_ptrs.size(); ++i) + { + CacheStatus& cache_status = m_status[*action_ptrs[i]->package_abi().get()]; + switch (availabilities[i]) { - CacheStatus& status = m_status[*actions[i].package_abi().get()]; - if (status.should_attempt_restore(provider.get())) + case CacheAvailability::available: { - action_ptrs.push_back(&actions[i]); - restores.push_back(RestoreResult::unavailable); - statuses.push_back(&status); + provider_actions.back().push_back(action_ptrs[i]); + break; + } + case CacheAvailability::unknown: [[fallthrough]]; + case CacheAvailability::unavailable: + { + /// Mark as unavailable, so the write-back happens properly. + cache_status.mark_unavailable(provider.get()); + break; } } } - if (action_ptrs.empty()) continue; + } + + for (size_t provider_idx = 0; provider_idx < m_config.read.size(); ++provider_idx) + { + auto& actions_for_provider = provider_actions[provider_idx]; + if (actions_for_provider.empty()) continue; + + auto& provider = m_config.read[provider_idx]; + std::vector restores{actions_for_provider.size(), RestoreResult::unavailable}; ElapsedTimer timer; - provider->fetch(action_ptrs, restores); + provider->fetch(actions_for_provider, restores); size_t num_restored = 0; for (size_t i = 0; i < restores.size(); ++i) { - if (restores[i] == RestoreResult::unavailable) - { - statuses[i]->mark_unavailable(provider.get()); - } - else + CacheStatus& cache_status = m_status[*actions_for_provider[i]->package_abi().get()]; + switch (restores[i]) { - statuses[i]->mark_restored(); - ++num_restored; + case RestoreResult::unavailable: + { + cache_status.mark_unavailable(provider.get()); + break; + } + case RestoreResult::restored: + { + cache_status.mark_restored(); + ++num_restored; + break; + } } } msg::println(provider->restored_message( @@ -2059,14 +2266,14 @@ namespace vcpkg } } - bool ReadOnlyBinaryCache::is_restored(const InstallPlanAction& action) const + Optional ReadOnlyBinaryCache::cache_status(const InstallPlanAction& action) const { if (auto abi = action.package_abi().get()) { auto it = m_status.find(*abi); - if (it != m_status.end()) return it->second.is_restored(); + if (it != m_status.end()) return it->second; } - return false; + return nullopt; } std::vector ReadOnlyBinaryCache::precheck(View actions) @@ -2100,6 +2307,7 @@ namespace vcpkg for (size_t i = 0; i < action_ptrs.size(); ++i) { auto&& this_status = m_status[*action_ptrs[i]->package_abi().get()]; + if (cache_result[i] == CacheAvailability::available) { this_status.mark_available(provider.get()); @@ -2149,12 +2357,9 @@ namespace vcpkg { if (auto abi = action.package_abi().get()) { - bool restored = m_status[*abi].is_restored(); - // Purge all status information on push_success (cache invalidation) - // - push_success may delete packages/ (invalidate restore) - // - push_success may make the package available from providers (invalidate unavailable) - m_status.erase(*abi); - if (!restored && !m_config.write.empty()) + CacheStatus& cache_status = m_status[*abi]; + + if (!cache_status.get_unavailable_providers().empty() || m_has_write_only_providers) { ElapsedTimer timer; BinaryPackageWriteInfo request{action}; @@ -2187,7 +2392,7 @@ namespace vcpkg { if (!provider->needs_zip_file() || request.zip_path.has_value()) { - num_destinations += provider->push_success(request, out_sink); + num_destinations += provider->push_success(request, out_sink, cache_status); } } if (request.zip_path) @@ -2226,6 +2431,11 @@ namespace vcpkg default: Checks::unreachable(VCPKG_LINE_INFO); } } + bool CacheStatus::should_attempt_write_back(ProviderId provider_id) const noexcept + { + return Util::Vectors::contains( + m_known_unavailable_providers, provider_id, [](const auto* p, ProviderId id) { return p->id() == id; }); + } bool CacheStatus::is_unavailable(const IReadBinaryProvider* sender) const noexcept { @@ -2241,6 +2451,7 @@ namespace vcpkg m_known_unavailable_providers.push_back(sender); } } + void CacheStatus::mark_available(const IReadBinaryProvider* sender) noexcept { switch (m_status) @@ -2259,12 +2470,16 @@ namespace vcpkg { switch (m_status) { - case CacheStatusState::unknown: m_known_unavailable_providers.clear(); [[fallthrough]]; + case CacheStatusState::unknown: [[fallthrough]]; case CacheStatusState::available: m_status = CacheStatusState::restored; break; case CacheStatusState::restored: break; default: Checks::unreachable(VCPKG_LINE_INFO); } } + void CacheStatus::mark_written_back(ProviderId provider_id) noexcept + { + Util::erase_remove_if(m_known_unavailable_providers, [=](const auto* p) { return p->id() == provider_id; }); + } const IReadBinaryProvider* CacheStatus::get_available_provider() const noexcept { @@ -2276,10 +2491,15 @@ namespace vcpkg default: Checks::unreachable(VCPKG_LINE_INFO); } } + CacheStatus::ReaderProviders& CacheStatus::get_unavailable_providers() noexcept + { + return m_known_unavailable_providers; + } void BinaryConfigParserState::clear() { *this = BinaryConfigParserState(); + binary_cache_providers.clear(); binary_cache_providers.insert("clear"); } @@ -2566,6 +2786,15 @@ std::string vcpkg::generate_nuget_packages_config(const ActionPlan& plan, String xml.close_tag("packages").line_break(); return std::move(xml.buf); } +bool vcpkg::HasWriteOnlyProviders(const BinaryProviders& providers) +{ + return Util::any_of(providers.write, [&](const auto& write_provider) { + return Util::any_of(write_provider->get_provider_ids(), [&](const auto& id) { + return providers.read.end() == + Util::find_if(providers.read, [&](const auto& read_provider) { return read_provider->id() == id; }); + }); + }); +} NugetReference vcpkg::make_nugetref(const InstallPlanAction& action, StringView prefix) { diff --git a/src/vcpkg/commands.install.cpp b/src/vcpkg/commands.install.cpp index 4e800b5b0c..d497bcbb35 100644 --- a/src/vcpkg/commands.install.cpp +++ b/src/vcpkg/commands.install.cpp @@ -341,12 +341,16 @@ namespace vcpkg if (plan_type == InstallPlanType::BUILD_AND_INSTALL) { std::unique_ptr bcf; - if (binary_cache.is_restored(action)) + if (Optional status = binary_cache.cache_status(action); + status.has_value() && status.get()->is_restored()) { auto maybe_bcf = Paragraphs::try_load_cached_package( fs, action.package_dir.value_or_exit(VCPKG_LINE_INFO), action.spec); bcf = std::make_unique(std::move(maybe_bcf).value_or_exit(VCPKG_LINE_INFO)); all_dependencies_satisfied = true; + + /// Write back the cached packages to other write-capable binary caches. + binary_cache.push_success(build_options.clean_packages, action); } else if (build_options.build_missing == BuildMissing::No) {