From 7c70e20b54bb74a389be32e2379020f3d8d8a631 Mon Sep 17 00:00:00 2001 From: Francisco Facioni Date: Wed, 12 Feb 2025 16:06:05 +0100 Subject: [PATCH 1/2] respect the order the providers are being defined --- include/vcpkg/binarycaching.h | 58 +++++- src/vcpkg-test/configparser.cpp | 358 +++++++++++++++++++++++++------- src/vcpkg/binarycaching.cpp | 329 +++++++++++++++++------------ 3 files changed, 522 insertions(+), 223 deletions(-) diff --git a/include/vcpkg/binarycaching.h b/include/vcpkg/binarycaching.h index 82328008c1..77d3883c5a 100644 --- a/include/vcpkg/binarycaching.h +++ b/include/vcpkg/binarycaching.h @@ -24,6 +24,7 @@ #include #include #include +#include #include namespace vcpkg @@ -131,42 +132,77 @@ namespace vcpkg std::string feed; }; - struct BinaryConfigParserState + struct FilesBinaryProviderConfig { - bool nuget_interactive = false; - std::set binary_cache_providers; - - std::string nugettimeout = "100"; - std::vector archives_to_read; std::vector archives_to_write; + }; + struct HttpBinaryProviderConfig + { + std::vector secrets; std::vector url_templates_to_get; std::vector url_templates_to_put; + }; + struct GcsBinaryProviderConfig + { std::vector gcs_read_prefixes; std::vector gcs_write_prefixes; + }; + struct AwsBinaryProviderConfig + { std::vector aws_read_prefixes; std::vector aws_write_prefixes; - bool aws_no_sign_request = false; + }; + struct CosBinaryProviderConfig + { std::vector cos_read_prefixes; std::vector cos_write_prefixes; + }; + struct GhaBinaryProviderConfig + { bool gha_write = false; bool gha_read = false; + }; - std::vector upkg_templates_to_get; - std::vector upkg_templates_to_put; - + struct NugetBinaryProviderConfig + { std::vector sources_to_read; std::vector sources_to_write; std::vector configs_to_read; std::vector configs_to_write; + }; - std::vector secrets; + struct AzureUpkgBinaryProviderConfig + { + std::vector upkg_templates_to_get; + std::vector upkg_templates_to_put; + }; + + using BinaryProviderConfig = std::variant; + + struct BinaryConfigParserState + { + bool nuget_interactive = false; + std::set binary_cache_providers; + + std::vector binary_providers; + + std::string nugettimeout = "100"; + + bool aws_no_sign_request = false; // These are filled in after construction by reading from args and environment std::string nuget_prefix; diff --git a/src/vcpkg-test/configparser.cpp b/src/vcpkg-test/configparser.cpp index 8e1a60d367..97fc89939f 100644 --- a/src/vcpkg-test/configparser.cpp +++ b/src/vcpkg-test/configparser.cpp @@ -17,16 +17,28 @@ namespace void validate_readonly_url(const BinaryConfigParserState& state, StringView url) { auto extended_url = url.to_string() + "/{sha}.zip?sas"; - CHECK(state.url_templates_to_put.empty()); - CHECK(state.url_templates_to_get.size() == 1); - CHECK(state.url_templates_to_get.front().url_template == extended_url); + CHECK(state.binary_providers.size() == 2); + auto config = std::get_if(&state.binary_providers.at(1)); + CHECK(config != nullptr); + CHECK(config->url_templates_to_put.empty()); + CHECK(config->url_templates_to_get.size() == 1); + CHECK(config->url_templates_to_get.front().url_template == extended_url); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_write.empty()); + REQUIRE(!filesConfig->archives_to_read.empty()); } void validate_readonly_sources(const BinaryConfigParserState& state, StringView sources) { - CHECK(state.sources_to_write.empty()); - CHECK(state.sources_to_read.size() == 1); - CHECK(state.sources_to_read.front() == sources); + CHECK(state.binary_providers.size() == 2); + auto config = std::get_if(&state.binary_providers.at(1)); + CHECK(config != nullptr); + CHECK(config->sources_to_write.empty()); + CHECK(config->sources_to_read.size() == 1); + CHECK(config->sources_to_read.front() == sources); } } @@ -71,23 +83,32 @@ TEST_CASE ("BinaryConfigParser files provider", "[binaryconfigparser]") auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"files"}}); - REQUIRE(!state.archives_to_read.empty()); - REQUIRE(!Util::Vectors::contains(state.archives_to_write, ABSOLUTE_PATH)); + REQUIRE(state.binary_providers.size() == 2); + auto config = std::get_if(&state.binary_providers.at(1)); + CHECK(config != nullptr); + REQUIRE(!config->archives_to_read.empty()); + REQUIRE(!Util::Vectors::contains(config->archives_to_write, ABSOLUTE_PATH)); } { auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH ",write", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"files"}}); - REQUIRE(!state.archives_to_write.empty()); + REQUIRE(state.binary_providers.size() == 2); + auto config = std::get_if(&state.binary_providers.at(1)); + CHECK(config != nullptr); + REQUIRE(!config->archives_to_write.empty()); } { auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH ",readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"files"}}); - REQUIRE(!state.archives_to_write.empty()); - REQUIRE(!state.archives_to_read.empty()); + REQUIRE(state.binary_providers.size() == 2); + auto config = std::get_if(&state.binary_providers.at(1)); + CHECK(config != nullptr); + REQUIRE(!config->archives_to_write.empty()); + REQUIRE(!config->archives_to_read.empty()); } { auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH ",readwrite,extra", {}); @@ -134,13 +155,21 @@ TEST_CASE ("BinaryConfigParser nuget source provider", "[binaryconfigparser]") auto parsed = parse_binary_provider_configs("nuget," ABSOLUTE_PATH ",readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - CHECK(state.sources_to_read.size() == 1); - CHECK(state.sources_to_write.size() == 1); - CHECK(state.sources_to_read.front() == state.sources_to_write.front()); - CHECK(state.sources_to_read.front() == ABSOLUTE_PATH); + REQUIRE(state.binary_providers.size() == 2); + auto nugetConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(nugetConfig != nullptr); + + CHECK(nugetConfig->sources_to_read.size() == 1); + CHECK(nugetConfig->sources_to_write.size() == 1); + CHECK(nugetConfig->sources_to_read.front() == nugetConfig->sources_to_write.front()); + CHECK(nugetConfig->sources_to_read.front() == ABSOLUTE_PATH); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"nuget"}}); - REQUIRE(!state.archives_to_write.empty()); - REQUIRE(!state.archives_to_read.empty()); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_write.empty()); + REQUIRE(!filesConfig->archives_to_read.empty()); } { auto parsed = parse_binary_provider_configs("nuget," ABSOLUTE_PATH ",readwrite,extra", {}); @@ -217,33 +246,55 @@ TEST_CASE ("BinaryConfigParser nuget config provider", "[binaryconfigparser]") auto parsed = parse_binary_provider_configs("nugetconfig," ABSOLUTE_PATH ",read", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - CHECK(state.configs_to_write.empty()); - CHECK(state.configs_to_read.size() == 1); - CHECK(state.configs_to_read.front() == ABSOLUTE_PATH); + REQUIRE(state.binary_providers.size() == 2); + auto nugetConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(nugetConfig != nullptr); + + CHECK(nugetConfig->configs_to_write.empty()); + CHECK(nugetConfig->configs_to_read.size() == 1); + CHECK(nugetConfig->configs_to_read.front() == ABSOLUTE_PATH); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"nuget"}}); - REQUIRE(!state.archives_to_read.empty()); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + REQUIRE(!filesConfig->archives_to_read.empty()); } { auto parsed = parse_binary_provider_configs("nugetconfig," ABSOLUTE_PATH ",write", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - CHECK(state.configs_to_read.empty()); - CHECK(state.configs_to_write.size() == 1); - CHECK(state.configs_to_write.front() == ABSOLUTE_PATH); + REQUIRE(state.binary_providers.size() == 2); + auto nugetConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(nugetConfig != nullptr); + + CHECK(nugetConfig->configs_to_read.empty()); + CHECK(nugetConfig->configs_to_write.size() == 1); + CHECK(nugetConfig->configs_to_write.front() == ABSOLUTE_PATH); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"nuget"}}); - REQUIRE(!state.archives_to_write.empty()); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + REQUIRE(!filesConfig->archives_to_write.empty()); } { auto parsed = parse_binary_provider_configs("nugetconfig," ABSOLUTE_PATH ",readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - CHECK(state.configs_to_read.size() == 1); - CHECK(state.configs_to_write.size() == 1); - CHECK(state.configs_to_read.front() == state.configs_to_write.front()); - CHECK(state.configs_to_read.front() == ABSOLUTE_PATH); + REQUIRE(state.binary_providers.size() == 2); + auto nugetConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(nugetConfig != nullptr); + + CHECK(nugetConfig->configs_to_read.size() == 1); + CHECK(nugetConfig->configs_to_write.size() == 1); + CHECK(nugetConfig->configs_to_read.front() == nugetConfig->configs_to_write.front()); + CHECK(nugetConfig->configs_to_read.front() == ABSOLUTE_PATH); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"nuget"}}); - REQUIRE(!state.archives_to_write.empty()); - REQUIRE(!state.archives_to_read.empty()); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_write.empty()); + REQUIRE(!filesConfig->archives_to_read.empty()); } { auto parsed = parse_binary_provider_configs("nugetconfig," ABSOLUTE_PATH ",readwrite,extra", {}); @@ -268,18 +319,33 @@ TEST_CASE ("BinaryConfigParser default provider", "[binaryconfigparser]") { auto parsed = parse_binary_provider_configs("default,read", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - REQUIRE(!state.archives_to_read.empty()); + + REQUIRE(state.binary_providers.size() == 2); + auto config = std::get_if(&state.binary_providers.at(0)); + CHECK(config != nullptr); + + REQUIRE(!config->archives_to_read.empty()); } { auto parsed = parse_binary_provider_configs("default,readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - REQUIRE(!state.archives_to_read.empty()); - REQUIRE(!state.archives_to_write.empty()); + + REQUIRE(state.binary_providers.size() == 2); + auto config = std::get_if(&state.binary_providers.at(0)); + CHECK(config != nullptr); + + REQUIRE(!config->archives_to_read.empty()); + REQUIRE(!config->archives_to_write.empty()); } { auto parsed = parse_binary_provider_configs("default,write", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - REQUIRE(!state.archives_to_write.empty()); + + REQUIRE(state.binary_providers.size() == 2); + auto config = std::get_if(&state.binary_providers.at(0)); + CHECK(config != nullptr); + + REQUIRE(!config->archives_to_write.empty()); } { auto parsed = parse_binary_provider_configs("default,read,extra", {}); @@ -426,7 +492,18 @@ TEST_CASE ("BinaryConfigParser azblob provider", "[binaryconfigparser]") REQUIRE(state.binary_cache_providers == std::set{{"azblob"}, {"default"}}); validate_readonly_url(state, "https://azure/container"); - REQUIRE(state.secrets == std::vector{"sas"}); + + REQUIRE(state.binary_providers.size() == 2); + auto httpConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(httpConfig != nullptr); + + REQUIRE(httpConfig->secrets == std::vector{"sas"}); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_write.empty()); + REQUIRE(!filesConfig->archives_to_read.empty()); } { auto parsed = parse_binary_provider_configs("x-azblob,https://azure/container,?sas", {}); @@ -450,32 +527,59 @@ TEST_CASE ("BinaryConfigParser azblob provider", "[binaryconfigparser]") REQUIRE(state.binary_cache_providers == std::set{{"azblob"}, {"default"}}); validate_readonly_url(state, "https://azure/container"); - REQUIRE(state.secrets == std::vector{"sas"}); - REQUIRE(!state.archives_to_read.empty()); + + REQUIRE(state.binary_providers.size() == 2); + auto azConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(azConfig != nullptr); + + REQUIRE(azConfig->secrets == std::vector{"sas"}); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_read.empty()); } { auto parsed = parse_binary_provider_configs("x-azblob,https://azure/container,sas,write", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"azblob"}, {"default"}}); - CHECK(state.url_templates_to_get.empty()); - CHECK(state.url_templates_to_put.size() == 1); - CHECK(state.url_templates_to_put.front().url_template == "https://azure/container/{sha}.zip?sas"); - REQUIRE(state.secrets == std::vector{"sas"}); - REQUIRE(!state.archives_to_write.empty()); + + REQUIRE(state.binary_providers.size() == 2); + auto azConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(azConfig != nullptr); + + CHECK(azConfig->url_templates_to_get.empty()); + CHECK(azConfig->url_templates_to_put.size() == 1); + CHECK(azConfig->url_templates_to_put.front().url_template == "https://azure/container/{sha}.zip?sas"); + REQUIRE(azConfig->secrets == std::vector{"sas"}); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_write.empty()); } { auto parsed = parse_binary_provider_configs("x-azblob,https://azure/container,sas,readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"azblob"}, {"default"}}); - CHECK(state.url_templates_to_get.size() == 1); - CHECK(state.url_templates_to_get.front().url_template == "https://azure/container/{sha}.zip?sas"); - CHECK(state.url_templates_to_put.size() == 1); - CHECK(state.url_templates_to_put.front().url_template == "https://azure/container/{sha}.zip?sas"); - REQUIRE(state.secrets == std::vector{"sas"}); - REQUIRE(!state.archives_to_read.empty()); - REQUIRE(!state.archives_to_write.empty()); + + REQUIRE(state.binary_providers.size() == 2); + auto azConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(azConfig != nullptr); + + CHECK(azConfig->url_templates_to_get.size() == 1); + CHECK(azConfig->url_templates_to_get.front().url_template == "https://azure/container/{sha}.zip?sas"); + CHECK(azConfig->url_templates_to_put.size() == 1); + CHECK(azConfig->url_templates_to_put.front().url_template == "https://azure/container/{sha}.zip?sas"); + REQUIRE(azConfig->secrets == std::vector{"sas"}); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_read.empty()); + REQUIRE(!filesConfig->archives_to_write.empty()); } } @@ -485,15 +589,37 @@ TEST_CASE ("BinaryConfigParser GCS provider", "[binaryconfigparser]") auto parsed = parse_binary_provider_configs("x-gcs,gs://my-bucket/", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - REQUIRE(state.gcs_read_prefixes == std::vector{"gs://my-bucket/"}); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"gcs"}}); + + REQUIRE(state.binary_providers.size() == 2); + auto gcsConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(gcsConfig != nullptr); + + REQUIRE(gcsConfig->gcs_read_prefixes == std::vector{"gs://my-bucket/"}); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_read.empty()); + REQUIRE(!filesConfig->archives_to_write.empty()); } { auto parsed = parse_binary_provider_configs("x-gcs,gs://my-bucket/my-folder", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - REQUIRE(state.gcs_read_prefixes == std::vector{"gs://my-bucket/my-folder/"}); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"gcs"}}); + + REQUIRE(state.binary_providers.size() == 2); + auto gcsConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(gcsConfig != nullptr); + + REQUIRE(gcsConfig->gcs_read_prefixes == std::vector{"gs://my-bucket/my-folder/"}); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_read.empty()); + REQUIRE(!filesConfig->archives_to_write.empty()); } { auto parsed = parse_binary_provider_configs("x-gcs,", {}); @@ -507,27 +633,51 @@ TEST_CASE ("BinaryConfigParser GCS provider", "[binaryconfigparser]") auto parsed = parse_binary_provider_configs("x-gcs,gs://my-bucket/my-folder,read", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); + REQUIRE(state.binary_providers.size() == 2); + auto gcsConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(gcsConfig != nullptr); + REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"gcs"}}); - REQUIRE(state.gcs_read_prefixes == std::vector{"gs://my-bucket/my-folder/"}); - REQUIRE(!state.archives_to_read.empty()); + REQUIRE(gcsConfig->gcs_read_prefixes == std::vector{"gs://my-bucket/my-folder/"}); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_read.empty()); } { auto parsed = parse_binary_provider_configs("x-gcs,gs://my-bucket/my-folder,write", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); + REQUIRE(state.binary_providers.size() == 2); + auto gcsConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(gcsConfig != nullptr); + REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"gcs"}}); - REQUIRE(state.gcs_write_prefixes == std::vector{"gs://my-bucket/my-folder/"}); - REQUIRE(!state.archives_to_write.empty()); + REQUIRE(gcsConfig->gcs_write_prefixes == std::vector{"gs://my-bucket/my-folder/"}); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_write.empty()); } { auto parsed = parse_binary_provider_configs("x-gcs,gs://my-bucket/my-folder,readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); + REQUIRE(state.binary_providers.size() == 2); + auto gcsConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(gcsConfig != nullptr); + REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"gcs"}}); - REQUIRE(state.gcs_write_prefixes == std::vector{"gs://my-bucket/my-folder/"}); - REQUIRE(state.gcs_read_prefixes == std::vector{"gs://my-bucket/my-folder/"}); - REQUIRE(!state.archives_to_write.empty()); - REQUIRE(!state.archives_to_read.empty()); + REQUIRE(gcsConfig->gcs_write_prefixes == std::vector{"gs://my-bucket/my-folder/"}); + REQUIRE(gcsConfig->gcs_read_prefixes == std::vector{"gs://my-bucket/my-folder/"}); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_write.empty()); + REQUIRE(!filesConfig->archives_to_read.empty()); } } @@ -537,22 +687,52 @@ TEST_CASE ("BinaryConfigParser HTTP provider", "[binaryconfigparser]") auto parsed = parse_binary_provider_configs("http,http://example.org/", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - REQUIRE(state.url_templates_to_get.size() == 1); - REQUIRE(state.url_templates_to_get[0].url_template == "http://example.org/{sha}.zip"); + REQUIRE(state.binary_providers.size() == 2); + auto httpConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(httpConfig != nullptr); + + REQUIRE(httpConfig->url_templates_to_get.size() == 1); + REQUIRE(httpConfig->url_templates_to_get[0].url_template == "http://example.org/{sha}.zip"); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_write.empty()); + REQUIRE(!filesConfig->archives_to_read.empty()); } { auto parsed = parse_binary_provider_configs("http,http://example.org", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - REQUIRE(state.url_templates_to_get.size() == 1); - REQUIRE(state.url_templates_to_get[0].url_template == "http://example.org/{sha}.zip"); + REQUIRE(state.binary_providers.size() == 2); + auto httpConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(httpConfig != nullptr); + + REQUIRE(httpConfig->url_templates_to_get.size() == 1); + REQUIRE(httpConfig->url_templates_to_get[0].url_template == "http://example.org/{sha}.zip"); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_write.empty()); + REQUIRE(!filesConfig->archives_to_read.empty()); } { auto parsed = parse_binary_provider_configs("http,http://example.org/{triplet}/{sha}", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - REQUIRE(state.url_templates_to_get.size() == 1); - REQUIRE(state.url_templates_to_get[0].url_template == "http://example.org/{triplet}/{sha}"); + REQUIRE(state.binary_providers.size() == 2); + auto httpConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(httpConfig != nullptr); + + REQUIRE(httpConfig->url_templates_to_get.size() == 1); + REQUIRE(httpConfig->url_templates_to_get[0].url_template == "http://example.org/{triplet}/{sha}"); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_write.empty()); + REQUIRE(!filesConfig->archives_to_read.empty()); } { auto parsed = parse_binary_provider_configs("http,http://example.org/{triplet}", {}); @@ -567,23 +747,45 @@ TEST_CASE ("BinaryConfigParser Universal Packages provider", "[binaryconfigparse auto parsed = parse_binary_provider_configs("x-az-universal,test_organization,test_project_name,test_feed,read", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - REQUIRE(state.upkg_templates_to_get.size() == 1); - REQUIRE(state.upkg_templates_to_get[0].feed == "test_feed"); - REQUIRE(state.upkg_templates_to_get[0].organization == "test_organization"); - REQUIRE(state.upkg_templates_to_get[0].project == "test_project_name"); + + REQUIRE(state.binary_providers.size() == 2); + auto upkgConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(upkgConfig != nullptr); + + REQUIRE(upkgConfig->upkg_templates_to_get.size() == 1); + REQUIRE(upkgConfig->upkg_templates_to_get[0].feed == "test_feed"); + REQUIRE(upkgConfig->upkg_templates_to_get[0].organization == "test_organization"); + REQUIRE(upkgConfig->upkg_templates_to_get[0].project == "test_project_name"); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_write.empty()); + REQUIRE(!filesConfig->archives_to_read.empty()); } { auto parsed = parse_binary_provider_configs("x-az-universal,test_organization,test_project_name,test_feed,readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); - REQUIRE(state.upkg_templates_to_get.size() == 1); - REQUIRE(state.upkg_templates_to_put.size() == 1); - REQUIRE(state.upkg_templates_to_get[0].feed == "test_feed"); - REQUIRE(state.upkg_templates_to_get[0].organization == "test_organization"); - REQUIRE(state.upkg_templates_to_get[0].project == "test_project_name"); - REQUIRE(state.upkg_templates_to_put[0].feed == "test_feed"); - REQUIRE(state.upkg_templates_to_put[0].organization == "test_organization"); - REQUIRE(state.upkg_templates_to_put[0].project == "test_project_name"); + + REQUIRE(state.binary_providers.size() == 2); + auto upkgConfig = std::get_if(&state.binary_providers.at(1)); + CHECK(upkgConfig != nullptr); + + REQUIRE(upkgConfig->upkg_templates_to_get.size() == 1); + REQUIRE(upkgConfig->upkg_templates_to_put.size() == 1); + REQUIRE(upkgConfig->upkg_templates_to_get[0].feed == "test_feed"); + REQUIRE(upkgConfig->upkg_templates_to_get[0].organization == "test_organization"); + REQUIRE(upkgConfig->upkg_templates_to_get[0].project == "test_project_name"); + REQUIRE(upkgConfig->upkg_templates_to_put[0].feed == "test_feed"); + REQUIRE(upkgConfig->upkg_templates_to_put[0].organization == "test_organization"); + REQUIRE(upkgConfig->upkg_templates_to_put[0].project == "test_project_name"); + + auto filesConfig = std::get_if(&state.binary_providers.at(0)); + CHECK(filesConfig != nullptr); + + REQUIRE(!filesConfig->archives_to_write.empty()); + REQUIRE(!filesConfig->archives_to_read.empty()); } { auto parsed = parse_binary_provider_configs( diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 3f5113f6d6..e3cf527d8b 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -1507,13 +1507,15 @@ namespace segments[1].first); } - handle_readwrite(state->archives_to_read, state->archives_to_write, std::move(p), segments, 2); + FilesBinaryProviderConfig config; + handle_readwrite(config.archives_to_read, config.archives_to_write, std::move(p), segments, 2); if (segments.size() > 3) { return add_error( msg::format(msgInvalidArgumentRequiresOneOrTwoArguments, msg::binary_source = "files"), segments[3].first); } + state->binary_providers.emplace_back(config); state->binary_cache_providers.insert("files"); } else if (segments[0].second == "interactive") @@ -1544,13 +1546,15 @@ namespace segments[1].first); } - handle_readwrite(state->configs_to_read, state->configs_to_write, std::move(p), segments, 2); + NugetBinaryProviderConfig config; + handle_readwrite(config.configs_to_read, config.configs_to_write, std::move(p), segments, 2); if (segments.size() > 3) { return add_error( msg::format(msgInvalidArgumentRequiresOneOrTwoArguments, msg::binary_source = "nugetconfig"), segments[3].first); } + state->binary_providers.emplace_back(config); state->binary_cache_providers.insert("nuget"); } else if (segments[0].second == "nuget") @@ -1569,13 +1573,15 @@ namespace msg::format(msgInvalidArgumentRequiresSourceArgument, msg::binary_source = "nuget")); } - handle_readwrite(state->sources_to_read, state->sources_to_write, std::move(p), segments, 2); + NugetBinaryProviderConfig config; + handle_readwrite(config.sources_to_read, config.sources_to_write, std::move(p), segments, 2); if (segments.size() > 3) { return add_error( msg::format(msgInvalidArgumentRequiresOneOrTwoArguments, msg::binary_source = "nuget"), segments[3].first); } + state->binary_providers.emplace_back(config); state->binary_cache_providers.insert("nuget"); } else if (segments[0].second == "nugettimeout") @@ -1609,8 +1615,10 @@ namespace return add_error(LocalizedString{maybe_home.error()}, segments[0].first); } + FilesBinaryProviderConfig config; handle_readwrite( - state->archives_to_read, state->archives_to_write, Path(*maybe_home.get()), segments, 1); + config.archives_to_read, config.archives_to_write, Path(*maybe_home.get()), segments, 1); + state->binary_providers.emplace_back(config); state->binary_cache_providers.insert("default"); } else if (segments[0].second == "x-azblob") @@ -1657,15 +1665,18 @@ namespace } p.append(segments[2].second); - state->secrets.push_back(segments[2].second); + + HttpBinaryProviderConfig config; + config.secrets.push_back(segments[2].second); UrlTemplate url_template = {p}; bool read = false, write = false; handle_readwrite(read, write, segments, 3); - if (read) state->url_templates_to_get.push_back(url_template); + if (read) config.url_templates_to_get.push_back(url_template); auto headers = azure_blob_headers(); url_template.headers.assign(headers.begin(), headers.end()); - if (write) state->url_templates_to_put.push_back(url_template); + if (write) config.url_templates_to_put.push_back(url_template); + state->binary_providers.emplace_back(config); state->binary_cache_providers.insert("azblob"); } else if (segments[0].second == "x-gcs") @@ -1698,8 +1709,9 @@ namespace p.push_back('/'); } - handle_readwrite(state->gcs_read_prefixes, state->gcs_write_prefixes, std::move(p), segments, 2); - + GcsBinaryProviderConfig config; + handle_readwrite(config.gcs_read_prefixes, config.gcs_write_prefixes, std::move(p), segments, 2); + state->binary_providers.emplace_back(config); state->binary_cache_providers.insert("gcs"); } else if (segments[0].second == "x-aws") @@ -1732,8 +1744,9 @@ namespace p.push_back('/'); } - handle_readwrite(state->aws_read_prefixes, state->aws_write_prefixes, std::move(p), segments, 2); - + AwsBinaryProviderConfig config; + handle_readwrite(config.aws_read_prefixes, config.aws_write_prefixes, std::move(p), segments, 2); + state->binary_providers.emplace_back(config); state->binary_cache_providers.insert("aws"); } else if (segments[0].second == "x-aws-config") @@ -1787,7 +1800,9 @@ namespace p.push_back('/'); } - handle_readwrite(state->cos_read_prefixes, state->cos_write_prefixes, std::move(p), segments, 2); + CosBinaryProviderConfig config; + handle_readwrite(config.cos_read_prefixes, config.cos_write_prefixes, std::move(p), segments, 2); + state->binary_providers.emplace_back(config); state->binary_cache_providers.insert("cos"); } else if (segments[0].second == "x-gha") @@ -1800,8 +1815,9 @@ namespace segments[2].first); } - handle_readwrite(state->gha_read, state->gha_write, segments, 1); - + GhaBinaryProviderConfig config; + handle_readwrite(config.gha_read, config.gha_write, segments, 1); + state->binary_providers.emplace_back(config); state->binary_cache_providers.insert("gha"); } else if (segments[0].second == "http") @@ -1866,8 +1882,10 @@ namespace url_template.headers.push_back(segments[3].second); } + HttpBinaryProviderConfig config; handle_readwrite( - state->url_templates_to_get, state->url_templates_to_put, std::move(url_template), segments, 2); + config.url_templates_to_get, config.url_templates_to_put, std::move(url_template), segments, 2); + state->binary_providers.emplace_back(config); state->binary_cache_providers.insert("http"); } else if (segments[0].second == "x-az-universal") @@ -1885,8 +1903,10 @@ namespace }; state->binary_cache_providers.insert("upkg"); + AzureUpkgBinaryProviderConfig config; handle_readwrite( - state->upkg_templates_to_get, state->upkg_templates_to_put, std::move(upkg_template), segments, 4); + config.upkg_templates_to_get, config.upkg_templates_to_put, std::move(upkg_template), segments, 4); + state->binary_providers.emplace_back(config); } else { @@ -2348,137 +2368,178 @@ namespace vcpkg m_config.nuget_prefix = s.nuget_prefix; - std::shared_ptr gcs_tool; - if (!s.gcs_read_prefixes.empty() || !s.gcs_write_prefixes.empty()) - { - gcs_tool = std::make_shared(tools, out_sink); - } - std::shared_ptr aws_tool; - if (!s.aws_read_prefixes.empty() || !s.aws_write_prefixes.empty()) + if (!s.binary_providers.empty()) { - aws_tool = std::make_shared(tools, out_sink, s.aws_no_sign_request); - } - std::shared_ptr cos_tool; - if (!s.cos_read_prefixes.empty() || !s.cos_write_prefixes.empty()) - { - cos_tool = std::make_shared(tools, out_sink); - } + std::shared_ptr gcs_tool; + std::shared_ptr aws_tool; + std::shared_ptr cos_tool; - if (s.gha_read || s.gha_write) - { - if (!args.actions_cache_url.has_value() || !args.actions_runtime_token.has_value()) - { - status_sink.println( - Color::error, - msg::format_error(msgGHAParametersMissing, msg::url = docs::binarycaching_gha_url)); - return false; - } - } - - if (!s.archives_to_read.empty() || !s.url_templates_to_get.empty() || !s.gcs_read_prefixes.empty() || - !s.aws_read_prefixes.empty() || !s.cos_read_prefixes.empty() || s.gha_read) - { ZipTool zip_tool; zip_tool.setup(tools, out_sink); - for (auto&& dir : s.archives_to_read) - { - m_config.read.push_back(std::make_unique(zip_tool, fs, std::move(dir))); - } + for (auto& config : s.binary_providers) + { + auto ok = std::visit( + [&](auto&& c) { + using T = std::decay_t; + if constexpr (std::is_same_v) + { + for (auto&& dir : c.archives_to_read) + { + m_config.read.push_back( + std::make_unique(zip_tool, fs, std::move(dir))); + } + if (!c.archives_to_write.empty()) + { + m_config.write.push_back( + std::make_unique(fs, std::move(c.archives_to_write))); + } + } + else if constexpr (std::is_same_v) + { + for (auto&& url : c.url_templates_to_get) + { + m_config.read.push_back(std::make_unique( + zip_tool, fs, buildtrees, std::move(url), c.secrets)); + } + if (!c.url_templates_to_put.empty()) + { + m_config.write.push_back(std::make_unique( + std::move(c.url_templates_to_put), c.secrets)); + } + } + else if constexpr (std::is_same_v) + { + if (!gcs_tool) + { + gcs_tool = std::make_shared(tools, out_sink); + } - for (auto&& url : s.url_templates_to_get) - { - m_config.read.push_back( - std::make_unique(zip_tool, fs, buildtrees, std::move(url), s.secrets)); - } + for (auto&& prefix : c.gcs_read_prefixes) + { + m_config.read.push_back(std::make_unique( + zip_tool, fs, buildtrees, std::move(prefix), gcs_tool)); + } + if (!c.gcs_write_prefixes.empty()) + { + m_config.write.push_back(std::make_unique( + std::move(c.gcs_write_prefixes), gcs_tool)); + } + } + else if constexpr (std::is_same_v) + { + if (!aws_tool) + { + aws_tool = std::make_shared(tools, out_sink, s.aws_no_sign_request); + } - for (auto&& prefix : s.gcs_read_prefixes) - { - m_config.read.push_back( - std::make_unique(zip_tool, fs, buildtrees, std::move(prefix), gcs_tool)); - } + for (auto&& prefix : c.aws_read_prefixes) + { + m_config.read.push_back(std::make_unique( + zip_tool, fs, buildtrees, std::move(prefix), aws_tool)); + } - for (auto&& prefix : s.aws_read_prefixes) - { - m_config.read.push_back( - std::make_unique(zip_tool, fs, buildtrees, std::move(prefix), aws_tool)); - } + if (!c.aws_write_prefixes.empty()) + { + m_config.write.push_back(std::make_unique( + std::move(c.aws_write_prefixes), aws_tool)); + } + } + else if constexpr (std::is_same_v) + { + if (!cos_tool) + { + cos_tool = std::make_shared(tools, out_sink); + } - for (auto&& prefix : s.cos_read_prefixes) - { - m_config.read.push_back( - std::make_unique(zip_tool, fs, buildtrees, std::move(prefix), cos_tool)); - } + for (auto&& prefix : c.cos_read_prefixes) + { + m_config.read.push_back(std::make_unique( + zip_tool, fs, buildtrees, std::move(prefix), cos_tool)); + } - if (s.gha_read) - { - const auto& url = *args.actions_cache_url.get(); - const auto& token = *args.actions_runtime_token.get(); - m_config.read.push_back(std::make_unique(zip_tool, fs, buildtrees, url, token)); - } - } - if (!s.archives_to_write.empty()) - { - m_config.write.push_back( - std::make_unique(fs, std::move(s.archives_to_write))); - } - if (!s.url_templates_to_put.empty()) - { - m_config.write.push_back( - std::make_unique(std::move(s.url_templates_to_put), s.secrets)); - } - if (!s.gcs_write_prefixes.empty()) - { - m_config.write.push_back( - std::make_unique(std::move(s.gcs_write_prefixes), gcs_tool)); - } - if (!s.aws_write_prefixes.empty()) - { - m_config.write.push_back( - std::make_unique(std::move(s.aws_write_prefixes), aws_tool)); - } - if (!s.cos_write_prefixes.empty()) - { - m_config.write.push_back( - std::make_unique(std::move(s.cos_write_prefixes), cos_tool)); - } - if (s.gha_write) - { - const auto& url = *args.actions_cache_url.get(); - const auto& token = *args.actions_runtime_token.get(); - m_config.write.push_back(std::make_unique(fs, url, token)); - } + if (!c.cos_write_prefixes.empty()) + { + m_config.write.push_back(std::make_unique( + std::move(c.cos_write_prefixes), cos_tool)); + } + } + else if constexpr (std::is_same_v) + { + if (c.gha_read || c.gha_write) + { + if (!args.actions_cache_url.has_value() || !args.actions_runtime_token.has_value()) + { + status_sink.println(Color::error, + msg::format_error(msgGHAParametersMissing, + msg::url = docs::binarycaching_gha_url)); + return false; + } + } - if (!s.sources_to_read.empty() || !s.configs_to_read.empty() || !s.sources_to_write.empty() || - !s.configs_to_write.empty()) - { - NugetBaseBinaryProvider nuget_base( - fs, NuGetTool(tools, out_sink, s), paths.packages(), buildtrees, s.nuget_prefix); - if (!s.sources_to_read.empty()) - m_config.read.push_back( - std::make_unique(nuget_base, nuget_sources_arg(s.sources_to_read))); - for (auto&& config : s.configs_to_read) - m_config.read.push_back( - std::make_unique(nuget_base, nuget_configfile_arg(config))); - if (!s.sources_to_write.empty() || !s.configs_to_write.empty()) - { - m_config.write.push_back(std::make_unique( - nuget_base, std::move(s.sources_to_write), std::move(s.configs_to_write))); - } - } + if (c.gha_read) + { + const auto& url = *args.actions_cache_url.get(); + const auto& token = *args.actions_runtime_token.get(); + m_config.read.push_back( + std::make_unique(zip_tool, fs, buildtrees, url, token)); + } + if (c.gha_write) + { + const auto& url = *args.actions_cache_url.get(); + const auto& token = *args.actions_runtime_token.get(); + m_config.write.push_back(std::make_unique(fs, url, token)); + } + } + else if constexpr (std::is_same_v) + { + if (!c.sources_to_read.empty() || !c.configs_to_read.empty() || + !c.sources_to_write.empty() || !c.configs_to_write.empty()) + { + NugetBaseBinaryProvider nuget_base(fs, + NuGetTool(tools, out_sink, s), + paths.packages(), + buildtrees, + s.nuget_prefix); + if (!c.sources_to_read.empty()) + m_config.read.push_back(std::make_unique( + nuget_base, nuget_sources_arg(c.sources_to_read))); + for (auto&& config : c.configs_to_read) + m_config.read.push_back(std::make_unique( + nuget_base, nuget_configfile_arg(config))); + if (!c.sources_to_write.empty() || !c.configs_to_write.empty()) + { + m_config.write.push_back(std::make_unique( + nuget_base, std::move(c.sources_to_write), std::move(c.configs_to_write))); + } + } + } + else if constexpr (std::is_same_v) + { + if (!c.upkg_templates_to_get.empty()) + { + for (auto&& src : c.upkg_templates_to_get) + { + m_config.read.push_back(std::make_unique( + tools, out_sink, std::move(src))); + } + } + if (!c.upkg_templates_to_put.empty()) + { + m_config.write.push_back(std::make_unique( + tools, out_sink, std::move(c.upkg_templates_to_put))); + } + } - if (!s.upkg_templates_to_get.empty()) - { - for (auto&& src : s.upkg_templates_to_get) - { - m_config.read.push_back( - std::make_unique(tools, out_sink, std::move(src))); + return true; + }, + config); + + if (!ok) + { + return false; + } } } - if (!s.upkg_templates_to_put.empty()) - { - m_config.write.push_back( - std::make_unique(tools, out_sink, std::move(s.upkg_templates_to_put))); - } } m_needs_nuspec_data = Util::any_of(m_config.write, [](auto&& p) { return p->needs_nuspec_data(); }); From 2b020bc7ba9725de3a3bc562c1ebfc0a9280f75f Mon Sep 17 00:00:00 2001 From: Francisco Facioni Date: Thu, 13 Feb 2025 01:49:16 +0100 Subject: [PATCH 2/2] implement writeback --- include/vcpkg/binarycaching.h | 28 ++- src/vcpkg-test/binarycaching.cpp | 2 +- src/vcpkg/binarycaching.cpp | 333 +++++++++++++++++++++---------- 3 files changed, 258 insertions(+), 105 deletions(-) diff --git a/include/vcpkg/binarycaching.h b/include/vcpkg/binarycaching.h index 77d3883c5a..d92b576081 100644 --- a/include/vcpkg/binarycaching.h +++ b/include/vcpkg/binarycaching.h @@ -29,6 +29,13 @@ namespace vcpkg { + struct BinaryProviderContainer + { + std::vector> read = {}; + std::unique_ptr write = nullptr; + bool write_back = false; + }; + struct CacheStatus { bool should_attempt_precheck(const IReadBinaryProvider* sender) const noexcept; @@ -36,11 +43,12 @@ namespace vcpkg bool is_unavailable(const IReadBinaryProvider* sender) const noexcept; const IReadBinaryProvider* get_available_provider() const noexcept; + const BinaryProviderContainer* get_restored_container() const noexcept; bool is_restored() const noexcept; void mark_unavailable(const IReadBinaryProvider* sender); void mark_available(const IReadBinaryProvider* sender) noexcept; - void mark_restored() noexcept; + void mark_restored(const BinaryProviderContainer* container) noexcept; private: CacheStatusState m_status = CacheStatusState::unknown; @@ -51,6 +59,9 @@ namespace vcpkg // The provider who affirmatively has the associated cache entry. const IReadBinaryProvider* m_available_provider = nullptr; // meaningful iff m_status == available + + // The provider who restored the associated cache entry. + const BinaryProviderContainer* m_restored_container = nullptr; // meaningful iff m_status == restored }; struct BinaryPackageReadInfo @@ -136,6 +147,7 @@ namespace vcpkg { std::vector archives_to_read; std::vector archives_to_write; + bool write_back = false; }; struct HttpBinaryProviderConfig @@ -143,30 +155,35 @@ namespace vcpkg std::vector secrets; std::vector url_templates_to_get; std::vector url_templates_to_put; + bool write_back = false; }; struct GcsBinaryProviderConfig { std::vector gcs_read_prefixes; std::vector gcs_write_prefixes; + bool write_back = false; }; struct AwsBinaryProviderConfig { std::vector aws_read_prefixes; std::vector aws_write_prefixes; + bool write_back = false; }; struct CosBinaryProviderConfig { std::vector cos_read_prefixes; std::vector cos_write_prefixes; + bool write_back = false; }; struct GhaBinaryProviderConfig { bool gha_write = false; bool gha_read = false; + bool write_back = false; }; struct NugetBinaryProviderConfig @@ -176,12 +193,16 @@ namespace vcpkg std::vector configs_to_read; std::vector configs_to_write; + + bool write_back = false; }; struct AzureUpkgBinaryProviderConfig { std::vector upkg_templates_to_get; std::vector upkg_templates_to_put; + + bool write_back = false; }; using BinaryProviderConfig = std::variant> read; - std::vector> write; + std::vector> containers; std::string nuget_prefix; NuGetRepoInfo nuget_repo; }; @@ -310,6 +330,8 @@ namespace vcpkg { BinaryPackageWriteInfo request; CleanPackages clean_after_push; + bool write_back = false; + const BinaryProviderContainer* restored_container = nullptr; }; ZipTool m_zip_tool; diff --git a/src/vcpkg-test/binarycaching.cpp b/src/vcpkg-test/binarycaching.cpp index 914ab144dd..6ae47c5dca 100644 --- a/src/vcpkg-test/binarycaching.cpp +++ b/src/vcpkg-test/binarycaching.cpp @@ -67,7 +67,7 @@ TEST_CASE ("CacheStatus operations", "[BinaryCache]") REQUIRE(!available.is_restored()); CacheStatus restored; - restored.mark_restored(); + restored.mark_restored(nullptr); REQUIRE(!restored.should_attempt_precheck(&know_nothing)); REQUIRE(!restored.should_attempt_restore(&know_nothing)); REQUIRE(!restored.is_unavailable(&know_nothing)); diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index e3cf527d8b..8aa6ab591c 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -40,6 +40,52 @@ namespace void parse_segments(std::vector>& out_segments); std::vector>> parse_all_segments(); + template + void handle_readwriteback(std::vector& read, + std::vector& write, + bool& writeback, + T&& t, + const std::vector>& segments, + size_t segment_idx) + { + if (segment_idx >= segments.size()) + { + read.push_back(std::move(t)); + return; + } + + auto& mode = segments[segment_idx].second; + + if (mode == "read") + { + read.push_back(std::move(t)); + } + else if (mode == "write") + { + write.push_back(std::move(t)); + } + else if (mode == "writeback") + { + write.push_back(t); + writeback = true; + } + else if (mode == "readwrite") + { + read.push_back(t); + write.push_back(std::move(t)); + } + else if (mode == "readwriteback") + { + read.push_back(t); + write.push_back(t); + writeback = true; + } + else + { + return add_error(msg::format(msgExpectedReadWriteReadWrite), segments[segment_idx].first); + } + } + template void handle_readwrite(std::vector& read, std::vector& write, @@ -74,10 +120,11 @@ namespace } } - void handle_readwrite(bool& read, - bool& write, - const std::vector>& segments, - size_t segment_idx) + void handle_readwriteback(bool& read, + bool& write, + bool& writeback, + const std::vector>& segments, + size_t segment_idx) { if (segment_idx >= segments.size()) { @@ -95,11 +142,22 @@ namespace { write = true; } + else if (mode == "writeback") + { + write = true; + writeback = true; + } else if (mode == "readwrite") { read = true; write = true; } + else if (mode == "readwriteback") + { + read = true; + write = true; + writeback = true; + } else { return add_error(msg::format(msgExpectedReadWriteReadWrite), segments[segment_idx].first); @@ -1508,7 +1566,8 @@ namespace } FilesBinaryProviderConfig config; - handle_readwrite(config.archives_to_read, config.archives_to_write, std::move(p), segments, 2); + handle_readwriteback( + config.archives_to_read, config.archives_to_write, config.write_back, std::move(p), segments, 2); if (segments.size() > 3) { return add_error( @@ -1547,7 +1606,8 @@ namespace } NugetBinaryProviderConfig config; - handle_readwrite(config.configs_to_read, config.configs_to_write, std::move(p), segments, 2); + handle_readwriteback( + config.configs_to_read, config.configs_to_write, config.write_back, std::move(p), segments, 2); if (segments.size() > 3) { return add_error( @@ -1574,7 +1634,8 @@ namespace } NugetBinaryProviderConfig config; - handle_readwrite(config.sources_to_read, config.sources_to_write, std::move(p), segments, 2); + handle_readwriteback( + config.sources_to_read, config.sources_to_write, config.write_back, std::move(p), segments, 2); if (segments.size() > 3) { return add_error( @@ -1616,8 +1677,12 @@ namespace } FilesBinaryProviderConfig config; - handle_readwrite( - config.archives_to_read, config.archives_to_write, Path(*maybe_home.get()), segments, 1); + handle_readwriteback(config.archives_to_read, + config.archives_to_write, + config.write_back, + Path(*maybe_home.get()), + segments, + 1); state->binary_providers.emplace_back(config); state->binary_cache_providers.insert("default"); } @@ -1670,7 +1735,7 @@ namespace config.secrets.push_back(segments[2].second); UrlTemplate url_template = {p}; bool read = false, write = false; - handle_readwrite(read, write, segments, 3); + handle_readwriteback(read, write, config.write_back, segments, 3); if (read) config.url_templates_to_get.push_back(url_template); auto headers = azure_blob_headers(); url_template.headers.assign(headers.begin(), headers.end()); @@ -1710,7 +1775,8 @@ namespace } GcsBinaryProviderConfig config; - handle_readwrite(config.gcs_read_prefixes, config.gcs_write_prefixes, std::move(p), segments, 2); + handle_readwriteback( + config.gcs_read_prefixes, config.gcs_write_prefixes, config.write_back, std::move(p), segments, 2); state->binary_providers.emplace_back(config); state->binary_cache_providers.insert("gcs"); } @@ -1745,7 +1811,8 @@ namespace } AwsBinaryProviderConfig config; - handle_readwrite(config.aws_read_prefixes, config.aws_write_prefixes, std::move(p), segments, 2); + handle_readwriteback( + config.aws_read_prefixes, config.aws_write_prefixes, config.write_back, std::move(p), segments, 2); state->binary_providers.emplace_back(config); state->binary_cache_providers.insert("aws"); } @@ -1801,7 +1868,8 @@ namespace } CosBinaryProviderConfig config; - handle_readwrite(config.cos_read_prefixes, config.cos_write_prefixes, std::move(p), segments, 2); + handle_readwriteback( + config.cos_read_prefixes, config.cos_write_prefixes, config.write_back, std::move(p), segments, 2); state->binary_providers.emplace_back(config); state->binary_cache_providers.insert("cos"); } @@ -1816,7 +1884,7 @@ namespace } GhaBinaryProviderConfig config; - handle_readwrite(config.gha_read, config.gha_write, segments, 1); + handle_readwriteback(config.gha_read, config.gha_write, config.write_back, segments, 1); state->binary_providers.emplace_back(config); state->binary_cache_providers.insert("gha"); } @@ -1883,8 +1951,12 @@ namespace } HttpBinaryProviderConfig config; - handle_readwrite( - config.url_templates_to_get, config.url_templates_to_put, std::move(url_template), segments, 2); + handle_readwriteback(config.url_templates_to_get, + config.url_templates_to_put, + config.write_back, + std::move(url_template), + segments, + 2); state->binary_providers.emplace_back(config); state->binary_cache_providers.insert("http"); } @@ -1904,8 +1976,12 @@ namespace state->binary_cache_providers.insert("upkg"); AzureUpkgBinaryProviderConfig config; - handle_readwrite( - config.upkg_templates_to_get, config.upkg_templates_to_put, std::move(upkg_template), segments, 4); + handle_readwriteback(config.upkg_templates_to_get, + config.upkg_templates_to_put, + config.write_back, + std::move(upkg_template), + segments, + 4); state->binary_providers.emplace_back(config); } else @@ -2131,43 +2207,46 @@ namespace vcpkg std::vector action_ptrs; std::vector restores; std::vector statuses; - for (auto&& provider : m_config.read) + for (const auto& container : m_config.containers) { - action_ptrs.clear(); - restores.clear(); - statuses.clear(); - for (size_t i = 0; i < actions.size(); ++i) + for (const auto& provider : container->read) { - if (actions[i].package_abi()) + action_ptrs.clear(); + restores.clear(); + statuses.clear(); + for (size_t i = 0; i < actions.size(); ++i) { - CacheStatus& status = m_status[*actions[i].package_abi().get()]; - if (status.should_attempt_restore(provider.get())) + if (actions[i].package_abi()) { - action_ptrs.push_back(&actions[i]); - restores.push_back(RestoreResult::unavailable); - statuses.push_back(&status); + CacheStatus& status = m_status[*actions[i].package_abi().get()]; + if (status.should_attempt_restore(provider.get())) + { + action_ptrs.push_back(&actions[i]); + restores.push_back(RestoreResult::unavailable); + statuses.push_back(&status); + } } } - } - if (action_ptrs.empty()) continue; + if (action_ptrs.empty()) continue; - ElapsedTimer timer; - provider->fetch(action_ptrs, restores); - size_t num_restored = 0; - for (size_t i = 0; i < restores.size(); ++i) - { - if (restores[i] == RestoreResult::unavailable) - { - statuses[i]->mark_unavailable(provider.get()); - } - else + ElapsedTimer timer; + provider->fetch(action_ptrs, restores); + size_t num_restored = 0; + for (size_t i = 0; i < restores.size(); ++i) { - statuses[i]->mark_restored(); - ++num_restored; + if (restores[i] == RestoreResult::unavailable) + { + statuses[i]->mark_unavailable(provider.get()); + } + else + { + statuses[i]->mark_restored(container.get()); + ++num_restored; + } } + msg::println(provider->restored_message( + num_restored, timer.elapsed().as())); } - msg::println(provider->restored_message( - num_restored, timer.elapsed().as())); } } @@ -2183,7 +2262,8 @@ namespace vcpkg void ReadOnlyBinaryCache::install_read_provider(std::unique_ptr&& provider) { - m_config.read.push_back(std::move(provider)); + auto& container = m_config.containers.emplace_back(std::make_unique()); + container->read.push_back(std::move(provider)); } std::vector ReadOnlyBinaryCache::precheck(View actions) @@ -2196,34 +2276,37 @@ namespace vcpkg std::vector action_ptrs; std::vector cache_result; std::vector indexes; - for (auto&& provider : m_config.read) + for (const auto& container : m_config.containers) { - action_ptrs.clear(); - cache_result.clear(); - indexes.clear(); - for (size_t i = 0; i < actions.size(); ++i) + for (const auto& provider : container->read) { - if (statuses[i]->should_attempt_precheck(provider.get())) + action_ptrs.clear(); + cache_result.clear(); + indexes.clear(); + for (size_t i = 0; i < actions.size(); ++i) { - action_ptrs.push_back(&actions[i]); - cache_result.push_back(CacheAvailability::unknown); - indexes.push_back(i); + if (statuses[i]->should_attempt_precheck(provider.get())) + { + action_ptrs.push_back(&actions[i]); + cache_result.push_back(CacheAvailability::unknown); + indexes.push_back(i); + } } - } - if (action_ptrs.empty()) continue; + if (action_ptrs.empty()) continue; - provider->precheck(action_ptrs, cache_result); + provider->precheck(action_ptrs, cache_result); - for (size_t i = 0; i < action_ptrs.size(); ++i) - { - auto&& this_status = m_status[*action_ptrs[i]->package_abi().get()]; - if (cache_result[i] == CacheAvailability::available) - { - this_status.mark_available(provider.get()); - } - else if (cache_result[i] == CacheAvailability::unavailable) + for (size_t i = 0; i < action_ptrs.size(); ++i) { - this_status.mark_unavailable(provider.get()); + auto&& this_status = m_status[*action_ptrs[i]->package_abi().get()]; + if (cache_result[i] == CacheAvailability::available) + { + this_status.mark_available(provider.get()); + } + else if (cache_result[i] == CacheAvailability::unavailable) + { + this_status.mark_unavailable(provider.get()); + } } } } @@ -2380,31 +2463,35 @@ namespace vcpkg { auto ok = std::visit( [&](auto&& c) { + auto& container = + m_config.containers.emplace_back(std::make_unique()); + container->write_back = c.write_back; + using T = std::decay_t; if constexpr (std::is_same_v) { for (auto&& dir : c.archives_to_read) { - m_config.read.push_back( + container->read.push_back( std::make_unique(zip_tool, fs, std::move(dir))); } if (!c.archives_to_write.empty()) { - m_config.write.push_back( - std::make_unique(fs, std::move(c.archives_to_write))); + container->write = + std::make_unique(fs, std::move(c.archives_to_write)); } } else if constexpr (std::is_same_v) { for (auto&& url : c.url_templates_to_get) { - m_config.read.push_back(std::make_unique( + container->read.push_back(std::make_unique( zip_tool, fs, buildtrees, std::move(url), c.secrets)); } if (!c.url_templates_to_put.empty()) { - m_config.write.push_back(std::make_unique( - std::move(c.url_templates_to_put), c.secrets)); + container->write = std::make_unique( + std::move(c.url_templates_to_put), c.secrets); } } else if constexpr (std::is_same_v) @@ -2416,13 +2503,13 @@ namespace vcpkg for (auto&& prefix : c.gcs_read_prefixes) { - m_config.read.push_back(std::make_unique( + container->read.push_back(std::make_unique( zip_tool, fs, buildtrees, std::move(prefix), gcs_tool)); } if (!c.gcs_write_prefixes.empty()) { - m_config.write.push_back(std::make_unique( - std::move(c.gcs_write_prefixes), gcs_tool)); + container->write = std::make_unique( + std::move(c.gcs_write_prefixes), gcs_tool); } } else if constexpr (std::is_same_v) @@ -2434,14 +2521,14 @@ namespace vcpkg for (auto&& prefix : c.aws_read_prefixes) { - m_config.read.push_back(std::make_unique( + container->read.push_back(std::make_unique( zip_tool, fs, buildtrees, std::move(prefix), aws_tool)); } if (!c.aws_write_prefixes.empty()) { - m_config.write.push_back(std::make_unique( - std::move(c.aws_write_prefixes), aws_tool)); + container->write = std::make_unique( + std::move(c.aws_write_prefixes), aws_tool); } } else if constexpr (std::is_same_v) @@ -2453,14 +2540,14 @@ namespace vcpkg for (auto&& prefix : c.cos_read_prefixes) { - m_config.read.push_back(std::make_unique( + container->read.push_back(std::make_unique( zip_tool, fs, buildtrees, std::move(prefix), cos_tool)); } if (!c.cos_write_prefixes.empty()) { - m_config.write.push_back(std::make_unique( - std::move(c.cos_write_prefixes), cos_tool)); + container->write = std::make_unique( + std::move(c.cos_write_prefixes), cos_tool); } } else if constexpr (std::is_same_v) @@ -2480,14 +2567,14 @@ namespace vcpkg { const auto& url = *args.actions_cache_url.get(); const auto& token = *args.actions_runtime_token.get(); - m_config.read.push_back( + container->read.push_back( std::make_unique(zip_tool, fs, buildtrees, url, token)); } if (c.gha_write) { const auto& url = *args.actions_cache_url.get(); const auto& token = *args.actions_runtime_token.get(); - m_config.write.push_back(std::make_unique(fs, url, token)); + container->write = std::make_unique(fs, url, token); } } else if constexpr (std::is_same_v) @@ -2501,15 +2588,15 @@ namespace vcpkg buildtrees, s.nuget_prefix); if (!c.sources_to_read.empty()) - m_config.read.push_back(std::make_unique( + container->read.push_back(std::make_unique( nuget_base, nuget_sources_arg(c.sources_to_read))); for (auto&& config : c.configs_to_read) - m_config.read.push_back(std::make_unique( + container->read.push_back(std::make_unique( nuget_base, nuget_configfile_arg(config))); if (!c.sources_to_write.empty() || !c.configs_to_write.empty()) { - m_config.write.push_back(std::make_unique( - nuget_base, std::move(c.sources_to_write), std::move(c.configs_to_write))); + container->write = std::make_unique( + nuget_base, std::move(c.sources_to_write), std::move(c.configs_to_write)); } } } @@ -2519,16 +2606,20 @@ namespace vcpkg { for (auto&& src : c.upkg_templates_to_get) { - m_config.read.push_back(std::make_unique( + container->read.push_back(std::make_unique( tools, out_sink, std::move(src))); } } if (!c.upkg_templates_to_put.empty()) { - m_config.write.push_back(std::make_unique( - tools, out_sink, std::move(c.upkg_templates_to_put))); + container->write = std::make_unique( + tools, out_sink, std::move(c.upkg_templates_to_put)); } } + else + { + static_assert(!std::is_same_v, "non-exhaustive visitor!"); + } return true; }, @@ -2542,8 +2633,10 @@ namespace vcpkg } } - m_needs_nuspec_data = Util::any_of(m_config.write, [](auto&& p) { return p->needs_nuspec_data(); }); - m_needs_zip_file = Util::any_of(m_config.write, [](auto&& p) { return p->needs_zip_file(); }); + m_needs_nuspec_data = + Util::any_of(m_config.containers, [](auto&& c) { return c->write && c->write->needs_nuspec_data(); }); + m_needs_zip_file = + Util::any_of(m_config.containers, [](auto&& c) { return c->write && c->write->needs_zip_file(); }); if (m_needs_zip_file) { m_zip_tool.setup(paths.get_tool_cache(), status_sink); @@ -2562,6 +2655,7 @@ namespace vcpkg if (auto abi = action.package_abi().get()) { bool restored; + const BinaryProviderContainer* restored_container = nullptr; auto it = m_status.find(*abi); if (it == m_status.end()) { @@ -2570,6 +2664,7 @@ namespace vcpkg else { restored = it->second.is_restored(); + restored_container = it->second.get_restored_container(); // Purge all status information on push_success (cache invalidation) // - push_success may delete packages/ (invalidate restore) @@ -2577,7 +2672,22 @@ namespace vcpkg m_status.erase(it); } - if (!restored && !m_config.write.empty()) + size_t count = 0; + if (!restored) + { + count = std::count_if(m_config.containers.begin(), m_config.containers.end(), [](const auto& c) { + return c->write != nullptr; + }); + } + else + { + count = std::count_if( + m_config.containers.begin(), m_config.containers.end(), [restored_container](const auto& c) { + return c->write && restored_container != c.get() && c->write_back; + }); + } + + if (count > 0) { ElapsedTimer timer; BinaryPackageWriteInfo request{action}; @@ -2589,9 +2699,9 @@ namespace vcpkg } m_synchronizer.add_submitted(); - msg::println(msg::format( - msgSubmittingBinaryCacheBackground, msg::spec = action.spec, msg::count = m_config.write.size())); - m_actions_to_push.push(ActionToPush{std::move(request), clean_packages}); + msg::println( + msg::format(msgSubmittingBinaryCacheBackground, msg::spec = action.spec, msg::count = count)); + m_actions_to_push.push(ActionToPush{std::move(request), clean_packages, restored, restored_container}); return; } } @@ -2640,8 +2750,15 @@ namespace vcpkg } size_t num_destinations = 0; - for (auto&& provider : m_config.write) + for (auto&& container : m_config.containers) { + if (!container->write) continue; + + if (action_to_push.write_back && + (!container->write_back || container.get() == action_to_push.restored_container)) + continue; + + const auto& provider = container->write; if (!provider->needs_zip_file() || action_to_push.request.zip_path.has_value()) { num_destinations += provider->push_success(action_to_push.request, m_bg_msg_sink); @@ -2723,12 +2840,15 @@ namespace vcpkg } } - void CacheStatus::mark_restored() noexcept + void CacheStatus::mark_restored(const BinaryProviderContainer* container) noexcept { switch (m_status) { case CacheStatusState::unknown: m_known_unavailable_providers.clear(); [[fallthrough]]; - case CacheStatusState::available: m_status = CacheStatusState::restored; break; + case CacheStatusState::available: + m_status = CacheStatusState::restored; + m_restored_container = container; + break; case CacheStatusState::restored: break; default: Checks::unreachable(VCPKG_LINE_INFO); } @@ -2745,6 +2865,17 @@ namespace vcpkg } } + const BinaryProviderContainer* CacheStatus::get_restored_container() const noexcept + { + switch (m_status) + { + case CacheStatusState::restored: return m_restored_container; + case CacheStatusState::unknown: + case CacheStatusState::available: return nullptr; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + void BinaryConfigParserState::clear() { *this = BinaryConfigParserState(); @@ -2817,7 +2948,7 @@ ExpectedL vcpkg::parse_binary_provider_configs(const st { BinaryConfigParserState s; - BinaryConfigParser default_parser("default,readwrite", "", &s); + BinaryConfigParser default_parser("default,readwriteback", "", &s); default_parser.parse(); if (auto err = default_parser.get_error()) {