feat(binary-cache): Add write-back support to binary cache

Author: sadroeck

include/vcpkg/base/util.h

@@ -44,6 +44,19 @@ namespace vcpkg::Util
 
             return false;
         }
+        template<class Vec, class Filter>
+        std::vector<ElementT<Vec>> filtered_copy(const Vec& container, const Filter&& filter)
+        {
+            std::vector<ElementT<Vec>> ret;
+            for (auto&& item : container)
+            {
+                if (filter(item))
+                {
+                    ret.push_back(item);
+                }
+            }
+            return ret;
+        }
         template<class Vec, class Key>
         bool contains(const Vec& container, const Key& item)
         {

include/vcpkg/binarycaching.h

@@ -25,18 +25,26 @@
 
 namespace vcpkg
 {
+    /// Unique identifier for a provider
+    using ProviderId = size_t;
+
     struct CacheStatus
     {
+        using ReaderProviders = std::vector<const IReadBinaryProvider*>;
+
         bool should_attempt_precheck(const IReadBinaryProvider* sender) const noexcept;
         bool should_attempt_restore(const IReadBinaryProvider* sender) const noexcept;
+        bool should_attempt_write_back(ProviderId provider_id) const noexcept;
 
         bool is_unavailable(const IReadBinaryProvider* sender) const noexcept;
         const IReadBinaryProvider* get_available_provider() const noexcept;
+        ReaderProviders& get_unavailable_providers() noexcept;
         bool is_restored() const noexcept;
 
         void mark_unavailable(const IReadBinaryProvider* sender);
         void mark_available(const IReadBinaryProvider* sender) noexcept;
         void mark_restored() noexcept;
+        void mark_written_back(ProviderId provider_id) noexcept;
 
     private:
         CacheStatusState m_status = CacheStatusState::unknown;
@@ -75,7 +83,9 @@ namespace vcpkg
 
         /// Called upon a successful build of `action` to store those contents in the binary cache.
         /// returns the number of successful uploads
-        virtual size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) = 0;
+        virtual size_t push_success(const BinaryPackageWriteInfo& request,
+                                    MessageSink& msg_sink,
+                                    CacheStatus& cache_status) = 0;
 
         virtual bool needs_nuspec_data() const = 0;
         virtual bool needs_zip_file() const = 0;
@@ -103,6 +113,11 @@ namespace vcpkg
 
         virtual LocalizedString restored_message(size_t count,
                                                  std::chrono::high_resolution_clock::duration elapsed) const = 0;
+
+        /// Unique identifier for this provider.
+        ///
+        /// Used by the cache to exclude cache providers during the write-back phase.
+        virtual ProviderId id() const = 0;
     };
 
     struct UrlTemplate
@@ -114,44 +129,62 @@ namespace vcpkg
         std::string instantiate_variables(const BinaryPackageReadInfo& info) const;
     };
 
+    struct GithubActionsInfo
+    {
+    };
+
     struct NuGetRepoInfo
     {
         std::string repo;
         std::string branch;
         std::string commit;
     };
 
+    enum class CacheType
+    {
+        Read,
+        Write,
+        ReadWrite
+    };
+
+    template<typename T>
+    struct CacheProvider
+    {
+        ProviderId id;
+        T source;
+        CacheType cache_type;
+
+        [[nodiscard]] constexpr bool is_read() const noexcept
+        {
+            return cache_type == CacheType::Read || cache_type == CacheType::ReadWrite;
+        }
+
+        [[nodiscard]] constexpr bool is_write() const noexcept
+        {
+            return cache_type == CacheType::Write || cache_type == CacheType::ReadWrite;
+        }
+    };
+
+    template<typename T>
+    using ProviderList = std::vector<CacheProvider<T>>;
+
     struct BinaryConfigParserState
     {
+        ProviderId provider_count = 0;
         bool nuget_interactive = false;
         std::set<StringLiteral> binary_cache_providers;
 
         std::string nugettimeout = "100";
 
-        std::vector<Path> archives_to_read;
-        std::vector<Path> archives_to_write;
-
-        std::vector<UrlTemplate> url_templates_to_get;
-        std::vector<UrlTemplate> url_templates_to_put;
-
-        std::vector<std::string> gcs_read_prefixes;
-        std::vector<std::string> gcs_write_prefixes;
-
-        std::vector<std::string> aws_read_prefixes;
-        std::vector<std::string> aws_write_prefixes;
+        ProviderList<Path> archives;
+        ProviderList<UrlTemplate> url_templates;
+        ProviderList<std::string> gcs_prefixes;
+        ProviderList<std::string> aws_prefixes;
         bool aws_no_sign_request = false;
-
-        std::vector<std::string> cos_read_prefixes;
-        std::vector<std::string> cos_write_prefixes;
-
-        bool gha_write = false;
-        bool gha_read = false;
-
-        std::vector<std::string> sources_to_read;
-        std::vector<std::string> sources_to_write;
-
-        std::vector<Path> configs_to_read;
-        std::vector<Path> configs_to_write;
+        ProviderList<std::string> cos_prefixes;
+        Optional<CacheProvider<GithubActionsInfo>> gha;
+        ProviderList<std::string> sources;
+        ProviderList<Path> configs;
 
         std::vector<std::string> secrets;
 
@@ -182,7 +215,7 @@ namespace vcpkg
         /// executing `actions`.
         void fetch(View<InstallPlanAction> actions);
 
-        bool is_restored(const InstallPlanAction& ipa) const;
+        Optional<CacheStatus> cache_status(const InstallPlanAction& ipa) const;
 
         /// Checks whether the `actions` are present in the cache, without restoring them. Used by CI to determine
         /// missing packages.

src/vcpkg-test/binarycaching.cpp

@@ -36,6 +36,8 @@ struct KnowNothingBinaryProvider : IReadBinaryProvider
     {
         return LocalizedString::from_raw("Nothing");
     }
+
+    ProviderId id() const override { return 1; }
 };
 
 TEST_CASE ("CacheStatus operations", "[BinaryCache]")