feat(binary-cache): Add write-back support to binary cache

Author: sadroeck

azure-pipelines/end-to-end-tests-dir/binary-caching.ps1

@@ -47,6 +47,27 @@ Require-FileExists "$installRoot/$Triplet/include/hello-1.h"
 Require-FileExists "$buildtreesRoot/vcpkg-hello-world-1/src"
 Require-FileNotExists "$buildtreesRoot/detect_compiler"
 
+# Test write back to files archive
+Remove-Item -Recurse -Force $installRoot
+Remove-Item -Recurse -Force $buildtreesRoot
+Run-Vcpkg -TestArgs ($commonArgs + @("install","vcpkg-hello-world-1", "vcpkg-cmake", "vcpkg-cmake-config", "--x-binarysource=clear;files,$ArchiveRoot;files,$ArchiveRootSecondary,write;files,$ArchiveRootUnused,read"))
+Throw-IfFailed
+Require-FileExists "$installRoot/$Triplet/include/hello-1.h"
+Require-FileNotExists "$buildtreesRoot/vcpkg-hello-world-1/src"
+if ((Get-ChildItem $ArchiveRootSecondary -Filter '*.zip' | Measure-Object).Count -ne 4) {
+    throw "In '$CurrentTest': did not write back exactly 4 packages to the secondary archive"
+}
+if ((Get-ChildItem $ArchiveRootUnused -Filter '*.zip' | Measure-Object).Count -ne 0) {
+    throw "In '$CurrentTest': some packages were written to the unused archive"
+}
+
+# Test restore from secondary archive
+Remove-Item -Recurse -Force $installRoot
+Run-Vcpkg -TestArgs ($commonArgs + @("install","vcpkg-hello-world-1", "vcpkg-cmake", "vcpkg-cmake-config", "--x-binarysource=clear;files,$ArchiveRootSecondary,read"))
+Throw-IfFailed
+Require-FileExists "$installRoot/$Triplet/include/hello-1.h"
+Require-FileNotExists "$buildtreesRoot/vcpkg-hello-world-1/src"
+
 if(-Not $IsLinux -and -Not $IsMacOS) {
     # Test restoring from nuget
     Remove-Item -Recurse -Force $installRoot
@@ -78,8 +99,8 @@ if(-Not $IsLinux -and -Not $IsMacOS) {
     Require-FileExists "$installRoot/$Triplet/include/hello-2.h"
     Require-FileNotExists "$buildtreesRoot/vcpkg-hello-world-1/src"
     Require-FileExists "$buildtreesRoot/vcpkg-hello-world-2/src"
-    if ((Get-ChildItem $NuGetRoot -Filter '*.nupkg' | Measure-Object).Count -ne 1) {
-        throw "In '$CurrentTest': did not create exactly 1 NuGet package"
+    if ((Get-ChildItem $NuGetRoot -Filter '*.nupkg' | Measure-Object).Count -ne 4) {
+        throw "In '$CurrentTest': did not create exactly 4 NuGet packages"
     }
 
     # Test export

azure-pipelines/end-to-end-tests-prelude.ps1

@@ -5,6 +5,8 @@ $packagesRoot = Join-Path $TestingRoot 'packages'
 $NuGetRoot = Join-Path $TestingRoot 'nuget'
 $NuGetRoot2 = Join-Path $TestingRoot 'nuget2'
 $ArchiveRoot = Join-Path $TestingRoot 'archives'
+$ArchiveRootSecondary = Join-Path $TestingRoot 'archives-secondary'
+$ArchiveRootUnused = Join-Path $TestingRoot 'archives-unused'
 $VersionFilesRoot = Join-Path $TestingRoot 'version-test'
 $directoryArgs = @(
     "--x-buildtrees-root=$buildtreesRoot",

include/vcpkg/base/util.h

@@ -44,6 +44,19 @@ namespace vcpkg::Util
 
             return false;
         }
+        template<class Vec, class Filter>
+        std::vector<ElementT<Vec>> filtered_copy(const Vec& container, const Filter&& filter)
+        {
+            std::vector<ElementT<Vec>> ret;
+            for (auto&& item : container)
+            {
+                if (filter(item))
+                {
+                    ret.push_back(item);
+                }
+            }
+            return ret;
+        }
         template<class Vec, class Key>
         bool contains(const Vec& container, const Key& item)
         {

include/vcpkg/binarycaching.h

@@ -25,18 +25,26 @@
 
 namespace vcpkg
 {
+    /// Unique identifier for a provider
+    using ProviderId = size_t;
+
     struct CacheStatus
     {
+        using ReaderProviders = std::vector<const IReadBinaryProvider*>;
+
         bool should_attempt_precheck(const IReadBinaryProvider* sender) const noexcept;
         bool should_attempt_restore(const IReadBinaryProvider* sender) const noexcept;
+        bool should_attempt_write_back(ProviderId provider_id) const noexcept;
 
         bool is_unavailable(const IReadBinaryProvider* sender) const noexcept;
         const IReadBinaryProvider* get_available_provider() const noexcept;
+        ReaderProviders& get_unavailable_providers() noexcept;
         bool is_restored() const noexcept;
 
         void mark_unavailable(const IReadBinaryProvider* sender);
         void mark_available(const IReadBinaryProvider* sender) noexcept;
         void mark_restored() noexcept;
+        void mark_written_back(ProviderId provider_id) noexcept;
 
     private:
         CacheStatusState m_status = CacheStatusState::unknown;
@@ -75,10 +83,13 @@ namespace vcpkg
 
         /// Called upon a successful build of `action` to store those contents in the binary cache.
         /// returns the number of successful uploads
-        virtual size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) = 0;
+        virtual size_t push_success(const BinaryPackageWriteInfo& request,
+                                    MessageSink& msg_sink,
+                                    CacheStatus& cache_status) = 0;
 
         virtual bool needs_nuspec_data() const = 0;
         virtual bool needs_zip_file() const = 0;
+        virtual std::vector<ProviderId> get_provider_ids() const = 0;
     };
 
     struct IReadBinaryProvider
@@ -93,6 +104,10 @@ namespace vcpkg
         /// Prerequisites: actions[i].package_abi(), out_status.size() == actions.size()
         virtual void fetch(View<const InstallPlanAction*> actions, Span<RestoreResult> out_status) const = 0;
 
+        /// Flag to indicate if the provider supports an efficient check to see if a certain set of packages are
+        /// available. If not, packages will assumed to be present & will always be fetched.
+        virtual bool can_precheck() const = 0;
+
         /// Checks whether the `actions` are present in the cache, without restoring them.
         ///
         /// Used by CI to determine missing packages. For each `i`, out_status[i] should be set to
@@ -103,6 +118,11 @@ namespace vcpkg
 
         virtual LocalizedString restored_message(size_t count,
                                                  std::chrono::high_resolution_clock::duration elapsed) const = 0;
+
+        /// Unique identifier for this provider.
+        ///
+        /// Used by the cache to exclude cache providers during the write-back phase.
+        virtual ProviderId id() const = 0;
     };
 
     struct UrlTemplate
@@ -114,44 +134,62 @@ namespace vcpkg
         std::string instantiate_variables(const BinaryPackageReadInfo& info) const;
     };
 
+    struct GithubActionsInfo
+    {
+    };
+
     struct NuGetRepoInfo
     {
         std::string repo;
         std::string branch;
         std::string commit;
     };
 
+    enum class CacheType
+    {
+        Read,
+        Write,
+        ReadWrite
+    };
+
+    template<typename T>
+    struct CacheProvider
+    {
+        ProviderId id;
+        T source;
+        CacheType cache_type;
+
+        [[nodiscard]] constexpr bool is_read() const noexcept
+        {
+            return cache_type == CacheType::Read || cache_type == CacheType::ReadWrite;
+        }
+
+        [[nodiscard]] constexpr bool is_write() const noexcept
+        {
+            return cache_type == CacheType::Write || cache_type == CacheType::ReadWrite;
+        }
+    };
+
+    template<typename T>
+    using ProviderList = std::vector<CacheProvider<T>>;
+
     struct BinaryConfigParserState
     {
+        ProviderId provider_count = 0;
         bool nuget_interactive = false;
         std::set<StringLiteral> binary_cache_providers;
 
         std::string nugettimeout = "100";
 
-        std::vector<Path> archives_to_read;
-        std::vector<Path> archives_to_write;
-
-        std::vector<UrlTemplate> url_templates_to_get;
-        std::vector<UrlTemplate> url_templates_to_put;
-
-        std::vector<std::string> gcs_read_prefixes;
-        std::vector<std::string> gcs_write_prefixes;
-
-        std::vector<std::string> aws_read_prefixes;
-        std::vector<std::string> aws_write_prefixes;
+        ProviderList<Path> archives;
+        ProviderList<UrlTemplate> url_templates;
+        ProviderList<std::string> gcs_prefixes;
+        ProviderList<std::string> aws_prefixes;
         bool aws_no_sign_request = false;
-
-        std::vector<std::string> cos_read_prefixes;
-        std::vector<std::string> cos_write_prefixes;
-
-        bool gha_write = false;
-        bool gha_read = false;
-
-        std::vector<std::string> sources_to_read;
-        std::vector<std::string> sources_to_write;
-
-        std::vector<Path> configs_to_read;
-        std::vector<Path> configs_to_write;
+        ProviderList<std::string> cos_prefixes;
+        Optional<CacheProvider<GithubActionsInfo>> gha;
+        ProviderList<std::string> sources;
+        ProviderList<Path> configs;
 
         std::vector<std::string> secrets;
 
@@ -173,6 +211,8 @@ namespace vcpkg
         NuGetRepoInfo nuget_repo;
     };
 
+    [[nodiscard]] bool HasWriteOnlyProviders(const BinaryProviders& providers);
+
     struct ReadOnlyBinaryCache
     {
         ReadOnlyBinaryCache() = default;
@@ -182,7 +222,7 @@ namespace vcpkg
         /// executing `actions`.
         void fetch(View<InstallPlanAction> actions);
 
-        bool is_restored(const InstallPlanAction& ipa) const;
+        Optional<CacheStatus> cache_status(const InstallPlanAction& ipa) const;
 
         /// Checks whether the `actions` are present in the cache, without restoring them. Used by CI to determine
         /// missing packages.
@@ -192,6 +232,10 @@ namespace vcpkg
     protected:
         BinaryProviders m_config;
 
+        /// Flag to indicate that at least one provider is write-only. This implies that the write-back phase should
+        /// always take place for every Cache item.
+        bool m_has_write_only_providers;
+
         std::unordered_map<std::string, CacheStatus> m_status;
     };
 

src/vcpkg-test/binarycaching.cpp

@@ -23,6 +23,9 @@ struct KnowNothingBinaryProvider : IReadBinaryProvider
             CHECK(out_status[idx] == RestoreResult::unavailable);
         }
     }
+
+    bool can_precheck() const override { return true; }
+
     void precheck(View<const InstallPlanAction*> actions, Span<CacheAvailability> out_status) const override
     {
         REQUIRE(actions.size() == out_status.size());
@@ -36,6 +39,8 @@ struct KnowNothingBinaryProvider : IReadBinaryProvider
     {
         return LocalizedString::from_raw("Nothing");
     }
+
+    ProviderId id() const override { return 1; }
 };
 
 TEST_CASE ("CacheStatus operations", "[BinaryCache]")