Skip to content

Commit 8adc114

Browse files
sadroecksadroeck
committed
feat(binary-cache): Add write-back support to binary cache
1 parent 6081c15 commit 8adc114

File tree

8 files changed

+558
-293
lines changed

8 files changed

+558
-293
lines changed

‎include/vcpkg/base/expected.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,8 @@ namespace vcpkg
102102
{
103103
}
104104

105-
ExpectedT(const ExpectedT& other) noexcept(
106-
std::is_nothrow_copy_constructible_v<Error>&& std::is_nothrow_copy_constructible_v<ExpectedHolder<T>>)
105+
ExpectedT(const ExpectedT& other) noexcept(std::is_nothrow_copy_constructible_v<Error> &&
106+
std::is_nothrow_copy_constructible_v<ExpectedHolder<T>>)
107107
: value_is_error(other.value_is_error)
108108
{
109109
if (value_is_error)
@@ -116,8 +116,8 @@ namespace vcpkg
116116
}
117117
}
118118

119-
ExpectedT(ExpectedT&& other) noexcept(
120-
std::is_nothrow_move_constructible_v<Error>&& std::is_nothrow_move_constructible_v<ExpectedHolder<T>>)
119+
ExpectedT(ExpectedT&& other) noexcept(std::is_nothrow_move_constructible_v<Error> &&
120+
std::is_nothrow_move_constructible_v<ExpectedHolder<T>>)
121121
: value_is_error(other.value_is_error)
122122
{
123123
if (value_is_error)

‎include/vcpkg/base/optional.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,8 @@ namespace vcpkg
7676
}
7777
}
7878

79-
OptionalStorage& operator=(const OptionalStorage& o) noexcept(
80-
std::is_nothrow_copy_constructible_v<T>&& std::is_nothrow_copy_assignable_v<T>)
79+
OptionalStorage& operator=(const OptionalStorage& o) noexcept(std::is_nothrow_copy_constructible_v<T> &&
80+
std::is_nothrow_copy_assignable_v<T>)
8181
{
8282
if (m_is_present && o.m_is_present)
8383
{

‎include/vcpkg/base/util.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,19 @@ namespace vcpkg::Util
4444

4545
return false;
4646
}
47+
template<class Vec, class Filter>
48+
std::vector<ElementT<Vec>> filtered_copy(const Vec& container, const Filter&& filter)
49+
{
50+
std::vector<ElementT<Vec>> ret;
51+
for (auto&& item : container)
52+
{
53+
if (filter(item))
54+
{
55+
ret.push_back(item);
56+
}
57+
}
58+
return ret;
59+
}
4760
template<class Vec, class Key>
4861
bool contains(const Vec& container, const Key& item)
4962
{

‎include/vcpkg/binarycaching.h‎

Lines changed: 58 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -25,18 +25,26 @@
2525

2626
namespace vcpkg
2727
{
28+
/// Unique identifier for a provider
29+
using ProviderId = size_t;
30+
2831
struct CacheStatus
2932
{
33+
using ReaderProviders = std::vector<const IReadBinaryProvider*>;
34+
3035
bool should_attempt_precheck(const IReadBinaryProvider* sender) const noexcept;
3136
bool should_attempt_restore(const IReadBinaryProvider* sender) const noexcept;
37+
bool should_attempt_write_back(ProviderId provider_id) const noexcept;
3238

3339
bool is_unavailable(const IReadBinaryProvider* sender) const noexcept;
3440
const IReadBinaryProvider* get_available_provider() const noexcept;
41+
ReaderProviders& get_unavailable_providers() noexcept;
3542
bool is_restored() const noexcept;
3643

3744
void mark_unavailable(const IReadBinaryProvider* sender);
3845
void mark_available(const IReadBinaryProvider* sender) noexcept;
3946
void mark_restored() noexcept;
47+
void mark_written_back(ProviderId provider_id) noexcept;
4048

4149
private:
4250
CacheStatusState m_status = CacheStatusState::unknown;
@@ -75,7 +83,9 @@ namespace vcpkg
7583

7684
/// Called upon a successful build of `action` to store those contents in the binary cache.
7785
/// returns the number of successful uploads
78-
virtual size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) = 0;
86+
virtual size_t push_success(const BinaryPackageWriteInfo& request,
87+
MessageSink& msg_sink,
88+
CacheStatus& cache_status) = 0;
7989

8090
virtual bool needs_nuspec_data() const = 0;
8191
virtual bool needs_zip_file() const = 0;
@@ -103,6 +113,11 @@ namespace vcpkg
103113

104114
virtual LocalizedString restored_message(size_t count,
105115
std::chrono::high_resolution_clock::duration elapsed) const = 0;
116+
117+
/// Unique identifier for this provider.
118+
///
119+
/// Used by the cache to exclude cache providers during the write-back phase.
120+
virtual ProviderId id() const = 0;
106121
};
107122

108123
struct UrlTemplate
@@ -114,44 +129,62 @@ namespace vcpkg
114129
std::string instantiate_variables(const BinaryPackageReadInfo& info) const;
115130
};
116131

132+
struct GithubActionsInfo
133+
{
134+
};
135+
117136
struct NuGetRepoInfo
118137
{
119138
std::string repo;
120139
std::string branch;
121140
std::string commit;
122141
};
123142

143+
enum class CacheType
144+
{
145+
Read,
146+
Write,
147+
ReadWrite
148+
};
149+
150+
template<typename T>
151+
struct CacheProvider
152+
{
153+
ProviderId id;
154+
T source;
155+
CacheType cache_type;
156+
157+
[[nodiscard]] constexpr bool is_read() const noexcept
158+
{
159+
return cache_type == CacheType::Read || cache_type == CacheType::ReadWrite;
160+
}
161+
162+
[[nodiscard]] constexpr bool is_write() const noexcept
163+
{
164+
return cache_type == CacheType::Write || cache_type == CacheType::ReadWrite;
165+
}
166+
};
167+
168+
template<typename T>
169+
using ProviderList = std::vector<CacheProvider<T>>;
170+
124171
struct BinaryConfigParserState
125172
{
173+
ProviderId provider_count = 0;
126174
bool nuget_interactive = false;
127175
std::set<StringLiteral> binary_cache_providers;
128176

129177
std::string nugettimeout = "100";
130178

131-
std::vector<Path> archives_to_read;
132-
std::vector<Path> archives_to_write;
133-
134-
std::vector<UrlTemplate> url_templates_to_get;
135-
std::vector<UrlTemplate> url_templates_to_put;
136-
137-
std::vector<std::string> gcs_read_prefixes;
138-
std::vector<std::string> gcs_write_prefixes;
139-
140-
std::vector<std::string> aws_read_prefixes;
141-
std::vector<std::string> aws_write_prefixes;
179+
ProviderList<Path> archives;
180+
ProviderList<UrlTemplate> url_templates;
181+
ProviderList<std::string> gcs_prefixes;
182+
ProviderList<std::string> aws_prefixes;
142183
bool aws_no_sign_request = false;
143-
144-
std::vector<std::string> cos_read_prefixes;
145-
std::vector<std::string> cos_write_prefixes;
146-
147-
bool gha_write = false;
148-
bool gha_read = false;
149-
150-
std::vector<std::string> sources_to_read;
151-
std::vector<std::string> sources_to_write;
152-
153-
std::vector<Path> configs_to_read;
154-
std::vector<Path> configs_to_write;
184+
ProviderList<std::string> cos_prefixes;
185+
Optional<CacheProvider<GithubActionsInfo>> gha;
186+
ProviderList<std::string> sources;
187+
ProviderList<Path> configs;
155188

156189
std::vector<std::string> secrets;
157190

@@ -182,7 +215,7 @@ namespace vcpkg
182215
/// executing `actions`.
183216
void fetch(View<InstallPlanAction> actions);
184217

185-
bool is_restored(const InstallPlanAction& ipa) const;
218+
Optional<CacheStatus> cache_status(const InstallPlanAction& ipa) const;
186219

187220
/// Checks whether the `actions` are present in the cache, without restoring them. Used by CI to determine
188221
/// missing packages.

‎src/vcpkg-test/binarycaching.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ struct KnowNothingBinaryProvider : IReadBinaryProvider
3636
{
3737
return LocalizedString::from_raw("Nothing");
3838
}
39+
40+
ProviderId id() const override { return 1; }
3941
};
4042

4143
TEST_CASE ("CacheStatus operations", "[BinaryCache]")

0 commit comments

Comments
 (0)