Support Azure universal packages as a binary caching provider (#1491)

* add universal support

* fix

* fix

* review

* review

* fix

Author: data-queue

include/vcpkg/base/message-data.inc.h

@@ -1490,6 +1490,12 @@ DECLARE_MESSAGE(HelpBinaryCachingAzBlob,
                 "**Experimental: will change or be removed without warning**\n"
                 "Adds an Azure Blob Storage source. Uses Shared Access Signature validation. <url> should include "
                 "the container path. <sas> must be be prefixed with a \"?\".")
+DECLARE_MESSAGE(HelpBinaryCachingAzUpkg,
+                (),
+                "Printed as the 'definition' for 'x-az-universal,<organization>,<project>,<feed>[,<rw>]'.",
+                "**Experimental: will change or be removed without warning**\n"
+                "Adds a Universal Package Azure Artifacts source. Uses the Azure CLI "
+                "(az artifacts) for uploads and downloads.")
 DECLARE_MESSAGE(HelpBinaryCachingCos,
                 (),
                 "Printed as the 'definition' for 'x-cos,<prefix>[,<rw>]'.",
@@ -1811,6 +1817,10 @@ DECLARE_MESSAGE(InvalidArgumentRequiresBaseUrlAndToken,
                 (msg::binary_source),
                 "",
                 "invalid argument: binary config '{binary_source}' requires at least a base-url and a SAS token")
+DECLARE_MESSAGE(InvalidArgumentRequiresFourOrFiveArguments,
+                (msg::binary_source),
+                "",
+                "invalid argument: binary config '{binary_source}' requires 4 or 5 arguments")
 DECLARE_MESSAGE(InvalidArgumentRequiresNoneArguments,
                 (msg::binary_source),
                 "",
@@ -2576,6 +2586,11 @@ DECLARE_MESSAGE(RestoredPackagesFromAWS,
                 (msg::count, msg::elapsed),
                 "",
                 "Restored {count} package(s) from AWS in {elapsed}. Use --debug to see more details.")
+DECLARE_MESSAGE(RestoredPackagesFromAZUPKG,
+                (msg::count, msg::elapsed),
+                "",
+                "Restored {count} package(s) from Universal Packages in {elapsed}. "
+                "Use --debug to see more details.")
 DECLARE_MESSAGE(RestoredPackagesFromCOS,
                 (msg::count, msg::elapsed),
                 "",

include/vcpkg/binarycaching.h

@@ -121,6 +121,13 @@ namespace vcpkg
         std::string commit;
     };
 
+    struct AzureUpkgSource
+    {
+        std::string organization;
+        std::string project;
+        std::string feed;
+    };
+
     struct BinaryConfigParserState
     {
         bool nuget_interactive = false;
@@ -147,6 +154,9 @@ namespace vcpkg
         bool gha_write = false;
         bool gha_read = false;
 
+        std::vector<AzureUpkgSource> upkg_templates_to_get;
+        std::vector<AzureUpkgSource> upkg_templates_to_put;
+
         std::vector<std::string> sources_to_read;
         std::vector<std::string> sources_to_write;
 

include/vcpkg/binarycaching.private.h

@@ -19,19 +19,19 @@ namespace vcpkg
     // - v?<X>.<Y><whatever> -> <X>.<Y>.0-vcpkg<abitag>
     // - v?<X>.<Y>.<Z><whatever> -> <X>.<Y>.<Z>-vcpkg<abitag>
     // - anything else -> 0.0.0-vcpkg<abitag>
-    std::string format_version_for_nugetref(StringView version_text, StringView abi_tag);
+    std::string format_version_for_feedref(StringView version_text, StringView abi_tag);
 
-    struct NugetReference
+    struct FeedReference
     {
-        NugetReference(std::string id, std::string version) : id(std::move(id)), version(std::move(version)) { }
+        FeedReference(std::string id, std::string version) : id(std::move(id)), version(std::move(version)) { }
 
         std::string id;
         std::string version;
 
         std::string nupkg_filename() const { return Strings::concat(id, '.', version, ".nupkg"); }
     };
 
-    NugetReference make_nugetref(const InstallPlanAction& action, StringView prefix);
+    FeedReference make_nugetref(const InstallPlanAction& action, StringView prefix);
 
     std::string generate_nuspec(const Path& package_dir,
                                 const InstallPlanAction& action,

include/vcpkg/metrics.h

@@ -24,6 +24,7 @@ namespace vcpkg
         BinaryCachingHttp,
         BinaryCachingNuget,
         BinaryCachingSource,
+        BinaryCachingUpkg,
         ErrorVersioningDisabled,
         ErrorVersioningNoBaseline,
         GitHubRepository,

include/vcpkg/tools.h

@@ -23,6 +23,7 @@ namespace vcpkg
         static constexpr StringLiteral GIT = "git";
         static constexpr StringLiteral GSUTIL = "gsutil";
         static constexpr StringLiteral AWSCLI = "aws";
+        static constexpr StringLiteral AZCLI = "az";
         static constexpr StringLiteral COSCLI = "coscli";
         static constexpr StringLiteral MONO = "mono";
         static constexpr StringLiteral NINJA = "ninja";
@@ -45,6 +46,11 @@ namespace vcpkg
         virtual const std::string& get_tool_version(StringView tool, MessageSink& status_sink) const = 0;
     };
 
+    ExpectedL<std::string> extract_prefixed_nonquote(StringLiteral prefix,
+                                                     StringLiteral tool_name,
+                                                     std::string&& output,
+                                                     const Path& exe_path);
+
     ExpectedL<std::string> extract_prefixed_nonwhitespace(StringLiteral prefix,
                                                           StringLiteral tool_name,
                                                           std::string&& output,

locales/messages.json

@@ -858,6 +858,8 @@
   "HelpBinaryCachingAwsHeader": "Azure Web Services sources",
   "HelpBinaryCachingAzBlob": "**Experimental: will change or be removed without warning**\nAdds an Azure Blob Storage source. Uses Shared Access Signature validation. <url> should include the container path. <sas> must be be prefixed with a \"?\".",
   "_HelpBinaryCachingAzBlob.comment": "Printed as the 'definition' for 'x-azblob,<url>,<sas>[,<rw>]'.",
+  "HelpBinaryCachingAzUpkg": "**Experimental: will change or be removed without warning**\nAdds a Universal Package Azure Artifacts source. Uses the Azure CLI (az artifacts) for uploads and downloads.",
+  "_HelpBinaryCachingAzUpkg.comment": "Printed as the 'definition' for 'x-az-universal,<organization>,<project>,<feed>[,<rw>]'.",
   "HelpBinaryCachingCos": "**Experimental: will change or be removed without warning**\nAdds an COS source. Uses the cos CLI for uploads and downloads. <prefix> should include the scheme 'cos://' and be suffixed with a \"/\".",
   "_HelpBinaryCachingCos.comment": "Printed as the 'definition' for 'x-cos,<prefix>[,<rw>]'.",
   "HelpBinaryCachingDefaults": "Adds the default file-based location. Based on your system settings, the default path to store binaries is \"{path}\". This consults %LOCALAPPDATA%/%APPDATA% on Windows and $XDG_CACHE_HOME or $HOME on other platforms.",
@@ -997,6 +999,8 @@
   "_InvalidArgumentRequiresBaseUrl.comment": "An example of {base_url} is azblob://. An example of {binary_source} is azblob.",
   "InvalidArgumentRequiresBaseUrlAndToken": "invalid argument: binary config '{binary_source}' requires at least a base-url and a SAS token",
   "_InvalidArgumentRequiresBaseUrlAndToken.comment": "An example of {binary_source} is azblob.",
+  "InvalidArgumentRequiresFourOrFiveArguments": "invalid argument: binary config '{binary_source}' requires 4 or 5 arguments",
+  "_InvalidArgumentRequiresFourOrFiveArguments.comment": "An example of {binary_source} is azblob.",
   "InvalidArgumentRequiresNoWildcards": "cannot fix Windows path case for path containing wildcards: {path}",
   "_InvalidArgumentRequiresNoWildcards.comment": "An example of {path} is /foo/bar.",
   "InvalidArgumentRequiresNoneArguments": "invalid argument: binary config '{binary_source}' does not take arguments",
@@ -1359,6 +1363,8 @@
   "_ResponseFileCode.comment": "Explains to the user that they can use response files on the command line, 'response_file' must have no spaces and be a legal file name.",
   "RestoredPackagesFromAWS": "Restored {count} package(s) from AWS in {elapsed}. Use --debug to see more details.",
   "_RestoredPackagesFromAWS.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.",
+  "RestoredPackagesFromAZUPKG": "Restored {count} package(s) from Universal Packages in {elapsed}. Use --debug to see more details.",
+  "_RestoredPackagesFromAZUPKG.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.",
   "RestoredPackagesFromCOS": "Restored {count} package(s) from COS in {elapsed}. Use --debug to see more details.",
   "_RestoredPackagesFromCOS.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.",
   "RestoredPackagesFromFiles": "Restored {count} package(s) from {path} in {elapsed}. Use --debug to see more details.",

src/vcpkg-test/binarycaching.cpp

@@ -166,30 +166,30 @@ TEST_CASE ("CacheStatus operations", "[BinaryCache]")
     REQUIRE(assignee.is_restored());
 }
 
-TEST_CASE ("format_version_for_nugetref semver-ish", "[format_version_for_nugetref]")
+TEST_CASE ("format_version_for_feedref semver-ish", "[format_version_for_feedref]")
 {
-    REQUIRE(format_version_for_nugetref("0.0.0", "abitag") == "0.0.0-vcpkgabitag");
-    REQUIRE(format_version_for_nugetref("1.0.1", "abitag") == "1.0.1-vcpkgabitag");
-    REQUIRE(format_version_for_nugetref("1.01.000", "abitag") == "1.1.0-vcpkgabitag");
-    REQUIRE(format_version_for_nugetref("1.2", "abitag") == "1.2.0-vcpkgabitag");
-    REQUIRE(format_version_for_nugetref("v52", "abitag") == "52.0.0-vcpkgabitag");
-    REQUIRE(format_version_for_nugetref("v09.01.02", "abitag") == "9.1.2-vcpkgabitag");
-    REQUIRE(format_version_for_nugetref("1.1.1q", "abitag") == "1.1.1-vcpkgabitag");
-    REQUIRE(format_version_for_nugetref("1", "abitag") == "1.0.0-vcpkgabitag");
+    REQUIRE(format_version_for_feedref("0.0.0", "abitag") == "0.0.0-vcpkgabitag");
+    REQUIRE(format_version_for_feedref("1.0.1", "abitag") == "1.0.1-vcpkgabitag");
+    REQUIRE(format_version_for_feedref("1.01.000", "abitag") == "1.1.0-vcpkgabitag");
+    REQUIRE(format_version_for_feedref("1.2", "abitag") == "1.2.0-vcpkgabitag");
+    REQUIRE(format_version_for_feedref("v52", "abitag") == "52.0.0-vcpkgabitag");
+    REQUIRE(format_version_for_feedref("v09.01.02", "abitag") == "9.1.2-vcpkgabitag");
+    REQUIRE(format_version_for_feedref("1.1.1q", "abitag") == "1.1.1-vcpkgabitag");
+    REQUIRE(format_version_for_feedref("1", "abitag") == "1.0.0-vcpkgabitag");
 }
 
-TEST_CASE ("format_version_for_nugetref date", "[format_version_for_nugetref]")
+TEST_CASE ("format_version_for_feedref date", "[format_version_for_feedref]")
 {
-    REQUIRE(format_version_for_nugetref("2020-06-26", "abitag") == "2020.6.26-vcpkgabitag");
-    REQUIRE(format_version_for_nugetref("20-06-26", "abitag") == "0.0.0-vcpkgabitag");
-    REQUIRE(format_version_for_nugetref("2020-06-26-release", "abitag") == "2020.6.26-vcpkgabitag");
-    REQUIRE(format_version_for_nugetref("2020-06-26000", "abitag") == "2020.6.26-vcpkgabitag");
+    REQUIRE(format_version_for_feedref("2020-06-26", "abitag") == "2020.6.26-vcpkgabitag");
+    REQUIRE(format_version_for_feedref("20-06-26", "abitag") == "0.0.0-vcpkgabitag");
+    REQUIRE(format_version_for_feedref("2020-06-26-release", "abitag") == "2020.6.26-vcpkgabitag");
+    REQUIRE(format_version_for_feedref("2020-06-26000", "abitag") == "2020.6.26-vcpkgabitag");
 }
 
-TEST_CASE ("format_version_for_nugetref generic", "[format_version_for_nugetref]")
+TEST_CASE ("format_version_for_feedref generic", "[format_version_for_feedref]")
 {
-    REQUIRE(format_version_for_nugetref("apr", "abitag") == "0.0.0-vcpkgabitag");
-    REQUIRE(format_version_for_nugetref("", "abitag") == "0.0.0-vcpkgabitag");
+    REQUIRE(format_version_for_feedref("apr", "abitag") == "0.0.0-vcpkgabitag");
+    REQUIRE(format_version_for_feedref("", "abitag") == "0.0.0-vcpkgabitag");
 }
 
 TEST_CASE ("generate_nuspec", "[generate_nuspec]")
@@ -236,11 +236,11 @@ Build-Depends: bzip
     compiler_info.version = "compilerversion";
     ipa.abi_info.get()->compiler_info = compiler_info;
 
-    NugetReference ref2 = make_nugetref(ipa, "prefix_");
+    FeedReference ref2 = make_nugetref(ipa, "prefix_");
 
     REQUIRE(ref2.nupkg_filename() == "prefix_zlib2_x64-windows.1.5.0-vcpkgpackageabi.nupkg");
 
-    NugetReference ref = make_nugetref(ipa, "");
+    FeedReference ref = make_nugetref(ipa, "");
 
     REQUIRE(ref.nupkg_filename() == "zlib2_x64-windows.1.5.0-vcpkgpackageabi.nupkg");
 

src/vcpkg-test/configparser.cpp

@@ -560,6 +560,42 @@ TEST_CASE ("BinaryConfigParser HTTP provider", "[binaryconfigparser]")
     }
 }
 
+TEST_CASE ("BinaryConfigParser Universal Packages provider", "[binaryconfigparser]")
+{
+    // Scheme: x-az-universal,<organization>,<project>,<feed>[,<readwrite>]
+    {
+        auto parsed =
+            parse_binary_provider_configs("x-az-universal,test_organization,test_project_name,test_feed,read", {});
+        auto state = parsed.value_or_exit(VCPKG_LINE_INFO);
+        REQUIRE(state.upkg_templates_to_get.size() == 1);
+        REQUIRE(state.upkg_templates_to_get[0].feed == "test_feed");
+        REQUIRE(state.upkg_templates_to_get[0].organization == "test_organization");
+        REQUIRE(state.upkg_templates_to_get[0].project == "test_project_name");
+    }
+    {
+        auto parsed =
+            parse_binary_provider_configs("x-az-universal,test_organization,test_project_name,test_feed,readwrite", {});
+        auto state = parsed.value_or_exit(VCPKG_LINE_INFO);
+        REQUIRE(state.upkg_templates_to_get.size() == 1);
+        REQUIRE(state.upkg_templates_to_put.size() == 1);
+        REQUIRE(state.upkg_templates_to_get[0].feed == "test_feed");
+        REQUIRE(state.upkg_templates_to_get[0].organization == "test_organization");
+        REQUIRE(state.upkg_templates_to_get[0].project == "test_project_name");
+        REQUIRE(state.upkg_templates_to_put[0].feed == "test_feed");
+        REQUIRE(state.upkg_templates_to_put[0].organization == "test_organization");
+        REQUIRE(state.upkg_templates_to_put[0].project == "test_project_name");
+    }
+    {
+        auto parsed = parse_binary_provider_configs(
+            "x-az-universal,test_organization,test_project_name,test_feed,extra_argument,readwrite", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = parse_binary_provider_configs("x-az-universal,missing_args,read", {});
+        REQUIRE(!parsed.has_value());
+    }
+}
+
 TEST_CASE ("AssetConfigParser azurl provider", "[assetconfigparser]")
 {
     CHECK(parse_download_configuration({}));

src/vcpkg-test/tools.cpp

@@ -128,3 +128,15 @@ TEST_CASE ("extract_prefixed_nonwhitespace", "[tools]")
     CHECK(error_result.error() == "error: fooutil (fooutil.exe) produced unexpected output when attempting to "
                                   "determine the version:\nmalformed output");
 }
+
+TEST_CASE ("extract_prefixed_nonquote", "[tools]")
+{
+    CHECK(extract_prefixed_nonquote("fooutil version ", "fooutil", "fooutil version 1.2\"", "fooutil.exe")
+              .value_or_exit(VCPKG_LINE_INFO) == "1.2");
+    CHECK(extract_prefixed_nonquote("fooutil version ", "fooutil", "fooutil version 1.2 \"  ", "fooutil.exe")
+              .value_or_exit(VCPKG_LINE_INFO) == "1.2 ");
+    auto error_result = extract_prefixed_nonquote("fooutil version ", "fooutil", "malformed output", "fooutil.exe");
+    CHECK(!error_result.has_value());
+    CHECK(error_result.error() == "error: fooutil (fooutil.exe) produced unexpected output when attempting to "
+                                  "determine the version:\nmalformed output");
+}