ci: baseline pass means pass (#1827)

Author: dg0yt

azure-pipelines/e2e-assets/ci/ci.pass.baseline.txt

@@ -0,0 +1,16 @@
+not-sup-host-b:arm64-osx=pass
+not-sup-host-b:x64-osx=pass
+not-sup-host-b:x86-windows=pass
+not-sup-host-b:x64-windows=pass
+not-sup-host-b:arm64-windows=pass
+not-sup-host-b:x64-linux=pass
+not-sup-host-b:arm64-linux=pass
+
+
+dep-on-feature-not-sup:arm64-osx=pass
+dep-on-feature-not-sup:x64-osx=pass
+dep-on-feature-not-sup:x86-windows=pass
+dep-on-feature-not-sup:x64-windows=pass
+dep-on-feature-not-sup:arm64-windows=pass
+dep-on-feature-not-sup:x64-linux=pass
+dep-on-feature-not-sup:arm64-linux=pass

azure-pipelines/end-to-end-tests-dir/ci.ps1

@@ -27,6 +27,18 @@ if (-not ($ErrorOutput.Contains("REGRESSION: dep-on-feature-not-sup:${Triplet} i
     throw "feature-not-sup's baseline fail entry should result in a regression because the port is cascade for this triplet"
 }
 
+# pass means pass
+$Output = Run-VcpkgAndCaptureOutput ci --dry-run --triplet=$Triplet --x-builtin-ports-root="$PSScriptRoot/../e2e-ports/ci"  --binarysource=clear --ci-baseline="$PSScriptRoot/../e2e-assets/ci/ci.baseline.txt"
+Throw-IfNotFailed
+$ErrorOutput = Run-VcpkgAndCaptureStdErr ci --dry-run --triplet=$Triplet --x-builtin-ports-root="$PSScriptRoot/../e2e-ports/ci"  --binarysource=clear --ci-baseline="$PSScriptRoot/../e2e-assets/ci/ci.pass.baseline.txt"
+Throw-IfNotFailed
+if (-not ($ErrorOutput.Contains("REGRESSION: not-sup-host-b:${Triplet} is marked as pass but not supported for ${Triplet}."))) {
+    throw "feature-not-sup's baseline pass entry should result in a regression because the port is not supported"
+}
+if (-not ($ErrorOutput.Contains("REGRESSION: dep-on-feature-not-sup:${Triplet} is marked as pass but one dependency is not supported for ${Triplet}."))) {
+    throw "feature-not-sup's baseline pass entry should result in a regression because the port is cascade for this triplet"
+}
+
 # any invalid manifest must raise an error
 $Output = Run-VcpkgAndCaptureOutput ci --dry-run --triplet=$Triplet --x-builtin-ports-root="$PSScriptRoot/../e2e-ports/broken-manifests"  --binarysource=clear --ci-baseline="$PSScriptRoot/../e2e-assets/ci/ci.baseline.txt"
 Throw-IfNotFailed

include/vcpkg/base/message-data.inc.h

@@ -495,6 +495,14 @@ DECLARE_MESSAGE(CiBaselineUnexpectedPass,
                 (msg::spec, msg::path),
                 "",
                 "PASSING, REMOVE FROM FAIL LIST: {spec} ({path}).")
+DECLARE_MESSAGE(CiBaselineUnexpectedPassCascade,
+                (msg::spec, msg::triplet),
+                "",
+                "REGRESSION: {spec} is marked as pass but one dependency is not supported for {triplet}.")
+DECLARE_MESSAGE(CiBaselineUnexpectedPassUnsupported,
+                (msg::spec, msg::triplet),
+                "",
+                "REGRESSION: {spec} is marked as pass but not supported for {triplet}.")
 DECLARE_MESSAGE(CISettingsOptCIBase,
                 (),
                 "",

locales/messages.json

@@ -311,6 +311,10 @@
   "_CiBaselineUnexpectedFailCascade.comment": "An example of {spec} is zlib:x64-windows. An example of {triplet} is x64-windows.",
   "CiBaselineUnexpectedPass": "PASSING, REMOVE FROM FAIL LIST: {spec} ({path}).",
   "_CiBaselineUnexpectedPass.comment": "An example of {spec} is zlib:x64-windows. An example of {path} is /foo/bar.",
+  "CiBaselineUnexpectedPassCascade": "REGRESSION: {spec} is marked as pass but one dependency is not supported for {triplet}.",
+  "_CiBaselineUnexpectedPassCascade.comment": "An example of {spec} is zlib:x64-windows. An example of {triplet} is x64-windows.",
+  "CiBaselineUnexpectedPassUnsupported": "REGRESSION: {spec} is marked as pass but not supported for {triplet}.",
+  "_CiBaselineUnexpectedPassUnsupported.comment": "An example of {spec} is zlib:x64-windows. An example of {triplet} is x64-windows.",
   "ClearingContents": "Clearing contents of {path}",
   "_ClearingContents.comment": "An example of {path} is /foo/bar.",
   "CmakeTargetsExcluded": "{count} additional targets are not displayed.",

src/vcpkg/commands.ci.cpp

@@ -417,6 +417,14 @@ namespace vcpkg
                                     msg::triplet = spec.package_spec.triplet())
                             .append_raw('\n');
                     }
+                    else if (cidata.required_success.contains(spec.package_spec))
+                    {
+                        not_supported_regressions
+                            .append(supp ? msgCiBaselineUnexpectedPassCascade : msgCiBaselineUnexpectedPassUnsupported,
+                                    msg::spec = spec.package_spec,
+                                    msg::triplet = spec.package_spec.triplet())
+                            .append_raw('\n');
+                    }
                     msg += fmt::format("{:>40}: {:>8}\n", spec.package_spec, supp ? "cascade" : "skip");
                 }
             }