Fix remaining test failures, remove "all results" from the report at the end.

Author: BillyONeal

azure-pipelines/end-to-end-tests-dir/ci.ps1

@@ -23,10 +23,9 @@ if ($Output.Split("*").Length -ne 4) {
 if (-not ($ErrorOutput.Contains("REGRESSION: not-sup-host-b:${Triplet} is marked as fail but not supported for ${Triplet}."))) {
     throw "feature-not-sup's baseline fail entry should result in a regression because the port is not supported"
 }
-# FIXME
-# if (-not ($ErrorOutput.Contains("REGRESSION: dep-on-feature-not-sup:${Triplet} is marked as fail but one dependency is not supported for ${Triplet}."))) {
-#     throw "feature-not-sup's baseline fail entry should result in a regression because the port is cascade for this triplet"
-# }
+if (-not ($ErrorOutput.Contains("REGRESSION: dep-on-feature-not-sup:${Triplet} is marked as fail but one dependency is not supported for ${Triplet}."))) {
+    throw "feature-not-sup's baseline fail entry should result in a regression because the port is cascade for this triplet"
+}
 
 # pass means pass
 $Output = Run-VcpkgAndCaptureOutput ci --dry-run --triplet=$Triplet --x-builtin-ports-root="$PSScriptRoot/../e2e-ports/ci"  --binarysource=clear --ci-baseline="$PSScriptRoot/../e2e-assets/ci/ci.baseline.txt"
@@ -36,10 +35,9 @@ Throw-IfNotFailed
 if (-not ($ErrorOutput.Contains("REGRESSION: not-sup-host-b:${Triplet} is marked as pass but not supported for ${Triplet}."))) {
     throw "feature-not-sup's baseline pass entry should result in a regression because the port is not supported"
 }
-# FIXME
-# if (-not ($ErrorOutput.Contains("REGRESSION: dep-on-feature-not-sup:${Triplet} is marked as pass but one dependency is not supported for ${Triplet}."))) {
-#     throw "feature-not-sup's baseline pass entry should result in a regression because the port is cascade for this triplet"
-# }
+if (-not ($ErrorOutput.Contains("REGRESSION: dep-on-feature-not-sup:${Triplet} is marked as pass but one dependency is not supported for ${Triplet}."))) {
+    throw "feature-not-sup's baseline pass entry should result in a regression because the port is cascade for this triplet"
+}
 
 # any invalid manifest must raise an error
 $Output = Run-VcpkgAndCaptureOutput ci --dry-run --triplet=$Triplet --x-builtin-ports-root="$PSScriptRoot/../e2e-ports/broken-manifests"  --binarysource=clear --ci-baseline="$PSScriptRoot/../e2e-assets/ci/ci.baseline.txt"

src/vcpkg/ci-baseline.cpp

@@ -192,7 +192,6 @@ namespace vcpkg
                                      const std::string* cifile,
                                      bool allow_unexpected_passing)
     {
-        // FIXME how to report msgCiBaselineUnexpectedFailCascade?
         switch (result)
         {
             case BuildResult::Succeeded:
@@ -221,9 +220,15 @@ namespace vcpkg
                 }
                 break;
             case BuildResult::CascadedDueToMissingDependencies:
-                if (cidata.required_success.contains(spec))
+                if (cidata.expected_failures.contains(spec))
+                {
+                    return msg::format(
+                        msgCiBaselineUnexpectedFailCascade, msg::spec = spec, msg::triplet = spec.triplet());
+                }
+                else if (cidata.required_success.contains(spec))
                 {
-                    return msg::format(msgCiBaselineDisallowedCascade, msg::spec = spec, msg::path = *cifile);
+                    return msg::format(
+                        msgCiBaselineUnexpectedPassCascade, msg::spec = spec, msg::triplet = spec.triplet());
                 }
                 break;
             case BuildResult::Unsupported:
@@ -238,12 +243,6 @@ namespace vcpkg
                 }
                 break;
             case BuildResult::Excluded:
-                if (cidata.required_success.contains(spec))
-                {
-                    return msg::format(
-                        msgCiBaselineUnexpectedPassCascade, msg::spec = spec, msg::triplet = spec.triplet());
-                }
-                break;
             case BuildResult::ExcludedByParent:
             case BuildResult::ExcludedByDryRun: break;
             case BuildResult::CacheMissing:

src/vcpkg/commands.ci.cpp

@@ -521,7 +521,7 @@ namespace vcpkg
         prune_entirely_known_action_branches(action_plan, pre_build_status.known);
 
         msg::println(msgElapsedTimeForChecks, msg::elapsed = timer.elapsed());
-        std::map<PackageSpec, CiResult> ci_results;
+        std::map<PackageSpec, CiResult> ci_plan_results;
         std::map<PackageSpec, CiResult> ci_full_results;
         for (auto&& pre_known_outcome : pre_build_status.known)
         {
@@ -565,7 +565,7 @@ namespace vcpkg
                                                             ipa->feature_list,
                                                             result.start_time,
                                                             result.timing}};
-                    ci_results.insert_or_assign(result.get_spec(), ci_result);
+                    ci_plan_results.insert_or_assign(result.get_spec(), ci_result);
                     ci_full_results.insert_or_assign(result.get_spec(), std::move(ci_result));
                 }
             }
@@ -579,21 +579,24 @@ namespace vcpkg
         summary_report.push_back(' ');
         target_triplet.to_string(summary_report);
         summary_report.push_back('\n');
-        for (auto&& ci_result : ci_full_results)
+        for (auto&& ci_result : ci_plan_results)
         {
-            summary_counts[ci_result.first.triplet()].increment(ci_result.second.code);
-
             summary_report.append(2, ' ');
             ci_result.first.to_string(summary_report);
             summary_report.append(": ");
             ci_result.second.to_string(summary_report);
             summary_report.push_back('\n');
         }
 
-        for (auto&& entry : summary_counts)
+        for (auto&& ci_result : ci_full_results)
+        {
+            summary_counts[ci_result.first.triplet()].increment(ci_result.second.code);
+        }
+
+        for (auto&& summary_count : summary_counts)
         {
             summary_report.push_back('\n');
-            summary_report.append(entry.second.format(entry.first).data());
+            summary_report.append(summary_count.second.format(summary_count.first).data());
         }
 
         summary_report.push_back('\n');
@@ -608,7 +611,7 @@ namespace vcpkg
         {
             XunitWriter xunitTestResults;
             const auto& xunit_results =
-                Util::Sets::contains(options.switches, SwitchXXUnitAll) ? ci_full_results : ci_results;
+                Util::Sets::contains(options.switches, SwitchXXUnitAll) ? ci_full_results : ci_plan_results;
             for (auto&& xunit_result : xunit_results)
             {
                 xunitTestResults.add_test_results(xunit_result.first, xunit_result.second);