ci: adjustments to the ecosystem compat tests

.github/workflows/ecosystem_compat_test.yml

@@ -26,12 +26,52 @@ jobs:
         uses: denoland/setup-deno@v2
         with:
           deno-version: canary
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+      - name: Authenticate with Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          project_id: denoland
+          credentials_json: '${{ secrets.GCP_SA_KEY }}'
+          export_environment_variables: true
+          create_credentials_file: true
       - name: Setup gcloud
         uses: google-github-actions/setup-gcloud@v2
         with:
           project_id: denoland
       - name: Run tests
         run: deno -A tools/ecosystem_compat_tests.ts
+      - name: Upload the report to dl.deno.land
+        run: |-
+          gsutil -h "Cache-Control: public, max-age=3600" cp tools/ecosystem_report.json gs://dl.deno.land/ecosystem-compat-test/$(date +%F)/report-${{matrix.os}}.json
+  summary:
+    runs-on: ubuntu-latest
+    needs: test
+    if: ${{ always() }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Setup Deno
+        uses: denoland/setup-deno@v2
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+      - name: Authenticate with Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          project_id: denoland
+          credentials_json: '${{ secrets.GCP_SA_KEY }}'
+          export_environment_variables: true
+          create_credentials_file: true
+      - name: Setup gcloud
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          project_id: denoland
       - name: Post message to slack channel
         run: deno -A tools/ecosystem_compat_slack.ts
         env:

tools/ecosystem_compat_slack.ts

@@ -29,44 +29,173 @@ const client = new WebClient(token, {
 });
 
 function formatDuration(duration: number) {
-  return (duration / 1000).toFixed(2) + "s";
+  return (duration / 1000).toFixed(0) + "s";
 }
 
-function createMessage(ecosystemReport: EcosystemReport) {
-  let mrkdwn = "Package manager report\n";
+function createMessage(ecosystemReports: Record<string, EcosystemReport>) {
+  const elements = [];
 
-  mrkdwn += `*npm*: exit code: ${ecosystemReport.npm.exitCode}, duration: ${
-    formatDuration(ecosystemReport.npm.duration)
-  }\n`;
-  mrkdwn += `*yarn*: exit code: ${ecosystemReport.yarn.exitCode}, duration: ${
-    formatDuration(ecosystemReport.yarn.duration)
-  }\n`;
-  mrkdwn += `*pnpm*: exit code: ${ecosystemReport.pnpm.exitCode}, duration: ${
-    formatDuration(ecosystemReport.pnpm.duration)
-  }\n`;
+  elements.push({
+    type: "section",
+    text: {
+      type: "mrkdwn",
+      text: "*Package manager report*\n\n",
+    },
+  });
 
-  return [
+  const tableHeader = [
     {
-      type: "section",
-      text: {
-        type: "mrkdwn",
-        text: mrkdwn,
-      },
+      type: "rich_text",
+      elements: [
+        {
+          type: "rich_text_section",
+          elements: [
+            {
+              type: "text",
+              text: "Program",
+              style: {
+                bold: true,
+              },
+            },
+          ],
+        },
+      ],
+    },
+    {
+      type: "rich_text",
+      elements: [
+        {
+          type: "rich_text_section",
+          elements: [
+            {
+              type: "text",
+              text: "Linux",
+              style: {
+                bold: true,
+              },
+            },
+          ],
+        },
+      ],
+    },
+    {
+      type: "rich_text",
+      elements: [
+        {
+          type: "rich_text_section",
+          elements: [
+            {
+              type: "text",
+              text: "macOS",
+              style: {
+                bold: true,
+              },
+            },
+          ],
+        },
+      ],
+    },
+    {
+      type: "rich_text",
+      elements: [
+        {
+          type: "rich_text_section",
+          elements: [
+            {
+              type: "text",
+              text: "Windows",
+              style: {
+                bold: true,
+              },
+            },
+          ],
+        },
+      ],
     },
   ];
+
+  const rows = [];
+
+  const programs = Object.keys(ecosystemReports["darwin"]);
+  for (const program of programs) {
+    const row = [
+      {
+        type: "rich_text",
+        elements: [
+          {
+            type: "rich_text_section",
+            elements: [
+              {
+                type: "text",
+                text: program,
+              },
+            ],
+          },
+        ],
+      },
+    ];
+
+    for (const os of ["darwin", "linux", "windows"]) {
+      const report = ecosystemReports[os][program] satisfies PmResult;
+
+      const text = `${
+        report.exitCode === 0 ? "✅" : "❌"
+      } code: ${report.exitCode}, (${formatDuration(report.duration)})`;
+      row.push({
+        type: "rich_text",
+        elements: [
+          {
+            type: "rich_text_section",
+            elements: [
+              {
+                type: "text",
+                text: text,
+                style: {
+                  code: true,
+                },
+              },
+            ],
+          },
+        ],
+      });
+    }
+
+    rows.push(row);
+  }
+
+  elements.push({
+    type: "table",
+    rows: [tableHeader, ...rows],
+  });
+  return elements;
+}
+
+async function downloadOsReports() {
+  const oses = ["windows", "linux", "darwin"];
+  const reports: Record<string, string> = {};
+  for (const os of oses) {
+    const res = await fetch(
+      `https://dl.deno.land/ecosystem-compat-test/${
+        new Date()
+          .toISOString()
+          .substring(0, 10)
+      }/report-${os}.json`,
+    );
+    if (res.status === 200) {
+      reports[os] = await res.json() satisfies EcosystemReport;
+    }
+  }
+  return reports;
 }
 
 async function main() {
-  const ecosystemReport = await Deno.readTextFile(
-    import.meta.resolve("./ecosystem_report.json"),
-  )
-    .then(JSON.parse) as EcosystemReport;
+  const ecosystemReports = await downloadOsReports();
 
   try {
     const result = await client.chat.postMessage({
       token,
       channel,
-      blocks: createMessage(ecosystemReport),
+      blocks: createMessage(ecosystemReports),
       unfurl_links: false,
       unfurl_media: false,
     });

tools/ecosystem_compat_tests.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env -S deno run --allow-all --config=tests/config/deno.json
 // Copyright 2018-2025 the Deno authors. MIT license.
 import $ from "jsr:@david/dax@^0.42.0";
+import { join } from "./util.js";
 
 async function setupTestRepo() {
   await teardownTestRepo();
@@ -11,7 +12,7 @@ async function teardownTestRepo() {
   await $`rm -rf nextjs-demo`;
 }
 
-async function runPackageManager(pm, cmd) {
+async function runPackageManager(pm: string, cmd: string) {
   const state = Date.now();
   const result =
     await $`cd nextjs-demo ; rm -rf node_modules ; ${Deno.execPath()} run -A --no-config npm:${pm} ${cmd}`
@@ -27,61 +28,66 @@ async function runPackageManager(pm, cmd) {
   };
 }
 
-function testNpm() {
-  return runPackageManager("npm", "install");
+async function testNpm() {
+  $.logStep("Testing npm...");
+  const report = await runPackageManager("npm", "install");
+  if (report.exitCode === 0) {
+    $.logStep("npm install succeeded");
+  } else {
+    $.logWarn(`npm install failed: ${report.stderr}`);
+  }
+  return report;
 }
 
-function testYarn() {
-  return runPackageManager("yarn", "install");
+async function testYarn() {
+  $.logStep("Testing yarn...");
+  const report = await runPackageManager("yarn", "install");
+  if (report.exitCode === 0) {
+    $.logStep("yarn install succeeded");
+  } else {
+    $.logWarn(`yarn install failed: ${report.stderr}`);
+  }
+  return report;
 }
 
-function testPnpm() {
-  return runPackageManager("pnpm", "install");
+async function testPnpm() {
+  $.logStep("Testing pnpm...");
+  const report = await runPackageManager("pnpm", "install");
+  if (report.exitCode === 0) {
+    $.logStep("pnpm install succeeded");
+  } else {
+    $.logWarn(`pnpm install failed: ${report.stderr}`);
+  }
+  return report;
 }
 
 async function main() {
-  let passed = true;
   $.logStep("Setting up test repo...");
   await setupTestRepo();
-  $.logStep("Testing npm...");
+
   const npmResult = await testNpm();
-  if (npmResult.exitCode !== 0) {
-    passed = false;
-    $.logWarn(`npm install failed: ${npmResult.stderr}`);
-  }
-  $.logStep("Testing yarn...");
   const yarnResult = await testYarn();
-  if (yarnResult.exitCode !== 0) {
-    passed = false;
-    $.logWarn(`yarn install failed: ${yarnResult.stderr}`);
-  }
-  $.logStep("Testing pnpm...");
   const pnpmResult = await testPnpm();
-  if (pnpmResult.exitCode !== 0) {
-    passed = false;
-    $.logWarn(`pnpm install failed: ${pnpmResult.stderr}`);
-  }
-  if (passed) {
-    $.logStep("All tests passed!");
-  } else {
-    $.logError("Some tests failed");
-  }
+  const reports = {
+    npm: {
+      exitCode: npmResult.exitCode,
+      duration: npmResult.duration,
+    },
+    yarn: {
+      exitCode: yarnResult.exitCode,
+      duration: yarnResult.duration,
+    },
+    pnpm: {
+      exitCode: pnpmResult.exitCode,
+      duration: pnpmResult.duration,
+    },
+  };
+
+  $.logStep("Final report:");
+  $.log(reports);
   await Deno.writeTextFile(
-    import.meta.resolve("./ecosystem_report.json"),
-    JSON.stringify({
-      npm: {
-        exitCode: npmResult.exitCode,
-        duration: npmResult.duration,
-      },
-      yarn: {
-        exitCode: yarnResult.exitCode,
-        duration: yarnResult.duration,
-      },
-      pnpm: {
-        exitCode: pnpmResult.exitCode,
-        duration: pnpmResult.duration,
-      },
-    }),
+    join(import.meta.dirname!, "ecosystem_report.json"),
+    JSON.stringify(reports),
   );
 }