From e98ca17d036d7e680c17384ac224e418bcc461d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Sun, 17 Nov 2024 09:49:35 +0100
Subject: [PATCH 1/8] Add backend to test case key

---
 site/frontend/src/pages/compare/compile/common.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/frontend/src/pages/compare/compile/common.ts b/site/frontend/src/pages/compare/compile/common.ts
index 7c034ba41..b128658e8 100644
--- a/site/frontend/src/pages/compare/compile/common.ts
+++ b/site/frontend/src/pages/compare/compile/common.ts
@@ -206,5 +206,5 @@ export function createCompileBenchmarkMap(
 }
 
 export function testCaseKey(testCase: CompileTestCase): string {
-  return `${testCase.benchmark};${testCase.profile};${testCase.scenario};${testCase.category}`;
+  return `${testCase.benchmark};${testCase.profile};${testCase.scenario};${testCase.backend};${testCase.category}`;
 }

From 69a77e60edf4b932c7dd69c37099912de84d9c14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Sun, 17 Nov 2024 10:08:24 +0100
Subject: [PATCH 2/8] Add comparison of LLVM against Cranelift

---
 .../src/pages/compare/compile/common.ts       | 55 +++++++++++++++++++
 .../pages/compare/compile/compile-page.vue    | 31 ++++++++++-
 2 files changed, 85 insertions(+), 1 deletion(-)

diff --git a/site/frontend/src/pages/compare/compile/common.ts b/site/frontend/src/pages/compare/compile/common.ts
index b128658e8..e33e53f7b 100644
--- a/site/frontend/src/pages/compare/compile/common.ts
+++ b/site/frontend/src/pages/compare/compile/common.ts
@@ -208,3 +208,58 @@ export function createCompileBenchmarkMap(
 export function testCaseKey(testCase: CompileTestCase): string {
   return `${testCase.benchmark};${testCase.profile};${testCase.scenario};${testCase.backend};${testCase.category}`;
 }
+
+// Transform compile comparisons to compare LLVM vs Cranelift, instead of
+// before/after. Assumes that the data comes from the same commit.
+export function transformDataForBackendComparison(
+  comparisons: CompileBenchmarkComparison[]
+): CompileBenchmarkComparison[] {
+  const benchmarkMap: Map<
+    string,
+    {
+      llvm: number | null;
+      cranelift: number | null;
+      benchmark: string;
+      profile: Profile;
+      scenario: string;
+    }
+  > = new Map();
+  for (const comparison of comparisons) {
+    const key = `${comparison.benchmark};${comparison.profile};${comparison.scenario}`;
+    if (!benchmarkMap.has(key)) {
+      benchmarkMap.set(key, {
+        llvm: null,
+        cranelift: null,
+        benchmark: comparison.benchmark,
+        profile: comparison.profile,
+        scenario: comparison.scenario,
+      });
+    }
+    const record = benchmarkMap.get(key);
+    if (comparison.backend === "llvm") {
+      record.llvm = comparison.comparison.statistics[0];
+    } else if (comparison.backend === "cranelift") {
+      record.cranelift = comparison.comparison.statistics[0];
+    }
+  }
+
+  const transformed = [];
+  benchmarkMap.forEach((entry) => {
+    const comparison: CompileBenchmarkComparison = {
+      benchmark: entry.benchmark,
+      profile: entry.profile,
+      scenario: entry.scenario,
+      // Treat the backend as LLVM
+      backend: "llvm",
+      comparison: {
+        statistics: [entry.llvm, entry.cranelift],
+        is_relevant: true,
+        significance_factor: 1.0,
+        significance_threshold: 1.0,
+      },
+    };
+    transformed.push(comparison);
+  });
+
+  return transformed;
+}
diff --git a/site/frontend/src/pages/compare/compile/compile-page.vue b/site/frontend/src/pages/compare/compile/compile-page.vue
index aeb7f4de8..f69a7df94 100644
--- a/site/frontend/src/pages/compare/compile/compile-page.vue
+++ b/site/frontend/src/pages/compare/compile/compile-page.vue
@@ -14,6 +14,7 @@ import {
   computeCompileComparisonsWithNonRelevant,
   createCompileBenchmarkMap,
   defaultCompileFilter,
+  transformDataForBackendComparison,
 } from "./common";
 import {BenchmarkInfo} from "../../../api";
 import {importantCompileMetrics} from "../metrics";
@@ -197,15 +198,31 @@ const urlParams = getUrlParams();
 const quickLinksKey = ref(0);
 const filter = ref(loadFilterFromUrl(urlParams, defaultCompileFilter));
 
+// Should we use the backend as the source of before/after data?
+const selfCompareBackend = ref(false);
+
 function exportData() {
   exportToMarkdown(comparisons.value, filter.value.showRawData);
 }
 
+// Are we currently comparing the same commit against each other?
+const comparesIdenticalCommits = computed(() => {
+  return props.data.a.commit === props.data.b.commit;
+});
 const benchmarkMap = createCompileBenchmarkMap(props.data);
+
+// Artificially restructure the data to create a comparison between backends
+const compileComparisons = computed(() => {
+  if (selfCompareBackend.value) {
+    return transformDataForBackendComparison(props.data.compile_comparisons);
+  } else {
+    return props.data.compile_comparisons;
+  }
+});
 const allComparisons = computed(() =>
   computeCompileComparisonsWithNonRelevant(
     filter.value,
-    props.data.compile_comparisons,
+    compileComparisons.value,
     benchmarkMap
   )
 );
@@ -222,6 +239,9 @@ const filteredSummary = computed(() => computeSummary(comparisons.value));
     :selected-metric="selector.stat"
     :metrics="benchmarkInfo.compile_metrics"
   />
+  <div v-if="comparesIdenticalCommits">
+    Self-compare backend: <input type="checkbox" v-model="selfCompareBackend" />
+  </div>
   <Filters
     :defaultFilter="defaultCompileFilter"
     :initialFilter="filter"
@@ -230,6 +250,9 @@ const filteredSummary = computed(() => computeSummary(comparisons.value));
   />
   <OverallSummary :summary="filteredSummary" />
   <Aggregations :cases="comparisons" />
+  <div class="warning" v-if="selfCompareBackend">
+    Comparing LLVM against Cranelift!
+  </div>
   <Benchmarks
     :data="data"
     :test-cases="comparisons"
@@ -239,3 +262,9 @@ const filteredSummary = computed(() => computeSummary(comparisons.value));
     :benchmark-map="benchmarkMap"
   ></Benchmarks>
 </template>
+<style lang="scss" scoped>
+.warning {
+  color: red;
+  font-weight: bold;
+}
+</style>

From cd5b340867b30e38e7892f8cdd4afae167b5ea98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Sun, 17 Nov 2024 10:11:09 +0100
Subject: [PATCH 3/8] Do not show backend column when comparing backends

---
 site/frontend/src/pages/compare/compile/benchmarks.vue       | 3 +++
 site/frontend/src/pages/compare/compile/compile-page.vue     | 1 +
 .../src/pages/compare/compile/table/comparisons-table.vue    | 5 +++--
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/site/frontend/src/pages/compare/compile/benchmarks.vue b/site/frontend/src/pages/compare/compile/benchmarks.vue
index fdc4e6dab..975b4e56a 100644
--- a/site/frontend/src/pages/compare/compile/benchmarks.vue
+++ b/site/frontend/src/pages/compare/compile/benchmarks.vue
@@ -16,6 +16,7 @@ export interface BenchmarkProps {
   benchmarkMap: CompileBenchmarkMap;
   filter: CompileBenchmarkFilter;
   stat: string;
+  showBackend: boolean;
 }
 
 const props = defineProps<BenchmarkProps>();
@@ -77,6 +78,7 @@ const secondaryHasNonRelevant = computed(
       :commit-b="data.b"
       :stat="stat"
       :benchmark-map="benchmarkMap"
+      :show-backend="showBackend"
     >
       <template #header>
         <Section title="Primary" link="secondary" :linkUp="false"></Section>
@@ -92,6 +94,7 @@ const secondaryHasNonRelevant = computed(
       :commit-b="data.b"
       :stat="stat"
       :benchmark-map="benchmarkMap"
+      :show-backend="showBackend"
     >
       <template #header>
         <Section title="Secondary" link="primary" :linkUp="true"></Section>
diff --git a/site/frontend/src/pages/compare/compile/compile-page.vue b/site/frontend/src/pages/compare/compile/compile-page.vue
index f69a7df94..a51023e3d 100644
--- a/site/frontend/src/pages/compare/compile/compile-page.vue
+++ b/site/frontend/src/pages/compare/compile/compile-page.vue
@@ -260,6 +260,7 @@ const filteredSummary = computed(() => computeSummary(comparisons.value));
     :filter="filter"
     :stat="selector.stat"
     :benchmark-map="benchmarkMap"
+    :show-backend="!selfCompareBackend"
   ></Benchmarks>
 </template>
 <style lang="scss" scoped>
diff --git a/site/frontend/src/pages/compare/compile/table/comparisons-table.vue b/site/frontend/src/pages/compare/compile/table/comparisons-table.vue
index 8d4f0d06a..362c54717 100644
--- a/site/frontend/src/pages/compare/compile/table/comparisons-table.vue
+++ b/site/frontend/src/pages/compare/compile/table/comparisons-table.vue
@@ -18,6 +18,7 @@ const props = defineProps<{
   commitA: ArtifactDescription;
   commitB: ArtifactDescription;
   stat: string;
+  showBackend: boolean;
 }>();
 
 function prettifyRawNumber(number: number): string {
@@ -56,7 +57,7 @@ const unit = computed(() => {
           <th>Benchmark</th>
           <th>Profile</th>
           <th>Scenario</th>
-          <th>Backend</th>
+          <th v-if="showBackend">Backend</th>
           <th>% Change</th>
           <th class="narrow">
             Significance Threshold
@@ -95,7 +96,7 @@ const unit = computed(() => {
                 {{ comparison.testCase.profile }}
               </td>
               <td>{{ comparison.testCase.scenario }}</td>
-              <td>{{ comparison.testCase.backend }}</td>
+              <td v-if="showBackend">{{ comparison.testCase.backend }}</td>
               <td>
                 <div class="numeric-aligned">
                   <span v-bind:class="percentClass(comparison.percent)">

From 9b2d3fb8995046c5a09a8ee49c6c28e2fc8709ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Sun, 17 Nov 2024 10:21:43 +0100
Subject: [PATCH 4/8] Store self compare backend checkbox in URL

---
 .../src/pages/compare/compile/common.ts       |  2 ++
 .../pages/compare/compile/compile-page.vue    | 25 +++++++++++++++++--
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/site/frontend/src/pages/compare/compile/common.ts b/site/frontend/src/pages/compare/compile/common.ts
index e33e53f7b..998be1430 100644
--- a/site/frontend/src/pages/compare/compile/common.ts
+++ b/site/frontend/src/pages/compare/compile/common.ts
@@ -27,6 +27,7 @@ export type CompileBenchmarkFilter = {
     binary: boolean;
     library: boolean;
   };
+  selfCompareBackend: boolean;
 } & BenchmarkFilter;
 
 export const defaultCompileFilter: CompileBenchmarkFilter = {
@@ -57,6 +58,7 @@ export const defaultCompileFilter: CompileBenchmarkFilter = {
     binary: true,
     library: true,
   },
+  selfCompareBackend: false,
 };
 
 export type Profile = "check" | "debug" | "opt" | "doc";
diff --git a/site/frontend/src/pages/compare/compile/compile-page.vue b/site/frontend/src/pages/compare/compile/compile-page.vue
index a51023e3d..6818a42e6 100644
--- a/site/frontend/src/pages/compare/compile/compile-page.vue
+++ b/site/frontend/src/pages/compare/compile/compile-page.vue
@@ -102,6 +102,11 @@ function loadFilterFromUrl(
         defaultFilter.artifact.library
       ),
     },
+    selfCompareBackend: getBoolOrDefault(
+      urlParams,
+      "selfCompareBackend",
+      defaultFilter.selfCompareBackend
+    ),
   };
 }
 
@@ -173,6 +178,11 @@ function storeFilterToUrl(
     filter.artifact.library,
     defaultFilter.artifact.library
   );
+  storeOrReset(
+    "selfCompareBackend",
+    filter.selfCompareBackend,
+    defaultFilter.selfCompareBackend
+  );
 
   changeUrl(urlParams);
 }
@@ -183,6 +193,10 @@ function updateFilter(newFilter: CompileBenchmarkFilter) {
   refreshQuickLinks();
 }
 
+function updateSelfCompareBackend(value: boolean) {
+  updateFilter({...filter.value, selfCompareBackend: value});
+}
+
 /**
  * When the filter changes, the URL is updated.
  * After that happens, we want to re-render the quick links component, because
@@ -199,7 +213,9 @@ const quickLinksKey = ref(0);
 const filter = ref(loadFilterFromUrl(urlParams, defaultCompileFilter));
 
 // Should we use the backend as the source of before/after data?
-const selfCompareBackend = ref(false);
+const selfCompareBackend = computed(
+  () => comparesIdenticalCommits.value && filter.value.selfCompareBackend
+);
 
 function exportData() {
   exportToMarkdown(comparisons.value, filter.value.showRawData);
@@ -240,7 +256,12 @@ const filteredSummary = computed(() => computeSummary(comparisons.value));
     :metrics="benchmarkInfo.compile_metrics"
   />
   <div v-if="comparesIdenticalCommits">
-    Self-compare backend: <input type="checkbox" v-model="selfCompareBackend" />
+    Self-compare backend:
+    <input
+      type="checkbox"
+      :checked="selfCompareBackend"
+      @change="(e) => updateSelfCompareBackend(e.target.checked)"
+    />
   </div>
   <Filters
     :defaultFilter="defaultCompileFilter"

From 4e647f8de36caf85426e48d95b140965155898ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Sun, 17 Nov 2024 10:28:23 +0100
Subject: [PATCH 5/8] Clarify comparison text

---
 site/frontend/src/pages/compare/compile/compile-page.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/frontend/src/pages/compare/compile/compile-page.vue b/site/frontend/src/pages/compare/compile/compile-page.vue
index 6818a42e6..e9d2a83f2 100644
--- a/site/frontend/src/pages/compare/compile/compile-page.vue
+++ b/site/frontend/src/pages/compare/compile/compile-page.vue
@@ -272,7 +272,7 @@ const filteredSummary = computed(() => computeSummary(comparisons.value));
   <OverallSummary :summary="filteredSummary" />
   <Aggregations :cases="comparisons" />
   <div class="warning" v-if="selfCompareBackend">
-    Comparing LLVM against Cranelift!
+    Comparing Cranelift against LLVM baseline!
   </div>
   <Benchmarks
     :data="data"

From e20f90ba31e435f7db58be776fd6522974add5cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Sun, 17 Nov 2024 10:34:57 +0100
Subject: [PATCH 6/8] Fix TS compilation error

---
 site/frontend/src/pages/compare/compile/compile-page.vue | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/site/frontend/src/pages/compare/compile/compile-page.vue b/site/frontend/src/pages/compare/compile/compile-page.vue
index e9d2a83f2..08c25b7fe 100644
--- a/site/frontend/src/pages/compare/compile/compile-page.vue
+++ b/site/frontend/src/pages/compare/compile/compile-page.vue
@@ -193,8 +193,11 @@ function updateFilter(newFilter: CompileBenchmarkFilter) {
   refreshQuickLinks();
 }
 
-function updateSelfCompareBackend(value: boolean) {
-  updateFilter({...filter.value, selfCompareBackend: value});
+// We pass the event target here, because Parcel cannot handle the `as`
+// cast directly in the template.
+function updateSelfCompareBackend(target: EventTarget) {
+  const element = target as HTMLInputElement;
+  updateFilter({...filter.value, selfCompareBackend: element.checked});
 }
 
 /**
@@ -260,7 +263,7 @@ const filteredSummary = computed(() => computeSummary(comparisons.value));
     <input
       type="checkbox"
       :checked="selfCompareBackend"
-      @change="(e) => updateSelfCompareBackend(e.target.checked)"
+      @change="(e) => updateSelfCompareBackend(e.target)"
     />
   </div>
   <Filters

From 915e09515281688a055e2f8720f86229a326d55e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Mon, 18 Nov 2024 11:45:35 +0100
Subject: [PATCH 7/8] Apply review comments

---
 .../src/pages/compare/compile/common.ts       |  9 +++----
 .../pages/compare/compile/compile-page.vue    | 26 +++++++++++++------
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/site/frontend/src/pages/compare/compile/common.ts b/site/frontend/src/pages/compare/compile/common.ts
index 998be1430..f2b17b0d5 100644
--- a/site/frontend/src/pages/compare/compile/common.ts
+++ b/site/frontend/src/pages/compare/compile/common.ts
@@ -245,13 +245,12 @@ export function transformDataForBackendComparison(
     }
   }
 
-  const transformed = [];
-  benchmarkMap.forEach((entry) => {
+  return Array.from(benchmarkMap, ([_, entry]) => {
     const comparison: CompileBenchmarkComparison = {
       benchmark: entry.benchmark,
       profile: entry.profile,
       scenario: entry.scenario,
-      // Treat the backend as LLVM
+      // Treat LLVM as the baseline
       backend: "llvm",
       comparison: {
         statistics: [entry.llvm, entry.cranelift],
@@ -260,8 +259,6 @@ export function transformDataForBackendComparison(
         significance_threshold: 1.0,
       },
     };
-    transformed.push(comparison);
+    return comparison;
   });
-
-  return transformed;
 }
diff --git a/site/frontend/src/pages/compare/compile/compile-page.vue b/site/frontend/src/pages/compare/compile/compile-page.vue
index 08c25b7fe..216fc08e3 100644
--- a/site/frontend/src/pages/compare/compile/compile-page.vue
+++ b/site/frontend/src/pages/compare/compile/compile-page.vue
@@ -216,22 +216,28 @@ const quickLinksKey = ref(0);
 const filter = ref(loadFilterFromUrl(urlParams, defaultCompileFilter));
 
 // Should we use the backend as the source of before/after data?
-const selfCompareBackend = computed(
-  () => comparesIdenticalCommits.value && filter.value.selfCompareBackend
-);
+const selfCompareBackend = computed(() => {
+  const hasMultipleBackends =
+    new Set(props.data.compile_comparisons.map((c) => c.backend)).size > 1;
+  return (
+    comparesIdenticalCommits.value &&
+    filter.value.selfCompareBackend &&
+    hasMultipleBackends
+  );
+});
 
 function exportData() {
   exportToMarkdown(comparisons.value, filter.value.showRawData);
 }
 
-// Are we currently comparing the same commit against each other?
+// Are we currently comparing the same commit in the before/after toolchains?
 const comparesIdenticalCommits = computed(() => {
   return props.data.a.commit === props.data.b.commit;
 });
 const benchmarkMap = createCompileBenchmarkMap(props.data);
 
-// Artificially restructure the data to create a comparison between backends
 const compileComparisons = computed(() => {
+  // If requested, artificially restructure the data to create a comparison between backends
   if (selfCompareBackend.value) {
     return transformDataForBackendComparison(props.data.compile_comparisons);
   } else {
@@ -258,8 +264,11 @@ const filteredSummary = computed(() => computeSummary(comparisons.value));
     :selected-metric="selector.stat"
     :metrics="benchmarkInfo.compile_metrics"
   />
-  <div v-if="comparesIdenticalCommits">
-    Self-compare backend:
+  <div
+    v-if="comparesIdenticalCommits"
+    :title="`Compare codegen backends for commit '${props.data.a.commit}'`"
+  >
+    Compare codegen backends for this commit:
     <input
       type="checkbox"
       :checked="selfCompareBackend"
@@ -275,7 +284,8 @@ const filteredSummary = computed(() => computeSummary(comparisons.value));
   <OverallSummary :summary="filteredSummary" />
   <Aggregations :cases="comparisons" />
   <div class="warning" v-if="selfCompareBackend">
-    Comparing Cranelift against LLVM baseline!
+    Note: comparing results of the baseline LLVM backend to the Cranelift
+    backend.
   </div>
   <Benchmarks
     :data="data"

From acc2d3c429b7c9c30fc9bc0eae1171036799b67f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <jakub.beranek@vsb.cz>
Date: Mon, 18 Nov 2024 15:17:55 +0100
Subject: [PATCH 8/8] Refactor logic for displaying self compare checkbox

---
 .../src/pages/compare/compile/compile-page.vue  | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/site/frontend/src/pages/compare/compile/compile-page.vue b/site/frontend/src/pages/compare/compile/compile-page.vue
index 216fc08e3..16219d054 100644
--- a/site/frontend/src/pages/compare/compile/compile-page.vue
+++ b/site/frontend/src/pages/compare/compile/compile-page.vue
@@ -217,23 +217,20 @@ const filter = ref(loadFilterFromUrl(urlParams, defaultCompileFilter));
 
 // Should we use the backend as the source of before/after data?
 const selfCompareBackend = computed(() => {
+  return canCompareBackends.value && filter.value.selfCompareBackend;
+});
+const canCompareBackends = computed(() => {
   const hasMultipleBackends =
     new Set(props.data.compile_comparisons.map((c) => c.backend)).size > 1;
-  return (
-    comparesIdenticalCommits.value &&
-    filter.value.selfCompareBackend &&
-    hasMultipleBackends
-  );
+  // Are we currently comparing the same commit in the before/after toolchains?
+  const comparesSameCommit = props.data.a.commit === props.data.b.commit;
+  return hasMultipleBackends && comparesSameCommit;
 });
 
 function exportData() {
   exportToMarkdown(comparisons.value, filter.value.showRawData);
 }
 
-// Are we currently comparing the same commit in the before/after toolchains?
-const comparesIdenticalCommits = computed(() => {
-  return props.data.a.commit === props.data.b.commit;
-});
 const benchmarkMap = createCompileBenchmarkMap(props.data);
 
 const compileComparisons = computed(() => {
@@ -265,7 +262,7 @@ const filteredSummary = computed(() => computeSummary(comparisons.value));
     :metrics="benchmarkInfo.compile_metrics"
   />
   <div
-    v-if="comparesIdenticalCommits"
+    v-if="canCompareBackends"
     :title="`Compare codegen backends for commit '${props.data.a.commit}'`"
   >
     Compare codegen backends for this commit: