Some updates to the Crossbow report

.github/workflows/crossbow-nightly-report-r-tests.yml

@@ -0,0 +1,53 @@
+name: Crossbow nightly report R tests
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - 'crossbow-nightly-report/**'
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'crossbow-nightly-report/**'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./crossbow-nightly-report
+
+    steps:
+    - uses: actions/checkout@v3
+
+
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libcurl4-openssl-dev libssl-dev libxml2-dev
+
+    - name: Set up R
+      uses: r-lib/actions/setup-r@v2
+      with:
+        r-version: '4.5.0'
+        use-public-rspm: true
+
+    - name: Restore packages using renv
+      uses: r-lib/actions/setup-renv@v2
+      with: 
+        working-directory: ./crossbow-nightly-report
+
+    - name: Install test dependencies
+      run: install.packages('testthat')
+      shell: Rscript {0}
+
+    - name: Run tests
+      run: |
+        library(testthat)
+        test_results <- test_dir("tests", reporter = "summary", stop_on_failure = FALSE)
+        test_result_df <- as.data.frame(test_results)
+        # Exit with error code if any tests failed
+        if (length(test_results) > 0 && any(c(test_result_df$failed, test_result_df$error))) {
+          quit(status = 1)
+        }
+      shell: Rscript {0}

.github/workflows/nightly_dashboard.yml

@@ -50,7 +50,7 @@ jobs:
 
       - uses: r-lib/actions/setup-r@v2
         with:
-          r-version: '4.4.0'
+          r-version: '4.5.0'
           use-public-rspm: true
 
       # Needed due to https://github.com/r-lib/actions/issues/618

crossbow-nightly-report/R/functions.R

@@ -1,11 +1,19 @@
+library(tibble)
+library(dplyr)
+library(lubridate)
+library(glue)
+library(tidyr)
+
 is_dev <- function() {
   Sys.getenv("GITHUB_ACTIONS") != "true"
 }
 
 dropdown_helper <- function(values, name, element_id) {
   htmltools::tags$select(
     # Set to undefined to clear the filter
-    onchange = glue("Reactable.setFilter('{element_id}', '{name}', event.target.value || undefined)"),
+    onchange = glue(
+      "Reactable.setFilter('{element_id}', '{name}', event.target.value || undefined)"
+    ),
     # "All" has an empty value to clear the filter, and is the default option
     htmltools::tags$option(value = "", "All"),
     lapply(unique(values), htmltools::tags$option),
@@ -14,11 +22,15 @@ dropdown_helper <- function(values, name, element_id) {
 }
 
 arrow_commit_links <- function(sha) {
-  glue("<a href='https://github.com/apache/arrow/commit/{sha}' target='_blank'>{substring(sha, 1, 7)}</a>")
+  glue(
+    "<a href='https://github.com/apache/arrow/commit/{sha}' target='_blank'>{substring(sha, 1, 7)}</a>"
+  )
 }
 
 arrow_compare_links <- function(sha1, sha2) {
-  comp_link <- glue("<a href='https://github.com/apache/arrow/compare/{sha1}...{sha2}' target='_blank'>{substring(sha1, 1, 7)}</a>")
+  comp_link <- glue(
+    "<a href='https://github.com/apache/arrow/compare/{sha1}...{sha2}' target='_blank'>{substring(sha1, 1, 7)}</a>"
+  )
 
   if (rlang::is_empty(comp_link)) {
     return(glue("Build has not yet been successful"))
@@ -34,86 +46,132 @@ make_nice_names <- function(x) {
   toTitleCase(gsub("_", " ", names(x)))
 }
 
-arrow_build_table <- function(nightly_data, type, task) {
+get_commit <- function(df, label) {
+  df$arrow_commit[df$fail_label == label]
+}
+
+arrow_build_table <- function(nightly_data, type, task, to_day = today()) {
+  # Filter data for a specific build type and task
   type_task_data <- nightly_data %>%
     filter(build_type == type) %>%
     filter(task_name == task)
 
-  ## filter for when the most recent run is a failure 
-  day_window <- today() - 2
+  # Look at yesterday's date to determine recent failures
+  # This is used as a window for identifying tasks that failed recently
+  day_window <- to_day - 1
+
+  # Get records where the task failed recently, order by date (newest first)
+  # and standardize task status values to "pass" and "fail"
   ordered_only_recent_fails <- type_task_data %>%
-    filter(task_name %in% task_name[nightly_date == day_window & task_status != "success"]) %>%
+    # Only keep records where the task name appears in yesterday's failures
+    filter(
+      task_name %in%
+        task_name[nightly_date == day_window & task_status != "success"]
+    ) %>%
     arrange(desc(nightly_date)) %>%
-    mutate(task_status = case_when(
-      task_status == "success" ~ "pass",
-      task_status == "failure" ~ "fail",
-      TRUE ~ task_status
-    ))
+    mutate(
+      task_status = case_when(
+        task_status == "success" ~ "pass",
+        task_status == "failure" ~ "fail",
+        TRUE ~ task_status
+      )
+    )
 
+  # If there are no recent failures, return a success summary or a null summary if the task is not active
   if (nrow(ordered_only_recent_fails) == 0) {
-    ## if there are no failures, return a version of the table that reflects that
+    # Calculate days since the last run (regardless of status)
     days <- as.numeric(
       difftime(
-        ymd(Sys.Date(), tz = "UTC"),
+        ymd(to_day, tz = "UTC"),
         max(type_task_data$nightly_date)
       )
     )
+    # Create a summary with success information
     success_df <- type_task_data %>%
+      # Remove stale data by filtering out everything but the last ~2 days of runs
+      # this makes it so that jobs that have been deleted (but are still in the 120 day look back)
+      # don't continue to show up.
+      filter(nightly_date >= to_day - 2) %>%
+      # Then, take the most recent run since that's all we care about if there are no failures.
       slice_max(order_by = nightly_date) %>%
       mutate(
         since_last_successful_build = days,
         last_successful_commit = arrow_commit_links(arrow_commit),
-        last_successful_build = glue("<a href='{build_links}' target='_blank'>{task_status}</a>"),
+        last_successful_build = glue(
+          "<a href='{build_links}' target='_blank'>{task_status}</a>"
+        ),
         most_recent_status = "passing"
       ) %>%
-      select(task_name, most_recent_status, since_last_successful_build, last_successful_commit, last_successful_build, build_type)
+      select(
+        task_name,
+        most_recent_status,
+        since_last_successful_build,
+        last_successful_commit,
+        last_successful_build,
+        build_type
+      )
+
     return(success_df)
   }
 
-  ## find first failure index
+  # Find the length of the most recent consecutive failure streak
+  # This uses run length encoding to identify the first sequence of failures
   idx_recent_fail <- rle(ordered_only_recent_fails$task_status)$lengths[1]
 
-  ## expand failure index and give it some names
+  # Create labels for the failure streak timeline
+  # This builds a dataframe with positions and labels for the recent failure sequence
   failure_df <- tibble(fails_plus_one = seq(1, idx_recent_fail + 1)) %>%
-    mutate(fail_label = case_when(
-      fails_plus_one == idx_recent_fail ~ "first_failure",
-      fails_plus_one == 1 ~ "most_recent_failure",
-      fails_plus_one == idx_recent_fail + 1 ~ "last_successful_build",
-      TRUE ~ paste0(fails_plus_one, " days ago")
-    )) %>%
+    mutate(
+      fail_label = case_when(
+        fails_plus_one == idx_recent_fail ~ "first_failure", # Where the failures began
+        fails_plus_one == 1 ~ "most_recent_failure", # The most recent failure
+        fails_plus_one == idx_recent_fail + 1 ~ "last_successful_build", # Last successful build before failures
+        TRUE ~ paste0(fails_plus_one, " days ago") # General failure timeline
+      )
+    ) %>%
+    # Only keep the most recent 9 days of failures or specific labeled events
     filter(fails_plus_one <= 9 | grepl("failure|build", fail_label))
 
-  ## inner_join to ordered data
+  # Join the failure timeline labels with the actual build data
   df <- ordered_only_recent_fails %>%
     rowid_to_column() %>%
     inner_join(failure_df, by = c("rowid" = "fails_plus_one"))
 
-
+  # Calculate days since last successful build
   if (all(type_task_data$task_status %in% "failure")) {
     days <- NA_real_
   } else {
-    ## days since last successful build (need to add one)
-    days <- sum(as.numeric(
-      difftime(
-        df$nightly_date[df$fail_label == "most_recent_failure"],
-        df$nightly_date[df$fail_label == "last_successful_build"]
-      )
-    ), 1)
-  }
-
-
-  get_commit <- function(label) {
-    df$arrow_commit[df$fail_label == label]
+    # Calculate days between most recent failure and last successful build
+    # Adding 1 to include the day of the failure
+    days <- sum(
+      as.numeric(
+        difftime(
+          df$nightly_date[df$fail_label == "most_recent_failure"],
+          df$nightly_date[df$fail_label == "last_successful_build"]
+        )
+      ),
+      1
+    )
   }
 
+  # Format the final result as a table with build status information (one row per task)
   df %>%
     arrange(desc(fail_label)) %>%
-    mutate(build_links = glue("<a href='{build_links}' target='_blank'>{task_status}</a>")) %>%
+    mutate(
+      build_links = glue(
+        "<a href='{build_links}' target='_blank'>{task_status}</a>"
+      )
+    ) %>%
     select(task_name, build_type, build_links, fail_label) %>%
+    # Reshape data to have one column for each failure stage
     pivot_wider(names_from = fail_label, values_from = build_links) %>%
+    # Add additional context columns
     mutate(
       since_last_successful_build = days,
-      last_successful_commit = arrow_compare_links(get_commit("last_successful_build"), get_commit("first_failure")),
+      last_successful_commit = arrow_compare_links(
+        get_commit(df, "last_successful_build"),
+        get_commit(df, "first_failure")
+      ),
       most_recent_status = "failing",
       .after = build_type
     )
@@ -124,6 +182,6 @@ crossbow_theme <- function(data, ...) {
     tab_options(
       table.font.size = 14,
       ...
-    ) %>% 
+    ) %>%
     opt_all_caps()
 }

crossbow-nightly-report/air.toml

@@ -0,0 +1,2 @@
+[format]
+line-width = 120

crossbow-nightly-report/crossbow-nightly-report.qmd

@@ -79,7 +79,7 @@ most_recent_commit <- unique(nightly$arrow_commit[nightly$nightly_date == max(ni
 
 <a href='https://arrow.apache.org/'><img src='https://arrow.apache.org/img/arrow-logo_hex_black-txt_white-bg.png' align="right" height="150" /></a>
 
-This report builds in sync with the email notifications (`[email protected]`). `r pluralize('Most recent commit{?s} are:  {arrow_commit_links(most_recent_commit)}')`. The report examines data from the last `r lookback_window` days.
+This report builds in sync with the email notifications (`[email protected]`). `r pluralize('Most recent commit{?s} {?is/are}:  {arrow_commit_links(most_recent_commit)}')`. The report examines data from the last `r lookback_window` days.
 
 
 # Summary 
@@ -280,7 +280,7 @@ build_table <- map2_df(map_params$build_type, map_params$task_name, ~ arrow_buil
 ```{r build_table}
 bs_nightly_tbl <- build_table %>%
   select(where(~ sum(!is.na(.x)) > 0)) %>% ## remove any columns with no data
-  arrange(build_type, most_recent_status, desc(since_last_successful_build)) %>%
+  arrange(most_recent_status, build_type, desc(since_last_successful_build)) %>%
   rowwise() %>%
   mutate(
     since_last_successful_build = ifelse(
@@ -297,12 +297,11 @@ bs_nightly_tbl_sorted <- bs_nightly_tbl[, rev(sort(names(bs_nightly_tbl)))] %>%
   relocate(any_of(c("last_successful_build", "first_failure")), .after = last_successful_commit) %>%
   relocate(build_type, .before = task_name)
 
-## set a row number such all the failing builds are always displayed
+## set a row number such all the failing builds are always displayed with a lower bound of 25
 tbl_row_n <- bs_nightly_tbl_sorted %>%
   filter(most_recent_status == "failing") %>%
-  count(build_type) %>%
-  filter(n == max(n)) %>%
-  pull(n)
+  count() %>%
+  max(., 25)
 
 names(bs_nightly_tbl_sorted) <- make_nice_names(bs_nightly_tbl_sorted)
 
@@ -332,13 +331,13 @@ bs_nightly_tbl_sorted %>%
     ),
     columns = list(
       `Build Type` = reactable::colDef(
-        width = 80,
+        width = 100,
         filterInput = function(values, name) {
           dropdown_helper(values, name, build_element_id)
         }
       ),
       `Task Name` = reactable::colDef(
-        width = 110
+        width = 150
       ),
       `Since Last Successful Build` = reactable::colDef(
         width = 100,
@@ -347,22 +346,22 @@ bs_nightly_tbl_sorted %>%
         }
       ),
       `Most Recent Status` = reactable::colDef(
-        width = 70,
+        width = 85,
         filterInput = function(values, name) {
           dropdown_helper(values, name, build_element_id)
         }
       ),
       `Last Successful Commit` = reactable::colDef(
-        width = 80
+        width = 90
       ),
       `Last Successful Build` = reactable::colDef(
         width = 80
       ),
       `First Failure` = reactable::colDef(
-        width = 60
+        width = 70
       ),
       `Most Recent Failure` = reactable::colDef(
-        width = 60
+        width = 70
       )
     ),
     elementId = build_element_id
@@ -408,16 +407,16 @@ nightly_sub_tbl %>%
     ),
     columns = list(
       `Task Status` = reactable::colDef(
-        width = 80,
+        width = 90,
         filterInput = function(values, name) {
           dropdown_helper(values, name, error_element_id)
         }
       ),
       `Task Name` = reactable::colDef(
-        width = 200,
-        filterInput = function(values, name) {
-          dropdown_helper(values, name, error_element_id)
-        }
+        width = 240,
+        # filterInput = function(values, name) {
+        #   dropdown_helper(values, name, error_element_id)
+        # }
       ),
       `Build Type` = reactable::colDef(
         width = 100,