Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/.sqlx/*.json binary linguist-generated=true

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions migrations/20260106212918_rollup_member.down.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
DROP TABLE IF EXISTS rollup_member;
11 changes: 11 additions & 0 deletions migrations/20260106212918_rollup_member.up.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
-- Add up migration script here
CREATE TABLE IF NOT EXISTS rollup_member
(
-- We have to store the PR through their numbers rather than a FK to the pull_request table. This is due to the rollup PR
-- being created at the time of record insertion in this table, so we won't have an entry in pull_request for it
rollup_pr_number BIGINT NOT NULL,
member_pr_number BIGINT NOT NULL,
repository TEXT NOT NULL,
Comment on lines +6 to +8
Copy link
Member

@Kobzol Kobzol Jan 12, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
rollup_pr_number BIGINT NOT NULL,
member_pr_number BIGINT NOT NULL,
repository TEXT NOT NULL,
repository TEXT NOT NULL,
rollup_pr_number BIGINT NOT NULL,
member_pr_number BIGINT NOT NULL,

Just a nit, it's easier to read like this when inspecting the DB state manually :) The migration data SQL file will also need to be updated after this change.


PRIMARY KEY (repository, rollup_pr_number, member_pr_number)
);
30 changes: 29 additions & 1 deletion src/database/client.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
use sqlx::PgPool;
use std::collections::{HashMap, HashSet};

use crate::bors::comment::CommentTag;
use crate::bors::{BuildKind, PullRequestStatus, RollupMode};
use crate::database::operations::update_pr_auto_build_id;
use crate::database::operations::{
get_nonclosed_rollups, register_rollup_pr_member, update_pr_auto_build_id,
};
use crate::database::{
BuildModel, BuildStatus, CommentModel, PullRequestModel, RepoModel, TreeState, WorkflowModel,
WorkflowStatus, WorkflowType,
Expand Down Expand Up @@ -336,4 +339,29 @@ impl PgDbClient {
pub async fn delete_tagged_bot_comment(&self, comment: &CommentModel) -> anyhow::Result<()> {
delete_tagged_bot_comment(&self.pool, comment.id).await
}

/// Register the contents of a rollup in the DB.
/// The contents are stored as rollup PR number - member PR number records, scoped under a specific repository.
/// All records are inserted in a single transaction.
pub async fn register_rollup_members(
&self,
repo: &GithubRepoName,
rollup_pr_number: &PullRequestNumber,
member_pr_numbers: &[PullRequestNumber],
) -> anyhow::Result<()> {
let mut tx = self.pool.begin().await?;
for member_pr_number in member_pr_numbers {
register_rollup_pr_member(&mut *tx, repo, rollup_pr_number, member_pr_number).await?;
}
tx.commit().await?;
Ok(())
}

/// Returns a map of rollup PR numbers to the set of member PR numbers that are part of that rollup.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
/// Returns a map of rollup PR numbers to the set of member PR numbers that are part of that rollup.
/// Returns a map of rollup PR numbers to the set of member PR numbers that are part of that rollup.
/// Only returns non-closed rollup PRs.

pub async fn get_nonclosed_rollups(
&self,
repo: &GithubRepoName,
) -> anyhow::Result<HashMap<PullRequestNumber, HashSet<PullRequestNumber>>> {
get_nonclosed_rollups(&self.pool, repo).await
}
}
58 changes: 58 additions & 0 deletions src/database/operations.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use chrono::DateTime;
use chrono::Utc;
use sqlx::postgres::PgExecutor;
use std::collections::{HashMap, HashSet};

use super::ApprovalInfo;
use super::ApprovalStatus;
Expand Down Expand Up @@ -1064,3 +1065,60 @@ pub(crate) async fn clear_auto_build(
})
.await
}

pub(crate) async fn register_rollup_pr_member(
executor: impl PgExecutor<'_>,
repo: &GithubRepoName,
rollup_pr_number: &PullRequestNumber,
member_pr_number: &PullRequestNumber,
) -> anyhow::Result<()> {
measure_db_query("register_rollup_pr_member", || async {
sqlx::query!(
r#"
INSERT INTO rollup_member (repository, rollup_pr_number, member_pr_number)
VALUES ($1, $2, $3)
"#,
repo as &GithubRepoName,
rollup_pr_number.0 as i32,
member_pr_number.0 as i32
)
.execute(executor)
.await?;
Ok(())
})
.await
}

pub(crate) async fn get_nonclosed_rollups(
executor: impl PgExecutor<'_>,
repo: &GithubRepoName,
) -> anyhow::Result<HashMap<PullRequestNumber, HashSet<PullRequestNumber>>> {
measure_db_query("get_nonclosed_rollups", || async {
let mut stream = sqlx::query!(
r#"
SELECT rm.rollup_pr_number, rm.member_pr_number
FROM rollup_member rm
JOIN pull_request pr
ON pr.repository = rm.repository
AND pr.number = rm.rollup_pr_number
WHERE rm.repository = $1
AND pr.status IN ('open', 'draft')
ORDER BY rm.rollup_pr_number;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
ORDER BY rm.rollup_pr_number;

Doesn't seem to be required, since we store the numbers in a hashmap/hashset anyway? Might help with cache locality for the hashmap, but that's not really needed here, and the DB won't have to sort the rollups.

"#,
repo as &GithubRepoName,
)
.fetch(executor);

let mut rollups_map: HashMap<PullRequestNumber, HashSet<PullRequestNumber>> =
HashMap::new();
while let Some(row) = stream.try_next().await? {
let rollup_pr = PullRequestNumber(row.rollup_pr_number as u64);
let member_pr = PullRequestNumber(row.member_pr_number as u64);

rollups_map.entry(rollup_pr).or_default().insert(member_pr);
}

Ok(rollups_map)
})
.await
}
87 changes: 80 additions & 7 deletions src/github/rollup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,6 @@ async fn create_rollup(
repo_owner,
mut pr_nums,
} = rollup_state;

// Repository where we will create the PR
let base_repo = GithubRepoName::new(&repo_owner, &repo_name);

Expand Down Expand Up @@ -388,6 +387,14 @@ async fn create_rollup(
.await
.context("Cannot create PR")?;

db.register_rollup_members(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was thinking about (many!) follow-up features that we could implement once this PR lands, and I realized that the problem with the missing PR id can be solved very easily. We can simply pre-insert the PR into the DB immediately here:

let pr_db = db.upsert_pull_request(..., pr.into());

and then use pr_db.id to record the rollup mapping in the DB.

Since we anyway have to do a JOIN with the PR table when selecting the rollup mapping due to filtering out open PRs, I think that storing the IDs in the mapping would be simpler, after all, would waste less DB space, and would provide proper FK mapping in the DB.

Could you please change it to store (FK to PR table, rollup PR number) in the rollup_member table? Sorry for the late notice 😅

Copy link
Author

@Carbonhell Carbonhell Jan 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That makes sense! Some questions:

  1. I assume the fact we perform another upsert from the webhook event received by GitHub for the same PR doesn't matter, as it'd just write the same data. Adding a check to prevent writing PRs that are already in the table in the webhook logic seems pointless - is my assumption correct?
  2. Wouldn't it make sense to change the member PR number in the same way? I assume we're guaranteed to have an entry in the PR table, as from what I know, rollups can only be created from the queue page, which implies PRs shown there have an entry in the table. Therefore we could use a FK for member PRs too for the same reasons. I think the only tradeoff is that writes would be slightly more expensive, but it shoudn't be a concern with the low amount of writes, and it might help future with future query patterns, if any.

I'm always looking for issues to contribute to (mostly during weekends, when i'm not too tired from working in a startup haha). If you want, write down the ideas in GitHub issues and I'll take care of them!

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. Yeah, the open PR webhook is not load-bearing. It can in theory race with the rollup handler, but that shouldn't matter much, Postgres will ensure atomicity of the PR creation.
  2. A very good point! Yeah, I agree. In all cases where we will query this table, the corresponding PRs should already be in the DB. Including when we record the member entries.

Btw, after this change you can remove refresh_prs from rollup tests.

Okay, I will create issues for the features, once this PR lands.

&base_repo,
&pr.number,
&successes.iter().map(|pr| pr.number).collect::<Vec<_>>(),
)
.await
.context("Cannot register rollup member record")?;

// Set the rollup label
gh_client
.add_labels(pr.number, &["rollup".to_string()])
Expand All @@ -406,6 +413,7 @@ mod tests {
User, default_repo_name, run_test,
};
use http::StatusCode;
use std::collections::{HashMap, HashSet};

#[sqlx::test]
async fn rollup_missing_fork(pool: sqlx::PgPool) {
Expand Down Expand Up @@ -528,6 +536,17 @@ mod tests {
make_rollup(ctx, &[&pr2, &pr3, &pr4, &pr5])
.await?
.assert_status(StatusCode::SEE_OTHER);
assert_eq!(
ctx.db().get_nonclosed_rollups(&default_repo_name()).await?,
HashMap::from([(
PullRequestNumber(6),
HashSet::from_iter(vec![
PullRequestNumber(2),
PullRequestNumber(3),
PullRequestNumber(5)
])
)])
);
Ok(())
})
.await;
Expand Down Expand Up @@ -650,7 +669,57 @@ mod tests {
insta::assert_snapshot!(pr_url, @"https://github.com/rust-lang/borstest/pull/4");

ctx.pr(4).await.expect_added_labels(&["rollup"]);
assert_eq!(
ctx.db().get_nonclosed_rollups(&pr2.repo).await?,
HashMap::from([(
PullRequestNumber(4),
HashSet::from_iter(vec![PullRequestNumber(3), PullRequestNumber(2)])
)])
);

Ok(())
})
.await;
let repo = gh.get_repo(());
insta::assert_snapshot!(repo.lock().get_pr(4).description, @"
Successful merges:

- rust-lang/borstest#2 (Title of PR 2)
- rust-lang/borstest#3 (Title of PR 3)

r? @ghost

<!-- homu-ignore:start -->
[Create a similar rollup](https://bors-test.com/queue/borstest?prs=2,3)
<!-- homu-ignore:end -->
");
}

/// Ensure that a PR can be associated to multiple rollups.
/// This can happen when opening a new rollup similar to an existing one, before closing the old one.
#[sqlx::test]
async fn multiple_rollups_same_pr(pool: sqlx::PgPool) {
let gh = run_test((pool, rollup_state()), async |ctx: &mut BorsTester| {
let pr2 = ctx.open_pr((), |_| {}).await?;
let pr3 = ctx.open_pr((), |_| {}).await?;
ctx.approve(pr2.id()).await?;
ctx.approve(pr3.id()).await?;

make_rollup(ctx, &[&pr2, &pr3]).await?;
make_rollup(ctx, &[&pr3]).await?;
assert_eq!(
ctx.db().get_nonclosed_rollups(&pr2.repo).await?,
HashMap::from([
(
PullRequestNumber(4),
HashSet::from_iter(vec![PullRequestNumber(2), PullRequestNumber(3)])
),
(
PullRequestNumber(5),
HashSet::from_iter(vec![PullRequestNumber(3)])
)
])
);
Ok(())
})
.await;
Expand Down Expand Up @@ -799,12 +868,16 @@ also include this pls"
prs: &[&PullRequest],
) -> anyhow::Result<ApiResponse> {
let prs = prs.iter().map(|pr| pr.number).collect::<Vec<_>>();
ctx.api_request(rollup_request(
&rollup_user().name,
default_repo_name(),
&prs,
))
.await
let response = ctx
.api_request(rollup_request(
&rollup_user().name,
default_repo_name(),
&prs,
))
.await;
// Trigger a refresh so that the new rollup PR can be upserted in the pull_request table
ctx.refresh_prs().await;
response
}

fn rollup_state() -> GitHub {
Expand Down
4 changes: 4 additions & 0 deletions src/server/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ use crate::database::{ApprovalStatus, PullRequestModel, QueueStatus};
use crate::github::{GithubRepoName, rollup};
use crate::templates::{
HelpTemplate, HtmlTemplate, NotFoundTemplate, PullRequestStats, QueueTemplate, RepositoryView,
RollupsInfo,
};
use crate::utils::sort_queue::sort_queue_prs;
use crate::{
Expand Down Expand Up @@ -377,6 +378,8 @@ pub async fn queue_handler(
(in_queue + in_queue_inc, failed + failed_inc)
});

let rollups = db.get_nonclosed_rollups(&repo.name).await?;

Ok(HtmlTemplate(QueueTemplate {
oauth_client_id: oauth
.as_ref()
Expand All @@ -393,6 +396,7 @@ pub async fn queue_handler(
prs,
pending_workflow,
selected_rollup_prs: params.pull_requests.map(|prs| prs.0).unwrap_or_default(),
rollups_info: RollupsInfo::from(rollups),
})
.into_response())
}
Expand Down
15 changes: 15 additions & 0 deletions src/templates.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,12 @@ use crate::database::{
BuildModel, BuildStatus, MergeableState::*, PullRequestModel, QueueStatus, TreeState,
WorkflowModel,
};
use crate::github::PullRequestNumber;
use askama::Template;
use axum::response::{Html, IntoResponse, Response};
use http::StatusCode;
use itertools::Itertools;
use std::collections::{HashMap, HashSet};

/// Build status to display on the queue page.
pub fn status_text(pr: &PullRequestModel) -> String {
Expand Down Expand Up @@ -55,6 +58,17 @@ pub struct PullRequestStats {
pub failed_count: usize,
}

/// A displayable view of a map of rollups to their members, meant to be used with data attributes
pub struct RollupsInfo {
pub rollups: HashMap<PullRequestNumber, HashSet<PullRequestNumber>>,
}

impl From<HashMap<PullRequestNumber, HashSet<PullRequestNumber>>> for RollupsInfo {
fn from(rollups: HashMap<PullRequestNumber, HashSet<PullRequestNumber>>) -> Self {
Self { rollups }
}
}

#[derive(Template)]
#[template(path = "queue.html", whitespace = "minimize")]
pub struct QueueTemplate {
Expand All @@ -63,6 +77,7 @@ pub struct QueueTemplate {
pub repo_url: String,
pub stats: PullRequestStats,
pub prs: Vec<PullRequestModel>,
pub rollups_info: RollupsInfo,
pub tree_state: TreeState,
pub oauth_client_id: Option<String>,
// PRs that should be pre-selected for being included in a rollup
Expand Down
Loading