feat: stream newest pull requests by category

This commit is contained in:
Valdemar Lorenzen
2026-10-14 14:08:21 +02:00
parent 5243de66fe
commit 0000089f42
8 changed files with 1465 additions and 377 deletions
+523 -135
View File
@@ -2,17 +2,26 @@
use crate::repository::{checked, run};
use serde_json::Value;
use std::{
collections::{BTreeMap, BTreeSet},
collections::BTreeMap,
path::Path,
process::Command,
sync::atomic::{AtomicBool, Ordering},
};
pub(crate) struct PullRequest {
pub(crate) struct PullRequestSummary {
pub number: u64,
pub title: String,
pub description: String,
pub closed: bool,
pub created_at: String,
}
pub(crate) struct PullRequestPage {
pub entries: Vec<PullRequestSummary>,
pub has_more: bool,
}
pub(crate) struct PullRequest {
pub revisions: Result<PullRequestRevisions, String>,
}
@@ -360,6 +369,246 @@ impl Forge {
}
}
impl Forge {
fn page_with(
&self,
closed: bool,
page: u32,
mut get: impl FnMut(&str) -> Result<Value, String>,
) -> Result<PullRequestPage, String> {
if page == 0 {
return Err("Forge page numbers start at 1".into());
}
let states: &[&str] = match (self.provider, closed) {
(Provider::Gitlab, true) => &["closed", "merged"],
(Provider::Gitlab, false) => &["opened"],
(_, true) => &["closed"],
(_, false) => &["open"],
};
let mut entries = BTreeMap::new();
let mut has_more = false;
for state in states {
let query = match self.provider {
Provider::Github => "per_page=50&sort=created&direction=desc",
Provider::Gitlab => "scope=all&per_page=50&order_by=created_at&sort=desc",
// Gitea's default sort is descending creation time; "oldest" reverses it.
Provider::Gitea => "limit=50",
};
let values = get(&format!(
"{}?state={state}&{query}&page={page}",
self.endpoint
))?;
let values = values
.as_array()
.ok_or("Forge API returned a non-array pull request list")?;
// Servers may cap page sizes. Only an empty page proves exhaustion.
has_more |= !values.is_empty();
for value in values {
let entry = summary(self.provider, value)?;
if entry.closed != closed {
return Err("Forge API returned a pull request in the wrong category".into());
}
entries.insert(entry.number, entry);
}
}
// GitLab needs two independently paged states for Closed. Later batches may
// precede older rows from the other state; consumers sort the partial catalog.
let mut entries: Vec<_> = entries.into_values().collect();
entries.sort_unstable_by(|a, b| {
b.created_at
.cmp(&a.created_at)
.then_with(|| b.number.cmp(&a.number))
});
Ok(PullRequestPage { entries, has_more })
}
fn resolve_with(
&self,
id: u64,
mut get: impl FnMut(&str) -> Result<Value, String>,
) -> Result<PullRequest, String> {
if id == 0 {
return Err("Pull request numbers start at 1".into());
}
let detail = get(&format!("{}/{id}", self.endpoint))?;
if number(
&detail,
if self.provider == Provider::Gitlab {
"iid"
} else {
"number"
},
)? != id
{
return Err("Forge API returned a different pull request".into());
}
let preserved_head = if self.provider == Provider::Gitea {
let repo = self.endpoint.strip_suffix("/pulls").unwrap();
let reference = get(&format!("{repo}/git/refs/pull/{id}/head"))?;
Some(gitea_head(&reference, id)?)
} else {
None
};
pull_request(self.provider, &detail, preserved_head)
}
}
fn gitea_head(value: &Value, id: u64) -> Result<String, String> {
// Gitea versions return either an object or an array for the filtered refs API.
let value = if let Some(values) = value.as_array() {
if values.len() != 1 {
return Err(format!(
"Gitea pull request #{id} has no unique preserved head reference"
));
}
&values[0]
} else {
value
};
if field(value, "ref")? != format!("refs/pull/{id}/head") {
return Err("Gitea API returned a different pull request reference".into());
}
let object = value
.get("object")
.ok_or("Gitea API returned no reference object")?;
if field(object, "type")? != "commit" {
return Err("Gitea API returned a non-commit pull request head".into());
}
sha(object, "sha")
}
fn created_at(value: &Value) -> Result<String, String> {
let invalid = || "Forge API returned invalid created_at".to_owned();
let text = field(value, "created_at")?;
let digits = |text: &str| -> Option<i32> {
(!text.is_empty() && text.bytes().all(|b| b.is_ascii_digit()))
.then(|| text.parse().ok())
.flatten()
};
if !text.is_ascii()
|| text.len() < 20
|| &text[4..5] != "-"
|| &text[7..8] != "-"
|| &text[10..11] != "T"
|| &text[13..14] != ":"
|| &text[16..17] != ":"
{
return Err(invalid());
}
let mut year = digits(&text[..4]).ok_or_else(invalid)?;
let mut month = digits(&text[5..7]).ok_or_else(invalid)?;
let mut day = digits(&text[8..10]).ok_or_else(invalid)?;
let hour = digits(&text[11..13]).ok_or_else(invalid)?;
let minute = digits(&text[14..16]).ok_or_else(invalid)?;
let second = digits(&text[17..19]).ok_or_else(invalid)?;
let days = |year: i32, month: i32| match month {
2 => 28 + i32::from(year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)),
4 | 6 | 9 | 11 => 30,
_ => 31,
};
if year == 0
|| !(1..=12).contains(&month)
|| !(1..=days(year, month)).contains(&day)
|| hour > 23
|| minute > 59
|| second > 59
{
return Err(invalid());
}
let mut rest = &text[19..];
let mut nanos = 0;
if let Some(fraction) = rest.strip_prefix('.') {
let end = fraction
.find(|c: char| !c.is_ascii_digit())
.ok_or_else(invalid)?;
if end == 0 || end > 9 {
return Err(invalid());
}
nanos = digits(&fraction[..end]).ok_or_else(invalid)? * 10i32.pow(9 - end as u32);
rest = &fraction[end..];
}
let offset = if rest == "Z" {
0
} else if rest.len() == 6 && matches!(&rest[..1], "+" | "-") && &rest[3..4] == ":" {
let hours = digits(&rest[1..3]).ok_or_else(invalid)?;
let minutes = digits(&rest[4..6]).ok_or_else(invalid)?;
if hours > 23 || minutes > 59 {
return Err(invalid());
}
(hours * 60 + minutes) * if rest.starts_with('+') { 1 } else { -1 }
} else {
return Err(invalid());
};
let minutes = hour * 60 + minute - offset;
day += minutes.div_euclid(1440);
if day == 0 {
month -= 1;
if month == 0 {
year -= 1;
month = 12;
}
day = days(year, month);
} else if day > days(year, month) {
day = 1;
month += 1;
if month == 13 {
year += 1;
month = 1;
}
}
if !(1..=9999).contains(&year) {
return Err(invalid());
}
let minutes = minutes.rem_euclid(1440);
Ok(format!(
"{year:04}-{month:02}-{day:02}T{:02}:{:02}:{second:02}.{nanos:09}Z",
minutes / 60,
minutes % 60
))
}
fn summary(provider: Provider, value: &Value) -> Result<PullRequestSummary, String> {
let gitlab = provider == Provider::Gitlab;
let (closed, merged) = status(provider, value)?;
let author = author(provider, value)?;
let state = if merged {
"merged"
} else if closed {
"closed"
} else {
"open"
};
let (base, head) = if gitlab {
(
value.get("target_branch").and_then(Value::as_str),
value.get("source_branch").and_then(Value::as_str),
)
} else {
(
value.pointer("/base/ref").and_then(Value::as_str),
value
.pointer(if provider == Provider::Gitea {
"/head/label"
} else {
"/head/ref"
})
.and_then(Value::as_str)
.filter(|s| !s.is_empty())
.or_else(|| value.pointer("/head/ref").and_then(Value::as_str)),
)
};
Ok(PullRequestSummary {
number: number(value, if gitlab { "iid" } else { "number" })?,
title: field(value, "title")?.to_owned(),
description: match (base, head) {
(Some(base), Some(head)) => format!("{author} · {state} · {head} → {base}"),
_ => format!("{author} · {state}"),
},
closed,
created_at: created_at(value)?,
})
}
fn field<'a>(value: &'a Value, key: &str) -> Result<&'a str, String> {
value
.get(key)
@@ -401,13 +650,8 @@ fn branch<'a>(value: &'a Value, key: &str) -> Result<&'a str, String> {
}
}
fn pull_request(
provider: Provider,
value: &Value,
preserved_head: Option<String>,
) -> Result<PullRequest, String> {
fn status(provider: Provider, value: &Value) -> Result<(bool, bool), String> {
let gitlab = provider == Provider::Gitlab;
let number = number(value, if gitlab { "iid" } else { "number" })?;
let state = field(value, "state")?;
let closed = match (gitlab, state) {
(false, "open") | (true, "opened" | "locked") => false,
@@ -429,6 +673,29 @@ fn pull_request(
if merged && !closed {
return Err("Forge API returned inconsistent merged state".into());
}
Ok((closed, merged))
}
fn author(provider: Provider, value: &Value) -> Result<&str, String> {
let gitlab = provider == Provider::Gitlab;
let author = value
.get(if gitlab { "author" } else { "user" })
.ok_or("Forge API returned no author")?;
if author.is_null() {
Ok("deleted user")
} else {
field(author, if gitlab { "username" } else { "login" })
}
}
fn pull_request(
provider: Provider,
value: &Value,
preserved_head: Option<String>,
) -> Result<PullRequest, String> {
let gitlab = provider == Provider::Gitlab;
let number = number(value, if gitlab { "iid" } else { "number" })?;
status(provider, value)?;
let revisions = (|| -> Result<_, String> {
let (base, head, base_sha, head_sha) = if gitlab {
let refs = value.get("diff_refs").filter(|v| v.is_object()).ok_or(
@@ -502,143 +769,46 @@ fn pull_request(
head,
))
})();
let author = value
.get(if gitlab { "author" } else { "user" })
.ok_or("Forge API returned no author")?;
let author = if author.is_null() {
"deleted user"
} else {
field(author, if gitlab { "username" } else { "login" })?
};
let state = if merged {
"merged"
} else if closed {
"closed"
} else {
"open"
};
Ok(PullRequest {
number,
title: field(value, "title")?.to_owned(),
description: match &revisions {
Ok((_, base, head)) => format!("{author} · {state} · {head} → {base}"),
Err(error) => format!("{author} · {state} · unavailable: {error}"),
},
closed,
revisions: revisions.map(|(revisions, _, _)| revisions),
})
}
pub(crate) fn list(
pub(crate) fn page(
url: &str,
provider: Option<&str>,
closed: bool,
page: u32,
root: &Path,
cancelled: &AtomicBool,
) -> Result<Vec<PullRequest>, String> {
) -> Result<PullRequestPage, String> {
let forge = Forge::parse(url, provider, |destination, port| {
let mut command = Command::new("ssh");
command.current_dir(root);
ssh_hostname(command, destination, port, cancelled)
})?;
let token = forge.token(root, cancelled)?;
let mut gitea_heads = BTreeMap::new();
let query = match forge.provider {
Provider::Github => "state=all&per_page=100&sort=created&direction=asc",
Provider::Gitlab => "state=all&scope=all&per_page=100&order_by=created_at&sort=asc",
Provider::Gitea => "state=all&limit=100&sort=oldest",
};
let mut requests = Vec::new();
let mut seen = BTreeSet::new();
for page in 1u64.. {
let values = forge.get(
&format!("{}?{query}&page={page}", forge.endpoint),
token.as_deref(),
root,
cancelled,
)?;
let values = values
.as_array()
.ok_or("Forge API returned a non-array pull request list")?;
// A server may cap the requested page size. Only an empty page proves exhaustion.
if values.is_empty() {
return Ok(requests);
}
if page == 1 && forge.provider == Provider::Gitea {
let repo = forge.endpoint.strip_suffix("/pulls").unwrap();
let refs = forge.get(
&format!("{repo}/git/refs/pull"),
token.as_deref(),
root,
cancelled,
)?;
for reference in refs
.as_array()
.ok_or("Gitea API returned a non-array reference list")?
{
let name = field(reference, "ref")?;
if let Some(id) = name
.strip_prefix("refs/pull/")
.and_then(|s| s.strip_suffix("/head"))
{
let id = id
.parse::<u64>()
.ok()
.filter(|n| *n > 0)
.ok_or("Gitea API returned an invalid pull request reference")?;
let object = reference
.get("object")
.ok_or("Gitea API returned no reference object")?;
if field(object, "type")? != "commit"
|| gitea_heads.insert(id, sha(object, "sha")?).is_some()
{
return Err(
"Gitea API returned an invalid or repeated pull request head".into(),
);
}
}
}
}
for value in values {
let id = number(
value,
if forge.provider == Provider::Gitlab {
"iid"
} else {
"number"
},
)?;
if !seen.insert(id) {
return Err(
"Forge API repeated a pull request while paginating; retry the list".into(),
);
}
let detail;
// ponytail: GitLab needs one detail read per MR; batch if large histories become slow.
let value = if forge.provider == Provider::Gitlab
&& value.get("diff_refs").is_none_or(Value::is_null)
{
detail = forge.get(
&format!("{}/{id}", forge.endpoint),
token.as_deref(),
root,
cancelled,
)?;
if number(&detail, "iid")? != id {
return Err("GitLab API returned a different merge request".into());
}
&detail
} else {
value
};
let preserved_head = if forge.provider == Provider::Gitea {
gitea_heads.remove(&id)
} else {
None
};
requests.push(pull_request(forge.provider, value, preserved_head)?);
}
}
Err("Forge pagination overflow".into())
forge.page_with(closed, page, |url| {
forge.get(url, token.as_deref(), root, cancelled)
})
}
pub(crate) fn resolve(
url: &str,
provider: Option<&str>,
number: u64,
root: &Path,
cancelled: &AtomicBool,
) -> Result<PullRequest, String> {
let forge = Forge::parse(url, provider, |destination, port| {
let mut command = Command::new("ssh");
command.current_dir(root);
ssh_hostname(command, destination, port, cancelled)
})?;
let token = forge.token(root, cancelled)?;
forge.resolve_with(number, |url| {
forge.get(url, token.as_deref(), root, cancelled)
})
}
#[cfg(test)]
@@ -652,6 +822,229 @@ mod tests {
})
}
#[test]
fn pages_are_bounded_newest_first_summaries_without_historical_metadata() {
for (provider, url, query) in [
(
Provider::Github,
"https://github.com/owner/repo",
"state=open&per_page=50&sort=created&direction=desc&page=3",
),
(
Provider::Gitlab,
"https://gitlab.com/owner/repo",
"state=opened&scope=all&per_page=50&order_by=created_at&sort=desc&page=3",
),
(
Provider::Gitea,
"https://codeberg.org/owner/repo",
"state=open&limit=50&page=3",
),
] {
let forge = parse(url, None).unwrap();
let mut calls = 0;
let page = forge.page_with(false, 3, |url| {
calls += 1;
assert_eq!(url, format!("{}?{query}", forge.endpoint));
Ok(json!([
{"number": 1, "iid": 1, "title": "Older", "state": if provider == Provider::Gitlab { "opened" } else { "open" },
"user": null, "author": null, "merged_at": null, "merged": false, "created_at": "2024-01-01T01:00:00+01:00"},
{"number": 2, "iid": 2, "title": "Newer", "state": if provider == Provider::Gitlab { "opened" } else { "open" },
"user": {"login": "author"}, "author": {"username": "author"}, "merged_at": null, "merged": false,
"created_at": "2024-01-01T00:00:00.001Z", "diff_refs": null, "head": {"sha": "not-a-commit"}}
]))
}).unwrap();
assert_eq!(calls, 1);
assert_eq!(
page.entries.iter().map(|p| p.number).collect::<Vec<_>>(),
[2, 1]
);
assert_eq!(page.entries[0].description, "author · open");
assert!(page.entries[1].description.contains("deleted user"));
assert!(
page.has_more,
"A short, server-capped page is not exhaustion"
);
let empty = forge.page_with(false, 4, |_| Ok(json!([]))).unwrap();
assert!(empty.entries.is_empty() && !empty.has_more);
assert!(
forge
.page_with(false, 0, |_| panic!("Page zero must not request"))
.is_err()
);
}
}
#[test]
fn gitlab_closed_combines_closed_and_merged_pages_without_open_requests() {
let forge = parse("https://gitlab.com/group/repo", None).unwrap();
let mut all = Vec::new();
for page in 1..=3 {
let mut states = Vec::new();
let result = forge
.page_with(true, page, |url| {
assert!(url.ends_with(&format!("&page={page}")));
let state = if url.contains("?state=closed&") {
"closed"
} else {
assert!(url.contains("?state=merged&"));
"merged"
};
states.push(state);
let (id, date) = match (state, page) {
("closed", 1) => (1, "2024-01-01T00:00:00Z"),
("merged", 1) => (3, "2024-03-01T00:00:00Z"),
("merged", 2) => (2, "2024-02-01T00:00:00Z"),
_ => return Ok(json!([])),
};
Ok(json!([{"iid": id, "title": state, "state": state,
"author": null, "created_at": date, "diff_refs": null}]))
})
.unwrap();
assert_eq!(states, ["closed", "merged"]);
assert_eq!(result.has_more, page != 3);
assert!(result.entries.iter().all(|p| p.closed));
if page == 1 {
assert_eq!(
result.entries.iter().map(|p| p.number).collect::<Vec<_>>(),
[3, 1]
);
}
// The second merged page still precedes the first closed page.
all.extend(result.entries);
}
all.sort_unstable_by(|a, b| b.created_at.cmp(&a.created_at));
assert_eq!(all.iter().map(|p| p.number).collect::<Vec<_>>(), [3, 2, 1]);
let raced = forge.page_with(true, 1, |url| {
Ok(json!([{"iid": 7, "title": "Raced merge", "state": if url.contains("?state=merged&") { "merged" } else { "closed" },
"author": null, "created_at": "2024-01-01T00:00:00Z"}]))
}).unwrap();
assert_eq!(raced.entries.len(), 1);
assert!(raced.entries[0].description.contains("merged"));
}
#[test]
fn timestamps_normalize_offsets_precision_and_calendar_boundaries() {
for (input, expected) in [
(
"2024-03-01T00:30:12.5+01:00",
"2024-02-29T23:30:12.500000000Z",
),
(
"2023-12-31T23:30:00-01:00",
"2024-01-01T00:30:00.000000000Z",
),
(
"2024-01-01T00:00:00.000000001Z",
"2024-01-01T00:00:00.000000001Z",
),
] {
assert_eq!(created_at(&json!({"created_at": input})).unwrap(), expected);
}
for input in [
"",
"not a date",
"2023-02-29T00:00:00Z",
"2024-13-01T00:00:00Z",
"2024-01-01T24:00:00Z",
"2024-01-01T00:00:00",
"2024-01-01T00:00:00+24:00",
"2024-01-01T00:00:00.Z",
"2024-01-01T00:00:00.1234567890Z",
"0001-01-01T00:00:00+01:00",
"9999-12-31T23:00:00-02:00",
] {
assert!(
created_at(&json!({"created_at": input})).is_err(),
"{input}"
);
}
}
#[test]
fn resolution_reads_only_selected_detail_and_exact_preserved_head() {
for (url, provider) in [
("https://github.com/owner/repo", Provider::Github),
("https://gitlab.com/group/repo", Provider::Gitlab),
("https://codeberg.org/owner/repo", Provider::Gitea),
] {
let forge = parse(url, None).unwrap();
let mut calls = Vec::new();
let pr = forge.resolve_with(7, |url| {
calls.push(url.to_owned());
if url == format!("{}/7", forge.endpoint) {
Ok(json!({"number": 7, "iid": 7, "title": "Selected", "state": if provider == Provider::Gitlab { "merged" } else { "closed" },
"merged_at": "2024-01-01T00:00:00Z", "merged": true, "user": null, "author": null,
"base": {"ref": "main", "sha": "1".repeat(40)}, "head": {"ref": "topic", "sha": "2".repeat(40)},
"merge_base": "3".repeat(40), "target_branch": "main", "source_branch": "topic",
"diff_refs": {"start_sha": null, "base_sha": "4".repeat(40), "head_sha": "5".repeat(40)},
"url": "https://evil.example/detail", "references": "https://evil.example/refs"}))
} else {
assert_eq!(url, format!("{}/git/refs/pull/7/head", forge.endpoint.strip_suffix("/pulls").unwrap()));
Ok(json!([{"ref": "refs/pull/7/head", "object": {"type": "commit", "sha": "6".repeat(40)}}]))
}
}).unwrap();
assert_eq!(calls.len(), if provider == Provider::Gitea { 2 } else { 1 });
let revisions = pr.revisions.unwrap();
assert_eq!(
revisions.base_sha,
match provider {
Provider::Github => "1",
Provider::Gitlab => "4",
Provider::Gitea => "3",
}
.repeat(40)
);
assert_eq!(
revisions.head_sha,
match provider {
Provider::Github => "2",
Provider::Gitlab => "5",
Provider::Gitea => "6",
}
.repeat(40)
);
assert!(
forge
.resolve_with(0, |_| panic!("Zero must not request"))
.is_err()
);
let mut calls = 0;
assert!(
forge
.resolve_with(7, |_| {
calls += 1;
Ok(json!({"number": 8, "iid": 8}))
})
.is_err()
);
assert_eq!(
calls, 1,
"Reject wrong detail identity before reading historical refs"
);
}
}
#[test]
fn gitea_preserved_heads_reject_missing_ambiguous_or_foreign_refs() {
let reference =
json!({"ref": "refs/pull/7/head", "object": {"type": "commit", "sha": "a".repeat(40)}});
assert_eq!(gitea_head(&reference, 7).unwrap(), "a".repeat(40));
assert_eq!(
gitea_head(&json!([reference.clone()]), 7).unwrap(),
"a".repeat(40)
);
for bad in [
json!([]),
json!([reference.clone(), reference.clone()]),
json!({"ref": "refs/pull/8/head", "object": reference["object"]}),
json!({"ref": "refs/pull/7/head", "object": {"type": "tag", "sha": "a".repeat(40)}}),
json!({"ref": "refs/pull/7/head", "object": {"type": "commit", "sha": "0".repeat(40)}}),
] {
assert!(gitea_head(&bad, 7).is_err());
}
}
#[test]
fn ssh_aliases_use_effective_hostname_and_preserve_user_matching() {
use std::{fs, io::Write, path::PathBuf};
@@ -788,9 +1181,7 @@ mod tests {
}
.repeat(40)
);
assert!(pr.closed && pr.description.contains("merged"));
if provider == Provider::Gitea {
assert!(pr.description.contains("fork:topic"));
let mut deleted = value.clone();
deleted["head"]["sha"] = Value::String(String::new());
assert_eq!(
@@ -802,9 +1193,6 @@ mod tests {
"5".repeat(40)
);
let unavailable = pull_request(provider, &deleted, None).unwrap();
assert_eq!(unavailable.number, 7);
assert!(unavailable.closed);
assert!(unavailable.description.contains("unavailable"));
assert!(unavailable.revisions.is_err());
}
}