bibfetch: a cover id is the edition's own, and a fetch has a deadline

Review id 3 on catena#2.

P0: Record.cover_id came from the search index's cover_i, the cover
chosen for the whole work, and the index is asked first, so cover()
fell back to another edition's image (Penguin War and Peace,
9780140447934, got Standard Ebooks' Maude cover 12621906). cover_i is
no longer asked for or read; cover_id is the first positive id in the
edition record's covers, or None. An ISBN request that fails, as
opposed to a 404, now ends the fetch with that failure instead of
going on to the cover id.

Redirects go only over HTTPS on the standard port to
covers.openlibrary.org, archive.org or a host under archive.org, and
never to an address by number; the covers store the fetch began at is
still allowed.

The 1 KB size floor is gone. With default=false a missing cover is a
404, and the stand-in without it is a 43-byte GIF the media type check
refuses, so the floor could only drop real Small covers.

Covers.deadline (DEADLINE, two minutes) bounds a whole fetch: every
address, attempt, redirect, body and wait. It starts once the fetch
holds the process lock. The timeout test no longer times a span that
includes the lock wait.
This commit is contained in:
funkpower committed 2026-11-13 15:04:34 -05:00
1 parent 19e5f3ddc3
commit 1abed4632d
4 files changed
+349 -73

No files matched your search

+41 -16
View File
@@ -85,10 +85,13 @@ impl Source {
}
// `fields` or the index answers with a handful of defaults and
// keeps the rest, which is how a language it holds went unread.
// Not `cover_i`: that is the cover chosen for the whole work, and
// on a work with hundreds of editions it is usually another
// edition's, in another translation or language.
Source::OpenLibrarySearch => format!(
"https://openlibrary.org/search.json?q=isbn:{}\
&fields=title,subtitle,author_name,first_publish_year,publisher,language,subject,\
number_of_pages_median,cover_i&limit=1",
number_of_pages_median&limit=1",
encode(id)
),
Source::GoogleBooks => format!(
@@ -278,6 +281,7 @@ fn open_library(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
.find_map(|l| l.key.rsplit('/').next().map(str::to_string))
.filter(|l| !l.is_empty());
record.pages = book.number_of_pages;
// The edition's own cover, and the only place a cover id is taken from.
record.cover_id = book
.covers
.into_iter()
@@ -309,8 +313,6 @@ struct SearchDoc {
subject: Vec<String>,
#[serde(rename = "number_of_pages_median", default)]
pages: Option<u32>,
#[serde(default)]
cover_i: Option<i64>,
}
/// How many of a work's subject headings are worth carrying.
@@ -342,10 +344,6 @@ fn open_library_search(id: &str, kind: &str, body: &str) -> Result<Record, Error
record.subjects = doc.subject;
record.subjects.truncate(SUBJECT_LIMIT);
record.pages = doc.pages;
record.cover_id = doc
.cover_i
.and_then(|id| u64::try_from(id).ok())
.filter(|id| *id > 0);
Ok(record)
}
@@ -542,18 +540,45 @@ mod tests {
assert_eq!(record.cover_id, Some(1162784));
let record = open_library("1", "isbn", r#"{"title":"A"}"#).expect("a record");
assert_eq!(record.cover_id, None);
let record = open_library_search("1", "isbn", r#"{"docs":[{"title":"A","cover_i":2468}]}"#)
.expect("a record");
assert_eq!(record.cover_id, Some(2468));
let record = open_library_search("1", "isbn", r#"{"docs":[{"title":"A","cover_i":-1}]}"#)
.expect("a record");
assert_eq!(record.cover_id, None);
}
#[test]
fn the_index_is_asked_for_the_cover_id() {
assert!(Source::OpenLibrarySearch.url("1").contains("cover_i"));
fn the_cover_id_is_the_editions_and_not_the_works() {
// ISBN 9780140447934 today: the index (asked first) names the work's
// cover; the edition record has none. Merged as `ask::lookup` merges.
let mut kept = open_library_search(
"9780140447934",
"isbn",
r#"{"docs":[{"title":"War and Peace","cover_i":12621906}]}"#,
)
.expect("a record");
let edition = open_library("9780140447934", "isbn", r#"{"title":"War and Peace"}"#)
.expect("a record");
kept.fill_from(edition);
// Cover 12621906 is OL34941467M, Standard Ebooks' Maude translation.
assert_eq!(kept.cover_id, None);
}
#[test]
fn an_edition_with_no_covers_gives_no_cover_id() {
let index = r#"{"docs":[{"title":"War and Peace","cover_i":12621906}]}"#;
// `lookup` often stops at the index, before the edition is asked.
let alone = open_library_search("9780140447934", "isbn", index).expect("a record");
assert_eq!(alone.cover_id, None);
// An edition whose only cover was deleted, and one with an empty list.
for edition in [
r#"{"title":"War and Peace","covers":[-1]}"#,
r#"{"title":"War and Peace","covers":[]}"#,
] {
let mut kept = open_library_search("9780140447934", "isbn", index).expect("a record");
kept.fill_from(open_library("9780140447934", "isbn", edition).expect("a record"));
assert_eq!(kept.cover_id, None, "{edition}");
}
}
#[test]
fn the_index_is_not_asked_for_the_works_cover() {
assert!(!Source::OpenLibrarySearch.url("1").contains("cover_i"));
}
#[test]