From dd41a93c9bc89886e176c68f8646966789d5d542 Mon Sep 17 00:00:00 2001 From: funkpower Date: Fri, 13 Nov 2026 16:19:54 -0500 Subject: [PATCH] bibfetch: take an ISBN's language, publisher, year and length from the edition, not the work Open Library's index answers about the work. Its publishers, languages, first year and median page count are every edition's at once, and the first of each is usually another edition's: ISBN 9780140447934, the Penguin Classics War and Peace, came back Spanish, from W.W. Norton, in 1845. The index is now asked only for what is true of the work (title, subtitle, authors, subjects). A record from it is never complete, so lookup always goes on to the edition record for container, language, year and pages, then Google Books for whatever the edition record does not say. A field neither gives is left out rather than filled from the work. Closes #6. --- crates/bibfetch/README.md | 10 ++- crates/bibfetch/src/lib.rs | 3 +- crates/bibfetch/src/source.rs | 152 ++++++++++++++++++++++++++-------- 3 files changed, 129 insertions(+), 36 deletions(-) diff --git a/crates/bibfetch/README.md b/crates/bibfetch/README.md index f643e08..642f4d4 100644 --- a/crates/bibfetch/README.md +++ b/crates/bibfetch/README.md @@ -86,7 +86,7 @@ is a failure — nobody replied, and the identifier may be perfectly good. |---|---|---| | `doi` | Crossref | It assigns most scholarly DOIs | | `arxiv` | the arXiv | Its own register of preprints | -| `isbn` | OpenLibrary's edition record, then its index, then Google Books | Books, without a key | +| `isbn` | OpenLibrary's index, then its edition record, then Google Books | Books, without a key | One register is enough for a DOI or a preprint: the body that assigns the identifier is the only one that has it. A book is different. An ISBN is @@ -97,6 +97,14 @@ names them. So the registers are asked in turn and the first complete answer wins. An incomplete one is kept in case nothing better comes, and a register being down is a failure only when every register is. +OpenLibrary's index answers about the work, not the edition: its publishers, +languages, years and page counts are every edition's at once, and the first +of each is usually another edition's. So for a book it gives only the title, +the authors and the subjects, and the `container`, `language`, `year` and +`pages` of a record come from the edition record for that ISBN, or from +Google Books when the edition record does not say. A field neither gives is +left out rather than filled from the work. + Google Books is asked last because it answers anonymous callers only until a daily quota runs out, after which it replies `429` to everyone sharing the address. diff --git a/crates/bibfetch/src/lib.rs b/crates/bibfetch/src/lib.rs index 516637b..7af67b7 100644 --- a/crates/bibfetch/src/lib.rs +++ b/crates/bibfetch/src/lib.rs @@ -265,11 +265,10 @@ mod merge_tests { let mut index = rec("openlibrary"); index.title = Some("The First World War".into()); index.authors = vec!["Michael Howard".into()]; - index.year = Some(2007); edition.fill_from(index); assert_eq!(edition.authors, ["Michael Howard"]); - assert_eq!(edition.year, Some(2007)); + assert_eq!(edition.language.as_deref(), Some("eng")); // The fuller title arrived first and is not displaced by the shorter. assert_eq!( edition.title.as_deref(), diff --git a/crates/bibfetch/src/source.rs b/crates/bibfetch/src/source.rs index c7b1c96..d4af50f 100644 --- a/crates/bibfetch/src/source.rs +++ b/crates/bibfetch/src/source.rs @@ -17,9 +17,10 @@ pub enum Source { Arxiv, /// OpenLibrary, for books by ISBN. OpenLibrary, - /// OpenLibrary's index, rather than its edition record. Asked when the - /// edition record has the book but not who wrote it, which is often: an - /// edition names its authors by key, and the index names them. + /// OpenLibrary's index, rather than its edition record. Asked for what is + /// true of the work — the title, the authors by name, the subjects — + /// because an edition names its authors by key and the index names them. + /// Never for what differs between editions. OpenLibrarySearch, /// Google Books, for books by ISBN. Asked last, because it answers /// anonymous callers only until a daily quota runs out. @@ -55,14 +56,14 @@ impl Source { match kind { "doi" => &[Source::Crossref], "arxiv" => &[Source::Arxiv], - // The index first. Asked for the fields by name it answers with - // the title, the authors, the publisher, the language, a page - // count and the subjects in one request, which is the whole of - // what a catalogue wants; the edition record is then needed only - // for a subtitle or a publisher the index did not give. Asking it - // second cost three requests per book where one usually does, and - // at three books in flight that is the difference between two - // days and two hours. + // The index first, for what is true of the work: the title, the + // authors by name, the subjects. Then the edition record, for + // what is true of this edition only: the publisher, the language, + // the year and the length. The index answers about the work, and + // its publishers, languages and years are every edition's at + // once, so they are not asked of it; the lookup goes on to the + // edition record for them. Google Books last, for whatever the + // edition record does not say. "isbn" => &[ Source::OpenLibrarySearch, Source::OpenLibrary, @@ -83,15 +84,16 @@ impl Source { Source::OpenLibrary => { format!("https://openlibrary.org/isbn/{}.json", encode(id)) } - // `fields` or the index answers with a handful of defaults and - // keeps the rest, which is how a language it holds went unread. - // Not `cover_i`: that is the cover chosen for the whole work, and - // on a work with hundreds of editions it is usually another - // edition's, in another translation or language. + // Only what is true of the work. Not `publisher`, `language`, + // `first_publish_year` or `number_of_pages_median`: those are every + // edition's at once, and the first of each is usually another + // edition's (the Penguin *War and Peace* came back Spanish, from + // W.W. Norton, in 1845). Not `cover_i`: that is the cover chosen + // for the whole work, often another edition's in another + // translation or language. Source::OpenLibrarySearch => format!( "https://openlibrary.org/search.json?q=isbn:{}\ -&fields=title,subtitle,author_name,first_publish_year,publisher,language,subject,\ -number_of_pages_median&limit=1", +&fields=title,subtitle,author_name,subject&limit=1", encode(id) ), Source::GoogleBooks => format!( @@ -295,6 +297,8 @@ struct SearchReply { docs: Vec, } +/// What the index says about a work. Only the fields true of every edition +/// are read, so an answer that carries the others anyway cannot leak them. #[derive(Deserialize)] struct SearchDoc { #[serde(default)] @@ -304,15 +308,7 @@ struct SearchDoc { #[serde(default)] author_name: Vec, #[serde(default)] - first_publish_year: Option, - #[serde(default)] - publisher: Vec, - #[serde(default)] - language: Vec, - #[serde(default)] subject: Vec, - #[serde(rename = "number_of_pages_median", default)] - pages: Option, } /// How many of a work's subject headings are worth carrying. @@ -324,6 +320,11 @@ pub(crate) const SUBJECT_LIMIT: usize = 12; /// A search that matched nothing answers with no documents rather than a /// status, so an empty reply is the absence and not a failure. +/// +/// The record says what is true of the work and nothing that belongs to one +/// edition: no publisher, language, year or page count. Those come from the +/// edition record, which [`crate::ask::lookup`] asks next because a record +/// without them is never complete. fn open_library_search(id: &str, kind: &str, body: &str) -> Result { let reply: SearchReply = serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?; let Some(doc) = reply.docs.into_iter().next() else { @@ -336,14 +337,10 @@ fn open_library_search(id: &str, kind: &str, body: &str) -> Result title, }; record.authors = doc.author_name; - record.year = doc.first_publish_year; - record.container = doc.publisher.into_iter().next(); - record.language = doc.language.into_iter().next(); // The index lists every subject any edition of the work carries, which // runs to hundreds on a well-known book. A shelf needs a handful. record.subjects = doc.subject; record.subjects.truncate(SUBJECT_LIMIT); - record.pages = doc.pages; Ok(record) } @@ -475,9 +472,8 @@ mod tests { fn each_kind_goes_to_the_body_that_assigns_it() { assert_eq!(Source::for_kind("doi"), Some(Source::Crossref)); assert_eq!(Source::for_kind("arxiv"), Some(Source::Arxiv)); - // The index, not the edition record: it answers with the whole of - // what a catalogue wants in one request, where the edition record - // takes up to twenty-eight seconds and names its authors by key. + // The index first, for the authors by name; the edition record names + // them by key, and is asked next for what only the edition knows. assert_eq!(Source::for_kind("isbn"), Some(Source::OpenLibrarySearch)); // No register answers about an ISSN without a key. assert_eq!(Source::for_kind("issn"), None); @@ -599,6 +595,96 @@ mod tests { assert!(!Source::OpenLibrarySearch.url("1").contains("cover_i")); } + /// What the index said for ISBN 9780140447934, the Penguin Classics *War + /// and Peace*, on 2026-11-13, asked for the fields the lookup asked for + /// before #6: the publishers, languages and first year of every edition + /// of the work. + const WAR_AND_PEACE_INDEX: &str = r#"{"docs":[{"title":"War and Peace", + "author_name":["Leo Tolstoy"],"first_publish_year":1845, + "publisher":["W.W. Norton","IndyPublish.com","Canterbury Classics"], + "language":["spa","gre","dut","eng"],"subject":["Fiction"], + "number_of_pages_median":1296}]}"#; + + /// The edition record for the same ISBN, `/isbn/9780140447934.json`, with + /// the English language its `languages` field should name. + const WAR_AND_PEACE_EDITION: &str = r#"{"title":"War and Peace (Penguin Classics)", + "publishers":["Penguin Books"],"publish_date":"September 27, 2007", + "languages":[{"key":"/languages/eng"}],"number_of_pages":1440}"#; + + /// Index first, then the edition record, merged as `ask::lookup` merges. + fn looked_up(edition: &str) -> Record { + let mut kept = + open_library_search("9780140447934", "isbn", WAR_AND_PEACE_INDEX).expect("a record"); + kept.fill_from(open_library("9780140447934", "isbn", edition).expect("a record")); + kept + } + + #[test] + fn the_index_says_nothing_only_an_edition_can() { + let record = + open_library_search("9780140447934", "isbn", WAR_AND_PEACE_INDEX).expect("a record"); + // What is true of the work is kept. + assert_eq!(record.title.as_deref(), Some("War and Peace")); + assert_eq!(record.authors, ["Leo Tolstoy"]); + // What belongs to one edition is not taken from all of them. + assert_eq!(record.language, None); + assert_eq!(record.container, None); + assert_eq!(record.year, None); + assert_eq!(record.pages, None); + } + + #[test] + fn a_lookup_by_isbn_gives_the_editions_language_publisher_and_year() { + let record = looked_up(WAR_AND_PEACE_EDITION); + assert_eq!(record.language.as_deref(), Some("eng")); + assert_eq!(record.container.as_deref(), Some("Penguin Books")); + assert_eq!(record.year, Some(2007)); + assert_eq!(record.pages, Some(1440)); + assert_eq!(record.authors, ["Leo Tolstoy"]); + } + + #[test] + fn what_the_edition_does_not_say_is_missing_rather_than_the_works() { + // The real edition record today names no language at all. + let record = looked_up( + r#"{"title":"War and Peace (Penguin Classics)","publishers":["Penguin Books"], + "publish_date":"September 27, 2007","number_of_pages":1440}"#, + ); + assert_eq!(record.language, None); + assert_eq!(record.container.as_deref(), Some("Penguin Books")); + assert_eq!(record.year, Some(2007)); + // Nor a date: the work's first year is not this edition's. + let record = looked_up(r#"{"title":"War and Peace (Penguin Classics)"}"#); + assert_eq!(record.year, None); + assert_eq!(record.container, None); + assert_eq!(record.pages, None); + } + + #[test] + fn the_index_alone_never_ends_a_lookup() { + // `lookup` stops at the first complete record; the index must not be + // one, or the edition record is never asked. + let record = + open_library_search("9780140447934", "isbn", WAR_AND_PEACE_INDEX).expect("a record"); + assert!(!record.complete()); + } + + #[test] + fn the_index_is_not_asked_for_what_its_editions_differ_in() { + let url = Source::OpenLibrarySearch.url("9780140447934"); + for field in [ + "publisher", + "language", + "first_publish_year", + "number_of_pages_median", + ] { + assert!( + !url.split(['=', ',', '&']).any(|f| f == field), + "{field} in {url}" + ); + } + } + #[test] fn an_answer_that_is_not_what_it_should_be_says_so() { assert!(matches!(