funkpower/catena
1
0 Fork 0
mirror of https://git.pumpin.online/funkpower/catena.git synced 2026-11-13 23:55:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request '0.5.0: review fixes, Python bindings for covers and search, version bump' (#11) from fixes-040 into main

Browse Source
This commit is contained in:
funkpower committed 2026-11-13 16:08:45 -05:00
2 parents 58573399e7 65fd4adaad
commit e9f9e75c1d
20 files changed
+1292 -227
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Cargo.lock
Generated
+26 -26
View File
@@ -101,7 +101,7 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "avplan"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -126,7 +126,7 @@ checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
[[package]]
name = "bibclean"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -143,7 +143,7 @@ dependencies = [
[[package]]
name = "bibfetch"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -196,7 +196,7 @@ dependencies = [
[[package]]
name = "bookmeta"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -210,7 +210,7 @@ dependencies = [
[[package]]
name = "booksniff"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -250,7 +250,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
[[package]]
name = "catena"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"bibclean",
"bibfetch",
@@ -277,7 +277,7 @@ dependencies = [
[[package]]
name = "catena-cli"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"clap",
"clap_mangen",
@@ -312,7 +312,7 @@ checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
[[package]]
name = "chunkmd"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -334,7 +334,7 @@ dependencies = [
[[package]]
name = "citefind"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"chunkmd",
@@ -399,7 +399,7 @@ dependencies = [
[[package]]
name = "colophon"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"bibclean",
"catena-cli",
@@ -448,7 +448,7 @@ dependencies = [
[[package]]
name = "corpus-core"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"schemars",
"serde",
@@ -492,7 +492,7 @@ dependencies = [
[[package]]
name = "cuekit"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"corpus-core",
"regex",
@@ -504,7 +504,7 @@ dependencies = [
[[package]]
name = "cuenorm"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -514,7 +514,7 @@ dependencies = [
[[package]]
name = "cuepack"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -525,7 +525,7 @@ dependencies = [
[[package]]
name = "cueseg"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -612,7 +612,7 @@ dependencies = [
[[package]]
name = "emb"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -876,7 +876,7 @@ dependencies = [
[[package]]
name = "idnorm"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"bibclean",
"catena-cli",
@@ -942,7 +942,7 @@ dependencies = [
[[package]]
name = "langid"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -1022,7 +1022,7 @@ dependencies = [
[[package]]
name = "mdscrub"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -1034,7 +1034,7 @@ dependencies = [
[[package]]
name = "mdsplit"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -1113,7 +1113,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
[[package]]
name = "pdfgutter"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -1126,7 +1126,7 @@ dependencies = [
[[package]]
name = "pdftriage"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -1370,7 +1370,7 @@ checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189"
[[package]]
name = "rrf"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -1517,7 +1517,7 @@ dependencies = [
[[package]]
name = "seriesof"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
@@ -1634,7 +1634,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
[[package]]
name = "textkit"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"regex",
"tiktoken-rs",
@@ -1724,7 +1724,7 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
[[package]]
name = "tocfind"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"catena-cli",
"clap",
Cargo.toml
+27 -27
View File
@@ -3,39 +3,39 @@ resolver = "3"
members = ["crates/*"]
[workspace.package]
version = "0.4.0"
version = "0.5.0"
edition = "2024"
rust-version = "1.85"
license = "MPL-2.0"
repository = "https://git.pumpin.online/funkpower/catena"
[workspace.dependencies]
catena-cli = { path = "crates/catena-cli", version = "0.4.0" }
corpus-core = { path = "crates/corpus-core", version = "0.4.0" }
cuekit = { path = "crates/cuekit", version = "0.4.0" }
textkit = { path = "crates/textkit", version = "0.4.0" }
chunkmd = { path = "crates/chunkmd", version = "0.4.0" }
mdscrub = { path = "crates/mdscrub", version = "0.4.0" }
mdsplit = { path = "crates/mdsplit", version = "0.4.0" }
pdftriage = { path = "crates/pdftriage", version = "0.4.0" }
cuenorm = { path = "crates/cuenorm", version = "0.4.0" }
cueseg = { path = "crates/cueseg", version = "0.4.0" }
rrf = { path = "crates/rrf", version = "0.4.0" }
cuepack = { path = "crates/cuepack", version = "0.4.0" }
bibclean = { path = "crates/bibclean", version = "0.4.0" }
avplan = { path = "crates/avplan", version = "0.4.0" }
pdfgutter = { path = "crates/pdfgutter", version = "0.4.0" }
catena = { path = "crates/bindings", version = "0.4.0" }
seriesof = { path = "crates/seriesof", version = "0.4.0" }
idnorm = { path = "crates/idnorm", version = "0.4.0" }
bibfetch = { path = "crates/bibfetch", version = "0.4.0" }
bookmeta = { path = "crates/bookmeta", version = "0.4.0" }
langid = { path = "crates/langid", version = "0.4.0" }
tocfind = { path = "crates/tocfind", version = "0.4.0" }
booksniff = { path = "crates/booksniff", version = "0.4.0" }
citefind = { path = "crates/citefind", version = "0.4.0" }
colophon = { path = "crates/colophon", version = "0.4.0" }
emb = { path = "crates/emb", version = "0.4.0" }
catena-cli = { path = "crates/catena-cli", version = "0.5.0" }
corpus-core = { path = "crates/corpus-core", version = "0.5.0" }
cuekit = { path = "crates/cuekit", version = "0.5.0" }
textkit = { path = "crates/textkit", version = "0.5.0" }
chunkmd = { path = "crates/chunkmd", version = "0.5.0" }
mdscrub = { path = "crates/mdscrub", version = "0.5.0" }
mdsplit = { path = "crates/mdsplit", version = "0.5.0" }
pdftriage = { path = "crates/pdftriage", version = "0.5.0" }
cuenorm = { path = "crates/cuenorm", version = "0.5.0" }
cueseg = { path = "crates/cueseg", version = "0.5.0" }
rrf = { path = "crates/rrf", version = "0.5.0" }
cuepack = { path = "crates/cuepack", version = "0.5.0" }
bibclean = { path = "crates/bibclean", version = "0.5.0" }
avplan = { path = "crates/avplan", version = "0.5.0" }
pdfgutter = { path = "crates/pdfgutter", version = "0.5.0" }
catena = { path = "crates/bindings", version = "0.5.0" }
seriesof = { path = "crates/seriesof", version = "0.5.0" }
idnorm = { path = "crates/idnorm", version = "0.5.0" }
bibfetch = { path = "crates/bibfetch", version = "0.5.0" }
bookmeta = { path = "crates/bookmeta", version = "0.5.0" }
langid = { path = "crates/langid", version = "0.5.0" }
tocfind = { path = "crates/tocfind", version = "0.5.0" }
booksniff = { path = "crates/booksniff", version = "0.5.0" }
citefind = { path = "crates/citefind", version = "0.5.0" }
colophon = { path = "crates/colophon", version = "0.5.0" }
emb = { path = "crates/emb", version = "0.5.0" }
clap = { version = "4", features = ["derive"] }
clap_mangen = "0.3"
crates/bibfetch/README.md
+45 -17
View File
@@ -136,6 +136,14 @@ redirect included, stops after two minutes. The two minutes start once it is
the fetch's turn, so a fetch queued behind others in the same process can wait
that long again first.
Open Library allows a hundred covers by ISBN every five minutes from one
address, and refuses the rest with `403`. Covers asked by ISBN are kept three
seconds apart across the whole process (`Covers::pace`), which stays under that
limit however long a run goes. A refusal all the same is not mistaken for an
outage: the other form of the ISBN is not asked, the cover id is, because
Open Library does not limit those, and with no cover found that way the fetch
fails with `Error::Limited`, which means ask again later.
| Option | Effect |
|---|---|
| `-o`, `--output <FILE>` | Where to write the image |
@@ -144,7 +152,8 @@ that long again first.
Exit `0` the cover was written, `1` Open Library has none, `65` the argument
was not an ISBN or the answer was not an image, `69` Open Library could not be
reached, `74` the file could not be written.
reached, `74` the file could not be written, `75` Open Library refused for
being asked too often and will answer later.
From Rust, `bibfetch::cover(&agent, isbn, cover_id, size)` returns
`Ok(None)` when there is no cover, and a [`Cover`](src/cover.rs) with the
@@ -154,7 +163,7 @@ bytes, the media type and the address asked when there is.
```
$ bibfetch search --title 'The First World War' --author Howard --limit 2
{"v":1,"id":"9780192853622","kind":"isbn","found":true,"source":"openlibrary","title":"The First World War","authors":["Michael Eliot Howard"],"year":2002,"container":"Oxford University Press","subjects":["World War, 1914-1918"],"language":"eng","pages":154,"cover_id":1123124}
{"v":1,"id":"9780192853622","kind":"isbn","found":true,"source":"openlibrary","title":"The First World War","authors":["Michael Eliot Howard"],"year":2002,"container":"Oxford University Press","subjects":["World War, 1914-1918"],"language":"eng","cover_id":1123124}
{"v":1,"id":"9780199205592","kind":"isbn","found":true,"source":"openlibrary","title":"The First World War: A Very Short Introduction","authors":["Michael Howard"],"year":2007,"container":"Oxford University Press","language":"eng","cover_id":14606475}
```
@@ -163,18 +172,33 @@ title and perhaps a name. This asks which books a register has under those,
and prints one record per candidate: Open Library's index first, and Google
Books only when Open Library has nothing.
Open Library's title search is strict. Given an author as well, it finds
nothing for *The Ladder of Divine Ascent* and four editions of *Ladder of
Divine Ascent*. So when the title as given finds nothing, the title without a
leading "The", "A" or "An" is asked next, and then the title and author as one
free-text query. Each is asked only when the one before found nothing, so a
title that matches costs one request. The looser questions can bring back
books that only share words with the title, which is one more reason the
candidates are for the caller to judge.
Each candidate is one edition. Open Library's index answers with works, and
names the edition of each that best matches the search; what belongs to an
edition — its ISBN, publisher, language, year and cover — is that edition's,
and what belongs to the work — the title, the authors, the subjects and the
length — is the work's. Google Books answers with volumes, which are editions
already.
names the edition of each that best matches the search. The ISBN, publisher,
language, year and cover are that edition's and no other's: a field the
edition does not give is left out rather than filled from the work, whose
publishers, years and ISBNs belong to all its editions at once. Only the
title, the authors and the subjects, which describe the work, are the work's.
Open Library's index gives no length for an edition, so its candidates carry
no `pages`. Google Books answers with volumes, which are editions already.
| Field | When the candidate has an ISBN | When it has none |
|---|---|---|
| `id` | The first ISBN whose check digit agrees, in thirteen digits; one written in thirteen is taken over one written in ten | The register's own key: an Open Library id such as `OL37863121M` for the edition, or `OL267171W` for the work when no edition was named; a Google Books volume id |
| `id` | The first ISBN of the edition whose check digit agrees, in thirteen digits; one written in thirteen is taken over one written in ten | The register's own key: an Open Library id such as `OL37863121M` for the edition, or `OL267171W` for the work when no edition was named; a Google Books volume id |
| `kind` | `isbn` | `olid` for Open Library, `googlebooks` for Google Books |
A work the index names no edition of, or names one it gives neither an ISBN
nor a key for, is a candidate named by the work's key, with its title,
authors and subjects and nothing an edition would say.
An ISBN whose check digit is wrong is passed over rather than corrected, and
Google's shelfmarks, typed `OTHER`, are not read as ISBNs even when their
digits would pass for one. A record of kind `isbn` can be handed straight back
@@ -185,9 +209,8 @@ to `bibfetch` for everything the registers say about that edition; `olid` and
Open Library's record of the edition the record names, by that edition's ISBN
or its key, so `bibfetch cover --cover-id` fetches that edition's cover and no
other. The cover the index picked for the whole work is never asked for,
because it is often another edition's. A work the index names no edition of
gives the first ISBN it lists that reads, which may be any edition's, and no
cover id; Google Books knows nothing of Open Library's covers and gives none.
because it is often another edition's. Google Books knows nothing of Open
Library's covers and gives none.
The candidates are what the register said, in the order it said them. This
makes no judgement about which one is the book in hand, and keeps one that is
@@ -201,10 +224,14 @@ bibfetch search --title 'Anathem' --author Stephenson \
| jq -r 'select(.kind == "isbn") | .id' | head -1
```
Open Library answering that it has nothing is an answer, and it stands when
Google Books cannot be reached; a register being down is a failure only when
both are. Each is asked the same way a lookup asks, retried with the same
waits. Open Library is sent the title and the author as written; Google Books
Open Library having nothing is an answer only once Google Books has been
asked and has nothing either. When Google Books cannot be reached, the search
fails rather than say no register has the title, and when it refuses for its
quota (`403`, or `429` on every attempt) the failure says so and the command
exits `75`. Open Library being down is a failure only when Google Books has
nothing to offer in its place. Each is asked the same way a lookup asks,
retried with the same waits, except that a `403` from Google Books is not
asked again. Open Library is sent the title and the author as written; Google Books
is sent each word as its own `intitle:` or `inauthor:` term, so nothing in a
title is read as search syntax.
@@ -215,8 +242,9 @@ title is read as search syntax.
| `--limit <COUNT>` | The most candidates to print, `1` to `40`. Default `10` |
Exit `0` at least one candidate was printed, `1` no register has the title,
`65` the title was empty, `69` no register could be reached, `74` the output
could not be written.
`65` the title was empty, `69` a register that had to answer could not be
reached, `74` the output could not be written, `75` Google Books refused for
its quota.
From Rust, `bibfetch::search(&agent, title, author, limit, attempts)` returns
the candidates, empty when no register has the title.
crates/bibfetch/src/cover.rs
+126 -8
View File
@@ -46,6 +46,11 @@ const REDIRECT_HOSTS: [&str; 2] = ["covers.openlibrary.org", "archive.org"];
/// agent's timeout each, and a minute's `Retry-After` between attempts.
pub const DEADLINE: Duration = Duration::from_secs(120);
/// How long to leave between two covers asked by ISBN. Open Library allows a
/// hundred every five minutes from one address and refuses the rest with
/// `403`; one every three seconds stays under that however long a run goes.
pub const PACE: Duration = Duration::from_secs(3);
/// The longest a `Retry-After` is honoured for. A register asking for more
/// than this is better treated as down than waited on.
const MAX_WAIT: Duration = Duration::from_secs(60);
@@ -54,7 +59,10 @@ const MAX_WAIT: Duration = Duration::from_secs(60);
/// this, and limits covers asked by ISBN to a hundred every five minutes
/// from one address; threads asking at once would only meet the limit
/// sooner and all back off together.
static ONE_AT_A_TIME: Mutex<()> = Mutex::new(());
///
/// It holds when the last cover was asked for by ISBN, so that [`PACE`] is
/// kept across fetches and threads.
static ONE_AT_A_TIME: Mutex<Option<Instant>> = Mutex::new(None);
/// Which of Open Library's three sizes to ask for.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
@@ -127,6 +135,9 @@ pub struct Covers {
/// request is given whatever is left of it, or the agent's own timeout
/// if that is shorter, and a wait that would run past it is not waited.
pub deadline: Duration,
/// The least time between two covers asked by ISBN; [`PACE`] by default.
/// Covers asked by id are not limited and are not spaced out.
pub pace: Duration,
}
impl Default for Covers {
@@ -135,6 +146,7 @@ impl Default for Covers {
base: OPEN_LIBRARY_COVERS.to_string(),
attempts: crate::ATTEMPTS,
deadline: DEADLINE,
pace: PACE,
}
}
}
@@ -162,6 +174,12 @@ impl Covers {
/// of the ISBN does not stop the other being asked, but it does stop the
/// cover id being asked: the ISBN's own cover may exist, and is the one
/// wanted. The failure is returned when no ISBN form gave a cover.
///
/// Refusal for asking too often is the exception. It is
/// [`Error::Limited`], the other form is not asked, since it would only
/// be refused too, and the cover id is still asked, because Open Library
/// does not limit those. A cover found that way is returned; otherwise the
/// refusal is, so that the caller knows to ask again later.
pub fn fetch(
&self,
agent: &ureq::Agent,
@@ -186,31 +204,54 @@ impl Covers {
.map(|form| format!("{}/b/isbn/{form}-{letter}.jpg?default=false", self.base))
.collect();
let _turn = ONE_AT_A_TIME
let mut last_by_isbn = ONE_AT_A_TIME
.lock()
.unwrap_or_else(|held| held.into_inner());
// From here, not from the call: waiting for the lock is time another
// fetch spent, and counting it would let a busy process starve this
// one of the time its own requests need.
let until = Instant::now() + self.deadline;
// one of the time its own requests need. A deadline too long to add
// to the clock is no deadline at all.
let now = Instant::now();
let until = now
.checked_add(self.deadline)
.unwrap_or_else(|| now + Duration::from_secs(365 * 24 * 60 * 60));
let mut failed = None;
for address in &addresses {
match self.ask(agent, address, until) {
if let Some(ready) = last_by_isbn.and_then(|at| at.checked_add(self.pace)) {
let now = Instant::now();
if ready > now {
if ready >= until {
return Err(Error::Unreachable(format!(
"{address}: past the {:?} deadline, waiting to keep {:?} \
between covers asked by ISBN",
self.deadline, self.pace
)));
}
std::thread::sleep(ready - now);
}
}
let answer = self.ask(agent, address, until);
*last_by_isbn = Some(Instant::now());
match answer {
Ok(Answer::Image(cover)) => return Ok(Some(cover)),
Ok(Answer::Absent) => {}
Err(why @ Error::Limited(_)) => {
failed = Some(why);
break;
}
Err(why) => failed = Some(why),
}
}
if let Some(why) = failed {
if let Some(why) = failed.take_if(|why| !matches!(why, Error::Limited(_))) {
return Err(why);
}
let Some(id) = cover_id else {
return Ok(None);
return failed.map_or(Ok(None), Err);
};
let address = format!("{}/b/id/{id}-{letter}.jpg?default=false", self.base);
match self.ask(agent, &address, until)? {
Answer::Image(cover) => Ok(Some(cover)),
Answer::Absent => Ok(None),
Answer::Absent => failed.map_or(Ok(None), Err),
}
}
@@ -311,6 +352,15 @@ Open Library's covers or the Internet Archive"
// `default=false` makes a missing cover a 404 rather than a
// stand-in image.
404 | 410 => return Ok(Answer::Absent),
// Open Library's answer to more than a hundred covers asked by
// ISBN in five minutes from one address. Asking again at once
// would only be refused again.
403 => {
return Err(Fault::Final(Error::Limited(format!(
"{at}: refused (403); Open Library allows a hundred covers \
by ISBN every five minutes from one address"
))));
}
429 | 503 => {
let wait = header(&response, "retry-after")
.and_then(|value| value.trim().parse::<u64>().ok())
@@ -603,6 +653,7 @@ mod tests {
base: self.base.clone(),
attempts: 3,
deadline: DEADLINE,
pace: Duration::ZERO,
}
}
@@ -1052,6 +1103,73 @@ mod tests {
assert_eq!(stub.asked(), [ISBN13, ISBN10]);
}
#[test]
fn a_refusal_for_asking_too_often_still_asks_the_cover_id() {
let stub = Stub::start(|_, path| match path {
ISBN13 => Reply::status(403),
"/b/id/42-L.jpg?default=false" => Reply::image("image/jpeg", jpeg(4096)),
_ => Reply::status(404),
});
let cover = stub
.covers()
.fetch(&agent(), "9780306406157", Some(42), CoverSize::Large)
.expect("an answer")
.expect("a cover");
assert!(cover.url.ends_with("/b/id/42-L.jpg?default=false"));
// The ten-digit form would only have been refused too.
assert_eq!(stub.asked(), [ISBN13, "/b/id/42-L.jpg?default=false"]);
}
#[test]
fn a_refusal_for_asking_too_often_is_not_an_absence() {
let stub = Stub::start(|_, path| match path {
ISBN13 => Reply::status(403),
_ => Reply::status(404),
});
let found = stub
.covers()
.fetch(&agent(), "9780306406157", None, CoverSize::Large)
.map(|cover| cover.map(|cover| cover.url));
assert!(matches!(found, Err(Error::Limited(_))), "{found:?}");
assert_eq!(stub.asked(), [ISBN13]);
// With a cover id that has no cover either, the refusal still stands:
// the ISBN's own cover may be there once the limit passes.
let found = stub
.covers()
.fetch(&agent(), "9780306406157", Some(42), CoverSize::Large)
.map(|cover| cover.map(|cover| cover.url));
assert!(matches!(found, Err(Error::Limited(_))), "{found:?}");
}
#[test]
fn a_deadline_too_long_to_count_is_no_deadline() {
let stub = Stub::start(|_, _| Reply::image("image/jpeg", jpeg(2048)));
let covers = Covers {
deadline: Duration::MAX,
..stub.covers()
};
let cover = covers
.fetch(&agent(), "9780306406157", None, CoverSize::Large)
.expect("an answer");
assert!(cover.is_some());
}
#[test]
fn covers_asked_by_isbn_are_spaced_out() {
let stub = Stub::start(|_, _| Reply::status(404));
let covers = Covers {
pace: Duration::from_millis(300),
..stub.covers()
};
let started = Instant::now();
let found = covers.fetch(&agent(), "9780306406157", None, CoverSize::Large);
assert!(matches!(found, Ok(None)), "{found:?}");
// Two forms of the ISBN, so at least one pause between them.
assert!(started.elapsed() >= Duration::from_millis(300));
assert_eq!(stub.asked(), [ISBN13, ISBN10]);
}
#[test]
fn a_redirect_to_an_address_by_number_is_refused() {
// A listener standing in for a service on the LAN; it only counts.
crates/bibfetch/src/lib.rs
+4
View File
@@ -112,6 +112,9 @@ pub enum Error {
NoRegister(String),
/// What was asked about is not an identifier of the kind it should be.
Invalid(String),
/// The register refused because it has been asked too often from this
/// address. It is not down, and asking again later will work.
Limited(String),
}
impl std::fmt::Display for Error {
@@ -123,6 +126,7 @@ impl std::fmt::Display for Error {
write!(f, "no register here answers about {kind} identifiers")
}
Error::Invalid(why) => write!(f, "{why}"),
Error::Limited(why) => write!(f, "{why}"),
}
}
}
crates/bibfetch/src/main.rs
+2
View File
@@ -225,6 +225,7 @@ fn cover(
eprintln!("bibfetch: {isbn}: {why}");
ExitCode::from(match why {
bibfetch::Error::Unreachable(_) => exit::UNAVAILABLE,
bibfetch::Error::Limited(_) => exit::TEMPFAIL,
_ => exit::DATA,
})
}
@@ -255,6 +256,7 @@ fn search(
eprintln!("bibfetch: {why}");
ExitCode::from(match why {
bibfetch::Error::Invalid(_) => exit::DATA,
bibfetch::Error::Limited(_) => exit::TEMPFAIL,
_ => exit::UNAVAILABLE,
})
}
crates/bibfetch/src/search.rs
+328 -104
View File
@@ -23,22 +23,34 @@ pub const MAX_LIMIT: usize = 40;
/// What Open Library's index is asked to say about each work, and about the
/// one edition of it that best matches the search.
///
/// Not the work's own `cover_i` or `cover_edition_key`: the cover the index
/// chose for a work is often another edition's. The edition's `cover_i` is
/// the first cover in that edition's record, which is what a lookup by ISBN
/// gives as `cover_id`.
const OPEN_LIBRARY_FIELDS: &str = "key,title,subtitle,author_name,first_publish_year,publisher,\
isbn,language,subject,number_of_pages_median,editions,editions.key,editions.isbn,\
editions.publisher,editions.language,editions.publish_date,editions.cover_i";
/// Of the work, only what describes the work: its publishers, ISBNs,
/// languages, years and page counts are every edition's at once, and a
/// record made from them would describe no edition. Not the work's own
/// `cover_i` or `cover_edition_key` either: the cover the index chose for a
/// work is often another edition's. The edition's `cover_i` is the first
/// cover in that edition's record, which is what a lookup by ISBN gives as
/// `cover_id`.
const OPEN_LIBRARY_FIELDS: &str = "key,title,subtitle,author_name,subject,editions,editions.key,\
editions.isbn,editions.publisher,editions.language,editions.publish_date,editions.cover_i";
/// Where Google Books is asked.
const GOOGLE_BOOKS: &str = "https://www.googleapis.com/books/v1/volumes";
/// Ask which books a register has under a title, and by an author if one is
/// named.
///
/// Open Library's index first, and Google Books only when that answered with
/// nothing. A register being down is a failure only when every register
/// asked is: Open Library answering that it has nothing is an answer, and it
/// stands when Google Books cannot be reached. Asks nothing when `limit` is
/// zero; a `limit` over [`MAX_LIMIT`] is taken as that.
/// nothing. Open Library's title search is strict: given an author too, it
/// finds nothing for "The Ladder of Divine Ascent" that it finds for "Ladder
/// of Divine Ascent". So when the title as given finds nothing, the title
/// without a leading article is asked, and then the title and author as one
/// free-text query, before Google Books is. Each later question is asked only
/// when the one before it found nothing, and a looser one can return books
/// that only share words with the title, which the caller judges. Once Google Books has been asked, its failure is the answer:
/// [`Error::Limited`] when it refused for its quota, [`Error::Unreachable`]
/// when it could not be reached, since saying no register has the title
/// would be a claim no register made. Asks nothing when `limit` is zero; a
/// `limit` over [`MAX_LIMIT`] is taken as that.
pub fn search(
agent: &ureq::Agent,
title: &str,
@@ -46,9 +58,7 @@ pub fn search(
limit: usize,
attempts: u32,
) -> Result<Vec<Record>, Error> {
search_with(title, author, limit, |url| {
crate::ask::get(agent, url, attempts)
})
search_with(title, author, limit, |url| get(agent, url, attempts))
}
/// [`search`], with how to fetch an address passed in.
@@ -75,25 +85,104 @@ fn search_with(
None => Err(Error::Unreachable(format!("{url}: answered 404"))),
};
let open_library =
fetch(open_library_url(title, author, limit)).and_then(|body| open_library(&body, limit));
let mut from_open_library = Ok(Vec::new());
for url in open_library_urls(title, author, limit) {
from_open_library = fetch(url).and_then(|body| open_library(&body, limit));
if !from_open_library.as_ref().is_ok_and(Vec::is_empty) {
break;
}
}
let open_library = from_open_library;
if open_library.as_ref().is_ok_and(|found| !found.is_empty()) {
return open_library;
}
let google = match google_url(title, author, limit) {
Some(url) => fetch(url).and_then(|body| google_books(&body, limit)),
let Some(url) = google_url(title, author, limit) else {
// Nothing in the title Google could be asked by.
None => Ok(Vec::new()),
return open_library;
};
match (open_library, google) {
(_, Ok(found)) if !found.is_empty() => Ok(found),
(Ok(none), _) => Ok(none),
(Err(why), _) => Err(why),
let google = fetch(url).and_then(|body| google_books(&body, limit))?;
if google.is_empty() {
// Open Library's nothing, or its failure, which Google Books having
// nothing either does not excuse.
return open_library;
}
Ok(google)
}
/// Where to ask Open Library.
/// The body at an address, asked as a lookup asks, or `None` when the
/// register says there is no such thing.
///
/// Except that Google Books refusing for its quota is [`Error::Limited`]: a
/// `403` at once, since it is the day's quota and asking again within
/// seconds will not move it, and a `429` once every attempt has been refused.
fn get(agent: &ureq::Agent, url: &str, attempts: u32) -> Result<Option<String>, Error> {
let attempts = attempts.max(1);
let mut last = Error::Unreachable("no attempt was made".into());
for attempt in 0..attempts {
match agent.get(url).call() {
Ok(mut response) => {
return response
.body_mut()
.read_to_string()
.map(Some)
.map_err(|e| Error::Unreachable(e.to_string()));
}
Err(ureq::Error::StatusCode(404)) => return Ok(None),
Err(ureq::Error::StatusCode(status)) => match refused(url, status) {
Some(why) if status == 403 => return Err(why),
Some(why) => last = why,
None => last = Error::Unreachable(format!("{url}: answered {status}")),
},
Err(e) => last = Error::Unreachable(e.to_string()),
}
if attempt + 1 < attempts {
std::thread::sleep(std::time::Duration::from_secs(1 << attempt.min(5)));
}
}
Err(last)
}
/// The refusal a status from a register is, when it is one for asking too
/// often.
fn refused(url: &str, status: u16) -> Option<Error> {
(url.starts_with(GOOGLE_BOOKS) && matches!(status, 403 | 429)).then(|| {
Error::Limited(format!(
"Google Books refused ({status}): its quota for this address is spent for now"
))
})
}
/// Every question to put to Open Library, strictest first.
fn open_library_urls(title: &str, author: Option<&str>, limit: usize) -> Vec<String> {
let mut urls = vec![open_library_url(title, author, limit)];
if let Some(rest) = without_article(title) {
urls.push(open_library_url(rest, author, limit));
}
let words = match author {
Some(author) => format!("{title} {author}"),
None => title.to_string(),
};
urls.push(format!(
"https://openlibrary.org/search.json?q={}&fields={OPEN_LIBRARY_FIELDS}&limit={limit}",
source::encode(&words),
));
urls
}
/// A title without its leading English article, when it has one and
/// something follows it.
fn without_article(title: &str) -> Option<&str> {
let (first, rest) = title.split_once(char::is_whitespace)?;
let rest = rest.trim_start();
(["the", "a", "an"]
.iter()
.any(|article| first.eq_ignore_ascii_case(article))
&& !rest.is_empty())
.then_some(rest)
}
/// Where to ask Open Library by title and author.
fn open_library_url(title: &str, author: Option<&str>, limit: usize) -> String {
let author = author
.map(|author| format!("&author={}", source::encode(author)))
@@ -124,7 +213,7 @@ fn google_url(title: &str, author: Option<&str>, limit: usize) -> Option<String>
return None;
}
Some(format!(
"https://www.googleapis.com/books/v1/volumes?q={}&maxResults={limit}",
"{GOOGLE_BOOKS}?q={}&maxResults={limit}",
source::encode(&terms.join(" "))
))
}
@@ -162,18 +251,8 @@ struct Work {
#[serde(default)]
author_name: Vec<String>,
#[serde(default)]
first_publish_year: Option<i64>,
#[serde(default)]
publisher: Vec<String>,
#[serde(default)]
isbn: Vec<String>,
#[serde(default)]
language: Vec<String>,
#[serde(default)]
subject: Vec<String>,
#[serde(default)]
number_of_pages_median: Option<u32>,
#[serde(default)]
editions: Option<Editions>,
}
@@ -212,10 +291,11 @@ fn olid(key: &str) -> Option<String> {
/// Read Open Library's answer into one candidate per work, in its order.
///
/// What belongs to an edition — its ISBN, publisher, language, year and
/// cover — is taken from the edition the index chose as matching best; what
/// belongs to the work — its title, authors, subjects and length — from the
/// work. A work the index names no edition of gives its first ISBN that reads,
/// which may be any edition's, and no cover id.
/// cover — is taken from the edition the index chose as matching best, and
/// left out when that edition does not give it; only the title, authors and
/// subjects, which describe the work, are the work's. A work the index names
/// no edition of is named by the work's key and says nothing an edition
/// would.
fn open_library(body: &str, limit: usize) -> Result<Vec<Record>, Error> {
let reply: OpenLibraryReply =
serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?;
@@ -228,26 +308,47 @@ fn open_library(body: &str, limit: usize) -> Result<Vec<Record>, Error> {
}
fn candidate(work: Work) -> Option<Record> {
let edition = work.editions.and_then(|e| e.docs.into_iter().next());
// The edition, and what it is named by: its ISBN, or its key when it has
// none. One with neither cannot be named, and is as good as none.
let edition = work
.editions
.and_then(|e| e.docs.into_iter().next())
.and_then(|edition| {
match first_isbn(edition.isbn.iter().map(String::as_str)) {
Some(isbn) => Some((isbn, "isbn")),
None => edition
.key
.as_deref()
.and_then(olid)
.map(|key| (key, "olid")),
}
.map(|named| (named, edition))
});
// The cover is kept only when the record names the edition it is the
// cover of, by that edition's ISBN or its key.
let (id, kind, edition_named) = match &edition {
Some(edition) => match first_isbn(edition.isbn.iter().map(String::as_str)) {
Some(isbn) => (isbn, "isbn", true),
None => match edition.key.as_deref().and_then(olid) {
Some(key) => (key, "olid", true),
None => (work.key.as_deref().and_then(olid)?, "olid", false),
},
},
None => match first_isbn(work.isbn.iter().map(String::as_str)) {
Some(isbn) => (isbn, "isbn", false),
let mut record = match edition {
Some(((id, kind), edition)) => {
let mut record = source::blank(&id, kind, Source::OpenLibrarySearch);
record.container = edition.publisher.into_iter().next();
record.language = edition.language.into_iter().next();
record.year = edition
.publish_date
.iter()
.find_map(|date| source::year_in(date));
record.cover_id = edition
.cover_i
.and_then(|id| u64::try_from(id).ok())
.filter(|id| *id > 0);
record
}
// The work's own ISBNs, publishers and years are its editions', and
// which belongs with which the index does not say.
None => source::blank(
// A work with no key cannot be named, so it is not a candidate.
None => (work.key.as_deref().and_then(olid)?, "olid", false),
},
&work.key.as_deref().and_then(olid)?,
"olid",
Source::OpenLibrarySearch,
),
};
let mut record = source::blank(&id, kind, Source::OpenLibrarySearch);
record.title = match (work.title, work.subtitle) {
(Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")),
(title, _) => title,
@@ -255,30 +356,6 @@ fn candidate(work: Work) -> Option<Record> {
record.authors = work.author_name;
record.subjects = work.subject;
record.subjects.truncate(SUBJECT_LIMIT);
record.pages = work.number_of_pages_median;
let (publisher, language, year) = match edition {
Some(edition) => {
if edition_named {
record.cover_id = edition
.cover_i
.and_then(|id| u64::try_from(id).ok())
.filter(|id| *id > 0);
}
(
edition.publisher.into_iter().next(),
edition.language.into_iter().next(),
edition
.publish_date
.iter()
.find_map(|date| source::year_in(date)),
)
}
None => (None, None, None),
};
record.container = publisher.or_else(|| work.publisher.into_iter().next());
record.language = language.or_else(|| work.language.into_iter().next());
record.year = year.or(work.first_publish_year);
Some(record)
}
@@ -312,8 +389,9 @@ mod tests {
use super::*;
/// Open Library's answer to `title=the first world war&author=howard`,
/// recorded 2026-11-13 with the fields above and `limit=2`, the ISBN and
/// subject lists shortened.
/// recorded 2026-11-13 with `limit=2`, the ISBN and subject lists
/// shortened, and with the work's own publishers, ISBNs, years and length,
/// which are not asked for and are read past when they come.
const HOWARD: &str = r#"{"numFound": 9, "start": 0, "numFoundExact": true, "num_found": 9,
"documentation_url": "https://openlibrary.org/dev/docs/api/search", "q": "", "offset": null,
"docs": [
@@ -420,7 +498,8 @@ mod tests {
assert_eq!(first.container.as_deref(), Some("Oxford University Press"));
assert_eq!(first.language.as_deref(), Some("eng"));
assert_eq!(first.subjects, ["World War, 1914-1918"]);
assert_eq!(first.pages, Some(154));
// The work's median length is no edition's.
assert_eq!(first.pages, None);
assert_eq!(first.cover_id, Some(1123124));
let second = &found[1];
@@ -474,17 +553,64 @@ mod tests {
}
#[test]
fn a_work_with_no_edition_named_gives_its_isbn_but_no_cover() {
let body = r#"{"docs": [{"key": "/works/OL1W", "title": "A Book",
"isbn": ["0-306-40615-2"], "publisher": ["Plenum"], "first_publish_year": 1981}]}"#;
fn what_the_edition_does_not_give_is_left_out_not_taken_from_the_work() {
let body = r#"{"docs": [{"key": "/works/OL267171W", "title": "War and Peace",
"author_name": ["Leo Tolstoy"], "subject": ["Napoleonic Wars"],
"first_publish_year": 1867, "publisher": ["Crowell"], "language": ["eng"],
"number_of_pages_median": 1296,
"editions": {"docs": [{"key": "/books/OL1M", "isbn": ["9780199232765"],
"cover_i": 5}]}}]}"#;
let found = open_library(body, 5).expect("candidates");
let only = &found[0];
assert_eq!(
(found[0].id.as_str(), found[0].kind.as_str()),
("9780306406157", "isbn")
(only.id.as_str(), only.kind.as_str()),
("9780199232765", "isbn")
);
assert_eq!(found[0].container.as_deref(), Some("Plenum"));
assert_eq!(found[0].year, Some(1981));
assert_eq!(found[0].cover_id, None);
assert_eq!(only.cover_id, Some(5));
// 1867 and Crowell are other editions'.
assert_eq!(only.year, None);
assert_eq!(only.container, None);
assert_eq!(only.language, None);
assert_eq!(only.pages, None);
// What describes the work is the work's.
assert_eq!(only.title.as_deref(), Some("War and Peace"));
assert_eq!(only.authors, ["Leo Tolstoy"]);
assert_eq!(only.subjects, ["Napoleonic Wars"]);
}
#[test]
fn a_work_with_no_edition_named_is_named_by_its_key_and_says_nothing_an_edition_would() {
let work = r#""key": "/works/OL1W", "title": "A Book", "author_name": ["A. Writer"],
"isbn": ["0-306-40615-2"], "publisher": ["Plenum"], "first_publish_year": 1981,
"language": ["eng"], "number_of_pages_median": 300"#;
// No edition, and an edition with neither an ISBN nor a key.
for editions in [
"",
r#", "editions": {"docs": [{"publisher": ["Plenum"], "cover_i": 7}]}"#,
] {
let body = format!(r#"{{"docs": [{{{work}{editions}}}]}}"#);
let found = open_library(&body, 5).expect("candidates");
let only = &found[0];
// Not 9780306406157: which edition that is, the index does not say.
assert_eq!(
(only.id.as_str(), only.kind.as_str()),
("OL1W", "olid"),
"{body}"
);
assert_eq!(only.title.as_deref(), Some("A Book"));
assert_eq!(only.authors, ["A. Writer"]);
assert_eq!(
(
only.year,
only.container.as_deref(),
only.language.as_deref(),
only.pages,
only.cover_id
),
(None, None, None, None, None),
"{body}"
);
}
}
#[test]
@@ -545,8 +671,19 @@ mod tests {
recorded(&mut asked, &bodies),
)
.expect("candidates");
assert_eq!(asked.len(), 2);
assert!(asked[1].starts_with("https://www.googleapis.com/books/v1/volumes?q="));
// The title as given, without its article, as free text, then Google.
assert_eq!(asked.len(), 4, "{asked:?}");
assert!(
asked[1].contains("?title=Google%20Story&author=Vise&"),
"{}",
asked[1]
);
assert!(
asked[2].contains("?q=The%20Google%20Story%20Vise&"),
"{}",
asked[2]
);
assert!(asked[3].starts_with("https://www.googleapis.com/books/v1/volumes?q="));
assert_eq!(found.len(), 2);
let first = &found[0];
@@ -593,15 +730,46 @@ mod tests {
}
#[test]
fn open_library_having_nothing_stands_when_google_books_is_down() {
fn google_books_down_after_open_library_had_nothing_is_a_failure() {
let mut asked = Vec::new();
let bodies = [
("openlibrary.org", Ok(NOTHING)),
("googleapis.com", Err(down())),
];
let found =
search_with("Nothing", None, 5, recorded(&mut asked, &bodies)).expect("an answer");
assert!(found.is_empty());
let failed = search_with("Nothing", None, 5, recorded(&mut asked, &bodies));
assert!(matches!(failed, Err(Error::Unreachable(_))), "{failed:?}");
// By title, as free text, then Google.
assert_eq!(asked.len(), 3, "{asked:?}");
}
#[test]
fn google_books_refusing_for_its_quota_is_said_to_be_that() {
let mut asked = Vec::new();
let limited = refused(&google_url("Nothing", None, 5).expect("a url"), 429);
let bodies = [
("openlibrary.org", Ok(NOTHING)),
("googleapis.com", Err(limited.expect("a refusal"))),
];
let failed = search_with("Nothing", None, 5, recorded(&mut asked, &bodies));
assert!(matches!(failed, Err(Error::Limited(_))), "{failed:?}");
}
#[test]
fn only_google_books_quota_statuses_are_refusals() {
let google = google_url("Anathem", None, 5).expect("a url");
for status in [403, 429] {
assert!(
matches!(refused(&google, status), Some(Error::Limited(_))),
"{status}"
);
}
for status in [400, 500, 503] {
assert!(refused(&google, status).is_none(), "{status}");
}
let open_library = open_library_url("Anathem", None, 5);
for status in [403, 429] {
assert!(refused(&open_library, status).is_none(), "{status}");
}
}
#[test]
@@ -650,8 +818,50 @@ mod tests {
];
search_with("Anathem", Some(" "), 5, recorded(&mut asked, &bodies)).expect("ok");
assert!(!asked[0].contains("author="), "{}", asked[0]);
assert!(!asked[1].contains("inauthor"), "{}", asked[1]);
assert!(asked[1].contains("q=intitle%3AAnathem&"), "{}", asked[1]);
assert!(asked[1].contains("?q=Anathem&"), "{}", asked[1]);
assert!(!asked[2].contains("inauthor"), "{}", asked[2]);
assert!(asked[2].contains("q=intitle%3AAnathem&"), "{}", asked[2]);
}
#[test]
fn a_leading_article_is_dropped_when_the_title_as_given_finds_nothing() {
let mut asked = Vec::new();
let mut answered = 0;
let found = search_with(
"The Ladder of Divine Ascent",
Some("John Climacus"),
5,
|url: &str| {
asked.push(url.to_string());
answered += 1;
Ok(Some(
if answered == 2 { TOLSTOY } else { NOTHING }.to_string(),
))
},
)
.expect("candidates");
assert!(!found.is_empty());
assert_eq!(
asked.len(),
2,
"nothing is asked after a question finds something"
);
assert!(
asked[1].contains("?title=Ladder%20of%20Divine%20Ascent&author=John%20Climacus&"),
"{}",
asked[1]
);
}
#[test]
fn only_a_leading_article_with_words_after_it_is_dropped() {
assert_eq!(without_article("The Republic"), Some("Republic"));
assert_eq!(without_article("an Essay"), Some("Essay"));
assert_eq!(without_article("A"), None);
assert_eq!(without_article("The "), None);
assert_eq!(without_article("Theology"), None);
assert_eq!(without_article("Anathem"), None);
assert_eq!(open_library_urls("Anathem", None, 5).len(), 2);
}
#[test]
@@ -723,7 +933,9 @@ intitle:y intitle:z inauthor:O'Brien inauthor:Sons";
let found = search_with("?!", Some("Someone"), 5, recorded(&mut asked, &bodies))
.expect("an answer");
assert!(found.is_empty());
assert_eq!(asked.len(), 1);
// By title and as free text; never Google.
assert_eq!(asked.len(), 2, "{asked:?}");
assert!(asked.iter().all(|url| url.contains("openlibrary.org")));
}
#[test]
@@ -744,13 +956,25 @@ intitle:y intitle:z inauthor:O'Brien inauthor:Sons";
let found =
search_with("The Google Story", None, 1, recorded(&mut asked, &bodies)).expect("ok");
assert_eq!(found.len(), 1);
assert!(asked[0].ends_with("&limit=1"), "{}", asked[0]);
assert!(asked[1].ends_with("&maxResults=1"), "{}", asked[1]);
let google = asked.last().expect("Google was asked");
assert!(
asked[..asked.len() - 1]
.iter()
.all(|url| url.ends_with("&limit=1")),
"{asked:?}"
);
assert!(google.ends_with("&maxResults=1"), "{google}");
let mut asked = Vec::new();
search_with("The Google Story", None, 500, recorded(&mut asked, &bodies)).expect("ok");
assert!(asked[0].ends_with("&limit=40"), "{}", asked[0]);
assert!(asked[1].ends_with("&maxResults=40"), "{}", asked[1]);
let google = asked.last().expect("Google was asked");
assert!(
asked[..asked.len() - 1]
.iter()
.all(|url| url.ends_with("&limit=40")),
"{asked:?}"
);
assert!(google.ends_with("&maxResults=40"), "{google}");
let mut asked = Vec::new();
let found =
crates/bindings/python/catena/__init__.py
+6
View File
@@ -21,6 +21,8 @@ from ._catena import (
compose_subjects,
count_tokens,
cue_windows,
Limited,
fetch_cover,
fetch_identifier,
find_citations,
fuse_hits,
@@ -41,6 +43,7 @@ from ._catena import (
record_severity,
same_work,
scrub,
search_title,
segment_cues,
series_of,
shelve,
@@ -53,6 +56,7 @@ from ._catena import (
__all__ = [
"__version__",
"Limited",
"canonical_subject",
"chunk_markdown",
"citation_prompt",
@@ -62,6 +66,7 @@ __all__ = [
"compose_subjects",
"count_tokens",
"cue_windows",
"fetch_cover",
"fetch_identifier",
"find_citations",
"fuse_hits",
@@ -82,6 +87,7 @@ __all__ = [
"record_severity",
"same_work",
"scrub",
"search_title",
"segment_cues",
"series_of",
"shelve",
crates/bindings/python/catena/__init__.pyi
+34
View File
@@ -103,6 +103,40 @@ def fetch_identifier(
"""
...
class Limited(Exception):
"""A register refused because it has been asked too often; ask again later."""
def fetch_cover(
isbn: str,
cover_id: int | None = ...,
size: str = ...,
polite: str | None = ...,
timeout: int = ...,
deadline: int = ...,
) -> dict[str, Any] | None:
"""The cover Open Library has for a book, by ISBN, then by cover id.
A dict with `bytes`, `media_type`, `extension`, `source` and `url`, or
`None` when there is none. Raises `Limited` when Open Library refused for
being asked too often. Blocks on the network.
"""
...
def search_title(
title: str,
author: str | None = ...,
limit: int = ...,
polite: str | None = ...,
timeout: int = ...,
attempts: int = ...,
) -> list[dict[str, Any]]:
"""The editions the registers list for a title and author, in their order.
Records shaped like `fetch_identifier`'s. Use `same_work` to decide which
one is the book. Blocks on the network.
"""
...
def colophon_prompt(
record: dict[str, Any], text: str, pages: int = ...
) -> str | None:
crates/bindings/src/python.rs
+104 -2
View File
@@ -10,7 +10,7 @@
use pyo3::exceptions::PyValueError;
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyList};
use pyo3::types::{PyBytes, PyDict, PyList};
use crate::{ChunkBudgets, CueBudgets};
@@ -434,6 +434,9 @@ fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> {
module.add_function(wrap_pyfunction!(read_colophon, module)?)?;
module.add_function(wrap_pyfunction!(shelve, module)?)?;
module.add_function(wrap_pyfunction!(fetch_identifier, module)?)?;
module.add_function(wrap_pyfunction!(fetch_cover, module)?)?;
module.add_function(wrap_pyfunction!(search_title, module)?)?;
module.add("Limited", module.py().get_type::<Limited>())?;
module.add_function(wrap_pyfunction!(colophon_prompt, module)?)?;
module.add_function(wrap_pyfunction!(merge_colophon, module)?)?;
Ok(())
@@ -512,13 +515,112 @@ fn fetch_identifier<'py>(
// would stop every other thread in the interpreter for the whole request.
let found = py
.detach(|| bibfetch::ask::lookup(&agent, id, kind, ask.attempts))
.map_err(|e| PyValueError::new_err(e.to_string()))?;
.map_err(register_error)?;
match found {
Some(record) => Ok(Some(to_py(py, &record)?)),
None => Ok(None),
}
}
pyo3::create_exception!(
_catena,
Limited,
pyo3::exceptions::PyException,
"A register refused because it has been asked too often; ask again later."
);
/// A register's error as the Python exception that says what to do about it.
fn register_error(error: bibfetch::Error) -> PyErr {
match error {
bibfetch::Error::Limited(why) => Limited::new_err(why),
other => PyValueError::new_err(other.to_string()),
}
}
/// The cover Open Library has for a book, by its ISBN, then by the cover id
/// a record named.
///
/// Returns a dict with `bytes`, `media_type` (`image/jpeg` or `image/png`),
/// `extension`, `source` and `url`, or `None` when there is no cover. Covers
/// asked by ISBN are kept three seconds apart across the process, and the
/// whole fetch stops after `deadline` seconds. Raises `catena.Limited` when
/// Open Library refused for being asked too often, and `ValueError` when the
/// ISBN is not one or the store could not be reached. Blocks on the network.
#[pyfunction]
#[pyo3(signature = (isbn, cover_id = None, size = "large", polite = None, timeout = 20, deadline = 120))]
fn fetch_cover<'py>(
py: Python<'py>,
isbn: &str,
cover_id: Option<u64>,
size: &str,
polite: Option<String>,
timeout: u64,
deadline: u64,
) -> PyResult<Option<Bound<'py, PyDict>>> {
let size = match size {
"small" => bibfetch::CoverSize::Small,
"medium" => bibfetch::CoverSize::Medium,
"large" => bibfetch::CoverSize::Large,
other => {
return Err(PyValueError::new_err(format!(
"size is small, medium or large, not {other:?}"
)));
}
};
let agent = bibfetch::ask::Ask {
polite,
timeout: std::time::Duration::from_secs(timeout.max(1)),
..Default::default()
}
.agent();
let covers = bibfetch::cover::Covers {
deadline: std::time::Duration::from_secs(deadline.max(1)),
..Default::default()
};
let found = py
.detach(|| covers.fetch(&agent, isbn, cover_id, size))
.map_err(register_error)?;
let Some(cover) = found else {
return Ok(None);
};
let out = PyDict::new(py);
out.set_item("bytes", PyBytes::new(py, &cover.bytes))?;
out.set_item("media_type", cover.media_type)?;
out.set_item("extension", cover.extension())?;
out.set_item("source", cover.source)?;
out.set_item("url", cover.url)?;
Ok(Some(out))
}
/// The editions the registers list for a title, and an author if one is known.
///
/// Open Library is asked first and Google Books only when it has nothing.
/// The candidates come back in the register's own order, as records shaped
/// like `fetch_identifier`'s; nothing here decides which is the book, which is
/// what `same_work` is for. Blocks on the network.
#[pyfunction]
#[pyo3(signature = (title, author = None, limit = 10, polite = None, timeout = 20, attempts = 3))]
fn search_title<'py>(
py: Python<'py>,
title: &str,
author: Option<&str>,
limit: usize,
polite: Option<String>,
timeout: u64,
attempts: u32,
) -> PyResult<Bound<'py, PyList>> {
let ask = bibfetch::ask::Ask {
polite,
timeout: std::time::Duration::from_secs(timeout.max(1)),
attempts: attempts.max(1),
};
let agent = ask.agent();
let found = py
.detach(|| bibfetch::search(&agent, title, author, limit, ask.attempts))
.map_err(register_error)?;
list_to_py(py, &found)
}
/// The question to put to a model about the fields a book's pages did not
/// settle, or `None` when they settled everything.
///
crates/bookmeta/README.md
+30 -8
View File
@@ -99,8 +99,10 @@ A Kindle book's title is EXTH 503 where it is there, then the MOBI header's
full name, then the Palm database name (cut to 31 bytes, with a note). Authors
(100) and subjects (105) are every record of their kind, in order; publisher
(101), description (103), date (106) and language (524) the first. EXTH 104 is
an `isbn` when it is shaped like one and `other` when not, and 113 is an
`asin`. A contributor (108) is not an author, the same as an EPUB's
an `isbn` when it is shaped like one and `other` when not. 113 is filed by
its shape: a `uuid` when it is one (calibre writes its book id there), an
`asin` when it is ten letters and digits, and `other` when it is neither. A
contributor (108) is not an author, the same as an EPUB's
`dc:contributor`, and is not reported. With no EXTH language, the language
comes from the header's locale. Strings are decoded as the header says,
UTF-8 or Windows-1252. A book whose text is encrypted still has its header
@@ -117,7 +119,10 @@ read off its first bytes: JPEG, PNG, GIF, WebP or BMP. It is for library
callers and is not part of the JSON record, which is a line of text.
A Kindle book's cover is EXTH 201, or the thumbnail, 202, when no cover is
named, and with neither the first image record.
named. With neither, it is the record at the header's first image index, as
calibre does, and only that record: when it is not a JPEG, PNG or GIF there is
no cover, because the records after it can be illustrations, and kindlegen
sometimes puts a font there first.
An EPUB's is the first of these that reaches an image:
@@ -125,10 +130,18 @@ An EPUB's is the first of these that reaches an image:
2. The item `<meta name="cover" content="…">` names (EPUB 2). Some
converters write the image's address there rather than its id, and that is
followed too.
3. The guide's `<reference type="cover">`, a page, and the first `<img src>`
or SVG `<image href>` on it. A guide that points straight at an image is
taken at its word.
4. Any image in the manifest whose id or address says `cover`.
3. The guide's `<reference type="cover">`, a page, and the `<img src>` or
SVG `<image href>` on it. Where the page has several, the one whose file
name, `alt`, `id` or `class` says `cover` is taken; where none does, the
first is, with a note that there were others. A guide that points straight
at an image is taken at its word.
4. Any image in the manifest whose id or file name says `cover`, one named
`cover` outright first and the rest in manifest order.
A name says `cover` when it has it as a word of its own, in any case. Words
break at anything not a letter or digit and where a lower case letter meets a
capital, so `CoverImage`, `Book_COVER` and `cover-thumb` say it, and
`discover-ch1` and `recovery-map` do not.
A route that names something unusable — not in the archive, not an image —
leaves a note, and the next route is asked. Addresses are resolved against the
@@ -140,7 +153,16 @@ What the bytes say beats what the manifest says: a JPEG listed as
`image/png` comes back as `image/jpeg`, with a note. A cover larger than
`bookmeta::MAX_COVER` (16 MiB) is left out with a note, in either format, and
an EPUB's is never decompressed further than that, whatever size the archive
claims for it.
claims for it. Nor are more than eight members read looking for one,
however many candidates the manifest lists: the search stops there, with a
note.
`Cover::guessed` says the file left the choice open: a Kindle book's first
image record, or an EPUB image picked because its name says `cover`. Either
can be an illustration from inside the book, so a caller that can get a cover
another way, from a register by ISBN, should prefer that one. A cover the file
names is never a guess. `Cover::extension` gives the usual file extension for
its media type.
## Asking a model
crates/bookmeta/src/epub.rs
+284 -16
View File
@@ -22,6 +22,13 @@ use crate::{Cover, MAX_COVER, Metadata, Source, image_type};
/// document or is trying to be a decompression bomb.
const MAX_PACKAGE: u64 = 4 * 1024 * 1024;
/// How many members one search for a cover will decompress.
///
/// Each is bounded by [`MAX_COVER`], but a manifest can list as many
/// cover-named images as it likes, and reading every one of them is time a
/// single book is not owed. A real cover is found in one or two reads.
const MAX_READS: usize = 8;
/// Read what an EPUB says about itself.
pub fn read(path: &str, bytes: &[u8]) -> Metadata {
let mut meta = Metadata::empty(path, Source::Opf);
@@ -295,6 +302,8 @@ fn cover<R: std::io::Read + std::io::Seek>(
archive,
items: &items,
tried: Vec::new(),
reads: 0,
spent: false,
meta,
};
@@ -344,13 +353,23 @@ fn cover<R: std::io::Read + std::io::Seek>(
}
}
// Last: an image whose id or address says cover.
for item in items.iter().filter(|item| {
item.media_type.starts_with("image/")
&& (item.id.to_lowercase().contains("cover")
|| item.href.to_lowercase().contains("cover"))
}) {
if let Some(found) = look.image(base, &item.href) {
// Last: an image whose id or file name has the word cover in it, one
// named `cover` outright first, then in manifest order.
let file_stem = |item: &Item| stem(item.path.as_deref().unwrap_or(&item.href)).to_string();
let mut named: Vec<&Item> = items
.iter()
.filter(|item| {
item.media_type.starts_with("image/")
&& (says_cover(&item.id) || says_cover(&file_stem(item)))
})
.collect();
named.sort_by_key(|item| !file_stem(item).eq_ignore_ascii_case("cover"));
for item in named {
if look.spent {
break;
}
if let Some(mut found) = look.image(base, &item.href) {
found.guessed = true;
return Some(found);
}
}
@@ -363,12 +382,19 @@ struct Look<'a, R> {
archive: &'a mut zip::ZipArchive<R>,
items: &'a [Item],
tried: Vec<String>,
/// How many members have been decompressed, against [`MAX_READS`].
reads: usize,
/// Whether the reads have run out, and said so.
spent: bool,
meta: &'a mut Metadata,
}
impl<R: std::io::Read + std::io::Seek> Look<'_, R> {
/// Where an address leads, or a note saying it leads nowhere it may.
fn locate(&mut self, base: &str, href: &str) -> Option<String> {
if self.spent {
return None;
}
let Some(path) = resolve(base, href) else {
self.meta.note(format!(
"the cover's address, {href}, is not a path inside the archive"
@@ -391,6 +417,11 @@ impl<R: std::io::Read + std::io::Seek> Look<'_, R> {
/// The image a cover page shows, or the page itself where the guide
/// points straight at an image.
///
/// A page with several images may lead with a logo or an ornament, so
/// the one whose file name, `alt`, `id` or `class` says cover is taken
/// over the first; with none of them saying so, the first is taken, and
/// a note says the page had others.
fn page(&mut self, base: &str, href: &str) -> Option<Cover> {
let path = self.locate(base, href)?;
let bytes = self.fetch(&path, MAX_PACKAGE as usize, "cover page")?;
@@ -398,23 +429,55 @@ impl<R: std::io::Read + std::io::Seek> Look<'_, R> {
return self.check(&path, bytes);
}
let text = String::from_utf8_lossy(&bytes);
let Some(src) = tags(&text).find_map(|(name, tag)| match name {
"img" => attr(tag, "src"),
"image" => attr(tag, "xlink:href").or_else(|| attr(tag, "href")),
_ => None,
}) else {
let images: Vec<(String, bool)> = tags(&text)
.filter_map(|(name, tag)| {
let src = match name {
"img" => attr(tag, "src"),
"image" => attr(tag, "xlink:href").or_else(|| attr(tag, "href")),
_ => None,
}?;
let named = says_cover(stem(src.split('#').next().unwrap_or("")))
|| ["alt", "id", "class"]
.iter()
.any(|name| attr(tag, name).is_some_and(|value| says_cover(&value)));
Some((src, named))
})
.collect();
let src = if let Some((src, _)) = images.iter().find(|(_, named)| *named) {
src
} else if let Some((first, _)) = images.first() {
if images.len() > 1 {
self.meta.note(format!(
"the cover page, {path}, shows {} images, and the first was taken",
images.len()
));
}
first
} else {
self.meta
.note(format!("the cover page, {path}, shows no image"));
return None;
};
self.image(directory_of(&path), &src)
self.image(directory_of(&path), src)
}
/// One member's bytes, read no further than `cap`.
///
/// The size the archive declares is checked first and the read is
/// bounded anyway, because the declaration may be a lie.
/// bounded anyway, because the declaration may be a lie. No more than
/// [`MAX_READS`] members are read in all; the first asked for after
/// that ends the search, with a note.
fn fetch(&mut self, path: &str, cap: usize, what: &str) -> Option<Vec<u8>> {
if self.spent {
return None;
}
if self.reads == MAX_READS {
self.spent = true;
self.meta.note(format!(
"the search for a cover stopped after reading {MAX_READS} members"
));
return None;
}
let file = match self.archive.by_name(path) {
Ok(file) => file,
Err(zip::result::ZipError::FileNotFound) => {
@@ -433,6 +496,7 @@ impl<R: std::io::Read + std::io::Seek> Look<'_, R> {
self.meta.note(too_large);
return None;
}
self.reads += 1;
let mut bytes = Vec::new();
if file.take(cap as u64 + 1).read_to_end(&mut bytes).is_err() {
self.meta
@@ -469,6 +533,7 @@ impl<R: std::io::Read + std::io::Seek> Look<'_, R> {
Some(Cover {
media_type: media_type.to_string(),
bytes,
guessed: false,
})
}
}
@@ -478,6 +543,50 @@ fn directory_of(path: &str) -> &str {
path.rsplit_once('/').map_or("", |(dir, _)| dir)
}
/// A path's file name without its extension.
fn stem(path: &str) -> &str {
let name = path.rsplit_once('/').map_or(path, |(_, name)| name);
match name.rsplit_once('.') {
Some((stem, _)) if !stem.is_empty() => stem,
_ => name,
}
}
/// Whether a name has `cover` in it as a word of its own, in any case.
///
/// Words are split at anything not a letter or a digit and where a lower
/// case letter meets a capital, so `CoverImage`, `Book_COVER` and
/// `cover-thumb` say cover, and `discover` and `recovery` do not.
fn says_cover(name: &str) -> bool {
words(name)
.iter()
.any(|word| word.eq_ignore_ascii_case("cover"))
}
/// The words of a name, as [`says_cover`] splits them.
fn words(name: &str) -> Vec<&str> {
let mut out = Vec::new();
let mut start: Option<usize> = None;
let mut prev = ' ';
for (at, c) in name.char_indices() {
let begins =
c.is_alphanumeric() && (start.is_none() || (prev.is_lowercase() && c.is_uppercase()));
if !c.is_alphanumeric() || begins {
if let Some(from) = start.take() {
out.push(&name[from..at]);
}
}
if begins {
start = Some(at);
}
prev = c;
}
if let Some(from) = start {
out.push(&name[from..]);
}
out
}
/// Where an address written in a member in `base` leads inside the archive.
///
/// The fragment comes off and percent-escapes are undone. An address with a
@@ -521,7 +630,13 @@ fn percent_decode(text: &str) -> Option<String> {
let mut at = 0;
while at < raw.len() {
if raw[at] == b'%' {
let hex = std::str::from_utf8(raw.get(at + 1..at + 3)?).ok()?;
// Two hex digits and nothing else: `from_str_radix` would also
// take a sign, and read `%+f` as 15.
let hex = raw.get(at + 1..at + 3)?;
if !hex.iter().all(u8::is_ascii_hexdigit) {
return None;
}
let hex = std::str::from_utf8(hex).ok()?;
out.push(u8::from_str_radix(hex, 16).ok()?);
at += 3;
} else {
@@ -767,6 +882,8 @@ mod tests {
let cover = meta.cover.expect("a cover");
assert_eq!(cover.media_type, "image/jpeg");
assert_eq!(cover.bytes, JPEG);
assert!(!cover.guessed, "a declared cover is not a guess");
assert_eq!(cover.extension(), "jpg");
assert!(meta.notes.is_empty(), "{:?}", meta.notes);
}
@@ -861,7 +978,10 @@ mod tests {
("OEBPS/images/i2.gif", GIF),
],
);
assert_eq!(meta.cover.expect("a cover").bytes, GIF);
let cover = meta.cover.expect("a cover");
assert_eq!(cover.bytes, GIF);
assert!(cover.guessed, "a cover found by its name is a guess");
assert_eq!(cover.extension(), "gif");
let by_address = opf(
r#"<item id="i9" href="images/Book_COVER.jpeg" media-type="image/jpeg"/>"#,
@@ -871,6 +991,147 @@ mod tests {
assert_eq!(meta.cover.expect("a cover").bytes, JPEG);
}
#[test]
fn a_name_says_cover_only_as_a_word_of_its_own() {
for yes in [
"cover",
"CoverImage",
"Book_COVER",
"cover-thumb",
"frontCover",
"x.cover",
] {
assert!(says_cover(yes), "{yes}");
}
for no in [
"discover-ch1",
"recovery-map",
"covers",
"Coverage",
"uncover",
"",
] {
assert!(!says_cover(no), "{no}");
}
assert_eq!(words("CoverImage_v2"), ["Cover", "Image", "v2"]);
}
#[test]
fn a_name_that_only_contains_cover_is_not_a_cover() {
let package = opf(
r#"<item id="ch1" href="images/discover-ch1.png" media-type="image/png"/>
<item id="i2" href="images/cover.jpg" media-type="image/jpeg"/>"#,
"",
);
let meta = book(
&package,
&[
("OEBPS/images/discover-ch1.png", PNG),
("OEBPS/images/cover.jpg", JPEG),
],
);
let cover = meta.cover.expect("a cover");
assert_eq!(cover.bytes, JPEG);
assert!(cover.guessed);
let alone = opf(
r#"<item id="map" href="images/recovery-map.png" media-type="image/png"/>"#,
"",
);
let meta = book(&alone, &[("OEBPS/images/recovery-map.png", PNG)]);
assert_eq!(meta.cover, None);
assert!(meta.notes.is_empty(), "{:?}", meta.notes);
}
#[test]
fn an_image_named_cover_outright_beats_one_listed_before_it() {
let package = opf(
r#"<item id="cover-thumb" href="images/thumb.png" media-type="image/png"/>
<item id="i2" href="images/Cover.jpg" media-type="image/jpeg"/>"#,
"",
);
let meta = book(
&package,
&[
("OEBPS/images/thumb.png", PNG),
("OEBPS/images/Cover.jpg", JPEG),
],
);
let cover = meta.cover.expect("a cover");
assert_eq!(cover.bytes, JPEG);
assert!(cover.guessed);
}
#[test]
fn the_guide_cover_page_prefers_the_image_that_says_cover() {
let package = opf(
"",
r#"<guide><reference type="cover" href="text/cover.xhtml"/></guide>"#,
);
for art in [
r#"<img src="../images/front-cover.jpg"/>"#,
r#"<img alt="Cover" src="../images/front-cover.jpg"/>"#,
r#"<img id="coverImage" src="../images/front-cover.jpg"/>"#,
r#"<img class="full cover" src="../images/front-cover.jpg"/>"#,
] {
let page = format!(r#"<html><body><img src="../images/logo.png"/>{art}</body></html>"#);
let meta = book(
&package,
&[
("OEBPS/text/cover.xhtml", page.as_bytes()),
("OEBPS/images/logo.png", PNG),
("OEBPS/images/front-cover.jpg", JPEG),
],
);
assert_eq!(meta.cover.expect(art).bytes, JPEG, "{art}");
assert!(meta.notes.is_empty(), "{art}: {:?}", meta.notes);
}
}
#[test]
fn a_guide_cover_page_with_several_unnamed_images_takes_the_first_and_says_so() {
let package = opf(
"",
r#"<guide><reference type="cover" href="text/cover.xhtml"/></guide>"#,
);
let page = br#"<html><body><img src="../images/logo.png"/><img src="../images/art.jpg"/></body></html>"#;
let meta = book(
&package,
&[
("OEBPS/text/cover.xhtml", page),
("OEBPS/images/logo.png", PNG),
("OEBPS/images/art.jpg", JPEG),
],
);
assert_eq!(meta.cover.expect("a cover").bytes, PNG);
assert_eq!(
meta.notes,
["the cover page, OEBPS/text/cover.xhtml, shows 2 images, and the first was taken"]
);
}
#[test]
fn a_search_for_a_cover_reads_no_more_than_its_budget() {
let count = MAX_READS + 4;
let manifest: String = (0..count)
.map(|n| format!(r#"<item id="cover-{n}" href="c{n}.jpg" media-type="image/jpeg"/>"#))
.collect();
let names: Vec<String> = (0..count).map(|n| format!("OEBPS/c{n}.jpg")).collect();
let members: Vec<(&str, &[u8])> = names
.iter()
.map(|name| (name.as_str(), &b"<html>not an image</html>"[..]))
.collect();
let meta = book(&opf(&manifest, ""), &members);
assert_eq!(meta.cover, None);
let mut expected: Vec<String> = (0..MAX_READS)
.map(|n| format!("the cover, OEBPS/c{n}.jpg, is not an image"))
.collect();
expected.push(format!(
"the search for a cover stopped after reading {MAX_READS} members"
));
assert_eq!(meta.notes, expected);
}
#[test]
fn a_book_that_names_no_cover_has_none_and_says_nothing() {
let package = opf(
@@ -931,6 +1192,8 @@ mod tests {
archive: &mut archive,
items: &[],
tried: Vec::new(),
reads: 0,
spent: false,
meta: &mut meta,
};
let got = look.fetch("big.jpg", cap, "cover").map(|bytes| bytes.len());
@@ -1023,6 +1286,8 @@ mod tests {
Some("OEBPS/text/b.jpg")
);
assert_eq!(resolve("", "a%2Fb.jpg").as_deref(), Some("a/b.jpg"));
assert_eq!(percent_decode("a%2fb%2F").as_deref(), Some("a/b/"));
assert_eq!(percent_decode("a%+fb"), None);
for hostile in [
"../../c.jpg",
"../../../etc/passwd",
@@ -1031,6 +1296,9 @@ mod tests {
"file:///etc/passwd",
"%ZZ.jpg",
"%ff.jpg",
"a%+fb",
"a%-1b",
"a% fb",
"c%",
"#only-a-fragment",
"",
crates/bookmeta/src/lib.rs
+18
View File
@@ -110,6 +110,24 @@ pub struct Cover {
pub media_type: String,
/// The image itself, never more than [`MAX_COVER`] bytes.
pub bytes: Vec<u8>,
/// Whether the file left the choice open. A MOBI that names no cover, or
/// an EPUB whose only cover is an image with "cover" in its name, has one
/// picked by position or by name, and that pick can be an illustration
/// from inside the book. A caller with a better source should prefer it.
pub guessed: bool,
}
impl Cover {
/// The usual file extension for the image, without the dot.
pub fn extension(&self) -> &'static str {
match self.media_type.as_str() {
"image/png" => "png",
"image/gif" => "gif",
"image/webp" => "webp",
"image/bmp" => "bmp",
_ => "jpg",
}
}
}
/// How large a cover is worth keeping.
crates/bookmeta/src/mobi.rs
+2
View File
@@ -342,6 +342,7 @@ fn cover(
Some(Cover {
media_type: media_type.to_string(),
bytes: data.to_vec(),
guessed: false,
})
}
@@ -361,6 +362,7 @@ fn first_image_record(bytes: &[u8], records: &[usize], first_image: Option<u32>)
Some(Cover {
media_type: media_type.to_string(),
bytes: data.to_vec(),
guessed: true,
})
}
crates/bookmeta/tests/mobi.rs
+2
View File
@@ -31,6 +31,7 @@ fn says_what_it_was_given(meta: &bookmeta::Metadata) {
let cover = meta.cover.as_ref().expect("a cover");
assert_eq!(cover.media_type, "image/jpeg");
assert!(!cover.guessed, "EXTH 201 names the cover");
let original = std::fs::read(fixture("lantern-cover.jpg")).expect("the cover fixture");
assert_eq!(cover.bytes, original);
}
@@ -84,6 +85,7 @@ fn a_file_naming_no_cover_gets_its_first_image_record() {
let meta = bookmeta::from_bytes("x.mobi", &lantern_naming_no_cover());
let cover = meta.cover.expect("the first image record");
assert_eq!(cover.media_type, "image/jpeg");
assert!(cover.guessed, "a cover taken by position is a guess");
let original = std::fs::read(fixture("lantern-cover.jpg")).expect("the cover fixture");
assert_eq!(cover.bytes, original);
}
crates/booksniff/README.md
+15 -6
View File
@@ -89,12 +89,12 @@ one bad file does not stop a sweep.
| `ole2` | The compound-file header every binary Word document starts with |
| `epub` | A zip holding `mimetype` and `META-INF/container.xml` |
| `zip-wrapped-pdf` | A zip whose only real member is one PDF |
| `zip-wrapped-fb2` | A zip whose only real member is one FictionBook — the `.fb2.zip` most of them are passed around as |
| `cbz` | A zip whose every real member is a page image: JPEG, PNG, GIF or WebP |
| `zip-wrapped-fb2` | A zip whose only real member is named `.fb2` — the `.fb2.zip` most FictionBooks are passed around as |
| `cbz` | A zip whose every real member is a page image (JPEG, PNG, GIF or WebP), and which carries a `ComicInfo.xml` or is itself named `.cbz` or `.cbr` |
| `azw3`, `mobi` | A Palm database tagged `BOOKMOBI`, told apart by the format version inside the first record |
| `lit` | `ITOLITLS`, Microsoft Reader's signature |
| `rtf` | `{\rtf` at the front, after any byte-order mark and blank space |
| `fb2` | XML whose root element is `FictionBook`, found by stepping over the declaration, comments and doctype in the first four kilobytes |
| `rtf` | `{\rtf` at the front, after any byte-order mark and blank space, followed by the version digit, a space or the next control word |
| `fb2` | XML whose root element is `FictionBook`, found by stepping over the declaration, comments and doctype in the first four kilobytes; UTF-16 with a byte-order mark is decoded first |
| `html` | Markup at the front, after any byte-order mark |
| `txt` | Text that is none of the above |
| `unknown` | Anything else, with a note saying what was seen |
@@ -110,8 +110,17 @@ kind is there to stop exactly that.
A comic is the pages and nothing else. Directories, the `ComicInfo.xml` a
comic reader writes into the archive, and the desktop clutter every archive
picks up are passed over; anything more — one PDF among the pages, a readme —
and the zip is not a comic, and says what it holds instead. Only names are
read, out of the archive's directory, so no member is ever unpacked.
and the zip is not a comic, and says what it holds instead.
A zip of pictures is as often a holiday as a comic, so images alone are not
enough. It is a comic when it carries a `ComicInfo.xml`, or when its own name
ends `.cbz` or `.cbr`. Otherwise it is `unknown`, with a note saying it is a
zip of so many images and would be a comic book if its name said `.cbz`.
Members are judged by name. Only the archive's directory is read, so no
member is ever unpacked and a page called `001.jpg` is taken to be an image
without its bytes being looked at. The same goes for a wrapped FictionBook:
the one member is taken to be a FictionBook because its name ends `.fb2`.
FictionBook is read as bytes rather than as text. A great many are written in
windows-1251, which spells its markup in ASCII but is not UTF-8, and would
crates/booksniff/src/fb2.rs
+99
View File
@@ -9,9 +9,22 @@
//! windows-1251, which is not UTF-8 but spells its markup in ASCII all the
//! same, and a reader that insisted on UTF-8 would call every one of them
//! binary.
//!
//! A FictionBook saved as UTF-16 is the one case where the markup is not
//! ASCII bytes. Its byte-order mark says so, and the front is decoded before
//! the same walk is made over it.
use crate::BOM;
/// The UTF-16 byte-order mark, little-endian.
const UTF16_LE: &[u8] = b"\xff\xfe";
/// The UTF-16 byte-order mark, big-endian.
const UTF16_BE: &[u8] = b"\xfe\xff";
/// Most UTF-16 code units decoded from the front: the same window, in
/// characters, as the bytes read for every other encoding.
const UTF16_UNITS: usize = 2048;
/// What the root element of a FictionBook is called.
const ROOT: &[u8] = b"FictionBook";
@@ -21,9 +34,41 @@ const ROOT: &[u8] = b"FictionBook";
/// does not begin inside it is not found, so the answer is no rather than a
/// guess.
pub fn is_fictionbook(head: &[u8]) -> bool {
match utf16(head) {
Some(decoded) => is_fictionbook_root(decoded.as_bytes()),
None => is_fictionbook_root(head),
}
}
fn is_fictionbook_root(head: &[u8]) -> bool {
root_element(head).is_some_and(|name| local_name(name) == ROOT)
}
/// The front of a UTF-16 file as UTF-8, if a byte-order mark says it is one.
///
/// Only a bounded prefix is decoded. A half character at the end, or a
/// surrogate without its pair, becomes a replacement character, which the
/// walk takes as part of a name and so never as the end of one.
fn utf16(head: &[u8]) -> Option<String> {
let (body, big_endian) = match head.strip_prefix(UTF16_LE) {
Some(body) => (body, false),
None => (head.strip_prefix(UTF16_BE)?, true),
};
let units = body.chunks_exact(2).take(UTF16_UNITS).map(|pair| {
let pair = [pair[0], pair[1]];
if big_endian {
u16::from_be_bytes(pair)
} else {
u16::from_le_bytes(pair)
}
});
Some(
char::decode_utf16(units)
.map(|c| c.unwrap_or(char::REPLACEMENT_CHARACTER))
.collect(),
)
}
/// The name of the first element, after the prolog.
fn root_element(head: &[u8]) -> Option<&[u8]> {
let mut rest = head.strip_prefix(BOM).unwrap_or(head);
@@ -146,6 +191,60 @@ mod tests {
assert!(!is_fictionbook(b"<!-- never closed <FictionBook>"));
}
fn utf16(text: &str, big_endian: bool) -> Vec<u8> {
let mut out = if big_endian {
UTF16_BE.to_vec()
} else {
UTF16_LE.to_vec()
};
for unit in text.encode_utf16() {
out.extend_from_slice(&if big_endian {
unit.to_be_bytes()
} else {
unit.to_le_bytes()
});
}
out
}
const UTF16_BOOK: &str = "<?xml version=\"1.0\" encoding=\"UTF-16\"?>\r\n\
<FictionBook xmlns=\"http://www.gribuser.ru/xml/fictionbook/2.0\">\
<description><book-title>Мельница</book-title>";
#[test]
fn a_little_endian_utf16_fictionbook_is_recognised() {
assert!(is_fictionbook(&utf16(UTF16_BOOK, false)));
assert!(!is_fictionbook(&utf16(
"<?xml version=\"1.0\"?><rss>",
false
)));
}
#[test]
fn a_big_endian_utf16_fictionbook_is_recognised() {
assert!(is_fictionbook(&utf16(UTF16_BOOK, true)));
assert!(!is_fictionbook(&utf16(
"<?xml version=\"1.0\"?><rss>",
true
)));
}
#[test]
fn utf16_read_in_the_wrong_order_is_not_a_fictionbook() {
let mut swapped = utf16(UTF16_BOOK, false);
swapped[..2].copy_from_slice(UTF16_BE);
assert!(!is_fictionbook(&swapped));
}
#[test]
fn a_utf16_root_the_window_cut_short_is_not_taken() {
let head = utf16("<FictionBook>", false);
for cut in 0..head.len() {
assert!(!is_fictionbook(&head[..cut]), "cut at {cut}");
}
assert!(is_fictionbook(&head));
}
#[test]
fn a_truncated_or_mangled_prolog_does_not_panic() {
let head = b"\xef\xbb\xbf<?xml?><!--c--><!DOCTYPE x [ <!ENTITY a '>'> ] \"]\">\
crates/booksniff/src/lib.rs
+137 -13
View File
@@ -36,7 +36,10 @@ const ARCHIVE_NOISE: &[&str] = &["__macosx/", ".ds_store", "thumbs.db"];
/// What a comic book archive carries besides its pages: the catalogue entry
/// comic readers write into it.
const COMIC_NOISE: &[&str] = &["comicinfo.xml"];
const COMIC_CATALOGUE: &str = "comicinfo.xml";
/// The extensions that say an archive is a comic book.
const COMIC_EXTENSIONS: &[&str] = &["cbz", "cbr"];
/// The extensions of the images a comic book archive is made of.
const PAGE_IMAGES: &[&str] = &[".jpg", ".jpeg", ".png", ".gif", ".webp"];
@@ -125,8 +128,23 @@ fn archive(source: &impl Bytes, doc: &mut Doc) -> Kind {
doc.note(format!("a zip holding one FictionBook, {fb2:?}"));
return Kind::ZipWrappedFb2;
}
if is_comic(&names) {
return Kind::Cbz;
if let Some(pages) = comic_pages(&names) {
// A zip of pictures is as often a holiday as a comic, and only the
// names have been read. It is a comic when something says so: the
// catalogue entry a comic reader writes, or a name that claims it.
let named_comic = doc
.declared_ext
.as_deref()
.is_some_and(|ext| COMIC_EXTENSIONS.contains(&ext));
if pages.catalogued || named_comic {
return Kind::Cbz;
}
doc.note(format!(
"a zip of {} image{}, which would be a comic book if its name said .cbz",
pages.count,
if pages.count == 1 { "" } else { "s" }
));
return Kind::Unknown;
}
if lower
.iter()
@@ -160,23 +178,39 @@ fn sole_member<'a>(names: &'a [String], extension: &str) -> Option<&'a str> {
}
}
/// Whether an archive is the pages of a comic and nothing else.
/// The pages of an archive that is images and nothing else.
struct Pages {
/// How many images there are.
count: usize,
/// Whether a comic reader's catalogue entry is there beside them.
catalogued: bool,
}
/// The pages, if an archive is images and nothing else.
///
/// Every real member has to be an image. One PDF or one text file among the
/// pages makes it something else, and saying what would be a guess.
fn is_comic(names: &[String]) -> bool {
/// pages makes it something else, and saying what would be a guess. Images
/// are known by their names alone: only the archive's directory is read.
fn comic_pages(names: &[String]) -> Option<Pages> {
let mut catalogued = false;
let pages: Vec<String> = real_members(names)
.into_iter()
.map(|name| name.to_lowercase())
.filter(|name| {
let file = name.rsplit('/').next().unwrap_or(name);
!COMIC_NOISE.contains(&file)
let catalogue = file == COMIC_CATALOGUE;
catalogued |= catalogue;
!catalogue
})
.collect();
!pages.is_empty()
let all_images = !pages.is_empty()
&& pages
.iter()
.all(|name| PAGE_IMAGES.iter().any(|image| name.ends_with(image)))
.all(|name| PAGE_IMAGES.iter().any(|image| name.ends_with(image)));
all_images.then_some(Pages {
count: pages.len(),
catalogued,
})
}
/// The members that carry content: no directories, no desktop clutter.
@@ -198,7 +232,7 @@ fn front(head: &[u8], doc: &mut Doc) -> Kind {
// design but not always in practice, and a FictionBook is as often
// windows-1251 as UTF-8.
let lead = head.strip_prefix(BOM).unwrap_or(head).trim_ascii_start();
if lead.starts_with(RTF) {
if is_rtf(lead) {
return Kind::Rtf;
}
if fb2::is_fictionbook(head) {
@@ -207,6 +241,17 @@ fn front(head: &[u8], doc: &mut Doc) -> Kind {
text(head, doc)
}
/// Whether `lead` opens an RTF document.
///
/// `{\rtf` has to be the whole control word: what follows it is the version,
/// normally `1`, or the space or next control word that ends it. `{\rtfoo` is
/// some other word that happens to start the same way.
fn is_rtf(lead: &[u8]) -> bool {
lead.strip_prefix(RTF)
.and_then(|rest| rest.first())
.is_some_and(|next| next.is_ascii_digit() || next.is_ascii_whitespace() || *next == b'\\')
}
/// Text, markup, or nothing this knows.
fn text(head: &[u8], doc: &mut Doc) -> Kind {
if head.is_empty() {
@@ -432,10 +477,23 @@ mod tests {
#[test]
fn rtf_cut_short_of_its_control_word_is_not_rtf() {
assert_eq!(sniff(&br"{\rt".as_slice(), "a.rtf").kind, Kind::Txt);
assert_eq!(sniff(&br"{\rtf".as_slice(), "a.rtf").kind, Kind::Txt);
assert_eq!(kind_of(br"{\rtx1", "a.rtf"), Kind::Txt);
assert_eq!(kind_of(br"text {\rtf1", "a.rtf"), Kind::Txt);
}
#[test]
fn rtf_is_the_whole_control_word() {
assert_eq!(kind_of(br"{\rtf\ansi\deff0", "a.rtf"), Kind::Rtf);
assert_eq!(kind_of(b"{\\rtf \\ansi", "a.rtf"), Kind::Rtf);
assert_eq!(kind_of(b"{\\rtf\r\n\\ansi", "a.rtf"), Kind::Rtf);
assert_eq!(kind_of(br"{\rtf2", "a.rtf"), Kind::Rtf);
assert_eq!(kind_of(br"{\rtfoo bar}", "a.rtf"), Kind::Txt);
assert_eq!(kind_of(br"{\rtf}", "a.rtf"), Kind::Txt);
assert_eq!(kind_of(br"{\rtf-1", "a.rtf"), Kind::Txt);
}
#[test]
fn rtf_named_as_a_word_document_is_remarked_on() {
let doc = sniff(&padded(br"{\rtf1\ansi").as_slice(), "letter.doc");
@@ -471,6 +529,28 @@ mod tests {
assert_eq!(kind_of(feed, "feed.xml"), Kind::Html);
}
#[test]
fn a_utf16_fictionbook_is_recognised_in_either_byte_order() {
let book = "<?xml version=\"1.0\" encoding=\"UTF-16\"?>\n<FictionBook>";
for big_endian in [false, true] {
let mut bytes = if big_endian {
vec![0xfe, 0xff]
} else {
vec![0xff, 0xfe]
};
for unit in book.encode_utf16() {
bytes.extend_from_slice(&if big_endian {
unit.to_be_bytes()
} else {
unit.to_le_bytes()
});
}
let doc = sniff(&padded(&bytes).as_slice(), "mill.fb2");
assert_eq!(doc.kind, Kind::Fb2, "big-endian: {big_endian}");
assert!(doc.notes.is_empty(), "{:?}", doc.notes);
}
}
#[test]
fn a_fictionbook_named_as_something_else_is_remarked_on() {
let doc = sniff(&padded(b"<FictionBook>").as_slice(), "mill.txt");
@@ -526,11 +606,55 @@ mod tests {
}
#[test]
fn a_comic_saved_as_a_zip_is_remarked_on() {
let doc = archive_of(&["001.jpg", "002.jpg"], "issue-1.zip");
fn a_zip_of_photographs_is_not_a_comic() {
let doc = archive_of(&["IMG_0001.jpg", "IMG_0002.jpg"], "holiday.zip");
assert_eq!(doc.kind, Kind::Unknown);
assert!(
doc.notes.iter().any(|note| note
== "a zip of 2 images, which would be a comic book if its name said .cbz"),
"{:?}",
doc.notes
);
let one = archive_of(&["IMG_0001.jpg"], "holiday.zip");
assert!(
one.notes
.iter()
.any(|note| note.starts_with("a zip of 1 image,")),
"{:?}",
one.notes
);
}
#[test]
fn the_same_images_named_as_a_comic_are_one() {
let doc = archive_of(&["IMG_0001.jpg", "IMG_0002.jpg"], "holiday.cbz");
assert_eq!(doc.kind, Kind::Cbz);
assert!(
doc.notes.iter().any(|note| note.contains("comic")),
!doc.notes.iter().any(|note| note.contains("named .")),
"{:?}",
doc.notes
);
// A comic zipped and named as if it were RAR is still a comic, and
// the name is remarked on.
let rar = archive_of(&["001.jpg", "002.jpg"], "issue-1.cbr");
assert_eq!(rar.kind, Kind::Cbz);
assert!(
rar.notes.iter().any(|note| note.contains("named .cbr")),
"{:?}",
rar.notes
);
}
#[test]
fn a_comic_saved_as_a_zip_is_known_by_its_catalogue_and_remarked_on() {
let doc = archive_of(&["001.jpg", "002.jpg", "ComicInfo.xml"], "issue-1.zip");
assert_eq!(doc.kind, Kind::Cbz);
assert!(
doc.notes
.iter()
.any(|note| note == "named .zip, but the bytes say a comic book archive"),
"{:?}",
doc.notes
);
crates/corpus-core/src/exit.rs
+2
View File
@@ -30,6 +30,8 @@ pub const UNAVAILABLE: u8 = 69;
pub const SOFTWARE: u8 = 70;
/// Reading or writing failed.
pub const IO: u8 = 74;
/// A service refused for now, and asking again later will work.
pub const TEMPFAIL: u8 = 75;
/// The answer a predicate tool gives.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
docs/contract.md
+1
View File
@@ -47,6 +47,7 @@ that was concatenated or written on Windows still reads.
| `69` | A program it needs is not installed |
| `70` | Its own fault |
| `74` | Reading or writing failed |
| `75` | A service refused for now; try again later |
Failures start where `sysexits.h` puts them, so "no" and "broke" are never the
same number. That is what makes this work: