mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-11-13 23:55:40 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 funkpower
|
5f2a471f58 | ||
|
funkpower
|
bd104834ed |
No files matched your search
Generated
+26
-26
@@ -101,7 +101,7 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "avplan"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -126,7 +126,7 @@ checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "bibclean"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -143,7 +143,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bibfetch"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -196,7 +196,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bookmeta"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -210,7 +210,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "booksniff"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -250,7 +250,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
|
||||
|
||||
[[package]]
|
||||
name = "catena"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"bibfetch",
|
||||
@@ -277,7 +277,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "catena-cli"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"clap_mangen",
|
||||
@@ -312,7 +312,7 @@ checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||
|
||||
[[package]]
|
||||
name = "chunkmd"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -334,7 +334,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "citefind"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"chunkmd",
|
||||
@@ -399,7 +399,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "colophon"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"catena-cli",
|
||||
@@ -448,7 +448,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "corpus-core"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"schemars",
|
||||
"serde",
|
||||
@@ -492,7 +492,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cuekit"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"corpus-core",
|
||||
"regex",
|
||||
@@ -504,7 +504,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cuenorm"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -514,7 +514,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cuepack"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -525,7 +525,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cueseg"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -612,7 +612,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "emb"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -876,7 +876,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "idnorm"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"catena-cli",
|
||||
@@ -942,7 +942,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "langid"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1022,7 +1022,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mdscrub"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1034,7 +1034,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mdsplit"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1113,7 +1113,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "pdfgutter"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1126,7 +1126,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdftriage"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1370,7 +1370,7 @@ checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189"
|
||||
|
||||
[[package]]
|
||||
name = "rrf"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1517,7 +1517,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "seriesof"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1634,7 +1634,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
||||
|
||||
[[package]]
|
||||
name = "textkit"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"regex",
|
||||
"tiktoken-rs",
|
||||
@@ -1724,7 +1724,7 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
|
||||
|
||||
[[package]]
|
||||
name = "tocfind"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
|
||||
+27
-27
@@ -3,39 +3,39 @@ resolver = "3"
|
||||
members = ["crates/*"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MPL-2.0"
|
||||
repository = "https://git.pumpin.online/funkpower/catena"
|
||||
|
||||
[workspace.dependencies]
|
||||
catena-cli = { path = "crates/catena-cli", version = "0.3.1" }
|
||||
corpus-core = { path = "crates/corpus-core", version = "0.3.1" }
|
||||
cuekit = { path = "crates/cuekit", version = "0.3.1" }
|
||||
textkit = { path = "crates/textkit", version = "0.3.1" }
|
||||
chunkmd = { path = "crates/chunkmd", version = "0.3.1" }
|
||||
mdscrub = { path = "crates/mdscrub", version = "0.3.1" }
|
||||
mdsplit = { path = "crates/mdsplit", version = "0.3.1" }
|
||||
pdftriage = { path = "crates/pdftriage", version = "0.3.1" }
|
||||
cuenorm = { path = "crates/cuenorm", version = "0.3.1" }
|
||||
cueseg = { path = "crates/cueseg", version = "0.3.1" }
|
||||
rrf = { path = "crates/rrf", version = "0.3.1" }
|
||||
cuepack = { path = "crates/cuepack", version = "0.3.1" }
|
||||
bibclean = { path = "crates/bibclean", version = "0.3.1" }
|
||||
avplan = { path = "crates/avplan", version = "0.3.1" }
|
||||
pdfgutter = { path = "crates/pdfgutter", version = "0.3.1" }
|
||||
catena = { path = "crates/bindings", version = "0.3.1" }
|
||||
seriesof = { path = "crates/seriesof", version = "0.3.1" }
|
||||
idnorm = { path = "crates/idnorm", version = "0.3.1" }
|
||||
bibfetch = { path = "crates/bibfetch", version = "0.3.1" }
|
||||
bookmeta = { path = "crates/bookmeta", version = "0.3.1" }
|
||||
langid = { path = "crates/langid", version = "0.3.1" }
|
||||
tocfind = { path = "crates/tocfind", version = "0.3.1" }
|
||||
booksniff = { path = "crates/booksniff", version = "0.3.1" }
|
||||
citefind = { path = "crates/citefind", version = "0.3.1" }
|
||||
colophon = { path = "crates/colophon", version = "0.3.1" }
|
||||
emb = { path = "crates/emb", version = "0.3.1" }
|
||||
catena-cli = { path = "crates/catena-cli", version = "0.4.0" }
|
||||
corpus-core = { path = "crates/corpus-core", version = "0.4.0" }
|
||||
cuekit = { path = "crates/cuekit", version = "0.4.0" }
|
||||
textkit = { path = "crates/textkit", version = "0.4.0" }
|
||||
chunkmd = { path = "crates/chunkmd", version = "0.4.0" }
|
||||
mdscrub = { path = "crates/mdscrub", version = "0.4.0" }
|
||||
mdsplit = { path = "crates/mdsplit", version = "0.4.0" }
|
||||
pdftriage = { path = "crates/pdftriage", version = "0.4.0" }
|
||||
cuenorm = { path = "crates/cuenorm", version = "0.4.0" }
|
||||
cueseg = { path = "crates/cueseg", version = "0.4.0" }
|
||||
rrf = { path = "crates/rrf", version = "0.4.0" }
|
||||
cuepack = { path = "crates/cuepack", version = "0.4.0" }
|
||||
bibclean = { path = "crates/bibclean", version = "0.4.0" }
|
||||
avplan = { path = "crates/avplan", version = "0.4.0" }
|
||||
pdfgutter = { path = "crates/pdfgutter", version = "0.4.0" }
|
||||
catena = { path = "crates/bindings", version = "0.4.0" }
|
||||
seriesof = { path = "crates/seriesof", version = "0.4.0" }
|
||||
idnorm = { path = "crates/idnorm", version = "0.4.0" }
|
||||
bibfetch = { path = "crates/bibfetch", version = "0.4.0" }
|
||||
bookmeta = { path = "crates/bookmeta", version = "0.4.0" }
|
||||
langid = { path = "crates/langid", version = "0.4.0" }
|
||||
tocfind = { path = "crates/tocfind", version = "0.4.0" }
|
||||
booksniff = { path = "crates/booksniff", version = "0.4.0" }
|
||||
citefind = { path = "crates/citefind", version = "0.4.0" }
|
||||
colophon = { path = "crates/colophon", version = "0.4.0" }
|
||||
emb = { path = "crates/emb", version = "0.4.0" }
|
||||
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
clap_mangen = "0.3"
|
||||
|
||||
@@ -34,7 +34,7 @@ Each tool installs by name, so you can take one without taking the rest.
|
||||
| [`vocabmerge`](crates/bibclean/README.md#vocabmerge--the-other-command) | The near-duplicate headings in a catalogue's vocabulary |
|
||||
| [`seriesof`](crates/seriesof/README.md) | The series a book belongs to, and which volume this is |
|
||||
| [`idnorm`](crates/idnorm/README.md) | DOIs, ISSNs, ISBNs and arXiv ids, read and checked |
|
||||
| [`bibfetch`](crates/bibfetch/README.md) | What a register says about an identifier, and which books it has under a title |
|
||||
| [`bibfetch`](crates/bibfetch/README.md) | What a register says about an identifier |
|
||||
| [`citefind`](crates/citefind/README.md) | The references in a bibliography or a page of notes |
|
||||
| [`mdscrub`](crates/mdscrub/README.md) | Converted markdown in, clean markdown out |
|
||||
| [`mdsplit`](crates/mdsplit/README.md) | One markdown file into the works inside it |
|
||||
|
||||
@@ -20,8 +20,6 @@ So this asks, and that is the only thing it does.
|
||||
- **Filling in a reference** you have only the identifier for.
|
||||
- **Before citing something**, to check the year and the authors are what you
|
||||
think they are.
|
||||
- **Cataloguing a book that came with no identifier**, to find the editions a
|
||||
register has under its title, and an ISBN to file it by.
|
||||
|
||||
## Install
|
||||
|
||||
@@ -136,14 +134,6 @@ redirect included, stops after two minutes. The two minutes start once it is
|
||||
the fetch's turn, so a fetch queued behind others in the same process can wait
|
||||
that long again first.
|
||||
|
||||
Open Library allows a hundred covers by ISBN every five minutes from one
|
||||
address, and refuses the rest with `403`. Covers asked by ISBN are kept three
|
||||
seconds apart across the whole process (`Covers::pace`), which stays under that
|
||||
limit however long a run goes. A refusal all the same is not mistaken for an
|
||||
outage: the other form of the ISBN is not asked, the cover id is, because
|
||||
Open Library does not limit those, and with no cover found that way the fetch
|
||||
fails with `Error::Limited`, which means ask again later.
|
||||
|
||||
| Option | Effect |
|
||||
|---|---|
|
||||
| `-o`, `--output <FILE>` | Where to write the image |
|
||||
@@ -152,94 +142,12 @@ fails with `Error::Limited`, which means ask again later.
|
||||
|
||||
Exit `0` the cover was written, `1` Open Library has none, `65` the argument
|
||||
was not an ISBN or the answer was not an image, `69` Open Library could not be
|
||||
reached, `74` the file could not be written, `75` Open Library refused for
|
||||
being asked too often and will answer later.
|
||||
reached, `74` the file could not be written.
|
||||
|
||||
From Rust, `bibfetch::cover(&agent, isbn, cover_id, size)` returns
|
||||
`Ok(None)` when there is no cover, and a [`Cover`](src/cover.rs) with the
|
||||
bytes, the media type and the address asked when there is.
|
||||
|
||||
## Searching by title
|
||||
|
||||
```
|
||||
$ bibfetch search --title 'The First World War' --author Howard --limit 2
|
||||
{"v":1,"id":"9780192853622","kind":"isbn","found":true,"source":"openlibrary","title":"The First World War","authors":["Michael Eliot Howard"],"year":2002,"container":"Oxford University Press","subjects":["World War, 1914-1918"],"language":"eng","cover_id":1123124}
|
||||
{"v":1,"id":"9780199205592","kind":"isbn","found":true,"source":"openlibrary","title":"The First World War: A Very Short Introduction","authors":["Michael Howard"],"year":2007,"container":"Oxford University Press","language":"eng","cover_id":14606475}
|
||||
```
|
||||
|
||||
A book that arrives as a file often has no identifier to ask about, only a
|
||||
title and perhaps a name. This asks which books a register has under those,
|
||||
and prints one record per candidate: Open Library's index first, and Google
|
||||
Books only when Open Library has nothing.
|
||||
|
||||
Each candidate is one edition. Open Library's index answers with works, and
|
||||
names the edition of each that best matches the search. The ISBN, publisher,
|
||||
language, year and cover are that edition's and no other's: a field the
|
||||
edition does not give is left out rather than filled from the work, whose
|
||||
publishers, years and ISBNs belong to all its editions at once. Only the
|
||||
title, the authors and the subjects, which describe the work, are the work's.
|
||||
Open Library's index gives no length for an edition, so its candidates carry
|
||||
no `pages`. Google Books answers with volumes, which are editions already.
|
||||
|
||||
| Field | When the candidate has an ISBN | When it has none |
|
||||
|---|---|---|
|
||||
| `id` | The first ISBN of the edition whose check digit agrees, in thirteen digits; one written in thirteen is taken over one written in ten | The register's own key: an Open Library id such as `OL37863121M` for the edition, or `OL267171W` for the work when no edition was named; a Google Books volume id |
|
||||
| `kind` | `isbn` | `olid` for Open Library, `googlebooks` for Google Books |
|
||||
|
||||
A work the index names no edition of, or names one it gives neither an ISBN
|
||||
nor a key for, is a candidate named by the work's key, with its title,
|
||||
authors and subjects and nothing an edition would say.
|
||||
|
||||
An ISBN whose check digit is wrong is passed over rather than corrected, and
|
||||
Google's shelfmarks, typed `OTHER`, are not read as ISBNs even when their
|
||||
digits would pass for one. A record of kind `isbn` can be handed straight back
|
||||
to `bibfetch` for everything the registers say about that edition; `olid` and
|
||||
`googlebooks` are names no lookup here answers about.
|
||||
|
||||
`cover_id` is the same thing it is in a lookup record: the first cover in
|
||||
Open Library's record of the edition the record names, by that edition's ISBN
|
||||
or its key, so `bibfetch cover --cover-id` fetches that edition's cover and no
|
||||
other. The cover the index picked for the whole work is never asked for,
|
||||
because it is often another edition's. Google Books knows nothing of Open
|
||||
Library's covers and gives none.
|
||||
|
||||
The candidates are what the register said, in the order it said them. This
|
||||
makes no judgement about which one is the book in hand, and keeps one that is
|
||||
plainly another book, because a title is not an identifier: *The Hamlet* comes
|
||||
back for *Hamlet*. Deciding whether a candidate is the same work is a
|
||||
comparison of titles, and [`bibclean`](../bibclean/README.md)'s `same_work`
|
||||
is one.
|
||||
|
||||
```sh
|
||||
bibfetch search --title 'Anathem' --author Stephenson \
|
||||
| jq -r 'select(.kind == "isbn") | .id' | head -1
|
||||
```
|
||||
|
||||
Open Library having nothing is an answer only once Google Books has been
|
||||
asked and has nothing either. When Google Books cannot be reached, the search
|
||||
fails rather than say no register has the title, and when it refuses for its
|
||||
quota (`403`, or `429` on every attempt) the failure says so and the command
|
||||
exits `75`. Open Library being down is a failure only when Google Books has
|
||||
nothing to offer in its place. Each is asked the same way a lookup asks,
|
||||
retried with the same waits, except that a `403` from Google Books is not
|
||||
asked again. Open Library is sent the title and the author as written; Google Books
|
||||
is sent each word as its own `intitle:` or `inauthor:` term, so nothing in a
|
||||
title is read as search syntax.
|
||||
|
||||
| Option | Effect |
|
||||
|---|---|
|
||||
| `--title <TITLE>` | The title, or as much of it as you have |
|
||||
| `--author <AUTHOR>` | An author's name, or part of one |
|
||||
| `--limit <COUNT>` | The most candidates to print, `1` to `40`. Default `10` |
|
||||
|
||||
Exit `0` at least one candidate was printed, `1` no register has the title,
|
||||
`65` the title was empty, `69` a register that had to answer could not be
|
||||
reached, `74` the output could not be written, `75` Google Books refused for
|
||||
its quota.
|
||||
|
||||
From Rust, `bibfetch::search(&agent, title, author, limit, attempts)` returns
|
||||
the candidates, empty when no register has the title.
|
||||
|
||||
## Reaching the network
|
||||
|
||||
This and [`emb`](../emb/README.md) are the only two tools here that do. Both
|
||||
@@ -248,11 +156,10 @@ something that happens on the way to something else — which is the condition
|
||||
under which [the contract](../../docs/contract.md) allows it at all.
|
||||
|
||||
Nothing is cached, nothing is stored but a cover you asked to have written,
|
||||
and nothing is sent but the identifier, or the title and author searched for,
|
||||
and the address you pass to `--polite`.
|
||||
and nothing is sent but the identifier and the address you pass to
|
||||
`--polite`.
|
||||
|
||||
## See also
|
||||
|
||||
- [`idnorm`](../idnorm/README.md) — reading the identifier in the first place.
|
||||
- [`bibclean`](../bibclean/README.md) — the rest of a catalogue record, and
|
||||
whether a candidate is the same work.
|
||||
- [`bibclean`](../bibclean/README.md) — the rest of a catalogue record.
|
||||
+25
-30
@@ -89,6 +89,10 @@ pub fn lookup(
|
||||
}
|
||||
|
||||
/// Ask a register, retrying a failure that might not be one.
|
||||
///
|
||||
/// Waits a second before the second attempt and two before the third: long
|
||||
/// enough for a register that is rate-limiting, short enough that one which
|
||||
/// is simply down does not hold up a pipeline.
|
||||
fn ask(
|
||||
agent: &ureq::Agent,
|
||||
source: Source,
|
||||
@@ -96,48 +100,39 @@ fn ask(
|
||||
kind: &str,
|
||||
attempts: u32,
|
||||
) -> Result<crate::Record, Error> {
|
||||
match get(agent, &source.url(id), attempts)? {
|
||||
Some(body) => source.read(id, kind, &body),
|
||||
// A register that does not have it says so with a status, and that is
|
||||
// an answer rather than a failure.
|
||||
None => Ok(crate::missing(id, kind, source)),
|
||||
}
|
||||
}
|
||||
|
||||
/// The body at an address, or `None` when the register says it has no such
|
||||
/// thing; a failure is asked again.
|
||||
///
|
||||
/// Waits a second before the second attempt and two before the third: long
|
||||
/// enough for a register that is rate-limiting, short enough that one which
|
||||
/// is simply down does not hold up a pipeline. Reading what came back is left
|
||||
/// to the caller, because an answer this cannot read will not read any better
|
||||
/// next time.
|
||||
pub(crate) fn get(agent: &ureq::Agent, url: &str, attempts: u32) -> Result<Option<String>, Error> {
|
||||
let attempts = attempts.max(1);
|
||||
let mut last = Error::Unreachable("no attempt was made".into());
|
||||
for attempt in 0..attempts {
|
||||
match once(agent, url) {
|
||||
Ok(answer) => return Ok(answer),
|
||||
|
||||
for attempt in 0..attempts.max(1) {
|
||||
match fetch(agent, source, id, kind) {
|
||||
Ok(record) => return Ok(record),
|
||||
// A shape this cannot read will not read any better next time.
|
||||
Err(e @ Error::Shape(_)) => return Err(e),
|
||||
Err(e) => last = e,
|
||||
}
|
||||
if attempt + 1 < attempts {
|
||||
std::thread::sleep(Duration::from_secs(1 << attempt.min(5)));
|
||||
}
|
||||
std::thread::sleep(Duration::from_secs(1 << attempt));
|
||||
}
|
||||
Err(last)
|
||||
}
|
||||
|
||||
fn once(agent: &ureq::Agent, url: &str) -> Result<Option<String>, Error> {
|
||||
let mut response = match agent.get(url).call() {
|
||||
fn fetch(
|
||||
agent: &ureq::Agent,
|
||||
source: Source,
|
||||
id: &str,
|
||||
kind: &str,
|
||||
) -> Result<crate::Record, Error> {
|
||||
let response = agent.get(source.url(id)).call();
|
||||
let mut response = match response {
|
||||
Ok(response) => response,
|
||||
Err(ureq::Error::StatusCode(404)) => return Ok(None),
|
||||
// A register that does not have it says so with a status, and that is
|
||||
// an answer rather than a failure.
|
||||
Err(ureq::Error::StatusCode(404)) => return Ok(crate::missing(id, kind, source)),
|
||||
Err(e) => return Err(Error::Unreachable(e.to_string())),
|
||||
};
|
||||
response
|
||||
let body = response
|
||||
.body_mut()
|
||||
.read_to_string()
|
||||
.map(Some)
|
||||
.map_err(|e| Error::Unreachable(e.to_string()))
|
||||
.map_err(|e| Error::Unreachable(e.to_string()))?;
|
||||
source.read(id, kind, &body)
|
||||
}
|
||||
|
||||
/// Who is asking.
|
||||
|
||||
@@ -46,11 +46,6 @@ const REDIRECT_HOSTS: [&str; 2] = ["covers.openlibrary.org", "archive.org"];
|
||||
/// agent's timeout each, and a minute's `Retry-After` between attempts.
|
||||
pub const DEADLINE: Duration = Duration::from_secs(120);
|
||||
|
||||
/// How long to leave between two covers asked by ISBN. Open Library allows a
|
||||
/// hundred every five minutes from one address and refuses the rest with
|
||||
/// `403`; one every three seconds stays under that however long a run goes.
|
||||
pub const PACE: Duration = Duration::from_secs(3);
|
||||
|
||||
/// The longest a `Retry-After` is honoured for. A register asking for more
|
||||
/// than this is better treated as down than waited on.
|
||||
const MAX_WAIT: Duration = Duration::from_secs(60);
|
||||
@@ -59,10 +54,7 @@ const MAX_WAIT: Duration = Duration::from_secs(60);
|
||||
/// this, and limits covers asked by ISBN to a hundred every five minutes
|
||||
/// from one address; threads asking at once would only meet the limit
|
||||
/// sooner and all back off together.
|
||||
///
|
||||
/// It holds when the last cover was asked for by ISBN, so that [`PACE`] is
|
||||
/// kept across fetches and threads.
|
||||
static ONE_AT_A_TIME: Mutex<Option<Instant>> = Mutex::new(None);
|
||||
static ONE_AT_A_TIME: Mutex<()> = Mutex::new(());
|
||||
|
||||
/// Which of Open Library's three sizes to ask for.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
@@ -135,9 +127,6 @@ pub struct Covers {
|
||||
/// request is given whatever is left of it, or the agent's own timeout
|
||||
/// if that is shorter, and a wait that would run past it is not waited.
|
||||
pub deadline: Duration,
|
||||
/// The least time between two covers asked by ISBN; [`PACE`] by default.
|
||||
/// Covers asked by id are not limited and are not spaced out.
|
||||
pub pace: Duration,
|
||||
}
|
||||
|
||||
impl Default for Covers {
|
||||
@@ -146,7 +135,6 @@ impl Default for Covers {
|
||||
base: OPEN_LIBRARY_COVERS.to_string(),
|
||||
attempts: crate::ATTEMPTS,
|
||||
deadline: DEADLINE,
|
||||
pace: PACE,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -174,12 +162,6 @@ impl Covers {
|
||||
/// of the ISBN does not stop the other being asked, but it does stop the
|
||||
/// cover id being asked: the ISBN's own cover may exist, and is the one
|
||||
/// wanted. The failure is returned when no ISBN form gave a cover.
|
||||
///
|
||||
/// Refusal for asking too often is the exception. It is
|
||||
/// [`Error::Limited`], the other form is not asked, since it would only
|
||||
/// be refused too, and the cover id is still asked, because Open Library
|
||||
/// does not limit those. A cover found that way is returned; otherwise the
|
||||
/// refusal is, so that the caller knows to ask again later.
|
||||
pub fn fetch(
|
||||
&self,
|
||||
agent: &ureq::Agent,
|
||||
@@ -204,54 +186,31 @@ impl Covers {
|
||||
.map(|form| format!("{}/b/isbn/{form}-{letter}.jpg?default=false", self.base))
|
||||
.collect();
|
||||
|
||||
let mut last_by_isbn = ONE_AT_A_TIME
|
||||
let _turn = ONE_AT_A_TIME
|
||||
.lock()
|
||||
.unwrap_or_else(|held| held.into_inner());
|
||||
// From here, not from the call: waiting for the lock is time another
|
||||
// fetch spent, and counting it would let a busy process starve this
|
||||
// one of the time its own requests need. A deadline too long to add
|
||||
// to the clock is no deadline at all.
|
||||
let now = Instant::now();
|
||||
let until = now
|
||||
.checked_add(self.deadline)
|
||||
.unwrap_or_else(|| now + Duration::from_secs(365 * 24 * 60 * 60));
|
||||
// one of the time its own requests need.
|
||||
let until = Instant::now() + self.deadline;
|
||||
let mut failed = None;
|
||||
for address in &addresses {
|
||||
if let Some(ready) = last_by_isbn.and_then(|at| at.checked_add(self.pace)) {
|
||||
let now = Instant::now();
|
||||
if ready > now {
|
||||
if ready >= until {
|
||||
return Err(Error::Unreachable(format!(
|
||||
"{address}: past the {:?} deadline, waiting to keep {:?} \
|
||||
between covers asked by ISBN",
|
||||
self.deadline, self.pace
|
||||
)));
|
||||
}
|
||||
std::thread::sleep(ready - now);
|
||||
}
|
||||
}
|
||||
let answer = self.ask(agent, address, until);
|
||||
*last_by_isbn = Some(Instant::now());
|
||||
match answer {
|
||||
match self.ask(agent, address, until) {
|
||||
Ok(Answer::Image(cover)) => return Ok(Some(cover)),
|
||||
Ok(Answer::Absent) => {}
|
||||
Err(why @ Error::Limited(_)) => {
|
||||
failed = Some(why);
|
||||
break;
|
||||
}
|
||||
Err(why) => failed = Some(why),
|
||||
}
|
||||
}
|
||||
if let Some(why) = failed.take_if(|why| !matches!(why, Error::Limited(_))) {
|
||||
if let Some(why) = failed {
|
||||
return Err(why);
|
||||
}
|
||||
let Some(id) = cover_id else {
|
||||
return failed.map_or(Ok(None), Err);
|
||||
return Ok(None);
|
||||
};
|
||||
let address = format!("{}/b/id/{id}-{letter}.jpg?default=false", self.base);
|
||||
match self.ask(agent, &address, until)? {
|
||||
Answer::Image(cover) => Ok(Some(cover)),
|
||||
Answer::Absent => failed.map_or(Ok(None), Err),
|
||||
Answer::Absent => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -352,15 +311,6 @@ Open Library's covers or the Internet Archive"
|
||||
// `default=false` makes a missing cover a 404 rather than a
|
||||
// stand-in image.
|
||||
404 | 410 => return Ok(Answer::Absent),
|
||||
// Open Library's answer to more than a hundred covers asked by
|
||||
// ISBN in five minutes from one address. Asking again at once
|
||||
// would only be refused again.
|
||||
403 => {
|
||||
return Err(Fault::Final(Error::Limited(format!(
|
||||
"{at}: refused (403); Open Library allows a hundred covers \
|
||||
by ISBN every five minutes from one address"
|
||||
))));
|
||||
}
|
||||
429 | 503 => {
|
||||
let wait = header(&response, "retry-after")
|
||||
.and_then(|value| value.trim().parse::<u64>().ok())
|
||||
@@ -653,7 +603,6 @@ mod tests {
|
||||
base: self.base.clone(),
|
||||
attempts: 3,
|
||||
deadline: DEADLINE,
|
||||
pace: Duration::ZERO,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1103,73 +1052,6 @@ mod tests {
|
||||
assert_eq!(stub.asked(), [ISBN13, ISBN10]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_refusal_for_asking_too_often_still_asks_the_cover_id() {
|
||||
let stub = Stub::start(|_, path| match path {
|
||||
ISBN13 => Reply::status(403),
|
||||
"/b/id/42-L.jpg?default=false" => Reply::image("image/jpeg", jpeg(4096)),
|
||||
_ => Reply::status(404),
|
||||
});
|
||||
let cover = stub
|
||||
.covers()
|
||||
.fetch(&agent(), "9780306406157", Some(42), CoverSize::Large)
|
||||
.expect("an answer")
|
||||
.expect("a cover");
|
||||
assert!(cover.url.ends_with("/b/id/42-L.jpg?default=false"));
|
||||
// The ten-digit form would only have been refused too.
|
||||
assert_eq!(stub.asked(), [ISBN13, "/b/id/42-L.jpg?default=false"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_refusal_for_asking_too_often_is_not_an_absence() {
|
||||
let stub = Stub::start(|_, path| match path {
|
||||
ISBN13 => Reply::status(403),
|
||||
_ => Reply::status(404),
|
||||
});
|
||||
let found = stub
|
||||
.covers()
|
||||
.fetch(&agent(), "9780306406157", None, CoverSize::Large)
|
||||
.map(|cover| cover.map(|cover| cover.url));
|
||||
assert!(matches!(found, Err(Error::Limited(_))), "{found:?}");
|
||||
assert_eq!(stub.asked(), [ISBN13]);
|
||||
|
||||
// With a cover id that has no cover either, the refusal still stands:
|
||||
// the ISBN's own cover may be there once the limit passes.
|
||||
let found = stub
|
||||
.covers()
|
||||
.fetch(&agent(), "9780306406157", Some(42), CoverSize::Large)
|
||||
.map(|cover| cover.map(|cover| cover.url));
|
||||
assert!(matches!(found, Err(Error::Limited(_))), "{found:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_deadline_too_long_to_count_is_no_deadline() {
|
||||
let stub = Stub::start(|_, _| Reply::image("image/jpeg", jpeg(2048)));
|
||||
let covers = Covers {
|
||||
deadline: Duration::MAX,
|
||||
..stub.covers()
|
||||
};
|
||||
let cover = covers
|
||||
.fetch(&agent(), "9780306406157", None, CoverSize::Large)
|
||||
.expect("an answer");
|
||||
assert!(cover.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn covers_asked_by_isbn_are_spaced_out() {
|
||||
let stub = Stub::start(|_, _| Reply::status(404));
|
||||
let covers = Covers {
|
||||
pace: Duration::from_millis(300),
|
||||
..stub.covers()
|
||||
};
|
||||
let started = Instant::now();
|
||||
let found = covers.fetch(&agent(), "9780306406157", None, CoverSize::Large);
|
||||
assert!(matches!(found, Ok(None)), "{found:?}");
|
||||
// Two forms of the ISBN, so at least one pause between them.
|
||||
assert!(started.elapsed() >= Duration::from_millis(300));
|
||||
assert_eq!(stub.asked(), [ISBN13, ISBN10]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redirect_to_an_address_by_number_is_refused() {
|
||||
// A listener standing in for a service on the LAN; it only counts.
|
||||
|
||||
@@ -12,14 +12,9 @@
|
||||
//! them: Crossref for a DOI, the arXiv for a preprint, OpenLibrary for a
|
||||
//! book. There is no register for an ISSN that answers without a key, so an
|
||||
//! ISSN is looked up as the journal of a work rather than on its own.
|
||||
//!
|
||||
//! A book with no identifier at all can still be looked for by its title,
|
||||
//! with [`search()`]: Open Library's index, then Google Books, each candidate
|
||||
//! a record of one edition, in the order the register gave them.
|
||||
|
||||
pub mod ask;
|
||||
pub mod cover;
|
||||
pub mod search;
|
||||
pub mod source;
|
||||
|
||||
use std::time::Duration;
|
||||
@@ -28,7 +23,6 @@ use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub use cover::{Cover, CoverSize, cover};
|
||||
pub use search::search;
|
||||
pub use source::Source;
|
||||
|
||||
/// How long to wait on a register before giving up.
|
||||
@@ -112,9 +106,6 @@ pub enum Error {
|
||||
NoRegister(String),
|
||||
/// What was asked about is not an identifier of the kind it should be.
|
||||
Invalid(String),
|
||||
/// The register refused because it has been asked too often from this
|
||||
/// address. It is not down, and asking again later will work.
|
||||
Limited(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Error {
|
||||
@@ -126,7 +117,6 @@ impl std::fmt::Display for Error {
|
||||
write!(f, "no register here answers about {kind} identifiers")
|
||||
}
|
||||
Error::Invalid(why) => write!(f, "{why}"),
|
||||
Error::Limited(why) => write!(f, "{why}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,8 +30,7 @@ Example:
|
||||
|
||||
idnorm refs.txt | bibfetch | jq -r 'select(.found) | .title'
|
||||
idnorm --kind doi refs.txt | bibfetch --polite you@example.org
|
||||
bibfetch cover 978-0-306-40615-7 -o cover.jpg
|
||||
bibfetch search --title 'The First World War' --author Howard"
|
||||
bibfetch cover 978-0-306-40615-7 -o cover.jpg"
|
||||
)]
|
||||
struct Args {
|
||||
#[command(subcommand)]
|
||||
@@ -75,26 +74,6 @@ enum Command {
|
||||
#[arg(long, value_name = "ID")]
|
||||
cover_id: Option<u64>,
|
||||
},
|
||||
/// Find the books a register has under a title, and an author.
|
||||
///
|
||||
/// Prints one record per candidate edition, in the order the register
|
||||
/// gave them: Open Library's index, or Google Books when that has
|
||||
/// nothing. Which one is the book in hand is left to you.
|
||||
///
|
||||
/// Exit codes: 0 at least one candidate, 1 none, 65 no title,
|
||||
/// 69 no register could be reached, 74 the output could not be written.
|
||||
Search {
|
||||
/// The title, or as much of it as you have.
|
||||
#[arg(long)]
|
||||
title: String,
|
||||
/// An author's name, or part of one.
|
||||
#[arg(long)]
|
||||
author: Option<String>,
|
||||
/// The most candidates to print.
|
||||
#[arg(long, default_value_t = 10, value_name = "COUNT",
|
||||
value_parser = clap::value_parser!(u16).range(1..=bibfetch::search::MAX_LIMIT as i64))]
|
||||
limit: u16,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, clap::ValueEnum)]
|
||||
@@ -117,27 +96,14 @@ fn main() -> ExitCode {
|
||||
}
|
||||
.agent();
|
||||
|
||||
match &args.command {
|
||||
Some(Command::Cover {
|
||||
isbn,
|
||||
output,
|
||||
size,
|
||||
cover_id,
|
||||
}) => return cover(&agent, args.attempts, isbn, output, *size, *cover_id),
|
||||
Some(Command::Search {
|
||||
title,
|
||||
author,
|
||||
limit,
|
||||
}) => {
|
||||
return search(
|
||||
&agent,
|
||||
args.attempts,
|
||||
title,
|
||||
author.as_deref(),
|
||||
usize::from(*limit),
|
||||
);
|
||||
}
|
||||
None => {}
|
||||
if let Some(Command::Cover {
|
||||
isbn,
|
||||
output,
|
||||
size,
|
||||
cover_id,
|
||||
}) = &args.command
|
||||
{
|
||||
return cover(&agent, args.attempts, isbn, output, *size, *cover_id);
|
||||
}
|
||||
|
||||
let text = match catena_cli::read_text(args.path.as_deref()) {
|
||||
@@ -225,44 +191,12 @@ fn cover(
|
||||
eprintln!("bibfetch: {isbn}: {why}");
|
||||
ExitCode::from(match why {
|
||||
bibfetch::Error::Unreachable(_) => exit::UNAVAILABLE,
|
||||
bibfetch::Error::Limited(_) => exit::TEMPFAIL,
|
||||
_ => exit::DATA,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Search by title and print the candidates.
|
||||
fn search(
|
||||
agent: &ureq::Agent,
|
||||
attempts: u32,
|
||||
title: &str,
|
||||
author: Option<&str>,
|
||||
limit: usize,
|
||||
) -> ExitCode {
|
||||
match bibfetch::search(agent, title, author, limit, attempts) {
|
||||
Ok(found) => {
|
||||
let wrote = catena_cli::stream("bibfetch", &found);
|
||||
if format!("{wrote:?}") != format!("{:?}", ExitCode::from(exit::OK)) {
|
||||
return wrote;
|
||||
}
|
||||
if found.is_empty() {
|
||||
eprintln!("bibfetch: no register has {title:?}");
|
||||
return ExitCode::from(exit::NO);
|
||||
}
|
||||
ExitCode::from(exit::OK)
|
||||
}
|
||||
Err(why) => {
|
||||
eprintln!("bibfetch: {why}");
|
||||
ExitCode::from(match why {
|
||||
bibfetch::Error::Invalid(_) => exit::DATA,
|
||||
bibfetch::Error::Limited(_) => exit::TEMPFAIL,
|
||||
_ => exit::UNAVAILABLE,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The identifier and its kind out of one record.
|
||||
fn read_identifier(line: &str) -> Result<(String, String), String> {
|
||||
let value: serde_json::Value =
|
||||
|
||||
@@ -1,895 +0,0 @@
|
||||
//! Books by their title, for when there is no identifier to ask about.
|
||||
//!
|
||||
//! A book that arrives as a file often carries no ISBN at all, only a title
|
||||
//! and perhaps a name. A register can still be asked which books it has under
|
||||
//! those, and what it answers is a list of candidates: each an edition the
|
||||
//! register holds, with whatever identifier, publisher and cover it gives.
|
||||
//!
|
||||
//! Which candidate is the book in hand is not decided here. The candidates
|
||||
//! are what the register said, in the order it said them; deciding whether
|
||||
//! one is the same work is a comparison of titles, and that belongs to
|
||||
//! whoever holds the book.
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::source::{self, GoogleIdentifier, GoogleReply, SUBJECT_LIMIT, Source};
|
||||
use crate::{Error, Record};
|
||||
|
||||
/// The most candidates one search asks for. Google Books answers with no more
|
||||
/// than forty at a time, and a title that needs more than forty candidates to
|
||||
/// find the right one needs a better title.
|
||||
pub const MAX_LIMIT: usize = 40;
|
||||
|
||||
/// What Open Library's index is asked to say about each work, and about the
|
||||
/// one edition of it that best matches the search.
|
||||
///
|
||||
/// Of the work, only what describes the work: its publishers, ISBNs,
|
||||
/// languages, years and page counts are every edition's at once, and a
|
||||
/// record made from them would describe no edition. Not the work's own
|
||||
/// `cover_i` or `cover_edition_key` either: the cover the index chose for a
|
||||
/// work is often another edition's. The edition's `cover_i` is the first
|
||||
/// cover in that edition's record, which is what a lookup by ISBN gives as
|
||||
/// `cover_id`.
|
||||
const OPEN_LIBRARY_FIELDS: &str = "key,title,subtitle,author_name,subject,editions,editions.key,\
|
||||
editions.isbn,editions.publisher,editions.language,editions.publish_date,editions.cover_i";
|
||||
|
||||
/// Where Google Books is asked.
|
||||
const GOOGLE_BOOKS: &str = "https://www.googleapis.com/books/v1/volumes";
|
||||
|
||||
/// Ask which books a register has under a title, and by an author if one is
|
||||
/// named.
|
||||
///
|
||||
/// Open Library's index first, and Google Books only when that answered with
|
||||
/// nothing. Once Google Books has been asked, its failure is the answer:
|
||||
/// [`Error::Limited`] when it refused for its quota, [`Error::Unreachable`]
|
||||
/// when it could not be reached, since saying no register has the title
|
||||
/// would be a claim no register made. Asks nothing when `limit` is zero; a
|
||||
/// `limit` over [`MAX_LIMIT`] is taken as that.
|
||||
pub fn search(
|
||||
agent: &ureq::Agent,
|
||||
title: &str,
|
||||
author: Option<&str>,
|
||||
limit: usize,
|
||||
attempts: u32,
|
||||
) -> Result<Vec<Record>, Error> {
|
||||
search_with(title, author, limit, |url| get(agent, url, attempts))
|
||||
}
|
||||
|
||||
/// [`search`], with how to fetch an address passed in.
|
||||
fn search_with(
|
||||
title: &str,
|
||||
author: Option<&str>,
|
||||
limit: usize,
|
||||
mut get: impl FnMut(&str) -> Result<Option<String>, Error>,
|
||||
) -> Result<Vec<Record>, Error> {
|
||||
let title = title.trim();
|
||||
if title.is_empty() {
|
||||
return Err(Error::Invalid("no title to search for".into()));
|
||||
}
|
||||
let author = author.map(str::trim).filter(|author| !author.is_empty());
|
||||
let limit = limit.min(MAX_LIMIT);
|
||||
if limit == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut fetch = |url: String| match get(&url)? {
|
||||
Some(body) => Ok(body),
|
||||
// A search that matches nothing answers with an empty list. A 404 is
|
||||
// the address being wrong, not the book being absent.
|
||||
None => Err(Error::Unreachable(format!("{url}: answered 404"))),
|
||||
};
|
||||
|
||||
let open_library =
|
||||
fetch(open_library_url(title, author, limit)).and_then(|body| open_library(&body, limit));
|
||||
if open_library.as_ref().is_ok_and(|found| !found.is_empty()) {
|
||||
return open_library;
|
||||
}
|
||||
|
||||
let Some(url) = google_url(title, author, limit) else {
|
||||
// Nothing in the title Google could be asked by.
|
||||
return open_library;
|
||||
};
|
||||
let google = fetch(url).and_then(|body| google_books(&body, limit))?;
|
||||
if google.is_empty() {
|
||||
// Open Library's nothing, or its failure, which Google Books having
|
||||
// nothing either does not excuse.
|
||||
return open_library;
|
||||
}
|
||||
Ok(google)
|
||||
}
|
||||
|
||||
/// The body at an address, asked as a lookup asks, or `None` when the
|
||||
/// register says there is no such thing.
|
||||
///
|
||||
/// Except that Google Books refusing for its quota is [`Error::Limited`]: a
|
||||
/// `403` at once, since it is the day's quota and asking again within
|
||||
/// seconds will not move it, and a `429` once every attempt has been refused.
|
||||
fn get(agent: &ureq::Agent, url: &str, attempts: u32) -> Result<Option<String>, Error> {
|
||||
let attempts = attempts.max(1);
|
||||
let mut last = Error::Unreachable("no attempt was made".into());
|
||||
for attempt in 0..attempts {
|
||||
match agent.get(url).call() {
|
||||
Ok(mut response) => {
|
||||
return response
|
||||
.body_mut()
|
||||
.read_to_string()
|
||||
.map(Some)
|
||||
.map_err(|e| Error::Unreachable(e.to_string()));
|
||||
}
|
||||
Err(ureq::Error::StatusCode(404)) => return Ok(None),
|
||||
Err(ureq::Error::StatusCode(status)) => match refused(url, status) {
|
||||
Some(why) if status == 403 => return Err(why),
|
||||
Some(why) => last = why,
|
||||
None => last = Error::Unreachable(format!("{url}: answered {status}")),
|
||||
},
|
||||
Err(e) => last = Error::Unreachable(e.to_string()),
|
||||
}
|
||||
if attempt + 1 < attempts {
|
||||
std::thread::sleep(std::time::Duration::from_secs(1 << attempt.min(5)));
|
||||
}
|
||||
}
|
||||
Err(last)
|
||||
}
|
||||
|
||||
/// The refusal a status from a register is, when it is one for asking too
|
||||
/// often.
|
||||
fn refused(url: &str, status: u16) -> Option<Error> {
|
||||
(url.starts_with(GOOGLE_BOOKS) && matches!(status, 403 | 429)).then(|| {
|
||||
Error::Limited(format!(
|
||||
"Google Books refused ({status}): its quota for this address is spent for now"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
/// Where to ask Open Library.
|
||||
fn open_library_url(title: &str, author: Option<&str>, limit: usize) -> String {
|
||||
let author = author
|
||||
.map(|author| format!("&author={}", source::encode(author)))
|
||||
.unwrap_or_default();
|
||||
format!(
|
||||
"https://openlibrary.org/search.json?title={}{author}&fields={OPEN_LIBRARY_FIELDS}&limit={limit}",
|
||||
source::encode(title),
|
||||
)
|
||||
}
|
||||
|
||||
/// Where to ask Google Books, or `None` when there is nothing to ask by.
|
||||
///
|
||||
/// Every word is its own `intitle:` or `inauthor:` term, so no character of
|
||||
/// the title can open a phrase, negate a term or name another field: a word is
|
||||
/// the letters, digits and apostrophes in it, and anything else separates
|
||||
/// words.
|
||||
fn google_url(title: &str, author: Option<&str>, limit: usize) -> Option<String> {
|
||||
let terms: Vec<String> = words(title)
|
||||
.map(|word| format!("intitle:{word}"))
|
||||
.chain(
|
||||
author
|
||||
.into_iter()
|
||||
.flat_map(words)
|
||||
.map(|word| format!("inauthor:{word}")),
|
||||
)
|
||||
.collect();
|
||||
if !terms.iter().any(|term| term.starts_with("intitle:")) {
|
||||
return None;
|
||||
}
|
||||
Some(format!(
|
||||
"{GOOGLE_BOOKS}?q={}&maxResults={limit}",
|
||||
source::encode(&terms.join(" "))
|
||||
))
|
||||
}
|
||||
|
||||
fn words(text: &str) -> impl Iterator<Item = &str> {
|
||||
text.split(|c: char| !(c.is_alphanumeric() || c == '\''))
|
||||
.map(|word| word.trim_matches('\''))
|
||||
.filter(|word| !word.is_empty())
|
||||
}
|
||||
|
||||
/// The first ISBN in a list whose check digit agrees, in its thirteen-digit
|
||||
/// form. One written in thirteen digits is taken over one written in ten.
|
||||
fn first_isbn<'a>(isbns: impl Iterator<Item = &'a str> + Clone) -> Option<String> {
|
||||
isbns
|
||||
.clone()
|
||||
.filter(|isbn| idnorm::isbn::digits(isbn).len() == 13)
|
||||
.find_map(idnorm::isbn::to_13)
|
||||
.or_else(|| isbns.into_iter().find_map(idnorm::isbn::to_13))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct OpenLibraryReply {
|
||||
#[serde(default)]
|
||||
docs: Vec<Work>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Work {
|
||||
#[serde(default)]
|
||||
key: Option<String>,
|
||||
#[serde(default)]
|
||||
title: Option<String>,
|
||||
#[serde(default)]
|
||||
subtitle: Option<String>,
|
||||
#[serde(default)]
|
||||
author_name: Vec<String>,
|
||||
#[serde(default)]
|
||||
subject: Vec<String>,
|
||||
#[serde(default)]
|
||||
editions: Option<Editions>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Editions {
|
||||
#[serde(default)]
|
||||
docs: Vec<Edition>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Edition {
|
||||
#[serde(default)]
|
||||
key: Option<String>,
|
||||
#[serde(default)]
|
||||
isbn: Vec<String>,
|
||||
#[serde(default)]
|
||||
publisher: Vec<String>,
|
||||
#[serde(default)]
|
||||
language: Vec<String>,
|
||||
#[serde(default)]
|
||||
publish_date: Vec<String>,
|
||||
#[serde(default)]
|
||||
cover_i: Option<i64>,
|
||||
}
|
||||
|
||||
/// The last segment of an Open Library key: `OL3585423M` out of
|
||||
/// `/books/OL3585423M`. The letter at the end says what it names, `M` an
|
||||
/// edition and `W` a work.
|
||||
fn olid(key: &str) -> Option<String> {
|
||||
key.rsplit('/')
|
||||
.next()
|
||||
.filter(|id| !id.is_empty())
|
||||
.map(str::to_string)
|
||||
}
|
||||
|
||||
/// Read Open Library's answer into one candidate per work, in its order.
|
||||
///
|
||||
/// What belongs to an edition — its ISBN, publisher, language, year and
|
||||
/// cover — is taken from the edition the index chose as matching best, and
|
||||
/// left out when that edition does not give it; only the title, authors and
|
||||
/// subjects, which describe the work, are the work's. A work the index names
|
||||
/// no edition of is named by the work's key and says nothing an edition
|
||||
/// would.
|
||||
fn open_library(body: &str, limit: usize) -> Result<Vec<Record>, Error> {
|
||||
let reply: OpenLibraryReply =
|
||||
serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?;
|
||||
Ok(reply
|
||||
.docs
|
||||
.into_iter()
|
||||
.filter_map(candidate)
|
||||
.take(limit)
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn candidate(work: Work) -> Option<Record> {
|
||||
// The edition, and what it is named by: its ISBN, or its key when it has
|
||||
// none. One with neither cannot be named, and is as good as none.
|
||||
let edition = work
|
||||
.editions
|
||||
.and_then(|e| e.docs.into_iter().next())
|
||||
.and_then(|edition| {
|
||||
match first_isbn(edition.isbn.iter().map(String::as_str)) {
|
||||
Some(isbn) => Some((isbn, "isbn")),
|
||||
None => edition
|
||||
.key
|
||||
.as_deref()
|
||||
.and_then(olid)
|
||||
.map(|key| (key, "olid")),
|
||||
}
|
||||
.map(|named| (named, edition))
|
||||
});
|
||||
|
||||
let mut record = match edition {
|
||||
Some(((id, kind), edition)) => {
|
||||
let mut record = source::blank(&id, kind, Source::OpenLibrarySearch);
|
||||
record.container = edition.publisher.into_iter().next();
|
||||
record.language = edition.language.into_iter().next();
|
||||
record.year = edition
|
||||
.publish_date
|
||||
.iter()
|
||||
.find_map(|date| source::year_in(date));
|
||||
record.cover_id = edition
|
||||
.cover_i
|
||||
.and_then(|id| u64::try_from(id).ok())
|
||||
.filter(|id| *id > 0);
|
||||
record
|
||||
}
|
||||
// The work's own ISBNs, publishers and years are its editions', and
|
||||
// which belongs with which the index does not say.
|
||||
None => source::blank(
|
||||
// A work with no key cannot be named, so it is not a candidate.
|
||||
&work.key.as_deref().and_then(olid)?,
|
||||
"olid",
|
||||
Source::OpenLibrarySearch,
|
||||
),
|
||||
};
|
||||
record.title = match (work.title, work.subtitle) {
|
||||
(Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")),
|
||||
(title, _) => title,
|
||||
};
|
||||
record.authors = work.author_name;
|
||||
record.subjects = work.subject;
|
||||
record.subjects.truncate(SUBJECT_LIMIT);
|
||||
Some(record)
|
||||
}
|
||||
|
||||
/// Read Google Books' answer into one candidate per volume, in its order.
|
||||
fn google_books(body: &str, limit: usize) -> Result<Vec<Record>, Error> {
|
||||
let reply: GoogleReply = serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?;
|
||||
Ok(reply
|
||||
.items
|
||||
.into_iter()
|
||||
.filter_map(|item| {
|
||||
// Only what Google types as an ISBN: its `OTHER` identifiers are
|
||||
// library shelfmarks, some of which are ten digits long.
|
||||
let isbns = item
|
||||
.volume_info
|
||||
.identifiers
|
||||
.iter()
|
||||
.filter(|GoogleIdentifier { kind, .. }| kind == "ISBN_13" || kind == "ISBN_10")
|
||||
.map(|identifier| identifier.identifier.as_str());
|
||||
let (id, kind) = match first_isbn(isbns) {
|
||||
Some(isbn) => (isbn, "isbn"),
|
||||
None => (item.id.filter(|id| !id.is_empty())?, "googlebooks"),
|
||||
};
|
||||
Some(source::google_volume(&id, kind, item.volume_info))
|
||||
})
|
||||
.take(limit)
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Open Library's answer to `title=the first world war&author=howard`,
|
||||
/// recorded 2026-11-13 with `limit=2`, the ISBN and subject lists
|
||||
/// shortened, and with the work's own publishers, ISBNs, years and length,
|
||||
/// which are not asked for and are read past when they come.
|
||||
const HOWARD: &str = r#"{"numFound": 9, "start": 0, "numFoundExact": true, "num_found": 9,
|
||||
"documentation_url": "https://openlibrary.org/dev/docs/api/search", "q": "", "offset": null,
|
||||
"docs": [
|
||||
{"author_name": ["Michael Eliot Howard"], "first_publish_year": 2002,
|
||||
"isbn": ["9780192853622", "0192853627"], "key": "/works/OL1382726W",
|
||||
"language": ["eng"], "number_of_pages_median": 154,
|
||||
"publisher": ["Oxford University Press"], "title": "The First World War",
|
||||
"subject": ["World War, 1914-1918"],
|
||||
"editions": {"numFound": 1, "start": 0, "numFoundExact": true, "docs": [
|
||||
{"key": "/books/OL3585423M", "cover_i": 1123124, "language": ["eng"],
|
||||
"publisher": ["Oxford University Press"], "publish_date": ["2002"],
|
||||
"isbn": ["0192853627", "9780192853622"]}]}},
|
||||
{"author_name": ["Michael Howard"], "first_publish_year": 2007,
|
||||
"isbn": ["0199205590", "9780199205592"], "key": "/works/OL4282399W",
|
||||
"language": ["eng"], "publisher": ["Oxford University Press"],
|
||||
"subtitle": "A Very Short Introduction", "title": "The First World War",
|
||||
"editions": {"numFound": 1, "start": 0, "numFoundExact": true, "docs": [
|
||||
{"key": "/books/OL7403004M", "cover_i": 14606475, "language": ["eng"],
|
||||
"publisher": ["Oxford University Press"], "publish_date": ["20 May 2007"],
|
||||
"isbn": ["9780199205592", "0199205590"]}]},
|
||||
"subject": []}]}"#;
|
||||
|
||||
/// Open Library's answer to `title=war and peace&author=tolstoy`, recorded
|
||||
/// 2026-11-13, first work only. The edition the index matched best is a
|
||||
/// 1917 printing that has no ISBN, though other editions of the work do.
|
||||
const TOLSTOY: &str = r#"{"numFound": 146, "docs": [
|
||||
{"author_name": ["Leo Tolstoy"], "first_publish_year": 1867,
|
||||
"isbn": ["9781524881993", "1645178889"], "key": "/works/OL267171W",
|
||||
"title": "War and Peace",
|
||||
"editions": {"numFound": 487, "docs": [
|
||||
{"key": "/books/OL37863121M", "cover_i": 12736340, "language": ["eng"],
|
||||
"publisher": ["Thomas Y. Crowell"], "publish_date": ["1917"]}]}}]}"#;
|
||||
|
||||
/// Open Library's answer when nothing has the title.
|
||||
const NOTHING: &str = r#"{"numFound": 0, "start": 0, "numFoundExact": true, "num_found": 0,
|
||||
"documentation_url": "https://openlibrary.org/dev/docs/api/search", "q": "",
|
||||
"offset": null, "docs": []}"#;
|
||||
|
||||
/// Google Books' answer for a volume, in the shape its API documents.
|
||||
const GOOGLE: &str = r#"{"kind": "books#volumes", "totalItems": 2, "items": [
|
||||
{"kind": "books#volume", "id": "zyTCAlFPjgYC", "volumeInfo": {
|
||||
"title": "The Google Story", "authors": ["David A. Vise", "Mark Malseed"],
|
||||
"publisher": "Random House Digital, Inc.", "publishedDate": "2005-11-15",
|
||||
"industryIdentifiers": [
|
||||
{"type": "ISBN_10", "identifier": "055380457X"},
|
||||
{"type": "ISBN_13", "identifier": "9780553804577"}],
|
||||
"pageCount": 207, "categories": ["Browsers (Computer programs)"],
|
||||
"language": "en"}},
|
||||
{"kind": "books#volume", "id": "a1b2C3d4", "volumeInfo": {
|
||||
"title": "The Google Story", "subtitle": "A Dissertation",
|
||||
"publishedDate": "2009",
|
||||
"industryIdentifiers": [
|
||||
{"type": "OTHER", "identifier": "UOM:0306406152"}]}}]}"#;
|
||||
|
||||
/// Fetch from a table of recorded bodies, keeping every address asked.
|
||||
fn recorded<'a>(
|
||||
asked: &'a mut Vec<String>,
|
||||
bodies: &'a [(&'a str, Result<&'a str, Error>)],
|
||||
) -> impl FnMut(&str) -> Result<Option<String>, Error> + 'a {
|
||||
move |url| {
|
||||
asked.push(url.to_string());
|
||||
let (_, body) = bodies
|
||||
.iter()
|
||||
.find(|(host, _)| url.contains(host))
|
||||
.unwrap_or_else(|| panic!("asked an address nothing was recorded for: {url}"));
|
||||
body.clone().map(|body| Some(body.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
fn down() -> Error {
|
||||
Error::Unreachable("connection refused".into())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_work_open_library_has_is_a_candidate_named_by_its_isbn() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [("openlibrary.org", Ok(HOWARD))];
|
||||
let found = search_with(
|
||||
"The First World War",
|
||||
Some("Howard"),
|
||||
5,
|
||||
recorded(&mut asked, &bodies),
|
||||
)
|
||||
.expect("candidates");
|
||||
assert_eq!(
|
||||
asked.len(),
|
||||
1,
|
||||
"Google is not asked when Open Library answers"
|
||||
);
|
||||
|
||||
assert_eq!(found.len(), 2);
|
||||
let first = &found[0];
|
||||
assert!(first.found);
|
||||
assert_eq!(first.v, corpus_core::V);
|
||||
assert_eq!(first.source, "openlibrary");
|
||||
// Written ten-digit first by the edition, given in thirteen.
|
||||
assert_eq!(
|
||||
(first.id.as_str(), first.kind.as_str()),
|
||||
("9780192853622", "isbn")
|
||||
);
|
||||
assert_eq!(first.title.as_deref(), Some("The First World War"));
|
||||
assert_eq!(first.authors, ["Michael Eliot Howard"]);
|
||||
assert_eq!(first.year, Some(2002));
|
||||
assert_eq!(first.container.as_deref(), Some("Oxford University Press"));
|
||||
assert_eq!(first.language.as_deref(), Some("eng"));
|
||||
assert_eq!(first.subjects, ["World War, 1914-1918"]);
|
||||
// The work's median length is no edition's.
|
||||
assert_eq!(first.pages, None);
|
||||
assert_eq!(first.cover_id, Some(1123124));
|
||||
|
||||
let second = &found[1];
|
||||
assert_eq!(second.id, "9780199205592");
|
||||
assert_eq!(
|
||||
second.title.as_deref(),
|
||||
Some("The First World War: A Very Short Introduction")
|
||||
);
|
||||
// The edition's year, not the work's first.
|
||||
assert_eq!(second.year, Some(2007));
|
||||
// The first cover in OL7403004M's own record; the work's cover is
|
||||
// 1137779, which is not.
|
||||
assert_eq!(second.cover_id, Some(14606475));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_candidates_stay_in_the_order_the_register_gave_them() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [("openlibrary.org", Ok(HOWARD))];
|
||||
let found =
|
||||
search_with("first world war", None, 5, recorded(&mut asked, &bodies)).expect("ok");
|
||||
let ids: Vec<&str> = found.iter().map(|r| r.id.as_str()).collect();
|
||||
assert_eq!(ids, ["9780192853622", "9780199205592"]);
|
||||
let found =
|
||||
search_with("first world war", None, 1, recorded(&mut asked, &bodies)).expect("ok");
|
||||
assert_eq!(found.len(), 1);
|
||||
assert_eq!(found[0].id, "9780192853622");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_edition_with_no_isbn_is_named_by_its_key_and_keeps_its_cover() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [("openlibrary.org", Ok(TOLSTOY))];
|
||||
let found = search_with(
|
||||
"War and Peace",
|
||||
Some("Tolstoy"),
|
||||
5,
|
||||
recorded(&mut asked, &bodies),
|
||||
)
|
||||
.expect("candidates");
|
||||
let only = &found[0];
|
||||
// Not 9781524881993: that is another edition's, and the publisher,
|
||||
// year and cover here are this one's.
|
||||
assert_eq!(
|
||||
(only.id.as_str(), only.kind.as_str()),
|
||||
("OL37863121M", "olid")
|
||||
);
|
||||
assert_eq!(only.container.as_deref(), Some("Thomas Y. Crowell"));
|
||||
assert_eq!(only.year, Some(1917));
|
||||
assert_eq!(only.cover_id, Some(12736340));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_the_edition_does_not_give_is_left_out_not_taken_from_the_work() {
|
||||
let body = r#"{"docs": [{"key": "/works/OL267171W", "title": "War and Peace",
|
||||
"author_name": ["Leo Tolstoy"], "subject": ["Napoleonic Wars"],
|
||||
"first_publish_year": 1867, "publisher": ["Crowell"], "language": ["eng"],
|
||||
"number_of_pages_median": 1296,
|
||||
"editions": {"docs": [{"key": "/books/OL1M", "isbn": ["9780199232765"],
|
||||
"cover_i": 5}]}}]}"#;
|
||||
let found = open_library(body, 5).expect("candidates");
|
||||
let only = &found[0];
|
||||
assert_eq!(
|
||||
(only.id.as_str(), only.kind.as_str()),
|
||||
("9780199232765", "isbn")
|
||||
);
|
||||
assert_eq!(only.cover_id, Some(5));
|
||||
// 1867 and Crowell are other editions'.
|
||||
assert_eq!(only.year, None);
|
||||
assert_eq!(only.container, None);
|
||||
assert_eq!(only.language, None);
|
||||
assert_eq!(only.pages, None);
|
||||
// What describes the work is the work's.
|
||||
assert_eq!(only.title.as_deref(), Some("War and Peace"));
|
||||
assert_eq!(only.authors, ["Leo Tolstoy"]);
|
||||
assert_eq!(only.subjects, ["Napoleonic Wars"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_work_with_no_edition_named_is_named_by_its_key_and_says_nothing_an_edition_would() {
|
||||
let work = r#""key": "/works/OL1W", "title": "A Book", "author_name": ["A. Writer"],
|
||||
"isbn": ["0-306-40615-2"], "publisher": ["Plenum"], "first_publish_year": 1981,
|
||||
"language": ["eng"], "number_of_pages_median": 300"#;
|
||||
// No edition, and an edition with neither an ISBN nor a key.
|
||||
for editions in [
|
||||
"",
|
||||
r#", "editions": {"docs": [{"publisher": ["Plenum"], "cover_i": 7}]}"#,
|
||||
] {
|
||||
let body = format!(r#"{{"docs": [{{{work}{editions}}}]}}"#);
|
||||
let found = open_library(&body, 5).expect("candidates");
|
||||
let only = &found[0];
|
||||
// Not 9780306406157: which edition that is, the index does not say.
|
||||
assert_eq!(
|
||||
(only.id.as_str(), only.kind.as_str()),
|
||||
("OL1W", "olid"),
|
||||
"{body}"
|
||||
);
|
||||
assert_eq!(only.title.as_deref(), Some("A Book"));
|
||||
assert_eq!(only.authors, ["A. Writer"]);
|
||||
assert_eq!(
|
||||
(
|
||||
only.year,
|
||||
only.container.as_deref(),
|
||||
only.language.as_deref(),
|
||||
only.pages,
|
||||
only.cover_id
|
||||
),
|
||||
(None, None, None, None, None),
|
||||
"{body}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_work_with_no_isbn_and_no_edition_is_named_by_the_works_key() {
|
||||
let body = r#"{"docs": [{"key": "/works/OL267171W", "title": "War and Peace"}]}"#;
|
||||
let found = open_library(body, 5).expect("candidates");
|
||||
assert_eq!(
|
||||
(found[0].id.as_str(), found[0].kind.as_str()),
|
||||
("OL267171W", "olid")
|
||||
);
|
||||
assert_eq!(found[0].cover_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn isbns_whose_check_digit_is_wrong_are_passed_over() {
|
||||
// 9780306406158 and 0306406153 are 9780306406157 mistyped.
|
||||
let body = r#"{"docs": [
|
||||
{"key": "/works/OL1W", "title": "A", "editions": {"docs": [
|
||||
{"key": "/books/OL1M", "isbn": ["9780306406158", "0306406153"], "cover_i": 7}]}},
|
||||
{"key": "/works/OL2W", "title": "B", "editions": {"docs": [
|
||||
{"key": "/books/OL2M", "isbn": ["9780306406158", "0306406152"]}]}}]}"#;
|
||||
let found = open_library(body, 5).expect("candidates");
|
||||
assert_eq!(
|
||||
(found[0].id.as_str(), found[0].kind.as_str()),
|
||||
("OL1M", "olid")
|
||||
);
|
||||
assert_eq!(found[0].cover_id, Some(7));
|
||||
// The ten-digit one reads, and is given in thirteen.
|
||||
assert_eq!(
|
||||
(found[1].id.as_str(), found[1].kind.as_str()),
|
||||
("9780306406157", "isbn")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thirteen_digits_are_taken_over_ten() {
|
||||
// Two different ISBNs on one edition: the thirteen-digit one wins
|
||||
// although the ten-digit one is listed first.
|
||||
let isbns = ["0306406152", "9780199205592"];
|
||||
assert_eq!(
|
||||
first_isbn(isbns.into_iter()).as_deref(),
|
||||
Some("9780199205592")
|
||||
);
|
||||
assert_eq!(first_isbn(["nonsense"].into_iter()), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_from_open_library_is_asked_of_google_books() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [
|
||||
("openlibrary.org", Ok(NOTHING)),
|
||||
("googleapis.com", Ok(GOOGLE)),
|
||||
];
|
||||
let found = search_with(
|
||||
"The Google Story",
|
||||
Some("Vise"),
|
||||
5,
|
||||
recorded(&mut asked, &bodies),
|
||||
)
|
||||
.expect("candidates");
|
||||
assert_eq!(asked.len(), 2);
|
||||
assert!(asked[1].starts_with("https://www.googleapis.com/books/v1/volumes?q="));
|
||||
|
||||
assert_eq!(found.len(), 2);
|
||||
let first = &found[0];
|
||||
assert_eq!(first.source, "googlebooks");
|
||||
assert_eq!(
|
||||
(first.id.as_str(), first.kind.as_str()),
|
||||
("9780553804577", "isbn")
|
||||
);
|
||||
assert_eq!(first.authors, ["David A. Vise", "Mark Malseed"]);
|
||||
assert_eq!(first.year, Some(2005));
|
||||
assert_eq!(
|
||||
first.container.as_deref(),
|
||||
Some("Random House Digital, Inc.")
|
||||
);
|
||||
assert_eq!(first.pages, Some(207));
|
||||
// Google Books knows nothing of Open Library's covers.
|
||||
assert_eq!(first.cover_id, None);
|
||||
|
||||
// A shelfmark is not an ISBN, though its digits would pass for one.
|
||||
let second = &found[1];
|
||||
assert_eq!(
|
||||
(second.id.as_str(), second.kind.as_str()),
|
||||
("a1b2C3d4", "googlebooks")
|
||||
);
|
||||
assert_eq!(
|
||||
second.title.as_deref(),
|
||||
Some("The Google Story: A Dissertation")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_from_either_register_is_no_candidates() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [
|
||||
("openlibrary.org", Ok(NOTHING)),
|
||||
(
|
||||
"googleapis.com",
|
||||
Ok(r#"{"kind": "books#volumes", "totalItems": 0}"#),
|
||||
),
|
||||
];
|
||||
let found =
|
||||
search_with("Nothing", None, 5, recorded(&mut asked, &bodies)).expect("an answer");
|
||||
assert!(found.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn google_books_down_after_open_library_had_nothing_is_a_failure() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [
|
||||
("openlibrary.org", Ok(NOTHING)),
|
||||
("googleapis.com", Err(down())),
|
||||
];
|
||||
let failed = search_with("Nothing", None, 5, recorded(&mut asked, &bodies));
|
||||
assert!(matches!(failed, Err(Error::Unreachable(_))), "{failed:?}");
|
||||
assert_eq!(asked.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn google_books_refusing_for_its_quota_is_said_to_be_that() {
|
||||
let mut asked = Vec::new();
|
||||
let limited = refused(&google_url("Nothing", None, 5).expect("a url"), 429);
|
||||
let bodies = [
|
||||
("openlibrary.org", Ok(NOTHING)),
|
||||
("googleapis.com", Err(limited.expect("a refusal"))),
|
||||
];
|
||||
let failed = search_with("Nothing", None, 5, recorded(&mut asked, &bodies));
|
||||
assert!(matches!(failed, Err(Error::Limited(_))), "{failed:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_google_books_quota_statuses_are_refusals() {
|
||||
let google = google_url("Anathem", None, 5).expect("a url");
|
||||
for status in [403, 429] {
|
||||
assert!(
|
||||
matches!(refused(&google, status), Some(Error::Limited(_))),
|
||||
"{status}"
|
||||
);
|
||||
}
|
||||
for status in [400, 500, 503] {
|
||||
assert!(refused(&google, status).is_none(), "{status}");
|
||||
}
|
||||
let open_library = open_library_url("Anathem", None, 5);
|
||||
for status in [403, 429] {
|
||||
assert!(refused(&open_library, status).is_none(), "{status}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_library_down_is_asked_of_google_books() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [
|
||||
("openlibrary.org", Err(down())),
|
||||
("googleapis.com", Ok(GOOGLE)),
|
||||
];
|
||||
let found =
|
||||
search_with("The Google Story", None, 5, recorded(&mut asked, &bodies)).expect("ok");
|
||||
assert_eq!(found[0].source, "googlebooks");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_register_down_is_a_failure() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [
|
||||
("openlibrary.org", Err(down())),
|
||||
("googleapis.com", Err(down())),
|
||||
];
|
||||
let failed = search_with("A Book", None, 5, recorded(&mut asked, &bodies));
|
||||
assert!(matches!(failed, Err(Error::Unreachable(_))), "{failed:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_that_cannot_be_read_says_so() {
|
||||
assert!(matches!(open_library("<html>", 5), Err(Error::Shape(_))));
|
||||
assert!(matches!(google_books("<html>", 5), Err(Error::Shape(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_author_is_left_out_when_there_is_none() {
|
||||
for author in [None, Some(""), Some(" ")] {
|
||||
let url = open_library_url(
|
||||
"Anathem",
|
||||
author.map(str::trim).filter(|a| !a.is_empty()),
|
||||
5,
|
||||
);
|
||||
assert!(!url.contains("author="), "{url}");
|
||||
}
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [
|
||||
("openlibrary.org", Ok(NOTHING)),
|
||||
("googleapis.com", Ok(GOOGLE)),
|
||||
];
|
||||
search_with("Anathem", Some(" "), 5, recorded(&mut asked, &bodies)).expect("ok");
|
||||
assert!(!asked[0].contains("author="), "{}", asked[0]);
|
||||
assert!(!asked[1].contains("inauthor"), "{}", asked[1]);
|
||||
assert!(asked[1].contains("q=intitle%3AAnathem&"), "{}", asked[1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_author_narrows_the_search_when_there_is_one() {
|
||||
let url = open_library_url("Anathem", Some("Neal Stephenson"), 5);
|
||||
assert!(
|
||||
url.contains("?title=Anathem&author=Neal%20Stephenson&fields="),
|
||||
"{url}"
|
||||
);
|
||||
let url = google_url("Anathem", Some("Neal Stephenson"), 5).expect("a url");
|
||||
assert!(
|
||||
url.contains("q=intitle%3AAnathem%20inauthor%3ANeal%20inauthor%3AStephenson&"),
|
||||
"{url}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_title_cannot_change_the_shape_of_the_address() {
|
||||
let hostile = r#"Fish & Chips: "A History" #2?=x/y+z"#;
|
||||
for url in [
|
||||
open_library_url(hostile, Some("O'Brien & Sons"), 5),
|
||||
google_url(hostile, Some("O'Brien & Sons"), 5).expect("a url"),
|
||||
] {
|
||||
assert_eq!(url.matches('?').count(), 1, "{url}");
|
||||
assert_eq!(url.matches('#').count(), 0, "{url}");
|
||||
assert_eq!(url.matches('"').count(), 0, "{url}");
|
||||
assert_eq!(url.matches('+').count(), 0, "{url}");
|
||||
let query = url.split_once('?').expect("a query").1;
|
||||
let names: Vec<&str> = query
|
||||
.split('&')
|
||||
.map(|pair| pair.split_once('=').expect("name=value").0)
|
||||
.collect();
|
||||
assert!(
|
||||
names == ["title", "author", "fields", "limit"] || names == ["q", "maxResults"],
|
||||
"{url}"
|
||||
);
|
||||
}
|
||||
// Open Library is asked the title as written.
|
||||
let url = open_library_url(hostile, None, 5);
|
||||
assert!(
|
||||
url.contains("title=Fish%20%26%20Chips%3A%20%22A%20History%22%20%232%3F%3Dx%2Fy%2Bz&"),
|
||||
"{url}"
|
||||
);
|
||||
// Google is asked word by word, so nothing in a title is syntax.
|
||||
let url = google_url(hostile, Some("O'Brien & Sons"), 5).expect("a url");
|
||||
let q = "intitle:Fish intitle:Chips intitle:A intitle:History intitle:2 intitle:x \
|
||||
intitle:y intitle:z inauthor:O'Brien inauthor:Sons";
|
||||
assert!(url.contains(&format!("q={}&", source::encode(q))), "{url}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_title_outside_ascii_is_sent_as_utf8() {
|
||||
let url = open_library_url("Преступление и наказание", Some("Dostoevský"), 5);
|
||||
assert!(
|
||||
url.contains("title=%D0%9F%D1%80%D0%B5%D1%81%D1%82%D1%83%D0%BF"),
|
||||
"{url}"
|
||||
);
|
||||
assert!(url.contains("author=Dostoevsk%C3%BD&"), "{url}");
|
||||
let url = google_url("Преступление и наказание", None, 5).expect("a url");
|
||||
// Letters in any script are words.
|
||||
assert!(url.contains("intitle%3A%D0%B8%20"), "{url}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_title_with_no_words_is_not_asked_of_google_books() {
|
||||
assert_eq!(google_url("?!", Some("Someone"), 5), None);
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [("openlibrary.org", Ok(NOTHING))];
|
||||
let found = search_with("?!", Some("Someone"), 5, recorded(&mut asked, &bodies))
|
||||
.expect("an answer");
|
||||
assert!(found.is_empty());
|
||||
assert_eq!(asked.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_title_is_refused_without_asking() {
|
||||
let mut asked = Vec::new();
|
||||
let failed = search_with(" ", Some("Someone"), 5, recorded(&mut asked, &[]));
|
||||
assert!(matches!(failed, Err(Error::Invalid(_))), "{failed:?}");
|
||||
assert!(asked.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_limit_is_asked_for_and_held_to() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [
|
||||
("openlibrary.org", Ok(NOTHING)),
|
||||
("googleapis.com", Ok(GOOGLE)),
|
||||
];
|
||||
let found =
|
||||
search_with("The Google Story", None, 1, recorded(&mut asked, &bodies)).expect("ok");
|
||||
assert_eq!(found.len(), 1);
|
||||
assert!(asked[0].ends_with("&limit=1"), "{}", asked[0]);
|
||||
assert!(asked[1].ends_with("&maxResults=1"), "{}", asked[1]);
|
||||
|
||||
let mut asked = Vec::new();
|
||||
search_with("The Google Story", None, 500, recorded(&mut asked, &bodies)).expect("ok");
|
||||
assert!(asked[0].ends_with("&limit=40"), "{}", asked[0]);
|
||||
assert!(asked[1].ends_with("&maxResults=40"), "{}", asked[1]);
|
||||
|
||||
let mut asked = Vec::new();
|
||||
let found =
|
||||
search_with("The Google Story", None, 0, recorded(&mut asked, &bodies)).expect("ok");
|
||||
assert!(found.is_empty());
|
||||
assert!(asked.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_index_is_not_asked_for_the_works_cover() {
|
||||
let url = open_library_url("A", None, 5);
|
||||
let fields = url
|
||||
.split('&')
|
||||
.find_map(|pair| pair.strip_prefix("fields="))
|
||||
.expect("fields");
|
||||
let fields: Vec<&str> = fields.split(',').collect();
|
||||
assert!(fields.contains(&"editions.cover_i"));
|
||||
assert!(!fields.contains(&"cover_i"));
|
||||
assert!(!fields.contains(&"cover_edition_key"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_404_from_a_search_is_a_failure_and_not_an_absence() {
|
||||
let failed = search_with("A Book", None, 5, |_| Ok(None));
|
||||
assert!(matches!(failed, Err(Error::Unreachable(_))), "{failed:?}");
|
||||
}
|
||||
}
|
||||
@@ -117,7 +117,7 @@ number_of_pages_median&limit=1",
|
||||
///
|
||||
/// A DOI suffix may hold a slash, a question mark or a hash, and every one of
|
||||
/// those means something else in a URL.
|
||||
pub(crate) fn encode(text: &str) -> String {
|
||||
fn encode(text: &str) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
for byte in text.bytes() {
|
||||
match byte {
|
||||
@@ -131,7 +131,7 @@ pub(crate) fn encode(text: &str) -> String {
|
||||
}
|
||||
|
||||
/// A record with the fields nothing found fills in.
|
||||
pub(crate) fn blank(id: &str, kind: &str, source: Source) -> Record {
|
||||
fn blank(id: &str, kind: &str, source: Source) -> Record {
|
||||
let mut record = crate::missing(id, kind, source);
|
||||
record.found = true;
|
||||
record
|
||||
@@ -320,7 +320,7 @@ struct SearchDoc {
|
||||
/// The index lists every heading every edition has ever been given, which on a
|
||||
/// well-known book is hundreds and on Hamlet is thousands. A shelf is filed
|
||||
/// from a handful, and the first are the broadest.
|
||||
pub(crate) const SUBJECT_LIMIT: usize = 12;
|
||||
const SUBJECT_LIMIT: usize = 12;
|
||||
|
||||
/// A search that matched nothing answers with no documents rather than a
|
||||
/// status, so an empty reply is the absence and not a failure.
|
||||
@@ -348,22 +348,19 @@ fn open_library_search(id: &str, kind: &str, body: &str) -> Result<Record, Error
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(crate) struct GoogleReply {
|
||||
struct GoogleReply {
|
||||
#[serde(default)]
|
||||
pub(crate) items: Vec<GoogleItem>,
|
||||
items: Vec<GoogleItem>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(crate) struct GoogleItem {
|
||||
/// Google's own key for the volume.
|
||||
#[serde(default)]
|
||||
pub(crate) id: Option<String>,
|
||||
struct GoogleItem {
|
||||
#[serde(rename = "volumeInfo", default)]
|
||||
pub(crate) volume_info: GoogleVolume,
|
||||
volume_info: GoogleVolume,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Default)]
|
||||
pub(crate) struct GoogleVolume {
|
||||
struct GoogleVolume {
|
||||
#[serde(default)]
|
||||
title: Option<String>,
|
||||
#[serde(default)]
|
||||
@@ -382,18 +379,6 @@ pub(crate) struct GoogleVolume {
|
||||
language: Option<String>,
|
||||
#[serde(rename = "pageCount", default)]
|
||||
page_count: Option<u32>,
|
||||
/// `[{"type": "ISBN_13", "identifier": "…"}]`, alongside library
|
||||
/// shelfmarks typed `OTHER`.
|
||||
#[serde(rename = "industryIdentifiers", default)]
|
||||
pub(crate) identifiers: Vec<GoogleIdentifier>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(crate) struct GoogleIdentifier {
|
||||
#[serde(rename = "type", default)]
|
||||
pub(crate) kind: String,
|
||||
#[serde(default)]
|
||||
pub(crate) identifier: String,
|
||||
}
|
||||
|
||||
/// A search that matched nothing answers with no items rather than a status,
|
||||
@@ -403,11 +388,8 @@ fn google_books(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
let Some(item) = reply.items.into_iter().next() else {
|
||||
return Ok(crate::missing(id, kind, Source::GoogleBooks));
|
||||
};
|
||||
Ok(google_volume(id, kind, item.volume_info))
|
||||
}
|
||||
let volume = item.volume_info;
|
||||
|
||||
/// One Google Books volume as a record.
|
||||
pub(crate) fn google_volume(id: &str, kind: &str, volume: GoogleVolume) -> Record {
|
||||
let mut record = blank(id, kind, Source::GoogleBooks);
|
||||
record.title = match (volume.title, volume.subtitle) {
|
||||
(Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")),
|
||||
@@ -420,11 +402,11 @@ pub(crate) fn google_volume(id: &str, kind: &str, volume: GoogleVolume) -> Recor
|
||||
record.subjects = volume.categories;
|
||||
record.language = volume.language;
|
||||
record.pages = volume.page_count;
|
||||
record
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
/// The first four-digit year in a string.
|
||||
pub(crate) fn year_in(text: &str) -> Option<i64> {
|
||||
fn year_in(text: &str) -> Option<i64> {
|
||||
let bytes: Vec<char> = text.chars().collect();
|
||||
for window in bytes.windows(4) {
|
||||
if window.iter().all(char::is_ascii_digit) {
|
||||
|
||||
@@ -21,8 +21,6 @@ from ._catena import (
|
||||
compose_subjects,
|
||||
count_tokens,
|
||||
cue_windows,
|
||||
Limited,
|
||||
fetch_cover,
|
||||
fetch_identifier,
|
||||
find_citations,
|
||||
fuse_hits,
|
||||
@@ -43,7 +41,6 @@ from ._catena import (
|
||||
record_severity,
|
||||
same_work,
|
||||
scrub,
|
||||
search_title,
|
||||
segment_cues,
|
||||
series_of,
|
||||
shelve,
|
||||
@@ -56,7 +53,6 @@ from ._catena import (
|
||||
|
||||
__all__ = [
|
||||
"__version__",
|
||||
"Limited",
|
||||
"canonical_subject",
|
||||
"chunk_markdown",
|
||||
"citation_prompt",
|
||||
@@ -66,7 +62,6 @@ __all__ = [
|
||||
"compose_subjects",
|
||||
"count_tokens",
|
||||
"cue_windows",
|
||||
"fetch_cover",
|
||||
"fetch_identifier",
|
||||
"find_citations",
|
||||
"fuse_hits",
|
||||
@@ -87,7 +82,6 @@ __all__ = [
|
||||
"record_severity",
|
||||
"same_work",
|
||||
"scrub",
|
||||
"search_title",
|
||||
"segment_cues",
|
||||
"series_of",
|
||||
"shelve",
|
||||
|
||||
@@ -103,40 +103,6 @@ def fetch_identifier(
|
||||
"""
|
||||
...
|
||||
|
||||
class Limited(Exception):
|
||||
"""A register refused because it has been asked too often; ask again later."""
|
||||
|
||||
def fetch_cover(
|
||||
isbn: str,
|
||||
cover_id: int | None = ...,
|
||||
size: str = ...,
|
||||
polite: str | None = ...,
|
||||
timeout: int = ...,
|
||||
deadline: int = ...,
|
||||
) -> dict[str, Any] | None:
|
||||
"""The cover Open Library has for a book, by ISBN, then by cover id.
|
||||
|
||||
A dict with `bytes`, `media_type`, `extension`, `source` and `url`, or
|
||||
`None` when there is none. Raises `Limited` when Open Library refused for
|
||||
being asked too often. Blocks on the network.
|
||||
"""
|
||||
...
|
||||
|
||||
def search_title(
|
||||
title: str,
|
||||
author: str | None = ...,
|
||||
limit: int = ...,
|
||||
polite: str | None = ...,
|
||||
timeout: int = ...,
|
||||
attempts: int = ...,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""The editions the registers list for a title and author, in their order.
|
||||
|
||||
Records shaped like `fetch_identifier`'s. Use `same_work` to decide which
|
||||
one is the book. Blocks on the network.
|
||||
"""
|
||||
...
|
||||
|
||||
def colophon_prompt(
|
||||
record: dict[str, Any], text: str, pages: int = ...
|
||||
) -> str | None:
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
use pyo3::exceptions::PyValueError;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyBytes, PyDict, PyList};
|
||||
use pyo3::types::{PyDict, PyList};
|
||||
|
||||
use crate::{ChunkBudgets, CueBudgets};
|
||||
|
||||
@@ -434,9 +434,6 @@ fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
module.add_function(wrap_pyfunction!(read_colophon, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(shelve, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(fetch_identifier, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(fetch_cover, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(search_title, module)?)?;
|
||||
module.add("Limited", module.py().get_type::<Limited>())?;
|
||||
module.add_function(wrap_pyfunction!(colophon_prompt, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(merge_colophon, module)?)?;
|
||||
Ok(())
|
||||
@@ -515,112 +512,13 @@ fn fetch_identifier<'py>(
|
||||
// would stop every other thread in the interpreter for the whole request.
|
||||
let found = py
|
||||
.detach(|| bibfetch::ask::lookup(&agent, id, kind, ask.attempts))
|
||||
.map_err(register_error)?;
|
||||
.map_err(|e| PyValueError::new_err(e.to_string()))?;
|
||||
match found {
|
||||
Some(record) => Ok(Some(to_py(py, &record)?)),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pyo3::create_exception!(
|
||||
_catena,
|
||||
Limited,
|
||||
pyo3::exceptions::PyException,
|
||||
"A register refused because it has been asked too often; ask again later."
|
||||
);
|
||||
|
||||
/// A register's error as the Python exception that says what to do about it.
|
||||
fn register_error(error: bibfetch::Error) -> PyErr {
|
||||
match error {
|
||||
bibfetch::Error::Limited(why) => Limited::new_err(why),
|
||||
other => PyValueError::new_err(other.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The cover Open Library has for a book, by its ISBN, then by the cover id
|
||||
/// a record named.
|
||||
///
|
||||
/// Returns a dict with `bytes`, `media_type` (`image/jpeg` or `image/png`),
|
||||
/// `extension`, `source` and `url`, or `None` when there is no cover. Covers
|
||||
/// asked by ISBN are kept three seconds apart across the process, and the
|
||||
/// whole fetch stops after `deadline` seconds. Raises `catena.Limited` when
|
||||
/// Open Library refused for being asked too often, and `ValueError` when the
|
||||
/// ISBN is not one or the store could not be reached. Blocks on the network.
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (isbn, cover_id = None, size = "large", polite = None, timeout = 20, deadline = 120))]
|
||||
fn fetch_cover<'py>(
|
||||
py: Python<'py>,
|
||||
isbn: &str,
|
||||
cover_id: Option<u64>,
|
||||
size: &str,
|
||||
polite: Option<String>,
|
||||
timeout: u64,
|
||||
deadline: u64,
|
||||
) -> PyResult<Option<Bound<'py, PyDict>>> {
|
||||
let size = match size {
|
||||
"small" => bibfetch::CoverSize::Small,
|
||||
"medium" => bibfetch::CoverSize::Medium,
|
||||
"large" => bibfetch::CoverSize::Large,
|
||||
other => {
|
||||
return Err(PyValueError::new_err(format!(
|
||||
"size is small, medium or large, not {other:?}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
let agent = bibfetch::ask::Ask {
|
||||
polite,
|
||||
timeout: std::time::Duration::from_secs(timeout.max(1)),
|
||||
..Default::default()
|
||||
}
|
||||
.agent();
|
||||
let covers = bibfetch::cover::Covers {
|
||||
deadline: std::time::Duration::from_secs(deadline.max(1)),
|
||||
..Default::default()
|
||||
};
|
||||
let found = py
|
||||
.detach(|| covers.fetch(&agent, isbn, cover_id, size))
|
||||
.map_err(register_error)?;
|
||||
let Some(cover) = found else {
|
||||
return Ok(None);
|
||||
};
|
||||
let out = PyDict::new(py);
|
||||
out.set_item("bytes", PyBytes::new(py, &cover.bytes))?;
|
||||
out.set_item("media_type", cover.media_type)?;
|
||||
out.set_item("extension", cover.extension())?;
|
||||
out.set_item("source", cover.source)?;
|
||||
out.set_item("url", cover.url)?;
|
||||
Ok(Some(out))
|
||||
}
|
||||
|
||||
/// The editions the registers list for a title, and an author if one is known.
|
||||
///
|
||||
/// Open Library is asked first and Google Books only when it has nothing.
|
||||
/// The candidates come back in the register's own order, as records shaped
|
||||
/// like `fetch_identifier`'s; nothing here decides which is the book, which is
|
||||
/// what `same_work` is for. Blocks on the network.
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (title, author = None, limit = 10, polite = None, timeout = 20, attempts = 3))]
|
||||
fn search_title<'py>(
|
||||
py: Python<'py>,
|
||||
title: &str,
|
||||
author: Option<&str>,
|
||||
limit: usize,
|
||||
polite: Option<String>,
|
||||
timeout: u64,
|
||||
attempts: u32,
|
||||
) -> PyResult<Bound<'py, PyList>> {
|
||||
let ask = bibfetch::ask::Ask {
|
||||
polite,
|
||||
timeout: std::time::Duration::from_secs(timeout.max(1)),
|
||||
attempts: attempts.max(1),
|
||||
};
|
||||
let agent = ask.agent();
|
||||
let found = py
|
||||
.detach(|| bibfetch::search(&agent, title, author, limit, ask.attempts))
|
||||
.map_err(register_error)?;
|
||||
list_to_py(py, &found)
|
||||
}
|
||||
|
||||
/// The question to put to a model about the fields a book's pages did not
|
||||
/// settle, or `None` when they settled everything.
|
||||
///
|
||||
|
||||
@@ -99,10 +99,8 @@ A Kindle book's title is EXTH 503 where it is there, then the MOBI header's
|
||||
full name, then the Palm database name (cut to 31 bytes, with a note). Authors
|
||||
(100) and subjects (105) are every record of their kind, in order; publisher
|
||||
(101), description (103), date (106) and language (524) the first. EXTH 104 is
|
||||
an `isbn` when it is shaped like one and `other` when not. 113 is filed by
|
||||
its shape: a `uuid` when it is one (calibre writes its book id there), an
|
||||
`asin` when it is ten letters and digits, and `other` when it is neither. A
|
||||
contributor (108) is not an author, the same as an EPUB's
|
||||
an `isbn` when it is shaped like one and `other` when not, and 113 is an
|
||||
`asin`. A contributor (108) is not an author, the same as an EPUB's
|
||||
`dc:contributor`, and is not reported. With no EXTH language, the language
|
||||
comes from the header's locale. Strings are decoded as the header says,
|
||||
UTF-8 or Windows-1252. A book whose text is encrypted still has its header
|
||||
@@ -114,45 +112,10 @@ an EXTH block that breaks off keeps the records before the break.
|
||||
|
||||
## Covers
|
||||
|
||||
`Metadata::cover` is the cover image the file carries, with its media type
|
||||
read off its first bytes: JPEG, PNG, GIF, WebP or BMP. It is for library
|
||||
callers and is not part of the JSON record, which is a line of text.
|
||||
|
||||
A Kindle book's cover is EXTH 201, or the thumbnail, 202, when no cover is
|
||||
named. With neither, it is the record at the header's first image index, as
|
||||
calibre does, and only that record: when it is not a JPEG, PNG or GIF there is
|
||||
no cover, because the records after it can be illustrations, and kindlegen
|
||||
sometimes puts a font there first.
|
||||
|
||||
An EPUB's is the first of these that reaches an image:
|
||||
|
||||
1. The manifest item whose `properties` include `cover-image` (EPUB 3).
|
||||
2. The item `<meta name="cover" content="…">` names (EPUB 2). Some
|
||||
converters write the image's address there rather than its id, and that is
|
||||
followed too.
|
||||
3. The guide's `<reference type="cover">`, a page, and the first `<img src>`
|
||||
or SVG `<image href>` on it. A guide that points straight at an image is
|
||||
taken at its word.
|
||||
4. Any image in the manifest whose id or address says `cover`.
|
||||
|
||||
A route that names something unusable — not in the archive, not an image —
|
||||
leaves a note, and the next route is asked. Addresses are resolved against the
|
||||
document they are written in, with their fragment off and their
|
||||
percent-escapes undone, and one that climbs out of the archive or names a
|
||||
scheme (`http:`, `data:`) is not followed.
|
||||
|
||||
What the bytes say beats what the manifest says: a JPEG listed as
|
||||
`image/png` comes back as `image/jpeg`, with a note. A cover larger than
|
||||
`bookmeta::MAX_COVER` (16 MiB) is left out with a note, in either format, and
|
||||
an EPUB's is never decompressed further than that, whatever size the archive
|
||||
claims for it.
|
||||
|
||||
`Cover::guessed` says the file left the choice open: a Kindle book's first
|
||||
image record, or an EPUB image picked because its name says `cover`. Either
|
||||
can be an illustration from inside the book, so a caller that can get a cover
|
||||
another way, from a register by ISBN, should prefer that one. A cover the file
|
||||
names is never a guess. `Cover::extension` gives the usual file extension for
|
||||
its media type.
|
||||
`Metadata::cover` is the cover image a Kindle book carries (EXTH 201, or the
|
||||
thumbnail, 202, when no cover is named), with its media type read off its
|
||||
first bytes. It is for library callers and is not part of the JSON record,
|
||||
which is a line of text.
|
||||
|
||||
## Asking a model
|
||||
|
||||
|
||||
+1
-731
@@ -4,17 +4,10 @@
|
||||
//! document carries Dublin Core elements: title, creator, publisher, date,
|
||||
//! language, identifier, subject. Calibre adds its own `<meta>` entries for
|
||||
//! the series, which nothing in the standard has a place for.
|
||||
//!
|
||||
//! The cover is whichever image the package names, asked in the order the
|
||||
//! standards came: EPUB 3's `cover-image` property, EPUB 2's
|
||||
//! `<meta name="cover">`, the guide's cover page and the image on it, and
|
||||
//! last any image whose name says cover. Every address is resolved inside the
|
||||
//! archive and nowhere else, and every member is read no further than a fixed
|
||||
//! bound, [`MAX_COVER`] for an image.
|
||||
|
||||
use std::io::Read;
|
||||
|
||||
use crate::{Cover, MAX_COVER, Metadata, Source, image_type};
|
||||
use crate::{Metadata, Source};
|
||||
|
||||
/// How large a package document is worth reading.
|
||||
///
|
||||
@@ -45,7 +38,6 @@ pub fn read(path: &str, bytes: &[u8]) -> Metadata {
|
||||
};
|
||||
|
||||
fill(&mut meta, &package);
|
||||
meta.cover = cover(&mut archive, &package_path, &package, &mut meta);
|
||||
if meta.is_empty() {
|
||||
meta.note("a package document with nothing filled in");
|
||||
}
|
||||
@@ -256,341 +248,6 @@ fn unescape(text: &str) -> String {
|
||||
.replace("&", "&")
|
||||
}
|
||||
|
||||
/// One entry of the package's manifest.
|
||||
struct Item {
|
||||
id: String,
|
||||
href: String,
|
||||
media_type: String,
|
||||
properties: String,
|
||||
/// Where `href` leads inside the archive, when it leads anywhere.
|
||||
path: Option<String>,
|
||||
}
|
||||
|
||||
/// The cover image, by the first of the four routes that reaches one.
|
||||
///
|
||||
/// A route that names something unusable leaves a note and the next route is
|
||||
/// asked, so a book whose declared cover is missing can still have the one
|
||||
/// its file name gives away.
|
||||
fn cover<R: std::io::Read + std::io::Seek>(
|
||||
archive: &mut zip::ZipArchive<R>,
|
||||
package_path: &str,
|
||||
package: &str,
|
||||
meta: &mut Metadata,
|
||||
) -> Option<Cover> {
|
||||
let base = directory_of(package_path);
|
||||
let items: Vec<Item> = tags(package)
|
||||
.filter(|(name, _)| *name == "item")
|
||||
.map(|(_, tag)| {
|
||||
let href = attr(tag, "href").unwrap_or_default();
|
||||
Item {
|
||||
id: attr(tag, "id").unwrap_or_default(),
|
||||
path: resolve(base, &href),
|
||||
href,
|
||||
media_type: attr(tag, "media-type").unwrap_or_default(),
|
||||
properties: attr(tag, "properties").unwrap_or_default(),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let mut look = Look {
|
||||
archive,
|
||||
items: &items,
|
||||
tried: Vec::new(),
|
||||
meta,
|
||||
};
|
||||
|
||||
// EPUB 3: the manifest item whose properties say so.
|
||||
for item in items.iter().filter(|item| {
|
||||
item.properties
|
||||
.split_ascii_whitespace()
|
||||
.any(|property| property == "cover-image")
|
||||
}) {
|
||||
if let Some(found) = look.image(base, &item.href) {
|
||||
return Some(found);
|
||||
}
|
||||
}
|
||||
|
||||
// EPUB 2: `<meta name="cover" content="…">` names an item by its id, or
|
||||
// in some converters' hands by its address.
|
||||
if let Some(named) = tags(package)
|
||||
.filter(|(name, _)| *name == "meta")
|
||||
.find(|(_, tag)| attr(tag, "name").as_deref() == Some("cover"))
|
||||
.and_then(|(_, tag)| attr(tag, "content"))
|
||||
.map(|content| content.trim().to_string())
|
||||
.filter(|content| !content.is_empty())
|
||||
{
|
||||
let as_path = resolve(base, &named);
|
||||
let href = items
|
||||
.iter()
|
||||
.find(|item| item.id == named)
|
||||
.or_else(|| {
|
||||
items
|
||||
.iter()
|
||||
.find(|item| item.path.is_some() && item.path == as_path)
|
||||
})
|
||||
.map_or(named.as_str(), |item| item.href.as_str());
|
||||
if let Some(found) = look.image(base, href) {
|
||||
return Some(found);
|
||||
}
|
||||
}
|
||||
|
||||
// The guide's cover page, and the first image on it.
|
||||
if let Some(page) = tags(package)
|
||||
.filter(|(name, _)| *name == "reference")
|
||||
.find(|(_, tag)| attr(tag, "type").is_some_and(|kind| kind.eq_ignore_ascii_case("cover")))
|
||||
.and_then(|(_, tag)| attr(tag, "href"))
|
||||
{
|
||||
if let Some(found) = look.page(base, &page) {
|
||||
return Some(found);
|
||||
}
|
||||
}
|
||||
|
||||
// Last: an image whose id or address says cover.
|
||||
for item in items.iter().filter(|item| {
|
||||
item.media_type.starts_with("image/")
|
||||
&& (item.id.to_lowercase().contains("cover")
|
||||
|| item.href.to_lowercase().contains("cover"))
|
||||
}) {
|
||||
if let Some(mut found) = look.image(base, &item.href) {
|
||||
found.guessed = true;
|
||||
return Some(found);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// The search for a cover, remembering what it has already looked at so
|
||||
/// that two routes to the same member read it, and note it, once.
|
||||
struct Look<'a, R> {
|
||||
archive: &'a mut zip::ZipArchive<R>,
|
||||
items: &'a [Item],
|
||||
tried: Vec<String>,
|
||||
meta: &'a mut Metadata,
|
||||
}
|
||||
|
||||
impl<R: std::io::Read + std::io::Seek> Look<'_, R> {
|
||||
/// Where an address leads, or a note saying it leads nowhere it may.
|
||||
fn locate(&mut self, base: &str, href: &str) -> Option<String> {
|
||||
let Some(path) = resolve(base, href) else {
|
||||
self.meta.note(format!(
|
||||
"the cover's address, {href}, is not a path inside the archive"
|
||||
));
|
||||
return None;
|
||||
};
|
||||
if self.tried.contains(&path) {
|
||||
return None;
|
||||
}
|
||||
self.tried.push(path.clone());
|
||||
Some(path)
|
||||
}
|
||||
|
||||
/// An image at an address, if that is what is there.
|
||||
fn image(&mut self, base: &str, href: &str) -> Option<Cover> {
|
||||
let path = self.locate(base, href)?;
|
||||
let bytes = self.fetch(&path, MAX_COVER, "cover")?;
|
||||
self.check(&path, bytes)
|
||||
}
|
||||
|
||||
/// The image a cover page shows, or the page itself where the guide
|
||||
/// points straight at an image.
|
||||
fn page(&mut self, base: &str, href: &str) -> Option<Cover> {
|
||||
let path = self.locate(base, href)?;
|
||||
let bytes = self.fetch(&path, MAX_PACKAGE as usize, "cover page")?;
|
||||
if image_type(&bytes).is_some() {
|
||||
return self.check(&path, bytes);
|
||||
}
|
||||
let text = String::from_utf8_lossy(&bytes);
|
||||
let Some(src) = tags(&text).find_map(|(name, tag)| match name {
|
||||
"img" => attr(tag, "src"),
|
||||
"image" => attr(tag, "xlink:href").or_else(|| attr(tag, "href")),
|
||||
_ => None,
|
||||
}) else {
|
||||
self.meta
|
||||
.note(format!("the cover page, {path}, shows no image"));
|
||||
return None;
|
||||
};
|
||||
self.image(directory_of(&path), &src)
|
||||
}
|
||||
|
||||
/// One member's bytes, read no further than `cap`.
|
||||
///
|
||||
/// The size the archive declares is checked first and the read is
|
||||
/// bounded anyway, because the declaration may be a lie.
|
||||
fn fetch(&mut self, path: &str, cap: usize, what: &str) -> Option<Vec<u8>> {
|
||||
let file = match self.archive.by_name(path) {
|
||||
Ok(file) => file,
|
||||
Err(zip::result::ZipError::FileNotFound) => {
|
||||
self.meta
|
||||
.note(format!("the {what}, {path}, is not in the archive"));
|
||||
return None;
|
||||
}
|
||||
Err(_) => {
|
||||
self.meta
|
||||
.note(format!("the {what}, {path}, could not be read"));
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let too_large = format!("the {what}, {path}, is larger than {}", crate::size(cap));
|
||||
if file.size() > cap as u64 {
|
||||
self.meta.note(too_large);
|
||||
return None;
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
if file.take(cap as u64 + 1).read_to_end(&mut bytes).is_err() {
|
||||
self.meta
|
||||
.note(format!("the {what}, {path}, could not be read"));
|
||||
return None;
|
||||
}
|
||||
if bytes.len() > cap {
|
||||
self.meta.note(too_large);
|
||||
return None;
|
||||
}
|
||||
Some(bytes)
|
||||
}
|
||||
|
||||
/// The bytes as a cover when they are an image, typed by what they are
|
||||
/// rather than by what the manifest says.
|
||||
fn check(&mut self, path: &str, bytes: Vec<u8>) -> Option<Cover> {
|
||||
let Some(media_type) = image_type(&bytes) else {
|
||||
self.meta
|
||||
.note(format!("the cover, {path}, is not an image"));
|
||||
return None;
|
||||
};
|
||||
if let Some(item) = self
|
||||
.items
|
||||
.iter()
|
||||
.find(|item| item.path.as_deref() == Some(path))
|
||||
{
|
||||
let declared = item.media_type.trim();
|
||||
if !declared.is_empty() && !declared.eq_ignore_ascii_case(media_type) {
|
||||
self.meta.note(format!(
|
||||
"the manifest calls the cover, {path}, {declared}, but it is {media_type}"
|
||||
));
|
||||
}
|
||||
}
|
||||
Some(Cover {
|
||||
media_type: media_type.to_string(),
|
||||
bytes,
|
||||
guessed: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The directory a member sits in, with no trailing slash.
|
||||
fn directory_of(path: &str) -> &str {
|
||||
path.rsplit_once('/').map_or("", |(dir, _)| dir)
|
||||
}
|
||||
|
||||
/// Where an address written in a member in `base` leads inside the archive.
|
||||
///
|
||||
/// The fragment comes off and percent-escapes are undone. An address with a
|
||||
/// scheme, or one that climbs above the root of the archive, leads nowhere
|
||||
/// this will follow.
|
||||
fn resolve(base: &str, href: &str) -> Option<String> {
|
||||
let href = href.split('#').next().unwrap_or("").trim();
|
||||
// `http:`, `data:`, `file:`: a scheme before the first slash.
|
||||
if let Some(colon) = href.find(':') {
|
||||
if !href[..colon].contains('/') {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
let href = percent_decode(href)?;
|
||||
let mut parts: Vec<&str> = if href.starts_with('/') {
|
||||
Vec::new()
|
||||
} else {
|
||||
base.split('/').filter(|part| !part.is_empty()).collect()
|
||||
};
|
||||
for part in href.split('/') {
|
||||
match part {
|
||||
"" | "." => {}
|
||||
".." => {
|
||||
parts.pop()?;
|
||||
}
|
||||
part => parts.push(part),
|
||||
}
|
||||
}
|
||||
// An address of nothing but the directory it is written in names no
|
||||
// member at all.
|
||||
if href.is_empty() || parts.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(parts.join("/"))
|
||||
}
|
||||
|
||||
/// `%20` back to a space, and so on, where the result is still UTF-8.
|
||||
fn percent_decode(text: &str) -> Option<String> {
|
||||
let raw = text.as_bytes();
|
||||
let mut out = Vec::with_capacity(raw.len());
|
||||
let mut at = 0;
|
||||
while at < raw.len() {
|
||||
if raw[at] == b'%' {
|
||||
let hex = std::str::from_utf8(raw.get(at + 1..at + 3)?).ok()?;
|
||||
out.push(u8::from_str_radix(hex, 16).ok()?);
|
||||
at += 3;
|
||||
} else {
|
||||
out.push(raw[at]);
|
||||
at += 1;
|
||||
}
|
||||
}
|
||||
String::from_utf8(out).ok()
|
||||
}
|
||||
|
||||
/// Every start tag in a document, in order, as its local name and its text.
|
||||
///
|
||||
/// Comments, declarations, processing instructions and end tags are passed
|
||||
/// over. The local name is what follows any namespace prefix, so
|
||||
/// `<opf:item>` is an `item`.
|
||||
fn tags(xml: &str) -> impl Iterator<Item = (&str, &str)> {
|
||||
let mut rest = xml;
|
||||
std::iter::from_fn(move || {
|
||||
loop {
|
||||
let at = rest.find('<')?;
|
||||
rest = &rest[at..];
|
||||
if let Some(after) = rest.strip_prefix("<!--") {
|
||||
rest = after.find("-->").map_or("", |end| &after[end + 3..]);
|
||||
continue;
|
||||
}
|
||||
let end = rest.find('>')?;
|
||||
let tag = &rest[1..end];
|
||||
rest = &rest[end + 1..];
|
||||
if tag.starts_with(['/', '!', '?']) {
|
||||
continue;
|
||||
}
|
||||
let name = tag
|
||||
.split(|c: char| c.is_ascii_whitespace() || c == '/')
|
||||
.next()
|
||||
.unwrap_or("");
|
||||
let local = name.rsplit_once(':').map_or(name, |(_, local)| local);
|
||||
return Some((local, tag));
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// The value of one attribute in a tag's text, entities undone.
|
||||
///
|
||||
/// The name has to stand on its own — `id` is not the end of `xml:id` or
|
||||
/// `data-id` — and may have space either side of its `=`.
|
||||
fn attr(tag: &str, name: &str) -> Option<String> {
|
||||
let mut from = 0;
|
||||
while let Some(found) = tag[from..].find(name) {
|
||||
let at = from + found;
|
||||
from = at + name.len();
|
||||
if !tag[..at].ends_with(|c: char| c.is_ascii_whitespace()) {
|
||||
continue;
|
||||
}
|
||||
let Some(rest) = tag[from..].trim_start().strip_prefix('=') else {
|
||||
continue;
|
||||
};
|
||||
let rest = rest.trim_start();
|
||||
let Some(quote) = rest.chars().next().filter(|c| matches!(c, '"' | '\'')) else {
|
||||
continue;
|
||||
};
|
||||
let value = &rest[1..];
|
||||
let end = value.find(quote)?;
|
||||
return Some(unescape(&value[..end]));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -699,391 +356,4 @@ mod tests {
|
||||
assert_eq!(meta.source, "none");
|
||||
assert!(!meta.notes.is_empty());
|
||||
}
|
||||
|
||||
// Covers. Each book is built here, a few members zipped in memory: the
|
||||
// container, a package in `OEBPS/`, and whatever the case is about.
|
||||
|
||||
const JPEG: &[u8] = b"\xff\xd8\xff\xe0\x00\x10JFIF\x00jpeg body";
|
||||
const PNG: &[u8] = b"\x89PNG\r\n\x1a\n\x00\x00\x00\x0dIHDR png body";
|
||||
const GIF: &[u8] = b"GIF89a\x01\x00\x01\x00 gif body";
|
||||
const WEBP: &[u8] = b"RIFF\x20\x00\x00\x00WEBPVP8 webp body";
|
||||
|
||||
const CONTAINER: &str = r#"<?xml version="1.0"?>
|
||||
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||
<rootfiles>
|
||||
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
|
||||
</rootfiles>
|
||||
</container>"#;
|
||||
|
||||
/// A package with this manifest and these extra elements, around a title.
|
||||
fn opf(manifest: &str, extra: &str) -> String {
|
||||
format!(
|
||||
r#"<?xml version="1.0"?>
|
||||
<package xmlns="http://www.idpf.org/2007/opf" version="3.0">
|
||||
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<dc:title>Covered</dc:title>
|
||||
</metadata>
|
||||
<manifest>
|
||||
<item id="text" href="text/one.xhtml" media-type="application/xhtml+xml"/>
|
||||
{manifest}
|
||||
</manifest>
|
||||
{extra}
|
||||
</package>"#
|
||||
)
|
||||
}
|
||||
|
||||
/// An EPUB of these members, after the container.
|
||||
fn epub(members: &[(&str, &[u8])]) -> Vec<u8> {
|
||||
use std::io::Write;
|
||||
let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
|
||||
let options = zip::write::SimpleFileOptions::default()
|
||||
.compression_method(zip::CompressionMethod::Deflated);
|
||||
let container: (&str, &[u8]) = ("META-INF/container.xml", CONTAINER.as_bytes());
|
||||
for (name, data) in std::iter::once(&container).chain(members) {
|
||||
zip.start_file(*name, options).unwrap();
|
||||
zip.write_all(data).unwrap();
|
||||
}
|
||||
zip.finish().unwrap().into_inner()
|
||||
}
|
||||
|
||||
fn book(package: &str, members: &[(&str, &[u8])]) -> Metadata {
|
||||
let mut all = vec![("OEBPS/content.opf", package.as_bytes())];
|
||||
all.extend_from_slice(members);
|
||||
read("x.epub", &epub(&all))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn epub_3_names_its_cover_with_a_property() {
|
||||
let package = opf(
|
||||
r#"<item id="decoy-cover" href="images/cover-thumb.png" media-type="image/png"/>
|
||||
<item id="c" href="images/front.jpg" media-type="image/jpeg" properties="svg cover-image"/>"#,
|
||||
r#"<metadata><meta name="cover" content="decoy-cover"/></metadata>"#,
|
||||
);
|
||||
let meta = book(
|
||||
&package,
|
||||
&[
|
||||
("OEBPS/images/front.jpg", JPEG),
|
||||
("OEBPS/images/cover-thumb.png", PNG),
|
||||
],
|
||||
);
|
||||
let cover = meta.cover.expect("a cover");
|
||||
assert_eq!(cover.media_type, "image/jpeg");
|
||||
assert_eq!(cover.bytes, JPEG);
|
||||
assert!(!cover.guessed, "a declared cover is not a guess");
|
||||
assert_eq!(cover.extension(), "jpg");
|
||||
assert!(meta.notes.is_empty(), "{:?}", meta.notes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn epub_2_names_its_cover_with_a_meta_holding_an_id() {
|
||||
let package = opf(
|
||||
r#"<opf:item id="cover-img" href="Images/Front%20Page.png#top" media-type="image/png"/>"#,
|
||||
r#"<metadata><meta content="cover-img" name="cover" /></metadata>"#,
|
||||
);
|
||||
let meta = book(&package, &[("OEBPS/Images/Front Page.png", PNG)]);
|
||||
let cover = meta.cover.expect("a cover");
|
||||
assert_eq!(cover.media_type, "image/png");
|
||||
assert_eq!(cover.bytes, PNG);
|
||||
assert!(meta.notes.is_empty(), "{:?}", meta.notes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cover_meta_holding_an_address_is_followed_as_one() {
|
||||
// Listed in the manifest under another id, and not listed at all.
|
||||
let listed = opf(
|
||||
r#"<item id="x1" href="../art/front.gif" media-type="image/gif"/>"#,
|
||||
r#"<metadata><meta name="cover" content="../art/front.gif"/></metadata>"#,
|
||||
);
|
||||
let meta = book(&listed, &[("art/front.gif", GIF)]);
|
||||
assert_eq!(meta.cover.expect("a cover").media_type, "image/gif");
|
||||
|
||||
let unlisted = opf(
|
||||
"",
|
||||
r#"<metadata><meta name="cover" content="front.webp"/></metadata>"#,
|
||||
);
|
||||
let meta = book(&unlisted, &[("OEBPS/front.webp", WEBP)]);
|
||||
let cover = meta.cover.expect("a cover");
|
||||
assert_eq!(cover.media_type, "image/webp");
|
||||
assert_eq!(cover.bytes, WEBP);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_guide_cover_page_gives_the_first_image_on_it() {
|
||||
let package = opf(
|
||||
r#"<item id="page" href="text/cover.xhtml" media-type="application/xhtml+xml"/>
|
||||
<item id="art" href="images/art.jpg" media-type="image/jpeg"/>"#,
|
||||
r#"<guide><reference type="Cover" title="Cover" href="text/cover.xhtml#start"/></guide>"#,
|
||||
);
|
||||
let page = br#"<html><body><!-- <img src="nothing.png"/> -->
|
||||
<img alt="cover" src="../images/art.jpg"/><img src="../images/later.png"/></body></html>"#;
|
||||
let meta = book(
|
||||
&package,
|
||||
&[
|
||||
("OEBPS/text/cover.xhtml", page),
|
||||
("OEBPS/images/art.jpg", JPEG),
|
||||
("OEBPS/images/later.png", PNG),
|
||||
],
|
||||
);
|
||||
let cover = meta.cover.expect("a cover");
|
||||
assert_eq!(cover.bytes, JPEG);
|
||||
assert!(meta.notes.is_empty(), "{:?}", meta.notes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_guide_cover_page_may_draw_its_image_in_svg() {
|
||||
let package = opf(
|
||||
"",
|
||||
r#"<guide><reference type="cover" href="titlepage.xhtml"/></guide>"#,
|
||||
);
|
||||
for image in [
|
||||
r#"<svg:image width="600" xlink:href="cover.png"/>"#,
|
||||
r#"<image href="cover.png" />"#,
|
||||
] {
|
||||
let page = format!(r#"<html><body><svg:svg>{image}</svg:svg></body></html>"#);
|
||||
let meta = book(
|
||||
&package,
|
||||
&[
|
||||
("OEBPS/titlepage.xhtml", page.as_bytes()),
|
||||
("OEBPS/cover.png", PNG),
|
||||
],
|
||||
);
|
||||
assert_eq!(meta.cover.expect(image).bytes, PNG);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn last_an_image_whose_name_says_cover() {
|
||||
let package = opf(
|
||||
r#"<item id="pic1" href="images/plate.jpg" media-type="image/jpeg"/>
|
||||
<item id="CoverImage" href="images/i2.gif" media-type="image/gif"/>"#,
|
||||
"",
|
||||
);
|
||||
let meta = book(
|
||||
&package,
|
||||
&[
|
||||
("OEBPS/images/plate.jpg", JPEG),
|
||||
("OEBPS/images/i2.gif", GIF),
|
||||
],
|
||||
);
|
||||
let cover = meta.cover.expect("a cover");
|
||||
assert_eq!(cover.bytes, GIF);
|
||||
assert!(cover.guessed, "a cover found by its name is a guess");
|
||||
assert_eq!(cover.extension(), "gif");
|
||||
|
||||
let by_address = opf(
|
||||
r#"<item id="i9" href="images/Book_COVER.jpeg" media-type="image/jpeg"/>"#,
|
||||
"",
|
||||
);
|
||||
let meta = book(&by_address, &[("OEBPS/images/Book_COVER.jpeg", JPEG)]);
|
||||
assert_eq!(meta.cover.expect("a cover").bytes, JPEG);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_book_that_names_no_cover_has_none_and_says_nothing() {
|
||||
let package = opf(
|
||||
r#"<item id="plate" href="images/plate.jpg" media-type="image/jpeg"/>"#,
|
||||
"",
|
||||
);
|
||||
let meta = book(&package, &[("OEBPS/images/plate.jpg", JPEG)]);
|
||||
assert_eq!(meta.cover, None);
|
||||
assert!(meta.notes.is_empty(), "{:?}", meta.notes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_mislabelled_cover_is_what_its_bytes_say() {
|
||||
let package = opf(
|
||||
r#"<item id="c" href="c.png" media-type="image/png" properties="cover-image"/>"#,
|
||||
"",
|
||||
);
|
||||
let meta = book(&package, &[("OEBPS/c.png", JPEG)]);
|
||||
assert_eq!(meta.cover.expect("a cover").media_type, "image/jpeg");
|
||||
assert_eq!(
|
||||
meta.notes,
|
||||
["the manifest calls the cover, OEBPS/c.png, image/png, but it is image/jpeg"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cover_that_is_not_an_image_is_a_note_and_not_a_cover() {
|
||||
let package = opf(
|
||||
r#"<item id="c" href="c.jpg" media-type="image/jpeg" properties="cover-image"/>"#,
|
||||
"",
|
||||
);
|
||||
let meta = book(&package, &[("OEBPS/c.jpg", b"<html>not an image</html>")]);
|
||||
assert_eq!(meta.cover, None);
|
||||
assert_eq!(meta.notes, ["the cover, OEBPS/c.jpg, is not an image"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cover_larger_than_the_cap_is_left_out() {
|
||||
let mut huge = JPEG.to_vec();
|
||||
huge.resize(MAX_COVER + 1, 0);
|
||||
let package = opf(
|
||||
r#"<item id="c" href="c.jpg" media-type="image/jpeg" properties="cover-image"/>"#,
|
||||
"",
|
||||
);
|
||||
let meta = book(&package, &[("OEBPS/c.jpg", &huge)]);
|
||||
assert_eq!(meta.cover, None);
|
||||
assert_eq!(
|
||||
meta.notes,
|
||||
["the cover, OEBPS/c.jpg, is larger than 16 MiB"]
|
||||
);
|
||||
}
|
||||
|
||||
/// Read one member through `Look::fetch` with a cap of `cap`.
|
||||
fn fetch(bytes: Vec<u8>, cap: usize) -> (Option<usize>, Vec<String>) {
|
||||
let mut meta = Metadata::empty("x.epub", Source::Opf);
|
||||
let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap();
|
||||
let mut look = Look {
|
||||
archive: &mut archive,
|
||||
items: &[],
|
||||
tried: Vec::new(),
|
||||
meta: &mut meta,
|
||||
};
|
||||
let got = look.fetch("big.jpg", cap, "cover").map(|bytes| bytes.len());
|
||||
(got, meta.notes)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_member_is_read_no_further_than_its_cap() {
|
||||
let bytes = epub(&[("big.jpg", &[0u8; 4096])]);
|
||||
assert_eq!(fetch(bytes.clone(), 4096).0, Some(4096));
|
||||
assert_eq!(
|
||||
fetch(bytes, 4095),
|
||||
(
|
||||
None,
|
||||
vec!["the cover, big.jpg, is larger than 4095 bytes".to_string()]
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_member_whose_archive_understates_its_size_is_still_bounded() {
|
||||
let mut bytes = epub(&[("big.jpg", &[0u8; 4096])]);
|
||||
// Declare 10 bytes uncompressed, in the local header and in the
|
||||
// central directory, where 4096 are compressed in.
|
||||
// The name comes 30 bytes into a local header and 46 into a central
|
||||
// directory entry, and the uncompressed size 22 and 24.
|
||||
let local = find(&bytes, b"big.jpg") - 30;
|
||||
bytes[local + 22..local + 26].copy_from_slice(&10u32.to_le_bytes());
|
||||
let central = local + 31 + find(&bytes[local + 31..], b"big.jpg") - 46;
|
||||
assert_eq!(&bytes[central..central + 4], b"PK\x01\x02");
|
||||
bytes[central + 24..central + 28].copy_from_slice(&10u32.to_le_bytes());
|
||||
|
||||
// The declaration passes the check, and the read stops at the cap.
|
||||
let (got, notes) = fetch(bytes, 100);
|
||||
assert_eq!(got, None);
|
||||
assert_eq!(notes, ["the cover, big.jpg, is larger than 100 bytes"]);
|
||||
}
|
||||
|
||||
fn find(haystack: &[u8], needle: &[u8]) -> usize {
|
||||
haystack
|
||||
.windows(needle.len())
|
||||
.position(|window| window == needle)
|
||||
.expect("present")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cover_named_but_missing_is_a_note_and_the_next_route_is_asked() {
|
||||
let package = opf(
|
||||
r#"<item id="c" href="gone.jpg" media-type="image/jpeg" properties="cover-image"/>"#,
|
||||
"",
|
||||
);
|
||||
let meta = book(&package, &[]);
|
||||
assert_eq!(meta.cover, None);
|
||||
assert_eq!(
|
||||
meta.notes,
|
||||
["the cover, OEBPS/gone.jpg, is not in the archive"]
|
||||
);
|
||||
|
||||
// A missing page is said the same way.
|
||||
let page = opf(
|
||||
"",
|
||||
r#"<guide><reference type="cover" href="none.xhtml"/></guide>"#,
|
||||
);
|
||||
assert_eq!(
|
||||
book(&page, &[]).notes,
|
||||
["the cover page, OEBPS/none.xhtml, is not in the archive"]
|
||||
);
|
||||
|
||||
// The same file reached again by another route is not read twice,
|
||||
// and an image the name gives away still stands in.
|
||||
let package = opf(
|
||||
r#"<item id="cover" href="gone.jpg" media-type="image/jpeg" properties="cover-image"/>
|
||||
<item id="cover-small" href="small.png" media-type="image/png"/>"#,
|
||||
r#"<metadata><meta name="cover" content="cover"/></metadata>"#,
|
||||
);
|
||||
let meta = book(&package, &[("OEBPS/small.png", PNG)]);
|
||||
assert_eq!(meta.cover.expect("the stand-in").bytes, PNG);
|
||||
assert_eq!(
|
||||
meta.notes,
|
||||
["the cover, OEBPS/gone.jpg, is not in the archive"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_address_never_leaves_the_archive() {
|
||||
assert_eq!(resolve("OEBPS", "../c.jpg").as_deref(), Some("c.jpg"));
|
||||
assert_eq!(resolve("OEBPS", "/c.jpg").as_deref(), Some("c.jpg"));
|
||||
assert_eq!(
|
||||
resolve("OEBPS/text", "./a/../b.jpg").as_deref(),
|
||||
Some("OEBPS/text/b.jpg")
|
||||
);
|
||||
assert_eq!(resolve("", "a%2Fb.jpg").as_deref(), Some("a/b.jpg"));
|
||||
for hostile in [
|
||||
"../../c.jpg",
|
||||
"../../../etc/passwd",
|
||||
"http://example.com/c.jpg",
|
||||
"data:image/png;base64,AAAA",
|
||||
"file:///etc/passwd",
|
||||
"%ZZ.jpg",
|
||||
"%ff.jpg",
|
||||
"c%",
|
||||
"#only-a-fragment",
|
||||
"",
|
||||
] {
|
||||
assert_eq!(resolve("OEBPS", hostile), None, "{hostile}");
|
||||
}
|
||||
|
||||
let package = opf(
|
||||
r#"<item id="c" href="../../c.jpg" media-type="image/jpeg" properties="cover-image"/>"#,
|
||||
"",
|
||||
);
|
||||
let meta = book(&package, &[("c.jpg", JPEG)]);
|
||||
assert_eq!(meta.cover, None);
|
||||
assert_eq!(
|
||||
meta.notes,
|
||||
["the cover's address, ../../c.jpg, is not a path inside the archive"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_attribute_is_matched_by_its_whole_name() {
|
||||
let tag = r#"item data-id="no" xml:id="no" id = 'yes' href="a&b.jpg""#;
|
||||
assert_eq!(attr(tag, "id").as_deref(), Some("yes"));
|
||||
assert_eq!(attr(tag, "href").as_deref(), Some("a&b.jpg"));
|
||||
assert_eq!(attr(r#"image xlink:href="x.png""#, "href"), None);
|
||||
assert_eq!(attr(r#"item id="unterminated"#, "id"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_truncation_of_a_book_with_a_cover_returns() {
|
||||
let package = opf(
|
||||
r#"<item id="c" href="c.jpg" media-type="image/jpeg" properties="cover-image"/>"#,
|
||||
r#"<guide><reference type="cover" href="p.xhtml"/></guide>"#,
|
||||
);
|
||||
let bytes = epub(&[
|
||||
("OEBPS/content.opf", package.as_bytes()),
|
||||
("OEBPS/c.jpg", JPEG),
|
||||
("OEBPS/p.xhtml", b"<img src=\"c.jpg\"/>"),
|
||||
]);
|
||||
for len in 0..=bytes.len() {
|
||||
let _ = read("cut.epub", &bytes[..len]);
|
||||
}
|
||||
for at in 0..bytes.len() {
|
||||
let mut broken = bytes.clone();
|
||||
broken[at] ^= 0xff;
|
||||
let _ = read("broken.epub", &broken);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -106,63 +106,10 @@ pub struct Metadata {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Cover {
|
||||
/// What kind of image it is, read off its first bytes: `image/jpeg`,
|
||||
/// `image/png`, `image/gif`, `image/webp` or `image/bmp`.
|
||||
/// `image/png`, `image/gif` or `image/bmp`.
|
||||
pub media_type: String,
|
||||
/// The image itself, never more than [`MAX_COVER`] bytes.
|
||||
/// The image itself.
|
||||
pub bytes: Vec<u8>,
|
||||
/// Whether the file left the choice open. A MOBI that names no cover, or
|
||||
/// an EPUB whose only cover is an image with "cover" in its name, has one
|
||||
/// picked by position or by name, and that pick can be an illustration
|
||||
/// from inside the book. A caller with a better source should prefer it.
|
||||
pub guessed: bool,
|
||||
}
|
||||
|
||||
impl Cover {
|
||||
/// The usual file extension for the image, without the dot.
|
||||
pub fn extension(&self) -> &'static str {
|
||||
match self.media_type.as_str() {
|
||||
"image/png" => "png",
|
||||
"image/gif" => "gif",
|
||||
"image/webp" => "webp",
|
||||
"image/bmp" => "bmp",
|
||||
_ => "jpg",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// How large a cover is worth keeping.
|
||||
///
|
||||
/// A real one is a few hundred kilobytes. An EPUB's cover has to be
|
||||
/// decompressed to be read, and this is as far as that goes: a larger one is
|
||||
/// left out with a note, in every format, so a hostile file cannot make a
|
||||
/// reader allocate whatever it likes.
|
||||
pub const MAX_COVER: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// A size, as a note says it.
|
||||
pub(crate) fn size(bytes: usize) -> String {
|
||||
const MIB: usize = 1024 * 1024;
|
||||
if bytes >= MIB && bytes % MIB == 0 {
|
||||
format!("{} MiB", bytes / MIB)
|
||||
} else {
|
||||
format!("{bytes} bytes")
|
||||
}
|
||||
}
|
||||
|
||||
/// What kind of image these bytes are, by their magic.
|
||||
pub(crate) fn image_type(data: &[u8]) -> Option<&'static str> {
|
||||
if data.starts_with(&[0xff, 0xd8, 0xff]) {
|
||||
Some("image/jpeg")
|
||||
} else if data.starts_with(b"\x89PNG\r\n\x1a\n") {
|
||||
Some("image/png")
|
||||
} else if data.starts_with(b"GIF87a") || data.starts_with(b"GIF89a") {
|
||||
Some("image/gif")
|
||||
} else if data.starts_with(b"RIFF") && data.get(8..12) == Some(b"WEBP") {
|
||||
Some("image/webp")
|
||||
} else if data.starts_with(b"BM") && data.len() > 14 {
|
||||
Some("image/bmp")
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn version() -> u32 {
|
||||
|
||||
+16
-13
@@ -13,7 +13,7 @@
|
||||
//! panics or reads past the end.
|
||||
|
||||
use crate::epub::{digits_of, is_isbn, is_uuid};
|
||||
use crate::{Cover, MAX_COVER, Metadata, Source, image_type};
|
||||
use crate::{Cover, Metadata, Source};
|
||||
|
||||
/// Where a Palm database says what it holds.
|
||||
const TYPE_AT: usize = 60;
|
||||
@@ -332,17 +332,9 @@ fn cover(
|
||||
meta.note("the cover record is not an image");
|
||||
return None;
|
||||
};
|
||||
if data.len() > MAX_COVER {
|
||||
meta.note(format!(
|
||||
"the cover record is larger than {}",
|
||||
crate::size(MAX_COVER)
|
||||
));
|
||||
return None;
|
||||
}
|
||||
Some(Cover {
|
||||
media_type: media_type.to_string(),
|
||||
bytes: data.to_vec(),
|
||||
guessed: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -355,14 +347,10 @@ fn cover(
|
||||
fn first_image_record(bytes: &[u8], records: &[usize], first_image: Option<u32>) -> Option<Cover> {
|
||||
let first = first_image.filter(|&index| index != NONE)?;
|
||||
let data = record(bytes, records, first as usize)?;
|
||||
if data.len() > MAX_COVER {
|
||||
return None;
|
||||
}
|
||||
let media_type = image_type(data).filter(|kind| *kind != "image/bmp")?;
|
||||
Some(Cover {
|
||||
media_type: media_type.to_string(),
|
||||
bytes: data.to_vec(),
|
||||
guessed: true,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -371,6 +359,21 @@ fn is_asin(value: &str) -> bool {
|
||||
value.len() == 10 && value.chars().all(|c| c.is_ascii_alphanumeric())
|
||||
}
|
||||
|
||||
/// What kind of image these bytes are, by their magic.
|
||||
fn image_type(data: &[u8]) -> Option<&'static str> {
|
||||
if data.starts_with(&[0xff, 0xd8, 0xff]) {
|
||||
Some("image/jpeg")
|
||||
} else if data.starts_with(b"\x89PNG\r\n\x1a\n") {
|
||||
Some("image/png")
|
||||
} else if data.starts_with(b"GIF87a") || data.starts_with(b"GIF89a") {
|
||||
Some("image/gif")
|
||||
} else if data.starts_with(b"BM") && data.len() > 14 {
|
||||
Some("image/bmp")
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// The Palm database's name, the last place a title can come from.
|
||||
///
|
||||
/// Converters write the title there with spaces made underscores; it is
|
||||
|
||||
@@ -31,7 +31,6 @@ fn says_what_it_was_given(meta: &bookmeta::Metadata) {
|
||||
|
||||
let cover = meta.cover.as_ref().expect("a cover");
|
||||
assert_eq!(cover.media_type, "image/jpeg");
|
||||
assert!(!cover.guessed, "EXTH 201 names the cover");
|
||||
let original = std::fs::read(fixture("lantern-cover.jpg")).expect("the cover fixture");
|
||||
assert_eq!(cover.bytes, original);
|
||||
}
|
||||
@@ -85,7 +84,6 @@ fn a_file_naming_no_cover_gets_its_first_image_record() {
|
||||
let meta = bookmeta::from_bytes("x.mobi", &lantern_naming_no_cover());
|
||||
let cover = meta.cover.expect("the first image record");
|
||||
assert_eq!(cover.media_type, "image/jpeg");
|
||||
assert!(cover.guessed, "a cover taken by position is a guess");
|
||||
let original = std::fs::read(fixture("lantern-cover.jpg")).expect("the cover fixture");
|
||||
assert_eq!(cover.bytes, original);
|
||||
}
|
||||
|
||||
@@ -26,8 +26,8 @@ file too small to be a book, a name that says "sample".
|
||||
cover images that ended up in a folder of PDFs.
|
||||
|
||||
A library accumulates files that lie: a DjVu named `.pdf`, a Word document
|
||||
named `.epub`, an RTF saved as `.doc`, an EPUB saved as `.zip`, a single PDF
|
||||
that somebody zipped and renamed back to `.pdf`.
|
||||
named `.epub`, an EPUB saved as `.zip`, a single PDF that somebody zipped and
|
||||
renamed back to `.pdf`.
|
||||
|
||||
## Install
|
||||
|
||||
@@ -89,12 +89,7 @@ one bad file does not stop a sweep.
|
||||
| `ole2` | The compound-file header every binary Word document starts with |
|
||||
| `epub` | A zip holding `mimetype` and `META-INF/container.xml` |
|
||||
| `zip-wrapped-pdf` | A zip whose only real member is one PDF |
|
||||
| `zip-wrapped-fb2` | A zip whose only real member is one FictionBook — the `.fb2.zip` most of them are passed around as |
|
||||
| `cbz` | A zip whose every real member is a page image: JPEG, PNG, GIF or WebP |
|
||||
| `azw3`, `mobi` | A Palm database tagged `BOOKMOBI`, told apart by the format version inside the first record |
|
||||
| `lit` | `ITOLITLS`, Microsoft Reader's signature |
|
||||
| `rtf` | `{\rtf` at the front, after any byte-order mark and blank space |
|
||||
| `fb2` | XML whose root element is `FictionBook`, found by stepping over the declaration, comments and doctype in the first four kilobytes |
|
||||
| `html` | Markup at the front, after any byte-order mark |
|
||||
| `txt` | Text that is none of the above |
|
||||
| `unknown` | Anything else, with a note saying what was seen |
|
||||
@@ -103,20 +98,6 @@ An archive holding several PDFs is a collection, not a wrapped book: choosing
|
||||
among them would be a guess, and naming one chapter as the whole work is worse
|
||||
than refusing.
|
||||
|
||||
A zipped FictionBook is reported as wrapped rather than as `fb2`, the same way
|
||||
a zipped PDF is: a converter handed it as FictionBook is handed a zip, and the
|
||||
kind is there to stop exactly that.
|
||||
|
||||
A comic is the pages and nothing else. Directories, the `ComicInfo.xml` a
|
||||
comic reader writes into the archive, and the desktop clutter every archive
|
||||
picks up are passed over; anything more — one PDF among the pages, a readme —
|
||||
and the zip is not a comic, and says what it holds instead. Only names are
|
||||
read, out of the archive's directory, so no member is ever unpacked.
|
||||
|
||||
FictionBook is read as bytes rather than as text. A great many are written in
|
||||
windows-1251, which spells its markup in ASCII but is not UTF-8, and would
|
||||
otherwise be called binary.
|
||||
|
||||
## As a library
|
||||
|
||||
```rust
|
||||
|
||||
@@ -1,162 +0,0 @@
|
||||
//! Telling a FictionBook from any other XML.
|
||||
//!
|
||||
//! A FictionBook is an XML document whose root element is `FictionBook`.
|
||||
//! The root comes after the prolog — a declaration, comments, perhaps a
|
||||
//! doctype — so it is found by stepping over those rather than by matching
|
||||
//! at a fixed offset.
|
||||
//!
|
||||
//! The walk is over bytes, not text. A great many FictionBooks are written in
|
||||
//! windows-1251, which is not UTF-8 but spells its markup in ASCII all the
|
||||
//! same, and a reader that insisted on UTF-8 would call every one of them
|
||||
//! binary.
|
||||
|
||||
use crate::BOM;
|
||||
|
||||
/// What the root element of a FictionBook is called.
|
||||
const ROOT: &[u8] = b"FictionBook";
|
||||
|
||||
/// Whether the front of a file is the start of a FictionBook.
|
||||
///
|
||||
/// `head` is however much of the front the caller read. A root element that
|
||||
/// does not begin inside it is not found, so the answer is no rather than a
|
||||
/// guess.
|
||||
pub fn is_fictionbook(head: &[u8]) -> bool {
|
||||
root_element(head).is_some_and(|name| local_name(name) == ROOT)
|
||||
}
|
||||
|
||||
/// The name of the first element, after the prolog.
|
||||
fn root_element(head: &[u8]) -> Option<&[u8]> {
|
||||
let mut rest = head.strip_prefix(BOM).unwrap_or(head);
|
||||
loop {
|
||||
rest = rest.trim_ascii_start();
|
||||
if let Some(after) = rest.strip_prefix(b"<?") {
|
||||
rest = after_marker(after, b"?>")?;
|
||||
} else if let Some(after) = rest.strip_prefix(b"<!--") {
|
||||
rest = after_marker(after, b"-->")?;
|
||||
} else if let Some(after) = rest.strip_prefix(b"<!") {
|
||||
rest = after_declaration(after)?;
|
||||
} else {
|
||||
// Anything but an element here is text before the root, which
|
||||
// XML does not allow.
|
||||
let after = rest.strip_prefix(b"<")?;
|
||||
// A name that runs to the end of what was read may be the front
|
||||
// of a longer one, so it is not taken.
|
||||
let end = after.iter().position(|byte| !is_name_byte(*byte))?;
|
||||
let ends_properly =
|
||||
matches!(after[end], b'>' | b'/') || after[end].is_ascii_whitespace();
|
||||
return (end > 0 && ends_properly).then_some(&after[..end]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What follows the first `marker`, if it is there at all.
|
||||
fn after_marker<'a>(rest: &'a [u8], marker: &[u8]) -> Option<&'a [u8]> {
|
||||
rest.windows(marker.len())
|
||||
.position(|window| window == marker)
|
||||
.map(|at| &rest[at + marker.len()..])
|
||||
}
|
||||
|
||||
/// What follows a doctype, whose internal subset may hold `>` of its own.
|
||||
fn after_declaration(rest: &[u8]) -> Option<&[u8]> {
|
||||
let mut depth = 0usize;
|
||||
let mut quote = None;
|
||||
for (at, &byte) in rest.iter().enumerate() {
|
||||
match quote {
|
||||
Some(open) if byte == open => quote = None,
|
||||
Some(_) => {}
|
||||
None => match byte {
|
||||
b'"' | b'\'' => quote = Some(byte),
|
||||
b'[' => depth += 1,
|
||||
b']' => depth = depth.saturating_sub(1),
|
||||
b'>' if depth == 0 => return Some(&rest[at + 1..]),
|
||||
_ => {}
|
||||
},
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// The name without its namespace prefix.
|
||||
fn local_name(name: &[u8]) -> &[u8] {
|
||||
match name.iter().rposition(|byte| *byte == b':') {
|
||||
Some(at) => &name[at + 1..],
|
||||
None => name,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_name_byte(byte: u8) -> bool {
|
||||
byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.' | b':') || byte >= 0x80
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const BOOK: &[u8] = b"<?xml version=\"1.0\" encoding=\"utf-8\"?>\n\
|
||||
<FictionBook xmlns=\"http://www.gribuser.ru/xml/fictionbook/2.0\">\n\
|
||||
<description><title-info><book-title>The Mill</book-title></title-info></description>";
|
||||
|
||||
#[test]
|
||||
fn a_fictionbook_is_recognised() {
|
||||
assert!(is_fictionbook(BOOK));
|
||||
assert!(is_fictionbook(b"<FictionBook>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_prolog_is_stepped_over() {
|
||||
let head = b"\xef\xbb\xbf <?xml version=\"1.0\"?>\r\n\
|
||||
<!-- converted by hand -->\n\
|
||||
<!DOCTYPE FictionBook [ <!ENTITY mill \"<b>the mill</b>\"> ]>\n\
|
||||
<?xml-stylesheet href=\"fb2.css\"?>\n\
|
||||
<FictionBook\n xmlns=\"http://www.gribuser.ru/xml/fictionbook/2.0\">";
|
||||
assert!(is_fictionbook(head));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_prefixed_root_is_still_a_fictionbook() {
|
||||
assert!(is_fictionbook(
|
||||
b"<fb:FictionBook xmlns:fb=\"http://www.gribuser.ru/xml/fictionbook/2.0\">"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn markup_need_not_be_utf8() {
|
||||
// windows-1251 for "Мельница" in the title.
|
||||
let head = b"<?xml version=\"1.0\" encoding=\"windows-1251\"?>\
|
||||
<FictionBook><description><book-title>\xcc\xe5\xeb\xfc\xed\xe8\xf6\xe0</book-title>";
|
||||
assert!(is_fictionbook(head));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_xml_is_not_a_fictionbook() {
|
||||
assert!(!is_fictionbook(
|
||||
b"<?xml version=\"1.0\"?>\n<rss version=\"2.0\">"
|
||||
));
|
||||
assert!(!is_fictionbook(b"<?xml version=\"1.0\"?><FictionBookish>"));
|
||||
assert!(!is_fictionbook(b"<html><FictionBook>"));
|
||||
assert!(!is_fictionbook(b"text before <FictionBook>"));
|
||||
assert!(!is_fictionbook(b""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_root_the_window_cut_short_is_not_taken() {
|
||||
assert!(!is_fictionbook(b"<?xml version=\"1.0\"?><FictionBook"));
|
||||
assert!(!is_fictionbook(b"<?xml version=\"1.0\"?><Fiction"));
|
||||
assert!(!is_fictionbook(b"<?xml version=\"1.0\""));
|
||||
assert!(!is_fictionbook(b"<!-- never closed <FictionBook>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_truncated_or_mangled_prolog_does_not_panic() {
|
||||
let head = b"\xef\xbb\xbf<?xml?><!--c--><!DOCTYPE x [ <!ENTITY a '>'> ] \"]\">\
|
||||
<FictionBook>";
|
||||
assert!(is_fictionbook(head));
|
||||
for cut in 0..head.len() {
|
||||
let _ = is_fictionbook(&head[..cut]);
|
||||
}
|
||||
for byte in 0..=255u8 {
|
||||
let _ = is_fictionbook(&[b'<', byte]);
|
||||
let _ = is_fictionbook(&[b'<', b'!', byte, b'>']);
|
||||
}
|
||||
}
|
||||
}
|
||||
+13
-254
@@ -9,7 +9,6 @@
|
||||
//! The containers are unambiguous, so they are read rather than guessed at.
|
||||
|
||||
pub mod bytes;
|
||||
pub mod fb2;
|
||||
pub mod mobi;
|
||||
pub mod zip;
|
||||
|
||||
@@ -23,32 +22,17 @@ const DJVU: &[u8] = b"AT&TFORM";
|
||||
const OLE2: &[u8] = &[0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1];
|
||||
/// A PDF announces its version in its first bytes.
|
||||
const PDF: &[u8] = b"%PDF-";
|
||||
/// Microsoft Reader's signature, ahead of everything else in the file.
|
||||
const LIT: &[u8] = b"ITOLITLS";
|
||||
/// The group every RTF document is, opened with the control word that names
|
||||
/// the format.
|
||||
const RTF: &[u8] = br"{\rtf";
|
||||
/// The UTF-8 byte-order mark.
|
||||
const BOM: &[u8] = b"\xef\xbb\xbf";
|
||||
|
||||
/// Archive members that carry no content: resource forks and desktop clutter.
|
||||
const ARCHIVE_NOISE: &[&str] = &["__macosx/", ".ds_store", "thumbs.db"];
|
||||
|
||||
/// What a comic book archive carries besides its pages: the catalogue entry
|
||||
/// comic readers write into it.
|
||||
const COMIC_NOISE: &[&str] = &["comicinfo.xml"];
|
||||
|
||||
/// The extensions of the images a comic book archive is made of.
|
||||
const PAGE_IMAGES: &[&str] = &[".jpg", ".jpeg", ".png", ".gif", ".webp"];
|
||||
|
||||
/// Substrings in a name that mark a file as a sample rather than a book.
|
||||
const SAMPLE_MARKERS: &[&str] = &["sample", "readme", "read me"];
|
||||
|
||||
/// Below this, a file is too small to be a book whatever it claims to be.
|
||||
const SMALLEST_BOOK: u64 = 4096;
|
||||
|
||||
/// How much of the front is read to tell RTF, FictionBook, markup and text
|
||||
/// apart.
|
||||
/// How much of the front is read to tell text from markup.
|
||||
const TEXT_WINDOW: usize = 4096;
|
||||
|
||||
/// Identify a file.
|
||||
@@ -65,17 +49,12 @@ pub fn sniff(source: &impl Bytes, name: &str) -> Doc {
|
||||
Kind::Pdf
|
||||
} else if source.magic_at(0, OLE2) {
|
||||
Kind::Ole2
|
||||
} else if source.magic_at(0, LIT) {
|
||||
Kind::Lit
|
||||
} else if source.magic_at(0, zip::LOCAL_HEADER) {
|
||||
archive(source, &mut doc)
|
||||
} else if let Some(kind) = mobi::kind(source) {
|
||||
kind
|
||||
} else {
|
||||
match source.slice(0, TEXT_WINDOW) {
|
||||
Ok(head) => front(&head, &mut doc),
|
||||
Err(_) => Kind::Unknown,
|
||||
}
|
||||
text(source, &mut doc)
|
||||
};
|
||||
|
||||
if doc.kind == Kind::Unknown {
|
||||
@@ -101,8 +80,7 @@ pub fn sniff(source: &impl Bytes, name: &str) -> Doc {
|
||||
doc
|
||||
}
|
||||
|
||||
/// What is inside a zip: an EPUB, a wrapped book, a comic, or a collection of
|
||||
/// pages.
|
||||
/// What is inside a zip: an EPUB, a wrapped PDF, or a collection of pages.
|
||||
fn archive(source: &impl Bytes, doc: &mut Doc) -> Kind {
|
||||
let Some(names) = zip::members(source) else {
|
||||
doc.note("a zip whose directory could not be read");
|
||||
@@ -121,13 +99,6 @@ fn archive(source: &impl Bytes, doc: &mut Doc) -> Kind {
|
||||
doc.note(format!("a zip holding one PDF, {pdf:?}"));
|
||||
return Kind::ZipWrappedPdf;
|
||||
}
|
||||
if let Some(fb2) = sole_member(&names, ".fb2") {
|
||||
doc.note(format!("a zip holding one FictionBook, {fb2:?}"));
|
||||
return Kind::ZipWrappedFb2;
|
||||
}
|
||||
if is_comic(&names) {
|
||||
return Kind::Cbz;
|
||||
}
|
||||
if lower
|
||||
.iter()
|
||||
.any(|name| name.ends_with(".htm") || name.ends_with(".html") || name.ends_with(".xhtml"))
|
||||
@@ -148,67 +119,25 @@ fn archive(source: &impl Bytes, doc: &mut Doc) -> Kind {
|
||||
/// a collection, and choosing among them would be a guess — better to say it
|
||||
/// is not a book than to name one chapter as the whole.
|
||||
pub fn sole_pdf(names: &[String]) -> Option<&str> {
|
||||
sole_member(names, ".pdf")
|
||||
}
|
||||
|
||||
/// The one member an archive exists to carry, if it is the only real member
|
||||
/// and its name ends in `extension`.
|
||||
fn sole_member<'a>(names: &'a [String], extension: &str) -> Option<&'a str> {
|
||||
match real_members(names).as_slice() {
|
||||
[only] if only.to_lowercase().ends_with(extension) => Some(only.as_str()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether an archive is the pages of a comic and nothing else.
|
||||
///
|
||||
/// Every real member has to be an image. One PDF or one text file among the
|
||||
/// pages makes it something else, and saying what would be a guess.
|
||||
fn is_comic(names: &[String]) -> bool {
|
||||
let pages: Vec<String> = real_members(names)
|
||||
.into_iter()
|
||||
.map(|name| name.to_lowercase())
|
||||
.filter(|name| {
|
||||
let file = name.rsplit('/').next().unwrap_or(name);
|
||||
!COMIC_NOISE.contains(&file)
|
||||
})
|
||||
.collect();
|
||||
!pages.is_empty()
|
||||
&& pages
|
||||
.iter()
|
||||
.all(|name| PAGE_IMAGES.iter().any(|image| name.ends_with(image)))
|
||||
}
|
||||
|
||||
/// The members that carry content: no directories, no desktop clutter.
|
||||
fn real_members(names: &[String]) -> Vec<&String> {
|
||||
names
|
||||
let real: Vec<&String> = names
|
||||
.iter()
|
||||
.filter(|name| !name.ends_with('/'))
|
||||
.filter(|name| {
|
||||
let lower = name.to_lowercase();
|
||||
!ARCHIVE_NOISE.iter().any(|noise| lower.contains(noise))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// What the front of a file says when it is not a container: RTF, a
|
||||
/// FictionBook, markup, text, or nothing this knows.
|
||||
fn front(head: &[u8], doc: &mut Doc) -> Kind {
|
||||
// Both are recognised on bytes rather than text. RTF is seven-bit by
|
||||
// design but not always in practice, and a FictionBook is as often
|
||||
// windows-1251 as UTF-8.
|
||||
let lead = head.strip_prefix(BOM).unwrap_or(head).trim_ascii_start();
|
||||
if lead.starts_with(RTF) {
|
||||
return Kind::Rtf;
|
||||
.collect();
|
||||
match real.as_slice() {
|
||||
[only] if only.to_lowercase().ends_with(".pdf") => Some(only.as_str()),
|
||||
_ => None,
|
||||
}
|
||||
if fb2::is_fictionbook(head) {
|
||||
return Kind::Fb2;
|
||||
}
|
||||
text(head, doc)
|
||||
}
|
||||
|
||||
/// Text, markup, or nothing this knows.
|
||||
fn text(head: &[u8], doc: &mut Doc) -> Kind {
|
||||
fn text(source: &impl Bytes, doc: &mut Doc) -> Kind {
|
||||
let Ok(head) = source.slice(0, TEXT_WINDOW) else {
|
||||
return Kind::Unknown;
|
||||
};
|
||||
if head.is_empty() {
|
||||
doc.note("the file is empty");
|
||||
return Kind::Unknown;
|
||||
@@ -220,7 +149,7 @@ fn text(head: &[u8], doc: &mut Doc) -> Kind {
|
||||
}
|
||||
// The window may have cut a character in half, which is not the same
|
||||
// thing as the file not being text.
|
||||
let head = match std::str::from_utf8(head) {
|
||||
let head = match std::str::from_utf8(&head) {
|
||||
Ok(text) => text,
|
||||
Err(e) if e.error_len().is_none() => {
|
||||
std::str::from_utf8(&head[..e.valid_up_to()]).expect("a valid prefix")
|
||||
@@ -249,15 +178,10 @@ fn names_the_same_thing(extension: &str, kind: Kind) -> bool {
|
||||
Kind::Djvu => matches!(extension, "djvu" | "djv"),
|
||||
Kind::Azw3 => matches!(extension, "azw3" | "azw" | "kfx"),
|
||||
Kind::Mobi => matches!(extension, "mobi" | "prc" | "azw"),
|
||||
Kind::Lit => extension == "lit",
|
||||
Kind::Fb2 => extension == "fb2",
|
||||
Kind::Rtf => extension == "rtf",
|
||||
Kind::Txt => matches!(extension, "txt" | "text" | "md" | "markdown"),
|
||||
Kind::Html => matches!(extension, "html" | "htm" | "xhtml" | "xml"),
|
||||
Kind::Ole2 => matches!(extension, "doc" | "xls" | "ppt"),
|
||||
Kind::ZipWrappedPdf => extension == "zip",
|
||||
Kind::ZipWrappedFb2 => matches!(extension, "zip" | "fbz"),
|
||||
Kind::Cbz => extension == "cbz",
|
||||
// Nothing was recognised, so there is nothing to disagree with.
|
||||
Kind::Unknown => true,
|
||||
}
|
||||
@@ -270,15 +194,10 @@ fn spelling(kind: Kind) -> &'static str {
|
||||
Kind::Djvu => "DjVu",
|
||||
Kind::Azw3 => "AZW3",
|
||||
Kind::Mobi => "MOBI",
|
||||
Kind::Lit => "Microsoft Reader",
|
||||
Kind::Fb2 => "FictionBook",
|
||||
Kind::Rtf => "RTF",
|
||||
Kind::Txt => "plain text",
|
||||
Kind::Html => "HTML",
|
||||
Kind::Ole2 => "an OLE2 document",
|
||||
Kind::ZipWrappedPdf => "a zip holding one PDF",
|
||||
Kind::ZipWrappedFb2 => "a zip holding one FictionBook",
|
||||
Kind::Cbz => "a comic book archive",
|
||||
Kind::Unknown => "nothing recognisable",
|
||||
}
|
||||
}
|
||||
@@ -413,164 +332,4 @@ mod tests {
|
||||
head.truncate(TEXT_WINDOW + 1);
|
||||
assert_eq!(sniff(&head.as_slice(), "a.txt").kind, Kind::Txt);
|
||||
}
|
||||
|
||||
fn archive_of(names: &[&str], name: &str) -> Doc {
|
||||
sniff(&zip::tests::zip(names).as_slice(), name)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rtf_is_recognised_by_its_opening_group() {
|
||||
assert_eq!(kind_of(br"{\rtf1\ansi\deff0 {\fonttbl", "a.rtf"), Kind::Rtf);
|
||||
assert_eq!(
|
||||
kind_of(b"\xef\xbb\xbf\r\n {\\rtf1\\ansi", "a.rtf"),
|
||||
Kind::Rtf
|
||||
);
|
||||
// Eight-bit text in the body does not stop it being RTF.
|
||||
assert_eq!(kind_of(b"{\\rtf1 caf\xe9 \x00", "a.rtf"), Kind::Rtf);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rtf_cut_short_of_its_control_word_is_not_rtf() {
|
||||
assert_eq!(sniff(&br"{\rt".as_slice(), "a.rtf").kind, Kind::Txt);
|
||||
assert_eq!(kind_of(br"{\rtx1", "a.rtf"), Kind::Txt);
|
||||
assert_eq!(kind_of(br"text {\rtf1", "a.rtf"), Kind::Txt);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rtf_named_as_a_word_document_is_remarked_on() {
|
||||
let doc = sniff(&padded(br"{\rtf1\ansi").as_slice(), "letter.doc");
|
||||
assert_eq!(doc.kind, Kind::Rtf);
|
||||
assert!(
|
||||
doc.notes
|
||||
.iter()
|
||||
.any(|note| note == "named .doc, but the bytes say RTF"),
|
||||
"{:?}",
|
||||
doc.notes
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn microsoft_reader_is_recognised_by_its_signature() {
|
||||
let doc = sniff(&padded(b"ITOLITLS\x01\x00\x00\x00").as_slice(), "book.lit");
|
||||
assert_eq!(doc.kind, Kind::Lit);
|
||||
assert!(doc.notes.is_empty(), "{:?}", doc.notes);
|
||||
assert_eq!(kind_of(b"ITOLITL", "book.lit"), Kind::Txt);
|
||||
assert_eq!(kind_of(b" ITOLITLS", "book.lit"), Kind::Txt);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fictionbook_is_recognised_and_other_xml_is_not() {
|
||||
let book = b"<?xml version=\"1.0\" encoding=\"windows-1251\"?>\n\
|
||||
<FictionBook xmlns=\"http://www.gribuser.ru/xml/fictionbook/2.0\">\
|
||||
<book-title>\xcc\xe5\xeb\xfc\xed\xe8\xf6\xe0</book-title>";
|
||||
let doc = sniff(&padded(book).as_slice(), "mill.fb2");
|
||||
assert_eq!(doc.kind, Kind::Fb2);
|
||||
assert!(doc.notes.is_empty(), "{:?}", doc.notes);
|
||||
|
||||
let feed = b"<?xml version=\"1.0\"?>\n<rss version=\"2.0\"><channel>";
|
||||
assert_eq!(kind_of(feed, "feed.xml"), Kind::Html);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fictionbook_named_as_something_else_is_remarked_on() {
|
||||
let doc = sniff(&padded(b"<FictionBook>").as_slice(), "mill.txt");
|
||||
assert_eq!(doc.kind, Kind::Fb2);
|
||||
assert!(
|
||||
doc.notes.iter().any(|note| note.contains("FictionBook")),
|
||||
"{:?}",
|
||||
doc.notes
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zip_holding_one_fictionbook_is_a_wrapped_one() {
|
||||
let doc = archive_of(&["__MACOSX/._mill.fb2", "mill.fb2"], "mill.fb2.zip");
|
||||
assert_eq!(doc.kind, Kind::ZipWrappedFb2);
|
||||
assert_eq!(doc.declared_ext.as_deref(), Some("zip"));
|
||||
assert!(
|
||||
doc.notes
|
||||
.iter()
|
||||
.any(|note| note.contains("one FictionBook")),
|
||||
"{:?}",
|
||||
doc.notes
|
||||
);
|
||||
assert!(!doc.notes.iter().any(|note| note.contains("named .")));
|
||||
|
||||
let two = archive_of(&["one.fb2", "two.fb2"], "pair.zip");
|
||||
assert_eq!(two.kind, Kind::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zip_of_page_images_is_a_comic() {
|
||||
let doc = archive_of(
|
||||
&[
|
||||
"Issue 1/",
|
||||
"Issue 1/001.jpg",
|
||||
"Issue 1/002.JPEG",
|
||||
"Issue 1/003.png",
|
||||
"Issue 1/004.gif",
|
||||
"Issue 1/005.webp",
|
||||
"ComicInfo.xml",
|
||||
"__MACOSX/Issue 1/._001.jpg",
|
||||
"Issue 1/Thumbs.db",
|
||||
".DS_Store",
|
||||
],
|
||||
"issue-1.cbz",
|
||||
);
|
||||
assert_eq!(doc.kind, Kind::Cbz);
|
||||
assert!(
|
||||
!doc.notes.iter().any(|note| note.contains("named .")),
|
||||
"{:?}",
|
||||
doc.notes
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_comic_saved_as_a_zip_is_remarked_on() {
|
||||
let doc = archive_of(&["001.jpg", "002.jpg"], "issue-1.zip");
|
||||
assert_eq!(doc.kind, Kind::Cbz);
|
||||
assert!(
|
||||
doc.notes.iter().any(|note| note.contains("comic")),
|
||||
"{:?}",
|
||||
doc.notes
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zip_with_anything_but_images_is_not_a_comic() {
|
||||
let one_pdf = archive_of(&["001.jpg", "002.jpg", "extras.pdf"], "issue-1.cbz");
|
||||
assert_eq!(one_pdf.kind, Kind::Unknown);
|
||||
assert!(
|
||||
one_pdf
|
||||
.notes
|
||||
.iter()
|
||||
.any(|note| note.contains("none of them a book")),
|
||||
"{:?}",
|
||||
one_pdf.notes
|
||||
);
|
||||
|
||||
let readme = archive_of(&["001.jpg", "readme.txt"], "issue-1.cbz");
|
||||
assert_eq!(readme.kind, Kind::Unknown);
|
||||
|
||||
// A name that ends like an image without being one.
|
||||
let lookalike = archive_of(&["001.jpg", "notes.jpg.txt"], "issue-1.cbz");
|
||||
assert_eq!(lookalike.kind, Kind::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zip_with_no_pages_is_not_a_comic() {
|
||||
let catalogue_only = archive_of(&["ComicInfo.xml", "pages/"], "issue-1.cbz");
|
||||
assert_eq!(catalogue_only.kind, Kind::Unknown);
|
||||
|
||||
let empty = archive_of(&[], "issue-1.cbz");
|
||||
assert_eq!(empty.kind, Kind::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_truncated_comic_does_not_panic() {
|
||||
let whole = zip::tests::zip(&["001.jpg", "002.png", "ComicInfo.xml"]);
|
||||
for cut in 0..whole.len() {
|
||||
let _ = sniff(&&whole[..cut], "issue-1.cbz");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -78,11 +78,11 @@ fn directory_start(source: &impl Bytes) -> Option<(usize, u64)> {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Build a zip holding `names`, each with empty stored content.
|
||||
pub(crate) fn zip(names: &[&str]) -> Vec<u8> {
|
||||
fn zip(names: &[&str]) -> Vec<u8> {
|
||||
let mut out: Vec<u8> = Vec::new();
|
||||
let mut offsets = Vec::new();
|
||||
for name in names {
|
||||
|
||||
@@ -64,30 +64,6 @@ fn several_files_come_back_as_several_records() {
|
||||
assert_eq!(kinds, ["\"pdf\"", "\"djvu\""]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_formats_read_from_the_front_come_back_by_their_wire_names() {
|
||||
let rtf = written("letter.rtf", &padded(br"{\rtf1\ansi"));
|
||||
let lit = written("book.lit", &padded(b"ITOLITLS"));
|
||||
let fb2 = written(
|
||||
"mill.fb2",
|
||||
&padded(b"<?xml version=\"1.0\"?>\n<FictionBook xmlns=\"http://www.gribuser.ru/xml/fictionbook/2.0\">"),
|
||||
);
|
||||
let out = run(&[
|
||||
rtf.to_str().expect("utf-8 path"),
|
||||
lit.to_str().expect("utf-8 path"),
|
||||
fb2.to_str().expect("utf-8 path"),
|
||||
]);
|
||||
assert_eq!(out.status.code(), Some(0));
|
||||
let kinds: Vec<String> = String::from_utf8(out.stdout)
|
||||
.expect("utf-8")
|
||||
.lines()
|
||||
.map(|line| {
|
||||
serde_json::from_str::<serde_json::Value>(line).expect("valid JSON")["kind"].to_string()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(kinds, ["\"rtf\"", "\"lit\"", "\"fb2\""]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_that_is_not_there_is_an_io_error_and_the_rest_still_run() {
|
||||
let path = written("kept.pdf", &padded(b"%PDF-1.7"));
|
||||
|
||||
@@ -77,12 +77,6 @@ pub enum Kind {
|
||||
Azw3,
|
||||
/// Amazon MOBI.
|
||||
Mobi,
|
||||
/// Microsoft Reader.
|
||||
Lit,
|
||||
/// FictionBook, the XML ebook format.
|
||||
Fb2,
|
||||
/// Rich Text Format.
|
||||
Rtf,
|
||||
/// Plain text.
|
||||
Txt,
|
||||
/// HTML or XHTML.
|
||||
@@ -91,10 +85,6 @@ pub enum Kind {
|
||||
Ole2,
|
||||
/// A zip archive whose only real content is a single PDF.
|
||||
ZipWrappedPdf,
|
||||
/// A zip archive whose only real content is a single FictionBook.
|
||||
ZipWrappedFb2,
|
||||
/// A comic book archive: a zip of page images.
|
||||
Cbz,
|
||||
/// Recognised as none of the above.
|
||||
Unknown,
|
||||
}
|
||||
@@ -128,11 +118,6 @@ mod tests {
|
||||
(Kind::Ole2, "\"ole2\""),
|
||||
(Kind::Azw3, "\"azw3\""),
|
||||
(Kind::ZipWrappedPdf, "\"zip-wrapped-pdf\""),
|
||||
(Kind::ZipWrappedFb2, "\"zip-wrapped-fb2\""),
|
||||
(Kind::Fb2, "\"fb2\""),
|
||||
(Kind::Lit, "\"lit\""),
|
||||
(Kind::Rtf, "\"rtf\""),
|
||||
(Kind::Cbz, "\"cbz\""),
|
||||
(Kind::Unknown, "\"unknown\""),
|
||||
];
|
||||
for (kind, wire) in cases {
|
||||
|
||||
@@ -30,8 +30,6 @@ pub const UNAVAILABLE: u8 = 69;
|
||||
pub const SOFTWARE: u8 = 70;
|
||||
/// Reading or writing failed.
|
||||
pub const IO: u8 = 74;
|
||||
/// A service refused for now, and asking again later will work.
|
||||
pub const TEMPFAIL: u8 = 75;
|
||||
|
||||
/// The answer a predicate tool gives.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
|
||||
+1
-2
@@ -47,7 +47,6 @@ that was concatenated or written on Windows still reads.
|
||||
| `69` | A program it needs is not installed |
|
||||
| `70` | Its own fault |
|
||||
| `74` | Reading or writing failed |
|
||||
| `75` | A service refused for now; try again later |
|
||||
|
||||
Failures start where `sysexits.h` puts them, so "no" and "broke" are never the
|
||||
same number. That is what makes this work:
|
||||
@@ -65,7 +64,7 @@ No network, no database, no configuration file, no state kept between runs.
|
||||
Five exceptions, each declared in `--help`:
|
||||
|
||||
- [`emb`](../crates/emb/README.md) makes an HTTP call, which is the whole of what it does.
|
||||
- [`bibfetch`](../crates/bibfetch/README.md) asks a register about an identifier or a title, likewise.
|
||||
- [`bibfetch`](../crates/bibfetch/README.md) asks a register about an identifier, likewise.
|
||||
- [`pdftriage`](../crates/pdftriage/README.md) runs `pdftotext` and `pdfinfo`.
|
||||
- [`pdfgutter`](../crates/pdfgutter/README.md) runs `pdftoppm` to look at a page.
|
||||
- [`avplan`](../crates/avplan/README.md) runs `ffprobe` to read a file's streams, and
|
||||
|
||||
@@ -20,7 +20,6 @@ to any of them is a change to a decision rather than to an accident.
|
||||
- [pdfgutter — a page tree that does not add up is refused](#pdfgutter--a-page-tree-that-does-not-add-up-is-refused)
|
||||
- [pdfgutter — a sheet on its side is cut the way it is read](#pdfgutter--a-sheet-on-its-side-is-cut-the-way-it-is-read)
|
||||
- [pdfgutter — an unreadable file is an answer, not a false](#pdfgutter--an-unreadable-file-is-an-answer-not-a-false)
|
||||
- [bookmeta — a cover is what its bytes say it is](#bookmeta--a-cover-is-what-its-bytes-say-it-is)
|
||||
- [Every pattern — two regex engines, two Unicode tables](#every-pattern--two-regex-engines-two-unicode-tables)
|
||||
|
||||
## cuenorm — settings after the end time are the format, not a malformation
|
||||
@@ -244,23 +243,6 @@ answers, and the [contract](contract.md) gives them three different numbers. A
|
||||
caller wanting them collapsed treats 1 and 2 alike; a caller wanting to know
|
||||
which files it failed on can now find out.
|
||||
|
||||
## bookmeta — a cover is what its bytes say it is
|
||||
|
||||
An EPUB's manifest gives every image a media type, and the cover's is not
|
||||
used: the type comes from the image's first bytes, and a disagreement is a
|
||||
note. Elsewhere `bookmeta` reports what the file says rather than what is
|
||||
true, and this is the one place it does not.
|
||||
|
||||
The difference is that a title nobody believes is still a title, while a
|
||||
media type that is wrong is an image that does not open. A caller handed
|
||||
`image/png` for a JPEG writes `cover.png`, serves it as PNG, or hands it to a
|
||||
decoder that refuses it. An image can be replaced by a converter or by hand
|
||||
without its manifest entry changing, and the note keeps what the file said,
|
||||
so nothing is lost by correcting it.
|
||||
|
||||
The same reasoning goes the other way: bytes that are not an image are not a
|
||||
cover, whatever the manifest calls them.
|
||||
|
||||
## Every pattern — two regex engines, two Unicode tables
|
||||
|
||||
Patterns here spell their character classes explicitly rather than leaning on
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ Part of [catena](../README.md).
|
||||
| Field | Meaning |
|
||||
|---|---|
|
||||
| `path` | Where the file was read from, as given |
|
||||
| `kind` | `pdf`, `epub`, `djvu`, `azw3`, `mobi`, `lit`, `fb2`, `rtf`, `txt`, `html`, `ole2`, `cbz`, `zip-wrapped-pdf`, `zip-wrapped-fb2`, `unknown` |
|
||||
| `kind` | `pdf`, `epub`, `djvu`, `azw3`, `mobi`, `txt`, `html`, `ole2`, `zip-wrapped-pdf`, `unknown` |
|
||||
| `declared_ext` | The extension the name claimed, lowercased, without the dot |
|
||||
| `bytes` | Size on disk |
|
||||
| `pages` | Page count, for paginated formats |
|
||||
|
||||
Reference in new issue
Block a user