mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-11-13 23:55:40 +00:00
Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 funkpower
|
03c19c7293 | ||
|
funkpower
|
ce8332d6d8 | ||
|
funkpower
|
b4000b9b67 | ||
|
funkpower
|
23969269e8 | ||
|
funkpower
|
452102639a | ||
|
funkpower
|
a105b1c9aa | ||
|
funkpower
|
1abed4632d | ||
|
funkpower
|
130324bbe0 | ||
|
funkpower
|
19e5f3ddc3 | ||
|
funkpower
|
b7d98e9d1a | ||
|
funkpower
|
6f32f52293 | ||
|
funkpower
|
e480f07c32 | ||
|
funkpower
|
d551adc14c |
No files matched your search
Generated
+26
-26
@@ -101,7 +101,7 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "avplan"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -126,7 +126,7 @@ checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "bibclean"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -143,7 +143,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bibfetch"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -196,7 +196,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bookmeta"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -210,7 +210,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "booksniff"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -250,7 +250,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
|
||||
|
||||
[[package]]
|
||||
name = "catena"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"bibfetch",
|
||||
@@ -277,7 +277,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "catena-cli"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"clap_mangen",
|
||||
@@ -312,7 +312,7 @@ checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||
|
||||
[[package]]
|
||||
name = "chunkmd"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -334,7 +334,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "citefind"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"chunkmd",
|
||||
@@ -399,7 +399,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "colophon"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"catena-cli",
|
||||
@@ -448,7 +448,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "corpus-core"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"schemars",
|
||||
"serde",
|
||||
@@ -492,7 +492,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cuekit"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"corpus-core",
|
||||
"regex",
|
||||
@@ -504,7 +504,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cuenorm"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -514,7 +514,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cuepack"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -525,7 +525,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cueseg"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -612,7 +612,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "emb"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -876,7 +876,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "idnorm"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"catena-cli",
|
||||
@@ -942,7 +942,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "langid"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1022,7 +1022,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mdscrub"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1034,7 +1034,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mdsplit"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1113,7 +1113,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "pdfgutter"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1126,7 +1126,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdftriage"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1370,7 +1370,7 @@ checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189"
|
||||
|
||||
[[package]]
|
||||
name = "rrf"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1517,7 +1517,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "seriesof"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1634,7 +1634,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
||||
|
||||
[[package]]
|
||||
name = "textkit"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"regex",
|
||||
"tiktoken-rs",
|
||||
@@ -1724,7 +1724,7 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
|
||||
|
||||
[[package]]
|
||||
name = "tocfind"
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
|
||||
+27
-27
@@ -3,39 +3,39 @@ resolver = "3"
|
||||
members = ["crates/*"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.2.3"
|
||||
version = "0.3.1"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MPL-2.0"
|
||||
repository = "https://git.pumpin.online/funkpower/catena"
|
||||
|
||||
[workspace.dependencies]
|
||||
catena-cli = { path = "crates/catena-cli", version = "0.2.3" }
|
||||
corpus-core = { path = "crates/corpus-core", version = "0.2.3" }
|
||||
cuekit = { path = "crates/cuekit", version = "0.2.3" }
|
||||
textkit = { path = "crates/textkit", version = "0.2.3" }
|
||||
chunkmd = { path = "crates/chunkmd", version = "0.2.3" }
|
||||
mdscrub = { path = "crates/mdscrub", version = "0.2.3" }
|
||||
mdsplit = { path = "crates/mdsplit", version = "0.2.3" }
|
||||
pdftriage = { path = "crates/pdftriage", version = "0.2.3" }
|
||||
cuenorm = { path = "crates/cuenorm", version = "0.2.3" }
|
||||
cueseg = { path = "crates/cueseg", version = "0.2.3" }
|
||||
rrf = { path = "crates/rrf", version = "0.2.3" }
|
||||
cuepack = { path = "crates/cuepack", version = "0.2.3" }
|
||||
bibclean = { path = "crates/bibclean", version = "0.2.3" }
|
||||
avplan = { path = "crates/avplan", version = "0.2.3" }
|
||||
pdfgutter = { path = "crates/pdfgutter", version = "0.2.3" }
|
||||
catena = { path = "crates/bindings", version = "0.2.3" }
|
||||
seriesof = { path = "crates/seriesof", version = "0.2.3" }
|
||||
idnorm = { path = "crates/idnorm", version = "0.2.3" }
|
||||
bibfetch = { path = "crates/bibfetch", version = "0.2.3" }
|
||||
bookmeta = { path = "crates/bookmeta", version = "0.2.3" }
|
||||
langid = { path = "crates/langid", version = "0.2.3" }
|
||||
tocfind = { path = "crates/tocfind", version = "0.2.3" }
|
||||
booksniff = { path = "crates/booksniff", version = "0.2.3" }
|
||||
citefind = { path = "crates/citefind", version = "0.2.3" }
|
||||
colophon = { path = "crates/colophon", version = "0.2.3" }
|
||||
emb = { path = "crates/emb", version = "0.2.3" }
|
||||
catena-cli = { path = "crates/catena-cli", version = "0.3.1" }
|
||||
corpus-core = { path = "crates/corpus-core", version = "0.3.1" }
|
||||
cuekit = { path = "crates/cuekit", version = "0.3.1" }
|
||||
textkit = { path = "crates/textkit", version = "0.3.1" }
|
||||
chunkmd = { path = "crates/chunkmd", version = "0.3.1" }
|
||||
mdscrub = { path = "crates/mdscrub", version = "0.3.1" }
|
||||
mdsplit = { path = "crates/mdsplit", version = "0.3.1" }
|
||||
pdftriage = { path = "crates/pdftriage", version = "0.3.1" }
|
||||
cuenorm = { path = "crates/cuenorm", version = "0.3.1" }
|
||||
cueseg = { path = "crates/cueseg", version = "0.3.1" }
|
||||
rrf = { path = "crates/rrf", version = "0.3.1" }
|
||||
cuepack = { path = "crates/cuepack", version = "0.3.1" }
|
||||
bibclean = { path = "crates/bibclean", version = "0.3.1" }
|
||||
avplan = { path = "crates/avplan", version = "0.3.1" }
|
||||
pdfgutter = { path = "crates/pdfgutter", version = "0.3.1" }
|
||||
catena = { path = "crates/bindings", version = "0.3.1" }
|
||||
seriesof = { path = "crates/seriesof", version = "0.3.1" }
|
||||
idnorm = { path = "crates/idnorm", version = "0.3.1" }
|
||||
bibfetch = { path = "crates/bibfetch", version = "0.3.1" }
|
||||
bookmeta = { path = "crates/bookmeta", version = "0.3.1" }
|
||||
langid = { path = "crates/langid", version = "0.3.1" }
|
||||
tocfind = { path = "crates/tocfind", version = "0.3.1" }
|
||||
booksniff = { path = "crates/booksniff", version = "0.3.1" }
|
||||
citefind = { path = "crates/citefind", version = "0.3.1" }
|
||||
colophon = { path = "crates/colophon", version = "0.3.1" }
|
||||
emb = { path = "crates/emb", version = "0.3.1" }
|
||||
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
clap_mangen = "0.3"
|
||||
|
||||
@@ -34,7 +34,7 @@ Each tool installs by name, so you can take one without taking the rest.
|
||||
| [`vocabmerge`](crates/bibclean/README.md#vocabmerge--the-other-command) | The near-duplicate headings in a catalogue's vocabulary |
|
||||
| [`seriesof`](crates/seriesof/README.md) | The series a book belongs to, and which volume this is |
|
||||
| [`idnorm`](crates/idnorm/README.md) | DOIs, ISSNs, ISBNs and arXiv ids, read and checked |
|
||||
| [`bibfetch`](crates/bibfetch/README.md) | What a register says about an identifier |
|
||||
| [`bibfetch`](crates/bibfetch/README.md) | What a register says about an identifier, and which books it has under a title |
|
||||
| [`citefind`](crates/citefind/README.md) | The references in a bibliography or a page of notes |
|
||||
| [`mdscrub`](crates/mdscrub/README.md) | Converted markdown in, clean markdown out |
|
||||
| [`mdsplit`](crates/mdsplit/README.md) | One markdown file into the works inside it |
|
||||
|
||||
+118
-2
@@ -20,6 +20,8 @@ So this asks, and that is the only thing it does.
|
||||
- **Filling in a reference** you have only the identifier for.
|
||||
- **Before citing something**, to check the year and the authors are what you
|
||||
think they are.
|
||||
- **Cataloguing a book that came with no identifier**, to find the editions a
|
||||
register has under its title, and an ISBN to file it by.
|
||||
|
||||
## Install
|
||||
|
||||
@@ -107,6 +109,118 @@ register that is rate-limiting, short enough that one which is simply down
|
||||
does not hold up a pipeline. An answer this cannot read is not retried,
|
||||
because it will not read any better next time.
|
||||
|
||||
## Covers
|
||||
|
||||
```sh
|
||||
bibfetch cover 978-0-306-40615-7 -o cover.jpg
|
||||
```
|
||||
|
||||
Fetches a book's cover from Open Library's covers store by its ISBN and
|
||||
writes the image. The thirteen-digit form is asked first and the ten-digit
|
||||
form second, because a book is often filed under only one of them; an ISBN
|
||||
whose check digit is wrong is asked about as written, since converting it
|
||||
would give another book's number. `--cover-id` names an Open Library cover id
|
||||
to ask by last, which is what the `cover_id` field of a lookup record holds:
|
||||
the first cover in Open Library's record of that edition, never the cover its
|
||||
index picked for the whole work, which is often another edition's. The cover
|
||||
id is asked only when every form of the ISBN asked was answered with no cover; if
|
||||
asking by ISBN failed, that failure is the result.
|
||||
|
||||
What comes back is checked before it is written: a JPEG or PNG by its bytes
|
||||
as well as by what the server says, no larger than ten megabytes, reached in at
|
||||
most three redirects, each over HTTPS to `covers.openlibrary.org`,
|
||||
`archive.org` or a host under `archive.org`, and never to an address by number.
|
||||
One request is made at a time, a `429` or `503` is waited out for as long as
|
||||
its `Retry-After` says, up to a minute, and the whole fetch, every attempt and
|
||||
redirect included, stops after two minutes. The two minutes start once it is
|
||||
the fetch's turn, so a fetch queued behind others in the same process can wait
|
||||
that long again first.
|
||||
|
||||
| Option | Effect |
|
||||
|---|---|
|
||||
| `-o`, `--output <FILE>` | Where to write the image |
|
||||
| `--size <SIZE>` | `small`, `medium` or `large`. Default `large` |
|
||||
| `--cover-id <ID>` | An Open Library cover id to ask by when the ISBN has none |
|
||||
|
||||
Exit `0` the cover was written, `1` Open Library has none, `65` the argument
|
||||
was not an ISBN or the answer was not an image, `69` Open Library could not be
|
||||
reached, `74` the file could not be written.
|
||||
|
||||
From Rust, `bibfetch::cover(&agent, isbn, cover_id, size)` returns
|
||||
`Ok(None)` when there is no cover, and a [`Cover`](src/cover.rs) with the
|
||||
bytes, the media type and the address asked when there is.
|
||||
|
||||
## Searching by title
|
||||
|
||||
```
|
||||
$ bibfetch search --title 'The First World War' --author Howard --limit 2
|
||||
{"v":1,"id":"9780192853622","kind":"isbn","found":true,"source":"openlibrary","title":"The First World War","authors":["Michael Eliot Howard"],"year":2002,"container":"Oxford University Press","subjects":["World War, 1914-1918"],"language":"eng","pages":154,"cover_id":1123124}
|
||||
{"v":1,"id":"9780199205592","kind":"isbn","found":true,"source":"openlibrary","title":"The First World War: A Very Short Introduction","authors":["Michael Howard"],"year":2007,"container":"Oxford University Press","language":"eng","cover_id":14606475}
|
||||
```
|
||||
|
||||
A book that arrives as a file often has no identifier to ask about, only a
|
||||
title and perhaps a name. This asks which books a register has under those,
|
||||
and prints one record per candidate: Open Library's index first, and Google
|
||||
Books only when Open Library has nothing.
|
||||
|
||||
Each candidate is one edition. Open Library's index answers with works, and
|
||||
names the edition of each that best matches the search; what belongs to an
|
||||
edition — its ISBN, publisher, language, year and cover — is that edition's,
|
||||
and what belongs to the work — the title, the authors, the subjects and the
|
||||
length — is the work's. Google Books answers with volumes, which are editions
|
||||
already.
|
||||
|
||||
| Field | When the candidate has an ISBN | When it has none |
|
||||
|---|---|---|
|
||||
| `id` | The first ISBN whose check digit agrees, in thirteen digits; one written in thirteen is taken over one written in ten | The register's own key: an Open Library id such as `OL37863121M` for the edition, or `OL267171W` for the work when no edition was named; a Google Books volume id |
|
||||
| `kind` | `isbn` | `olid` for Open Library, `googlebooks` for Google Books |
|
||||
|
||||
An ISBN whose check digit is wrong is passed over rather than corrected, and
|
||||
Google's shelfmarks, typed `OTHER`, are not read as ISBNs even when their
|
||||
digits would pass for one. A record of kind `isbn` can be handed straight back
|
||||
to `bibfetch` for everything the registers say about that edition; `olid` and
|
||||
`googlebooks` are names no lookup here answers about.
|
||||
|
||||
`cover_id` is the same thing it is in a lookup record: the first cover in
|
||||
Open Library's record of the edition the record names, by that edition's ISBN
|
||||
or its key, so `bibfetch cover --cover-id` fetches that edition's cover and no
|
||||
other. The cover the index picked for the whole work is never asked for,
|
||||
because it is often another edition's. A work the index names no edition of
|
||||
gives the first ISBN it lists that reads, which may be any edition's, and no
|
||||
cover id; Google Books knows nothing of Open Library's covers and gives none.
|
||||
|
||||
The candidates are what the register said, in the order it said them. This
|
||||
makes no judgement about which one is the book in hand, and keeps one that is
|
||||
plainly another book, because a title is not an identifier: *The Hamlet* comes
|
||||
back for *Hamlet*. Deciding whether a candidate is the same work is a
|
||||
comparison of titles, and [`bibclean`](../bibclean/README.md)'s `same_work`
|
||||
is one.
|
||||
|
||||
```sh
|
||||
bibfetch search --title 'Anathem' --author Stephenson \
|
||||
| jq -r 'select(.kind == "isbn") | .id' | head -1
|
||||
```
|
||||
|
||||
Open Library answering that it has nothing is an answer, and it stands when
|
||||
Google Books cannot be reached; a register being down is a failure only when
|
||||
both are. Each is asked the same way a lookup asks, retried with the same
|
||||
waits. Open Library is sent the title and the author as written; Google Books
|
||||
is sent each word as its own `intitle:` or `inauthor:` term, so nothing in a
|
||||
title is read as search syntax.
|
||||
|
||||
| Option | Effect |
|
||||
|---|---|
|
||||
| `--title <TITLE>` | The title, or as much of it as you have |
|
||||
| `--author <AUTHOR>` | An author's name, or part of one |
|
||||
| `--limit <COUNT>` | The most candidates to print, `1` to `40`. Default `10` |
|
||||
|
||||
Exit `0` at least one candidate was printed, `1` no register has the title,
|
||||
`65` the title was empty, `69` no register could be reached, `74` the output
|
||||
could not be written.
|
||||
|
||||
From Rust, `bibfetch::search(&agent, title, author, limit, attempts)` returns
|
||||
the candidates, empty when no register has the title.
|
||||
|
||||
## Reaching the network
|
||||
|
||||
This and [`emb`](../emb/README.md) are the only two tools here that do. Both
|
||||
@@ -114,10 +228,12 @@ say so in `--help`, and for both it is the whole of what they do rather than
|
||||
something that happens on the way to something else — which is the condition
|
||||
under which [the contract](../../docs/contract.md) allows it at all.
|
||||
|
||||
Nothing is cached, nothing is stored, and nothing is sent but the identifier
|
||||
Nothing is cached, nothing is stored but a cover you asked to have written,
|
||||
and nothing is sent but the identifier, or the title and author searched for,
|
||||
and the address you pass to `--polite`.
|
||||
|
||||
## See also
|
||||
|
||||
- [`idnorm`](../idnorm/README.md) — reading the identifier in the first place.
|
||||
- [`bibclean`](../bibclean/README.md) — the rest of a catalogue record.
|
||||
- [`bibclean`](../bibclean/README.md) — the rest of a catalogue record, and
|
||||
whether a candidate is the same work.
|
||||
+41
-31
@@ -63,13 +63,18 @@ pub fn lookup(
|
||||
for source in chain {
|
||||
match ask(agent, *source, id, kind, attempts) {
|
||||
Ok(record) => {
|
||||
let complete = record.found && record.title.is_some() && !record.authors.is_empty();
|
||||
let better = best.as_ref().is_none_or(|kept| record.found && !kept.found);
|
||||
if complete {
|
||||
return Ok(Some(record));
|
||||
match &mut best {
|
||||
// Merged rather than replaced. The registers answer about
|
||||
// different parts of one book: OpenLibrary's edition record
|
||||
// carries the publisher, the pagination and the language
|
||||
// and names its authors by key, while its search index
|
||||
// carries the authors and not the publisher. Keeping
|
||||
// whichever arrived first throws the other half away.
|
||||
Some(kept) => kept.fill_from(record),
|
||||
None => best = Some(record),
|
||||
}
|
||||
if better {
|
||||
best = Some(record);
|
||||
if best.as_ref().is_some_and(Record::complete) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(why) => last_error = Some(why),
|
||||
@@ -84,10 +89,6 @@ pub fn lookup(
|
||||
}
|
||||
|
||||
/// Ask a register, retrying a failure that might not be one.
|
||||
///
|
||||
/// Waits a second before the second attempt and two before the third: long
|
||||
/// enough for a register that is rate-limiting, short enough that one which
|
||||
/// is simply down does not hold up a pipeline.
|
||||
fn ask(
|
||||
agent: &ureq::Agent,
|
||||
source: Source,
|
||||
@@ -95,39 +96,48 @@ fn ask(
|
||||
kind: &str,
|
||||
attempts: u32,
|
||||
) -> Result<crate::Record, Error> {
|
||||
let mut last = Error::Unreachable("no attempt was made".into());
|
||||
match get(agent, &source.url(id), attempts)? {
|
||||
Some(body) => source.read(id, kind, &body),
|
||||
// A register that does not have it says so with a status, and that is
|
||||
// an answer rather than a failure.
|
||||
None => Ok(crate::missing(id, kind, source)),
|
||||
}
|
||||
}
|
||||
|
||||
for attempt in 0..attempts.max(1) {
|
||||
match fetch(agent, source, id, kind) {
|
||||
Ok(record) => return Ok(record),
|
||||
// A shape this cannot read will not read any better next time.
|
||||
Err(e @ Error::Shape(_)) => return Err(e),
|
||||
/// The body at an address, or `None` when the register says it has no such
|
||||
/// thing; a failure is asked again.
|
||||
///
|
||||
/// Waits a second before the second attempt and two before the third: long
|
||||
/// enough for a register that is rate-limiting, short enough that one which
|
||||
/// is simply down does not hold up a pipeline. Reading what came back is left
|
||||
/// to the caller, because an answer this cannot read will not read any better
|
||||
/// next time.
|
||||
pub(crate) fn get(agent: &ureq::Agent, url: &str, attempts: u32) -> Result<Option<String>, Error> {
|
||||
let attempts = attempts.max(1);
|
||||
let mut last = Error::Unreachable("no attempt was made".into());
|
||||
for attempt in 0..attempts {
|
||||
match once(agent, url) {
|
||||
Ok(answer) => return Ok(answer),
|
||||
Err(e) => last = e,
|
||||
}
|
||||
std::thread::sleep(Duration::from_secs(1 << attempt));
|
||||
if attempt + 1 < attempts {
|
||||
std::thread::sleep(Duration::from_secs(1 << attempt.min(5)));
|
||||
}
|
||||
}
|
||||
Err(last)
|
||||
}
|
||||
|
||||
fn fetch(
|
||||
agent: &ureq::Agent,
|
||||
source: Source,
|
||||
id: &str,
|
||||
kind: &str,
|
||||
) -> Result<crate::Record, Error> {
|
||||
let response = agent.get(source.url(id)).call();
|
||||
let mut response = match response {
|
||||
fn once(agent: &ureq::Agent, url: &str) -> Result<Option<String>, Error> {
|
||||
let mut response = match agent.get(url).call() {
|
||||
Ok(response) => response,
|
||||
// A register that does not have it says so with a status, and that is
|
||||
// an answer rather than a failure.
|
||||
Err(ureq::Error::StatusCode(404)) => return Ok(crate::missing(id, kind, source)),
|
||||
Err(ureq::Error::StatusCode(404)) => return Ok(None),
|
||||
Err(e) => return Err(Error::Unreachable(e.to_string())),
|
||||
};
|
||||
let body = response
|
||||
response
|
||||
.body_mut()
|
||||
.read_to_string()
|
||||
.map_err(|e| Error::Unreachable(e.to_string()))?;
|
||||
source.read(id, kind, &body)
|
||||
.map(Some)
|
||||
.map_err(|e| Error::Unreachable(e.to_string()))
|
||||
}
|
||||
|
||||
/// Who is asking.
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -12,8 +12,14 @@
|
||||
//! them: Crossref for a DOI, the arXiv for a preprint, OpenLibrary for a
|
||||
//! book. There is no register for an ISSN that answers without a key, so an
|
||||
//! ISSN is looked up as the journal of a work rather than on its own.
|
||||
//!
|
||||
//! A book with no identifier at all can still be looked for by its title,
|
||||
//! with [`search()`]: Open Library's index, then Google Books, each candidate
|
||||
//! a record of one edition, in the order the register gave them.
|
||||
|
||||
pub mod ask;
|
||||
pub mod cover;
|
||||
pub mod search;
|
||||
pub mod source;
|
||||
|
||||
use std::time::Duration;
|
||||
@@ -21,6 +27,8 @@ use std::time::Duration;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub use cover::{Cover, CoverSize, cover};
|
||||
pub use search::search;
|
||||
pub use source::Source;
|
||||
|
||||
/// How long to wait on a register before giving up.
|
||||
@@ -59,6 +67,28 @@ pub struct Record {
|
||||
/// The abstract, where the register carries one.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub summary: Option<String>,
|
||||
/// The subject headings the register files it under, as it words them.
|
||||
///
|
||||
/// A register states these for far more books than print a cataloguing
|
||||
/// block, which makes them the second source a shelf otherwise lacks.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub subjects: Vec<String>,
|
||||
/// The language it is written in, as the register gives it: a two- or
|
||||
/// three-letter code, or a name. Not normalised here — what a register
|
||||
/// said is the record, and deciding that "eng" and "English" are one
|
||||
/// thing belongs to whoever is filing them.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub language: Option<String>,
|
||||
/// How many pages the register says it has.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub pages: Option<u32>,
|
||||
/// The first cover id in Open Library's record of this edition, for
|
||||
/// [`cover()`] to ask by when the ISBN has none. Taken from the edition
|
||||
/// record only, never from the index's cover for the work, which is
|
||||
/// often another edition's; `None` when the edition names no cover or
|
||||
/// the lookup stopped before asking for the edition record.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cover_id: Option<u64>,
|
||||
}
|
||||
|
||||
fn version() -> u32 {
|
||||
@@ -80,6 +110,8 @@ pub enum Error {
|
||||
Shape(String),
|
||||
/// Nothing here knows where to ask about this kind.
|
||||
NoRegister(String),
|
||||
/// What was asked about is not an identifier of the kind it should be.
|
||||
Invalid(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Error {
|
||||
@@ -90,6 +122,7 @@ impl std::fmt::Display for Error {
|
||||
Error::NoRegister(kind) => {
|
||||
write!(f, "no register here answers about {kind} identifiers")
|
||||
}
|
||||
Error::Invalid(why) => write!(f, "{why}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -112,6 +145,74 @@ pub fn missing(id: &str, kind: &str, source: Source) -> Record {
|
||||
year: None,
|
||||
container: None,
|
||||
summary: None,
|
||||
subjects: Vec::new(),
|
||||
language: None,
|
||||
pages: None,
|
||||
cover_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
impl Record {
|
||||
/// Whether this record says everything a catalogue wants.
|
||||
///
|
||||
/// Used to decide when to stop asking. Title and authors are the minimum
|
||||
/// that makes a record usable; the rest is worth another register only
|
||||
/// while one is left to ask.
|
||||
pub fn complete(&self) -> bool {
|
||||
self.found
|
||||
&& self.title.is_some()
|
||||
&& !self.authors.is_empty()
|
||||
&& self.container.is_some()
|
||||
&& self.language.is_some()
|
||||
}
|
||||
|
||||
/// Take from another record whatever this one does not have.
|
||||
///
|
||||
/// The registers are complementary rather than redundant, and asking them
|
||||
/// in turn and keeping one answer throws away the difference. OpenLibrary's
|
||||
/// edition record gives the publisher, the pagination and the language but
|
||||
/// names its authors by key; its search index gives the authors but not the
|
||||
/// publisher. One book, two half-answers, and the whole of it only if they
|
||||
/// are put together.
|
||||
///
|
||||
/// Nothing already answered is overwritten, so the order of the chain is
|
||||
/// the order of trust.
|
||||
pub fn fill_from(&mut self, other: Record) {
|
||||
if !other.found {
|
||||
return;
|
||||
}
|
||||
self.found = true;
|
||||
if self.title.is_none() {
|
||||
self.title = other.title;
|
||||
}
|
||||
if self.authors.is_empty() {
|
||||
self.authors = other.authors;
|
||||
}
|
||||
if self.year.is_none() {
|
||||
self.year = other.year;
|
||||
}
|
||||
if self.container.is_none() {
|
||||
self.container = other.container;
|
||||
}
|
||||
if self.summary.is_none() {
|
||||
self.summary = other.summary;
|
||||
}
|
||||
if self.subjects.is_empty() {
|
||||
self.subjects = other.subjects;
|
||||
}
|
||||
if self.language.is_none() {
|
||||
self.language = other.language;
|
||||
}
|
||||
if self.pages.is_none() {
|
||||
self.pages = other.pages;
|
||||
}
|
||||
if self.cover_id.is_none() {
|
||||
self.cover_id = other.cover_id;
|
||||
}
|
||||
// Which register answered becomes which registers did.
|
||||
if !self.source.split('+').any(|s| s == other.source) {
|
||||
self.source = format!("{}+{}", self.source, other.source);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,3 +236,74 @@ mod tests {
|
||||
assert!(json.contains("\"found\":false"), "{json}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod merge_tests {
|
||||
use super::*;
|
||||
|
||||
fn rec(source: &str) -> Record {
|
||||
let mut r = missing("9780199205592", "isbn", Source::OpenLibrary);
|
||||
r.source = source.to_string();
|
||||
r.found = true;
|
||||
r
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_half_answers_make_one_record() {
|
||||
// Exactly what the two OpenLibrary endpoints give for one book: the
|
||||
// edition record has the publisher and the language and names its
|
||||
// authors by key; the index has the authors and not the publisher.
|
||||
let mut edition = rec("openlibrary");
|
||||
edition.title = Some("The First World War: A Very Short Introduction".into());
|
||||
edition.container = Some("Oxford University Press".into());
|
||||
edition.language = Some("eng".into());
|
||||
|
||||
let mut index = rec("openlibrary");
|
||||
index.title = Some("The First World War".into());
|
||||
index.authors = vec!["Michael Howard".into()];
|
||||
index.year = Some(2007);
|
||||
|
||||
edition.fill_from(index);
|
||||
assert_eq!(edition.authors, ["Michael Howard"]);
|
||||
assert_eq!(edition.year, Some(2007));
|
||||
// The fuller title arrived first and is not displaced by the shorter.
|
||||
assert_eq!(
|
||||
edition.title.as_deref(),
|
||||
Some("The First World War: A Very Short Introduction")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_register_that_has_nothing_contributes_nothing() {
|
||||
let mut kept = rec("openlibrary");
|
||||
kept.title = Some("A Book".into());
|
||||
let absent = missing("9780199205592", "isbn", Source::GoogleBooks);
|
||||
kept.fill_from(absent);
|
||||
assert_eq!(kept.title.as_deref(), Some("A Book"));
|
||||
assert_eq!(kept.source, "openlibrary");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_record_names_every_register_that_answered() {
|
||||
let mut a = rec("openlibrary");
|
||||
a.title = Some("A Book".into());
|
||||
let mut b = rec("googlebooks");
|
||||
b.found = true;
|
||||
b.pages = Some(134);
|
||||
a.fill_from(b);
|
||||
assert_eq!(a.source, "openlibrary+googlebooks");
|
||||
assert_eq!(a.pages, Some(134));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_means_a_catalogue_has_what_it_needs() {
|
||||
let mut r = rec("openlibrary");
|
||||
assert!(!r.complete());
|
||||
r.title = Some("A Book".into());
|
||||
r.authors = vec!["Someone".into()];
|
||||
assert!(!r.complete(), "still missing the publisher and language");
|
||||
r.container = Some("A Press".into());
|
||||
r.language = Some("eng".into());
|
||||
assert!(r.complete());
|
||||
}
|
||||
}
|
||||
+158
-11
@@ -29,33 +29,117 @@ Exit codes: 0 every identifier was found, 1 at least one was not,
|
||||
Example:
|
||||
|
||||
idnorm refs.txt | bibfetch | jq -r 'select(.found) | .title'
|
||||
idnorm --kind doi refs.txt | bibfetch --polite you@example.org"
|
||||
idnorm --kind doi refs.txt | bibfetch --polite you@example.org
|
||||
bibfetch cover 978-0-306-40615-7 -o cover.jpg
|
||||
bibfetch search --title 'The First World War' --author Howard"
|
||||
)]
|
||||
struct Args {
|
||||
#[command(subcommand)]
|
||||
command: Option<Command>,
|
||||
|
||||
/// Identifier records to look up, as `idnorm` writes them. Reads stdin
|
||||
/// when absent.
|
||||
path: Option<PathBuf>,
|
||||
|
||||
/// An address to put in the request, which the registers ask for and
|
||||
/// answer faster when they get.
|
||||
#[arg(long, value_name = "EMAIL")]
|
||||
#[arg(long, global = true, value_name = "EMAIL")]
|
||||
polite: Option<String>,
|
||||
|
||||
/// How long to wait on a register, in seconds.
|
||||
#[arg(long, default_value_t = bibfetch::TIMEOUT.as_secs(), value_name = "SECONDS")]
|
||||
#[arg(long, global = true, default_value_t = bibfetch::TIMEOUT.as_secs(), value_name = "SECONDS")]
|
||||
timeout: u64,
|
||||
|
||||
/// How many times to ask before believing a failure.
|
||||
#[arg(long, default_value_t = bibfetch::ATTEMPTS, value_name = "COUNT")]
|
||||
#[arg(long, global = true, default_value_t = bibfetch::ATTEMPTS, value_name = "COUNT")]
|
||||
attempts: u32,
|
||||
}
|
||||
|
||||
#[derive(clap::Subcommand)]
|
||||
enum Command {
|
||||
/// Fetch a book's cover from Open Library by its ISBN.
|
||||
///
|
||||
/// Exit codes: 0 the cover was written, 1 Open Library has none,
|
||||
/// 65 not an ISBN or not an image, 69 Open Library could not be reached,
|
||||
/// 74 the file could not be written.
|
||||
Cover {
|
||||
/// The ISBN, ten or thirteen digits, hyphens or not.
|
||||
isbn: String,
|
||||
/// Where to write the image.
|
||||
#[arg(short, long, value_name = "FILE")]
|
||||
output: PathBuf,
|
||||
/// Which size to ask for.
|
||||
#[arg(long, value_enum, default_value_t = Size::Large)]
|
||||
size: Size,
|
||||
/// An Open Library cover id to ask by when the ISBN has no cover.
|
||||
#[arg(long, value_name = "ID")]
|
||||
cover_id: Option<u64>,
|
||||
},
|
||||
/// Find the books a register has under a title, and an author.
|
||||
///
|
||||
/// Prints one record per candidate edition, in the order the register
|
||||
/// gave them: Open Library's index, or Google Books when that has
|
||||
/// nothing. Which one is the book in hand is left to you.
|
||||
///
|
||||
/// Exit codes: 0 at least one candidate, 1 none, 65 no title,
|
||||
/// 69 no register could be reached, 74 the output could not be written.
|
||||
Search {
|
||||
/// The title, or as much of it as you have.
|
||||
#[arg(long)]
|
||||
title: String,
|
||||
/// An author's name, or part of one.
|
||||
#[arg(long)]
|
||||
author: Option<String>,
|
||||
/// The most candidates to print.
|
||||
#[arg(long, default_value_t = 10, value_name = "COUNT",
|
||||
value_parser = clap::value_parser!(u16).range(1..=bibfetch::search::MAX_LIMIT as i64))]
|
||||
limit: u16,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, clap::ValueEnum)]
|
||||
enum Size {
|
||||
Small,
|
||||
Medium,
|
||||
Large,
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let args: Args = match catena_cli::parse() {
|
||||
Ok(args) => args,
|
||||
Err(code) => return code,
|
||||
};
|
||||
|
||||
let agent = bibfetch::ask::Ask {
|
||||
polite: args.polite.clone(),
|
||||
timeout: Duration::from_secs(args.timeout),
|
||||
attempts: args.attempts,
|
||||
}
|
||||
.agent();
|
||||
|
||||
match &args.command {
|
||||
Some(Command::Cover {
|
||||
isbn,
|
||||
output,
|
||||
size,
|
||||
cover_id,
|
||||
}) => return cover(&agent, args.attempts, isbn, output, *size, *cover_id),
|
||||
Some(Command::Search {
|
||||
title,
|
||||
author,
|
||||
limit,
|
||||
}) => {
|
||||
return search(
|
||||
&agent,
|
||||
args.attempts,
|
||||
title,
|
||||
author.as_deref(),
|
||||
usize::from(*limit),
|
||||
);
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
|
||||
let text = match catena_cli::read_text(args.path.as_deref()) {
|
||||
Ok(text) => text,
|
||||
Err((code, why)) => {
|
||||
@@ -64,13 +148,6 @@ fn main() -> ExitCode {
|
||||
}
|
||||
};
|
||||
|
||||
let agent = bibfetch::ask::Ask {
|
||||
polite: args.polite.clone(),
|
||||
timeout: Duration::from_secs(args.timeout),
|
||||
attempts: args.attempts,
|
||||
}
|
||||
.agent();
|
||||
|
||||
let mut records = Vec::new();
|
||||
let mut missing = 0usize;
|
||||
|
||||
@@ -114,6 +191,76 @@ fn main() -> ExitCode {
|
||||
})
|
||||
}
|
||||
|
||||
/// Fetch one cover and write it where asked.
|
||||
fn cover(
|
||||
agent: &ureq::Agent,
|
||||
attempts: u32,
|
||||
isbn: &str,
|
||||
output: &std::path::Path,
|
||||
size: Size,
|
||||
cover_id: Option<u64>,
|
||||
) -> ExitCode {
|
||||
let covers = bibfetch::cover::Covers {
|
||||
attempts,
|
||||
..Default::default()
|
||||
};
|
||||
let size = match size {
|
||||
Size::Small => bibfetch::CoverSize::Small,
|
||||
Size::Medium => bibfetch::CoverSize::Medium,
|
||||
Size::Large => bibfetch::CoverSize::Large,
|
||||
};
|
||||
match covers.fetch(agent, isbn, cover_id, size) {
|
||||
Ok(Some(found)) => match std::fs::write(output, &found.bytes) {
|
||||
Ok(()) => ExitCode::from(exit::OK),
|
||||
Err(why) => {
|
||||
eprintln!("bibfetch: {}: {why}", output.display());
|
||||
ExitCode::from(exit::IO)
|
||||
}
|
||||
},
|
||||
Ok(None) => {
|
||||
eprintln!("bibfetch: {isbn}: Open Library has no cover");
|
||||
ExitCode::from(exit::NO)
|
||||
}
|
||||
Err(why) => {
|
||||
eprintln!("bibfetch: {isbn}: {why}");
|
||||
ExitCode::from(match why {
|
||||
bibfetch::Error::Unreachable(_) => exit::UNAVAILABLE,
|
||||
_ => exit::DATA,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Search by title and print the candidates.
|
||||
fn search(
|
||||
agent: &ureq::Agent,
|
||||
attempts: u32,
|
||||
title: &str,
|
||||
author: Option<&str>,
|
||||
limit: usize,
|
||||
) -> ExitCode {
|
||||
match bibfetch::search(agent, title, author, limit, attempts) {
|
||||
Ok(found) => {
|
||||
let wrote = catena_cli::stream("bibfetch", &found);
|
||||
if format!("{wrote:?}") != format!("{:?}", ExitCode::from(exit::OK)) {
|
||||
return wrote;
|
||||
}
|
||||
if found.is_empty() {
|
||||
eprintln!("bibfetch: no register has {title:?}");
|
||||
return ExitCode::from(exit::NO);
|
||||
}
|
||||
ExitCode::from(exit::OK)
|
||||
}
|
||||
Err(why) => {
|
||||
eprintln!("bibfetch: {why}");
|
||||
ExitCode::from(match why {
|
||||
bibfetch::Error::Invalid(_) => exit::DATA,
|
||||
_ => exit::UNAVAILABLE,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The identifier and its kind out of one record.
|
||||
fn read_identifier(line: &str) -> Result<(String, String), String> {
|
||||
let value: serde_json::Value =
|
||||
|
||||
@@ -0,0 +1,780 @@
|
||||
//! Books by their title, for when there is no identifier to ask about.
|
||||
//!
|
||||
//! A book that arrives as a file often carries no ISBN at all, only a title
|
||||
//! and perhaps a name. A register can still be asked which books it has under
|
||||
//! those, and what it answers is a list of candidates: each an edition the
|
||||
//! register holds, with whatever identifier, publisher and cover it gives.
|
||||
//!
|
||||
//! Which candidate is the book in hand is not decided here. The candidates
|
||||
//! are what the register said, in the order it said them; deciding whether
|
||||
//! one is the same work is a comparison of titles, and that belongs to
|
||||
//! whoever holds the book.
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::source::{self, GoogleIdentifier, GoogleReply, SUBJECT_LIMIT, Source};
|
||||
use crate::{Error, Record};
|
||||
|
||||
/// The most candidates one search asks for. Google Books answers with no more
|
||||
/// than forty at a time, and a title that needs more than forty candidates to
|
||||
/// find the right one needs a better title.
|
||||
pub const MAX_LIMIT: usize = 40;
|
||||
|
||||
/// What Open Library's index is asked to say about each work, and about the
|
||||
/// one edition of it that best matches the search.
|
||||
///
|
||||
/// Not the work's own `cover_i` or `cover_edition_key`: the cover the index
|
||||
/// chose for a work is often another edition's. The edition's `cover_i` is
|
||||
/// the first cover in that edition's record, which is what a lookup by ISBN
|
||||
/// gives as `cover_id`.
|
||||
const OPEN_LIBRARY_FIELDS: &str = "key,title,subtitle,author_name,first_publish_year,publisher,\
|
||||
isbn,language,subject,number_of_pages_median,editions,editions.key,editions.isbn,\
|
||||
editions.publisher,editions.language,editions.publish_date,editions.cover_i";
|
||||
|
||||
/// Ask which books a register has under a title, and by an author if one is
|
||||
/// named.
|
||||
///
|
||||
/// Open Library's index first, and Google Books only when that answered with
|
||||
/// nothing. A register being down is a failure only when every register
|
||||
/// asked is: Open Library answering that it has nothing is an answer, and it
|
||||
/// stands when Google Books cannot be reached. Asks nothing when `limit` is
|
||||
/// zero; a `limit` over [`MAX_LIMIT`] is taken as that.
|
||||
pub fn search(
|
||||
agent: &ureq::Agent,
|
||||
title: &str,
|
||||
author: Option<&str>,
|
||||
limit: usize,
|
||||
attempts: u32,
|
||||
) -> Result<Vec<Record>, Error> {
|
||||
search_with(title, author, limit, |url| {
|
||||
crate::ask::get(agent, url, attempts)
|
||||
})
|
||||
}
|
||||
|
||||
/// [`search`], with how to fetch an address passed in.
|
||||
fn search_with(
|
||||
title: &str,
|
||||
author: Option<&str>,
|
||||
limit: usize,
|
||||
mut get: impl FnMut(&str) -> Result<Option<String>, Error>,
|
||||
) -> Result<Vec<Record>, Error> {
|
||||
let title = title.trim();
|
||||
if title.is_empty() {
|
||||
return Err(Error::Invalid("no title to search for".into()));
|
||||
}
|
||||
let author = author.map(str::trim).filter(|author| !author.is_empty());
|
||||
let limit = limit.min(MAX_LIMIT);
|
||||
if limit == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut fetch = |url: String| match get(&url)? {
|
||||
Some(body) => Ok(body),
|
||||
// A search that matches nothing answers with an empty list. A 404 is
|
||||
// the address being wrong, not the book being absent.
|
||||
None => Err(Error::Unreachable(format!("{url}: answered 404"))),
|
||||
};
|
||||
|
||||
let open_library =
|
||||
fetch(open_library_url(title, author, limit)).and_then(|body| open_library(&body, limit));
|
||||
if open_library.as_ref().is_ok_and(|found| !found.is_empty()) {
|
||||
return open_library;
|
||||
}
|
||||
|
||||
let google = match google_url(title, author, limit) {
|
||||
Some(url) => fetch(url).and_then(|body| google_books(&body, limit)),
|
||||
// Nothing in the title Google could be asked by.
|
||||
None => Ok(Vec::new()),
|
||||
};
|
||||
match (open_library, google) {
|
||||
(_, Ok(found)) if !found.is_empty() => Ok(found),
|
||||
(Ok(none), _) => Ok(none),
|
||||
(Err(why), _) => Err(why),
|
||||
}
|
||||
}
|
||||
|
||||
/// Where to ask Open Library.
|
||||
fn open_library_url(title: &str, author: Option<&str>, limit: usize) -> String {
|
||||
let author = author
|
||||
.map(|author| format!("&author={}", source::encode(author)))
|
||||
.unwrap_or_default();
|
||||
format!(
|
||||
"https://openlibrary.org/search.json?title={}{author}&fields={OPEN_LIBRARY_FIELDS}&limit={limit}",
|
||||
source::encode(title),
|
||||
)
|
||||
}
|
||||
|
||||
/// Where to ask Google Books, or `None` when there is nothing to ask by.
|
||||
///
|
||||
/// Every word is its own `intitle:` or `inauthor:` term, so no character of
|
||||
/// the title can open a phrase, negate a term or name another field: a word is
|
||||
/// the letters, digits and apostrophes in it, and anything else separates
|
||||
/// words.
|
||||
fn google_url(title: &str, author: Option<&str>, limit: usize) -> Option<String> {
|
||||
let terms: Vec<String> = words(title)
|
||||
.map(|word| format!("intitle:{word}"))
|
||||
.chain(
|
||||
author
|
||||
.into_iter()
|
||||
.flat_map(words)
|
||||
.map(|word| format!("inauthor:{word}")),
|
||||
)
|
||||
.collect();
|
||||
if !terms.iter().any(|term| term.starts_with("intitle:")) {
|
||||
return None;
|
||||
}
|
||||
Some(format!(
|
||||
"https://www.googleapis.com/books/v1/volumes?q={}&maxResults={limit}",
|
||||
source::encode(&terms.join(" "))
|
||||
))
|
||||
}
|
||||
|
||||
fn words(text: &str) -> impl Iterator<Item = &str> {
|
||||
text.split(|c: char| !(c.is_alphanumeric() || c == '\''))
|
||||
.map(|word| word.trim_matches('\''))
|
||||
.filter(|word| !word.is_empty())
|
||||
}
|
||||
|
||||
/// The first ISBN in a list whose check digit agrees, in its thirteen-digit
|
||||
/// form. One written in thirteen digits is taken over one written in ten.
|
||||
fn first_isbn<'a>(isbns: impl Iterator<Item = &'a str> + Clone) -> Option<String> {
|
||||
isbns
|
||||
.clone()
|
||||
.filter(|isbn| idnorm::isbn::digits(isbn).len() == 13)
|
||||
.find_map(idnorm::isbn::to_13)
|
||||
.or_else(|| isbns.into_iter().find_map(idnorm::isbn::to_13))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct OpenLibraryReply {
|
||||
#[serde(default)]
|
||||
docs: Vec<Work>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Work {
|
||||
#[serde(default)]
|
||||
key: Option<String>,
|
||||
#[serde(default)]
|
||||
title: Option<String>,
|
||||
#[serde(default)]
|
||||
subtitle: Option<String>,
|
||||
#[serde(default)]
|
||||
author_name: Vec<String>,
|
||||
#[serde(default)]
|
||||
first_publish_year: Option<i64>,
|
||||
#[serde(default)]
|
||||
publisher: Vec<String>,
|
||||
#[serde(default)]
|
||||
isbn: Vec<String>,
|
||||
#[serde(default)]
|
||||
language: Vec<String>,
|
||||
#[serde(default)]
|
||||
subject: Vec<String>,
|
||||
#[serde(default)]
|
||||
number_of_pages_median: Option<u32>,
|
||||
#[serde(default)]
|
||||
editions: Option<Editions>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Editions {
|
||||
#[serde(default)]
|
||||
docs: Vec<Edition>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Edition {
|
||||
#[serde(default)]
|
||||
key: Option<String>,
|
||||
#[serde(default)]
|
||||
isbn: Vec<String>,
|
||||
#[serde(default)]
|
||||
publisher: Vec<String>,
|
||||
#[serde(default)]
|
||||
language: Vec<String>,
|
||||
#[serde(default)]
|
||||
publish_date: Vec<String>,
|
||||
#[serde(default)]
|
||||
cover_i: Option<i64>,
|
||||
}
|
||||
|
||||
/// The last segment of an Open Library key: `OL3585423M` out of
|
||||
/// `/books/OL3585423M`. The letter at the end says what it names, `M` an
|
||||
/// edition and `W` a work.
|
||||
fn olid(key: &str) -> Option<String> {
|
||||
key.rsplit('/')
|
||||
.next()
|
||||
.filter(|id| !id.is_empty())
|
||||
.map(str::to_string)
|
||||
}
|
||||
|
||||
/// Read Open Library's answer into one candidate per work, in its order.
|
||||
///
|
||||
/// What belongs to an edition — its ISBN, publisher, language, year and
|
||||
/// cover — is taken from the edition the index chose as matching best; what
|
||||
/// belongs to the work — its title, authors, subjects and length — from the
|
||||
/// work. A work the index names no edition of gives its first ISBN that reads,
|
||||
/// which may be any edition's, and no cover id.
|
||||
fn open_library(body: &str, limit: usize) -> Result<Vec<Record>, Error> {
|
||||
let reply: OpenLibraryReply =
|
||||
serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?;
|
||||
Ok(reply
|
||||
.docs
|
||||
.into_iter()
|
||||
.filter_map(candidate)
|
||||
.take(limit)
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn candidate(work: Work) -> Option<Record> {
|
||||
let edition = work.editions.and_then(|e| e.docs.into_iter().next());
|
||||
|
||||
// The cover is kept only when the record names the edition it is the
|
||||
// cover of, by that edition's ISBN or its key.
|
||||
let (id, kind, edition_named) = match &edition {
|
||||
Some(edition) => match first_isbn(edition.isbn.iter().map(String::as_str)) {
|
||||
Some(isbn) => (isbn, "isbn", true),
|
||||
None => match edition.key.as_deref().and_then(olid) {
|
||||
Some(key) => (key, "olid", true),
|
||||
None => (work.key.as_deref().and_then(olid)?, "olid", false),
|
||||
},
|
||||
},
|
||||
None => match first_isbn(work.isbn.iter().map(String::as_str)) {
|
||||
Some(isbn) => (isbn, "isbn", false),
|
||||
// A work with no key cannot be named, so it is not a candidate.
|
||||
None => (work.key.as_deref().and_then(olid)?, "olid", false),
|
||||
},
|
||||
};
|
||||
|
||||
let mut record = source::blank(&id, kind, Source::OpenLibrarySearch);
|
||||
record.title = match (work.title, work.subtitle) {
|
||||
(Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")),
|
||||
(title, _) => title,
|
||||
};
|
||||
record.authors = work.author_name;
|
||||
record.subjects = work.subject;
|
||||
record.subjects.truncate(SUBJECT_LIMIT);
|
||||
record.pages = work.number_of_pages_median;
|
||||
|
||||
let (publisher, language, year) = match edition {
|
||||
Some(edition) => {
|
||||
if edition_named {
|
||||
record.cover_id = edition
|
||||
.cover_i
|
||||
.and_then(|id| u64::try_from(id).ok())
|
||||
.filter(|id| *id > 0);
|
||||
}
|
||||
(
|
||||
edition.publisher.into_iter().next(),
|
||||
edition.language.into_iter().next(),
|
||||
edition
|
||||
.publish_date
|
||||
.iter()
|
||||
.find_map(|date| source::year_in(date)),
|
||||
)
|
||||
}
|
||||
None => (None, None, None),
|
||||
};
|
||||
record.container = publisher.or_else(|| work.publisher.into_iter().next());
|
||||
record.language = language.or_else(|| work.language.into_iter().next());
|
||||
record.year = year.or(work.first_publish_year);
|
||||
Some(record)
|
||||
}
|
||||
|
||||
/// Read Google Books' answer into one candidate per volume, in its order.
|
||||
fn google_books(body: &str, limit: usize) -> Result<Vec<Record>, Error> {
|
||||
let reply: GoogleReply = serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?;
|
||||
Ok(reply
|
||||
.items
|
||||
.into_iter()
|
||||
.filter_map(|item| {
|
||||
// Only what Google types as an ISBN: its `OTHER` identifiers are
|
||||
// library shelfmarks, some of which are ten digits long.
|
||||
let isbns = item
|
||||
.volume_info
|
||||
.identifiers
|
||||
.iter()
|
||||
.filter(|GoogleIdentifier { kind, .. }| kind == "ISBN_13" || kind == "ISBN_10")
|
||||
.map(|identifier| identifier.identifier.as_str());
|
||||
let (id, kind) = match first_isbn(isbns) {
|
||||
Some(isbn) => (isbn, "isbn"),
|
||||
None => (item.id.filter(|id| !id.is_empty())?, "googlebooks"),
|
||||
};
|
||||
Some(source::google_volume(&id, kind, item.volume_info))
|
||||
})
|
||||
.take(limit)
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Open Library's answer to `title=the first world war&author=howard`,
|
||||
/// recorded 2026-11-13 with the fields above and `limit=2`, the ISBN and
|
||||
/// subject lists shortened.
|
||||
const HOWARD: &str = r#"{"numFound": 9, "start": 0, "numFoundExact": true, "num_found": 9,
|
||||
"documentation_url": "https://openlibrary.org/dev/docs/api/search", "q": "", "offset": null,
|
||||
"docs": [
|
||||
{"author_name": ["Michael Eliot Howard"], "first_publish_year": 2002,
|
||||
"isbn": ["9780192853622", "0192853627"], "key": "/works/OL1382726W",
|
||||
"language": ["eng"], "number_of_pages_median": 154,
|
||||
"publisher": ["Oxford University Press"], "title": "The First World War",
|
||||
"subject": ["World War, 1914-1918"],
|
||||
"editions": {"numFound": 1, "start": 0, "numFoundExact": true, "docs": [
|
||||
{"key": "/books/OL3585423M", "cover_i": 1123124, "language": ["eng"],
|
||||
"publisher": ["Oxford University Press"], "publish_date": ["2002"],
|
||||
"isbn": ["0192853627", "9780192853622"]}]}},
|
||||
{"author_name": ["Michael Howard"], "first_publish_year": 2007,
|
||||
"isbn": ["0199205590", "9780199205592"], "key": "/works/OL4282399W",
|
||||
"language": ["eng"], "publisher": ["Oxford University Press"],
|
||||
"subtitle": "A Very Short Introduction", "title": "The First World War",
|
||||
"editions": {"numFound": 1, "start": 0, "numFoundExact": true, "docs": [
|
||||
{"key": "/books/OL7403004M", "cover_i": 14606475, "language": ["eng"],
|
||||
"publisher": ["Oxford University Press"], "publish_date": ["20 May 2007"],
|
||||
"isbn": ["9780199205592", "0199205590"]}]},
|
||||
"subject": []}]}"#;
|
||||
|
||||
/// Open Library's answer to `title=war and peace&author=tolstoy`, recorded
|
||||
/// 2026-11-13, first work only. The edition the index matched best is a
|
||||
/// 1917 printing that has no ISBN, though other editions of the work do.
|
||||
const TOLSTOY: &str = r#"{"numFound": 146, "docs": [
|
||||
{"author_name": ["Leo Tolstoy"], "first_publish_year": 1867,
|
||||
"isbn": ["9781524881993", "1645178889"], "key": "/works/OL267171W",
|
||||
"title": "War and Peace",
|
||||
"editions": {"numFound": 487, "docs": [
|
||||
{"key": "/books/OL37863121M", "cover_i": 12736340, "language": ["eng"],
|
||||
"publisher": ["Thomas Y. Crowell"], "publish_date": ["1917"]}]}}]}"#;
|
||||
|
||||
/// Open Library's answer when nothing has the title.
|
||||
const NOTHING: &str = r#"{"numFound": 0, "start": 0, "numFoundExact": true, "num_found": 0,
|
||||
"documentation_url": "https://openlibrary.org/dev/docs/api/search", "q": "",
|
||||
"offset": null, "docs": []}"#;
|
||||
|
||||
/// Google Books' answer for a volume, in the shape its API documents.
|
||||
const GOOGLE: &str = r#"{"kind": "books#volumes", "totalItems": 2, "items": [
|
||||
{"kind": "books#volume", "id": "zyTCAlFPjgYC", "volumeInfo": {
|
||||
"title": "The Google Story", "authors": ["David A. Vise", "Mark Malseed"],
|
||||
"publisher": "Random House Digital, Inc.", "publishedDate": "2005-11-15",
|
||||
"industryIdentifiers": [
|
||||
{"type": "ISBN_10", "identifier": "055380457X"},
|
||||
{"type": "ISBN_13", "identifier": "9780553804577"}],
|
||||
"pageCount": 207, "categories": ["Browsers (Computer programs)"],
|
||||
"language": "en"}},
|
||||
{"kind": "books#volume", "id": "a1b2C3d4", "volumeInfo": {
|
||||
"title": "The Google Story", "subtitle": "A Dissertation",
|
||||
"publishedDate": "2009",
|
||||
"industryIdentifiers": [
|
||||
{"type": "OTHER", "identifier": "UOM:0306406152"}]}}]}"#;
|
||||
|
||||
/// Fetch from a table of recorded bodies, keeping every address asked.
|
||||
fn recorded<'a>(
|
||||
asked: &'a mut Vec<String>,
|
||||
bodies: &'a [(&'a str, Result<&'a str, Error>)],
|
||||
) -> impl FnMut(&str) -> Result<Option<String>, Error> + 'a {
|
||||
move |url| {
|
||||
asked.push(url.to_string());
|
||||
let (_, body) = bodies
|
||||
.iter()
|
||||
.find(|(host, _)| url.contains(host))
|
||||
.unwrap_or_else(|| panic!("asked an address nothing was recorded for: {url}"));
|
||||
body.clone().map(|body| Some(body.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
fn down() -> Error {
|
||||
Error::Unreachable("connection refused".into())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_work_open_library_has_is_a_candidate_named_by_its_isbn() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [("openlibrary.org", Ok(HOWARD))];
|
||||
let found = search_with(
|
||||
"The First World War",
|
||||
Some("Howard"),
|
||||
5,
|
||||
recorded(&mut asked, &bodies),
|
||||
)
|
||||
.expect("candidates");
|
||||
assert_eq!(
|
||||
asked.len(),
|
||||
1,
|
||||
"Google is not asked when Open Library answers"
|
||||
);
|
||||
|
||||
assert_eq!(found.len(), 2);
|
||||
let first = &found[0];
|
||||
assert!(first.found);
|
||||
assert_eq!(first.v, corpus_core::V);
|
||||
assert_eq!(first.source, "openlibrary");
|
||||
// Written ten-digit first by the edition, given in thirteen.
|
||||
assert_eq!(
|
||||
(first.id.as_str(), first.kind.as_str()),
|
||||
("9780192853622", "isbn")
|
||||
);
|
||||
assert_eq!(first.title.as_deref(), Some("The First World War"));
|
||||
assert_eq!(first.authors, ["Michael Eliot Howard"]);
|
||||
assert_eq!(first.year, Some(2002));
|
||||
assert_eq!(first.container.as_deref(), Some("Oxford University Press"));
|
||||
assert_eq!(first.language.as_deref(), Some("eng"));
|
||||
assert_eq!(first.subjects, ["World War, 1914-1918"]);
|
||||
assert_eq!(first.pages, Some(154));
|
||||
assert_eq!(first.cover_id, Some(1123124));
|
||||
|
||||
let second = &found[1];
|
||||
assert_eq!(second.id, "9780199205592");
|
||||
assert_eq!(
|
||||
second.title.as_deref(),
|
||||
Some("The First World War: A Very Short Introduction")
|
||||
);
|
||||
// The edition's year, not the work's first.
|
||||
assert_eq!(second.year, Some(2007));
|
||||
// The first cover in OL7403004M's own record; the work's cover is
|
||||
// 1137779, which is not.
|
||||
assert_eq!(second.cover_id, Some(14606475));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_candidates_stay_in_the_order_the_register_gave_them() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [("openlibrary.org", Ok(HOWARD))];
|
||||
let found =
|
||||
search_with("first world war", None, 5, recorded(&mut asked, &bodies)).expect("ok");
|
||||
let ids: Vec<&str> = found.iter().map(|r| r.id.as_str()).collect();
|
||||
assert_eq!(ids, ["9780192853622", "9780199205592"]);
|
||||
let found =
|
||||
search_with("first world war", None, 1, recorded(&mut asked, &bodies)).expect("ok");
|
||||
assert_eq!(found.len(), 1);
|
||||
assert_eq!(found[0].id, "9780192853622");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_edition_with_no_isbn_is_named_by_its_key_and_keeps_its_cover() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [("openlibrary.org", Ok(TOLSTOY))];
|
||||
let found = search_with(
|
||||
"War and Peace",
|
||||
Some("Tolstoy"),
|
||||
5,
|
||||
recorded(&mut asked, &bodies),
|
||||
)
|
||||
.expect("candidates");
|
||||
let only = &found[0];
|
||||
// Not 9781524881993: that is another edition's, and the publisher,
|
||||
// year and cover here are this one's.
|
||||
assert_eq!(
|
||||
(only.id.as_str(), only.kind.as_str()),
|
||||
("OL37863121M", "olid")
|
||||
);
|
||||
assert_eq!(only.container.as_deref(), Some("Thomas Y. Crowell"));
|
||||
assert_eq!(only.year, Some(1917));
|
||||
assert_eq!(only.cover_id, Some(12736340));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_work_with_no_edition_named_gives_its_isbn_but_no_cover() {
|
||||
let body = r#"{"docs": [{"key": "/works/OL1W", "title": "A Book",
|
||||
"isbn": ["0-306-40615-2"], "publisher": ["Plenum"], "first_publish_year": 1981}]}"#;
|
||||
let found = open_library(body, 5).expect("candidates");
|
||||
assert_eq!(
|
||||
(found[0].id.as_str(), found[0].kind.as_str()),
|
||||
("9780306406157", "isbn")
|
||||
);
|
||||
assert_eq!(found[0].container.as_deref(), Some("Plenum"));
|
||||
assert_eq!(found[0].year, Some(1981));
|
||||
assert_eq!(found[0].cover_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_work_with_no_isbn_and_no_edition_is_named_by_the_works_key() {
|
||||
let body = r#"{"docs": [{"key": "/works/OL267171W", "title": "War and Peace"}]}"#;
|
||||
let found = open_library(body, 5).expect("candidates");
|
||||
assert_eq!(
|
||||
(found[0].id.as_str(), found[0].kind.as_str()),
|
||||
("OL267171W", "olid")
|
||||
);
|
||||
assert_eq!(found[0].cover_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn isbns_whose_check_digit_is_wrong_are_passed_over() {
|
||||
// 9780306406158 and 0306406153 are 9780306406157 mistyped.
|
||||
let body = r#"{"docs": [
|
||||
{"key": "/works/OL1W", "title": "A", "editions": {"docs": [
|
||||
{"key": "/books/OL1M", "isbn": ["9780306406158", "0306406153"], "cover_i": 7}]}},
|
||||
{"key": "/works/OL2W", "title": "B", "editions": {"docs": [
|
||||
{"key": "/books/OL2M", "isbn": ["9780306406158", "0306406152"]}]}}]}"#;
|
||||
let found = open_library(body, 5).expect("candidates");
|
||||
assert_eq!(
|
||||
(found[0].id.as_str(), found[0].kind.as_str()),
|
||||
("OL1M", "olid")
|
||||
);
|
||||
assert_eq!(found[0].cover_id, Some(7));
|
||||
// The ten-digit one reads, and is given in thirteen.
|
||||
assert_eq!(
|
||||
(found[1].id.as_str(), found[1].kind.as_str()),
|
||||
("9780306406157", "isbn")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thirteen_digits_are_taken_over_ten() {
|
||||
// Two different ISBNs on one edition: the thirteen-digit one wins
|
||||
// although the ten-digit one is listed first.
|
||||
let isbns = ["0306406152", "9780199205592"];
|
||||
assert_eq!(
|
||||
first_isbn(isbns.into_iter()).as_deref(),
|
||||
Some("9780199205592")
|
||||
);
|
||||
assert_eq!(first_isbn(["nonsense"].into_iter()), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_from_open_library_is_asked_of_google_books() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [
|
||||
("openlibrary.org", Ok(NOTHING)),
|
||||
("googleapis.com", Ok(GOOGLE)),
|
||||
];
|
||||
let found = search_with(
|
||||
"The Google Story",
|
||||
Some("Vise"),
|
||||
5,
|
||||
recorded(&mut asked, &bodies),
|
||||
)
|
||||
.expect("candidates");
|
||||
assert_eq!(asked.len(), 2);
|
||||
assert!(asked[1].starts_with("https://www.googleapis.com/books/v1/volumes?q="));
|
||||
|
||||
assert_eq!(found.len(), 2);
|
||||
let first = &found[0];
|
||||
assert_eq!(first.source, "googlebooks");
|
||||
assert_eq!(
|
||||
(first.id.as_str(), first.kind.as_str()),
|
||||
("9780553804577", "isbn")
|
||||
);
|
||||
assert_eq!(first.authors, ["David A. Vise", "Mark Malseed"]);
|
||||
assert_eq!(first.year, Some(2005));
|
||||
assert_eq!(
|
||||
first.container.as_deref(),
|
||||
Some("Random House Digital, Inc.")
|
||||
);
|
||||
assert_eq!(first.pages, Some(207));
|
||||
// Google Books knows nothing of Open Library's covers.
|
||||
assert_eq!(first.cover_id, None);
|
||||
|
||||
// A shelfmark is not an ISBN, though its digits would pass for one.
|
||||
let second = &found[1];
|
||||
assert_eq!(
|
||||
(second.id.as_str(), second.kind.as_str()),
|
||||
("a1b2C3d4", "googlebooks")
|
||||
);
|
||||
assert_eq!(
|
||||
second.title.as_deref(),
|
||||
Some("The Google Story: A Dissertation")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_from_either_register_is_no_candidates() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [
|
||||
("openlibrary.org", Ok(NOTHING)),
|
||||
(
|
||||
"googleapis.com",
|
||||
Ok(r#"{"kind": "books#volumes", "totalItems": 0}"#),
|
||||
),
|
||||
];
|
||||
let found =
|
||||
search_with("Nothing", None, 5, recorded(&mut asked, &bodies)).expect("an answer");
|
||||
assert!(found.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_library_having_nothing_stands_when_google_books_is_down() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [
|
||||
("openlibrary.org", Ok(NOTHING)),
|
||||
("googleapis.com", Err(down())),
|
||||
];
|
||||
let found =
|
||||
search_with("Nothing", None, 5, recorded(&mut asked, &bodies)).expect("an answer");
|
||||
assert!(found.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_library_down_is_asked_of_google_books() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [
|
||||
("openlibrary.org", Err(down())),
|
||||
("googleapis.com", Ok(GOOGLE)),
|
||||
];
|
||||
let found =
|
||||
search_with("The Google Story", None, 5, recorded(&mut asked, &bodies)).expect("ok");
|
||||
assert_eq!(found[0].source, "googlebooks");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_register_down_is_a_failure() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [
|
||||
("openlibrary.org", Err(down())),
|
||||
("googleapis.com", Err(down())),
|
||||
];
|
||||
let failed = search_with("A Book", None, 5, recorded(&mut asked, &bodies));
|
||||
assert!(matches!(failed, Err(Error::Unreachable(_))), "{failed:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_that_cannot_be_read_says_so() {
|
||||
assert!(matches!(open_library("<html>", 5), Err(Error::Shape(_))));
|
||||
assert!(matches!(google_books("<html>", 5), Err(Error::Shape(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_author_is_left_out_when_there_is_none() {
|
||||
for author in [None, Some(""), Some(" ")] {
|
||||
let url = open_library_url(
|
||||
"Anathem",
|
||||
author.map(str::trim).filter(|a| !a.is_empty()),
|
||||
5,
|
||||
);
|
||||
assert!(!url.contains("author="), "{url}");
|
||||
}
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [
|
||||
("openlibrary.org", Ok(NOTHING)),
|
||||
("googleapis.com", Ok(GOOGLE)),
|
||||
];
|
||||
search_with("Anathem", Some(" "), 5, recorded(&mut asked, &bodies)).expect("ok");
|
||||
assert!(!asked[0].contains("author="), "{}", asked[0]);
|
||||
assert!(!asked[1].contains("inauthor"), "{}", asked[1]);
|
||||
assert!(asked[1].contains("q=intitle%3AAnathem&"), "{}", asked[1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_author_narrows_the_search_when_there_is_one() {
|
||||
let url = open_library_url("Anathem", Some("Neal Stephenson"), 5);
|
||||
assert!(
|
||||
url.contains("?title=Anathem&author=Neal%20Stephenson&fields="),
|
||||
"{url}"
|
||||
);
|
||||
let url = google_url("Anathem", Some("Neal Stephenson"), 5).expect("a url");
|
||||
assert!(
|
||||
url.contains("q=intitle%3AAnathem%20inauthor%3ANeal%20inauthor%3AStephenson&"),
|
||||
"{url}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_title_cannot_change_the_shape_of_the_address() {
|
||||
let hostile = r#"Fish & Chips: "A History" #2?=x/y+z"#;
|
||||
for url in [
|
||||
open_library_url(hostile, Some("O'Brien & Sons"), 5),
|
||||
google_url(hostile, Some("O'Brien & Sons"), 5).expect("a url"),
|
||||
] {
|
||||
assert_eq!(url.matches('?').count(), 1, "{url}");
|
||||
assert_eq!(url.matches('#').count(), 0, "{url}");
|
||||
assert_eq!(url.matches('"').count(), 0, "{url}");
|
||||
assert_eq!(url.matches('+').count(), 0, "{url}");
|
||||
let query = url.split_once('?').expect("a query").1;
|
||||
let names: Vec<&str> = query
|
||||
.split('&')
|
||||
.map(|pair| pair.split_once('=').expect("name=value").0)
|
||||
.collect();
|
||||
assert!(
|
||||
names == ["title", "author", "fields", "limit"] || names == ["q", "maxResults"],
|
||||
"{url}"
|
||||
);
|
||||
}
|
||||
// Open Library is asked the title as written.
|
||||
let url = open_library_url(hostile, None, 5);
|
||||
assert!(
|
||||
url.contains("title=Fish%20%26%20Chips%3A%20%22A%20History%22%20%232%3F%3Dx%2Fy%2Bz&"),
|
||||
"{url}"
|
||||
);
|
||||
// Google is asked word by word, so nothing in a title is syntax.
|
||||
let url = google_url(hostile, Some("O'Brien & Sons"), 5).expect("a url");
|
||||
let q = "intitle:Fish intitle:Chips intitle:A intitle:History intitle:2 intitle:x \
|
||||
intitle:y intitle:z inauthor:O'Brien inauthor:Sons";
|
||||
assert!(url.contains(&format!("q={}&", source::encode(q))), "{url}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_title_outside_ascii_is_sent_as_utf8() {
|
||||
let url = open_library_url("Преступление и наказание", Some("Dostoevský"), 5);
|
||||
assert!(
|
||||
url.contains("title=%D0%9F%D1%80%D0%B5%D1%81%D1%82%D1%83%D0%BF"),
|
||||
"{url}"
|
||||
);
|
||||
assert!(url.contains("author=Dostoevsk%C3%BD&"), "{url}");
|
||||
let url = google_url("Преступление и наказание", None, 5).expect("a url");
|
||||
// Letters in any script are words.
|
||||
assert!(url.contains("intitle%3A%D0%B8%20"), "{url}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_title_with_no_words_is_not_asked_of_google_books() {
|
||||
assert_eq!(google_url("?!", Some("Someone"), 5), None);
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [("openlibrary.org", Ok(NOTHING))];
|
||||
let found = search_with("?!", Some("Someone"), 5, recorded(&mut asked, &bodies))
|
||||
.expect("an answer");
|
||||
assert!(found.is_empty());
|
||||
assert_eq!(asked.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_title_is_refused_without_asking() {
|
||||
let mut asked = Vec::new();
|
||||
let failed = search_with(" ", Some("Someone"), 5, recorded(&mut asked, &[]));
|
||||
assert!(matches!(failed, Err(Error::Invalid(_))), "{failed:?}");
|
||||
assert!(asked.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_limit_is_asked_for_and_held_to() {
|
||||
let mut asked = Vec::new();
|
||||
let bodies = [
|
||||
("openlibrary.org", Ok(NOTHING)),
|
||||
("googleapis.com", Ok(GOOGLE)),
|
||||
];
|
||||
let found =
|
||||
search_with("The Google Story", None, 1, recorded(&mut asked, &bodies)).expect("ok");
|
||||
assert_eq!(found.len(), 1);
|
||||
assert!(asked[0].ends_with("&limit=1"), "{}", asked[0]);
|
||||
assert!(asked[1].ends_with("&maxResults=1"), "{}", asked[1]);
|
||||
|
||||
let mut asked = Vec::new();
|
||||
search_with("The Google Story", None, 500, recorded(&mut asked, &bodies)).expect("ok");
|
||||
assert!(asked[0].ends_with("&limit=40"), "{}", asked[0]);
|
||||
assert!(asked[1].ends_with("&maxResults=40"), "{}", asked[1]);
|
||||
|
||||
let mut asked = Vec::new();
|
||||
let found =
|
||||
search_with("The Google Story", None, 0, recorded(&mut asked, &bodies)).expect("ok");
|
||||
assert!(found.is_empty());
|
||||
assert!(asked.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_index_is_not_asked_for_the_works_cover() {
|
||||
let url = open_library_url("A", None, 5);
|
||||
let fields = url
|
||||
.split('&')
|
||||
.find_map(|pair| pair.strip_prefix("fields="))
|
||||
.expect("fields");
|
||||
let fields: Vec<&str> = fields.split(',').collect();
|
||||
assert!(fields.contains(&"editions.cover_i"));
|
||||
assert!(!fields.contains(&"cover_i"));
|
||||
assert!(!fields.contains(&"cover_edition_key"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_404_from_a_search_is_a_failure_and_not_an_absence() {
|
||||
let failed = search_with("A Book", None, 5, |_| Ok(None));
|
||||
assert!(matches!(failed, Err(Error::Unreachable(_))), "{failed:?}");
|
||||
}
|
||||
}
|
||||
+163
-14
@@ -55,9 +55,17 @@ impl Source {
|
||||
match kind {
|
||||
"doi" => &[Source::Crossref],
|
||||
"arxiv" => &[Source::Arxiv],
|
||||
// The index first. Asked for the fields by name it answers with
|
||||
// the title, the authors, the publisher, the language, a page
|
||||
// count and the subjects in one request, which is the whole of
|
||||
// what a catalogue wants; the edition record is then needed only
|
||||
// for a subtitle or a publisher the index did not give. Asking it
|
||||
// second cost three requests per book where one usually does, and
|
||||
// at three books in flight that is the difference between two
|
||||
// days and two hours.
|
||||
"isbn" => &[
|
||||
Source::OpenLibrary,
|
||||
Source::OpenLibrarySearch,
|
||||
Source::OpenLibrary,
|
||||
Source::GoogleBooks,
|
||||
],
|
||||
_ => &[],
|
||||
@@ -75,9 +83,15 @@ impl Source {
|
||||
Source::OpenLibrary => {
|
||||
format!("https://openlibrary.org/isbn/{}.json", encode(id))
|
||||
}
|
||||
// `fields` or the index answers with a handful of defaults and
|
||||
// keeps the rest, which is how a language it holds went unread.
|
||||
// Not `cover_i`: that is the cover chosen for the whole work, and
|
||||
// on a work with hundreds of editions it is usually another
|
||||
// edition's, in another translation or language.
|
||||
Source::OpenLibrarySearch => format!(
|
||||
"https://openlibrary.org/search.json?q=isbn:{}\
|
||||
&fields=title,subtitle,author_name,first_publish_year,publisher&limit=1",
|
||||
&fields=title,subtitle,author_name,first_publish_year,publisher,language,subject,\
|
||||
number_of_pages_median&limit=1",
|
||||
encode(id)
|
||||
),
|
||||
Source::GoogleBooks => format!(
|
||||
@@ -103,7 +117,7 @@ impl Source {
|
||||
///
|
||||
/// A DOI suffix may hold a slash, a question mark or a hash, and every one of
|
||||
/// those means something else in a URL.
|
||||
fn encode(text: &str) -> String {
|
||||
pub(crate) fn encode(text: &str) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
for byte in text.bytes() {
|
||||
match byte {
|
||||
@@ -117,7 +131,7 @@ fn encode(text: &str) -> String {
|
||||
}
|
||||
|
||||
/// A record with the fields nothing found fills in.
|
||||
fn blank(id: &str, kind: &str, source: Source) -> Record {
|
||||
pub(crate) fn blank(id: &str, kind: &str, source: Source) -> Record {
|
||||
let mut record = crate::missing(id, kind, source);
|
||||
record.found = true;
|
||||
record
|
||||
@@ -222,9 +236,27 @@ struct OpenLibraryBook {
|
||||
#[serde(default)]
|
||||
title: Option<String>,
|
||||
#[serde(default)]
|
||||
subtitle: Option<String>,
|
||||
#[serde(default)]
|
||||
publish_date: Option<String>,
|
||||
#[serde(default)]
|
||||
publishers: Vec<String>,
|
||||
#[serde(default)]
|
||||
subjects: Vec<String>,
|
||||
/// `[{"key": "/languages/eng"}]`, which is how OpenLibrary names one.
|
||||
#[serde(default)]
|
||||
languages: Vec<OpenLibraryKey>,
|
||||
#[serde(default)]
|
||||
number_of_pages: Option<u32>,
|
||||
/// Cover ids, best first. A deleted cover is left in the list as `-1`.
|
||||
#[serde(default)]
|
||||
covers: Vec<i64>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct OpenLibraryKey {
|
||||
#[serde(default)]
|
||||
key: String,
|
||||
}
|
||||
|
||||
fn open_library(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
@@ -232,10 +264,28 @@ fn open_library(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?;
|
||||
|
||||
let mut record = blank(id, kind, Source::OpenLibrary);
|
||||
record.title = book.title;
|
||||
// An edition record keeps the subtitle in its own field, and a catalogue
|
||||
// wants the whole title. Google Books is already joined the same way.
|
||||
record.title = match (book.title, book.subtitle) {
|
||||
(Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")),
|
||||
(title, _) => title,
|
||||
};
|
||||
record.container = book.publishers.into_iter().next();
|
||||
// The date is free text — "1999", "March 1999", "1999-03-01".
|
||||
record.year = book.publish_date.as_deref().and_then(year_in);
|
||||
record.subjects = book.subjects;
|
||||
// "/languages/eng" names the language by the last segment of a key.
|
||||
record.language = book
|
||||
.languages
|
||||
.into_iter()
|
||||
.find_map(|l| l.key.rsplit('/').next().map(str::to_string))
|
||||
.filter(|l| !l.is_empty());
|
||||
record.pages = book.number_of_pages;
|
||||
// The edition's own cover, and the only place a cover id is taken from.
|
||||
record.cover_id = book
|
||||
.covers
|
||||
.into_iter()
|
||||
.find_map(|id| u64::try_from(id).ok().filter(|id| *id > 0));
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
@@ -257,8 +307,21 @@ struct SearchDoc {
|
||||
first_publish_year: Option<i64>,
|
||||
#[serde(default)]
|
||||
publisher: Vec<String>,
|
||||
#[serde(default)]
|
||||
language: Vec<String>,
|
||||
#[serde(default)]
|
||||
subject: Vec<String>,
|
||||
#[serde(rename = "number_of_pages_median", default)]
|
||||
pages: Option<u32>,
|
||||
}
|
||||
|
||||
/// How many of a work's subject headings are worth carrying.
|
||||
///
|
||||
/// The index lists every heading every edition has ever been given, which on a
|
||||
/// well-known book is hundreds and on Hamlet is thousands. A shelf is filed
|
||||
/// from a handful, and the first are the broadest.
|
||||
pub(crate) const SUBJECT_LIMIT: usize = 12;
|
||||
|
||||
/// A search that matched nothing answers with no documents rather than a
|
||||
/// status, so an empty reply is the absence and not a failure.
|
||||
fn open_library_search(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
@@ -275,23 +338,32 @@ fn open_library_search(id: &str, kind: &str, body: &str) -> Result<Record, Error
|
||||
record.authors = doc.author_name;
|
||||
record.year = doc.first_publish_year;
|
||||
record.container = doc.publisher.into_iter().next();
|
||||
record.language = doc.language.into_iter().next();
|
||||
// The index lists every subject any edition of the work carries, which
|
||||
// runs to hundreds on a well-known book. A shelf needs a handful.
|
||||
record.subjects = doc.subject;
|
||||
record.subjects.truncate(SUBJECT_LIMIT);
|
||||
record.pages = doc.pages;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct GoogleReply {
|
||||
pub(crate) struct GoogleReply {
|
||||
#[serde(default)]
|
||||
items: Vec<GoogleItem>,
|
||||
pub(crate) items: Vec<GoogleItem>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct GoogleItem {
|
||||
pub(crate) struct GoogleItem {
|
||||
/// Google's own key for the volume.
|
||||
#[serde(default)]
|
||||
pub(crate) id: Option<String>,
|
||||
#[serde(rename = "volumeInfo", default)]
|
||||
volume_info: GoogleVolume,
|
||||
pub(crate) volume_info: GoogleVolume,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Default)]
|
||||
struct GoogleVolume {
|
||||
pub(crate) struct GoogleVolume {
|
||||
#[serde(default)]
|
||||
title: Option<String>,
|
||||
#[serde(default)]
|
||||
@@ -304,6 +376,24 @@ struct GoogleVolume {
|
||||
publisher: Option<String>,
|
||||
#[serde(default)]
|
||||
description: Option<String>,
|
||||
#[serde(default)]
|
||||
categories: Vec<String>,
|
||||
#[serde(rename = "language", default)]
|
||||
language: Option<String>,
|
||||
#[serde(rename = "pageCount", default)]
|
||||
page_count: Option<u32>,
|
||||
/// `[{"type": "ISBN_13", "identifier": "…"}]`, alongside library
|
||||
/// shelfmarks typed `OTHER`.
|
||||
#[serde(rename = "industryIdentifiers", default)]
|
||||
pub(crate) identifiers: Vec<GoogleIdentifier>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(crate) struct GoogleIdentifier {
|
||||
#[serde(rename = "type", default)]
|
||||
pub(crate) kind: String,
|
||||
#[serde(default)]
|
||||
pub(crate) identifier: String,
|
||||
}
|
||||
|
||||
/// A search that matched nothing answers with no items rather than a status,
|
||||
@@ -313,8 +403,11 @@ fn google_books(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
let Some(item) = reply.items.into_iter().next() else {
|
||||
return Ok(crate::missing(id, kind, Source::GoogleBooks));
|
||||
};
|
||||
let volume = item.volume_info;
|
||||
Ok(google_volume(id, kind, item.volume_info))
|
||||
}
|
||||
|
||||
/// One Google Books volume as a record.
|
||||
pub(crate) fn google_volume(id: &str, kind: &str, volume: GoogleVolume) -> Record {
|
||||
let mut record = blank(id, kind, Source::GoogleBooks);
|
||||
record.title = match (volume.title, volume.subtitle) {
|
||||
(Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")),
|
||||
@@ -324,11 +417,14 @@ fn google_books(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
record.year = volume.published_date.as_deref().and_then(year_in);
|
||||
record.container = volume.publisher;
|
||||
record.summary = volume.description.map(|text| collapse(&text));
|
||||
Ok(record)
|
||||
record.subjects = volume.categories;
|
||||
record.language = volume.language;
|
||||
record.pages = volume.page_count;
|
||||
record
|
||||
}
|
||||
|
||||
/// The first four-digit year in a string.
|
||||
fn year_in(text: &str) -> Option<i64> {
|
||||
pub(crate) fn year_in(text: &str) -> Option<i64> {
|
||||
let bytes: Vec<char> = text.chars().collect();
|
||||
for window in bytes.windows(4) {
|
||||
if window.iter().all(char::is_ascii_digit) {
|
||||
@@ -379,7 +475,10 @@ mod tests {
|
||||
fn each_kind_goes_to_the_body_that_assigns_it() {
|
||||
assert_eq!(Source::for_kind("doi"), Some(Source::Crossref));
|
||||
assert_eq!(Source::for_kind("arxiv"), Some(Source::Arxiv));
|
||||
assert_eq!(Source::for_kind("isbn"), Some(Source::OpenLibrary));
|
||||
// The index, not the edition record: it answers with the whole of
|
||||
// what a catalogue wants in one request, where the edition record
|
||||
// takes up to twenty-eight seconds and names its authors by key.
|
||||
assert_eq!(Source::for_kind("isbn"), Some(Source::OpenLibrarySearch));
|
||||
// No register answers about an ISSN without a key.
|
||||
assert_eq!(Source::for_kind("issn"), None);
|
||||
}
|
||||
@@ -450,6 +549,56 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_cover_id_a_record_names_is_kept_for_the_cover_lookup() {
|
||||
// A deleted cover stays in an edition's list as -1, ahead of the
|
||||
// ones still there.
|
||||
let record = open_library("1", "isbn", r#"{"title":"A","covers":[-1,1162784,99]}"#)
|
||||
.expect("a record");
|
||||
assert_eq!(record.cover_id, Some(1162784));
|
||||
let record = open_library("1", "isbn", r#"{"title":"A"}"#).expect("a record");
|
||||
assert_eq!(record.cover_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_cover_id_is_the_editions_and_not_the_works() {
|
||||
// ISBN 9780140447934 today: the index (asked first) names the work's
|
||||
// cover; the edition record has none. Merged as `ask::lookup` merges.
|
||||
let mut kept = open_library_search(
|
||||
"9780140447934",
|
||||
"isbn",
|
||||
r#"{"docs":[{"title":"War and Peace","cover_i":12621906}]}"#,
|
||||
)
|
||||
.expect("a record");
|
||||
let edition = open_library("9780140447934", "isbn", r#"{"title":"War and Peace"}"#)
|
||||
.expect("a record");
|
||||
kept.fill_from(edition);
|
||||
// Cover 12621906 is OL34941467M, Standard Ebooks' Maude translation.
|
||||
assert_eq!(kept.cover_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_edition_with_no_covers_gives_no_cover_id() {
|
||||
let index = r#"{"docs":[{"title":"War and Peace","cover_i":12621906}]}"#;
|
||||
// `lookup` often stops at the index, before the edition is asked.
|
||||
let alone = open_library_search("9780140447934", "isbn", index).expect("a record");
|
||||
assert_eq!(alone.cover_id, None);
|
||||
// An edition whose only cover was deleted, and one with an empty list.
|
||||
for edition in [
|
||||
r#"{"title":"War and Peace","covers":[-1]}"#,
|
||||
r#"{"title":"War and Peace","covers":[]}"#,
|
||||
] {
|
||||
let mut kept = open_library_search("9780140447934", "isbn", index).expect("a record");
|
||||
kept.fill_from(open_library("9780140447934", "isbn", edition).expect("a record"));
|
||||
assert_eq!(kept.cover_id, None, "{edition}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_index_is_not_asked_for_the_works_cover() {
|
||||
assert!(!Source::OpenLibrarySearch.url("1").contains("cover_i"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_that_is_not_what_it_should_be_says_so() {
|
||||
assert!(matches!(
|
||||
|
||||
@@ -84,6 +84,7 @@ bookmeta library/*.epub \
|
||||
|---|---|
|
||||
| EPUB | The package document named by `META-INF/container.xml` — Dublin Core, plus the `<meta>` entries Calibre adds for a series |
|
||||
| PDF | The trailer's information dictionary |
|
||||
| MOBI, AZW, AZW3 | The MOBI header in the first record, and the EXTH records after it |
|
||||
|
||||
An EPUB whose container is malformed falls back to the first `.opf` in the
|
||||
archive, because a converter that wrote a bad container often still wrote a
|
||||
@@ -94,6 +95,28 @@ both become `isbn`, a UUID becomes `uuid`, anything else `other`. PDF dates
|
||||
are reduced to the day where they parse and kept verbatim where they do not,
|
||||
because a date nobody can read is still evidence.
|
||||
|
||||
A Kindle book's title is EXTH 503 where it is there, then the MOBI header's
|
||||
full name, then the Palm database name (cut to 31 bytes, with a note). Authors
|
||||
(100) and subjects (105) are every record of their kind, in order; publisher
|
||||
(101), description (103), date (106) and language (524) the first. EXTH 104 is
|
||||
an `isbn` when it is shaped like one and `other` when not, and 113 is an
|
||||
`asin`. A contributor (108) is not an author, the same as an EPUB's
|
||||
`dc:contributor`, and is not reported. With no EXTH language, the language
|
||||
comes from the header's locale. Strings are decoded as the header says,
|
||||
UTF-8 or Windows-1252. A book whose text is encrypted still has its header
|
||||
read, and carries the note `drm`.
|
||||
|
||||
Every offset in a Kindle book is checked before it is followed: headers that
|
||||
are cut short or point outside the file give an empty record with a note, and
|
||||
an EXTH block that breaks off keeps the records before the break.
|
||||
|
||||
## Covers
|
||||
|
||||
`Metadata::cover` is the cover image a Kindle book carries (EXTH 201, or the
|
||||
thumbnail, 202, when no cover is named), with its media type read off its
|
||||
first bytes. It is for library callers and is not part of the JSON record,
|
||||
which is a line of text.
|
||||
|
||||
## Asking a model
|
||||
|
||||
Some files say nothing useful. `--prompt` writes the question and **sends
|
||||
@@ -147,7 +170,7 @@ if let Some(prompt) = bookmeta::prompt::compose(&meta, Some(opening)) {
|
||||
}
|
||||
```
|
||||
|
||||
`bookmeta::epub` and `bookmeta::pdf` read one format each, and
|
||||
`bookmeta::epub`, `bookmeta::pdf` and `bookmeta::mobi` read one format each, and
|
||||
`Metadata::missing` says which fields a record still wants.
|
||||
|
||||
## See also
|
||||
|
||||
@@ -85,6 +85,7 @@ fn fill(meta: &mut Metadata, package: &str) {
|
||||
meta.title = first(package, "title");
|
||||
meta.authors = creators(package);
|
||||
meta.publisher = first(package, "publisher");
|
||||
meta.description = first(package, "description");
|
||||
meta.date = first(package, "date");
|
||||
meta.language = first(package, "language");
|
||||
meta.subjects = all(package, "subject");
|
||||
@@ -166,7 +167,7 @@ fn identifiers(package: &str) -> Vec<(String, String)> {
|
||||
("doi", value.to_string())
|
||||
} else if is_isbn(value) {
|
||||
("isbn", digits_of(value))
|
||||
} else if lower.len() == 36 && lower.matches('-').count() == 4 {
|
||||
} else if is_uuid(&lower) {
|
||||
("uuid", lower.clone())
|
||||
} else {
|
||||
("other", value.to_string())
|
||||
@@ -179,7 +180,7 @@ fn identifiers(package: &str) -> Vec<(String, String)> {
|
||||
}
|
||||
|
||||
/// Whether a string is ten or thirteen digits once its hyphens come off.
|
||||
fn is_isbn(value: &str) -> bool {
|
||||
pub(crate) fn is_isbn(value: &str) -> bool {
|
||||
let digits = digits_of(value);
|
||||
matches!(digits.len(), 10 | 13)
|
||||
&& value
|
||||
@@ -187,7 +188,13 @@ fn is_isbn(value: &str) -> bool {
|
||||
.all(|c| c.is_ascii_digit() || "- xX".contains(c))
|
||||
}
|
||||
|
||||
fn digits_of(value: &str) -> String {
|
||||
/// Whether a value is shaped like a UUID: 36 characters with four hyphens.
|
||||
pub(crate) fn is_uuid(value: &str) -> bool {
|
||||
value.len() == 36 && value.matches('-').count() == 4
|
||||
}
|
||||
|
||||
/// The digits of an ISBN, with its check character, and nothing else.
|
||||
pub(crate) fn digits_of(value: &str) -> String {
|
||||
value
|
||||
.chars()
|
||||
.filter(|c| c.is_ascii_digit() || matches!(c, 'x' | 'X'))
|
||||
@@ -253,6 +260,7 @@ mod tests {
|
||||
<dc:creator opf:role="aut">Kallistos Ware</dc:creator>
|
||||
<dc:contributor>A Typesetter</dc:contributor>
|
||||
<dc:publisher>Faber & Faber</dc:publisher>
|
||||
<dc:description><p>Writings of the Fathers.</p></dc:description>
|
||||
<dc:date>1984-09-17</dc:date>
|
||||
<dc:language>en</dc:language>
|
||||
<dc:subject>Orthodox Christianity</dc:subject>
|
||||
@@ -275,6 +283,10 @@ mod tests {
|
||||
let meta = read_package();
|
||||
assert_eq!(meta.title.as_deref(), Some("The Philokalia, Volume 3"));
|
||||
assert_eq!(meta.publisher.as_deref(), Some("Faber & Faber"));
|
||||
assert_eq!(
|
||||
meta.description.as_deref(),
|
||||
Some("<p>Writings of the Fathers.</p>")
|
||||
);
|
||||
assert_eq!(meta.date.as_deref(), Some("1984-09-17"));
|
||||
assert_eq!(meta.language.as_deref(), Some("en"));
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
//! [`bibfetch`]: https://git.pumpin.online/funkpower/catena
|
||||
|
||||
pub mod epub;
|
||||
pub mod mobi;
|
||||
pub mod pdf;
|
||||
pub mod prompt;
|
||||
|
||||
@@ -30,6 +31,8 @@ pub enum Source {
|
||||
Opf,
|
||||
/// A PDF's information dictionary.
|
||||
PdfInfo,
|
||||
/// A Kindle book's MOBI header and the EXTH records after it.
|
||||
Mobi,
|
||||
/// Nothing was found to read.
|
||||
None,
|
||||
}
|
||||
@@ -40,6 +43,7 @@ impl Source {
|
||||
match self {
|
||||
Source::Opf => "opf",
|
||||
Source::PdfInfo => "pdf_info",
|
||||
Source::Mobi => "mobi",
|
||||
Source::None => "none",
|
||||
}
|
||||
}
|
||||
@@ -64,6 +68,9 @@ pub struct Metadata {
|
||||
/// Who published it.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub publisher: Option<String>,
|
||||
/// The publisher's description of the book, as written: often HTML.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub description: Option<String>,
|
||||
/// The publication date, in whatever form the file wrote it.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub date: Option<String>,
|
||||
@@ -86,6 +93,23 @@ pub struct Metadata {
|
||||
/// What was worth saying about the reading.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub notes: Vec<String>,
|
||||
/// The cover image the file carries.
|
||||
///
|
||||
/// Not part of the record: a record is a line of text and this is an
|
||||
/// image. A library caller takes it from here.
|
||||
#[serde(skip)]
|
||||
#[schemars(skip)]
|
||||
pub cover: Option<Cover>,
|
||||
}
|
||||
|
||||
/// A cover image, as the file stores it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Cover {
|
||||
/// What kind of image it is, read off its first bytes: `image/jpeg`,
|
||||
/// `image/png`, `image/gif` or `image/bmp`.
|
||||
pub media_type: String,
|
||||
/// The image itself.
|
||||
pub bytes: Vec<u8>,
|
||||
}
|
||||
|
||||
fn version() -> u32 {
|
||||
@@ -143,6 +167,7 @@ impl Metadata {
|
||||
self.title.is_none()
|
||||
&& self.authors.is_empty()
|
||||
&& self.publisher.is_none()
|
||||
&& self.description.is_none()
|
||||
&& self.date.is_none()
|
||||
&& self.language.is_none()
|
||||
&& self.identifiers.is_empty()
|
||||
@@ -171,6 +196,9 @@ pub fn from_bytes(path: &str, bytes: &[u8]) -> Metadata {
|
||||
if bytes.starts_with(b"%PDF") {
|
||||
return pdf::read(path, bytes);
|
||||
}
|
||||
if mobi::is_mobi(bytes) {
|
||||
return mobi::read(path, bytes);
|
||||
}
|
||||
let mut meta = Metadata::empty(path, Source::None);
|
||||
meta.note("not a format this reads metadata out of");
|
||||
meta
|
||||
|
||||
@@ -0,0 +1,873 @@
|
||||
//! A Kindle book's MOBI header and the EXTH records after it.
|
||||
//!
|
||||
//! MOBI, AZW and AZW3 files are Palm databases. The first record opens with a
|
||||
//! PalmDOC header and a MOBI header, which gives the book's full name and says
|
||||
//! how its strings are encoded. When a flag in that header is set, an EXTH
|
||||
//! block follows with everything else: authors, publisher, ISBN, subjects, and
|
||||
//! which image record is the cover. AZW3 (KF8) keeps the same layout in its
|
||||
//! first record, so one reader covers all three.
|
||||
//!
|
||||
//! Every offset in the file is checked before it is followed. A file whose
|
||||
//! headers are cut short or written wrongly gives an empty record with a note;
|
||||
//! an EXTH block that breaks off keeps the records before the break. Neither
|
||||
//! panics or reads past the end.
|
||||
|
||||
use crate::epub::{digits_of, is_isbn, is_uuid};
|
||||
use crate::{Cover, Metadata, Source};
|
||||
|
||||
/// Where a Palm database says what it holds.
|
||||
const TYPE_AT: usize = 60;
|
||||
/// What a Kindle book says there.
|
||||
const BOOK_TYPE: &[u8] = b"BOOKMOBI";
|
||||
/// The Palm database's own name: 32 bytes, NUL-padded.
|
||||
const NAME_LEN: usize = 32;
|
||||
/// Where the number of records is written.
|
||||
const RECORD_COUNT_AT: usize = 76;
|
||||
/// Where the record list starts: eight bytes per record, offset first.
|
||||
const RECORD_LIST_AT: usize = 78;
|
||||
|
||||
/// Where the PalmDOC header says whether the text is encrypted.
|
||||
const ENCRYPTION_AT: usize = 12;
|
||||
/// Where the MOBI header begins inside the first record.
|
||||
const MOBI_AT: usize = 16;
|
||||
/// The MOBI header's own magic.
|
||||
const MOBI_MAGIC: &[u8] = b"MOBI";
|
||||
|
||||
// Fields of the MOBI header, as offsets from its start. Each is only there
|
||||
// when the header's declared length reaches past it.
|
||||
const HEADER_LENGTH: usize = 4;
|
||||
const TEXT_ENCODING: usize = 12;
|
||||
const FULL_NAME_OFFSET: usize = 68;
|
||||
const FULL_NAME_LENGTH: usize = 72;
|
||||
const LOCALE: usize = 76;
|
||||
const FIRST_IMAGE: usize = 92;
|
||||
const EXTH_FLAGS: usize = 112;
|
||||
|
||||
/// The flag that says an EXTH block follows the MOBI header.
|
||||
const HAS_EXTH: u32 = 0x40;
|
||||
/// The EXTH block's magic.
|
||||
const EXTH_MAGIC: &[u8] = b"EXTH";
|
||||
|
||||
/// The text encodings a MOBI header names.
|
||||
const UTF_8: u32 = 65001;
|
||||
const WINDOWS_1252: u32 = 1252;
|
||||
|
||||
/// What an index or offset says when it points at nothing.
|
||||
const NONE: u32 = 0xffff_ffff;
|
||||
|
||||
// EXTH record types.
|
||||
const AUTHOR: u32 = 100;
|
||||
const PUBLISHER: u32 = 101;
|
||||
const DESCRIPTION: u32 = 103;
|
||||
const ISBN: u32 = 104;
|
||||
const SUBJECT: u32 = 105;
|
||||
const PUBLISHED: u32 = 106;
|
||||
const ASIN: u32 = 113;
|
||||
const COVER_OFFSET: u32 = 201;
|
||||
const THUMB_OFFSET: u32 = 202;
|
||||
const UPDATED_TITLE: u32 = 503;
|
||||
const LANGUAGE: u32 = 524;
|
||||
|
||||
/// Whether these bytes are a Kindle book this module can read.
|
||||
pub fn is_mobi(bytes: &[u8]) -> bool {
|
||||
bytes.get(TYPE_AT..TYPE_AT + BOOK_TYPE.len()) == Some(BOOK_TYPE)
|
||||
}
|
||||
|
||||
/// Read what a MOBI, AZW or AZW3 file says about itself.
|
||||
pub fn read(path: &str, bytes: &[u8]) -> Metadata {
|
||||
let mut meta = Metadata::empty(path, Source::Mobi);
|
||||
if let Err(why) = fill(&mut meta, bytes) {
|
||||
let mut empty = Metadata::empty(path, Source::None);
|
||||
empty.note(why);
|
||||
return empty;
|
||||
}
|
||||
if meta.is_empty() {
|
||||
meta.note("a MOBI header with nothing filled in");
|
||||
}
|
||||
meta
|
||||
}
|
||||
|
||||
/// The strings an EXTH block carries, before they are placed in a record.
|
||||
#[derive(Default)]
|
||||
struct Exth {
|
||||
authors: Vec<String>,
|
||||
publisher: Option<String>,
|
||||
description: Option<String>,
|
||||
isbn: Option<String>,
|
||||
subjects: Vec<String>,
|
||||
published: Option<String>,
|
||||
asin: Option<String>,
|
||||
cover: Option<u32>,
|
||||
thumb: Option<u32>,
|
||||
title: Option<String>,
|
||||
language: Option<String>,
|
||||
}
|
||||
|
||||
fn fill(meta: &mut Metadata, bytes: &[u8]) -> Result<(), String> {
|
||||
if !is_mobi(bytes) {
|
||||
return Err("not a Kindle book: the Palm database type is not BOOKMOBI".into());
|
||||
}
|
||||
let records = records(bytes)?;
|
||||
let first = record(bytes, &records, 0).ok_or("the first record lies outside the file")?;
|
||||
|
||||
if first.get(MOBI_AT..MOBI_AT + MOBI_MAGIC.len()) != Some(MOBI_MAGIC) {
|
||||
return Err("no MOBI header in the first record".into());
|
||||
}
|
||||
let header_length =
|
||||
u32_at(first, MOBI_AT + HEADER_LENGTH).ok_or("the MOBI header is cut short")? as usize;
|
||||
let header = Header {
|
||||
record: first,
|
||||
length: header_length,
|
||||
};
|
||||
|
||||
let encoding = header.field(TEXT_ENCODING).unwrap_or(WINDOWS_1252);
|
||||
if encoding != UTF_8 && encoding != WINDOWS_1252 {
|
||||
meta.note(format!(
|
||||
"text encoding {encoding}, which is neither UTF-8 nor Windows-1252; read as Windows-1252"
|
||||
));
|
||||
}
|
||||
let decode = |raw: &[u8]| text(raw, encoding);
|
||||
|
||||
if u16_at(first, ENCRYPTION_AT).is_some_and(|kind| kind != 0) {
|
||||
meta.note("drm");
|
||||
}
|
||||
|
||||
let exth = if header
|
||||
.field(EXTH_FLAGS)
|
||||
.is_some_and(|flags| flags & HAS_EXTH != 0)
|
||||
{
|
||||
let at = MOBI_AT
|
||||
.checked_add(header_length)
|
||||
.ok_or("the MOBI header length is out of range")?;
|
||||
exth(first, at, &decode, meta)
|
||||
} else {
|
||||
Exth::default()
|
||||
};
|
||||
|
||||
let full_name = header
|
||||
.field(FULL_NAME_OFFSET)
|
||||
.zip(header.field(FULL_NAME_LENGTH))
|
||||
.and_then(|(at, len)| slice(first, at as usize, len as usize))
|
||||
.and_then(decode);
|
||||
meta.title = match (exth.title, full_name) {
|
||||
(Some(title), _) | (None, Some(title)) => Some(title),
|
||||
(None, None) => {
|
||||
let name = palm_name(bytes);
|
||||
if name.is_some() {
|
||||
meta.note("title from the Palm database name, which is cut to 31 bytes");
|
||||
}
|
||||
name
|
||||
}
|
||||
};
|
||||
|
||||
meta.authors = exth.authors;
|
||||
meta.publisher = exth.publisher;
|
||||
meta.description = exth.description;
|
||||
meta.date = exth.published;
|
||||
meta.subjects = exth.subjects;
|
||||
meta.language = exth.language.or_else(|| {
|
||||
header
|
||||
.field(LOCALE)
|
||||
.and_then(|locale| language_of(locale).map(str::to_string))
|
||||
});
|
||||
if let Some(isbn) = exth.isbn {
|
||||
if is_isbn(&isbn) {
|
||||
meta.identifiers.insert("isbn".into(), digits_of(&isbn));
|
||||
} else {
|
||||
meta.identifiers.insert("other".into(), isbn);
|
||||
}
|
||||
}
|
||||
// EXTH 113 holds an ASIN when the book was made for Amazon, and calibre's
|
||||
// own book UUID when it was converted without one.
|
||||
if let Some(value) = exth.asin {
|
||||
let lower = value.to_lowercase();
|
||||
if is_uuid(&lower) {
|
||||
meta.identifiers.insert("uuid".into(), lower);
|
||||
} else if is_asin(&value) {
|
||||
meta.identifiers.insert("asin".into(), value);
|
||||
} else {
|
||||
meta.identifiers.entry("other".into()).or_insert(value);
|
||||
}
|
||||
}
|
||||
|
||||
let first_image = header.field(FIRST_IMAGE);
|
||||
if let Some(offset) = exth.cover.or(exth.thumb) {
|
||||
meta.cover = cover(bytes, &records, first_image, offset, meta);
|
||||
} else {
|
||||
meta.cover = first_image_record(bytes, &records, first_image);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The MOBI header, for reading fields that may or may not be in it.
|
||||
struct Header<'a> {
|
||||
record: &'a [u8],
|
||||
length: usize,
|
||||
}
|
||||
|
||||
impl Header<'_> {
|
||||
/// A field at `at` bytes into the header, when the header reaches it.
|
||||
fn field(&self, at: usize) -> Option<u32> {
|
||||
if at.checked_add(4)? > self.length {
|
||||
return None;
|
||||
}
|
||||
u32_at(self.record, MOBI_AT.checked_add(at)?)
|
||||
}
|
||||
}
|
||||
|
||||
/// Where each record starts, as the record list says.
|
||||
fn records(bytes: &[u8]) -> Result<Vec<usize>, String> {
|
||||
let count = u16_at(bytes, RECORD_COUNT_AT).ok_or("the Palm database header is cut short")?;
|
||||
if count == 0 {
|
||||
return Err("a Palm database with no records".into());
|
||||
}
|
||||
let mut out = Vec::with_capacity(count as usize);
|
||||
for at in 0..count as usize {
|
||||
let entry = RECORD_LIST_AT + at * 8;
|
||||
let Some(offset) = u32_at(bytes, entry) else {
|
||||
return Err("the record list runs past the end of the file".into());
|
||||
};
|
||||
out.push(offset as usize);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// One record's bytes: from its offset to the next record's, or to the end.
|
||||
fn record<'a>(bytes: &'a [u8], records: &[usize], index: usize) -> Option<&'a [u8]> {
|
||||
let start = *records.get(index)?;
|
||||
let end = records.get(index + 1).copied().unwrap_or(bytes.len());
|
||||
bytes.get(start..end)
|
||||
}
|
||||
|
||||
/// Read the EXTH block at `at` inside the first record.
|
||||
///
|
||||
/// A block that ends early keeps what was read before the break, with a note:
|
||||
/// the records already read are as good as they would have been.
|
||||
fn exth(
|
||||
first: &[u8],
|
||||
at: usize,
|
||||
decode: &dyn Fn(&[u8]) -> Option<String>,
|
||||
meta: &mut Metadata,
|
||||
) -> Exth {
|
||||
let mut out = Exth::default();
|
||||
if at >= first.len() || first.get(at..at + EXTH_MAGIC.len()) != Some(EXTH_MAGIC) {
|
||||
meta.note("the header says an EXTH block follows, but none does");
|
||||
return out;
|
||||
}
|
||||
let (Some(length), Some(count)) = (u32_at(first, at + 4), u32_at(first, at + 8)) else {
|
||||
meta.note("the EXTH block ends early");
|
||||
return out;
|
||||
};
|
||||
let end = at.saturating_add(length as usize).min(first.len());
|
||||
|
||||
let mut cursor = at + 12;
|
||||
for _ in 0..count {
|
||||
let (Some(kind), Some(size)) = (u32_at(first, cursor), u32_at(first, cursor + 4)) else {
|
||||
meta.note("the EXTH block ends early");
|
||||
break;
|
||||
};
|
||||
let size = size as usize;
|
||||
let Some(data) = size
|
||||
.checked_sub(8)
|
||||
.and_then(|len| slice(first, cursor + 8, len))
|
||||
.filter(|_| cursor.checked_add(size).is_some_and(|stop| stop <= end))
|
||||
else {
|
||||
meta.note("the EXTH block ends early");
|
||||
break;
|
||||
};
|
||||
cursor += size;
|
||||
|
||||
match kind {
|
||||
AUTHOR => out.authors.extend(decode(data)),
|
||||
SUBJECT => out.subjects.extend(decode(data)),
|
||||
PUBLISHER => set(&mut out.publisher, decode(data)),
|
||||
DESCRIPTION => set(&mut out.description, decode(data)),
|
||||
ISBN => set(&mut out.isbn, decode(data)),
|
||||
PUBLISHED => set(&mut out.published, decode(data)),
|
||||
ASIN => set(&mut out.asin, decode(data)),
|
||||
UPDATED_TITLE => set(&mut out.title, decode(data)),
|
||||
LANGUAGE => set(&mut out.language, decode(data)),
|
||||
COVER_OFFSET => set(&mut out.cover, number(data)),
|
||||
THUMB_OFFSET => set(&mut out.thumb, number(data)),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Keep the first value a field is given: a later duplicate is usually a
|
||||
/// converter writing the same thing twice, and the first is what it meant.
|
||||
fn set<T>(slot: &mut Option<T>, value: Option<T>) {
|
||||
if slot.is_none() {
|
||||
*slot = value;
|
||||
}
|
||||
}
|
||||
|
||||
/// A four-byte number record that points somewhere.
|
||||
fn number(data: &[u8]) -> Option<u32> {
|
||||
let value = u32::from_be_bytes(data.try_into().ok()?);
|
||||
(value != NONE).then_some(value)
|
||||
}
|
||||
|
||||
/// The cover image, from the record the header and the EXTH offset name.
|
||||
fn cover(
|
||||
bytes: &[u8],
|
||||
records: &[usize],
|
||||
first_image: Option<u32>,
|
||||
offset: u32,
|
||||
meta: &mut Metadata,
|
||||
) -> Option<Cover> {
|
||||
let Some(first_image) = first_image.filter(|&index| index != NONE) else {
|
||||
meta.note("a cover is named, but the header has no image records");
|
||||
return None;
|
||||
};
|
||||
let Some(data) = first_image
|
||||
.checked_add(offset)
|
||||
.and_then(|index| record(bytes, records, index as usize))
|
||||
else {
|
||||
meta.note("the cover points past the last record");
|
||||
return None;
|
||||
};
|
||||
let Some(media_type) = image_type(data) else {
|
||||
meta.note("the cover record is not an image");
|
||||
return None;
|
||||
};
|
||||
Some(Cover {
|
||||
media_type: media_type.to_string(),
|
||||
bytes: data.to_vec(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The cover when the file names none: calibre takes the record at the first
|
||||
/// image index, and only that one.
|
||||
///
|
||||
/// When that record is not a JPEG, PNG or GIF there is no cover. The search
|
||||
/// does not walk on to the next image, because a later record may be an inline
|
||||
/// illustration, and kindlegen puts a font or a RESC record first.
|
||||
fn first_image_record(bytes: &[u8], records: &[usize], first_image: Option<u32>) -> Option<Cover> {
|
||||
let first = first_image.filter(|&index| index != NONE)?;
|
||||
let data = record(bytes, records, first as usize)?;
|
||||
let media_type = image_type(data).filter(|kind| *kind != "image/bmp")?;
|
||||
Some(Cover {
|
||||
media_type: media_type.to_string(),
|
||||
bytes: data.to_vec(),
|
||||
})
|
||||
}
|
||||
|
||||
/// An ASIN is ten letters and digits; an ISBN-10 is one too.
|
||||
fn is_asin(value: &str) -> bool {
|
||||
value.len() == 10 && value.chars().all(|c| c.is_ascii_alphanumeric())
|
||||
}
|
||||
|
||||
/// What kind of image these bytes are, by their magic.
|
||||
fn image_type(data: &[u8]) -> Option<&'static str> {
|
||||
if data.starts_with(&[0xff, 0xd8, 0xff]) {
|
||||
Some("image/jpeg")
|
||||
} else if data.starts_with(b"\x89PNG\r\n\x1a\n") {
|
||||
Some("image/png")
|
||||
} else if data.starts_with(b"GIF87a") || data.starts_with(b"GIF89a") {
|
||||
Some("image/gif")
|
||||
} else if data.starts_with(b"BM") && data.len() > 14 {
|
||||
Some("image/bmp")
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// The Palm database's name, the last place a title can come from.
|
||||
///
|
||||
/// Converters write the title there with spaces made underscores; it is
|
||||
/// reported as written.
|
||||
fn palm_name(bytes: &[u8]) -> Option<String> {
|
||||
let raw = bytes.get(..NAME_LEN)?;
|
||||
let raw = raw.split(|&byte| byte == 0).next().unwrap_or(raw);
|
||||
text(raw, WINDOWS_1252)
|
||||
}
|
||||
|
||||
/// A string in the book's declared encoding, trimmed; empty is none.
|
||||
fn text(raw: &[u8], encoding: u32) -> Option<String> {
|
||||
let decoded: String = match (encoding, std::str::from_utf8(raw)) {
|
||||
(UTF_8, Ok(text)) => text.to_string(),
|
||||
(UTF_8, Err(_)) => String::from_utf8_lossy(raw).into_owned(),
|
||||
_ => raw.iter().map(|&byte| windows_1252(byte)).collect(),
|
||||
};
|
||||
let trimmed = decoded.trim_matches(|c: char| c == '\0' || c.is_whitespace());
|
||||
(!trimmed.is_empty()).then(|| trimmed.to_string())
|
||||
}
|
||||
|
||||
/// One Windows-1252 byte as a character.
|
||||
fn windows_1252(byte: u8) -> char {
|
||||
match byte {
|
||||
0x80..=0x9f => HIGH_CONTROLS[(byte - 0x80) as usize],
|
||||
_ => byte as char,
|
||||
}
|
||||
}
|
||||
|
||||
/// The characters Windows-1252 puts where Latin-1 has C1 controls. The five
|
||||
/// positions it leaves undefined decode to themselves, as browsers do.
|
||||
const HIGH_CONTROLS: [char; 32] = [
|
||||
'\u{20ac}', '\u{81}', '\u{201a}', '\u{192}', '\u{201e}', '\u{2026}', '\u{2020}', '\u{2021}',
|
||||
'\u{2c6}', '\u{2030}', '\u{160}', '\u{2039}', '\u{152}', '\u{8d}', '\u{17d}', '\u{8f}',
|
||||
'\u{90}', '\u{2018}', '\u{2019}', '\u{201c}', '\u{201d}', '\u{2022}', '\u{2013}', '\u{2014}',
|
||||
'\u{2dc}', '\u{2122}', '\u{161}', '\u{203a}', '\u{153}', '\u{9d}', '\u{17e}', '\u{178}',
|
||||
];
|
||||
|
||||
/// The language a MOBI locale names, for books with no EXTH language.
|
||||
///
|
||||
/// The low byte is a Windows primary language id; the region above it is
|
||||
/// dropped, since the file's own language field never carries one either.
|
||||
fn language_of(locale: u32) -> Option<&'static str> {
|
||||
Some(match locale & 0xff {
|
||||
0x01 => "ar",
|
||||
0x02 => "bg",
|
||||
0x03 => "ca",
|
||||
0x04 => "zh",
|
||||
0x05 => "cs",
|
||||
0x06 => "da",
|
||||
0x07 => "de",
|
||||
0x08 => "el",
|
||||
0x09 => "en",
|
||||
0x0a => "es",
|
||||
0x0b => "fi",
|
||||
0x0c => "fr",
|
||||
0x0d => "he",
|
||||
0x0e => "hu",
|
||||
0x0f => "is",
|
||||
0x10 => "it",
|
||||
0x11 => "ja",
|
||||
0x12 => "ko",
|
||||
0x13 => "nl",
|
||||
0x14 => "no",
|
||||
0x15 => "pl",
|
||||
0x16 => "pt",
|
||||
0x18 => "ro",
|
||||
0x19 => "ru",
|
||||
0x1a => "hr",
|
||||
0x1b => "sk",
|
||||
0x1d => "sv",
|
||||
0x1f => "tr",
|
||||
0x22 => "uk",
|
||||
0x29 => "fa",
|
||||
0x2a => "vi",
|
||||
0x36 => "af",
|
||||
0x39 => "hi",
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
fn slice(bytes: &[u8], at: usize, len: usize) -> Option<&[u8]> {
|
||||
bytes.get(at..at.checked_add(len)?)
|
||||
}
|
||||
|
||||
fn u16_at(bytes: &[u8], at: usize) -> Option<u16> {
|
||||
Some(u16::from_be_bytes(slice(bytes, at, 2)?.try_into().ok()?))
|
||||
}
|
||||
|
||||
fn u32_at(bytes: &[u8], at: usize) -> Option<u32> {
|
||||
Some(u32::from_be_bytes(slice(bytes, at, 4)?.try_into().ok()?))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const JPEG: &[u8] = &[0xff, 0xd8, 0xff, 0xe0, 0, 16, b'J', b'F', b'I', b'F'];
|
||||
|
||||
/// A Kindle book built field by field, so each test can break one thing.
|
||||
struct Book {
|
||||
name: &'static [u8],
|
||||
encoding: u32,
|
||||
encryption: u16,
|
||||
locale: u32,
|
||||
full_name: Vec<u8>,
|
||||
exth: Option<Vec<(u32, Vec<u8>)>>,
|
||||
first_image: u32,
|
||||
images: Vec<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl Default for Book {
|
||||
fn default() -> Self {
|
||||
Book {
|
||||
name: b"A_Test_Book",
|
||||
encoding: UTF_8,
|
||||
encryption: 0,
|
||||
locale: 0x09,
|
||||
full_name: b"A Test Book".to_vec(),
|
||||
exth: Some(Vec::new()),
|
||||
first_image: 1,
|
||||
images: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Book {
|
||||
fn bytes(&self) -> Vec<u8> {
|
||||
const HEADER: usize = 232;
|
||||
let mut first = vec![0u8; MOBI_AT + HEADER];
|
||||
first[ENCRYPTION_AT..ENCRYPTION_AT + 2].copy_from_slice(&self.encryption.to_be_bytes());
|
||||
first[MOBI_AT..MOBI_AT + 4].copy_from_slice(MOBI_MAGIC);
|
||||
let put = |first: &mut Vec<u8>, at: usize, value: u32| {
|
||||
first[MOBI_AT + at..MOBI_AT + at + 4].copy_from_slice(&value.to_be_bytes());
|
||||
};
|
||||
put(&mut first, HEADER_LENGTH, HEADER as u32);
|
||||
put(&mut first, TEXT_ENCODING, self.encoding);
|
||||
put(&mut first, LOCALE, self.locale);
|
||||
put(&mut first, FIRST_IMAGE, self.first_image);
|
||||
if let Some(records) = &self.exth {
|
||||
put(&mut first, EXTH_FLAGS, HAS_EXTH);
|
||||
let mut body = Vec::new();
|
||||
for (kind, data) in records {
|
||||
body.extend_from_slice(&kind.to_be_bytes());
|
||||
body.extend_from_slice(&(data.len() as u32 + 8).to_be_bytes());
|
||||
body.extend_from_slice(data);
|
||||
}
|
||||
first.extend_from_slice(EXTH_MAGIC);
|
||||
first.extend_from_slice(&(body.len() as u32 + 12).to_be_bytes());
|
||||
first.extend_from_slice(&(records.len() as u32).to_be_bytes());
|
||||
first.extend_from_slice(&body);
|
||||
}
|
||||
let name_at = first.len();
|
||||
first.extend_from_slice(&self.full_name);
|
||||
put(&mut first, FULL_NAME_OFFSET, name_at as u32);
|
||||
put(&mut first, FULL_NAME_LENGTH, self.full_name.len() as u32);
|
||||
|
||||
let mut records = vec![first];
|
||||
records.extend(self.images.iter().cloned());
|
||||
|
||||
let mut out = vec![0u8; RECORD_LIST_AT + records.len() * 8];
|
||||
out[..self.name.len()].copy_from_slice(self.name);
|
||||
out[TYPE_AT..TYPE_AT + 8].copy_from_slice(BOOK_TYPE);
|
||||
out[RECORD_COUNT_AT..RECORD_COUNT_AT + 2]
|
||||
.copy_from_slice(&(records.len() as u16).to_be_bytes());
|
||||
for (at, record) in records.iter().enumerate() {
|
||||
let offset = out.len() as u32;
|
||||
let entry = RECORD_LIST_AT + at * 8;
|
||||
out[entry..entry + 4].copy_from_slice(&offset.to_be_bytes());
|
||||
out.extend_from_slice(record);
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
fn exth(records: &[(u32, &[u8])]) -> Option<Vec<(u32, Vec<u8>)>> {
|
||||
Some(records.iter().map(|(k, v)| (*k, v.to_vec())).collect())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_exth_records_fill_the_record() {
|
||||
let book = Book {
|
||||
exth: exth(&[
|
||||
(UPDATED_TITLE, b"The Real Title"),
|
||||
(AUTHOR, b"First Author"),
|
||||
(AUTHOR, b"Second Author"),
|
||||
(PUBLISHER, b"A Press"),
|
||||
(DESCRIPTION, b"What it is about."),
|
||||
(ISBN, b"978-0-306-40615-7"),
|
||||
(SUBJECT, b"History"),
|
||||
(SUBJECT, b"Ships"),
|
||||
(PUBLISHED, b"1999-01-02"),
|
||||
(ASIN, b"B000TEST00"),
|
||||
(LANGUAGE, b"de"),
|
||||
]),
|
||||
..Book::default()
|
||||
};
|
||||
let meta = read("x.mobi", &book.bytes());
|
||||
assert_eq!(meta.source, "mobi");
|
||||
assert_eq!(meta.title.as_deref(), Some("The Real Title"));
|
||||
assert_eq!(meta.authors, ["First Author", "Second Author"]);
|
||||
assert_eq!(meta.publisher.as_deref(), Some("A Press"));
|
||||
assert_eq!(meta.description.as_deref(), Some("What it is about."));
|
||||
assert_eq!(meta.date.as_deref(), Some("1999-01-02"));
|
||||
assert_eq!(meta.subjects, ["History", "Ships"]);
|
||||
assert_eq!(meta.language.as_deref(), Some("de"));
|
||||
assert_eq!(meta.identifiers["isbn"], "9780306406157");
|
||||
assert_eq!(meta.identifiers["asin"], "B000TEST00");
|
||||
assert!(meta.notes.is_empty(), "{:?}", meta.notes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_title_falls_back_to_the_full_name_and_then_the_database_name() {
|
||||
let meta = read("x.mobi", &Book::default().bytes());
|
||||
assert_eq!(meta.title.as_deref(), Some("A Test Book"));
|
||||
|
||||
let book = Book {
|
||||
full_name: Vec::new(),
|
||||
..Book::default()
|
||||
};
|
||||
let meta = read("x.mobi", &book.bytes());
|
||||
assert_eq!(meta.title.as_deref(), Some("A_Test_Book"));
|
||||
assert!(meta.notes[0].contains("Palm database name"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_language_falls_back_to_the_locale() {
|
||||
let book = Book {
|
||||
locale: 0x040c,
|
||||
..Book::default()
|
||||
};
|
||||
assert_eq!(
|
||||
read("x.mobi", &book.bytes()).language.as_deref(),
|
||||
Some("fr")
|
||||
);
|
||||
|
||||
let book = Book {
|
||||
locale: 0,
|
||||
..Book::default()
|
||||
};
|
||||
assert_eq!(read("x.mobi", &book.bytes()).language, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn windows_1252_strings_decode_as_windows_1252() {
|
||||
let book = Book {
|
||||
encoding: WINDOWS_1252,
|
||||
exth: exth(&[
|
||||
(AUTHOR, b"Ren\xe9 Ch\x96ar"),
|
||||
(PUBLISHER, b"\x93Quoted\x94"),
|
||||
]),
|
||||
..Book::default()
|
||||
};
|
||||
let meta = read("x.mobi", &book.bytes());
|
||||
assert_eq!(meta.authors, ["René Ch–ar"]);
|
||||
assert_eq!(meta.publisher.as_deref(), Some("“Quoted”"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_cover_is_the_image_record_the_offset_names() {
|
||||
let book = Book {
|
||||
exth: exth(&[(COVER_OFFSET, &1u32.to_be_bytes())]),
|
||||
images: vec![b"GIF89a-thumb".to_vec(), JPEG.to_vec()],
|
||||
..Book::default()
|
||||
};
|
||||
let cover = read("x.mobi", &book.bytes()).cover.expect("a cover");
|
||||
assert_eq!(cover.media_type, "image/jpeg");
|
||||
assert_eq!(cover.bytes, JPEG);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_thumbnail_stands_in_when_no_cover_is_named() {
|
||||
let book = Book {
|
||||
exth: exth(&[(THUMB_OFFSET, &0u32.to_be_bytes())]),
|
||||
images: vec![b"\x89PNG\r\n\x1a\nrest".to_vec()],
|
||||
..Book::default()
|
||||
};
|
||||
let cover = read("x.mobi", &book.bytes()).cover.expect("a cover");
|
||||
assert_eq!(cover.media_type, "image/png");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_no_cover_named_only_the_record_at_first_image_can_be_the_cover() {
|
||||
let book = Book {
|
||||
images: vec![JPEG.to_vec(), b"GIF89a-later".to_vec()],
|
||||
..Book::default()
|
||||
};
|
||||
let cover = read("x.mobi", &book.bytes()).cover.expect("a cover");
|
||||
assert_eq!(cover.media_type, "image/jpeg");
|
||||
assert_eq!(cover.bytes, JPEG);
|
||||
|
||||
// The record at first_image is not an image: no cover, even though a
|
||||
// later record is one.
|
||||
let book = Book {
|
||||
images: vec![
|
||||
b"not an image".to_vec(),
|
||||
JPEG.to_vec(),
|
||||
b"GIF89a-later".to_vec(),
|
||||
],
|
||||
..Book::default()
|
||||
};
|
||||
assert_eq!(read("x.mobi", &book.bytes()).cover, None);
|
||||
|
||||
let book = Book {
|
||||
images: vec![b"not an image".to_vec()],
|
||||
..Book::default()
|
||||
};
|
||||
assert_eq!(read("x.mobi", &book.bytes()).cover, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_font_at_first_image_is_not_skipped_past_to_an_inline_image() {
|
||||
// No EXTH 201 or 202. The first resource record is a font, as in
|
||||
// kindlegen output; calibre reads only that record and finds no cover.
|
||||
let book = Book {
|
||||
images: vec![b"FONT\0\0\0\x18".to_vec(), JPEG.to_vec()],
|
||||
..Book::default()
|
||||
};
|
||||
assert_eq!(read("x.mobi", &book.bytes()).cover, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stale_kf8_boundary_below_first_image_does_not_hide_the_cover() {
|
||||
let book = Book {
|
||||
exth: exth(&[(121, &1u32.to_be_bytes())]),
|
||||
first_image: 2,
|
||||
images: vec![b"text".to_vec(), JPEG.to_vec()],
|
||||
..Book::default()
|
||||
};
|
||||
assert!(read("x.mobi", &book.bytes()).cover.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_first_image_search_stops_at_the_kf8_boundary() {
|
||||
let book = Book {
|
||||
exth: exth(&[(121, &2u32.to_be_bytes())]),
|
||||
images: vec![b"text".to_vec(), JPEG.to_vec()],
|
||||
..Book::default()
|
||||
};
|
||||
// Record 1 is not an image, and record 2 is past the boundary.
|
||||
assert_eq!(read("x.mobi", &book.bytes()).cover, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exth_113_is_an_asin_a_uuid_or_something_else() {
|
||||
let id = |value: &[u8]| {
|
||||
let book = Book {
|
||||
exth: exth(&[(ASIN, value)]),
|
||||
..Book::default()
|
||||
};
|
||||
read("x.mobi", &book.bytes()).identifiers
|
||||
};
|
||||
assert_eq!(id(b"B01A7RJGIW")["asin"], "B01A7RJGIW");
|
||||
assert_eq!(id(b"0306406152")["asin"], "0306406152");
|
||||
let uuid = id(b"69AE65CA-B644-499F-A78C-139771C7140A");
|
||||
assert_eq!(uuid["uuid"], "69ae65ca-b644-499f-a78c-139771c7140a");
|
||||
assert_eq!(uuid.get("asin"), None);
|
||||
let other = id(b"2AW6TL2JJEYYGXCOV3PWN6ZOYUORXAVX");
|
||||
assert_eq!(other["other"], "2AW6TL2JJEYYGXCOV3PWN6ZOYUORXAVX");
|
||||
assert_eq!(other.get("asin"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cover_that_points_nowhere_is_a_note_and_not_a_cover() {
|
||||
let book = Book {
|
||||
exth: exth(&[(COVER_OFFSET, &7u32.to_be_bytes())]),
|
||||
images: vec![JPEG.to_vec()],
|
||||
..Book::default()
|
||||
};
|
||||
let meta = read("x.mobi", &book.bytes());
|
||||
assert_eq!(meta.cover, None);
|
||||
assert_eq!(meta.notes, ["the cover points past the last record"]);
|
||||
|
||||
let book = Book {
|
||||
exth: exth(&[(COVER_OFFSET, &u32::MAX.to_be_bytes())]),
|
||||
first_image: u32::MAX - 1,
|
||||
images: vec![JPEG.to_vec()],
|
||||
..Book::default()
|
||||
};
|
||||
assert_eq!(read("x.mobi", &book.bytes()).cover, None);
|
||||
|
||||
let book = Book {
|
||||
exth: exth(&[(COVER_OFFSET, &0u32.to_be_bytes())]),
|
||||
images: vec![b"not an image".to_vec()],
|
||||
..Book::default()
|
||||
};
|
||||
let meta = read("x.mobi", &book.bytes());
|
||||
assert_eq!(meta.cover, None);
|
||||
assert_eq!(meta.notes, ["the cover record is not an image"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_encrypted_book_still_has_its_header_read() {
|
||||
let book = Book {
|
||||
encryption: 2,
|
||||
exth: exth(&[(AUTHOR, b"Somebody")]),
|
||||
..Book::default()
|
||||
};
|
||||
let meta = read("x.azw", &book.bytes());
|
||||
assert_eq!(meta.authors, ["Somebody"]);
|
||||
assert_eq!(meta.notes, ["drm"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_exth_record_that_overruns_its_block_stops_the_reading() {
|
||||
let mut bytes = Book {
|
||||
exth: exth(&[(AUTHOR, b"Kept"), (PUBLISHER, b"Lost")]),
|
||||
..Book::default()
|
||||
}
|
||||
.bytes();
|
||||
// The second record's length, made to reach past the block.
|
||||
let at = find(&bytes, b"Lost") - 4;
|
||||
bytes[at..at + 4].copy_from_slice(&0x7fff_ffffu32.to_be_bytes());
|
||||
let meta = read("x.mobi", &bytes);
|
||||
assert_eq!(meta.authors, ["Kept"]);
|
||||
assert_eq!(meta.publisher, None);
|
||||
assert!(
|
||||
meta.notes
|
||||
.contains(&"the EXTH block ends early".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_record_length_shorter_than_its_own_header_stops_the_reading() {
|
||||
let mut bytes = Book {
|
||||
exth: exth(&[(AUTHOR, b"Kept")]),
|
||||
..Book::default()
|
||||
}
|
||||
.bytes();
|
||||
let at = find(&bytes, b"Kept") - 4;
|
||||
bytes[at..at + 4].copy_from_slice(&3u32.to_be_bytes());
|
||||
let meta = read("x.mobi", &bytes);
|
||||
assert!(meta.authors.is_empty());
|
||||
assert!(
|
||||
meta.notes
|
||||
.contains(&"the EXTH block ends early".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_is_not_a_kindle_book_says_so() {
|
||||
let meta = read("x.mobi", b"not a palm database at all");
|
||||
assert_eq!(meta.source, "none");
|
||||
assert!(meta.notes[0].starts_with("not a Kindle book"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_first_record_with_no_mobi_header_is_refused() {
|
||||
let mut bytes = Book::default().bytes();
|
||||
let at = mobi_header(&bytes);
|
||||
bytes[at] = b'X';
|
||||
let meta = read("x.mobi", &bytes);
|
||||
assert_eq!(meta.source, "none");
|
||||
assert_eq!(meta.notes, ["no MOBI header in the first record"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_header_too_short_for_a_field_does_not_have_that_field() {
|
||||
let mut bytes = Book::default().bytes();
|
||||
let at = mobi_header(&bytes) + HEADER_LENGTH;
|
||||
// Long enough for the encoding, too short for the full name.
|
||||
bytes[at..at + 4].copy_from_slice(&24u32.to_be_bytes());
|
||||
let meta = read("x.mobi", &bytes);
|
||||
assert_eq!(meta.title.as_deref(), Some("A_Test_Book"));
|
||||
assert_eq!(meta.language, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_truncation_returns_without_panicking() {
|
||||
let bytes = Book {
|
||||
exth: exth(&[
|
||||
(UPDATED_TITLE, b"Title"),
|
||||
(AUTHOR, b"Author"),
|
||||
(COVER_OFFSET, &0u32.to_be_bytes()),
|
||||
]),
|
||||
images: vec![JPEG.to_vec()],
|
||||
..Book::default()
|
||||
}
|
||||
.bytes();
|
||||
for len in 0..=bytes.len() {
|
||||
let _ = read("x.mobi", &bytes[..len]);
|
||||
}
|
||||
}
|
||||
|
||||
/// Where the MOBI header starts: after the record list, so the `MOBI`
|
||||
/// at the end of the database type is not it.
|
||||
fn mobi_header(bytes: &[u8]) -> usize {
|
||||
RECORD_LIST_AT + find(&bytes[RECORD_LIST_AT..], MOBI_MAGIC)
|
||||
}
|
||||
|
||||
fn find(haystack: &[u8], needle: &[u8]) -> usize {
|
||||
haystack
|
||||
.windows(needle.len())
|
||||
.position(|window| window == needle)
|
||||
.expect("present")
|
||||
}
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
# Fixtures
|
||||
|
||||
## MOBI and AZW3
|
||||
|
||||
`lantern.html` is a two-paragraph story written for these tests, and
|
||||
`lantern-cover.jpg` a 120×180 cover drawn for them with Pillow. Both are
|
||||
dedicated to the public domain (CC0 1.0), and so is everything made from them.
|
||||
|
||||
`lantern.mobi` (MOBI 6) and `lantern.azw3` (KF8) were made from them with
|
||||
calibre 9.14:
|
||||
|
||||
```sh
|
||||
common=(--title "A Lantern in the Orchard" --authors "Ada Quill & Tomás Reed"
|
||||
--publisher "Orchard Press" --isbn 9780306406157 --tags "Fiction,Villages"
|
||||
--language en --pubdate 2001-04-05 --comments "A short story written for testing."
|
||||
--cover lantern-cover.jpg)
|
||||
ebook-convert lantern.html lantern.mobi --mobi-file-type old "${common[@]}"
|
||||
ebook-convert lantern.html lantern.azw3 "${common[@]}"
|
||||
```
|
||||
|
||||
The other three are `lantern.mobi` with one thing broken. `r0` is the first
|
||||
record's offset, the big-endian 32-bit number at byte 78 (136 here).
|
||||
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| `lantern-no-exth.mobi` | The EXTH flag (`0x40`) cleared in the MOBI header's flags, at `r0 + 16 + 112` |
|
||||
| `lantern-drm.mobi` | The PalmDOC encryption type, at `r0 + 12`, set to 2 |
|
||||
| `lantern-truncated.mobi` | The first 1024 bytes only |
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 2.7 KiB |
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
+5
@@ -0,0 +1,5 @@
|
||||
<html><head><meta charset="utf-8"><title>x</title></head><body>
|
||||
<h1>A Lantern in the Orchard</h1>
|
||||
<p>The lantern hung from the lowest branch of the pear tree, and every evening someone lit it. Nobody in the village could say who.</p>
|
||||
<p>Café owners swore it was the miller; the miller swore it was the wind.</p>
|
||||
</body></html>
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,144 @@
|
||||
//! MOBI and AZW3 files as a converter writes them, and as they break.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
fn fixture(name: &str) -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/fixtures")
|
||||
.join(name)
|
||||
}
|
||||
|
||||
fn read(name: &str) -> bookmeta::Metadata {
|
||||
bookmeta::read(&fixture(name))
|
||||
}
|
||||
|
||||
/// What the two converted fixtures were given, and so must say.
|
||||
fn says_what_it_was_given(meta: &bookmeta::Metadata) {
|
||||
assert_eq!(meta.source, "mobi");
|
||||
assert_eq!(meta.title.as_deref(), Some("A Lantern in the Orchard"));
|
||||
assert_eq!(meta.authors, ["Ada Quill", "Tomás Reed"]);
|
||||
assert_eq!(meta.publisher.as_deref(), Some("Orchard Press"));
|
||||
assert_eq!(
|
||||
meta.description.as_deref(),
|
||||
Some("A short story written for testing.")
|
||||
);
|
||||
assert_eq!(meta.date.as_deref(), Some("2001-04-05T04:00:00+00:00"));
|
||||
assert_eq!(meta.language.as_deref(), Some("en"));
|
||||
assert_eq!(meta.subjects, ["Fiction", "Villages"]);
|
||||
assert_eq!(meta.identifiers["isbn"], "9780306406157");
|
||||
assert_eq!(meta.identifiers.get("asin"), None);
|
||||
assert_eq!(meta.identifiers["uuid"].len(), 36);
|
||||
|
||||
let cover = meta.cover.as_ref().expect("a cover");
|
||||
assert_eq!(cover.media_type, "image/jpeg");
|
||||
let original = std::fs::read(fixture("lantern-cover.jpg")).expect("the cover fixture");
|
||||
assert_eq!(cover.bytes, original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_mobi_6_file_says_what_it_was_given() {
|
||||
let meta = read("lantern.mobi");
|
||||
says_what_it_was_given(&meta);
|
||||
assert!(meta.notes.is_empty(), "{:?}", meta.notes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_azw3_file_says_what_it_was_given() {
|
||||
let meta = read("lantern.azw3");
|
||||
says_what_it_was_given(&meta);
|
||||
assert!(meta.notes.is_empty(), "{:?}", meta.notes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exth_113_uuid_is_not_an_asin() {
|
||||
for name in ["lantern.mobi", "lantern.azw3"] {
|
||||
let meta = read(name);
|
||||
assert_eq!(meta.identifiers.get("asin"), None, "{name}");
|
||||
assert!(meta.identifiers.contains_key("uuid"), "{name}");
|
||||
}
|
||||
}
|
||||
|
||||
/// `lantern.mobi` with its cover and thumbnail offsets (EXTH 201 and 202)
|
||||
/// retyped to an unused record type, so the file names no cover.
|
||||
fn lantern_naming_no_cover() -> Vec<u8> {
|
||||
let mut bytes = std::fs::read(fixture("lantern.mobi")).expect("the fixture");
|
||||
let exth = bytes.windows(4).position(|w| w == b"EXTH").expect("EXTH");
|
||||
let count = u32::from_be_bytes(bytes[exth + 8..exth + 12].try_into().unwrap());
|
||||
let mut cursor = exth + 12;
|
||||
let mut retyped = 0;
|
||||
for _ in 0..count {
|
||||
let kind = u32::from_be_bytes(bytes[cursor..cursor + 4].try_into().unwrap());
|
||||
let size = u32::from_be_bytes(bytes[cursor + 4..cursor + 8].try_into().unwrap());
|
||||
if kind == 201 || kind == 202 {
|
||||
bytes[cursor..cursor + 4].copy_from_slice(&999u32.to_be_bytes());
|
||||
retyped += 1;
|
||||
}
|
||||
cursor += size as usize;
|
||||
}
|
||||
assert!(retyped > 0, "the fixture names a cover");
|
||||
bytes
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_naming_no_cover_gets_its_first_image_record() {
|
||||
let meta = bookmeta::from_bytes("x.mobi", &lantern_naming_no_cover());
|
||||
let cover = meta.cover.expect("the first image record");
|
||||
assert_eq!(cover.media_type, "image/jpeg");
|
||||
let original = std::fs::read(fixture("lantern-cover.jpg")).expect("the cover fixture");
|
||||
assert_eq!(cover.bytes, original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_with_no_exth_still_has_its_full_name_and_locale() {
|
||||
let meta = read("lantern-no-exth.mobi");
|
||||
assert_eq!(meta.source, "mobi");
|
||||
assert_eq!(meta.title.as_deref(), Some("A Lantern in the Orchard"));
|
||||
assert_eq!(meta.language.as_deref(), Some("en"));
|
||||
assert!(meta.authors.is_empty());
|
||||
assert!(meta.identifiers.is_empty());
|
||||
// No EXTH names no cover, so the first image record stands in for one.
|
||||
let cover = meta.cover.expect("the first image record");
|
||||
assert_eq!(cover.media_type, "image/jpeg");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_truncated_file_is_an_empty_record_with_a_reason() {
|
||||
let meta = read("lantern-truncated.mobi");
|
||||
assert_eq!(meta.source, "none");
|
||||
assert!(meta.is_empty());
|
||||
assert_eq!(meta.notes, ["the first record lies outside the file"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_drm_flagged_file_has_its_metadata_and_says_drm() {
|
||||
let meta = read("lantern-drm.mobi");
|
||||
says_what_it_was_given(&meta);
|
||||
assert_eq!(meta.notes, ["drm"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_truncation_of_the_mobi_fixture_returns() {
|
||||
let bytes = std::fs::read(fixture("lantern.mobi")).expect("the fixture");
|
||||
let first_record_end = u32::from_be_bytes(bytes[86..90].try_into().unwrap()) as usize;
|
||||
for len in 0..=bytes.len() {
|
||||
let meta = bookmeta::from_bytes("cut.mobi", &bytes[..len]);
|
||||
if len < first_record_end {
|
||||
assert!(meta.is_empty(), "cut at {len}: {meta:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_single_byte_corruption_of_the_header_returns() {
|
||||
let bytes = std::fs::read(fixture("lantern.mobi")).expect("the fixture");
|
||||
// The Palm header, the record list, and the whole of the first record:
|
||||
// every offset the reader follows is in there.
|
||||
let first_record_end = u32::from_be_bytes(bytes[86..90].try_into().unwrap()) as usize;
|
||||
for at in 0..first_record_end {
|
||||
for value in [0x00, 0x7f, 0xff] {
|
||||
let mut broken = bytes.clone();
|
||||
broken[at] = value;
|
||||
let _ = bookmeta::from_bytes("broken.mobi", &broken);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
//! An ISBN's two forms, ten digits and thirteen.
|
||||
//!
|
||||
//! A book printed before 2007 carries a ten-digit ISBN, one printed since
|
||||
//! carries thirteen, and every ten-digit one has a thirteen-digit twin: `978`
|
||||
//! in front and the check digit worked out again. A catalogue may file a
|
||||
//! book under either, so asking about one form and not the other misses
|
||||
//! books that are there.
|
||||
//!
|
||||
//! Only a well-formed ISBN is converted. Working out a fresh check digit for
|
||||
//! one whose own check digit is wrong would not correct the typing error, it
|
||||
//! would turn the number into a different book's.
|
||||
|
||||
/// Just the digits of an ISBN, with a check character of ten as `X`.
|
||||
pub fn digits(text: &str) -> String {
|
||||
bibclean::isbn::digits(text).to_ascii_uppercase()
|
||||
}
|
||||
|
||||
/// The thirteen-digit form of an ISBN, from either form.
|
||||
///
|
||||
/// `None` when the text is not an ISBN whose check digit agrees.
|
||||
pub fn to_13(text: &str) -> Option<String> {
|
||||
let digits = digits(text);
|
||||
if !bibclean::isbn::valid(&digits) {
|
||||
return None;
|
||||
}
|
||||
match digits.len() {
|
||||
13 => Some(digits),
|
||||
10 => {
|
||||
let body = format!("978{}", &digits[..9]);
|
||||
let sum: u32 = body
|
||||
.bytes()
|
||||
.enumerate()
|
||||
.map(|(at, b)| u32::from(b - b'0') * if at % 2 == 0 { 1 } else { 3 })
|
||||
.sum();
|
||||
Some(format!("{body}{}", (10 - sum % 10) % 10))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The ten-digit form of an ISBN, from either form.
|
||||
///
|
||||
/// `None` when the text is not an ISBN whose check digit agrees, or when it
|
||||
/// is a thirteen-digit one starting `979`, which was never issued in ten.
|
||||
pub fn to_10(text: &str) -> Option<String> {
|
||||
let digits = digits(text);
|
||||
if !bibclean::isbn::valid(&digits) {
|
||||
return None;
|
||||
}
|
||||
match digits.len() {
|
||||
10 => Some(digits),
|
||||
13 if digits.starts_with("978") => {
|
||||
let body = &digits[3..12];
|
||||
let sum: u32 = body
|
||||
.bytes()
|
||||
.enumerate()
|
||||
.map(|(at, b)| u32::from(b - b'0') * (10 - at as u32))
|
||||
.sum();
|
||||
let check = match (11 - sum % 11) % 11 {
|
||||
10 => 'X',
|
||||
n => char::from(b'0' + n as u8),
|
||||
};
|
||||
Some(format!("{body}{check}"))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn each_form_becomes_the_other() {
|
||||
assert_eq!(to_13("0-306-40615-2").as_deref(), Some("9780306406157"));
|
||||
assert_eq!(to_10("978-0-306-40615-7").as_deref(), Some("0306406152"));
|
||||
assert_eq!(to_13("9780306406157").as_deref(), Some("9780306406157"));
|
||||
assert_eq!(to_10("0306406152").as_deref(), Some("0306406152"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_check_digit_of_ten_is_written_x() {
|
||||
assert_eq!(to_10("9780804429573").as_deref(), Some("080442957X"));
|
||||
assert_eq!(to_13("080442957x").as_deref(), Some("9780804429573"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_979_isbn_has_no_ten_digit_form() {
|
||||
assert_eq!(to_10("979-10-90636-07-1"), None);
|
||||
assert_eq!(to_13("979-10-90636-07-1").as_deref(), Some("9791090636071"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_mistyped_isbn_is_not_converted_into_a_different_book() {
|
||||
assert_eq!(to_13("0-306-40615-3"), None);
|
||||
assert_eq!(to_10("978-0-306-40615-8"), None);
|
||||
assert_eq!(to_13("12345"), None);
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@
|
||||
|
||||
pub mod arxiv;
|
||||
pub mod doi;
|
||||
pub mod isbn;
|
||||
pub mod issn;
|
||||
|
||||
use schemars::JsonSchema;
|
||||
|
||||
+1
-1
@@ -64,7 +64,7 @@ No network, no database, no configuration file, no state kept between runs.
|
||||
Five exceptions, each declared in `--help`:
|
||||
|
||||
- [`emb`](../crates/emb/README.md) makes an HTTP call, which is the whole of what it does.
|
||||
- [`bibfetch`](../crates/bibfetch/README.md) asks a register about an identifier, likewise.
|
||||
- [`bibfetch`](../crates/bibfetch/README.md) asks a register about an identifier or a title, likewise.
|
||||
- [`pdftriage`](../crates/pdftriage/README.md) runs `pdftotext` and `pdfinfo`.
|
||||
- [`pdfgutter`](../crates/pdfgutter/README.md) runs `pdftoppm` to look at a page.
|
||||
- [`avplan`](../crates/avplan/README.md) runs `ffprobe` to read a file's streams, and
|
||||
|
||||
Reference in new issue
Block a user