mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-11-13 23:55:40 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 funkpower
|
bd104834ed |
No files matched your search
Generated
+26
-26
@@ -101,7 +101,7 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "avplan"
|
name = "avplan"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -126,7 +126,7 @@ checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bibclean"
|
name = "bibclean"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -143,7 +143,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bibfetch"
|
name = "bibfetch"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -196,7 +196,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bookmeta"
|
name = "bookmeta"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -210,7 +210,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "booksniff"
|
name = "booksniff"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -250,7 +250,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "catena"
|
name = "catena"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bibclean",
|
"bibclean",
|
||||||
"bibfetch",
|
"bibfetch",
|
||||||
@@ -277,7 +277,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "catena-cli"
|
name = "catena-cli"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"clap",
|
"clap",
|
||||||
"clap_mangen",
|
"clap_mangen",
|
||||||
@@ -312,7 +312,7 @@ checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "chunkmd"
|
name = "chunkmd"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -334,7 +334,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "citefind"
|
name = "citefind"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"chunkmd",
|
"chunkmd",
|
||||||
@@ -399,7 +399,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "colophon"
|
name = "colophon"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bibclean",
|
"bibclean",
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
@@ -448,7 +448,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "corpus-core"
|
name = "corpus-core"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"schemars",
|
"schemars",
|
||||||
"serde",
|
"serde",
|
||||||
@@ -492,7 +492,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cuekit"
|
name = "cuekit"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"corpus-core",
|
"corpus-core",
|
||||||
"regex",
|
"regex",
|
||||||
@@ -504,7 +504,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cuenorm"
|
name = "cuenorm"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -514,7 +514,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cuepack"
|
name = "cuepack"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -525,7 +525,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cueseg"
|
name = "cueseg"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -612,7 +612,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "emb"
|
name = "emb"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -876,7 +876,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "idnorm"
|
name = "idnorm"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bibclean",
|
"bibclean",
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
@@ -942,7 +942,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "langid"
|
name = "langid"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -1022,7 +1022,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mdscrub"
|
name = "mdscrub"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -1034,7 +1034,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mdsplit"
|
name = "mdsplit"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -1113,7 +1113,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pdfgutter"
|
name = "pdfgutter"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -1126,7 +1126,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pdftriage"
|
name = "pdftriage"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -1370,7 +1370,7 @@ checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rrf"
|
name = "rrf"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -1517,7 +1517,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "seriesof"
|
name = "seriesof"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -1634,7 +1634,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "textkit"
|
name = "textkit"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"regex",
|
"regex",
|
||||||
"tiktoken-rs",
|
"tiktoken-rs",
|
||||||
@@ -1724,7 +1724,7 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tocfind"
|
name = "tocfind"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"catena-cli",
|
"catena-cli",
|
||||||
"clap",
|
"clap",
|
||||||
|
|||||||
+27
-27
@@ -3,39 +3,39 @@ resolver = "3"
|
|||||||
members = ["crates/*"]
|
members = ["crates/*"]
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
rust-version = "1.85"
|
rust-version = "1.85"
|
||||||
license = "MPL-2.0"
|
license = "MPL-2.0"
|
||||||
repository = "https://git.pumpin.online/funkpower/catena"
|
repository = "https://git.pumpin.online/funkpower/catena"
|
||||||
|
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
catena-cli = { path = "crates/catena-cli", version = "0.3.1" }
|
catena-cli = { path = "crates/catena-cli", version = "0.4.0" }
|
||||||
corpus-core = { path = "crates/corpus-core", version = "0.3.1" }
|
corpus-core = { path = "crates/corpus-core", version = "0.4.0" }
|
||||||
cuekit = { path = "crates/cuekit", version = "0.3.1" }
|
cuekit = { path = "crates/cuekit", version = "0.4.0" }
|
||||||
textkit = { path = "crates/textkit", version = "0.3.1" }
|
textkit = { path = "crates/textkit", version = "0.4.0" }
|
||||||
chunkmd = { path = "crates/chunkmd", version = "0.3.1" }
|
chunkmd = { path = "crates/chunkmd", version = "0.4.0" }
|
||||||
mdscrub = { path = "crates/mdscrub", version = "0.3.1" }
|
mdscrub = { path = "crates/mdscrub", version = "0.4.0" }
|
||||||
mdsplit = { path = "crates/mdsplit", version = "0.3.1" }
|
mdsplit = { path = "crates/mdsplit", version = "0.4.0" }
|
||||||
pdftriage = { path = "crates/pdftriage", version = "0.3.1" }
|
pdftriage = { path = "crates/pdftriage", version = "0.4.0" }
|
||||||
cuenorm = { path = "crates/cuenorm", version = "0.3.1" }
|
cuenorm = { path = "crates/cuenorm", version = "0.4.0" }
|
||||||
cueseg = { path = "crates/cueseg", version = "0.3.1" }
|
cueseg = { path = "crates/cueseg", version = "0.4.0" }
|
||||||
rrf = { path = "crates/rrf", version = "0.3.1" }
|
rrf = { path = "crates/rrf", version = "0.4.0" }
|
||||||
cuepack = { path = "crates/cuepack", version = "0.3.1" }
|
cuepack = { path = "crates/cuepack", version = "0.4.0" }
|
||||||
bibclean = { path = "crates/bibclean", version = "0.3.1" }
|
bibclean = { path = "crates/bibclean", version = "0.4.0" }
|
||||||
avplan = { path = "crates/avplan", version = "0.3.1" }
|
avplan = { path = "crates/avplan", version = "0.4.0" }
|
||||||
pdfgutter = { path = "crates/pdfgutter", version = "0.3.1" }
|
pdfgutter = { path = "crates/pdfgutter", version = "0.4.0" }
|
||||||
catena = { path = "crates/bindings", version = "0.3.1" }
|
catena = { path = "crates/bindings", version = "0.4.0" }
|
||||||
seriesof = { path = "crates/seriesof", version = "0.3.1" }
|
seriesof = { path = "crates/seriesof", version = "0.4.0" }
|
||||||
idnorm = { path = "crates/idnorm", version = "0.3.1" }
|
idnorm = { path = "crates/idnorm", version = "0.4.0" }
|
||||||
bibfetch = { path = "crates/bibfetch", version = "0.3.1" }
|
bibfetch = { path = "crates/bibfetch", version = "0.4.0" }
|
||||||
bookmeta = { path = "crates/bookmeta", version = "0.3.1" }
|
bookmeta = { path = "crates/bookmeta", version = "0.4.0" }
|
||||||
langid = { path = "crates/langid", version = "0.3.1" }
|
langid = { path = "crates/langid", version = "0.4.0" }
|
||||||
tocfind = { path = "crates/tocfind", version = "0.3.1" }
|
tocfind = { path = "crates/tocfind", version = "0.4.0" }
|
||||||
booksniff = { path = "crates/booksniff", version = "0.3.1" }
|
booksniff = { path = "crates/booksniff", version = "0.4.0" }
|
||||||
citefind = { path = "crates/citefind", version = "0.3.1" }
|
citefind = { path = "crates/citefind", version = "0.4.0" }
|
||||||
colophon = { path = "crates/colophon", version = "0.3.1" }
|
colophon = { path = "crates/colophon", version = "0.4.0" }
|
||||||
emb = { path = "crates/emb", version = "0.3.1" }
|
emb = { path = "crates/emb", version = "0.4.0" }
|
||||||
|
|
||||||
clap = { version = "4", features = ["derive"] }
|
clap = { version = "4", features = ["derive"] }
|
||||||
clap_mangen = "0.3"
|
clap_mangen = "0.3"
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ Each tool installs by name, so you can take one without taking the rest.
|
|||||||
| [`vocabmerge`](crates/bibclean/README.md#vocabmerge--the-other-command) | The near-duplicate headings in a catalogue's vocabulary |
|
| [`vocabmerge`](crates/bibclean/README.md#vocabmerge--the-other-command) | The near-duplicate headings in a catalogue's vocabulary |
|
||||||
| [`seriesof`](crates/seriesof/README.md) | The series a book belongs to, and which volume this is |
|
| [`seriesof`](crates/seriesof/README.md) | The series a book belongs to, and which volume this is |
|
||||||
| [`idnorm`](crates/idnorm/README.md) | DOIs, ISSNs, ISBNs and arXiv ids, read and checked |
|
| [`idnorm`](crates/idnorm/README.md) | DOIs, ISSNs, ISBNs and arXiv ids, read and checked |
|
||||||
| [`bibfetch`](crates/bibfetch/README.md) | What a register says about an identifier, and which books it has under a title |
|
| [`bibfetch`](crates/bibfetch/README.md) | What a register says about an identifier |
|
||||||
| [`citefind`](crates/citefind/README.md) | The references in a bibliography or a page of notes |
|
| [`citefind`](crates/citefind/README.md) | The references in a bibliography or a page of notes |
|
||||||
| [`mdscrub`](crates/mdscrub/README.md) | Converted markdown in, clean markdown out |
|
| [`mdscrub`](crates/mdscrub/README.md) | Converted markdown in, clean markdown out |
|
||||||
| [`mdsplit`](crates/mdsplit/README.md) | One markdown file into the works inside it |
|
| [`mdsplit`](crates/mdsplit/README.md) | One markdown file into the works inside it |
|
||||||
|
|||||||
@@ -20,8 +20,6 @@ So this asks, and that is the only thing it does.
|
|||||||
- **Filling in a reference** you have only the identifier for.
|
- **Filling in a reference** you have only the identifier for.
|
||||||
- **Before citing something**, to check the year and the authors are what you
|
- **Before citing something**, to check the year and the authors are what you
|
||||||
think they are.
|
think they are.
|
||||||
- **Cataloguing a book that came with no identifier**, to find the editions a
|
|
||||||
register has under its title, and an ISBN to file it by.
|
|
||||||
|
|
||||||
## Install
|
## Install
|
||||||
|
|
||||||
@@ -150,77 +148,6 @@ From Rust, `bibfetch::cover(&agent, isbn, cover_id, size)` returns
|
|||||||
`Ok(None)` when there is no cover, and a [`Cover`](src/cover.rs) with the
|
`Ok(None)` when there is no cover, and a [`Cover`](src/cover.rs) with the
|
||||||
bytes, the media type and the address asked when there is.
|
bytes, the media type and the address asked when there is.
|
||||||
|
|
||||||
## Searching by title
|
|
||||||
|
|
||||||
```
|
|
||||||
$ bibfetch search --title 'The First World War' --author Howard --limit 2
|
|
||||||
{"v":1,"id":"9780192853622","kind":"isbn","found":true,"source":"openlibrary","title":"The First World War","authors":["Michael Eliot Howard"],"year":2002,"container":"Oxford University Press","subjects":["World War, 1914-1918"],"language":"eng","pages":154,"cover_id":1123124}
|
|
||||||
{"v":1,"id":"9780199205592","kind":"isbn","found":true,"source":"openlibrary","title":"The First World War: A Very Short Introduction","authors":["Michael Howard"],"year":2007,"container":"Oxford University Press","language":"eng","cover_id":14606475}
|
|
||||||
```
|
|
||||||
|
|
||||||
A book that arrives as a file often has no identifier to ask about, only a
|
|
||||||
title and perhaps a name. This asks which books a register has under those,
|
|
||||||
and prints one record per candidate: Open Library's index first, and Google
|
|
||||||
Books only when Open Library has nothing.
|
|
||||||
|
|
||||||
Each candidate is one edition. Open Library's index answers with works, and
|
|
||||||
names the edition of each that best matches the search; what belongs to an
|
|
||||||
edition — its ISBN, publisher, language, year and cover — is that edition's,
|
|
||||||
and what belongs to the work — the title, the authors, the subjects and the
|
|
||||||
length — is the work's. Google Books answers with volumes, which are editions
|
|
||||||
already.
|
|
||||||
|
|
||||||
| Field | When the candidate has an ISBN | When it has none |
|
|
||||||
|---|---|---|
|
|
||||||
| `id` | The first ISBN whose check digit agrees, in thirteen digits; one written in thirteen is taken over one written in ten | The register's own key: an Open Library id such as `OL37863121M` for the edition, or `OL267171W` for the work when no edition was named; a Google Books volume id |
|
|
||||||
| `kind` | `isbn` | `olid` for Open Library, `googlebooks` for Google Books |
|
|
||||||
|
|
||||||
An ISBN whose check digit is wrong is passed over rather than corrected, and
|
|
||||||
Google's shelfmarks, typed `OTHER`, are not read as ISBNs even when their
|
|
||||||
digits would pass for one. A record of kind `isbn` can be handed straight back
|
|
||||||
to `bibfetch` for everything the registers say about that edition; `olid` and
|
|
||||||
`googlebooks` are names no lookup here answers about.
|
|
||||||
|
|
||||||
`cover_id` is the same thing it is in a lookup record: the first cover in
|
|
||||||
Open Library's record of the edition the record names, by that edition's ISBN
|
|
||||||
or its key, so `bibfetch cover --cover-id` fetches that edition's cover and no
|
|
||||||
other. The cover the index picked for the whole work is never asked for,
|
|
||||||
because it is often another edition's. A work the index names no edition of
|
|
||||||
gives the first ISBN it lists that reads, which may be any edition's, and no
|
|
||||||
cover id; Google Books knows nothing of Open Library's covers and gives none.
|
|
||||||
|
|
||||||
The candidates are what the register said, in the order it said them. This
|
|
||||||
makes no judgement about which one is the book in hand, and keeps one that is
|
|
||||||
plainly another book, because a title is not an identifier: *The Hamlet* comes
|
|
||||||
back for *Hamlet*. Deciding whether a candidate is the same work is a
|
|
||||||
comparison of titles, and [`bibclean`](../bibclean/README.md)'s `same_work`
|
|
||||||
is one.
|
|
||||||
|
|
||||||
```sh
|
|
||||||
bibfetch search --title 'Anathem' --author Stephenson \
|
|
||||||
| jq -r 'select(.kind == "isbn") | .id' | head -1
|
|
||||||
```
|
|
||||||
|
|
||||||
Open Library answering that it has nothing is an answer, and it stands when
|
|
||||||
Google Books cannot be reached; a register being down is a failure only when
|
|
||||||
both are. Each is asked the same way a lookup asks, retried with the same
|
|
||||||
waits. Open Library is sent the title and the author as written; Google Books
|
|
||||||
is sent each word as its own `intitle:` or `inauthor:` term, so nothing in a
|
|
||||||
title is read as search syntax.
|
|
||||||
|
|
||||||
| Option | Effect |
|
|
||||||
|---|---|
|
|
||||||
| `--title <TITLE>` | The title, or as much of it as you have |
|
|
||||||
| `--author <AUTHOR>` | An author's name, or part of one |
|
|
||||||
| `--limit <COUNT>` | The most candidates to print, `1` to `40`. Default `10` |
|
|
||||||
|
|
||||||
Exit `0` at least one candidate was printed, `1` no register has the title,
|
|
||||||
`65` the title was empty, `69` no register could be reached, `74` the output
|
|
||||||
could not be written.
|
|
||||||
|
|
||||||
From Rust, `bibfetch::search(&agent, title, author, limit, attempts)` returns
|
|
||||||
the candidates, empty when no register has the title.
|
|
||||||
|
|
||||||
## Reaching the network
|
## Reaching the network
|
||||||
|
|
||||||
This and [`emb`](../emb/README.md) are the only two tools here that do. Both
|
This and [`emb`](../emb/README.md) are the only two tools here that do. Both
|
||||||
@@ -229,11 +156,10 @@ something that happens on the way to something else — which is the condition
|
|||||||
under which [the contract](../../docs/contract.md) allows it at all.
|
under which [the contract](../../docs/contract.md) allows it at all.
|
||||||
|
|
||||||
Nothing is cached, nothing is stored but a cover you asked to have written,
|
Nothing is cached, nothing is stored but a cover you asked to have written,
|
||||||
and nothing is sent but the identifier, or the title and author searched for,
|
and nothing is sent but the identifier and the address you pass to
|
||||||
and the address you pass to `--polite`.
|
`--polite`.
|
||||||
|
|
||||||
## See also
|
## See also
|
||||||
|
|
||||||
- [`idnorm`](../idnorm/README.md) — reading the identifier in the first place.
|
- [`idnorm`](../idnorm/README.md) — reading the identifier in the first place.
|
||||||
- [`bibclean`](../bibclean/README.md) — the rest of a catalogue record, and
|
- [`bibclean`](../bibclean/README.md) — the rest of a catalogue record.
|
||||||
whether a candidate is the same work.
|
|
||||||
+25
-30
@@ -89,6 +89,10 @@ pub fn lookup(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Ask a register, retrying a failure that might not be one.
|
/// Ask a register, retrying a failure that might not be one.
|
||||||
|
///
|
||||||
|
/// Waits a second before the second attempt and two before the third: long
|
||||||
|
/// enough for a register that is rate-limiting, short enough that one which
|
||||||
|
/// is simply down does not hold up a pipeline.
|
||||||
fn ask(
|
fn ask(
|
||||||
agent: &ureq::Agent,
|
agent: &ureq::Agent,
|
||||||
source: Source,
|
source: Source,
|
||||||
@@ -96,48 +100,39 @@ fn ask(
|
|||||||
kind: &str,
|
kind: &str,
|
||||||
attempts: u32,
|
attempts: u32,
|
||||||
) -> Result<crate::Record, Error> {
|
) -> Result<crate::Record, Error> {
|
||||||
match get(agent, &source.url(id), attempts)? {
|
|
||||||
Some(body) => source.read(id, kind, &body),
|
|
||||||
// A register that does not have it says so with a status, and that is
|
|
||||||
// an answer rather than a failure.
|
|
||||||
None => Ok(crate::missing(id, kind, source)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The body at an address, or `None` when the register says it has no such
|
|
||||||
/// thing; a failure is asked again.
|
|
||||||
///
|
|
||||||
/// Waits a second before the second attempt and two before the third: long
|
|
||||||
/// enough for a register that is rate-limiting, short enough that one which
|
|
||||||
/// is simply down does not hold up a pipeline. Reading what came back is left
|
|
||||||
/// to the caller, because an answer this cannot read will not read any better
|
|
||||||
/// next time.
|
|
||||||
pub(crate) fn get(agent: &ureq::Agent, url: &str, attempts: u32) -> Result<Option<String>, Error> {
|
|
||||||
let attempts = attempts.max(1);
|
|
||||||
let mut last = Error::Unreachable("no attempt was made".into());
|
let mut last = Error::Unreachable("no attempt was made".into());
|
||||||
for attempt in 0..attempts {
|
|
||||||
match once(agent, url) {
|
for attempt in 0..attempts.max(1) {
|
||||||
Ok(answer) => return Ok(answer),
|
match fetch(agent, source, id, kind) {
|
||||||
|
Ok(record) => return Ok(record),
|
||||||
|
// A shape this cannot read will not read any better next time.
|
||||||
|
Err(e @ Error::Shape(_)) => return Err(e),
|
||||||
Err(e) => last = e,
|
Err(e) => last = e,
|
||||||
}
|
}
|
||||||
if attempt + 1 < attempts {
|
std::thread::sleep(Duration::from_secs(1 << attempt));
|
||||||
std::thread::sleep(Duration::from_secs(1 << attempt.min(5)));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Err(last)
|
Err(last)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn once(agent: &ureq::Agent, url: &str) -> Result<Option<String>, Error> {
|
fn fetch(
|
||||||
let mut response = match agent.get(url).call() {
|
agent: &ureq::Agent,
|
||||||
|
source: Source,
|
||||||
|
id: &str,
|
||||||
|
kind: &str,
|
||||||
|
) -> Result<crate::Record, Error> {
|
||||||
|
let response = agent.get(source.url(id)).call();
|
||||||
|
let mut response = match response {
|
||||||
Ok(response) => response,
|
Ok(response) => response,
|
||||||
Err(ureq::Error::StatusCode(404)) => return Ok(None),
|
// A register that does not have it says so with a status, and that is
|
||||||
|
// an answer rather than a failure.
|
||||||
|
Err(ureq::Error::StatusCode(404)) => return Ok(crate::missing(id, kind, source)),
|
||||||
Err(e) => return Err(Error::Unreachable(e.to_string())),
|
Err(e) => return Err(Error::Unreachable(e.to_string())),
|
||||||
};
|
};
|
||||||
response
|
let body = response
|
||||||
.body_mut()
|
.body_mut()
|
||||||
.read_to_string()
|
.read_to_string()
|
||||||
.map(Some)
|
.map_err(|e| Error::Unreachable(e.to_string()))?;
|
||||||
.map_err(|e| Error::Unreachable(e.to_string()))
|
source.read(id, kind, &body)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Who is asking.
|
/// Who is asking.
|
||||||
|
|||||||
@@ -12,14 +12,9 @@
|
|||||||
//! them: Crossref for a DOI, the arXiv for a preprint, OpenLibrary for a
|
//! them: Crossref for a DOI, the arXiv for a preprint, OpenLibrary for a
|
||||||
//! book. There is no register for an ISSN that answers without a key, so an
|
//! book. There is no register for an ISSN that answers without a key, so an
|
||||||
//! ISSN is looked up as the journal of a work rather than on its own.
|
//! ISSN is looked up as the journal of a work rather than on its own.
|
||||||
//!
|
|
||||||
//! A book with no identifier at all can still be looked for by its title,
|
|
||||||
//! with [`search()`]: Open Library's index, then Google Books, each candidate
|
|
||||||
//! a record of one edition, in the order the register gave them.
|
|
||||||
|
|
||||||
pub mod ask;
|
pub mod ask;
|
||||||
pub mod cover;
|
pub mod cover;
|
||||||
pub mod search;
|
|
||||||
pub mod source;
|
pub mod source;
|
||||||
|
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
@@ -28,7 +23,6 @@ use schemars::JsonSchema;
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
pub use cover::{Cover, CoverSize, cover};
|
pub use cover::{Cover, CoverSize, cover};
|
||||||
pub use search::search;
|
|
||||||
pub use source::Source;
|
pub use source::Source;
|
||||||
|
|
||||||
/// How long to wait on a register before giving up.
|
/// How long to wait on a register before giving up.
|
||||||
|
|||||||
@@ -30,8 +30,7 @@ Example:
|
|||||||
|
|
||||||
idnorm refs.txt | bibfetch | jq -r 'select(.found) | .title'
|
idnorm refs.txt | bibfetch | jq -r 'select(.found) | .title'
|
||||||
idnorm --kind doi refs.txt | bibfetch --polite you@example.org
|
idnorm --kind doi refs.txt | bibfetch --polite you@example.org
|
||||||
bibfetch cover 978-0-306-40615-7 -o cover.jpg
|
bibfetch cover 978-0-306-40615-7 -o cover.jpg"
|
||||||
bibfetch search --title 'The First World War' --author Howard"
|
|
||||||
)]
|
)]
|
||||||
struct Args {
|
struct Args {
|
||||||
#[command(subcommand)]
|
#[command(subcommand)]
|
||||||
@@ -75,26 +74,6 @@ enum Command {
|
|||||||
#[arg(long, value_name = "ID")]
|
#[arg(long, value_name = "ID")]
|
||||||
cover_id: Option<u64>,
|
cover_id: Option<u64>,
|
||||||
},
|
},
|
||||||
/// Find the books a register has under a title, and an author.
|
|
||||||
///
|
|
||||||
/// Prints one record per candidate edition, in the order the register
|
|
||||||
/// gave them: Open Library's index, or Google Books when that has
|
|
||||||
/// nothing. Which one is the book in hand is left to you.
|
|
||||||
///
|
|
||||||
/// Exit codes: 0 at least one candidate, 1 none, 65 no title,
|
|
||||||
/// 69 no register could be reached, 74 the output could not be written.
|
|
||||||
Search {
|
|
||||||
/// The title, or as much of it as you have.
|
|
||||||
#[arg(long)]
|
|
||||||
title: String,
|
|
||||||
/// An author's name, or part of one.
|
|
||||||
#[arg(long)]
|
|
||||||
author: Option<String>,
|
|
||||||
/// The most candidates to print.
|
|
||||||
#[arg(long, default_value_t = 10, value_name = "COUNT",
|
|
||||||
value_parser = clap::value_parser!(u16).range(1..=bibfetch::search::MAX_LIMIT as i64))]
|
|
||||||
limit: u16,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, clap::ValueEnum)]
|
#[derive(Clone, Copy, clap::ValueEnum)]
|
||||||
@@ -117,27 +96,14 @@ fn main() -> ExitCode {
|
|||||||
}
|
}
|
||||||
.agent();
|
.agent();
|
||||||
|
|
||||||
match &args.command {
|
if let Some(Command::Cover {
|
||||||
Some(Command::Cover {
|
isbn,
|
||||||
isbn,
|
output,
|
||||||
output,
|
size,
|
||||||
size,
|
cover_id,
|
||||||
cover_id,
|
}) = &args.command
|
||||||
}) => return cover(&agent, args.attempts, isbn, output, *size, *cover_id),
|
{
|
||||||
Some(Command::Search {
|
return cover(&agent, args.attempts, isbn, output, *size, *cover_id);
|
||||||
title,
|
|
||||||
author,
|
|
||||||
limit,
|
|
||||||
}) => {
|
|
||||||
return search(
|
|
||||||
&agent,
|
|
||||||
args.attempts,
|
|
||||||
title,
|
|
||||||
author.as_deref(),
|
|
||||||
usize::from(*limit),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
None => {}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let text = match catena_cli::read_text(args.path.as_deref()) {
|
let text = match catena_cli::read_text(args.path.as_deref()) {
|
||||||
@@ -231,36 +197,6 @@ fn cover(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Search by title and print the candidates.
|
|
||||||
fn search(
|
|
||||||
agent: &ureq::Agent,
|
|
||||||
attempts: u32,
|
|
||||||
title: &str,
|
|
||||||
author: Option<&str>,
|
|
||||||
limit: usize,
|
|
||||||
) -> ExitCode {
|
|
||||||
match bibfetch::search(agent, title, author, limit, attempts) {
|
|
||||||
Ok(found) => {
|
|
||||||
let wrote = catena_cli::stream("bibfetch", &found);
|
|
||||||
if format!("{wrote:?}") != format!("{:?}", ExitCode::from(exit::OK)) {
|
|
||||||
return wrote;
|
|
||||||
}
|
|
||||||
if found.is_empty() {
|
|
||||||
eprintln!("bibfetch: no register has {title:?}");
|
|
||||||
return ExitCode::from(exit::NO);
|
|
||||||
}
|
|
||||||
ExitCode::from(exit::OK)
|
|
||||||
}
|
|
||||||
Err(why) => {
|
|
||||||
eprintln!("bibfetch: {why}");
|
|
||||||
ExitCode::from(match why {
|
|
||||||
bibfetch::Error::Invalid(_) => exit::DATA,
|
|
||||||
_ => exit::UNAVAILABLE,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The identifier and its kind out of one record.
|
/// The identifier and its kind out of one record.
|
||||||
fn read_identifier(line: &str) -> Result<(String, String), String> {
|
fn read_identifier(line: &str) -> Result<(String, String), String> {
|
||||||
let value: serde_json::Value =
|
let value: serde_json::Value =
|
||||||
|
|||||||
@@ -1,780 +0,0 @@
|
|||||||
//! Books by their title, for when there is no identifier to ask about.
|
|
||||||
//!
|
|
||||||
//! A book that arrives as a file often carries no ISBN at all, only a title
|
|
||||||
//! and perhaps a name. A register can still be asked which books it has under
|
|
||||||
//! those, and what it answers is a list of candidates: each an edition the
|
|
||||||
//! register holds, with whatever identifier, publisher and cover it gives.
|
|
||||||
//!
|
|
||||||
//! Which candidate is the book in hand is not decided here. The candidates
|
|
||||||
//! are what the register said, in the order it said them; deciding whether
|
|
||||||
//! one is the same work is a comparison of titles, and that belongs to
|
|
||||||
//! whoever holds the book.
|
|
||||||
|
|
||||||
use serde::Deserialize;
|
|
||||||
|
|
||||||
use crate::source::{self, GoogleIdentifier, GoogleReply, SUBJECT_LIMIT, Source};
|
|
||||||
use crate::{Error, Record};
|
|
||||||
|
|
||||||
/// The most candidates one search asks for. Google Books answers with no more
|
|
||||||
/// than forty at a time, and a title that needs more than forty candidates to
|
|
||||||
/// find the right one needs a better title.
|
|
||||||
pub const MAX_LIMIT: usize = 40;
|
|
||||||
|
|
||||||
/// What Open Library's index is asked to say about each work, and about the
|
|
||||||
/// one edition of it that best matches the search.
|
|
||||||
///
|
|
||||||
/// Not the work's own `cover_i` or `cover_edition_key`: the cover the index
|
|
||||||
/// chose for a work is often another edition's. The edition's `cover_i` is
|
|
||||||
/// the first cover in that edition's record, which is what a lookup by ISBN
|
|
||||||
/// gives as `cover_id`.
|
|
||||||
const OPEN_LIBRARY_FIELDS: &str = "key,title,subtitle,author_name,first_publish_year,publisher,\
|
|
||||||
isbn,language,subject,number_of_pages_median,editions,editions.key,editions.isbn,\
|
|
||||||
editions.publisher,editions.language,editions.publish_date,editions.cover_i";
|
|
||||||
|
|
||||||
/// Ask which books a register has under a title, and by an author if one is
|
|
||||||
/// named.
|
|
||||||
///
|
|
||||||
/// Open Library's index first, and Google Books only when that answered with
|
|
||||||
/// nothing. A register being down is a failure only when every register
|
|
||||||
/// asked is: Open Library answering that it has nothing is an answer, and it
|
|
||||||
/// stands when Google Books cannot be reached. Asks nothing when `limit` is
|
|
||||||
/// zero; a `limit` over [`MAX_LIMIT`] is taken as that.
|
|
||||||
pub fn search(
|
|
||||||
agent: &ureq::Agent,
|
|
||||||
title: &str,
|
|
||||||
author: Option<&str>,
|
|
||||||
limit: usize,
|
|
||||||
attempts: u32,
|
|
||||||
) -> Result<Vec<Record>, Error> {
|
|
||||||
search_with(title, author, limit, |url| {
|
|
||||||
crate::ask::get(agent, url, attempts)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// [`search`], with how to fetch an address passed in.
|
|
||||||
fn search_with(
|
|
||||||
title: &str,
|
|
||||||
author: Option<&str>,
|
|
||||||
limit: usize,
|
|
||||||
mut get: impl FnMut(&str) -> Result<Option<String>, Error>,
|
|
||||||
) -> Result<Vec<Record>, Error> {
|
|
||||||
let title = title.trim();
|
|
||||||
if title.is_empty() {
|
|
||||||
return Err(Error::Invalid("no title to search for".into()));
|
|
||||||
}
|
|
||||||
let author = author.map(str::trim).filter(|author| !author.is_empty());
|
|
||||||
let limit = limit.min(MAX_LIMIT);
|
|
||||||
if limit == 0 {
|
|
||||||
return Ok(Vec::new());
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut fetch = |url: String| match get(&url)? {
|
|
||||||
Some(body) => Ok(body),
|
|
||||||
// A search that matches nothing answers with an empty list. A 404 is
|
|
||||||
// the address being wrong, not the book being absent.
|
|
||||||
None => Err(Error::Unreachable(format!("{url}: answered 404"))),
|
|
||||||
};
|
|
||||||
|
|
||||||
let open_library =
|
|
||||||
fetch(open_library_url(title, author, limit)).and_then(|body| open_library(&body, limit));
|
|
||||||
if open_library.as_ref().is_ok_and(|found| !found.is_empty()) {
|
|
||||||
return open_library;
|
|
||||||
}
|
|
||||||
|
|
||||||
let google = match google_url(title, author, limit) {
|
|
||||||
Some(url) => fetch(url).and_then(|body| google_books(&body, limit)),
|
|
||||||
// Nothing in the title Google could be asked by.
|
|
||||||
None => Ok(Vec::new()),
|
|
||||||
};
|
|
||||||
match (open_library, google) {
|
|
||||||
(_, Ok(found)) if !found.is_empty() => Ok(found),
|
|
||||||
(Ok(none), _) => Ok(none),
|
|
||||||
(Err(why), _) => Err(why),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Where to ask Open Library.
|
|
||||||
fn open_library_url(title: &str, author: Option<&str>, limit: usize) -> String {
|
|
||||||
let author = author
|
|
||||||
.map(|author| format!("&author={}", source::encode(author)))
|
|
||||||
.unwrap_or_default();
|
|
||||||
format!(
|
|
||||||
"https://openlibrary.org/search.json?title={}{author}&fields={OPEN_LIBRARY_FIELDS}&limit={limit}",
|
|
||||||
source::encode(title),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Where to ask Google Books, or `None` when there is nothing to ask by.
|
|
||||||
///
|
|
||||||
/// Every word is its own `intitle:` or `inauthor:` term, so no character of
|
|
||||||
/// the title can open a phrase, negate a term or name another field: a word is
|
|
||||||
/// the letters, digits and apostrophes in it, and anything else separates
|
|
||||||
/// words.
|
|
||||||
fn google_url(title: &str, author: Option<&str>, limit: usize) -> Option<String> {
|
|
||||||
let terms: Vec<String> = words(title)
|
|
||||||
.map(|word| format!("intitle:{word}"))
|
|
||||||
.chain(
|
|
||||||
author
|
|
||||||
.into_iter()
|
|
||||||
.flat_map(words)
|
|
||||||
.map(|word| format!("inauthor:{word}")),
|
|
||||||
)
|
|
||||||
.collect();
|
|
||||||
if !terms.iter().any(|term| term.starts_with("intitle:")) {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
Some(format!(
|
|
||||||
"https://www.googleapis.com/books/v1/volumes?q={}&maxResults={limit}",
|
|
||||||
source::encode(&terms.join(" "))
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn words(text: &str) -> impl Iterator<Item = &str> {
|
|
||||||
text.split(|c: char| !(c.is_alphanumeric() || c == '\''))
|
|
||||||
.map(|word| word.trim_matches('\''))
|
|
||||||
.filter(|word| !word.is_empty())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The first ISBN in a list whose check digit agrees, in its thirteen-digit
|
|
||||||
/// form. One written in thirteen digits is taken over one written in ten.
|
|
||||||
fn first_isbn<'a>(isbns: impl Iterator<Item = &'a str> + Clone) -> Option<String> {
|
|
||||||
isbns
|
|
||||||
.clone()
|
|
||||||
.filter(|isbn| idnorm::isbn::digits(isbn).len() == 13)
|
|
||||||
.find_map(idnorm::isbn::to_13)
|
|
||||||
.or_else(|| isbns.into_iter().find_map(idnorm::isbn::to_13))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
|
||||||
struct OpenLibraryReply {
|
|
||||||
#[serde(default)]
|
|
||||||
docs: Vec<Work>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
|
||||||
struct Work {
|
|
||||||
#[serde(default)]
|
|
||||||
key: Option<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
title: Option<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
subtitle: Option<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
author_name: Vec<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
first_publish_year: Option<i64>,
|
|
||||||
#[serde(default)]
|
|
||||||
publisher: Vec<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
isbn: Vec<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
language: Vec<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
subject: Vec<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
number_of_pages_median: Option<u32>,
|
|
||||||
#[serde(default)]
|
|
||||||
editions: Option<Editions>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
|
||||||
struct Editions {
|
|
||||||
#[serde(default)]
|
|
||||||
docs: Vec<Edition>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
|
||||||
struct Edition {
|
|
||||||
#[serde(default)]
|
|
||||||
key: Option<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
isbn: Vec<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
publisher: Vec<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
language: Vec<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
publish_date: Vec<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
cover_i: Option<i64>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The last segment of an Open Library key: `OL3585423M` out of
|
|
||||||
/// `/books/OL3585423M`. The letter at the end says what it names, `M` an
|
|
||||||
/// edition and `W` a work.
|
|
||||||
fn olid(key: &str) -> Option<String> {
|
|
||||||
key.rsplit('/')
|
|
||||||
.next()
|
|
||||||
.filter(|id| !id.is_empty())
|
|
||||||
.map(str::to_string)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Read Open Library's answer into one candidate per work, in its order.
|
|
||||||
///
|
|
||||||
/// What belongs to an edition — its ISBN, publisher, language, year and
|
|
||||||
/// cover — is taken from the edition the index chose as matching best; what
|
|
||||||
/// belongs to the work — its title, authors, subjects and length — from the
|
|
||||||
/// work. A work the index names no edition of gives its first ISBN that reads,
|
|
||||||
/// which may be any edition's, and no cover id.
|
|
||||||
fn open_library(body: &str, limit: usize) -> Result<Vec<Record>, Error> {
|
|
||||||
let reply: OpenLibraryReply =
|
|
||||||
serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?;
|
|
||||||
Ok(reply
|
|
||||||
.docs
|
|
||||||
.into_iter()
|
|
||||||
.filter_map(candidate)
|
|
||||||
.take(limit)
|
|
||||||
.collect())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn candidate(work: Work) -> Option<Record> {
|
|
||||||
let edition = work.editions.and_then(|e| e.docs.into_iter().next());
|
|
||||||
|
|
||||||
// The cover is kept only when the record names the edition it is the
|
|
||||||
// cover of, by that edition's ISBN or its key.
|
|
||||||
let (id, kind, edition_named) = match &edition {
|
|
||||||
Some(edition) => match first_isbn(edition.isbn.iter().map(String::as_str)) {
|
|
||||||
Some(isbn) => (isbn, "isbn", true),
|
|
||||||
None => match edition.key.as_deref().and_then(olid) {
|
|
||||||
Some(key) => (key, "olid", true),
|
|
||||||
None => (work.key.as_deref().and_then(olid)?, "olid", false),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
None => match first_isbn(work.isbn.iter().map(String::as_str)) {
|
|
||||||
Some(isbn) => (isbn, "isbn", false),
|
|
||||||
// A work with no key cannot be named, so it is not a candidate.
|
|
||||||
None => (work.key.as_deref().and_then(olid)?, "olid", false),
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut record = source::blank(&id, kind, Source::OpenLibrarySearch);
|
|
||||||
record.title = match (work.title, work.subtitle) {
|
|
||||||
(Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")),
|
|
||||||
(title, _) => title,
|
|
||||||
};
|
|
||||||
record.authors = work.author_name;
|
|
||||||
record.subjects = work.subject;
|
|
||||||
record.subjects.truncate(SUBJECT_LIMIT);
|
|
||||||
record.pages = work.number_of_pages_median;
|
|
||||||
|
|
||||||
let (publisher, language, year) = match edition {
|
|
||||||
Some(edition) => {
|
|
||||||
if edition_named {
|
|
||||||
record.cover_id = edition
|
|
||||||
.cover_i
|
|
||||||
.and_then(|id| u64::try_from(id).ok())
|
|
||||||
.filter(|id| *id > 0);
|
|
||||||
}
|
|
||||||
(
|
|
||||||
edition.publisher.into_iter().next(),
|
|
||||||
edition.language.into_iter().next(),
|
|
||||||
edition
|
|
||||||
.publish_date
|
|
||||||
.iter()
|
|
||||||
.find_map(|date| source::year_in(date)),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
None => (None, None, None),
|
|
||||||
};
|
|
||||||
record.container = publisher.or_else(|| work.publisher.into_iter().next());
|
|
||||||
record.language = language.or_else(|| work.language.into_iter().next());
|
|
||||||
record.year = year.or(work.first_publish_year);
|
|
||||||
Some(record)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Read Google Books' answer into one candidate per volume, in its order.
|
|
||||||
fn google_books(body: &str, limit: usize) -> Result<Vec<Record>, Error> {
|
|
||||||
let reply: GoogleReply = serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?;
|
|
||||||
Ok(reply
|
|
||||||
.items
|
|
||||||
.into_iter()
|
|
||||||
.filter_map(|item| {
|
|
||||||
// Only what Google types as an ISBN: its `OTHER` identifiers are
|
|
||||||
// library shelfmarks, some of which are ten digits long.
|
|
||||||
let isbns = item
|
|
||||||
.volume_info
|
|
||||||
.identifiers
|
|
||||||
.iter()
|
|
||||||
.filter(|GoogleIdentifier { kind, .. }| kind == "ISBN_13" || kind == "ISBN_10")
|
|
||||||
.map(|identifier| identifier.identifier.as_str());
|
|
||||||
let (id, kind) = match first_isbn(isbns) {
|
|
||||||
Some(isbn) => (isbn, "isbn"),
|
|
||||||
None => (item.id.filter(|id| !id.is_empty())?, "googlebooks"),
|
|
||||||
};
|
|
||||||
Some(source::google_volume(&id, kind, item.volume_info))
|
|
||||||
})
|
|
||||||
.take(limit)
|
|
||||||
.collect())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
/// Open Library's answer to `title=the first world war&author=howard`,
|
|
||||||
/// recorded 2026-11-13 with the fields above and `limit=2`, the ISBN and
|
|
||||||
/// subject lists shortened.
|
|
||||||
const HOWARD: &str = r#"{"numFound": 9, "start": 0, "numFoundExact": true, "num_found": 9,
|
|
||||||
"documentation_url": "https://openlibrary.org/dev/docs/api/search", "q": "", "offset": null,
|
|
||||||
"docs": [
|
|
||||||
{"author_name": ["Michael Eliot Howard"], "first_publish_year": 2002,
|
|
||||||
"isbn": ["9780192853622", "0192853627"], "key": "/works/OL1382726W",
|
|
||||||
"language": ["eng"], "number_of_pages_median": 154,
|
|
||||||
"publisher": ["Oxford University Press"], "title": "The First World War",
|
|
||||||
"subject": ["World War, 1914-1918"],
|
|
||||||
"editions": {"numFound": 1, "start": 0, "numFoundExact": true, "docs": [
|
|
||||||
{"key": "/books/OL3585423M", "cover_i": 1123124, "language": ["eng"],
|
|
||||||
"publisher": ["Oxford University Press"], "publish_date": ["2002"],
|
|
||||||
"isbn": ["0192853627", "9780192853622"]}]}},
|
|
||||||
{"author_name": ["Michael Howard"], "first_publish_year": 2007,
|
|
||||||
"isbn": ["0199205590", "9780199205592"], "key": "/works/OL4282399W",
|
|
||||||
"language": ["eng"], "publisher": ["Oxford University Press"],
|
|
||||||
"subtitle": "A Very Short Introduction", "title": "The First World War",
|
|
||||||
"editions": {"numFound": 1, "start": 0, "numFoundExact": true, "docs": [
|
|
||||||
{"key": "/books/OL7403004M", "cover_i": 14606475, "language": ["eng"],
|
|
||||||
"publisher": ["Oxford University Press"], "publish_date": ["20 May 2007"],
|
|
||||||
"isbn": ["9780199205592", "0199205590"]}]},
|
|
||||||
"subject": []}]}"#;
|
|
||||||
|
|
||||||
/// Open Library's answer to `title=war and peace&author=tolstoy`, recorded
|
|
||||||
/// 2026-11-13, first work only. The edition the index matched best is a
|
|
||||||
/// 1917 printing that has no ISBN, though other editions of the work do.
|
|
||||||
const TOLSTOY: &str = r#"{"numFound": 146, "docs": [
|
|
||||||
{"author_name": ["Leo Tolstoy"], "first_publish_year": 1867,
|
|
||||||
"isbn": ["9781524881993", "1645178889"], "key": "/works/OL267171W",
|
|
||||||
"title": "War and Peace",
|
|
||||||
"editions": {"numFound": 487, "docs": [
|
|
||||||
{"key": "/books/OL37863121M", "cover_i": 12736340, "language": ["eng"],
|
|
||||||
"publisher": ["Thomas Y. Crowell"], "publish_date": ["1917"]}]}}]}"#;
|
|
||||||
|
|
||||||
/// Open Library's answer when nothing has the title.
|
|
||||||
const NOTHING: &str = r#"{"numFound": 0, "start": 0, "numFoundExact": true, "num_found": 0,
|
|
||||||
"documentation_url": "https://openlibrary.org/dev/docs/api/search", "q": "",
|
|
||||||
"offset": null, "docs": []}"#;
|
|
||||||
|
|
||||||
/// Google Books' answer for a volume, in the shape its API documents.
|
|
||||||
const GOOGLE: &str = r#"{"kind": "books#volumes", "totalItems": 2, "items": [
|
|
||||||
{"kind": "books#volume", "id": "zyTCAlFPjgYC", "volumeInfo": {
|
|
||||||
"title": "The Google Story", "authors": ["David A. Vise", "Mark Malseed"],
|
|
||||||
"publisher": "Random House Digital, Inc.", "publishedDate": "2005-11-15",
|
|
||||||
"industryIdentifiers": [
|
|
||||||
{"type": "ISBN_10", "identifier": "055380457X"},
|
|
||||||
{"type": "ISBN_13", "identifier": "9780553804577"}],
|
|
||||||
"pageCount": 207, "categories": ["Browsers (Computer programs)"],
|
|
||||||
"language": "en"}},
|
|
||||||
{"kind": "books#volume", "id": "a1b2C3d4", "volumeInfo": {
|
|
||||||
"title": "The Google Story", "subtitle": "A Dissertation",
|
|
||||||
"publishedDate": "2009",
|
|
||||||
"industryIdentifiers": [
|
|
||||||
{"type": "OTHER", "identifier": "UOM:0306406152"}]}}]}"#;
|
|
||||||
|
|
||||||
/// Fetch from a table of recorded bodies, keeping every address asked.
|
|
||||||
fn recorded<'a>(
|
|
||||||
asked: &'a mut Vec<String>,
|
|
||||||
bodies: &'a [(&'a str, Result<&'a str, Error>)],
|
|
||||||
) -> impl FnMut(&str) -> Result<Option<String>, Error> + 'a {
|
|
||||||
move |url| {
|
|
||||||
asked.push(url.to_string());
|
|
||||||
let (_, body) = bodies
|
|
||||||
.iter()
|
|
||||||
.find(|(host, _)| url.contains(host))
|
|
||||||
.unwrap_or_else(|| panic!("asked an address nothing was recorded for: {url}"));
|
|
||||||
body.clone().map(|body| Some(body.to_string()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn down() -> Error {
|
|
||||||
Error::Unreachable("connection refused".into())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn each_work_open_library_has_is_a_candidate_named_by_its_isbn() {
|
|
||||||
let mut asked = Vec::new();
|
|
||||||
let bodies = [("openlibrary.org", Ok(HOWARD))];
|
|
||||||
let found = search_with(
|
|
||||||
"The First World War",
|
|
||||||
Some("Howard"),
|
|
||||||
5,
|
|
||||||
recorded(&mut asked, &bodies),
|
|
||||||
)
|
|
||||||
.expect("candidates");
|
|
||||||
assert_eq!(
|
|
||||||
asked.len(),
|
|
||||||
1,
|
|
||||||
"Google is not asked when Open Library answers"
|
|
||||||
);
|
|
||||||
|
|
||||||
assert_eq!(found.len(), 2);
|
|
||||||
let first = &found[0];
|
|
||||||
assert!(first.found);
|
|
||||||
assert_eq!(first.v, corpus_core::V);
|
|
||||||
assert_eq!(first.source, "openlibrary");
|
|
||||||
// Written ten-digit first by the edition, given in thirteen.
|
|
||||||
assert_eq!(
|
|
||||||
(first.id.as_str(), first.kind.as_str()),
|
|
||||||
("9780192853622", "isbn")
|
|
||||||
);
|
|
||||||
assert_eq!(first.title.as_deref(), Some("The First World War"));
|
|
||||||
assert_eq!(first.authors, ["Michael Eliot Howard"]);
|
|
||||||
assert_eq!(first.year, Some(2002));
|
|
||||||
assert_eq!(first.container.as_deref(), Some("Oxford University Press"));
|
|
||||||
assert_eq!(first.language.as_deref(), Some("eng"));
|
|
||||||
assert_eq!(first.subjects, ["World War, 1914-1918"]);
|
|
||||||
assert_eq!(first.pages, Some(154));
|
|
||||||
assert_eq!(first.cover_id, Some(1123124));
|
|
||||||
|
|
||||||
let second = &found[1];
|
|
||||||
assert_eq!(second.id, "9780199205592");
|
|
||||||
assert_eq!(
|
|
||||||
second.title.as_deref(),
|
|
||||||
Some("The First World War: A Very Short Introduction")
|
|
||||||
);
|
|
||||||
// The edition's year, not the work's first.
|
|
||||||
assert_eq!(second.year, Some(2007));
|
|
||||||
// The first cover in OL7403004M's own record; the work's cover is
|
|
||||||
// 1137779, which is not.
|
|
||||||
assert_eq!(second.cover_id, Some(14606475));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn the_candidates_stay_in_the_order_the_register_gave_them() {
|
|
||||||
let mut asked = Vec::new();
|
|
||||||
let bodies = [("openlibrary.org", Ok(HOWARD))];
|
|
||||||
let found =
|
|
||||||
search_with("first world war", None, 5, recorded(&mut asked, &bodies)).expect("ok");
|
|
||||||
let ids: Vec<&str> = found.iter().map(|r| r.id.as_str()).collect();
|
|
||||||
assert_eq!(ids, ["9780192853622", "9780199205592"]);
|
|
||||||
let found =
|
|
||||||
search_with("first world war", None, 1, recorded(&mut asked, &bodies)).expect("ok");
|
|
||||||
assert_eq!(found.len(), 1);
|
|
||||||
assert_eq!(found[0].id, "9780192853622");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn an_edition_with_no_isbn_is_named_by_its_key_and_keeps_its_cover() {
|
|
||||||
let mut asked = Vec::new();
|
|
||||||
let bodies = [("openlibrary.org", Ok(TOLSTOY))];
|
|
||||||
let found = search_with(
|
|
||||||
"War and Peace",
|
|
||||||
Some("Tolstoy"),
|
|
||||||
5,
|
|
||||||
recorded(&mut asked, &bodies),
|
|
||||||
)
|
|
||||||
.expect("candidates");
|
|
||||||
let only = &found[0];
|
|
||||||
// Not 9781524881993: that is another edition's, and the publisher,
|
|
||||||
// year and cover here are this one's.
|
|
||||||
assert_eq!(
|
|
||||||
(only.id.as_str(), only.kind.as_str()),
|
|
||||||
("OL37863121M", "olid")
|
|
||||||
);
|
|
||||||
assert_eq!(only.container.as_deref(), Some("Thomas Y. Crowell"));
|
|
||||||
assert_eq!(only.year, Some(1917));
|
|
||||||
assert_eq!(only.cover_id, Some(12736340));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_work_with_no_edition_named_gives_its_isbn_but_no_cover() {
|
|
||||||
let body = r#"{"docs": [{"key": "/works/OL1W", "title": "A Book",
|
|
||||||
"isbn": ["0-306-40615-2"], "publisher": ["Plenum"], "first_publish_year": 1981}]}"#;
|
|
||||||
let found = open_library(body, 5).expect("candidates");
|
|
||||||
assert_eq!(
|
|
||||||
(found[0].id.as_str(), found[0].kind.as_str()),
|
|
||||||
("9780306406157", "isbn")
|
|
||||||
);
|
|
||||||
assert_eq!(found[0].container.as_deref(), Some("Plenum"));
|
|
||||||
assert_eq!(found[0].year, Some(1981));
|
|
||||||
assert_eq!(found[0].cover_id, None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_work_with_no_isbn_and_no_edition_is_named_by_the_works_key() {
|
|
||||||
let body = r#"{"docs": [{"key": "/works/OL267171W", "title": "War and Peace"}]}"#;
|
|
||||||
let found = open_library(body, 5).expect("candidates");
|
|
||||||
assert_eq!(
|
|
||||||
(found[0].id.as_str(), found[0].kind.as_str()),
|
|
||||||
("OL267171W", "olid")
|
|
||||||
);
|
|
||||||
assert_eq!(found[0].cover_id, None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn isbns_whose_check_digit_is_wrong_are_passed_over() {
|
|
||||||
// 9780306406158 and 0306406153 are 9780306406157 mistyped.
|
|
||||||
let body = r#"{"docs": [
|
|
||||||
{"key": "/works/OL1W", "title": "A", "editions": {"docs": [
|
|
||||||
{"key": "/books/OL1M", "isbn": ["9780306406158", "0306406153"], "cover_i": 7}]}},
|
|
||||||
{"key": "/works/OL2W", "title": "B", "editions": {"docs": [
|
|
||||||
{"key": "/books/OL2M", "isbn": ["9780306406158", "0306406152"]}]}}]}"#;
|
|
||||||
let found = open_library(body, 5).expect("candidates");
|
|
||||||
assert_eq!(
|
|
||||||
(found[0].id.as_str(), found[0].kind.as_str()),
|
|
||||||
("OL1M", "olid")
|
|
||||||
);
|
|
||||||
assert_eq!(found[0].cover_id, Some(7));
|
|
||||||
// The ten-digit one reads, and is given in thirteen.
|
|
||||||
assert_eq!(
|
|
||||||
(found[1].id.as_str(), found[1].kind.as_str()),
|
|
||||||
("9780306406157", "isbn")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn thirteen_digits_are_taken_over_ten() {
|
|
||||||
// Two different ISBNs on one edition: the thirteen-digit one wins
|
|
||||||
// although the ten-digit one is listed first.
|
|
||||||
let isbns = ["0306406152", "9780199205592"];
|
|
||||||
assert_eq!(
|
|
||||||
first_isbn(isbns.into_iter()).as_deref(),
|
|
||||||
Some("9780199205592")
|
|
||||||
);
|
|
||||||
assert_eq!(first_isbn(["nonsense"].into_iter()), None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn nothing_from_open_library_is_asked_of_google_books() {
|
|
||||||
let mut asked = Vec::new();
|
|
||||||
let bodies = [
|
|
||||||
("openlibrary.org", Ok(NOTHING)),
|
|
||||||
("googleapis.com", Ok(GOOGLE)),
|
|
||||||
];
|
|
||||||
let found = search_with(
|
|
||||||
"The Google Story",
|
|
||||||
Some("Vise"),
|
|
||||||
5,
|
|
||||||
recorded(&mut asked, &bodies),
|
|
||||||
)
|
|
||||||
.expect("candidates");
|
|
||||||
assert_eq!(asked.len(), 2);
|
|
||||||
assert!(asked[1].starts_with("https://www.googleapis.com/books/v1/volumes?q="));
|
|
||||||
|
|
||||||
assert_eq!(found.len(), 2);
|
|
||||||
let first = &found[0];
|
|
||||||
assert_eq!(first.source, "googlebooks");
|
|
||||||
assert_eq!(
|
|
||||||
(first.id.as_str(), first.kind.as_str()),
|
|
||||||
("9780553804577", "isbn")
|
|
||||||
);
|
|
||||||
assert_eq!(first.authors, ["David A. Vise", "Mark Malseed"]);
|
|
||||||
assert_eq!(first.year, Some(2005));
|
|
||||||
assert_eq!(
|
|
||||||
first.container.as_deref(),
|
|
||||||
Some("Random House Digital, Inc.")
|
|
||||||
);
|
|
||||||
assert_eq!(first.pages, Some(207));
|
|
||||||
// Google Books knows nothing of Open Library's covers.
|
|
||||||
assert_eq!(first.cover_id, None);
|
|
||||||
|
|
||||||
// A shelfmark is not an ISBN, though its digits would pass for one.
|
|
||||||
let second = &found[1];
|
|
||||||
assert_eq!(
|
|
||||||
(second.id.as_str(), second.kind.as_str()),
|
|
||||||
("a1b2C3d4", "googlebooks")
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
second.title.as_deref(),
|
|
||||||
Some("The Google Story: A Dissertation")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn nothing_from_either_register_is_no_candidates() {
|
|
||||||
let mut asked = Vec::new();
|
|
||||||
let bodies = [
|
|
||||||
("openlibrary.org", Ok(NOTHING)),
|
|
||||||
(
|
|
||||||
"googleapis.com",
|
|
||||||
Ok(r#"{"kind": "books#volumes", "totalItems": 0}"#),
|
|
||||||
),
|
|
||||||
];
|
|
||||||
let found =
|
|
||||||
search_with("Nothing", None, 5, recorded(&mut asked, &bodies)).expect("an answer");
|
|
||||||
assert!(found.is_empty());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn open_library_having_nothing_stands_when_google_books_is_down() {
|
|
||||||
let mut asked = Vec::new();
|
|
||||||
let bodies = [
|
|
||||||
("openlibrary.org", Ok(NOTHING)),
|
|
||||||
("googleapis.com", Err(down())),
|
|
||||||
];
|
|
||||||
let found =
|
|
||||||
search_with("Nothing", None, 5, recorded(&mut asked, &bodies)).expect("an answer");
|
|
||||||
assert!(found.is_empty());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn open_library_down_is_asked_of_google_books() {
|
|
||||||
let mut asked = Vec::new();
|
|
||||||
let bodies = [
|
|
||||||
("openlibrary.org", Err(down())),
|
|
||||||
("googleapis.com", Ok(GOOGLE)),
|
|
||||||
];
|
|
||||||
let found =
|
|
||||||
search_with("The Google Story", None, 5, recorded(&mut asked, &bodies)).expect("ok");
|
|
||||||
assert_eq!(found[0].source, "googlebooks");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn every_register_down_is_a_failure() {
|
|
||||||
let mut asked = Vec::new();
|
|
||||||
let bodies = [
|
|
||||||
("openlibrary.org", Err(down())),
|
|
||||||
("googleapis.com", Err(down())),
|
|
||||||
];
|
|
||||||
let failed = search_with("A Book", None, 5, recorded(&mut asked, &bodies));
|
|
||||||
assert!(matches!(failed, Err(Error::Unreachable(_))), "{failed:?}");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn an_answer_that_cannot_be_read_says_so() {
|
|
||||||
assert!(matches!(open_library("<html>", 5), Err(Error::Shape(_))));
|
|
||||||
assert!(matches!(google_books("<html>", 5), Err(Error::Shape(_))));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn the_author_is_left_out_when_there_is_none() {
|
|
||||||
for author in [None, Some(""), Some(" ")] {
|
|
||||||
let url = open_library_url(
|
|
||||||
"Anathem",
|
|
||||||
author.map(str::trim).filter(|a| !a.is_empty()),
|
|
||||||
5,
|
|
||||||
);
|
|
||||||
assert!(!url.contains("author="), "{url}");
|
|
||||||
}
|
|
||||||
let mut asked = Vec::new();
|
|
||||||
let bodies = [
|
|
||||||
("openlibrary.org", Ok(NOTHING)),
|
|
||||||
("googleapis.com", Ok(GOOGLE)),
|
|
||||||
];
|
|
||||||
search_with("Anathem", Some(" "), 5, recorded(&mut asked, &bodies)).expect("ok");
|
|
||||||
assert!(!asked[0].contains("author="), "{}", asked[0]);
|
|
||||||
assert!(!asked[1].contains("inauthor"), "{}", asked[1]);
|
|
||||||
assert!(asked[1].contains("q=intitle%3AAnathem&"), "{}", asked[1]);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn the_author_narrows_the_search_when_there_is_one() {
|
|
||||||
let url = open_library_url("Anathem", Some("Neal Stephenson"), 5);
|
|
||||||
assert!(
|
|
||||||
url.contains("?title=Anathem&author=Neal%20Stephenson&fields="),
|
|
||||||
"{url}"
|
|
||||||
);
|
|
||||||
let url = google_url("Anathem", Some("Neal Stephenson"), 5).expect("a url");
|
|
||||||
assert!(
|
|
||||||
url.contains("q=intitle%3AAnathem%20inauthor%3ANeal%20inauthor%3AStephenson&"),
|
|
||||||
"{url}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_title_cannot_change_the_shape_of_the_address() {
|
|
||||||
let hostile = r#"Fish & Chips: "A History" #2?=x/y+z"#;
|
|
||||||
for url in [
|
|
||||||
open_library_url(hostile, Some("O'Brien & Sons"), 5),
|
|
||||||
google_url(hostile, Some("O'Brien & Sons"), 5).expect("a url"),
|
|
||||||
] {
|
|
||||||
assert_eq!(url.matches('?').count(), 1, "{url}");
|
|
||||||
assert_eq!(url.matches('#').count(), 0, "{url}");
|
|
||||||
assert_eq!(url.matches('"').count(), 0, "{url}");
|
|
||||||
assert_eq!(url.matches('+').count(), 0, "{url}");
|
|
||||||
let query = url.split_once('?').expect("a query").1;
|
|
||||||
let names: Vec<&str> = query
|
|
||||||
.split('&')
|
|
||||||
.map(|pair| pair.split_once('=').expect("name=value").0)
|
|
||||||
.collect();
|
|
||||||
assert!(
|
|
||||||
names == ["title", "author", "fields", "limit"] || names == ["q", "maxResults"],
|
|
||||||
"{url}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
// Open Library is asked the title as written.
|
|
||||||
let url = open_library_url(hostile, None, 5);
|
|
||||||
assert!(
|
|
||||||
url.contains("title=Fish%20%26%20Chips%3A%20%22A%20History%22%20%232%3F%3Dx%2Fy%2Bz&"),
|
|
||||||
"{url}"
|
|
||||||
);
|
|
||||||
// Google is asked word by word, so nothing in a title is syntax.
|
|
||||||
let url = google_url(hostile, Some("O'Brien & Sons"), 5).expect("a url");
|
|
||||||
let q = "intitle:Fish intitle:Chips intitle:A intitle:History intitle:2 intitle:x \
|
|
||||||
intitle:y intitle:z inauthor:O'Brien inauthor:Sons";
|
|
||||||
assert!(url.contains(&format!("q={}&", source::encode(q))), "{url}");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_title_outside_ascii_is_sent_as_utf8() {
|
|
||||||
let url = open_library_url("Преступление и наказание", Some("Dostoevský"), 5);
|
|
||||||
assert!(
|
|
||||||
url.contains("title=%D0%9F%D1%80%D0%B5%D1%81%D1%82%D1%83%D0%BF"),
|
|
||||||
"{url}"
|
|
||||||
);
|
|
||||||
assert!(url.contains("author=Dostoevsk%C3%BD&"), "{url}");
|
|
||||||
let url = google_url("Преступление и наказание", None, 5).expect("a url");
|
|
||||||
// Letters in any script are words.
|
|
||||||
assert!(url.contains("intitle%3A%D0%B8%20"), "{url}");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_title_with_no_words_is_not_asked_of_google_books() {
|
|
||||||
assert_eq!(google_url("?!", Some("Someone"), 5), None);
|
|
||||||
let mut asked = Vec::new();
|
|
||||||
let bodies = [("openlibrary.org", Ok(NOTHING))];
|
|
||||||
let found = search_with("?!", Some("Someone"), 5, recorded(&mut asked, &bodies))
|
|
||||||
.expect("an answer");
|
|
||||||
assert!(found.is_empty());
|
|
||||||
assert_eq!(asked.len(), 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn no_title_is_refused_without_asking() {
|
|
||||||
let mut asked = Vec::new();
|
|
||||||
let failed = search_with(" ", Some("Someone"), 5, recorded(&mut asked, &[]));
|
|
||||||
assert!(matches!(failed, Err(Error::Invalid(_))), "{failed:?}");
|
|
||||||
assert!(asked.is_empty());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn the_limit_is_asked_for_and_held_to() {
|
|
||||||
let mut asked = Vec::new();
|
|
||||||
let bodies = [
|
|
||||||
("openlibrary.org", Ok(NOTHING)),
|
|
||||||
("googleapis.com", Ok(GOOGLE)),
|
|
||||||
];
|
|
||||||
let found =
|
|
||||||
search_with("The Google Story", None, 1, recorded(&mut asked, &bodies)).expect("ok");
|
|
||||||
assert_eq!(found.len(), 1);
|
|
||||||
assert!(asked[0].ends_with("&limit=1"), "{}", asked[0]);
|
|
||||||
assert!(asked[1].ends_with("&maxResults=1"), "{}", asked[1]);
|
|
||||||
|
|
||||||
let mut asked = Vec::new();
|
|
||||||
search_with("The Google Story", None, 500, recorded(&mut asked, &bodies)).expect("ok");
|
|
||||||
assert!(asked[0].ends_with("&limit=40"), "{}", asked[0]);
|
|
||||||
assert!(asked[1].ends_with("&maxResults=40"), "{}", asked[1]);
|
|
||||||
|
|
||||||
let mut asked = Vec::new();
|
|
||||||
let found =
|
|
||||||
search_with("The Google Story", None, 0, recorded(&mut asked, &bodies)).expect("ok");
|
|
||||||
assert!(found.is_empty());
|
|
||||||
assert!(asked.is_empty());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn the_index_is_not_asked_for_the_works_cover() {
|
|
||||||
let url = open_library_url("A", None, 5);
|
|
||||||
let fields = url
|
|
||||||
.split('&')
|
|
||||||
.find_map(|pair| pair.strip_prefix("fields="))
|
|
||||||
.expect("fields");
|
|
||||||
let fields: Vec<&str> = fields.split(',').collect();
|
|
||||||
assert!(fields.contains(&"editions.cover_i"));
|
|
||||||
assert!(!fields.contains(&"cover_i"));
|
|
||||||
assert!(!fields.contains(&"cover_edition_key"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_404_from_a_search_is_a_failure_and_not_an_absence() {
|
|
||||||
let failed = search_with("A Book", None, 5, |_| Ok(None));
|
|
||||||
assert!(matches!(failed, Err(Error::Unreachable(_))), "{failed:?}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -117,7 +117,7 @@ number_of_pages_median&limit=1",
|
|||||||
///
|
///
|
||||||
/// A DOI suffix may hold a slash, a question mark or a hash, and every one of
|
/// A DOI suffix may hold a slash, a question mark or a hash, and every one of
|
||||||
/// those means something else in a URL.
|
/// those means something else in a URL.
|
||||||
pub(crate) fn encode(text: &str) -> String {
|
fn encode(text: &str) -> String {
|
||||||
let mut out = String::with_capacity(text.len());
|
let mut out = String::with_capacity(text.len());
|
||||||
for byte in text.bytes() {
|
for byte in text.bytes() {
|
||||||
match byte {
|
match byte {
|
||||||
@@ -131,7 +131,7 @@ pub(crate) fn encode(text: &str) -> String {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// A record with the fields nothing found fills in.
|
/// A record with the fields nothing found fills in.
|
||||||
pub(crate) fn blank(id: &str, kind: &str, source: Source) -> Record {
|
fn blank(id: &str, kind: &str, source: Source) -> Record {
|
||||||
let mut record = crate::missing(id, kind, source);
|
let mut record = crate::missing(id, kind, source);
|
||||||
record.found = true;
|
record.found = true;
|
||||||
record
|
record
|
||||||
@@ -320,7 +320,7 @@ struct SearchDoc {
|
|||||||
/// The index lists every heading every edition has ever been given, which on a
|
/// The index lists every heading every edition has ever been given, which on a
|
||||||
/// well-known book is hundreds and on Hamlet is thousands. A shelf is filed
|
/// well-known book is hundreds and on Hamlet is thousands. A shelf is filed
|
||||||
/// from a handful, and the first are the broadest.
|
/// from a handful, and the first are the broadest.
|
||||||
pub(crate) const SUBJECT_LIMIT: usize = 12;
|
const SUBJECT_LIMIT: usize = 12;
|
||||||
|
|
||||||
/// A search that matched nothing answers with no documents rather than a
|
/// A search that matched nothing answers with no documents rather than a
|
||||||
/// status, so an empty reply is the absence and not a failure.
|
/// status, so an empty reply is the absence and not a failure.
|
||||||
@@ -348,22 +348,19 @@ fn open_library_search(id: &str, kind: &str, body: &str) -> Result<Record, Error
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
pub(crate) struct GoogleReply {
|
struct GoogleReply {
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub(crate) items: Vec<GoogleItem>,
|
items: Vec<GoogleItem>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
pub(crate) struct GoogleItem {
|
struct GoogleItem {
|
||||||
/// Google's own key for the volume.
|
|
||||||
#[serde(default)]
|
|
||||||
pub(crate) id: Option<String>,
|
|
||||||
#[serde(rename = "volumeInfo", default)]
|
#[serde(rename = "volumeInfo", default)]
|
||||||
pub(crate) volume_info: GoogleVolume,
|
volume_info: GoogleVolume,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize, Default)]
|
#[derive(Deserialize, Default)]
|
||||||
pub(crate) struct GoogleVolume {
|
struct GoogleVolume {
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
title: Option<String>,
|
title: Option<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
@@ -382,18 +379,6 @@ pub(crate) struct GoogleVolume {
|
|||||||
language: Option<String>,
|
language: Option<String>,
|
||||||
#[serde(rename = "pageCount", default)]
|
#[serde(rename = "pageCount", default)]
|
||||||
page_count: Option<u32>,
|
page_count: Option<u32>,
|
||||||
/// `[{"type": "ISBN_13", "identifier": "…"}]`, alongside library
|
|
||||||
/// shelfmarks typed `OTHER`.
|
|
||||||
#[serde(rename = "industryIdentifiers", default)]
|
|
||||||
pub(crate) identifiers: Vec<GoogleIdentifier>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
|
||||||
pub(crate) struct GoogleIdentifier {
|
|
||||||
#[serde(rename = "type", default)]
|
|
||||||
pub(crate) kind: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub(crate) identifier: String,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A search that matched nothing answers with no items rather than a status,
|
/// A search that matched nothing answers with no items rather than a status,
|
||||||
@@ -403,11 +388,8 @@ fn google_books(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
|||||||
let Some(item) = reply.items.into_iter().next() else {
|
let Some(item) = reply.items.into_iter().next() else {
|
||||||
return Ok(crate::missing(id, kind, Source::GoogleBooks));
|
return Ok(crate::missing(id, kind, Source::GoogleBooks));
|
||||||
};
|
};
|
||||||
Ok(google_volume(id, kind, item.volume_info))
|
let volume = item.volume_info;
|
||||||
}
|
|
||||||
|
|
||||||
/// One Google Books volume as a record.
|
|
||||||
pub(crate) fn google_volume(id: &str, kind: &str, volume: GoogleVolume) -> Record {
|
|
||||||
let mut record = blank(id, kind, Source::GoogleBooks);
|
let mut record = blank(id, kind, Source::GoogleBooks);
|
||||||
record.title = match (volume.title, volume.subtitle) {
|
record.title = match (volume.title, volume.subtitle) {
|
||||||
(Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")),
|
(Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")),
|
||||||
@@ -420,11 +402,11 @@ pub(crate) fn google_volume(id: &str, kind: &str, volume: GoogleVolume) -> Recor
|
|||||||
record.subjects = volume.categories;
|
record.subjects = volume.categories;
|
||||||
record.language = volume.language;
|
record.language = volume.language;
|
||||||
record.pages = volume.page_count;
|
record.pages = volume.page_count;
|
||||||
record
|
Ok(record)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The first four-digit year in a string.
|
/// The first four-digit year in a string.
|
||||||
pub(crate) fn year_in(text: &str) -> Option<i64> {
|
fn year_in(text: &str) -> Option<i64> {
|
||||||
let bytes: Vec<char> = text.chars().collect();
|
let bytes: Vec<char> = text.chars().collect();
|
||||||
for window in bytes.windows(4) {
|
for window in bytes.windows(4) {
|
||||||
if window.iter().all(char::is_ascii_digit) {
|
if window.iter().all(char::is_ascii_digit) {
|
||||||
|
|||||||
+1
-1
@@ -64,7 +64,7 @@ No network, no database, no configuration file, no state kept between runs.
|
|||||||
Five exceptions, each declared in `--help`:
|
Five exceptions, each declared in `--help`:
|
||||||
|
|
||||||
- [`emb`](../crates/emb/README.md) makes an HTTP call, which is the whole of what it does.
|
- [`emb`](../crates/emb/README.md) makes an HTTP call, which is the whole of what it does.
|
||||||
- [`bibfetch`](../crates/bibfetch/README.md) asks a register about an identifier or a title, likewise.
|
- [`bibfetch`](../crates/bibfetch/README.md) asks a register about an identifier, likewise.
|
||||||
- [`pdftriage`](../crates/pdftriage/README.md) runs `pdftotext` and `pdfinfo`.
|
- [`pdftriage`](../crates/pdftriage/README.md) runs `pdftotext` and `pdfinfo`.
|
||||||
- [`pdfgutter`](../crates/pdfgutter/README.md) runs `pdftoppm` to look at a page.
|
- [`pdfgutter`](../crates/pdfgutter/README.md) runs `pdftoppm` to look at a page.
|
||||||
- [`avplan`](../crates/avplan/README.md) runs `ffprobe` to read a file's streams, and
|
- [`avplan`](../crates/avplan/README.md) runs `ffprobe` to read a file's streams, and
|
||||||
|
|||||||
Reference in new issue
Block a user