mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-11-13 23:55:40 +00:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 funkpower
|
5c4f770d76 | ||
|
funkpower
|
6f32f52293 | ||
|
funkpower
|
e480f07c32 | ||
|
funkpower
|
d551adc14c | ||
|
funkpower
|
90fb977a10 |
No files matched your search
Generated
+26
-26
@@ -101,7 +101,7 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "avplan"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -126,7 +126,7 @@ checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "bibclean"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -143,7 +143,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bibfetch"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -196,7 +196,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bookmeta"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -210,7 +210,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "booksniff"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -250,7 +250,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
|
||||
|
||||
[[package]]
|
||||
name = "catena"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"bibfetch",
|
||||
@@ -277,7 +277,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "catena-cli"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"clap_mangen",
|
||||
@@ -312,7 +312,7 @@ checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||
|
||||
[[package]]
|
||||
name = "chunkmd"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -334,7 +334,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "citefind"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"chunkmd",
|
||||
@@ -399,7 +399,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "colophon"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"catena-cli",
|
||||
@@ -448,7 +448,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "corpus-core"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"schemars",
|
||||
"serde",
|
||||
@@ -492,7 +492,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cuekit"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"corpus-core",
|
||||
"regex",
|
||||
@@ -504,7 +504,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cuenorm"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -514,7 +514,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cuepack"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -525,7 +525,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cueseg"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -612,7 +612,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "emb"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -876,7 +876,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "idnorm"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"catena-cli",
|
||||
@@ -942,7 +942,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "langid"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1022,7 +1022,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mdscrub"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1034,7 +1034,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mdsplit"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1113,7 +1113,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "pdfgutter"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1126,7 +1126,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdftriage"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1370,7 +1370,7 @@ checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189"
|
||||
|
||||
[[package]]
|
||||
name = "rrf"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1517,7 +1517,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "seriesof"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1634,7 +1634,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
||||
|
||||
[[package]]
|
||||
name = "textkit"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"regex",
|
||||
"tiktoken-rs",
|
||||
@@ -1724,7 +1724,7 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
|
||||
|
||||
[[package]]
|
||||
name = "tocfind"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
|
||||
+27
-27
@@ -3,39 +3,39 @@ resolver = "3"
|
||||
members = ["crates/*"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MPL-2.0"
|
||||
repository = "https://git.pumpin.online/funkpower/catena"
|
||||
|
||||
[workspace.dependencies]
|
||||
catena-cli = { path = "crates/catena-cli", version = "0.2.2" }
|
||||
corpus-core = { path = "crates/corpus-core", version = "0.2.2" }
|
||||
cuekit = { path = "crates/cuekit", version = "0.2.2" }
|
||||
textkit = { path = "crates/textkit", version = "0.2.2" }
|
||||
chunkmd = { path = "crates/chunkmd", version = "0.2.2" }
|
||||
mdscrub = { path = "crates/mdscrub", version = "0.2.2" }
|
||||
mdsplit = { path = "crates/mdsplit", version = "0.2.2" }
|
||||
pdftriage = { path = "crates/pdftriage", version = "0.2.2" }
|
||||
cuenorm = { path = "crates/cuenorm", version = "0.2.2" }
|
||||
cueseg = { path = "crates/cueseg", version = "0.2.2" }
|
||||
rrf = { path = "crates/rrf", version = "0.2.2" }
|
||||
cuepack = { path = "crates/cuepack", version = "0.2.2" }
|
||||
bibclean = { path = "crates/bibclean", version = "0.2.2" }
|
||||
avplan = { path = "crates/avplan", version = "0.2.2" }
|
||||
pdfgutter = { path = "crates/pdfgutter", version = "0.2.2" }
|
||||
catena = { path = "crates/bindings", version = "0.2.2" }
|
||||
seriesof = { path = "crates/seriesof", version = "0.2.2" }
|
||||
idnorm = { path = "crates/idnorm", version = "0.2.2" }
|
||||
bibfetch = { path = "crates/bibfetch", version = "0.2.2" }
|
||||
bookmeta = { path = "crates/bookmeta", version = "0.2.2" }
|
||||
langid = { path = "crates/langid", version = "0.2.2" }
|
||||
tocfind = { path = "crates/tocfind", version = "0.2.2" }
|
||||
booksniff = { path = "crates/booksniff", version = "0.2.2" }
|
||||
citefind = { path = "crates/citefind", version = "0.2.2" }
|
||||
colophon = { path = "crates/colophon", version = "0.2.2" }
|
||||
emb = { path = "crates/emb", version = "0.2.2" }
|
||||
catena-cli = { path = "crates/catena-cli", version = "0.3.1" }
|
||||
corpus-core = { path = "crates/corpus-core", version = "0.3.1" }
|
||||
cuekit = { path = "crates/cuekit", version = "0.3.1" }
|
||||
textkit = { path = "crates/textkit", version = "0.3.1" }
|
||||
chunkmd = { path = "crates/chunkmd", version = "0.3.1" }
|
||||
mdscrub = { path = "crates/mdscrub", version = "0.3.1" }
|
||||
mdsplit = { path = "crates/mdsplit", version = "0.3.1" }
|
||||
pdftriage = { path = "crates/pdftriage", version = "0.3.1" }
|
||||
cuenorm = { path = "crates/cuenorm", version = "0.3.1" }
|
||||
cueseg = { path = "crates/cueseg", version = "0.3.1" }
|
||||
rrf = { path = "crates/rrf", version = "0.3.1" }
|
||||
cuepack = { path = "crates/cuepack", version = "0.3.1" }
|
||||
bibclean = { path = "crates/bibclean", version = "0.3.1" }
|
||||
avplan = { path = "crates/avplan", version = "0.3.1" }
|
||||
pdfgutter = { path = "crates/pdfgutter", version = "0.3.1" }
|
||||
catena = { path = "crates/bindings", version = "0.3.1" }
|
||||
seriesof = { path = "crates/seriesof", version = "0.3.1" }
|
||||
idnorm = { path = "crates/idnorm", version = "0.3.1" }
|
||||
bibfetch = { path = "crates/bibfetch", version = "0.3.1" }
|
||||
bookmeta = { path = "crates/bookmeta", version = "0.3.1" }
|
||||
langid = { path = "crates/langid", version = "0.3.1" }
|
||||
tocfind = { path = "crates/tocfind", version = "0.3.1" }
|
||||
booksniff = { path = "crates/booksniff", version = "0.3.1" }
|
||||
citefind = { path = "crates/citefind", version = "0.3.1" }
|
||||
colophon = { path = "crates/colophon", version = "0.3.1" }
|
||||
emb = { path = "crates/emb", version = "0.3.1" }
|
||||
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
clap_mangen = "0.3"
|
||||
|
||||
@@ -63,13 +63,18 @@ pub fn lookup(
|
||||
for source in chain {
|
||||
match ask(agent, *source, id, kind, attempts) {
|
||||
Ok(record) => {
|
||||
let complete = record.found && record.title.is_some() && !record.authors.is_empty();
|
||||
let better = best.as_ref().is_none_or(|kept| record.found && !kept.found);
|
||||
if complete {
|
||||
return Ok(Some(record));
|
||||
match &mut best {
|
||||
// Merged rather than replaced. The registers answer about
|
||||
// different parts of one book: OpenLibrary's edition record
|
||||
// carries the publisher, the pagination and the language
|
||||
// and names its authors by key, while its search index
|
||||
// carries the authors and not the publisher. Keeping
|
||||
// whichever arrived first throws the other half away.
|
||||
Some(kept) => kept.fill_from(record),
|
||||
None => best = Some(record),
|
||||
}
|
||||
if better {
|
||||
best = Some(record);
|
||||
if best.as_ref().is_some_and(Record::complete) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(why) => last_error = Some(why),
|
||||
|
||||
@@ -59,6 +59,21 @@ pub struct Record {
|
||||
/// The abstract, where the register carries one.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub summary: Option<String>,
|
||||
/// The subject headings the register files it under, as it words them.
|
||||
///
|
||||
/// A register states these for far more books than print a cataloguing
|
||||
/// block, which makes them the second source a shelf otherwise lacks.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub subjects: Vec<String>,
|
||||
/// The language it is written in, as the register gives it: a two- or
|
||||
/// three-letter code, or a name. Not normalised here — what a register
|
||||
/// said is the record, and deciding that "eng" and "English" are one
|
||||
/// thing belongs to whoever is filing them.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub language: Option<String>,
|
||||
/// How many pages the register says it has.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub pages: Option<u32>,
|
||||
}
|
||||
|
||||
fn version() -> u32 {
|
||||
@@ -112,6 +127,70 @@ pub fn missing(id: &str, kind: &str, source: Source) -> Record {
|
||||
year: None,
|
||||
container: None,
|
||||
summary: None,
|
||||
subjects: Vec::new(),
|
||||
language: None,
|
||||
pages: None,
|
||||
}
|
||||
}
|
||||
|
||||
impl Record {
|
||||
/// Whether this record says everything a catalogue wants.
|
||||
///
|
||||
/// Used to decide when to stop asking. Title and authors are the minimum
|
||||
/// that makes a record usable; the rest is worth another register only
|
||||
/// while one is left to ask.
|
||||
pub fn complete(&self) -> bool {
|
||||
self.found
|
||||
&& self.title.is_some()
|
||||
&& !self.authors.is_empty()
|
||||
&& self.container.is_some()
|
||||
&& self.language.is_some()
|
||||
}
|
||||
|
||||
/// Take from another record whatever this one does not have.
|
||||
///
|
||||
/// The registers are complementary rather than redundant, and asking them
|
||||
/// in turn and keeping one answer throws away the difference. OpenLibrary's
|
||||
/// edition record gives the publisher, the pagination and the language but
|
||||
/// names its authors by key; its search index gives the authors but not the
|
||||
/// publisher. One book, two half-answers, and the whole of it only if they
|
||||
/// are put together.
|
||||
///
|
||||
/// Nothing already answered is overwritten, so the order of the chain is
|
||||
/// the order of trust.
|
||||
pub fn fill_from(&mut self, other: Record) {
|
||||
if !other.found {
|
||||
return;
|
||||
}
|
||||
self.found = true;
|
||||
if self.title.is_none() {
|
||||
self.title = other.title;
|
||||
}
|
||||
if self.authors.is_empty() {
|
||||
self.authors = other.authors;
|
||||
}
|
||||
if self.year.is_none() {
|
||||
self.year = other.year;
|
||||
}
|
||||
if self.container.is_none() {
|
||||
self.container = other.container;
|
||||
}
|
||||
if self.summary.is_none() {
|
||||
self.summary = other.summary;
|
||||
}
|
||||
if self.subjects.is_empty() {
|
||||
self.subjects = other.subjects;
|
||||
}
|
||||
if self.language.is_none() {
|
||||
self.language = other.language;
|
||||
}
|
||||
if self.pages.is_none() {
|
||||
self.pages = other.pages;
|
||||
}
|
||||
// Which register answered becomes which registers did.
|
||||
if !self.source.split('+').any(|s| s == other.source) {
|
||||
self.source = format!("{}+{}", self.source, other.source);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,3 +214,74 @@ mod tests {
|
||||
assert!(json.contains("\"found\":false"), "{json}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod merge_tests {
|
||||
use super::*;
|
||||
|
||||
fn rec(source: &str) -> Record {
|
||||
let mut r = missing("9780199205592", "isbn", Source::OpenLibrary);
|
||||
r.source = source.to_string();
|
||||
r.found = true;
|
||||
r
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_half_answers_make_one_record() {
|
||||
// Exactly what the two OpenLibrary endpoints give for one book: the
|
||||
// edition record has the publisher and the language and names its
|
||||
// authors by key; the index has the authors and not the publisher.
|
||||
let mut edition = rec("openlibrary");
|
||||
edition.title = Some("The First World War: A Very Short Introduction".into());
|
||||
edition.container = Some("Oxford University Press".into());
|
||||
edition.language = Some("eng".into());
|
||||
|
||||
let mut index = rec("openlibrary");
|
||||
index.title = Some("The First World War".into());
|
||||
index.authors = vec!["Michael Howard".into()];
|
||||
index.year = Some(2007);
|
||||
|
||||
edition.fill_from(index);
|
||||
assert_eq!(edition.authors, ["Michael Howard"]);
|
||||
assert_eq!(edition.year, Some(2007));
|
||||
// The fuller title arrived first and is not displaced by the shorter.
|
||||
assert_eq!(
|
||||
edition.title.as_deref(),
|
||||
Some("The First World War: A Very Short Introduction")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_register_that_has_nothing_contributes_nothing() {
|
||||
let mut kept = rec("openlibrary");
|
||||
kept.title = Some("A Book".into());
|
||||
let absent = missing("9780199205592", "isbn", Source::GoogleBooks);
|
||||
kept.fill_from(absent);
|
||||
assert_eq!(kept.title.as_deref(), Some("A Book"));
|
||||
assert_eq!(kept.source, "openlibrary");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_record_names_every_register_that_answered() {
|
||||
let mut a = rec("openlibrary");
|
||||
a.title = Some("A Book".into());
|
||||
let mut b = rec("googlebooks");
|
||||
b.found = true;
|
||||
b.pages = Some(134);
|
||||
a.fill_from(b);
|
||||
assert_eq!(a.source, "openlibrary+googlebooks");
|
||||
assert_eq!(a.pages, Some(134));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_means_a_catalogue_has_what_it_needs() {
|
||||
let mut r = rec("openlibrary");
|
||||
assert!(!r.complete());
|
||||
r.title = Some("A Book".into());
|
||||
r.authors = vec!["Someone".into()];
|
||||
assert!(!r.complete(), "still missing the publisher and language");
|
||||
r.container = Some("A Press".into());
|
||||
r.language = Some("eng".into());
|
||||
assert!(r.complete());
|
||||
}
|
||||
}
|
||||
@@ -55,9 +55,17 @@ impl Source {
|
||||
match kind {
|
||||
"doi" => &[Source::Crossref],
|
||||
"arxiv" => &[Source::Arxiv],
|
||||
// The index first. Asked for the fields by name it answers with
|
||||
// the title, the authors, the publisher, the language, a page
|
||||
// count and the subjects in one request, which is the whole of
|
||||
// what a catalogue wants; the edition record is then needed only
|
||||
// for a subtitle or a publisher the index did not give. Asking it
|
||||
// second cost three requests per book where one usually does, and
|
||||
// at three books in flight that is the difference between two
|
||||
// days and two hours.
|
||||
"isbn" => &[
|
||||
Source::OpenLibrary,
|
||||
Source::OpenLibrarySearch,
|
||||
Source::OpenLibrary,
|
||||
Source::GoogleBooks,
|
||||
],
|
||||
_ => &[],
|
||||
@@ -75,9 +83,12 @@ impl Source {
|
||||
Source::OpenLibrary => {
|
||||
format!("https://openlibrary.org/isbn/{}.json", encode(id))
|
||||
}
|
||||
// `fields` or the index answers with a handful of defaults and
|
||||
// keeps the rest, which is how a language it holds went unread.
|
||||
Source::OpenLibrarySearch => format!(
|
||||
"https://openlibrary.org/search.json?q=isbn:{}\
|
||||
&fields=title,subtitle,author_name,first_publish_year,publisher&limit=1",
|
||||
&fields=title,subtitle,author_name,first_publish_year,publisher,language,subject,\
|
||||
number_of_pages_median&limit=1",
|
||||
encode(id)
|
||||
),
|
||||
Source::GoogleBooks => format!(
|
||||
@@ -222,9 +233,24 @@ struct OpenLibraryBook {
|
||||
#[serde(default)]
|
||||
title: Option<String>,
|
||||
#[serde(default)]
|
||||
subtitle: Option<String>,
|
||||
#[serde(default)]
|
||||
publish_date: Option<String>,
|
||||
#[serde(default)]
|
||||
publishers: Vec<String>,
|
||||
#[serde(default)]
|
||||
subjects: Vec<String>,
|
||||
/// `[{"key": "/languages/eng"}]`, which is how OpenLibrary names one.
|
||||
#[serde(default)]
|
||||
languages: Vec<OpenLibraryKey>,
|
||||
#[serde(default)]
|
||||
number_of_pages: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct OpenLibraryKey {
|
||||
#[serde(default)]
|
||||
key: String,
|
||||
}
|
||||
|
||||
fn open_library(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
@@ -232,10 +258,23 @@ fn open_library(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?;
|
||||
|
||||
let mut record = blank(id, kind, Source::OpenLibrary);
|
||||
record.title = book.title;
|
||||
// An edition record keeps the subtitle in its own field, and a catalogue
|
||||
// wants the whole title. Google Books is already joined the same way.
|
||||
record.title = match (book.title, book.subtitle) {
|
||||
(Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")),
|
||||
(title, _) => title,
|
||||
};
|
||||
record.container = book.publishers.into_iter().next();
|
||||
// The date is free text — "1999", "March 1999", "1999-03-01".
|
||||
record.year = book.publish_date.as_deref().and_then(year_in);
|
||||
record.subjects = book.subjects;
|
||||
// "/languages/eng" names the language by the last segment of a key.
|
||||
record.language = book
|
||||
.languages
|
||||
.into_iter()
|
||||
.find_map(|l| l.key.rsplit('/').next().map(str::to_string))
|
||||
.filter(|l| !l.is_empty());
|
||||
record.pages = book.number_of_pages;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
@@ -257,8 +296,21 @@ struct SearchDoc {
|
||||
first_publish_year: Option<i64>,
|
||||
#[serde(default)]
|
||||
publisher: Vec<String>,
|
||||
#[serde(default)]
|
||||
language: Vec<String>,
|
||||
#[serde(default)]
|
||||
subject: Vec<String>,
|
||||
#[serde(rename = "number_of_pages_median", default)]
|
||||
pages: Option<u32>,
|
||||
}
|
||||
|
||||
/// How many of a work's subject headings are worth carrying.
|
||||
///
|
||||
/// The index lists every heading every edition has ever been given, which on a
|
||||
/// well-known book is hundreds and on Hamlet is thousands. A shelf is filed
|
||||
/// from a handful, and the first are the broadest.
|
||||
const SUBJECT_LIMIT: usize = 12;
|
||||
|
||||
/// A search that matched nothing answers with no documents rather than a
|
||||
/// status, so an empty reply is the absence and not a failure.
|
||||
fn open_library_search(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
@@ -275,6 +327,12 @@ fn open_library_search(id: &str, kind: &str, body: &str) -> Result<Record, Error
|
||||
record.authors = doc.author_name;
|
||||
record.year = doc.first_publish_year;
|
||||
record.container = doc.publisher.into_iter().next();
|
||||
record.language = doc.language.into_iter().next();
|
||||
// The index lists every subject any edition of the work carries, which
|
||||
// runs to hundreds on a well-known book. A shelf needs a handful.
|
||||
record.subjects = doc.subject;
|
||||
record.subjects.truncate(SUBJECT_LIMIT);
|
||||
record.pages = doc.pages;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
@@ -304,6 +362,12 @@ struct GoogleVolume {
|
||||
publisher: Option<String>,
|
||||
#[serde(default)]
|
||||
description: Option<String>,
|
||||
#[serde(default)]
|
||||
categories: Vec<String>,
|
||||
#[serde(rename = "language", default)]
|
||||
language: Option<String>,
|
||||
#[serde(rename = "pageCount", default)]
|
||||
page_count: Option<u32>,
|
||||
}
|
||||
|
||||
/// A search that matched nothing answers with no items rather than a status,
|
||||
@@ -324,6 +388,9 @@ fn google_books(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
record.year = volume.published_date.as_deref().and_then(year_in);
|
||||
record.container = volume.publisher;
|
||||
record.summary = volume.description.map(|text| collapse(&text));
|
||||
record.subjects = volume.categories;
|
||||
record.language = volume.language;
|
||||
record.pages = volume.page_count;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
@@ -379,7 +446,10 @@ mod tests {
|
||||
fn each_kind_goes_to_the_body_that_assigns_it() {
|
||||
assert_eq!(Source::for_kind("doi"), Some(Source::Crossref));
|
||||
assert_eq!(Source::for_kind("arxiv"), Some(Source::Arxiv));
|
||||
assert_eq!(Source::for_kind("isbn"), Some(Source::OpenLibrary));
|
||||
// The index, not the edition record: it answers with the whole of
|
||||
// what a catalogue wants in one request, where the edition record
|
||||
// takes up to twenty-eight seconds and names its authors by key.
|
||||
assert_eq!(Source::for_kind("isbn"), Some(Source::OpenLibrarySearch));
|
||||
// No register answers about an ISSN without a key.
|
||||
assert_eq!(Source::for_kind("issn"), None);
|
||||
}
|
||||
|
||||
@@ -26,8 +26,8 @@ file too small to be a book, a name that says "sample".
|
||||
cover images that ended up in a folder of PDFs.
|
||||
|
||||
A library accumulates files that lie: a DjVu named `.pdf`, a Word document
|
||||
named `.epub`, an EPUB saved as `.zip`, a single PDF that somebody zipped and
|
||||
renamed back to `.pdf`.
|
||||
named `.epub`, an RTF saved as `.doc`, an EPUB saved as `.zip`, a single PDF
|
||||
that somebody zipped and renamed back to `.pdf`.
|
||||
|
||||
## Install
|
||||
|
||||
@@ -89,7 +89,12 @@ one bad file does not stop a sweep.
|
||||
| `ole2` | The compound-file header every binary Word document starts with |
|
||||
| `epub` | A zip holding `mimetype` and `META-INF/container.xml` |
|
||||
| `zip-wrapped-pdf` | A zip whose only real member is one PDF |
|
||||
| `zip-wrapped-fb2` | A zip whose only real member is one FictionBook — the `.fb2.zip` most of them are passed around as |
|
||||
| `cbz` | A zip whose every real member is a page image: JPEG, PNG, GIF or WebP |
|
||||
| `azw3`, `mobi` | A Palm database tagged `BOOKMOBI`, told apart by the format version inside the first record |
|
||||
| `lit` | `ITOLITLS`, Microsoft Reader's signature |
|
||||
| `rtf` | `{\rtf` at the front, after any byte-order mark and blank space |
|
||||
| `fb2` | XML whose root element is `FictionBook`, found by stepping over the declaration, comments and doctype in the first four kilobytes |
|
||||
| `html` | Markup at the front, after any byte-order mark |
|
||||
| `txt` | Text that is none of the above |
|
||||
| `unknown` | Anything else, with a note saying what was seen |
|
||||
@@ -98,6 +103,20 @@ An archive holding several PDFs is a collection, not a wrapped book: choosing
|
||||
among them would be a guess, and naming one chapter as the whole work is worse
|
||||
than refusing.
|
||||
|
||||
A zipped FictionBook is reported as wrapped rather than as `fb2`, the same way
|
||||
a zipped PDF is: a converter handed it as FictionBook is handed a zip, and the
|
||||
kind is there to stop exactly that.
|
||||
|
||||
A comic is the pages and nothing else. Directories, the `ComicInfo.xml` a
|
||||
comic reader writes into the archive, and the desktop clutter every archive
|
||||
picks up are passed over; anything more — one PDF among the pages, a readme —
|
||||
and the zip is not a comic, and says what it holds instead. Only names are
|
||||
read, out of the archive's directory, so no member is ever unpacked.
|
||||
|
||||
FictionBook is read as bytes rather than as text. A great many are written in
|
||||
windows-1251, which spells its markup in ASCII but is not UTF-8, and would
|
||||
otherwise be called binary.
|
||||
|
||||
## As a library
|
||||
|
||||
```rust
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
//! Telling a FictionBook from any other XML.
|
||||
//!
|
||||
//! A FictionBook is an XML document whose root element is `FictionBook`.
|
||||
//! The root comes after the prolog — a declaration, comments, perhaps a
|
||||
//! doctype — so it is found by stepping over those rather than by matching
|
||||
//! at a fixed offset.
|
||||
//!
|
||||
//! The walk is over bytes, not text. A great many FictionBooks are written in
|
||||
//! windows-1251, which is not UTF-8 but spells its markup in ASCII all the
|
||||
//! same, and a reader that insisted on UTF-8 would call every one of them
|
||||
//! binary.
|
||||
|
||||
use crate::BOM;
|
||||
|
||||
/// What the root element of a FictionBook is called.
|
||||
const ROOT: &[u8] = b"FictionBook";
|
||||
|
||||
/// Whether the front of a file is the start of a FictionBook.
|
||||
///
|
||||
/// `head` is however much of the front the caller read. A root element that
|
||||
/// does not begin inside it is not found, so the answer is no rather than a
|
||||
/// guess.
|
||||
pub fn is_fictionbook(head: &[u8]) -> bool {
|
||||
root_element(head).is_some_and(|name| local_name(name) == ROOT)
|
||||
}
|
||||
|
||||
/// The name of the first element, after the prolog.
|
||||
fn root_element(head: &[u8]) -> Option<&[u8]> {
|
||||
let mut rest = head.strip_prefix(BOM).unwrap_or(head);
|
||||
loop {
|
||||
rest = rest.trim_ascii_start();
|
||||
if let Some(after) = rest.strip_prefix(b"<?") {
|
||||
rest = after_marker(after, b"?>")?;
|
||||
} else if let Some(after) = rest.strip_prefix(b"<!--") {
|
||||
rest = after_marker(after, b"-->")?;
|
||||
} else if let Some(after) = rest.strip_prefix(b"<!") {
|
||||
rest = after_declaration(after)?;
|
||||
} else {
|
||||
// Anything but an element here is text before the root, which
|
||||
// XML does not allow.
|
||||
let after = rest.strip_prefix(b"<")?;
|
||||
// A name that runs to the end of what was read may be the front
|
||||
// of a longer one, so it is not taken.
|
||||
let end = after.iter().position(|byte| !is_name_byte(*byte))?;
|
||||
let ends_properly =
|
||||
matches!(after[end], b'>' | b'/') || after[end].is_ascii_whitespace();
|
||||
return (end > 0 && ends_properly).then_some(&after[..end]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What follows the first `marker`, if it is there at all.
|
||||
fn after_marker<'a>(rest: &'a [u8], marker: &[u8]) -> Option<&'a [u8]> {
|
||||
rest.windows(marker.len())
|
||||
.position(|window| window == marker)
|
||||
.map(|at| &rest[at + marker.len()..])
|
||||
}
|
||||
|
||||
/// What follows a doctype, whose internal subset may hold `>` of its own.
|
||||
fn after_declaration(rest: &[u8]) -> Option<&[u8]> {
|
||||
let mut depth = 0usize;
|
||||
let mut quote = None;
|
||||
for (at, &byte) in rest.iter().enumerate() {
|
||||
match quote {
|
||||
Some(open) if byte == open => quote = None,
|
||||
Some(_) => {}
|
||||
None => match byte {
|
||||
b'"' | b'\'' => quote = Some(byte),
|
||||
b'[' => depth += 1,
|
||||
b']' => depth = depth.saturating_sub(1),
|
||||
b'>' if depth == 0 => return Some(&rest[at + 1..]),
|
||||
_ => {}
|
||||
},
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// The name without its namespace prefix.
|
||||
fn local_name(name: &[u8]) -> &[u8] {
|
||||
match name.iter().rposition(|byte| *byte == b':') {
|
||||
Some(at) => &name[at + 1..],
|
||||
None => name,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_name_byte(byte: u8) -> bool {
|
||||
byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.' | b':') || byte >= 0x80
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const BOOK: &[u8] = b"<?xml version=\"1.0\" encoding=\"utf-8\"?>\n\
|
||||
<FictionBook xmlns=\"http://www.gribuser.ru/xml/fictionbook/2.0\">\n\
|
||||
<description><title-info><book-title>The Mill</book-title></title-info></description>";
|
||||
|
||||
#[test]
|
||||
fn a_fictionbook_is_recognised() {
|
||||
assert!(is_fictionbook(BOOK));
|
||||
assert!(is_fictionbook(b"<FictionBook>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_prolog_is_stepped_over() {
|
||||
let head = b"\xef\xbb\xbf <?xml version=\"1.0\"?>\r\n\
|
||||
<!-- converted by hand -->\n\
|
||||
<!DOCTYPE FictionBook [ <!ENTITY mill \"<b>the mill</b>\"> ]>\n\
|
||||
<?xml-stylesheet href=\"fb2.css\"?>\n\
|
||||
<FictionBook\n xmlns=\"http://www.gribuser.ru/xml/fictionbook/2.0\">";
|
||||
assert!(is_fictionbook(head));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_prefixed_root_is_still_a_fictionbook() {
|
||||
assert!(is_fictionbook(
|
||||
b"<fb:FictionBook xmlns:fb=\"http://www.gribuser.ru/xml/fictionbook/2.0\">"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn markup_need_not_be_utf8() {
|
||||
// windows-1251 for "Мельница" in the title.
|
||||
let head = b"<?xml version=\"1.0\" encoding=\"windows-1251\"?>\
|
||||
<FictionBook><description><book-title>\xcc\xe5\xeb\xfc\xed\xe8\xf6\xe0</book-title>";
|
||||
assert!(is_fictionbook(head));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_xml_is_not_a_fictionbook() {
|
||||
assert!(!is_fictionbook(
|
||||
b"<?xml version=\"1.0\"?>\n<rss version=\"2.0\">"
|
||||
));
|
||||
assert!(!is_fictionbook(b"<?xml version=\"1.0\"?><FictionBookish>"));
|
||||
assert!(!is_fictionbook(b"<html><FictionBook>"));
|
||||
assert!(!is_fictionbook(b"text before <FictionBook>"));
|
||||
assert!(!is_fictionbook(b""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_root_the_window_cut_short_is_not_taken() {
|
||||
assert!(!is_fictionbook(b"<?xml version=\"1.0\"?><FictionBook"));
|
||||
assert!(!is_fictionbook(b"<?xml version=\"1.0\"?><Fiction"));
|
||||
assert!(!is_fictionbook(b"<?xml version=\"1.0\""));
|
||||
assert!(!is_fictionbook(b"<!-- never closed <FictionBook>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_truncated_or_mangled_prolog_does_not_panic() {
|
||||
let head = b"\xef\xbb\xbf<?xml?><!--c--><!DOCTYPE x [ <!ENTITY a '>'> ] \"]\">\
|
||||
<FictionBook>";
|
||||
assert!(is_fictionbook(head));
|
||||
for cut in 0..head.len() {
|
||||
let _ = is_fictionbook(&head[..cut]);
|
||||
}
|
||||
for byte in 0..=255u8 {
|
||||
let _ = is_fictionbook(&[b'<', byte]);
|
||||
let _ = is_fictionbook(&[b'<', b'!', byte, b'>']);
|
||||
}
|
||||
}
|
||||
}
|
||||
+254
-13
@@ -9,6 +9,7 @@
|
||||
//! The containers are unambiguous, so they are read rather than guessed at.
|
||||
|
||||
pub mod bytes;
|
||||
pub mod fb2;
|
||||
pub mod mobi;
|
||||
pub mod zip;
|
||||
|
||||
@@ -22,17 +23,32 @@ const DJVU: &[u8] = b"AT&TFORM";
|
||||
const OLE2: &[u8] = &[0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1];
|
||||
/// A PDF announces its version in its first bytes.
|
||||
const PDF: &[u8] = b"%PDF-";
|
||||
/// Microsoft Reader's signature, ahead of everything else in the file.
|
||||
const LIT: &[u8] = b"ITOLITLS";
|
||||
/// The group every RTF document is, opened with the control word that names
|
||||
/// the format.
|
||||
const RTF: &[u8] = br"{\rtf";
|
||||
/// The UTF-8 byte-order mark.
|
||||
const BOM: &[u8] = b"\xef\xbb\xbf";
|
||||
|
||||
/// Archive members that carry no content: resource forks and desktop clutter.
|
||||
const ARCHIVE_NOISE: &[&str] = &["__macosx/", ".ds_store", "thumbs.db"];
|
||||
|
||||
/// What a comic book archive carries besides its pages: the catalogue entry
|
||||
/// comic readers write into it.
|
||||
const COMIC_NOISE: &[&str] = &["comicinfo.xml"];
|
||||
|
||||
/// The extensions of the images a comic book archive is made of.
|
||||
const PAGE_IMAGES: &[&str] = &[".jpg", ".jpeg", ".png", ".gif", ".webp"];
|
||||
|
||||
/// Substrings in a name that mark a file as a sample rather than a book.
|
||||
const SAMPLE_MARKERS: &[&str] = &["sample", "readme", "read me"];
|
||||
|
||||
/// Below this, a file is too small to be a book whatever it claims to be.
|
||||
const SMALLEST_BOOK: u64 = 4096;
|
||||
|
||||
/// How much of the front is read to tell text from markup.
|
||||
/// How much of the front is read to tell RTF, FictionBook, markup and text
|
||||
/// apart.
|
||||
const TEXT_WINDOW: usize = 4096;
|
||||
|
||||
/// Identify a file.
|
||||
@@ -49,12 +65,17 @@ pub fn sniff(source: &impl Bytes, name: &str) -> Doc {
|
||||
Kind::Pdf
|
||||
} else if source.magic_at(0, OLE2) {
|
||||
Kind::Ole2
|
||||
} else if source.magic_at(0, LIT) {
|
||||
Kind::Lit
|
||||
} else if source.magic_at(0, zip::LOCAL_HEADER) {
|
||||
archive(source, &mut doc)
|
||||
} else if let Some(kind) = mobi::kind(source) {
|
||||
kind
|
||||
} else {
|
||||
text(source, &mut doc)
|
||||
match source.slice(0, TEXT_WINDOW) {
|
||||
Ok(head) => front(&head, &mut doc),
|
||||
Err(_) => Kind::Unknown,
|
||||
}
|
||||
};
|
||||
|
||||
if doc.kind == Kind::Unknown {
|
||||
@@ -80,7 +101,8 @@ pub fn sniff(source: &impl Bytes, name: &str) -> Doc {
|
||||
doc
|
||||
}
|
||||
|
||||
/// What is inside a zip: an EPUB, a wrapped PDF, or a collection of pages.
|
||||
/// What is inside a zip: an EPUB, a wrapped book, a comic, or a collection of
|
||||
/// pages.
|
||||
fn archive(source: &impl Bytes, doc: &mut Doc) -> Kind {
|
||||
let Some(names) = zip::members(source) else {
|
||||
doc.note("a zip whose directory could not be read");
|
||||
@@ -99,6 +121,13 @@ fn archive(source: &impl Bytes, doc: &mut Doc) -> Kind {
|
||||
doc.note(format!("a zip holding one PDF, {pdf:?}"));
|
||||
return Kind::ZipWrappedPdf;
|
||||
}
|
||||
if let Some(fb2) = sole_member(&names, ".fb2") {
|
||||
doc.note(format!("a zip holding one FictionBook, {fb2:?}"));
|
||||
return Kind::ZipWrappedFb2;
|
||||
}
|
||||
if is_comic(&names) {
|
||||
return Kind::Cbz;
|
||||
}
|
||||
if lower
|
||||
.iter()
|
||||
.any(|name| name.ends_with(".htm") || name.ends_with(".html") || name.ends_with(".xhtml"))
|
||||
@@ -119,25 +148,67 @@ fn archive(source: &impl Bytes, doc: &mut Doc) -> Kind {
|
||||
/// a collection, and choosing among them would be a guess — better to say it
|
||||
/// is not a book than to name one chapter as the whole.
|
||||
pub fn sole_pdf(names: &[String]) -> Option<&str> {
|
||||
let real: Vec<&String> = names
|
||||
sole_member(names, ".pdf")
|
||||
}
|
||||
|
||||
/// The one member an archive exists to carry, if it is the only real member
|
||||
/// and its name ends in `extension`.
|
||||
fn sole_member<'a>(names: &'a [String], extension: &str) -> Option<&'a str> {
|
||||
match real_members(names).as_slice() {
|
||||
[only] if only.to_lowercase().ends_with(extension) => Some(only.as_str()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether an archive is the pages of a comic and nothing else.
|
||||
///
|
||||
/// Every real member has to be an image. One PDF or one text file among the
|
||||
/// pages makes it something else, and saying what would be a guess.
|
||||
fn is_comic(names: &[String]) -> bool {
|
||||
let pages: Vec<String> = real_members(names)
|
||||
.into_iter()
|
||||
.map(|name| name.to_lowercase())
|
||||
.filter(|name| {
|
||||
let file = name.rsplit('/').next().unwrap_or(name);
|
||||
!COMIC_NOISE.contains(&file)
|
||||
})
|
||||
.collect();
|
||||
!pages.is_empty()
|
||||
&& pages
|
||||
.iter()
|
||||
.all(|name| PAGE_IMAGES.iter().any(|image| name.ends_with(image)))
|
||||
}
|
||||
|
||||
/// The members that carry content: no directories, no desktop clutter.
|
||||
fn real_members(names: &[String]) -> Vec<&String> {
|
||||
names
|
||||
.iter()
|
||||
.filter(|name| !name.ends_with('/'))
|
||||
.filter(|name| {
|
||||
let lower = name.to_lowercase();
|
||||
!ARCHIVE_NOISE.iter().any(|noise| lower.contains(noise))
|
||||
})
|
||||
.collect();
|
||||
match real.as_slice() {
|
||||
[only] if only.to_lowercase().ends_with(".pdf") => Some(only.as_str()),
|
||||
_ => None,
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// What the front of a file says when it is not a container: RTF, a
|
||||
/// FictionBook, markup, text, or nothing this knows.
|
||||
fn front(head: &[u8], doc: &mut Doc) -> Kind {
|
||||
// Both are recognised on bytes rather than text. RTF is seven-bit by
|
||||
// design but not always in practice, and a FictionBook is as often
|
||||
// windows-1251 as UTF-8.
|
||||
let lead = head.strip_prefix(BOM).unwrap_or(head).trim_ascii_start();
|
||||
if lead.starts_with(RTF) {
|
||||
return Kind::Rtf;
|
||||
}
|
||||
if fb2::is_fictionbook(head) {
|
||||
return Kind::Fb2;
|
||||
}
|
||||
text(head, doc)
|
||||
}
|
||||
|
||||
/// Text, markup, or nothing this knows.
|
||||
fn text(source: &impl Bytes, doc: &mut Doc) -> Kind {
|
||||
let Ok(head) = source.slice(0, TEXT_WINDOW) else {
|
||||
return Kind::Unknown;
|
||||
};
|
||||
fn text(head: &[u8], doc: &mut Doc) -> Kind {
|
||||
if head.is_empty() {
|
||||
doc.note("the file is empty");
|
||||
return Kind::Unknown;
|
||||
@@ -149,7 +220,7 @@ fn text(source: &impl Bytes, doc: &mut Doc) -> Kind {
|
||||
}
|
||||
// The window may have cut a character in half, which is not the same
|
||||
// thing as the file not being text.
|
||||
let head = match std::str::from_utf8(&head) {
|
||||
let head = match std::str::from_utf8(head) {
|
||||
Ok(text) => text,
|
||||
Err(e) if e.error_len().is_none() => {
|
||||
std::str::from_utf8(&head[..e.valid_up_to()]).expect("a valid prefix")
|
||||
@@ -178,10 +249,15 @@ fn names_the_same_thing(extension: &str, kind: Kind) -> bool {
|
||||
Kind::Djvu => matches!(extension, "djvu" | "djv"),
|
||||
Kind::Azw3 => matches!(extension, "azw3" | "azw" | "kfx"),
|
||||
Kind::Mobi => matches!(extension, "mobi" | "prc" | "azw"),
|
||||
Kind::Lit => extension == "lit",
|
||||
Kind::Fb2 => extension == "fb2",
|
||||
Kind::Rtf => extension == "rtf",
|
||||
Kind::Txt => matches!(extension, "txt" | "text" | "md" | "markdown"),
|
||||
Kind::Html => matches!(extension, "html" | "htm" | "xhtml" | "xml"),
|
||||
Kind::Ole2 => matches!(extension, "doc" | "xls" | "ppt"),
|
||||
Kind::ZipWrappedPdf => extension == "zip",
|
||||
Kind::ZipWrappedFb2 => matches!(extension, "zip" | "fbz"),
|
||||
Kind::Cbz => extension == "cbz",
|
||||
// Nothing was recognised, so there is nothing to disagree with.
|
||||
Kind::Unknown => true,
|
||||
}
|
||||
@@ -194,10 +270,15 @@ fn spelling(kind: Kind) -> &'static str {
|
||||
Kind::Djvu => "DjVu",
|
||||
Kind::Azw3 => "AZW3",
|
||||
Kind::Mobi => "MOBI",
|
||||
Kind::Lit => "Microsoft Reader",
|
||||
Kind::Fb2 => "FictionBook",
|
||||
Kind::Rtf => "RTF",
|
||||
Kind::Txt => "plain text",
|
||||
Kind::Html => "HTML",
|
||||
Kind::Ole2 => "an OLE2 document",
|
||||
Kind::ZipWrappedPdf => "a zip holding one PDF",
|
||||
Kind::ZipWrappedFb2 => "a zip holding one FictionBook",
|
||||
Kind::Cbz => "a comic book archive",
|
||||
Kind::Unknown => "nothing recognisable",
|
||||
}
|
||||
}
|
||||
@@ -332,4 +413,164 @@ mod tests {
|
||||
head.truncate(TEXT_WINDOW + 1);
|
||||
assert_eq!(sniff(&head.as_slice(), "a.txt").kind, Kind::Txt);
|
||||
}
|
||||
|
||||
fn archive_of(names: &[&str], name: &str) -> Doc {
|
||||
sniff(&zip::tests::zip(names).as_slice(), name)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rtf_is_recognised_by_its_opening_group() {
|
||||
assert_eq!(kind_of(br"{\rtf1\ansi\deff0 {\fonttbl", "a.rtf"), Kind::Rtf);
|
||||
assert_eq!(
|
||||
kind_of(b"\xef\xbb\xbf\r\n {\\rtf1\\ansi", "a.rtf"),
|
||||
Kind::Rtf
|
||||
);
|
||||
// Eight-bit text in the body does not stop it being RTF.
|
||||
assert_eq!(kind_of(b"{\\rtf1 caf\xe9 \x00", "a.rtf"), Kind::Rtf);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rtf_cut_short_of_its_control_word_is_not_rtf() {
|
||||
assert_eq!(sniff(&br"{\rt".as_slice(), "a.rtf").kind, Kind::Txt);
|
||||
assert_eq!(kind_of(br"{\rtx1", "a.rtf"), Kind::Txt);
|
||||
assert_eq!(kind_of(br"text {\rtf1", "a.rtf"), Kind::Txt);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rtf_named_as_a_word_document_is_remarked_on() {
|
||||
let doc = sniff(&padded(br"{\rtf1\ansi").as_slice(), "letter.doc");
|
||||
assert_eq!(doc.kind, Kind::Rtf);
|
||||
assert!(
|
||||
doc.notes
|
||||
.iter()
|
||||
.any(|note| note == "named .doc, but the bytes say RTF"),
|
||||
"{:?}",
|
||||
doc.notes
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn microsoft_reader_is_recognised_by_its_signature() {
|
||||
let doc = sniff(&padded(b"ITOLITLS\x01\x00\x00\x00").as_slice(), "book.lit");
|
||||
assert_eq!(doc.kind, Kind::Lit);
|
||||
assert!(doc.notes.is_empty(), "{:?}", doc.notes);
|
||||
assert_eq!(kind_of(b"ITOLITL", "book.lit"), Kind::Txt);
|
||||
assert_eq!(kind_of(b" ITOLITLS", "book.lit"), Kind::Txt);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fictionbook_is_recognised_and_other_xml_is_not() {
|
||||
let book = b"<?xml version=\"1.0\" encoding=\"windows-1251\"?>\n\
|
||||
<FictionBook xmlns=\"http://www.gribuser.ru/xml/fictionbook/2.0\">\
|
||||
<book-title>\xcc\xe5\xeb\xfc\xed\xe8\xf6\xe0</book-title>";
|
||||
let doc = sniff(&padded(book).as_slice(), "mill.fb2");
|
||||
assert_eq!(doc.kind, Kind::Fb2);
|
||||
assert!(doc.notes.is_empty(), "{:?}", doc.notes);
|
||||
|
||||
let feed = b"<?xml version=\"1.0\"?>\n<rss version=\"2.0\"><channel>";
|
||||
assert_eq!(kind_of(feed, "feed.xml"), Kind::Html);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fictionbook_named_as_something_else_is_remarked_on() {
|
||||
let doc = sniff(&padded(b"<FictionBook>").as_slice(), "mill.txt");
|
||||
assert_eq!(doc.kind, Kind::Fb2);
|
||||
assert!(
|
||||
doc.notes.iter().any(|note| note.contains("FictionBook")),
|
||||
"{:?}",
|
||||
doc.notes
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zip_holding_one_fictionbook_is_a_wrapped_one() {
|
||||
let doc = archive_of(&["__MACOSX/._mill.fb2", "mill.fb2"], "mill.fb2.zip");
|
||||
assert_eq!(doc.kind, Kind::ZipWrappedFb2);
|
||||
assert_eq!(doc.declared_ext.as_deref(), Some("zip"));
|
||||
assert!(
|
||||
doc.notes
|
||||
.iter()
|
||||
.any(|note| note.contains("one FictionBook")),
|
||||
"{:?}",
|
||||
doc.notes
|
||||
);
|
||||
assert!(!doc.notes.iter().any(|note| note.contains("named .")));
|
||||
|
||||
let two = archive_of(&["one.fb2", "two.fb2"], "pair.zip");
|
||||
assert_eq!(two.kind, Kind::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zip_of_page_images_is_a_comic() {
|
||||
let doc = archive_of(
|
||||
&[
|
||||
"Issue 1/",
|
||||
"Issue 1/001.jpg",
|
||||
"Issue 1/002.JPEG",
|
||||
"Issue 1/003.png",
|
||||
"Issue 1/004.gif",
|
||||
"Issue 1/005.webp",
|
||||
"ComicInfo.xml",
|
||||
"__MACOSX/Issue 1/._001.jpg",
|
||||
"Issue 1/Thumbs.db",
|
||||
".DS_Store",
|
||||
],
|
||||
"issue-1.cbz",
|
||||
);
|
||||
assert_eq!(doc.kind, Kind::Cbz);
|
||||
assert!(
|
||||
!doc.notes.iter().any(|note| note.contains("named .")),
|
||||
"{:?}",
|
||||
doc.notes
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_comic_saved_as_a_zip_is_remarked_on() {
|
||||
let doc = archive_of(&["001.jpg", "002.jpg"], "issue-1.zip");
|
||||
assert_eq!(doc.kind, Kind::Cbz);
|
||||
assert!(
|
||||
doc.notes.iter().any(|note| note.contains("comic")),
|
||||
"{:?}",
|
||||
doc.notes
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zip_with_anything_but_images_is_not_a_comic() {
|
||||
let one_pdf = archive_of(&["001.jpg", "002.jpg", "extras.pdf"], "issue-1.cbz");
|
||||
assert_eq!(one_pdf.kind, Kind::Unknown);
|
||||
assert!(
|
||||
one_pdf
|
||||
.notes
|
||||
.iter()
|
||||
.any(|note| note.contains("none of them a book")),
|
||||
"{:?}",
|
||||
one_pdf.notes
|
||||
);
|
||||
|
||||
let readme = archive_of(&["001.jpg", "readme.txt"], "issue-1.cbz");
|
||||
assert_eq!(readme.kind, Kind::Unknown);
|
||||
|
||||
// A name that ends like an image without being one.
|
||||
let lookalike = archive_of(&["001.jpg", "notes.jpg.txt"], "issue-1.cbz");
|
||||
assert_eq!(lookalike.kind, Kind::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zip_with_no_pages_is_not_a_comic() {
|
||||
let catalogue_only = archive_of(&["ComicInfo.xml", "pages/"], "issue-1.cbz");
|
||||
assert_eq!(catalogue_only.kind, Kind::Unknown);
|
||||
|
||||
let empty = archive_of(&[], "issue-1.cbz");
|
||||
assert_eq!(empty.kind, Kind::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_truncated_comic_does_not_panic() {
|
||||
let whole = zip::tests::zip(&["001.jpg", "002.png", "ComicInfo.xml"]);
|
||||
for cut in 0..whole.len() {
|
||||
let _ = sniff(&&whole[..cut], "issue-1.cbz");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -78,11 +78,11 @@ fn directory_start(source: &impl Bytes) -> Option<(usize, u64)> {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Build a zip holding `names`, each with empty stored content.
|
||||
fn zip(names: &[&str]) -> Vec<u8> {
|
||||
pub(crate) fn zip(names: &[&str]) -> Vec<u8> {
|
||||
let mut out: Vec<u8> = Vec::new();
|
||||
let mut offsets = Vec::new();
|
||||
for name in names {
|
||||
|
||||
@@ -64,6 +64,30 @@ fn several_files_come_back_as_several_records() {
|
||||
assert_eq!(kinds, ["\"pdf\"", "\"djvu\""]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_formats_read_from_the_front_come_back_by_their_wire_names() {
|
||||
let rtf = written("letter.rtf", &padded(br"{\rtf1\ansi"));
|
||||
let lit = written("book.lit", &padded(b"ITOLITLS"));
|
||||
let fb2 = written(
|
||||
"mill.fb2",
|
||||
&padded(b"<?xml version=\"1.0\"?>\n<FictionBook xmlns=\"http://www.gribuser.ru/xml/fictionbook/2.0\">"),
|
||||
);
|
||||
let out = run(&[
|
||||
rtf.to_str().expect("utf-8 path"),
|
||||
lit.to_str().expect("utf-8 path"),
|
||||
fb2.to_str().expect("utf-8 path"),
|
||||
]);
|
||||
assert_eq!(out.status.code(), Some(0));
|
||||
let kinds: Vec<String> = String::from_utf8(out.stdout)
|
||||
.expect("utf-8")
|
||||
.lines()
|
||||
.map(|line| {
|
||||
serde_json::from_str::<serde_json::Value>(line).expect("valid JSON")["kind"].to_string()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(kinds, ["\"rtf\"", "\"lit\"", "\"fb2\""]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_that_is_not_there_is_an_io_error_and_the_rest_still_run() {
|
||||
let path = written("kept.pdf", &padded(b"%PDF-1.7"));
|
||||
|
||||
@@ -35,6 +35,29 @@ static LIFESPAN: LazyLock<Regex> = LazyLock::new(|| {
|
||||
static BARE_DATE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)^\d{1,4}\s*(bc|ad|bce|ce)?\.?$").expect("a valid pattern"));
|
||||
/// `Copyright © 2013 by`, `\u{a9} 2013 Richard Swinburne`, `Copyright 1971 by`.
|
||||
/// `1. General Remarks`, `2 James W. McAllister` — a numbered heading.
|
||||
static NUMBERED_HEADING: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"^\s*\d+\s*[.)]\s+\S|^\s*\d+\s{2,}\S").expect("a valid pattern"));
|
||||
/// `5222 What Is This`, `827 North Fourth Street` — a number that is not a
|
||||
/// year, leading a line.
|
||||
static LEADING_NUMBER: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"^\s*(\d{1,5})\s+\S").expect("a valid pattern"));
|
||||
/// `Page xxxvi`, `Chapter I`, `Fig. 3` — a marker, not a name.
|
||||
static MARKER: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(
|
||||
r"(?i)^\s*(page|pages|p\.|pp\.|fig\.|figure|plate|chapter|ch\.|section|part)\s+[\divxlcm]",
|
||||
)
|
||||
.expect("a valid pattern")
|
||||
});
|
||||
/// `Sidebottom, Harry. Ancient warfare` — a bibliography entry, which front
|
||||
/// matter prints in its cataloguing block and in its further reading.
|
||||
static CATALOGUE_ENTRY: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"^[A-Z][A-Za-z'\-]+,\s+[A-Z][A-Za-z'.\-]*\.\s+\S").expect("a valid pattern")
|
||||
});
|
||||
/// `To Mackenzie,` — a dedication, set alone on its own page.
|
||||
static DEDICATION: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"(?i)^\s*(to|for|in memory of)\s+\S.*,\s*$").expect("a valid pattern")
|
||||
});
|
||||
static COPYRIGHT_NOTICE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
// The year need not follow the symbol directly: "(c) Alfredo Ferrarin
|
||||
// 2004" puts the holder between them, and was read as a title.
|
||||
@@ -79,10 +102,13 @@ copenhagen|stockholm|moscow|beijing|tokyo|delhi|nairobi)\b[\s,·•|and]*",
|
||||
static STRUCTURAL: &[&str] = &[
|
||||
"abbreviations",
|
||||
"about the author",
|
||||
"abstract",
|
||||
"acknowledgements",
|
||||
"acknowledgments",
|
||||
"afterword",
|
||||
"appendix",
|
||||
"author",
|
||||
"authors",
|
||||
"bibliography",
|
||||
"contents",
|
||||
"dedication",
|
||||
@@ -95,6 +121,7 @@ static STRUCTURAL: &[&str] = &[
|
||||
"illustrations",
|
||||
"index",
|
||||
"introduction",
|
||||
"keywords",
|
||||
"list of contributors",
|
||||
"list of illustrations",
|
||||
"maps",
|
||||
@@ -102,7 +129,9 @@ static STRUCTURAL: &[&str] = &[
|
||||
"postscript",
|
||||
"preface",
|
||||
"prologue",
|
||||
"subtitle",
|
||||
"table of contents",
|
||||
"title",
|
||||
"title page",
|
||||
"translator's note",
|
||||
];
|
||||
@@ -182,6 +211,22 @@ fn is_furniture(line: &str) -> bool {
|
||||
}
|
||||
// Sentences a copyright page sets, which are not notices and so not caught
|
||||
// by the notice pattern.
|
||||
if NUMBERED_HEADING.is_match(t) || MARKER.is_match(t) || CATALOGUE_ENTRY.is_match(t) {
|
||||
return true;
|
||||
}
|
||||
// A line opening with a number that is not a year. "5222 What Is This" and
|
||||
// "827 North Fourth Street" are a stray folio and an address; "1984
|
||||
// Revisited" is a book, and the difference is whether the number could be
|
||||
// the year of one.
|
||||
if let Some(c) = LEADING_NUMBER.captures(t) {
|
||||
let n: u32 = c[1].parse().unwrap_or(0);
|
||||
if !(1000..=2099).contains(&n) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if DEDICATION.is_match(t) {
|
||||
return true;
|
||||
}
|
||||
let lower = t.to_lowercase();
|
||||
for phrase in [
|
||||
"moral rights of the author",
|
||||
@@ -496,3 +541,54 @@ mod conversion_tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod review_pile_tests {
|
||||
use super::*;
|
||||
|
||||
/// Everything here was proposed as a title by a real run over this
|
||||
/// library, and sat in the review queue waiting for a person to say no.
|
||||
#[test]
|
||||
fn what_the_review_pile_was_full_of() {
|
||||
for line in [
|
||||
"1. General Remarks",
|
||||
"2 James W. McAllister",
|
||||
"5222 What Is This",
|
||||
"827 North Fourth Street",
|
||||
"Page xxxvi",
|
||||
"Page xxxviii",
|
||||
"Chapter I",
|
||||
"title",
|
||||
"Sidebottom, Harry. Ancient warfare",
|
||||
"To Mackenzie,",
|
||||
] {
|
||||
assert!(
|
||||
is_structural(line) || is_furniture(line),
|
||||
"still read as a title: {line:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// And none of it catches a book. The numbers are the delicate part: a
|
||||
/// year can open a title, and a folio cannot.
|
||||
#[test]
|
||||
fn a_book_is_still_a_book() {
|
||||
for line in [
|
||||
"1984 Revisited",
|
||||
"1914: The Year the World Ended",
|
||||
"Chapter and Verse",
|
||||
"Part of the Furniture",
|
||||
"To Kill a Mockingbird",
|
||||
"For Whom the Bell Tolls",
|
||||
"The Title of This Book",
|
||||
"Pages from a Cold Island",
|
||||
"Sections of a Broken Whole",
|
||||
"In Memory of Bread",
|
||||
] {
|
||||
assert!(
|
||||
!(is_structural(line) || is_furniture(line)),
|
||||
"rejected a real title: {line:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -77,6 +77,12 @@ pub enum Kind {
|
||||
Azw3,
|
||||
/// Amazon MOBI.
|
||||
Mobi,
|
||||
/// Microsoft Reader.
|
||||
Lit,
|
||||
/// FictionBook, the XML ebook format.
|
||||
Fb2,
|
||||
/// Rich Text Format.
|
||||
Rtf,
|
||||
/// Plain text.
|
||||
Txt,
|
||||
/// HTML or XHTML.
|
||||
@@ -85,6 +91,10 @@ pub enum Kind {
|
||||
Ole2,
|
||||
/// A zip archive whose only real content is a single PDF.
|
||||
ZipWrappedPdf,
|
||||
/// A zip archive whose only real content is a single FictionBook.
|
||||
ZipWrappedFb2,
|
||||
/// A comic book archive: a zip of page images.
|
||||
Cbz,
|
||||
/// Recognised as none of the above.
|
||||
Unknown,
|
||||
}
|
||||
@@ -118,6 +128,11 @@ mod tests {
|
||||
(Kind::Ole2, "\"ole2\""),
|
||||
(Kind::Azw3, "\"azw3\""),
|
||||
(Kind::ZipWrappedPdf, "\"zip-wrapped-pdf\""),
|
||||
(Kind::ZipWrappedFb2, "\"zip-wrapped-fb2\""),
|
||||
(Kind::Fb2, "\"fb2\""),
|
||||
(Kind::Lit, "\"lit\""),
|
||||
(Kind::Rtf, "\"rtf\""),
|
||||
(Kind::Cbz, "\"cbz\""),
|
||||
(Kind::Unknown, "\"unknown\""),
|
||||
];
|
||||
for (kind, wire) in cases {
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ Part of [catena](../README.md).
|
||||
| Field | Meaning |
|
||||
|---|---|
|
||||
| `path` | Where the file was read from, as given |
|
||||
| `kind` | `pdf`, `epub`, `djvu`, `azw3`, `mobi`, `txt`, `html`, `ole2`, `zip-wrapped-pdf`, `unknown` |
|
||||
| `kind` | `pdf`, `epub`, `djvu`, `azw3`, `mobi`, `lit`, `fb2`, `rtf`, `txt`, `html`, `ole2`, `cbz`, `zip-wrapped-pdf`, `zip-wrapped-fb2`, `unknown` |
|
||||
| `declared_ext` | The extension the name claimed, lowercased, without the dot |
|
||||
| `bytes` | Size on disk |
|
||||
| `pages` | Page count, for paginated formats |
|
||||
|
||||
Reference in new issue
Block a user