booksniff: RTF, Microsoft Reader, FictionBook and comic archives

Four formats a library of ebooks holds and this had no name for, so each
came back as text, as HTML or as unknown.

RTF is its opening group, after any byte-order mark and blank space.
Microsoft Reader is its signature at the front. A FictionBook is XML
whose root element is FictionBook, found by stepping over the
declaration, comments and doctype in the first four kilobytes. That walk
is over bytes rather than text, because a great many FictionBooks are
windows-1251 and would otherwise be called binary; other XML is still
HTML, as before.

A zip whose only real member is one FictionBook is zip-wrapped-fb2, the
same way a zipped PDF is zip-wrapped-pdf: handed to a converter as
FictionBook it is still a zip. A zip whose every real member is a JPEG,
PNG, GIF or WebP is cbz, once directories, ComicInfo.xml and desktop
clutter are passed over. One PDF or one readme among the pages and it is
not a comic. Only the names in the archive's directory are read, as for
every other zip.

The kind vocabulary grows by five: lit, fb2, rtf, cbz and
zip-wrapped-fb2.

Cargo.lock

@@ -101,7 +101,7 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
 
 [[package]]
 name = "avplan"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -126,7 +126,7 @@ checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
 
 [[package]]
 name = "bibclean"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -143,7 +143,7 @@ dependencies = [
 
 [[package]]
 name = "bibfetch"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -196,7 +196,7 @@ dependencies = [
 
 [[package]]
 name = "bookmeta"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -210,7 +210,7 @@ dependencies = [
 
 [[package]]
 name = "booksniff"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -250,7 +250,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "catena"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "bibclean",
  "bibfetch",
@@ -277,7 +277,7 @@ dependencies = [
 
 [[package]]
 name = "catena-cli"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "clap",
  "clap_mangen",
@@ -312,7 +312,7 @@ checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
 
 [[package]]
 name = "chunkmd"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -334,7 +334,7 @@ dependencies = [
 
 [[package]]
 name = "citefind"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "chunkmd",
@@ -399,7 +399,7 @@ dependencies = [
 
 [[package]]
 name = "colophon"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "bibclean",
  "catena-cli",
@@ -448,7 +448,7 @@ dependencies = [
 
 [[package]]
 name = "corpus-core"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "schemars",
  "serde",
@@ -492,7 +492,7 @@ dependencies = [
 
 [[package]]
 name = "cuekit"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "corpus-core",
  "regex",
@@ -504,7 +504,7 @@ dependencies = [
 
 [[package]]
 name = "cuenorm"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -514,7 +514,7 @@ dependencies = [
 
 [[package]]
 name = "cuepack"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -525,7 +525,7 @@ dependencies = [
 
 [[package]]
 name = "cueseg"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -612,7 +612,7 @@ dependencies = [
 
 [[package]]
 name = "emb"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -876,7 +876,7 @@ dependencies = [
 
 [[package]]
 name = "idnorm"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "bibclean",
  "catena-cli",
@@ -942,7 +942,7 @@ dependencies = [
 
 [[package]]
 name = "langid"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -1022,7 +1022,7 @@ dependencies = [
 
 [[package]]
 name = "mdscrub"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -1034,7 +1034,7 @@ dependencies = [
 
 [[package]]
 name = "mdsplit"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -1113,7 +1113,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
 
 [[package]]
 name = "pdfgutter"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -1126,7 +1126,7 @@ dependencies = [
 
 [[package]]
 name = "pdftriage"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -1370,7 +1370,7 @@ checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189"
 
 [[package]]
 name = "rrf"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -1517,7 +1517,7 @@ dependencies = [
 
 [[package]]
 name = "seriesof"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -1634,7 +1634,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "textkit"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "regex",
  "tiktoken-rs",
@@ -1724,7 +1724,7 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
 
 [[package]]
 name = "tocfind"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",

Cargo.toml

@@ -3,39 +3,39 @@ resolver = "3"
 members = ["crates/*"]
 
 [workspace.package]
-version = "0.2.2"
+version = "0.3.1"
 edition = "2024"
 rust-version = "1.85"
 license = "MPL-2.0"
 repository = "https://git.pumpin.online/funkpower/catena"
 
 [workspace.dependencies]
-catena-cli = { path = "crates/catena-cli", version = "0.2.2" }
+catena-cli = { path = "crates/catena-cli", version = "0.3.1" }
-corpus-core = { path = "crates/corpus-core", version = "0.2.2" }
+corpus-core = { path = "crates/corpus-core", version = "0.3.1" }
-cuekit = { path = "crates/cuekit", version = "0.2.2" }
+cuekit = { path = "crates/cuekit", version = "0.3.1" }
-textkit = { path = "crates/textkit", version = "0.2.2" }
+textkit = { path = "crates/textkit", version = "0.3.1" }
-chunkmd = { path = "crates/chunkmd", version = "0.2.2" }
+chunkmd = { path = "crates/chunkmd", version = "0.3.1" }
-mdscrub = { path = "crates/mdscrub", version = "0.2.2" }
+mdscrub = { path = "crates/mdscrub", version = "0.3.1" }
-mdsplit = { path = "crates/mdsplit", version = "0.2.2" }
+mdsplit = { path = "crates/mdsplit", version = "0.3.1" }
-pdftriage = { path = "crates/pdftriage", version = "0.2.2" }
+pdftriage = { path = "crates/pdftriage", version = "0.3.1" }
-cuenorm = { path = "crates/cuenorm", version = "0.2.2" }
+cuenorm = { path = "crates/cuenorm", version = "0.3.1" }
-cueseg = { path = "crates/cueseg", version = "0.2.2" }
+cueseg = { path = "crates/cueseg", version = "0.3.1" }
-rrf = { path = "crates/rrf", version = "0.2.2" }
+rrf = { path = "crates/rrf", version = "0.3.1" }
-cuepack = { path = "crates/cuepack", version = "0.2.2" }
+cuepack = { path = "crates/cuepack", version = "0.3.1" }
-bibclean = { path = "crates/bibclean", version = "0.2.2" }
+bibclean = { path = "crates/bibclean", version = "0.3.1" }
-avplan = { path = "crates/avplan", version = "0.2.2" }
+avplan = { path = "crates/avplan", version = "0.3.1" }
-pdfgutter = { path = "crates/pdfgutter", version = "0.2.2" }
+pdfgutter = { path = "crates/pdfgutter", version = "0.3.1" }
-catena = { path = "crates/bindings", version = "0.2.2" }
+catena = { path = "crates/bindings", version = "0.3.1" }
-seriesof = { path = "crates/seriesof", version = "0.2.2" }
+seriesof = { path = "crates/seriesof", version = "0.3.1" }
-idnorm = { path = "crates/idnorm", version = "0.2.2" }
+idnorm = { path = "crates/idnorm", version = "0.3.1" }
-bibfetch = { path = "crates/bibfetch", version = "0.2.2" }
+bibfetch = { path = "crates/bibfetch", version = "0.3.1" }
-bookmeta = { path = "crates/bookmeta", version = "0.2.2" }
+bookmeta = { path = "crates/bookmeta", version = "0.3.1" }
-langid = { path = "crates/langid", version = "0.2.2" }
+langid = { path = "crates/langid", version = "0.3.1" }
-tocfind = { path = "crates/tocfind", version = "0.2.2" }
+tocfind = { path = "crates/tocfind", version = "0.3.1" }
-booksniff = { path = "crates/booksniff", version = "0.2.2" }
+booksniff = { path = "crates/booksniff", version = "0.3.1" }
-citefind = { path = "crates/citefind", version = "0.2.2" }
+citefind = { path = "crates/citefind", version = "0.3.1" }
-colophon = { path = "crates/colophon", version = "0.2.2" }
+colophon = { path = "crates/colophon", version = "0.3.1" }
-emb = { path = "crates/emb", version = "0.2.2" }
+emb = { path = "crates/emb", version = "0.3.1" }
 
 clap = { version = "4", features = ["derive"] }
 clap_mangen = "0.3"

crates/bibfetch/src/ask.rs

@@ -63,13 +63,18 @@ pub fn lookup(
     for source in chain {
         match ask(agent, *source, id, kind, attempts) {
             Ok(record) => {
-                let complete = record.found && record.title.is_some() && !record.authors.is_empty();
+                match &mut best {
-                let better = best.as_ref().is_none_or(|kept| record.found && !kept.found);
+                    // Merged rather than replaced. The registers answer about
-                if complete {
+                    // different parts of one book: OpenLibrary's edition record
-                    return Ok(Some(record));
+                    // carries the publisher, the pagination and the language
+                    // and names its authors by key, while its search index
+                    // carries the authors and not the publisher. Keeping
+                    // whichever arrived first throws the other half away.
+                    Some(kept) => kept.fill_from(record),
+                    None => best = Some(record),
                 }
-                if better {
+                if best.as_ref().is_some_and(Record::complete) {
-                    best = Some(record);
+                    break;
                 }
             }
             Err(why) => last_error = Some(why),

crates/bibfetch/src/lib.rs

@@ -59,6 +59,21 @@ pub struct Record {
     /// The abstract, where the register carries one.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub summary: Option<String>,
+    /// The subject headings the register files it under, as it words them.
+    ///
+    /// A register states these for far more books than print a cataloguing
+    /// block, which makes them the second source a shelf otherwise lacks.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub subjects: Vec<String>,
+    /// The language it is written in, as the register gives it: a two- or
+    /// three-letter code, or a name. Not normalised here — what a register
+    /// said is the record, and deciding that "eng" and "English" are one
+    /// thing belongs to whoever is filing them.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub language: Option<String>,
+    /// How many pages the register says it has.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub pages: Option<u32>,
 }
 
 fn version() -> u32 {
@@ -112,6 +127,70 @@ pub fn missing(id: &str, kind: &str, source: Source) -> Record {
         year: None,
         container: None,
         summary: None,
+        subjects: Vec::new(),
+        language: None,
+        pages: None,
+    }
+}
+
+impl Record {
+    /// Whether this record says everything a catalogue wants.
+    ///
+    /// Used to decide when to stop asking. Title and authors are the minimum
+    /// that makes a record usable; the rest is worth another register only
+    /// while one is left to ask.
+    pub fn complete(&self) -> bool {
+        self.found
+            && self.title.is_some()
+            && !self.authors.is_empty()
+            && self.container.is_some()
+            && self.language.is_some()
+    }
+
+    /// Take from another record whatever this one does not have.
+    ///
+    /// The registers are complementary rather than redundant, and asking them
+    /// in turn and keeping one answer throws away the difference. OpenLibrary's
+    /// edition record gives the publisher, the pagination and the language but
+    /// names its authors by key; its search index gives the authors but not the
+    /// publisher. One book, two half-answers, and the whole of it only if they
+    /// are put together.
+    ///
+    /// Nothing already answered is overwritten, so the order of the chain is
+    /// the order of trust.
+    pub fn fill_from(&mut self, other: Record) {
+        if !other.found {
+            return;
+        }
+        self.found = true;
+        if self.title.is_none() {
+            self.title = other.title;
+        }
+        if self.authors.is_empty() {
+            self.authors = other.authors;
+        }
+        if self.year.is_none() {
+            self.year = other.year;
+        }
+        if self.container.is_none() {
+            self.container = other.container;
+        }
+        if self.summary.is_none() {
+            self.summary = other.summary;
+        }
+        if self.subjects.is_empty() {
+            self.subjects = other.subjects;
+        }
+        if self.language.is_none() {
+            self.language = other.language;
+        }
+        if self.pages.is_none() {
+            self.pages = other.pages;
+        }
+        // Which register answered becomes which registers did.
+        if !self.source.split('+').any(|s| s == other.source) {
+            self.source = format!("{}+{}", self.source, other.source);
+        }
     }
 }
 
@@ -135,3 +214,74 @@ mod tests {
         assert!(json.contains("\"found\":false"), "{json}");
     }
 }
+
+#[cfg(test)]
+mod merge_tests {
+    use super::*;
+
+    fn rec(source: &str) -> Record {
+        let mut r = missing("9780199205592", "isbn", Source::OpenLibrary);
+        r.source = source.to_string();
+        r.found = true;
+        r
+    }
+
+    #[test]
+    fn two_half_answers_make_one_record() {
+        // Exactly what the two OpenLibrary endpoints give for one book: the
+        // edition record has the publisher and the language and names its
+        // authors by key; the index has the authors and not the publisher.
+        let mut edition = rec("openlibrary");
+        edition.title = Some("The First World War: A Very Short Introduction".into());
+        edition.container = Some("Oxford University Press".into());
+        edition.language = Some("eng".into());
+
+        let mut index = rec("openlibrary");
+        index.title = Some("The First World War".into());
+        index.authors = vec!["Michael Howard".into()];
+        index.year = Some(2007);
+
+        edition.fill_from(index);
+        assert_eq!(edition.authors, ["Michael Howard"]);
+        assert_eq!(edition.year, Some(2007));
+        // The fuller title arrived first and is not displaced by the shorter.
+        assert_eq!(
+            edition.title.as_deref(),
+            Some("The First World War: A Very Short Introduction")
+        );
+    }
+
+    #[test]
+    fn a_register_that_has_nothing_contributes_nothing() {
+        let mut kept = rec("openlibrary");
+        kept.title = Some("A Book".into());
+        let absent = missing("9780199205592", "isbn", Source::GoogleBooks);
+        kept.fill_from(absent);
+        assert_eq!(kept.title.as_deref(), Some("A Book"));
+        assert_eq!(kept.source, "openlibrary");
+    }
+
+    #[test]
+    fn the_record_names_every_register_that_answered() {
+        let mut a = rec("openlibrary");
+        a.title = Some("A Book".into());
+        let mut b = rec("googlebooks");
+        b.found = true;
+        b.pages = Some(134);
+        a.fill_from(b);
+        assert_eq!(a.source, "openlibrary+googlebooks");
+        assert_eq!(a.pages, Some(134));
+    }
+
+    #[test]
+    fn complete_means_a_catalogue_has_what_it_needs() {
+        let mut r = rec("openlibrary");
+        assert!(!r.complete());
+        r.title = Some("A Book".into());
+        r.authors = vec!["Someone".into()];
+        assert!(!r.complete(), "still missing the publisher and language");
+        r.container = Some("A Press".into());
+        r.language = Some("eng".into());
+        assert!(r.complete());
+    }
+}

crates/bibfetch/src/source.rs

@@ -55,9 +55,17 @@ impl Source {
         match kind {
             "doi" => &[Source::Crossref],
             "arxiv" => &[Source::Arxiv],
+            // The index first. Asked for the fields by name it answers with
+            // the title, the authors, the publisher, the language, a page
+            // count and the subjects in one request, which is the whole of
+            // what a catalogue wants; the edition record is then needed only
+            // for a subtitle or a publisher the index did not give. Asking it
+            // second cost three requests per book where one usually does, and
+            // at three books in flight that is the difference between two
+            // days and two hours.
             "isbn" => &[
-                Source::OpenLibrary,
                 Source::OpenLibrarySearch,
+                Source::OpenLibrary,
                 Source::GoogleBooks,
             ],
             _ => &[],
@@ -75,9 +83,12 @@ impl Source {
             Source::OpenLibrary => {
                 format!("https://openlibrary.org/isbn/{}.json", encode(id))
             }
+            // `fields` or the index answers with a handful of defaults and
+            // keeps the rest, which is how a language it holds went unread.
             Source::OpenLibrarySearch => format!(
                 "https://openlibrary.org/search.json?q=isbn:{}\
-&fields=title,subtitle,author_name,first_publish_year,publisher&limit=1",
+&fields=title,subtitle,author_name,first_publish_year,publisher,language,subject,\
+number_of_pages_median&limit=1",
                 encode(id)
             ),
             Source::GoogleBooks => format!(
@@ -222,9 +233,24 @@ struct OpenLibraryBook {
     #[serde(default)]
     title: Option<String>,
     #[serde(default)]
+    subtitle: Option<String>,
+    #[serde(default)]
     publish_date: Option<String>,
     #[serde(default)]
     publishers: Vec<String>,
+    #[serde(default)]
+    subjects: Vec<String>,
+    /// `[{"key": "/languages/eng"}]`, which is how OpenLibrary names one.
+    #[serde(default)]
+    languages: Vec<OpenLibraryKey>,
+    #[serde(default)]
+    number_of_pages: Option<u32>,
+}
+
+#[derive(serde::Deserialize)]
+struct OpenLibraryKey {
+    #[serde(default)]
+    key: String,
 }
 
 fn open_library(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
@@ -232,10 +258,23 @@ fn open_library(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
         serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?;
 
     let mut record = blank(id, kind, Source::OpenLibrary);
-    record.title = book.title;
+    // An edition record keeps the subtitle in its own field, and a catalogue
+    // wants the whole title. Google Books is already joined the same way.
+    record.title = match (book.title, book.subtitle) {
+        (Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")),
+        (title, _) => title,
+    };
     record.container = book.publishers.into_iter().next();
     // The date is free text — "1999", "March 1999", "1999-03-01".
     record.year = book.publish_date.as_deref().and_then(year_in);
+    record.subjects = book.subjects;
+    // "/languages/eng" names the language by the last segment of a key.
+    record.language = book
+        .languages
+        .into_iter()
+        .find_map(|l| l.key.rsplit('/').next().map(str::to_string))
+        .filter(|l| !l.is_empty());
+    record.pages = book.number_of_pages;
     Ok(record)
 }
 
@@ -257,8 +296,21 @@ struct SearchDoc {
     first_publish_year: Option<i64>,
     #[serde(default)]
     publisher: Vec<String>,
+    #[serde(default)]
+    language: Vec<String>,
+    #[serde(default)]
+    subject: Vec<String>,
+    #[serde(rename = "number_of_pages_median", default)]
+    pages: Option<u32>,
 }
 
+/// How many of a work's subject headings are worth carrying.
+///
+/// The index lists every heading every edition has ever been given, which on a
+/// well-known book is hundreds and on Hamlet is thousands. A shelf is filed
+/// from a handful, and the first are the broadest.
+const SUBJECT_LIMIT: usize = 12;
+
 /// A search that matched nothing answers with no documents rather than a
 /// status, so an empty reply is the absence and not a failure.
 fn open_library_search(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
@@ -275,6 +327,12 @@ fn open_library_search(id: &str, kind: &str, body: &str) -> Result<Record, Error
     record.authors = doc.author_name;
     record.year = doc.first_publish_year;
     record.container = doc.publisher.into_iter().next();
+    record.language = doc.language.into_iter().next();
+    // The index lists every subject any edition of the work carries, which
+    // runs to hundreds on a well-known book. A shelf needs a handful.
+    record.subjects = doc.subject;
+    record.subjects.truncate(SUBJECT_LIMIT);
+    record.pages = doc.pages;
     Ok(record)
 }
 
@@ -304,6 +362,12 @@ struct GoogleVolume {
     publisher: Option<String>,
     #[serde(default)]
     description: Option<String>,
+    #[serde(default)]
+    categories: Vec<String>,
+    #[serde(rename = "language", default)]
+    language: Option<String>,
+    #[serde(rename = "pageCount", default)]
+    page_count: Option<u32>,
 }
 
 /// A search that matched nothing answers with no items rather than a status,
@@ -324,6 +388,9 @@ fn google_books(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
     record.year = volume.published_date.as_deref().and_then(year_in);
     record.container = volume.publisher;
     record.summary = volume.description.map(|text| collapse(&text));
+    record.subjects = volume.categories;
+    record.language = volume.language;
+    record.pages = volume.page_count;
     Ok(record)
 }
 
@@ -379,7 +446,10 @@ mod tests {
     fn each_kind_goes_to_the_body_that_assigns_it() {
         assert_eq!(Source::for_kind("doi"), Some(Source::Crossref));
         assert_eq!(Source::for_kind("arxiv"), Some(Source::Arxiv));
-        assert_eq!(Source::for_kind("isbn"), Some(Source::OpenLibrary));
+        // The index, not the edition record: it answers with the whole of
+        // what a catalogue wants in one request, where the edition record
+        // takes up to twenty-eight seconds and names its authors by key.
+        assert_eq!(Source::for_kind("isbn"), Some(Source::OpenLibrarySearch));
         // No register answers about an ISSN without a key.
         assert_eq!(Source::for_kind("issn"), None);
     }

crates/booksniff/README.md

@@ -26,8 +26,8 @@ file too small to be a book, a name that says "sample".
   cover images that ended up in a folder of PDFs.
 
 A library accumulates files that lie: a DjVu named `.pdf`, a Word document
-named `.epub`, an EPUB saved as `.zip`, a single PDF that somebody zipped and
+named `.epub`, an RTF saved as `.doc`, an EPUB saved as `.zip`, a single PDF
-renamed back to `.pdf`.
+that somebody zipped and renamed back to `.pdf`.
 
 ## Install
 
@@ -89,7 +89,12 @@ one bad file does not stop a sweep.
 | `ole2` | The compound-file header every binary Word document starts with |
 | `epub` | A zip holding `mimetype` and `META-INF/container.xml` |
 | `zip-wrapped-pdf` | A zip whose only real member is one PDF |
+| `zip-wrapped-fb2` | A zip whose only real member is one FictionBook — the `.fb2.zip` most of them are passed around as |
+| `cbz` | A zip whose every real member is a page image: JPEG, PNG, GIF or WebP |
 | `azw3`, `mobi` | A Palm database tagged `BOOKMOBI`, told apart by the format version inside the first record |
+| `lit` | `ITOLITLS`, Microsoft Reader's signature |
+| `rtf` | `{\rtf` at the front, after any byte-order mark and blank space |
+| `fb2` | XML whose root element is `FictionBook`, found by stepping over the declaration, comments and doctype in the first four kilobytes |
 | `html` | Markup at the front, after any byte-order mark |
 | `txt` | Text that is none of the above |
 | `unknown` | Anything else, with a note saying what was seen |
@@ -98,6 +103,20 @@ An archive holding several PDFs is a collection, not a wrapped book: choosing
 among them would be a guess, and naming one chapter as the whole work is worse
 than refusing.
 
+A zipped FictionBook is reported as wrapped rather than as `fb2`, the same way
+a zipped PDF is: a converter handed it as FictionBook is handed a zip, and the
+kind is there to stop exactly that.
+
+A comic is the pages and nothing else. Directories, the `ComicInfo.xml` a
+comic reader writes into the archive, and the desktop clutter every archive
+picks up are passed over; anything more — one PDF among the pages, a readme —
+and the zip is not a comic, and says what it holds instead. Only names are
+read, out of the archive's directory, so no member is ever unpacked.
+
+FictionBook is read as bytes rather than as text. A great many are written in
+windows-1251, which spells its markup in ASCII but is not UTF-8, and would
+otherwise be called binary.
+
 ## As a library
 
 ```rust

crates/booksniff/src/fb2.rs

@@ -0,0 +1,162 @@
+//! Telling a FictionBook from any other XML.
+//!
+//! A FictionBook is an XML document whose root element is `FictionBook`.
+//! The root comes after the prolog — a declaration, comments, perhaps a
+//! doctype — so it is found by stepping over those rather than by matching
+//! at a fixed offset.
+//!
+//! The walk is over bytes, not text. A great many FictionBooks are written in
+//! windows-1251, which is not UTF-8 but spells its markup in ASCII all the
+//! same, and a reader that insisted on UTF-8 would call every one of them
+//! binary.
+
+use crate::BOM;
+
+/// What the root element of a FictionBook is called.
+const ROOT: &[u8] = b"FictionBook";
+
+/// Whether the front of a file is the start of a FictionBook.
+///
+/// `head` is however much of the front the caller read. A root element that
+/// does not begin inside it is not found, so the answer is no rather than a
+/// guess.
+pub fn is_fictionbook(head: &[u8]) -> bool {
+    root_element(head).is_some_and(|name| local_name(name) == ROOT)
+}
+
+/// The name of the first element, after the prolog.
+fn root_element(head: &[u8]) -> Option<&[u8]> {
+    let mut rest = head.strip_prefix(BOM).unwrap_or(head);
+    loop {
+        rest = rest.trim_ascii_start();
+        if let Some(after) = rest.strip_prefix(b"<?") {
+            rest = after_marker(after, b"?>")?;
+        } else if let Some(after) = rest.strip_prefix(b"<!--") {
+            rest = after_marker(after, b"-->")?;
+        } else if let Some(after) = rest.strip_prefix(b"<!") {
+            rest = after_declaration(after)?;
+        } else {
+            // Anything but an element here is text before the root, which
+            // XML does not allow.
+            let after = rest.strip_prefix(b"<")?;
+            // A name that runs to the end of what was read may be the front
+            // of a longer one, so it is not taken.
+            let end = after.iter().position(|byte| !is_name_byte(*byte))?;
+            let ends_properly =
+                matches!(after[end], b'>' | b'/') || after[end].is_ascii_whitespace();
+            return (end > 0 && ends_properly).then_some(&after[..end]);
+        }
+    }
+}
+
+/// What follows the first `marker`, if it is there at all.
+fn after_marker<'a>(rest: &'a [u8], marker: &[u8]) -> Option<&'a [u8]> {
+    rest.windows(marker.len())
+        .position(|window| window == marker)
+        .map(|at| &rest[at + marker.len()..])
+}
+
+/// What follows a doctype, whose internal subset may hold `>` of its own.
+fn after_declaration(rest: &[u8]) -> Option<&[u8]> {
+    let mut depth = 0usize;
+    let mut quote = None;
+    for (at, &byte) in rest.iter().enumerate() {
+        match quote {
+            Some(open) if byte == open => quote = None,
+            Some(_) => {}
+            None => match byte {
+                b'"' | b'\'' => quote = Some(byte),
+                b'[' => depth += 1,
+                b']' => depth = depth.saturating_sub(1),
+                b'>' if depth == 0 => return Some(&rest[at + 1..]),
+                _ => {}
+            },
+        }
+    }
+    None
+}
+
+/// The name without its namespace prefix.
+fn local_name(name: &[u8]) -> &[u8] {
+    match name.iter().rposition(|byte| *byte == b':') {
+        Some(at) => &name[at + 1..],
+        None => name,
+    }
+}
+
+fn is_name_byte(byte: u8) -> bool {
+    byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.' | b':') || byte >= 0x80
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const BOOK: &[u8] = b"<?xml version=\"1.0\" encoding=\"utf-8\"?>\n\
+        <FictionBook xmlns=\"http://www.gribuser.ru/xml/fictionbook/2.0\">\n\
+        <description><title-info><book-title>The Mill</book-title></title-info></description>";
+
+    #[test]
+    fn a_fictionbook_is_recognised() {
+        assert!(is_fictionbook(BOOK));
+        assert!(is_fictionbook(b"<FictionBook>"));
+    }
+
+    #[test]
+    fn the_prolog_is_stepped_over() {
+        let head = b"\xef\xbb\xbf  <?xml version=\"1.0\"?>\r\n\
+            <!-- converted by hand -->\n\
+            <!DOCTYPE FictionBook [ <!ENTITY mill \"<b>the mill</b>\"> ]>\n\
+            <?xml-stylesheet href=\"fb2.css\"?>\n\
+            <FictionBook\n xmlns=\"http://www.gribuser.ru/xml/fictionbook/2.0\">";
+        assert!(is_fictionbook(head));
+    }
+
+    #[test]
+    fn a_prefixed_root_is_still_a_fictionbook() {
+        assert!(is_fictionbook(
+            b"<fb:FictionBook xmlns:fb=\"http://www.gribuser.ru/xml/fictionbook/2.0\">"
+        ));
+    }
+
+    #[test]
+    fn markup_need_not_be_utf8() {
+        // windows-1251 for "Мельница" in the title.
+        let head = b"<?xml version=\"1.0\" encoding=\"windows-1251\"?>\
+            <FictionBook><description><book-title>\xcc\xe5\xeb\xfc\xed\xe8\xf6\xe0</book-title>";
+        assert!(is_fictionbook(head));
+    }
+
+    #[test]
+    fn other_xml_is_not_a_fictionbook() {
+        assert!(!is_fictionbook(
+            b"<?xml version=\"1.0\"?>\n<rss version=\"2.0\">"
+        ));
+        assert!(!is_fictionbook(b"<?xml version=\"1.0\"?><FictionBookish>"));
+        assert!(!is_fictionbook(b"<html><FictionBook>"));
+        assert!(!is_fictionbook(b"text before <FictionBook>"));
+        assert!(!is_fictionbook(b""));
+    }
+
+    #[test]
+    fn a_root_the_window_cut_short_is_not_taken() {
+        assert!(!is_fictionbook(b"<?xml version=\"1.0\"?><FictionBook"));
+        assert!(!is_fictionbook(b"<?xml version=\"1.0\"?><Fiction"));
+        assert!(!is_fictionbook(b"<?xml version=\"1.0\""));
+        assert!(!is_fictionbook(b"<!-- never closed <FictionBook>"));
+    }
+
+    #[test]
+    fn a_truncated_or_mangled_prolog_does_not_panic() {
+        let head = b"\xef\xbb\xbf<?xml?><!--c--><!DOCTYPE x [ <!ENTITY a '>'> ] \"]\">\
+            <FictionBook>";
+        assert!(is_fictionbook(head));
+        for cut in 0..head.len() {
+            let _ = is_fictionbook(&head[..cut]);
+        }
+        for byte in 0..=255u8 {
+            let _ = is_fictionbook(&[b'<', byte]);
+            let _ = is_fictionbook(&[b'<', b'!', byte, b'>']);
+        }
+    }
+}

crates/booksniff/src/lib.rs

@@ -9,6 +9,7 @@
 //! The containers are unambiguous, so they are read rather than guessed at.
 
 pub mod bytes;
+pub mod fb2;
 pub mod mobi;
 pub mod zip;
 
@@ -22,17 +23,32 @@ const DJVU: &[u8] = b"AT&TFORM";
 const OLE2: &[u8] = &[0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1];
 /// A PDF announces its version in its first bytes.
 const PDF: &[u8] = b"%PDF-";
+/// Microsoft Reader's signature, ahead of everything else in the file.
+const LIT: &[u8] = b"ITOLITLS";
+/// The group every RTF document is, opened with the control word that names
+/// the format.
+const RTF: &[u8] = br"{\rtf";
+/// The UTF-8 byte-order mark.
+const BOM: &[u8] = b"\xef\xbb\xbf";
 
 /// Archive members that carry no content: resource forks and desktop clutter.
 const ARCHIVE_NOISE: &[&str] = &["__macosx/", ".ds_store", "thumbs.db"];
 
+/// What a comic book archive carries besides its pages: the catalogue entry
+/// comic readers write into it.
+const COMIC_NOISE: &[&str] = &["comicinfo.xml"];
+
+/// The extensions of the images a comic book archive is made of.
+const PAGE_IMAGES: &[&str] = &[".jpg", ".jpeg", ".png", ".gif", ".webp"];
+
 /// Substrings in a name that mark a file as a sample rather than a book.
 const SAMPLE_MARKERS: &[&str] = &["sample", "readme", "read me"];
 
 /// Below this, a file is too small to be a book whatever it claims to be.
 const SMALLEST_BOOK: u64 = 4096;
 
-/// How much of the front is read to tell text from markup.
+/// How much of the front is read to tell RTF, FictionBook, markup and text
+/// apart.
 const TEXT_WINDOW: usize = 4096;
 
 /// Identify a file.
@@ -49,12 +65,17 @@ pub fn sniff(source: &impl Bytes, name: &str) -> Doc {
         Kind::Pdf
     } else if source.magic_at(0, OLE2) {
         Kind::Ole2
+    } else if source.magic_at(0, LIT) {
+        Kind::Lit
     } else if source.magic_at(0, zip::LOCAL_HEADER) {
         archive(source, &mut doc)
     } else if let Some(kind) = mobi::kind(source) {
         kind
     } else {
-        text(source, &mut doc)
+        match source.slice(0, TEXT_WINDOW) {
+            Ok(head) => front(&head, &mut doc),
+            Err(_) => Kind::Unknown,
+        }
     };
 
     if doc.kind == Kind::Unknown {
@@ -80,7 +101,8 @@ pub fn sniff(source: &impl Bytes, name: &str) -> Doc {
     doc
 }
 
-/// What is inside a zip: an EPUB, a wrapped PDF, or a collection of pages.
+/// What is inside a zip: an EPUB, a wrapped book, a comic, or a collection of
+/// pages.
 fn archive(source: &impl Bytes, doc: &mut Doc) -> Kind {
     let Some(names) = zip::members(source) else {
         doc.note("a zip whose directory could not be read");
@@ -99,6 +121,13 @@ fn archive(source: &impl Bytes, doc: &mut Doc) -> Kind {
         doc.note(format!("a zip holding one PDF, {pdf:?}"));
         return Kind::ZipWrappedPdf;
     }
+    if let Some(fb2) = sole_member(&names, ".fb2") {
+        doc.note(format!("a zip holding one FictionBook, {fb2:?}"));
+        return Kind::ZipWrappedFb2;
+    }
+    if is_comic(&names) {
+        return Kind::Cbz;
+    }
     if lower
         .iter()
         .any(|name| name.ends_with(".htm") || name.ends_with(".html") || name.ends_with(".xhtml"))
@@ -119,25 +148,67 @@ fn archive(source: &impl Bytes, doc: &mut Doc) -> Kind {
 /// a collection, and choosing among them would be a guess — better to say it
 /// is not a book than to name one chapter as the whole.
 pub fn sole_pdf(names: &[String]) -> Option<&str> {
-    let real: Vec<&String> = names
+    sole_member(names, ".pdf")
+}
+
+/// The one member an archive exists to carry, if it is the only real member
+/// and its name ends in `extension`.
+fn sole_member<'a>(names: &'a [String], extension: &str) -> Option<&'a str> {
+    match real_members(names).as_slice() {
+        [only] if only.to_lowercase().ends_with(extension) => Some(only.as_str()),
+        _ => None,
+    }
+}
+
+/// Whether an archive is the pages of a comic and nothing else.
+///
+/// Every real member has to be an image. One PDF or one text file among the
+/// pages makes it something else, and saying what would be a guess.
+fn is_comic(names: &[String]) -> bool {
+    let pages: Vec<String> = real_members(names)
+        .into_iter()
+        .map(|name| name.to_lowercase())
+        .filter(|name| {
+            let file = name.rsplit('/').next().unwrap_or(name);
+            !COMIC_NOISE.contains(&file)
+        })
+        .collect();
+    !pages.is_empty()
+        && pages
+            .iter()
+            .all(|name| PAGE_IMAGES.iter().any(|image| name.ends_with(image)))
+}
+
+/// The members that carry content: no directories, no desktop clutter.
+fn real_members(names: &[String]) -> Vec<&String> {
+    names
         .iter()
         .filter(|name| !name.ends_with('/'))
         .filter(|name| {
             let lower = name.to_lowercase();
             !ARCHIVE_NOISE.iter().any(|noise| lower.contains(noise))
         })
-        .collect();
+        .collect()
-    match real.as_slice() {
+}
-        [only] if only.to_lowercase().ends_with(".pdf") => Some(only.as_str()),
+
-        _ => None,
+/// What the front of a file says when it is not a container: RTF, a
+/// FictionBook, markup, text, or nothing this knows.
+fn front(head: &[u8], doc: &mut Doc) -> Kind {
+    // Both are recognised on bytes rather than text. RTF is seven-bit by
+    // design but not always in practice, and a FictionBook is as often
+    // windows-1251 as UTF-8.
+    let lead = head.strip_prefix(BOM).unwrap_or(head).trim_ascii_start();
+    if lead.starts_with(RTF) {
+        return Kind::Rtf;
     }
+    if fb2::is_fictionbook(head) {
+        return Kind::Fb2;
+    }
+    text(head, doc)
 }
 
 /// Text, markup, or nothing this knows.
-fn text(source: &impl Bytes, doc: &mut Doc) -> Kind {
+fn text(head: &[u8], doc: &mut Doc) -> Kind {
-    let Ok(head) = source.slice(0, TEXT_WINDOW) else {
-        return Kind::Unknown;
-    };
     if head.is_empty() {
         doc.note("the file is empty");
         return Kind::Unknown;
@@ -149,7 +220,7 @@ fn text(source: &impl Bytes, doc: &mut Doc) -> Kind {
     }
     // The window may have cut a character in half, which is not the same
     // thing as the file not being text.
-    let head = match std::str::from_utf8(&head) {
+    let head = match std::str::from_utf8(head) {
         Ok(text) => text,
         Err(e) if e.error_len().is_none() => {
             std::str::from_utf8(&head[..e.valid_up_to()]).expect("a valid prefix")
@@ -178,10 +249,15 @@ fn names_the_same_thing(extension: &str, kind: Kind) -> bool {
         Kind::Djvu => matches!(extension, "djvu" | "djv"),
         Kind::Azw3 => matches!(extension, "azw3" | "azw" | "kfx"),
         Kind::Mobi => matches!(extension, "mobi" | "prc" | "azw"),
+        Kind::Lit => extension == "lit",
+        Kind::Fb2 => extension == "fb2",
+        Kind::Rtf => extension == "rtf",
         Kind::Txt => matches!(extension, "txt" | "text" | "md" | "markdown"),
         Kind::Html => matches!(extension, "html" | "htm" | "xhtml" | "xml"),
         Kind::Ole2 => matches!(extension, "doc" | "xls" | "ppt"),
         Kind::ZipWrappedPdf => extension == "zip",
+        Kind::ZipWrappedFb2 => matches!(extension, "zip" | "fbz"),
+        Kind::Cbz => extension == "cbz",
         // Nothing was recognised, so there is nothing to disagree with.
         Kind::Unknown => true,
     }
@@ -194,10 +270,15 @@ fn spelling(kind: Kind) -> &'static str {
         Kind::Djvu => "DjVu",
         Kind::Azw3 => "AZW3",
         Kind::Mobi => "MOBI",
+        Kind::Lit => "Microsoft Reader",
+        Kind::Fb2 => "FictionBook",
+        Kind::Rtf => "RTF",
         Kind::Txt => "plain text",
         Kind::Html => "HTML",
         Kind::Ole2 => "an OLE2 document",
         Kind::ZipWrappedPdf => "a zip holding one PDF",
+        Kind::ZipWrappedFb2 => "a zip holding one FictionBook",
+        Kind::Cbz => "a comic book archive",
         Kind::Unknown => "nothing recognisable",
     }
 }
@@ -332,4 +413,164 @@ mod tests {
         head.truncate(TEXT_WINDOW + 1);
         assert_eq!(sniff(&head.as_slice(), "a.txt").kind, Kind::Txt);
     }
+
+    fn archive_of(names: &[&str], name: &str) -> Doc {
+        sniff(&zip::tests::zip(names).as_slice(), name)
+    }
+
+    #[test]
+    fn rtf_is_recognised_by_its_opening_group() {
+        assert_eq!(kind_of(br"{\rtf1\ansi\deff0 {\fonttbl", "a.rtf"), Kind::Rtf);
+        assert_eq!(
+            kind_of(b"\xef\xbb\xbf\r\n  {\\rtf1\\ansi", "a.rtf"),
+            Kind::Rtf
+        );
+        // Eight-bit text in the body does not stop it being RTF.
+        assert_eq!(kind_of(b"{\\rtf1 caf\xe9 \x00", "a.rtf"), Kind::Rtf);
+    }
+
+    #[test]
+    fn rtf_cut_short_of_its_control_word_is_not_rtf() {
+        assert_eq!(sniff(&br"{\rt".as_slice(), "a.rtf").kind, Kind::Txt);
+        assert_eq!(kind_of(br"{\rtx1", "a.rtf"), Kind::Txt);
+        assert_eq!(kind_of(br"text {\rtf1", "a.rtf"), Kind::Txt);
+    }
+
+    #[test]
+    fn rtf_named_as_a_word_document_is_remarked_on() {
+        let doc = sniff(&padded(br"{\rtf1\ansi").as_slice(), "letter.doc");
+        assert_eq!(doc.kind, Kind::Rtf);
+        assert!(
+            doc.notes
+                .iter()
+                .any(|note| note == "named .doc, but the bytes say RTF"),
+            "{:?}",
+            doc.notes
+        );
+    }
+
+    #[test]
+    fn microsoft_reader_is_recognised_by_its_signature() {
+        let doc = sniff(&padded(b"ITOLITLS\x01\x00\x00\x00").as_slice(), "book.lit");
+        assert_eq!(doc.kind, Kind::Lit);
+        assert!(doc.notes.is_empty(), "{:?}", doc.notes);
+        assert_eq!(kind_of(b"ITOLITL", "book.lit"), Kind::Txt);
+        assert_eq!(kind_of(b" ITOLITLS", "book.lit"), Kind::Txt);
+    }
+
+    #[test]
+    fn a_fictionbook_is_recognised_and_other_xml_is_not() {
+        let book = b"<?xml version=\"1.0\" encoding=\"windows-1251\"?>\n\
+            <FictionBook xmlns=\"http://www.gribuser.ru/xml/fictionbook/2.0\">\
+            <book-title>\xcc\xe5\xeb\xfc\xed\xe8\xf6\xe0</book-title>";
+        let doc = sniff(&padded(book).as_slice(), "mill.fb2");
+        assert_eq!(doc.kind, Kind::Fb2);
+        assert!(doc.notes.is_empty(), "{:?}", doc.notes);
+
+        let feed = b"<?xml version=\"1.0\"?>\n<rss version=\"2.0\"><channel>";
+        assert_eq!(kind_of(feed, "feed.xml"), Kind::Html);
+    }
+
+    #[test]
+    fn a_fictionbook_named_as_something_else_is_remarked_on() {
+        let doc = sniff(&padded(b"<FictionBook>").as_slice(), "mill.txt");
+        assert_eq!(doc.kind, Kind::Fb2);
+        assert!(
+            doc.notes.iter().any(|note| note.contains("FictionBook")),
+            "{:?}",
+            doc.notes
+        );
+    }
+
+    #[test]
+    fn a_zip_holding_one_fictionbook_is_a_wrapped_one() {
+        let doc = archive_of(&["__MACOSX/._mill.fb2", "mill.fb2"], "mill.fb2.zip");
+        assert_eq!(doc.kind, Kind::ZipWrappedFb2);
+        assert_eq!(doc.declared_ext.as_deref(), Some("zip"));
+        assert!(
+            doc.notes
+                .iter()
+                .any(|note| note.contains("one FictionBook")),
+            "{:?}",
+            doc.notes
+        );
+        assert!(!doc.notes.iter().any(|note| note.contains("named .")));
+
+        let two = archive_of(&["one.fb2", "two.fb2"], "pair.zip");
+        assert_eq!(two.kind, Kind::Unknown);
+    }
+
+    #[test]
+    fn a_zip_of_page_images_is_a_comic() {
+        let doc = archive_of(
+            &[
+                "Issue 1/",
+                "Issue 1/001.jpg",
+                "Issue 1/002.JPEG",
+                "Issue 1/003.png",
+                "Issue 1/004.gif",
+                "Issue 1/005.webp",
+                "ComicInfo.xml",
+                "__MACOSX/Issue 1/._001.jpg",
+                "Issue 1/Thumbs.db",
+                ".DS_Store",
+            ],
+            "issue-1.cbz",
+        );
+        assert_eq!(doc.kind, Kind::Cbz);
+        assert!(
+            !doc.notes.iter().any(|note| note.contains("named .")),
+            "{:?}",
+            doc.notes
+        );
+    }
+
+    #[test]
+    fn a_comic_saved_as_a_zip_is_remarked_on() {
+        let doc = archive_of(&["001.jpg", "002.jpg"], "issue-1.zip");
+        assert_eq!(doc.kind, Kind::Cbz);
+        assert!(
+            doc.notes.iter().any(|note| note.contains("comic")),
+            "{:?}",
+            doc.notes
+        );
+    }
+
+    #[test]
+    fn a_zip_with_anything_but_images_is_not_a_comic() {
+        let one_pdf = archive_of(&["001.jpg", "002.jpg", "extras.pdf"], "issue-1.cbz");
+        assert_eq!(one_pdf.kind, Kind::Unknown);
+        assert!(
+            one_pdf
+                .notes
+                .iter()
+                .any(|note| note.contains("none of them a book")),
+            "{:?}",
+            one_pdf.notes
+        );
+
+        let readme = archive_of(&["001.jpg", "readme.txt"], "issue-1.cbz");
+        assert_eq!(readme.kind, Kind::Unknown);
+
+        // A name that ends like an image without being one.
+        let lookalike = archive_of(&["001.jpg", "notes.jpg.txt"], "issue-1.cbz");
+        assert_eq!(lookalike.kind, Kind::Unknown);
+    }
+
+    #[test]
+    fn a_zip_with_no_pages_is_not_a_comic() {
+        let catalogue_only = archive_of(&["ComicInfo.xml", "pages/"], "issue-1.cbz");
+        assert_eq!(catalogue_only.kind, Kind::Unknown);
+
+        let empty = archive_of(&[], "issue-1.cbz");
+        assert_eq!(empty.kind, Kind::Unknown);
+    }
+
+    #[test]
+    fn a_truncated_comic_does_not_panic() {
+        let whole = zip::tests::zip(&["001.jpg", "002.png", "ComicInfo.xml"]);
+        for cut in 0..whole.len() {
+            let _ = sniff(&&whole[..cut], "issue-1.cbz");
+        }
+    }
 }

crates/booksniff/src/zip.rs

@@ -78,11 +78,11 @@ fn directory_start(source: &impl Bytes) -> Option<(usize, u64)> {
 }
 
 #[cfg(test)]
-mod tests {
+pub(crate) mod tests {
     use super::*;
 
     /// Build a zip holding `names`, each with empty stored content.
-    fn zip(names: &[&str]) -> Vec<u8> {
+    pub(crate) fn zip(names: &[&str]) -> Vec<u8> {
         let mut out: Vec<u8> = Vec::new();
         let mut offsets = Vec::new();
         for name in names {

crates/booksniff/tests/cli.rs

@@ -64,6 +64,30 @@ fn several_files_come_back_as_several_records() {
     assert_eq!(kinds, ["\"pdf\"", "\"djvu\""]);
 }
 
+#[test]
+fn the_formats_read_from_the_front_come_back_by_their_wire_names() {
+    let rtf = written("letter.rtf", &padded(br"{\rtf1\ansi"));
+    let lit = written("book.lit", &padded(b"ITOLITLS"));
+    let fb2 = written(
+        "mill.fb2",
+        &padded(b"<?xml version=\"1.0\"?>\n<FictionBook xmlns=\"http://www.gribuser.ru/xml/fictionbook/2.0\">"),
+    );
+    let out = run(&[
+        rtf.to_str().expect("utf-8 path"),
+        lit.to_str().expect("utf-8 path"),
+        fb2.to_str().expect("utf-8 path"),
+    ]);
+    assert_eq!(out.status.code(), Some(0));
+    let kinds: Vec<String> = String::from_utf8(out.stdout)
+        .expect("utf-8")
+        .lines()
+        .map(|line| {
+            serde_json::from_str::<serde_json::Value>(line).expect("valid JSON")["kind"].to_string()
+        })
+        .collect();
+    assert_eq!(kinds, ["\"rtf\"", "\"lit\"", "\"fb2\""]);
+}
+
 #[test]
 fn a_file_that_is_not_there_is_an_io_error_and_the_rest_still_run() {
     let path = written("kept.pdf", &padded(b"%PDF-1.7"));

crates/colophon/src/title.rs

@@ -35,6 +35,29 @@ static LIFESPAN: LazyLock<Regex> = LazyLock::new(|| {
 static BARE_DATE: LazyLock<Regex> =
     LazyLock::new(|| Regex::new(r"(?i)^\d{1,4}\s*(bc|ad|bce|ce)?\.?$").expect("a valid pattern"));
 /// `Copyright © 2013 by`, `\u{a9} 2013 Richard Swinburne`, `Copyright 1971 by`.
+/// `1. General Remarks`, `2   James W. McAllister` — a numbered heading.
+static NUMBERED_HEADING: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"^\s*\d+\s*[.)]\s+\S|^\s*\d+\s{2,}\S").expect("a valid pattern"));
+/// `5222 What Is This`, `827 North Fourth Street` — a number that is not a
+/// year, leading a line.
+static LEADING_NUMBER: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"^\s*(\d{1,5})\s+\S").expect("a valid pattern"));
+/// `Page xxxvi`, `Chapter I`, `Fig. 3` — a marker, not a name.
+static MARKER: LazyLock<Regex> = LazyLock::new(|| {
+    Regex::new(
+        r"(?i)^\s*(page|pages|p\.|pp\.|fig\.|figure|plate|chapter|ch\.|section|part)\s+[\divxlcm]",
+    )
+    .expect("a valid pattern")
+});
+/// `Sidebottom, Harry. Ancient warfare` — a bibliography entry, which front
+/// matter prints in its cataloguing block and in its further reading.
+static CATALOGUE_ENTRY: LazyLock<Regex> = LazyLock::new(|| {
+    Regex::new(r"^[A-Z][A-Za-z'\-]+,\s+[A-Z][A-Za-z'.\-]*\.\s+\S").expect("a valid pattern")
+});
+/// `To Mackenzie,` — a dedication, set alone on its own page.
+static DEDICATION: LazyLock<Regex> = LazyLock::new(|| {
+    Regex::new(r"(?i)^\s*(to|for|in memory of)\s+\S.*,\s*$").expect("a valid pattern")
+});
 static COPYRIGHT_NOTICE: LazyLock<Regex> = LazyLock::new(|| {
     // The year need not follow the symbol directly: "(c) Alfredo Ferrarin
     // 2004" puts the holder between them, and was read as a title.
@@ -79,10 +102,13 @@ copenhagen|stockholm|moscow|beijing|tokyo|delhi|nairobi)\b[\s,·•|and]*",
 static STRUCTURAL: &[&str] = &[
     "abbreviations",
     "about the author",
+    "abstract",
     "acknowledgements",
     "acknowledgments",
     "afterword",
     "appendix",
+    "author",
+    "authors",
     "bibliography",
     "contents",
     "dedication",
@@ -95,6 +121,7 @@ static STRUCTURAL: &[&str] = &[
     "illustrations",
     "index",
     "introduction",
+    "keywords",
     "list of contributors",
     "list of illustrations",
     "maps",
@@ -102,7 +129,9 @@ static STRUCTURAL: &[&str] = &[
     "postscript",
     "preface",
     "prologue",
+    "subtitle",
     "table of contents",
+    "title",
     "title page",
     "translator's note",
 ];
@@ -182,6 +211,22 @@ fn is_furniture(line: &str) -> bool {
     }
     // Sentences a copyright page sets, which are not notices and so not caught
     // by the notice pattern.
+    if NUMBERED_HEADING.is_match(t) || MARKER.is_match(t) || CATALOGUE_ENTRY.is_match(t) {
+        return true;
+    }
+    // A line opening with a number that is not a year. "5222 What Is This" and
+    // "827 North Fourth Street" are a stray folio and an address; "1984
+    // Revisited" is a book, and the difference is whether the number could be
+    // the year of one.
+    if let Some(c) = LEADING_NUMBER.captures(t) {
+        let n: u32 = c[1].parse().unwrap_or(0);
+        if !(1000..=2099).contains(&n) {
+            return true;
+        }
+    }
+    if DEDICATION.is_match(t) {
+        return true;
+    }
     let lower = t.to_lowercase();
     for phrase in [
         "moral rights of the author",
@@ -496,3 +541,54 @@ mod conversion_tests {
         }
     }
 }
+
+#[cfg(test)]
+mod review_pile_tests {
+    use super::*;
+
+    /// Everything here was proposed as a title by a real run over this
+    /// library, and sat in the review queue waiting for a person to say no.
+    #[test]
+    fn what_the_review_pile_was_full_of() {
+        for line in [
+            "1. General Remarks",
+            "2   James W. McAllister",
+            "5222 What Is This",
+            "827 North Fourth Street",
+            "Page xxxvi",
+            "Page xxxviii",
+            "Chapter I",
+            "title",
+            "Sidebottom, Harry. Ancient warfare",
+            "To Mackenzie,",
+        ] {
+            assert!(
+                is_structural(line) || is_furniture(line),
+                "still read as a title: {line:?}"
+            );
+        }
+    }
+
+    /// And none of it catches a book. The numbers are the delicate part: a
+    /// year can open a title, and a folio cannot.
+    #[test]
+    fn a_book_is_still_a_book() {
+        for line in [
+            "1984 Revisited",
+            "1914: The Year the World Ended",
+            "Chapter and Verse",
+            "Part of the Furniture",
+            "To Kill a Mockingbird",
+            "For Whom the Bell Tolls",
+            "The Title of This Book",
+            "Pages from a Cold Island",
+            "Sections of a Broken Whole",
+            "In Memory of Bread",
+        ] {
+            assert!(
+                !(is_structural(line) || is_furniture(line)),
+                "rejected a real title: {line:?}"
+            );
+        }
+    }
+}

crates/corpus-core/src/doc.rs

@@ -77,6 +77,12 @@ pub enum Kind {
     Azw3,
     /// Amazon MOBI.
     Mobi,
+    /// Microsoft Reader.
+    Lit,
+    /// FictionBook, the XML ebook format.
+    Fb2,
+    /// Rich Text Format.
+    Rtf,
     /// Plain text.
     Txt,
     /// HTML or XHTML.
@@ -85,6 +91,10 @@ pub enum Kind {
     Ole2,
     /// A zip archive whose only real content is a single PDF.
     ZipWrappedPdf,
+    /// A zip archive whose only real content is a single FictionBook.
+    ZipWrappedFb2,
+    /// A comic book archive: a zip of page images.
+    Cbz,
     /// Recognised as none of the above.
     Unknown,
 }
@@ -118,6 +128,11 @@ mod tests {
             (Kind::Ole2, "\"ole2\""),
             (Kind::Azw3, "\"azw3\""),
             (Kind::ZipWrappedPdf, "\"zip-wrapped-pdf\""),
+            (Kind::ZipWrappedFb2, "\"zip-wrapped-fb2\""),
+            (Kind::Fb2, "\"fb2\""),
+            (Kind::Lit, "\"lit\""),
+            (Kind::Rtf, "\"rtf\""),
+            (Kind::Cbz, "\"cbz\""),
             (Kind::Unknown, "\"unknown\""),
         ];
         for (kind, wire) in cases {

docs/records.md

@@ -19,7 +19,7 @@ Part of [catena](../README.md).
 | Field | Meaning |
 |---|---|
 | `path` | Where the file was read from, as given |
-| `kind` | `pdf`, `epub`, `djvu`, `azw3`, `mobi`, `txt`, `html`, `ole2`, `zip-wrapped-pdf`, `unknown` |
+| `kind` | `pdf`, `epub`, `djvu`, `azw3`, `mobi`, `lit`, `fb2`, `rtf`, `txt`, `html`, `ole2`, `cbz`, `zip-wrapped-pdf`, `zip-wrapped-fb2`, `unknown` |
 | `declared_ext` | The extension the name claimed, lowercased, without the dot |
 | `bytes` | Size on disk |
 | `pages` | Page count, for paginated formats |