diff --git a/crates/booksniff/README.md b/crates/booksniff/README.md index 499b698..2861a89 100644 --- a/crates/booksniff/README.md +++ b/crates/booksniff/README.md @@ -89,12 +89,12 @@ one bad file does not stop a sweep. | `ole2` | The compound-file header every binary Word document starts with | | `epub` | A zip holding `mimetype` and `META-INF/container.xml` | | `zip-wrapped-pdf` | A zip whose only real member is one PDF | -| `zip-wrapped-fb2` | A zip whose only real member is one FictionBook — the `.fb2.zip` most of them are passed around as | -| `cbz` | A zip whose every real member is a page image: JPEG, PNG, GIF or WebP | +| `zip-wrapped-fb2` | A zip whose only real member is named `.fb2` — the `.fb2.zip` most FictionBooks are passed around as | +| `cbz` | A zip whose every real member is a page image (JPEG, PNG, GIF or WebP), and which carries a `ComicInfo.xml` or is itself named `.cbz` or `.cbr` | | `azw3`, `mobi` | A Palm database tagged `BOOKMOBI`, told apart by the format version inside the first record | | `lit` | `ITOLITLS`, Microsoft Reader's signature | -| `rtf` | `{\rtf` at the front, after any byte-order mark and blank space | -| `fb2` | XML whose root element is `FictionBook`, found by stepping over the declaration, comments and doctype in the first four kilobytes | +| `rtf` | `{\rtf` at the front, after any byte-order mark and blank space, followed by the version digit, a space or the next control word | +| `fb2` | XML whose root element is `FictionBook`, found by stepping over the declaration, comments and doctype in the first four kilobytes; UTF-16 with a byte-order mark is decoded first | | `html` | Markup at the front, after any byte-order mark | | `txt` | Text that is none of the above | | `unknown` | Anything else, with a note saying what was seen | @@ -110,8 +110,17 @@ kind is there to stop exactly that. A comic is the pages and nothing else. Directories, the `ComicInfo.xml` a comic reader writes into the archive, and the desktop clutter every archive picks up are passed over; anything more — one PDF among the pages, a readme — -and the zip is not a comic, and says what it holds instead. Only names are -read, out of the archive's directory, so no member is ever unpacked. +and the zip is not a comic, and says what it holds instead. + +A zip of pictures is as often a holiday as a comic, so images alone are not +enough. It is a comic when it carries a `ComicInfo.xml`, or when its own name +ends `.cbz` or `.cbr`. Otherwise it is `unknown`, with a note saying it is a +zip of so many images and would be a comic book if its name said `.cbz`. + +Members are judged by name. Only the archive's directory is read, so no +member is ever unpacked and a page called `001.jpg` is taken to be an image +without its bytes being looked at. The same goes for a wrapped FictionBook: +the one member is taken to be a FictionBook because its name ends `.fb2`. FictionBook is read as bytes rather than as text. A great many are written in windows-1251, which spells its markup in ASCII but is not UTF-8, and would diff --git a/crates/booksniff/src/fb2.rs b/crates/booksniff/src/fb2.rs index a182e2b..9b5f9e6 100644 --- a/crates/booksniff/src/fb2.rs +++ b/crates/booksniff/src/fb2.rs @@ -9,9 +9,22 @@ //! windows-1251, which is not UTF-8 but spells its markup in ASCII all the //! same, and a reader that insisted on UTF-8 would call every one of them //! binary. +//! +//! A FictionBook saved as UTF-16 is the one case where the markup is not +//! ASCII bytes. Its byte-order mark says so, and the front is decoded before +//! the same walk is made over it. use crate::BOM; +/// The UTF-16 byte-order mark, little-endian. +const UTF16_LE: &[u8] = b"\xff\xfe"; +/// The UTF-16 byte-order mark, big-endian. +const UTF16_BE: &[u8] = b"\xfe\xff"; + +/// Most UTF-16 code units decoded from the front: the same window, in +/// characters, as the bytes read for every other encoding. +const UTF16_UNITS: usize = 2048; + /// What the root element of a FictionBook is called. const ROOT: &[u8] = b"FictionBook"; @@ -21,9 +34,41 @@ const ROOT: &[u8] = b"FictionBook"; /// does not begin inside it is not found, so the answer is no rather than a /// guess. pub fn is_fictionbook(head: &[u8]) -> bool { + match utf16(head) { + Some(decoded) => is_fictionbook_root(decoded.as_bytes()), + None => is_fictionbook_root(head), + } +} + +fn is_fictionbook_root(head: &[u8]) -> bool { root_element(head).is_some_and(|name| local_name(name) == ROOT) } +/// The front of a UTF-16 file as UTF-8, if a byte-order mark says it is one. +/// +/// Only a bounded prefix is decoded. A half character at the end, or a +/// surrogate without its pair, becomes a replacement character, which the +/// walk takes as part of a name and so never as the end of one. +fn utf16(head: &[u8]) -> Option { + let (body, big_endian) = match head.strip_prefix(UTF16_LE) { + Some(body) => (body, false), + None => (head.strip_prefix(UTF16_BE)?, true), + }; + let units = body.chunks_exact(2).take(UTF16_UNITS).map(|pair| { + let pair = [pair[0], pair[1]]; + if big_endian { + u16::from_be_bytes(pair) + } else { + u16::from_le_bytes(pair) + } + }); + Some( + char::decode_utf16(units) + .map(|c| c.unwrap_or(char::REPLACEMENT_CHARACTER)) + .collect(), + ) +} + /// The name of the first element, after the prolog. fn root_element(head: &[u8]) -> Option<&[u8]> { let mut rest = head.strip_prefix(BOM).unwrap_or(head); @@ -146,6 +191,60 @@ mod tests { assert!(!is_fictionbook(b"'> ] \"]\">\ diff --git a/crates/booksniff/src/lib.rs b/crates/booksniff/src/lib.rs index 9a47220..e4f8b6f 100644 --- a/crates/booksniff/src/lib.rs +++ b/crates/booksniff/src/lib.rs @@ -36,7 +36,10 @@ const ARCHIVE_NOISE: &[&str] = &["__macosx/", ".ds_store", "thumbs.db"]; /// What a comic book archive carries besides its pages: the catalogue entry /// comic readers write into it. -const COMIC_NOISE: &[&str] = &["comicinfo.xml"]; +const COMIC_CATALOGUE: &str = "comicinfo.xml"; + +/// The extensions that say an archive is a comic book. +const COMIC_EXTENSIONS: &[&str] = &["cbz", "cbr"]; /// The extensions of the images a comic book archive is made of. const PAGE_IMAGES: &[&str] = &[".jpg", ".jpeg", ".png", ".gif", ".webp"]; @@ -125,8 +128,23 @@ fn archive(source: &impl Bytes, doc: &mut Doc) -> Kind { doc.note(format!("a zip holding one FictionBook, {fb2:?}")); return Kind::ZipWrappedFb2; } - if is_comic(&names) { - return Kind::Cbz; + if let Some(pages) = comic_pages(&names) { + // A zip of pictures is as often a holiday as a comic, and only the + // names have been read. It is a comic when something says so: the + // catalogue entry a comic reader writes, or a name that claims it. + let named_comic = doc + .declared_ext + .as_deref() + .is_some_and(|ext| COMIC_EXTENSIONS.contains(&ext)); + if pages.catalogued || named_comic { + return Kind::Cbz; + } + doc.note(format!( + "a zip of {} image{}, which would be a comic book if its name said .cbz", + pages.count, + if pages.count == 1 { "" } else { "s" } + )); + return Kind::Unknown; } if lower .iter() @@ -160,23 +178,39 @@ fn sole_member<'a>(names: &'a [String], extension: &str) -> Option<&'a str> { } } -/// Whether an archive is the pages of a comic and nothing else. +/// The pages of an archive that is images and nothing else. +struct Pages { + /// How many images there are. + count: usize, + /// Whether a comic reader's catalogue entry is there beside them. + catalogued: bool, +} + +/// The pages, if an archive is images and nothing else. /// /// Every real member has to be an image. One PDF or one text file among the -/// pages makes it something else, and saying what would be a guess. -fn is_comic(names: &[String]) -> bool { +/// pages makes it something else, and saying what would be a guess. Images +/// are known by their names alone: only the archive's directory is read. +fn comic_pages(names: &[String]) -> Option { + let mut catalogued = false; let pages: Vec = real_members(names) .into_iter() .map(|name| name.to_lowercase()) .filter(|name| { let file = name.rsplit('/').next().unwrap_or(name); - !COMIC_NOISE.contains(&file) + let catalogue = file == COMIC_CATALOGUE; + catalogued |= catalogue; + !catalogue }) .collect(); - !pages.is_empty() + let all_images = !pages.is_empty() && pages .iter() - .all(|name| PAGE_IMAGES.iter().any(|image| name.ends_with(image))) + .all(|name| PAGE_IMAGES.iter().any(|image| name.ends_with(image))); + all_images.then_some(Pages { + count: pages.len(), + catalogued, + }) } /// The members that carry content: no directories, no desktop clutter. @@ -198,7 +232,7 @@ fn front(head: &[u8], doc: &mut Doc) -> Kind { // design but not always in practice, and a FictionBook is as often // windows-1251 as UTF-8. let lead = head.strip_prefix(BOM).unwrap_or(head).trim_ascii_start(); - if lead.starts_with(RTF) { + if is_rtf(lead) { return Kind::Rtf; } if fb2::is_fictionbook(head) { @@ -207,6 +241,17 @@ fn front(head: &[u8], doc: &mut Doc) -> Kind { text(head, doc) } +/// Whether `lead` opens an RTF document. +/// +/// `{\rtf` has to be the whole control word: what follows it is the version, +/// normally `1`, or the space or next control word that ends it. `{\rtfoo` is +/// some other word that happens to start the same way. +fn is_rtf(lead: &[u8]) -> bool { + lead.strip_prefix(RTF) + .and_then(|rest| rest.first()) + .is_some_and(|next| next.is_ascii_digit() || next.is_ascii_whitespace() || *next == b'\\') +} + /// Text, markup, or nothing this knows. fn text(head: &[u8], doc: &mut Doc) -> Kind { if head.is_empty() { @@ -432,10 +477,23 @@ mod tests { #[test] fn rtf_cut_short_of_its_control_word_is_not_rtf() { assert_eq!(sniff(&br"{\rt".as_slice(), "a.rtf").kind, Kind::Txt); + assert_eq!(sniff(&br"{\rtf".as_slice(), "a.rtf").kind, Kind::Txt); assert_eq!(kind_of(br"{\rtx1", "a.rtf"), Kind::Txt); assert_eq!(kind_of(br"text {\rtf1", "a.rtf"), Kind::Txt); } + #[test] + fn rtf_is_the_whole_control_word() { + assert_eq!(kind_of(br"{\rtf\ansi\deff0", "a.rtf"), Kind::Rtf); + assert_eq!(kind_of(b"{\\rtf \\ansi", "a.rtf"), Kind::Rtf); + assert_eq!(kind_of(b"{\\rtf\r\n\\ansi", "a.rtf"), Kind::Rtf); + assert_eq!(kind_of(br"{\rtf2", "a.rtf"), Kind::Rtf); + + assert_eq!(kind_of(br"{\rtfoo bar}", "a.rtf"), Kind::Txt); + assert_eq!(kind_of(br"{\rtf}", "a.rtf"), Kind::Txt); + assert_eq!(kind_of(br"{\rtf-1", "a.rtf"), Kind::Txt); + } + #[test] fn rtf_named_as_a_word_document_is_remarked_on() { let doc = sniff(&padded(br"{\rtf1\ansi").as_slice(), "letter.doc"); @@ -471,6 +529,28 @@ mod tests { assert_eq!(kind_of(feed, "feed.xml"), Kind::Html); } + #[test] + fn a_utf16_fictionbook_is_recognised_in_either_byte_order() { + let book = "\n"; + for big_endian in [false, true] { + let mut bytes = if big_endian { + vec![0xfe, 0xff] + } else { + vec![0xff, 0xfe] + }; + for unit in book.encode_utf16() { + bytes.extend_from_slice(&if big_endian { + unit.to_be_bytes() + } else { + unit.to_le_bytes() + }); + } + let doc = sniff(&padded(&bytes).as_slice(), "mill.fb2"); + assert_eq!(doc.kind, Kind::Fb2, "big-endian: {big_endian}"); + assert!(doc.notes.is_empty(), "{:?}", doc.notes); + } + } + #[test] fn a_fictionbook_named_as_something_else_is_remarked_on() { let doc = sniff(&padded(b"").as_slice(), "mill.txt"); @@ -526,11 +606,55 @@ mod tests { } #[test] - fn a_comic_saved_as_a_zip_is_remarked_on() { - let doc = archive_of(&["001.jpg", "002.jpg"], "issue-1.zip"); + fn a_zip_of_photographs_is_not_a_comic() { + let doc = archive_of(&["IMG_0001.jpg", "IMG_0002.jpg"], "holiday.zip"); + assert_eq!(doc.kind, Kind::Unknown); + assert!( + doc.notes.iter().any(|note| note + == "a zip of 2 images, which would be a comic book if its name said .cbz"), + "{:?}", + doc.notes + ); + + let one = archive_of(&["IMG_0001.jpg"], "holiday.zip"); + assert!( + one.notes + .iter() + .any(|note| note.starts_with("a zip of 1 image,")), + "{:?}", + one.notes + ); + } + + #[test] + fn the_same_images_named_as_a_comic_are_one() { + let doc = archive_of(&["IMG_0001.jpg", "IMG_0002.jpg"], "holiday.cbz"); assert_eq!(doc.kind, Kind::Cbz); assert!( - doc.notes.iter().any(|note| note.contains("comic")), + !doc.notes.iter().any(|note| note.contains("named .")), + "{:?}", + doc.notes + ); + + // A comic zipped and named as if it were RAR is still a comic, and + // the name is remarked on. + let rar = archive_of(&["001.jpg", "002.jpg"], "issue-1.cbr"); + assert_eq!(rar.kind, Kind::Cbz); + assert!( + rar.notes.iter().any(|note| note.contains("named .cbr")), + "{:?}", + rar.notes + ); + } + + #[test] + fn a_comic_saved_as_a_zip_is_known_by_its_catalogue_and_remarked_on() { + let doc = archive_of(&["001.jpg", "002.jpg", "ComicInfo.xml"], "issue-1.zip"); + assert_eq!(doc.kind, Kind::Cbz); + assert!( + doc.notes + .iter() + .any(|note| note == "named .zip, but the bytes say a comic book archive"), "{:?}", doc.notes );