Merge remote-tracking branch 'origin/main' into bibfetch-search

This commit is contained in:
funkpower committed 2026-11-13 15:29:05 -05:00
commit 03c19c7293
13 files changed
+1117 -4

No files matched your search

+24 -1
View File
@@ -84,6 +84,7 @@ bookmeta library/*.epub \
|---|---| |---|---|
| EPUB | The package document named by `META-INF/container.xml` — Dublin Core, plus the `<meta>` entries Calibre adds for a series | | EPUB | The package document named by `META-INF/container.xml` — Dublin Core, plus the `<meta>` entries Calibre adds for a series |
| PDF | The trailer's information dictionary | | PDF | The trailer's information dictionary |
| MOBI, AZW, AZW3 | The MOBI header in the first record, and the EXTH records after it |
An EPUB whose container is malformed falls back to the first `.opf` in the An EPUB whose container is malformed falls back to the first `.opf` in the
archive, because a converter that wrote a bad container often still wrote a archive, because a converter that wrote a bad container often still wrote a
@@ -94,6 +95,28 @@ both become `isbn`, a UUID becomes `uuid`, anything else `other`. PDF dates
are reduced to the day where they parse and kept verbatim where they do not, are reduced to the day where they parse and kept verbatim where they do not,
because a date nobody can read is still evidence. because a date nobody can read is still evidence.
A Kindle book's title is EXTH 503 where it is there, then the MOBI header's
full name, then the Palm database name (cut to 31 bytes, with a note). Authors
(100) and subjects (105) are every record of their kind, in order; publisher
(101), description (103), date (106) and language (524) the first. EXTH 104 is
an `isbn` when it is shaped like one and `other` when not, and 113 is an
`asin`. A contributor (108) is not an author, the same as an EPUB's
`dc:contributor`, and is not reported. With no EXTH language, the language
comes from the header's locale. Strings are decoded as the header says,
UTF-8 or Windows-1252. A book whose text is encrypted still has its header
read, and carries the note `drm`.
Every offset in a Kindle book is checked before it is followed: headers that
are cut short or point outside the file give an empty record with a note, and
an EXTH block that breaks off keeps the records before the break.
## Covers
`Metadata::cover` is the cover image a Kindle book carries (EXTH 201, or the
thumbnail, 202, when no cover is named), with its media type read off its
first bytes. It is for library callers and is not part of the JSON record,
which is a line of text.
## Asking a model ## Asking a model
Some files say nothing useful. `--prompt` writes the question and **sends Some files say nothing useful. `--prompt` writes the question and **sends
@@ -147,7 +170,7 @@ if let Some(prompt) = bookmeta::prompt::compose(&meta, Some(opening)) {
} }
``` ```
`bookmeta::epub` and `bookmeta::pdf` read one format each, and `bookmeta::epub`, `bookmeta::pdf` and `bookmeta::mobi` read one format each, and
`Metadata::missing` says which fields a record still wants. `Metadata::missing` says which fields a record still wants.
## See also ## See also
+15 -3
View File
@@ -85,6 +85,7 @@ fn fill(meta: &mut Metadata, package: &str) {
meta.title = first(package, "title"); meta.title = first(package, "title");
meta.authors = creators(package); meta.authors = creators(package);
meta.publisher = first(package, "publisher"); meta.publisher = first(package, "publisher");
meta.description = first(package, "description");
meta.date = first(package, "date"); meta.date = first(package, "date");
meta.language = first(package, "language"); meta.language = first(package, "language");
meta.subjects = all(package, "subject"); meta.subjects = all(package, "subject");
@@ -166,7 +167,7 @@ fn identifiers(package: &str) -> Vec<(String, String)> {
("doi", value.to_string()) ("doi", value.to_string())
} else if is_isbn(value) { } else if is_isbn(value) {
("isbn", digits_of(value)) ("isbn", digits_of(value))
} else if lower.len() == 36 && lower.matches('-').count() == 4 { } else if is_uuid(&lower) {
("uuid", lower.clone()) ("uuid", lower.clone())
} else { } else {
("other", value.to_string()) ("other", value.to_string())
@@ -179,7 +180,7 @@ fn identifiers(package: &str) -> Vec<(String, String)> {
} }
/// Whether a string is ten or thirteen digits once its hyphens come off. /// Whether a string is ten or thirteen digits once its hyphens come off.
fn is_isbn(value: &str) -> bool { pub(crate) fn is_isbn(value: &str) -> bool {
let digits = digits_of(value); let digits = digits_of(value);
matches!(digits.len(), 10 | 13) matches!(digits.len(), 10 | 13)
&& value && value
@@ -187,7 +188,13 @@ fn is_isbn(value: &str) -> bool {
.all(|c| c.is_ascii_digit() || "- xX".contains(c)) .all(|c| c.is_ascii_digit() || "- xX".contains(c))
} }
fn digits_of(value: &str) -> String { /// Whether a value is shaped like a UUID: 36 characters with four hyphens.
pub(crate) fn is_uuid(value: &str) -> bool {
value.len() == 36 && value.matches('-').count() == 4
}
/// The digits of an ISBN, with its check character, and nothing else.
pub(crate) fn digits_of(value: &str) -> String {
value value
.chars() .chars()
.filter(|c| c.is_ascii_digit() || matches!(c, 'x' | 'X')) .filter(|c| c.is_ascii_digit() || matches!(c, 'x' | 'X'))
@@ -253,6 +260,7 @@ mod tests {
<dc:creator opf:role="aut">Kallistos Ware</dc:creator> <dc:creator opf:role="aut">Kallistos Ware</dc:creator>
<dc:contributor>A Typesetter</dc:contributor> <dc:contributor>A Typesetter</dc:contributor>
<dc:publisher>Faber &amp; Faber</dc:publisher> <dc:publisher>Faber &amp; Faber</dc:publisher>
<dc:description>&lt;p&gt;Writings of the Fathers.&lt;/p&gt;</dc:description>
<dc:date>1984-09-17</dc:date> <dc:date>1984-09-17</dc:date>
<dc:language>en</dc:language> <dc:language>en</dc:language>
<dc:subject>Orthodox Christianity</dc:subject> <dc:subject>Orthodox Christianity</dc:subject>
@@ -275,6 +283,10 @@ mod tests {
let meta = read_package(); let meta = read_package();
assert_eq!(meta.title.as_deref(), Some("The Philokalia, Volume 3")); assert_eq!(meta.title.as_deref(), Some("The Philokalia, Volume 3"));
assert_eq!(meta.publisher.as_deref(), Some("Faber & Faber")); assert_eq!(meta.publisher.as_deref(), Some("Faber & Faber"));
assert_eq!(
meta.description.as_deref(),
Some("<p>Writings of the Fathers.</p>")
);
assert_eq!(meta.date.as_deref(), Some("1984-09-17")); assert_eq!(meta.date.as_deref(), Some("1984-09-17"));
assert_eq!(meta.language.as_deref(), Some("en")); assert_eq!(meta.language.as_deref(), Some("en"));
} }
+28
View File
@@ -14,6 +14,7 @@
//! [`bibfetch`]: https://git.pumpin.online/funkpower/catena //! [`bibfetch`]: https://git.pumpin.online/funkpower/catena
pub mod epub; pub mod epub;
pub mod mobi;
pub mod pdf; pub mod pdf;
pub mod prompt; pub mod prompt;
@@ -30,6 +31,8 @@ pub enum Source {
Opf, Opf,
/// A PDF's information dictionary. /// A PDF's information dictionary.
PdfInfo, PdfInfo,
/// A Kindle book's MOBI header and the EXTH records after it.
Mobi,
/// Nothing was found to read. /// Nothing was found to read.
None, None,
} }
@@ -40,6 +43,7 @@ impl Source {
match self { match self {
Source::Opf => "opf", Source::Opf => "opf",
Source::PdfInfo => "pdf_info", Source::PdfInfo => "pdf_info",
Source::Mobi => "mobi",
Source::None => "none", Source::None => "none",
} }
} }
@@ -64,6 +68,9 @@ pub struct Metadata {
/// Who published it. /// Who published it.
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub publisher: Option<String>, pub publisher: Option<String>,
/// The publisher's description of the book, as written: often HTML.
#[serde(skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
/// The publication date, in whatever form the file wrote it. /// The publication date, in whatever form the file wrote it.
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub date: Option<String>, pub date: Option<String>,
@@ -86,6 +93,23 @@ pub struct Metadata {
/// What was worth saying about the reading. /// What was worth saying about the reading.
#[serde(default, skip_serializing_if = "Vec::is_empty")] #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub notes: Vec<String>, pub notes: Vec<String>,
/// The cover image the file carries.
///
/// Not part of the record: a record is a line of text and this is an
/// image. A library caller takes it from here.
#[serde(skip)]
#[schemars(skip)]
pub cover: Option<Cover>,
}
/// A cover image, as the file stores it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Cover {
/// What kind of image it is, read off its first bytes: `image/jpeg`,
/// `image/png`, `image/gif` or `image/bmp`.
pub media_type: String,
/// The image itself.
pub bytes: Vec<u8>,
} }
fn version() -> u32 { fn version() -> u32 {
@@ -143,6 +167,7 @@ impl Metadata {
self.title.is_none() self.title.is_none()
&& self.authors.is_empty() && self.authors.is_empty()
&& self.publisher.is_none() && self.publisher.is_none()
&& self.description.is_none()
&& self.date.is_none() && self.date.is_none()
&& self.language.is_none() && self.language.is_none()
&& self.identifiers.is_empty() && self.identifiers.is_empty()
@@ -171,6 +196,9 @@ pub fn from_bytes(path: &str, bytes: &[u8]) -> Metadata {
if bytes.starts_with(b"%PDF") { if bytes.starts_with(b"%PDF") {
return pdf::read(path, bytes); return pdf::read(path, bytes);
} }
if mobi::is_mobi(bytes) {
return mobi::read(path, bytes);
}
let mut meta = Metadata::empty(path, Source::None); let mut meta = Metadata::empty(path, Source::None);
meta.note("not a format this reads metadata out of"); meta.note("not a format this reads metadata out of");
meta meta
+873
View File
@@ -0,0 +1,873 @@
//! A Kindle book's MOBI header and the EXTH records after it.
//!
//! MOBI, AZW and AZW3 files are Palm databases. The first record opens with a
//! PalmDOC header and a MOBI header, which gives the book's full name and says
//! how its strings are encoded. When a flag in that header is set, an EXTH
//! block follows with everything else: authors, publisher, ISBN, subjects, and
//! which image record is the cover. AZW3 (KF8) keeps the same layout in its
//! first record, so one reader covers all three.
//!
//! Every offset in the file is checked before it is followed. A file whose
//! headers are cut short or written wrongly gives an empty record with a note;
//! an EXTH block that breaks off keeps the records before the break. Neither
//! panics or reads past the end.
use crate::epub::{digits_of, is_isbn, is_uuid};
use crate::{Cover, Metadata, Source};
/// Where a Palm database says what it holds.
const TYPE_AT: usize = 60;
/// What a Kindle book says there.
const BOOK_TYPE: &[u8] = b"BOOKMOBI";
/// The Palm database's own name: 32 bytes, NUL-padded.
const NAME_LEN: usize = 32;
/// Where the number of records is written.
const RECORD_COUNT_AT: usize = 76;
/// Where the record list starts: eight bytes per record, offset first.
const RECORD_LIST_AT: usize = 78;
/// Where the PalmDOC header says whether the text is encrypted.
const ENCRYPTION_AT: usize = 12;
/// Where the MOBI header begins inside the first record.
const MOBI_AT: usize = 16;
/// The MOBI header's own magic.
const MOBI_MAGIC: &[u8] = b"MOBI";
// Fields of the MOBI header, as offsets from its start. Each is only there
// when the header's declared length reaches past it.
const HEADER_LENGTH: usize = 4;
const TEXT_ENCODING: usize = 12;
const FULL_NAME_OFFSET: usize = 68;
const FULL_NAME_LENGTH: usize = 72;
const LOCALE: usize = 76;
const FIRST_IMAGE: usize = 92;
const EXTH_FLAGS: usize = 112;
/// The flag that says an EXTH block follows the MOBI header.
const HAS_EXTH: u32 = 0x40;
/// The EXTH block's magic.
const EXTH_MAGIC: &[u8] = b"EXTH";
/// The text encodings a MOBI header names.
const UTF_8: u32 = 65001;
const WINDOWS_1252: u32 = 1252;
/// What an index or offset says when it points at nothing.
const NONE: u32 = 0xffff_ffff;
// EXTH record types.
const AUTHOR: u32 = 100;
const PUBLISHER: u32 = 101;
const DESCRIPTION: u32 = 103;
const ISBN: u32 = 104;
const SUBJECT: u32 = 105;
const PUBLISHED: u32 = 106;
const ASIN: u32 = 113;
const COVER_OFFSET: u32 = 201;
const THUMB_OFFSET: u32 = 202;
const UPDATED_TITLE: u32 = 503;
const LANGUAGE: u32 = 524;
/// Whether these bytes are a Kindle book this module can read.
pub fn is_mobi(bytes: &[u8]) -> bool {
bytes.get(TYPE_AT..TYPE_AT + BOOK_TYPE.len()) == Some(BOOK_TYPE)
}
/// Read what a MOBI, AZW or AZW3 file says about itself.
pub fn read(path: &str, bytes: &[u8]) -> Metadata {
let mut meta = Metadata::empty(path, Source::Mobi);
if let Err(why) = fill(&mut meta, bytes) {
let mut empty = Metadata::empty(path, Source::None);
empty.note(why);
return empty;
}
if meta.is_empty() {
meta.note("a MOBI header with nothing filled in");
}
meta
}
/// The strings an EXTH block carries, before they are placed in a record.
#[derive(Default)]
struct Exth {
authors: Vec<String>,
publisher: Option<String>,
description: Option<String>,
isbn: Option<String>,
subjects: Vec<String>,
published: Option<String>,
asin: Option<String>,
cover: Option<u32>,
thumb: Option<u32>,
title: Option<String>,
language: Option<String>,
}
fn fill(meta: &mut Metadata, bytes: &[u8]) -> Result<(), String> {
if !is_mobi(bytes) {
return Err("not a Kindle book: the Palm database type is not BOOKMOBI".into());
}
let records = records(bytes)?;
let first = record(bytes, &records, 0).ok_or("the first record lies outside the file")?;
if first.get(MOBI_AT..MOBI_AT + MOBI_MAGIC.len()) != Some(MOBI_MAGIC) {
return Err("no MOBI header in the first record".into());
}
let header_length =
u32_at(first, MOBI_AT + HEADER_LENGTH).ok_or("the MOBI header is cut short")? as usize;
let header = Header {
record: first,
length: header_length,
};
let encoding = header.field(TEXT_ENCODING).unwrap_or(WINDOWS_1252);
if encoding != UTF_8 && encoding != WINDOWS_1252 {
meta.note(format!(
"text encoding {encoding}, which is neither UTF-8 nor Windows-1252; read as Windows-1252"
));
}
let decode = |raw: &[u8]| text(raw, encoding);
if u16_at(first, ENCRYPTION_AT).is_some_and(|kind| kind != 0) {
meta.note("drm");
}
let exth = if header
.field(EXTH_FLAGS)
.is_some_and(|flags| flags & HAS_EXTH != 0)
{
let at = MOBI_AT
.checked_add(header_length)
.ok_or("the MOBI header length is out of range")?;
exth(first, at, &decode, meta)
} else {
Exth::default()
};
let full_name = header
.field(FULL_NAME_OFFSET)
.zip(header.field(FULL_NAME_LENGTH))
.and_then(|(at, len)| slice(first, at as usize, len as usize))
.and_then(decode);
meta.title = match (exth.title, full_name) {
(Some(title), _) | (None, Some(title)) => Some(title),
(None, None) => {
let name = palm_name(bytes);
if name.is_some() {
meta.note("title from the Palm database name, which is cut to 31 bytes");
}
name
}
};
meta.authors = exth.authors;
meta.publisher = exth.publisher;
meta.description = exth.description;
meta.date = exth.published;
meta.subjects = exth.subjects;
meta.language = exth.language.or_else(|| {
header
.field(LOCALE)
.and_then(|locale| language_of(locale).map(str::to_string))
});
if let Some(isbn) = exth.isbn {
if is_isbn(&isbn) {
meta.identifiers.insert("isbn".into(), digits_of(&isbn));
} else {
meta.identifiers.insert("other".into(), isbn);
}
}
// EXTH 113 holds an ASIN when the book was made for Amazon, and calibre's
// own book UUID when it was converted without one.
if let Some(value) = exth.asin {
let lower = value.to_lowercase();
if is_uuid(&lower) {
meta.identifiers.insert("uuid".into(), lower);
} else if is_asin(&value) {
meta.identifiers.insert("asin".into(), value);
} else {
meta.identifiers.entry("other".into()).or_insert(value);
}
}
let first_image = header.field(FIRST_IMAGE);
if let Some(offset) = exth.cover.or(exth.thumb) {
meta.cover = cover(bytes, &records, first_image, offset, meta);
} else {
meta.cover = first_image_record(bytes, &records, first_image);
}
Ok(())
}
/// The MOBI header, for reading fields that may or may not be in it.
struct Header<'a> {
record: &'a [u8],
length: usize,
}
impl Header<'_> {
/// A field at `at` bytes into the header, when the header reaches it.
fn field(&self, at: usize) -> Option<u32> {
if at.checked_add(4)? > self.length {
return None;
}
u32_at(self.record, MOBI_AT.checked_add(at)?)
}
}
/// Where each record starts, as the record list says.
fn records(bytes: &[u8]) -> Result<Vec<usize>, String> {
let count = u16_at(bytes, RECORD_COUNT_AT).ok_or("the Palm database header is cut short")?;
if count == 0 {
return Err("a Palm database with no records".into());
}
let mut out = Vec::with_capacity(count as usize);
for at in 0..count as usize {
let entry = RECORD_LIST_AT + at * 8;
let Some(offset) = u32_at(bytes, entry) else {
return Err("the record list runs past the end of the file".into());
};
out.push(offset as usize);
}
Ok(out)
}
/// One record's bytes: from its offset to the next record's, or to the end.
fn record<'a>(bytes: &'a [u8], records: &[usize], index: usize) -> Option<&'a [u8]> {
let start = *records.get(index)?;
let end = records.get(index + 1).copied().unwrap_or(bytes.len());
bytes.get(start..end)
}
/// Read the EXTH block at `at` inside the first record.
///
/// A block that ends early keeps what was read before the break, with a note:
/// the records already read are as good as they would have been.
fn exth(
first: &[u8],
at: usize,
decode: &dyn Fn(&[u8]) -> Option<String>,
meta: &mut Metadata,
) -> Exth {
let mut out = Exth::default();
if at >= first.len() || first.get(at..at + EXTH_MAGIC.len()) != Some(EXTH_MAGIC) {
meta.note("the header says an EXTH block follows, but none does");
return out;
}
let (Some(length), Some(count)) = (u32_at(first, at + 4), u32_at(first, at + 8)) else {
meta.note("the EXTH block ends early");
return out;
};
let end = at.saturating_add(length as usize).min(first.len());
let mut cursor = at + 12;
for _ in 0..count {
let (Some(kind), Some(size)) = (u32_at(first, cursor), u32_at(first, cursor + 4)) else {
meta.note("the EXTH block ends early");
break;
};
let size = size as usize;
let Some(data) = size
.checked_sub(8)
.and_then(|len| slice(first, cursor + 8, len))
.filter(|_| cursor.checked_add(size).is_some_and(|stop| stop <= end))
else {
meta.note("the EXTH block ends early");
break;
};
cursor += size;
match kind {
AUTHOR => out.authors.extend(decode(data)),
SUBJECT => out.subjects.extend(decode(data)),
PUBLISHER => set(&mut out.publisher, decode(data)),
DESCRIPTION => set(&mut out.description, decode(data)),
ISBN => set(&mut out.isbn, decode(data)),
PUBLISHED => set(&mut out.published, decode(data)),
ASIN => set(&mut out.asin, decode(data)),
UPDATED_TITLE => set(&mut out.title, decode(data)),
LANGUAGE => set(&mut out.language, decode(data)),
COVER_OFFSET => set(&mut out.cover, number(data)),
THUMB_OFFSET => set(&mut out.thumb, number(data)),
_ => {}
}
}
out
}
/// Keep the first value a field is given: a later duplicate is usually a
/// converter writing the same thing twice, and the first is what it meant.
fn set<T>(slot: &mut Option<T>, value: Option<T>) {
if slot.is_none() {
*slot = value;
}
}
/// A four-byte number record that points somewhere.
fn number(data: &[u8]) -> Option<u32> {
let value = u32::from_be_bytes(data.try_into().ok()?);
(value != NONE).then_some(value)
}
/// The cover image, from the record the header and the EXTH offset name.
fn cover(
bytes: &[u8],
records: &[usize],
first_image: Option<u32>,
offset: u32,
meta: &mut Metadata,
) -> Option<Cover> {
let Some(first_image) = first_image.filter(|&index| index != NONE) else {
meta.note("a cover is named, but the header has no image records");
return None;
};
let Some(data) = first_image
.checked_add(offset)
.and_then(|index| record(bytes, records, index as usize))
else {
meta.note("the cover points past the last record");
return None;
};
let Some(media_type) = image_type(data) else {
meta.note("the cover record is not an image");
return None;
};
Some(Cover {
media_type: media_type.to_string(),
bytes: data.to_vec(),
})
}
/// The cover when the file names none: calibre takes the record at the first
/// image index, and only that one.
///
/// When that record is not a JPEG, PNG or GIF there is no cover. The search
/// does not walk on to the next image, because a later record may be an inline
/// illustration, and kindlegen puts a font or a RESC record first.
fn first_image_record(bytes: &[u8], records: &[usize], first_image: Option<u32>) -> Option<Cover> {
let first = first_image.filter(|&index| index != NONE)?;
let data = record(bytes, records, first as usize)?;
let media_type = image_type(data).filter(|kind| *kind != "image/bmp")?;
Some(Cover {
media_type: media_type.to_string(),
bytes: data.to_vec(),
})
}
/// An ASIN is ten letters and digits; an ISBN-10 is one too.
fn is_asin(value: &str) -> bool {
value.len() == 10 && value.chars().all(|c| c.is_ascii_alphanumeric())
}
/// What kind of image these bytes are, by their magic.
fn image_type(data: &[u8]) -> Option<&'static str> {
if data.starts_with(&[0xff, 0xd8, 0xff]) {
Some("image/jpeg")
} else if data.starts_with(b"\x89PNG\r\n\x1a\n") {
Some("image/png")
} else if data.starts_with(b"GIF87a") || data.starts_with(b"GIF89a") {
Some("image/gif")
} else if data.starts_with(b"BM") && data.len() > 14 {
Some("image/bmp")
} else {
None
}
}
/// The Palm database's name, the last place a title can come from.
///
/// Converters write the title there with spaces made underscores; it is
/// reported as written.
fn palm_name(bytes: &[u8]) -> Option<String> {
let raw = bytes.get(..NAME_LEN)?;
let raw = raw.split(|&byte| byte == 0).next().unwrap_or(raw);
text(raw, WINDOWS_1252)
}
/// A string in the book's declared encoding, trimmed; empty is none.
fn text(raw: &[u8], encoding: u32) -> Option<String> {
let decoded: String = match (encoding, std::str::from_utf8(raw)) {
(UTF_8, Ok(text)) => text.to_string(),
(UTF_8, Err(_)) => String::from_utf8_lossy(raw).into_owned(),
_ => raw.iter().map(|&byte| windows_1252(byte)).collect(),
};
let trimmed = decoded.trim_matches(|c: char| c == '\0' || c.is_whitespace());
(!trimmed.is_empty()).then(|| trimmed.to_string())
}
/// One Windows-1252 byte as a character.
fn windows_1252(byte: u8) -> char {
match byte {
0x80..=0x9f => HIGH_CONTROLS[(byte - 0x80) as usize],
_ => byte as char,
}
}
/// The characters Windows-1252 puts where Latin-1 has C1 controls. The five
/// positions it leaves undefined decode to themselves, as browsers do.
const HIGH_CONTROLS: [char; 32] = [
'\u{20ac}', '\u{81}', '\u{201a}', '\u{192}', '\u{201e}', '\u{2026}', '\u{2020}', '\u{2021}',
'\u{2c6}', '\u{2030}', '\u{160}', '\u{2039}', '\u{152}', '\u{8d}', '\u{17d}', '\u{8f}',
'\u{90}', '\u{2018}', '\u{2019}', '\u{201c}', '\u{201d}', '\u{2022}', '\u{2013}', '\u{2014}',
'\u{2dc}', '\u{2122}', '\u{161}', '\u{203a}', '\u{153}', '\u{9d}', '\u{17e}', '\u{178}',
];
/// The language a MOBI locale names, for books with no EXTH language.
///
/// The low byte is a Windows primary language id; the region above it is
/// dropped, since the file's own language field never carries one either.
fn language_of(locale: u32) -> Option<&'static str> {
Some(match locale & 0xff {
0x01 => "ar",
0x02 => "bg",
0x03 => "ca",
0x04 => "zh",
0x05 => "cs",
0x06 => "da",
0x07 => "de",
0x08 => "el",
0x09 => "en",
0x0a => "es",
0x0b => "fi",
0x0c => "fr",
0x0d => "he",
0x0e => "hu",
0x0f => "is",
0x10 => "it",
0x11 => "ja",
0x12 => "ko",
0x13 => "nl",
0x14 => "no",
0x15 => "pl",
0x16 => "pt",
0x18 => "ro",
0x19 => "ru",
0x1a => "hr",
0x1b => "sk",
0x1d => "sv",
0x1f => "tr",
0x22 => "uk",
0x29 => "fa",
0x2a => "vi",
0x36 => "af",
0x39 => "hi",
_ => return None,
})
}
fn slice(bytes: &[u8], at: usize, len: usize) -> Option<&[u8]> {
bytes.get(at..at.checked_add(len)?)
}
fn u16_at(bytes: &[u8], at: usize) -> Option<u16> {
Some(u16::from_be_bytes(slice(bytes, at, 2)?.try_into().ok()?))
}
fn u32_at(bytes: &[u8], at: usize) -> Option<u32> {
Some(u32::from_be_bytes(slice(bytes, at, 4)?.try_into().ok()?))
}
#[cfg(test)]
mod tests {
use super::*;
const JPEG: &[u8] = &[0xff, 0xd8, 0xff, 0xe0, 0, 16, b'J', b'F', b'I', b'F'];
/// A Kindle book built field by field, so each test can break one thing.
struct Book {
name: &'static [u8],
encoding: u32,
encryption: u16,
locale: u32,
full_name: Vec<u8>,
exth: Option<Vec<(u32, Vec<u8>)>>,
first_image: u32,
images: Vec<Vec<u8>>,
}
impl Default for Book {
fn default() -> Self {
Book {
name: b"A_Test_Book",
encoding: UTF_8,
encryption: 0,
locale: 0x09,
full_name: b"A Test Book".to_vec(),
exth: Some(Vec::new()),
first_image: 1,
images: Vec::new(),
}
}
}
impl Book {
fn bytes(&self) -> Vec<u8> {
const HEADER: usize = 232;
let mut first = vec![0u8; MOBI_AT + HEADER];
first[ENCRYPTION_AT..ENCRYPTION_AT + 2].copy_from_slice(&self.encryption.to_be_bytes());
first[MOBI_AT..MOBI_AT + 4].copy_from_slice(MOBI_MAGIC);
let put = |first: &mut Vec<u8>, at: usize, value: u32| {
first[MOBI_AT + at..MOBI_AT + at + 4].copy_from_slice(&value.to_be_bytes());
};
put(&mut first, HEADER_LENGTH, HEADER as u32);
put(&mut first, TEXT_ENCODING, self.encoding);
put(&mut first, LOCALE, self.locale);
put(&mut first, FIRST_IMAGE, self.first_image);
if let Some(records) = &self.exth {
put(&mut first, EXTH_FLAGS, HAS_EXTH);
let mut body = Vec::new();
for (kind, data) in records {
body.extend_from_slice(&kind.to_be_bytes());
body.extend_from_slice(&(data.len() as u32 + 8).to_be_bytes());
body.extend_from_slice(data);
}
first.extend_from_slice(EXTH_MAGIC);
first.extend_from_slice(&(body.len() as u32 + 12).to_be_bytes());
first.extend_from_slice(&(records.len() as u32).to_be_bytes());
first.extend_from_slice(&body);
}
let name_at = first.len();
first.extend_from_slice(&self.full_name);
put(&mut first, FULL_NAME_OFFSET, name_at as u32);
put(&mut first, FULL_NAME_LENGTH, self.full_name.len() as u32);
let mut records = vec![first];
records.extend(self.images.iter().cloned());
let mut out = vec![0u8; RECORD_LIST_AT + records.len() * 8];
out[..self.name.len()].copy_from_slice(self.name);
out[TYPE_AT..TYPE_AT + 8].copy_from_slice(BOOK_TYPE);
out[RECORD_COUNT_AT..RECORD_COUNT_AT + 2]
.copy_from_slice(&(records.len() as u16).to_be_bytes());
for (at, record) in records.iter().enumerate() {
let offset = out.len() as u32;
let entry = RECORD_LIST_AT + at * 8;
out[entry..entry + 4].copy_from_slice(&offset.to_be_bytes());
out.extend_from_slice(record);
}
out
}
}
fn exth(records: &[(u32, &[u8])]) -> Option<Vec<(u32, Vec<u8>)>> {
Some(records.iter().map(|(k, v)| (*k, v.to_vec())).collect())
}
#[test]
fn the_exth_records_fill_the_record() {
let book = Book {
exth: exth(&[
(UPDATED_TITLE, b"The Real Title"),
(AUTHOR, b"First Author"),
(AUTHOR, b"Second Author"),
(PUBLISHER, b"A Press"),
(DESCRIPTION, b"What it is about."),
(ISBN, b"978-0-306-40615-7"),
(SUBJECT, b"History"),
(SUBJECT, b"Ships"),
(PUBLISHED, b"1999-01-02"),
(ASIN, b"B000TEST00"),
(LANGUAGE, b"de"),
]),
..Book::default()
};
let meta = read("x.mobi", &book.bytes());
assert_eq!(meta.source, "mobi");
assert_eq!(meta.title.as_deref(), Some("The Real Title"));
assert_eq!(meta.authors, ["First Author", "Second Author"]);
assert_eq!(meta.publisher.as_deref(), Some("A Press"));
assert_eq!(meta.description.as_deref(), Some("What it is about."));
assert_eq!(meta.date.as_deref(), Some("1999-01-02"));
assert_eq!(meta.subjects, ["History", "Ships"]);
assert_eq!(meta.language.as_deref(), Some("de"));
assert_eq!(meta.identifiers["isbn"], "9780306406157");
assert_eq!(meta.identifiers["asin"], "B000TEST00");
assert!(meta.notes.is_empty(), "{:?}", meta.notes);
}
#[test]
fn the_title_falls_back_to_the_full_name_and_then_the_database_name() {
let meta = read("x.mobi", &Book::default().bytes());
assert_eq!(meta.title.as_deref(), Some("A Test Book"));
let book = Book {
full_name: Vec::new(),
..Book::default()
};
let meta = read("x.mobi", &book.bytes());
assert_eq!(meta.title.as_deref(), Some("A_Test_Book"));
assert!(meta.notes[0].contains("Palm database name"));
}
#[test]
fn the_language_falls_back_to_the_locale() {
let book = Book {
locale: 0x040c,
..Book::default()
};
assert_eq!(
read("x.mobi", &book.bytes()).language.as_deref(),
Some("fr")
);
let book = Book {
locale: 0,
..Book::default()
};
assert_eq!(read("x.mobi", &book.bytes()).language, None);
}
#[test]
fn windows_1252_strings_decode_as_windows_1252() {
let book = Book {
encoding: WINDOWS_1252,
exth: exth(&[
(AUTHOR, b"Ren\xe9 Ch\x96ar"),
(PUBLISHER, b"\x93Quoted\x94"),
]),
..Book::default()
};
let meta = read("x.mobi", &book.bytes());
assert_eq!(meta.authors, ["René Ch–ar"]);
assert_eq!(meta.publisher.as_deref(), Some("“Quoted”"));
}
#[test]
fn the_cover_is_the_image_record_the_offset_names() {
let book = Book {
exth: exth(&[(COVER_OFFSET, &1u32.to_be_bytes())]),
images: vec![b"GIF89a-thumb".to_vec(), JPEG.to_vec()],
..Book::default()
};
let cover = read("x.mobi", &book.bytes()).cover.expect("a cover");
assert_eq!(cover.media_type, "image/jpeg");
assert_eq!(cover.bytes, JPEG);
}
#[test]
fn the_thumbnail_stands_in_when_no_cover_is_named() {
let book = Book {
exth: exth(&[(THUMB_OFFSET, &0u32.to_be_bytes())]),
images: vec![b"\x89PNG\r\n\x1a\nrest".to_vec()],
..Book::default()
};
let cover = read("x.mobi", &book.bytes()).cover.expect("a cover");
assert_eq!(cover.media_type, "image/png");
}
#[test]
fn with_no_cover_named_only_the_record_at_first_image_can_be_the_cover() {
let book = Book {
images: vec![JPEG.to_vec(), b"GIF89a-later".to_vec()],
..Book::default()
};
let cover = read("x.mobi", &book.bytes()).cover.expect("a cover");
assert_eq!(cover.media_type, "image/jpeg");
assert_eq!(cover.bytes, JPEG);
// The record at first_image is not an image: no cover, even though a
// later record is one.
let book = Book {
images: vec![
b"not an image".to_vec(),
JPEG.to_vec(),
b"GIF89a-later".to_vec(),
],
..Book::default()
};
assert_eq!(read("x.mobi", &book.bytes()).cover, None);
let book = Book {
images: vec![b"not an image".to_vec()],
..Book::default()
};
assert_eq!(read("x.mobi", &book.bytes()).cover, None);
}
#[test]
fn a_font_at_first_image_is_not_skipped_past_to_an_inline_image() {
// No EXTH 201 or 202. The first resource record is a font, as in
// kindlegen output; calibre reads only that record and finds no cover.
let book = Book {
images: vec![b"FONT\0\0\0\x18".to_vec(), JPEG.to_vec()],
..Book::default()
};
assert_eq!(read("x.mobi", &book.bytes()).cover, None);
}
#[test]
fn a_stale_kf8_boundary_below_first_image_does_not_hide_the_cover() {
let book = Book {
exth: exth(&[(121, &1u32.to_be_bytes())]),
first_image: 2,
images: vec![b"text".to_vec(), JPEG.to_vec()],
..Book::default()
};
assert!(read("x.mobi", &book.bytes()).cover.is_some());
}
#[test]
fn the_first_image_search_stops_at_the_kf8_boundary() {
let book = Book {
exth: exth(&[(121, &2u32.to_be_bytes())]),
images: vec![b"text".to_vec(), JPEG.to_vec()],
..Book::default()
};
// Record 1 is not an image, and record 2 is past the boundary.
assert_eq!(read("x.mobi", &book.bytes()).cover, None);
}
#[test]
fn exth_113_is_an_asin_a_uuid_or_something_else() {
let id = |value: &[u8]| {
let book = Book {
exth: exth(&[(ASIN, value)]),
..Book::default()
};
read("x.mobi", &book.bytes()).identifiers
};
assert_eq!(id(b"B01A7RJGIW")["asin"], "B01A7RJGIW");
assert_eq!(id(b"0306406152")["asin"], "0306406152");
let uuid = id(b"69AE65CA-B644-499F-A78C-139771C7140A");
assert_eq!(uuid["uuid"], "69ae65ca-b644-499f-a78c-139771c7140a");
assert_eq!(uuid.get("asin"), None);
let other = id(b"2AW6TL2JJEYYGXCOV3PWN6ZOYUORXAVX");
assert_eq!(other["other"], "2AW6TL2JJEYYGXCOV3PWN6ZOYUORXAVX");
assert_eq!(other.get("asin"), None);
}
#[test]
fn a_cover_that_points_nowhere_is_a_note_and_not_a_cover() {
let book = Book {
exth: exth(&[(COVER_OFFSET, &7u32.to_be_bytes())]),
images: vec![JPEG.to_vec()],
..Book::default()
};
let meta = read("x.mobi", &book.bytes());
assert_eq!(meta.cover, None);
assert_eq!(meta.notes, ["the cover points past the last record"]);
let book = Book {
exth: exth(&[(COVER_OFFSET, &u32::MAX.to_be_bytes())]),
first_image: u32::MAX - 1,
images: vec![JPEG.to_vec()],
..Book::default()
};
assert_eq!(read("x.mobi", &book.bytes()).cover, None);
let book = Book {
exth: exth(&[(COVER_OFFSET, &0u32.to_be_bytes())]),
images: vec![b"not an image".to_vec()],
..Book::default()
};
let meta = read("x.mobi", &book.bytes());
assert_eq!(meta.cover, None);
assert_eq!(meta.notes, ["the cover record is not an image"]);
}
#[test]
fn an_encrypted_book_still_has_its_header_read() {
let book = Book {
encryption: 2,
exth: exth(&[(AUTHOR, b"Somebody")]),
..Book::default()
};
let meta = read("x.azw", &book.bytes());
assert_eq!(meta.authors, ["Somebody"]);
assert_eq!(meta.notes, ["drm"]);
}
#[test]
fn an_exth_record_that_overruns_its_block_stops_the_reading() {
let mut bytes = Book {
exth: exth(&[(AUTHOR, b"Kept"), (PUBLISHER, b"Lost")]),
..Book::default()
}
.bytes();
// The second record's length, made to reach past the block.
let at = find(&bytes, b"Lost") - 4;
bytes[at..at + 4].copy_from_slice(&0x7fff_ffffu32.to_be_bytes());
let meta = read("x.mobi", &bytes);
assert_eq!(meta.authors, ["Kept"]);
assert_eq!(meta.publisher, None);
assert!(
meta.notes
.contains(&"the EXTH block ends early".to_string())
);
}
#[test]
fn a_record_length_shorter_than_its_own_header_stops_the_reading() {
let mut bytes = Book {
exth: exth(&[(AUTHOR, b"Kept")]),
..Book::default()
}
.bytes();
let at = find(&bytes, b"Kept") - 4;
bytes[at..at + 4].copy_from_slice(&3u32.to_be_bytes());
let meta = read("x.mobi", &bytes);
assert!(meta.authors.is_empty());
assert!(
meta.notes
.contains(&"the EXTH block ends early".to_string())
);
}
#[test]
fn what_is_not_a_kindle_book_says_so() {
let meta = read("x.mobi", b"not a palm database at all");
assert_eq!(meta.source, "none");
assert!(meta.notes[0].starts_with("not a Kindle book"));
}
#[test]
fn a_first_record_with_no_mobi_header_is_refused() {
let mut bytes = Book::default().bytes();
let at = mobi_header(&bytes);
bytes[at] = b'X';
let meta = read("x.mobi", &bytes);
assert_eq!(meta.source, "none");
assert_eq!(meta.notes, ["no MOBI header in the first record"]);
}
#[test]
fn a_header_too_short_for_a_field_does_not_have_that_field() {
let mut bytes = Book::default().bytes();
let at = mobi_header(&bytes) + HEADER_LENGTH;
// Long enough for the encoding, too short for the full name.
bytes[at..at + 4].copy_from_slice(&24u32.to_be_bytes());
let meta = read("x.mobi", &bytes);
assert_eq!(meta.title.as_deref(), Some("A_Test_Book"));
assert_eq!(meta.language, None);
}
#[test]
fn every_truncation_returns_without_panicking() {
let bytes = Book {
exth: exth(&[
(UPDATED_TITLE, b"Title"),
(AUTHOR, b"Author"),
(COVER_OFFSET, &0u32.to_be_bytes()),
]),
images: vec![JPEG.to_vec()],
..Book::default()
}
.bytes();
for len in 0..=bytes.len() {
let _ = read("x.mobi", &bytes[..len]);
}
}
/// Where the MOBI header starts: after the record list, so the `MOBI`
/// at the end of the database type is not it.
fn mobi_header(bytes: &[u8]) -> usize {
RECORD_LIST_AT + find(&bytes[RECORD_LIST_AT..], MOBI_MAGIC)
}
fn find(haystack: &[u8], needle: &[u8]) -> usize {
haystack
.windows(needle.len())
.position(|window| window == needle)
.expect("present")
}
}
+28
View File
@@ -0,0 +1,28 @@
# Fixtures
## MOBI and AZW3
`lantern.html` is a two-paragraph story written for these tests, and
`lantern-cover.jpg` a 120×180 cover drawn for them with Pillow. Both are
dedicated to the public domain (CC0 1.0), and so is everything made from them.
`lantern.mobi` (MOBI 6) and `lantern.azw3` (KF8) were made from them with
calibre 9.14:
```sh
common=(--title "A Lantern in the Orchard" --authors "Ada Quill & Tomás Reed"
--publisher "Orchard Press" --isbn 9780306406157 --tags "Fiction,Villages"
--language en --pubdate 2001-04-05 --comments "A short story written for testing."
--cover lantern-cover.jpg)
ebook-convert lantern.html lantern.mobi --mobi-file-type old "${common[@]}"
ebook-convert lantern.html lantern.azw3 "${common[@]}"
```
The other three are `lantern.mobi` with one thing broken. `r0` is the first
record's offset, the big-endian 32-bit number at byte 78 (136 here).
| File | Change |
|---|---|
| `lantern-no-exth.mobi` | The EXTH flag (`0x40`) cleared in the MOBI header's flags, at `r0 + 16 + 112` |
| `lantern-drm.mobi` | The PalmDOC encryption type, at `r0 + 12`, set to 2 |
| `lantern-truncated.mobi` | The first 1024 bytes only |
Binary file not shown.

After

Width:  |  Height:  |  Size: 2.7 KiB

Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+5
View File
@@ -0,0 +1,5 @@
<html><head><meta charset="utf-8"><title>x</title></head><body>
<h1>A Lantern in the Orchard</h1>
<p>The lantern hung from the lowest branch of the pear tree, and every evening someone lit it. Nobody in the village could say who.</p>
<p>Café owners swore it was the miller; the miller swore it was the wind.</p>
</body></html>
Binary file not shown.
+144
View File
@@ -0,0 +1,144 @@
//! MOBI and AZW3 files as a converter writes them, and as they break.
use std::path::{Path, PathBuf};
fn fixture(name: &str) -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures")
.join(name)
}
fn read(name: &str) -> bookmeta::Metadata {
bookmeta::read(&fixture(name))
}
/// What the two converted fixtures were given, and so must say.
fn says_what_it_was_given(meta: &bookmeta::Metadata) {
assert_eq!(meta.source, "mobi");
assert_eq!(meta.title.as_deref(), Some("A Lantern in the Orchard"));
assert_eq!(meta.authors, ["Ada Quill", "Tomás Reed"]);
assert_eq!(meta.publisher.as_deref(), Some("Orchard Press"));
assert_eq!(
meta.description.as_deref(),
Some("A short story written for testing.")
);
assert_eq!(meta.date.as_deref(), Some("2001-04-05T04:00:00+00:00"));
assert_eq!(meta.language.as_deref(), Some("en"));
assert_eq!(meta.subjects, ["Fiction", "Villages"]);
assert_eq!(meta.identifiers["isbn"], "9780306406157");
assert_eq!(meta.identifiers.get("asin"), None);
assert_eq!(meta.identifiers["uuid"].len(), 36);
let cover = meta.cover.as_ref().expect("a cover");
assert_eq!(cover.media_type, "image/jpeg");
let original = std::fs::read(fixture("lantern-cover.jpg")).expect("the cover fixture");
assert_eq!(cover.bytes, original);
}
#[test]
fn a_mobi_6_file_says_what_it_was_given() {
let meta = read("lantern.mobi");
says_what_it_was_given(&meta);
assert!(meta.notes.is_empty(), "{:?}", meta.notes);
}
#[test]
fn an_azw3_file_says_what_it_was_given() {
let meta = read("lantern.azw3");
says_what_it_was_given(&meta);
assert!(meta.notes.is_empty(), "{:?}", meta.notes);
}
#[test]
fn exth_113_uuid_is_not_an_asin() {
for name in ["lantern.mobi", "lantern.azw3"] {
let meta = read(name);
assert_eq!(meta.identifiers.get("asin"), None, "{name}");
assert!(meta.identifiers.contains_key("uuid"), "{name}");
}
}
/// `lantern.mobi` with its cover and thumbnail offsets (EXTH 201 and 202)
/// retyped to an unused record type, so the file names no cover.
fn lantern_naming_no_cover() -> Vec<u8> {
let mut bytes = std::fs::read(fixture("lantern.mobi")).expect("the fixture");
let exth = bytes.windows(4).position(|w| w == b"EXTH").expect("EXTH");
let count = u32::from_be_bytes(bytes[exth + 8..exth + 12].try_into().unwrap());
let mut cursor = exth + 12;
let mut retyped = 0;
for _ in 0..count {
let kind = u32::from_be_bytes(bytes[cursor..cursor + 4].try_into().unwrap());
let size = u32::from_be_bytes(bytes[cursor + 4..cursor + 8].try_into().unwrap());
if kind == 201 || kind == 202 {
bytes[cursor..cursor + 4].copy_from_slice(&999u32.to_be_bytes());
retyped += 1;
}
cursor += size as usize;
}
assert!(retyped > 0, "the fixture names a cover");
bytes
}
#[test]
fn a_file_naming_no_cover_gets_its_first_image_record() {
let meta = bookmeta::from_bytes("x.mobi", &lantern_naming_no_cover());
let cover = meta.cover.expect("the first image record");
assert_eq!(cover.media_type, "image/jpeg");
let original = std::fs::read(fixture("lantern-cover.jpg")).expect("the cover fixture");
assert_eq!(cover.bytes, original);
}
#[test]
fn a_file_with_no_exth_still_has_its_full_name_and_locale() {
let meta = read("lantern-no-exth.mobi");
assert_eq!(meta.source, "mobi");
assert_eq!(meta.title.as_deref(), Some("A Lantern in the Orchard"));
assert_eq!(meta.language.as_deref(), Some("en"));
assert!(meta.authors.is_empty());
assert!(meta.identifiers.is_empty());
// No EXTH names no cover, so the first image record stands in for one.
let cover = meta.cover.expect("the first image record");
assert_eq!(cover.media_type, "image/jpeg");
}
#[test]
fn a_truncated_file_is_an_empty_record_with_a_reason() {
let meta = read("lantern-truncated.mobi");
assert_eq!(meta.source, "none");
assert!(meta.is_empty());
assert_eq!(meta.notes, ["the first record lies outside the file"]);
}
#[test]
fn a_drm_flagged_file_has_its_metadata_and_says_drm() {
let meta = read("lantern-drm.mobi");
says_what_it_was_given(&meta);
assert_eq!(meta.notes, ["drm"]);
}
#[test]
fn every_truncation_of_the_mobi_fixture_returns() {
let bytes = std::fs::read(fixture("lantern.mobi")).expect("the fixture");
let first_record_end = u32::from_be_bytes(bytes[86..90].try_into().unwrap()) as usize;
for len in 0..=bytes.len() {
let meta = bookmeta::from_bytes("cut.mobi", &bytes[..len]);
if len < first_record_end {
assert!(meta.is_empty(), "cut at {len}: {meta:?}");
}
}
}
#[test]
fn every_single_byte_corruption_of_the_header_returns() {
let bytes = std::fs::read(fixture("lantern.mobi")).expect("the fixture");
// The Palm header, the record list, and the whole of the first record:
// every offset the reader follows is in there.
let first_record_end = u32::from_be_bytes(bytes[86..90].try_into().unwrap()) as usize;
for at in 0..first_record_end {
for value in [0x00, 0x7f, 0xff] {
let mut broken = bytes.clone();
broken[at] = value;
let _ = bookmeta::from_bytes("broken.mobi", &broken);
}
}
}