//! The identifiers a bibliography is held together by, read and checked. //! //! A DOI arrives as a bare string, a URL, a `doi:` prefix, with a trailing //! full stop the sentence around it left behind, in whatever case the //! publisher felt like. All of those are the same identifier, and a catalogue //! that stores them as they arrived cannot tell that. //! //! Nothing here asks anybody whether an identifier exists. What it can say is //! whether a string is *well formed*: an ISBN and an ISSN carry a check digit //! that catches a mistyped one, and a DOI or an arXiv id has a shape. That is //! a different question from whether the work is real, and the honest answer //! to the second needs a catalogue. pub mod arxiv; pub mod doi; pub mod isbn; pub mod issn; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; /// What kind of identifier something is. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum Kind { /// A digital object identifier: `10.1000/182`. Doi, /// An international standard serial number, for a journal: `0028-0836`. Issn, /// An international standard book number, for a book. Isbn, /// An arXiv preprint identifier: `2401.01234` or `math.GT/0309136`. Arxiv, } impl Kind { /// What it is called. pub fn name(self) -> &'static str { match self { Kind::Doi => "doi", Kind::Issn => "issn", Kind::Isbn => "isbn", Kind::Arxiv => "arxiv", } } } /// One identifier, as it arrived and as it should be stored. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct Identifier { /// Record version. #[serde(default = "version")] pub v: u32, /// The text it was read from, with any invisible characters removed. pub input: String, /// Which kind it turned out to be. pub kind: Kind, /// The identifier, in the form it should be stored and compared in. pub id: String, /// Whether the check digit agrees, where the kind carries one. `None` /// where the kind has nothing to check against. pub checksum: Option, /// A URL that resolves it, where the kind has a resolver. #[serde(skip_serializing_if = "Option::is_none")] pub url: Option, } fn version() -> u32 { corpus_core::V } impl corpus_core::Record for Identifier { fn version(&self) -> u32 { self.v } } /// Read whatever kind of identifier a string holds. /// /// Tried in order of how distinctive each shape is: a DOI announces itself /// with `10.`, an arXiv id with its own prefix or a dotted archive name, and /// what is left is a run of digits that is either thirteen or ten long (an /// ISBN) or eight (an ISSN). pub fn parse(text: &str) -> Option { parse_as(text, None) } /// A character that takes no width and carries no meaning. fn is_invisible(c: char) -> bool { matches!( c, '\u{00ad}' // soft hyphen | '\u{200b}'..='\u{200f}' // zero-width space, joiners, marks | '\u{2060}'..='\u{2064}' // word joiner and friends | '\u{feff}' // byte-order mark ) } /// Read a string as one particular kind, or as whatever it looks like. pub fn parse_as(text: &str, kind: Option) -> Option { // Text extracted from a PDF or an EPUB is full of characters that take // no width and mean nothing: zero-width spaces and joiners where the // typesetter allowed a line to break, a soft hyphen, a byte-order mark // left at the front of a fragment. None of them is part of an // identifier, and a DOI carrying one does not resolve. let cleaned: String = text.chars().filter(|c| !is_invisible(*c)).collect(); let input = cleaned.trim(); if input.is_empty() { return None; } // Order matters: a DOI announces itself with `10.`, an arXiv id with its // own prefix, and what is left is a run of digits whose length decides // between a book and a journal. const EVERY: [Kind; 4] = [Kind::Doi, Kind::Arxiv, Kind::Isbn, Kind::Issn]; let one = [kind.unwrap_or(Kind::Doi)]; let kinds: &[Kind] = match kind { Some(_) => &one, None => &EVERY, }; for kind in kinds { let found = match kind { Kind::Doi => doi::parse(input).map(|id| Identifier { v: corpus_core::V, url: Some(format!("https://doi.org/{id}")), input: input.to_string(), kind: Kind::Doi, id, checksum: None, }), Kind::Arxiv => arxiv::parse(input).map(|id| Identifier { v: corpus_core::V, url: Some(format!("https://arxiv.org/abs/{id}")), input: input.to_string(), kind: Kind::Arxiv, id, checksum: None, }), Kind::Issn => issn::parse(input).map(|id| Identifier { v: corpus_core::V, checksum: Some(issn::valid(&id)), url: Some(format!("https://portal.issn.org/resource/ISSN/{id}")), input: input.to_string(), kind: Kind::Issn, id, }), Kind::Isbn => isbn(input), }; if let Some(found) = found { return Some(found); } } None } /// An ISBN, using the reader the catalogue tools already share. fn isbn(input: &str) -> Option { let digits = bibclean::isbn::digits(input); if !matches!(digits.len(), 10 | 13) { return None; } // Only where the input is the identifier and nothing else. An ISBN has no // prefix that announces it, so a sentence holding a thirteen-digit number // would otherwise read as one. if !input .chars() .all(|c| c.is_ascii_digit() || "- xX".contains(c)) { return None; } Some(Identifier { v: corpus_core::V, checksum: Some(bibclean::isbn::valid(&digits)), url: Some(format!("https://openlibrary.org/isbn/{digits}")), input: input.to_string(), kind: Kind::Isbn, id: digits, }) } #[cfg(test)] mod tests { #[test] fn an_identifier_copied_out_of_a_pdf_is_full_of_nothing() { // A typesetter's zero-width spaces, where the line was allowed to // break. The DOI does not resolve with them in it. let found = parse("DOI: 10.1093/\u{200b}oso/\u{200b}9780190065065.003.0001").expect("a doi"); assert_eq!(found.id, "10.1093/oso/9780190065065.003.0001"); assert_eq!(found.kind, Kind::Doi); } #[test] fn a_prefix_is_not_cut_off_inside_a_character() { // `10.1093/\u{200b}oso/\u{200b}` is exactly as many bytes as // `https://doi.org/`, and a slice at that length used to land in the // middle of the third zero-width space and bring the process down. for text in [ "10.1093/\u{200b}oso/\u{200b}9780190065065", "\u{feff}10.1000/182", "arxiv:\u{200b}1706.03762", "\u{200b}\u{200b}\u{200b}\u{200b}\u{200b}\u{200b}", "\u{00ad}", ] { // The answer may be None; what it may not do is panic. let _ = parse(text); } } #[test] fn a_soft_hyphen_in_an_isbn_is_not_a_digit() { let found = parse("978\u{00ad}0\u{00ad}306\u{00ad}40615\u{00ad}7").expect("an isbn"); assert_eq!(found.kind, Kind::Isbn); assert_eq!(found.checksum, Some(true)); } use super::*; #[test] fn a_doi_is_read_however_it_was_written() { for text in [ "10.1000/182", "doi:10.1000/182", "https://doi.org/10.1000/182", "http://dx.doi.org/10.1000/182", " 10.1000/182. ", ] { let found = parse(text).unwrap_or_else(|| panic!("{text}")); assert_eq!(found.kind, Kind::Doi, "{text}"); assert_eq!(found.id, "10.1000/182", "{text}"); } } #[test] fn an_issn_carries_a_check_digit() { let found = parse("0028-0836").expect("an issn"); assert_eq!(found.kind, Kind::Issn); assert_eq!(found.id, "0028-0836"); assert_eq!(found.checksum, Some(true)); let wrong = parse("0028-0837").expect("an issn"); assert_eq!(wrong.checksum, Some(false), "one digit out is caught"); } #[test] fn an_isbn_carries_one_too() { let found = parse("978-0-306-40615-7").expect("an isbn"); assert_eq!(found.kind, Kind::Isbn); assert_eq!(found.id, "9780306406157"); assert_eq!(found.checksum, Some(true)); } #[test] fn an_arxiv_identifier_is_read_in_both_its_forms() { assert_eq!(parse("arXiv:2401.01234").expect("an id").id, "2401.01234"); assert_eq!(parse("2401.01234v2").expect("an id").id, "2401.01234v2"); assert_eq!( parse("math.GT/0309136").expect("an id").id, "math.GT/0309136" ); } #[test] fn every_kind_resolves_somewhere() { for text in [ "10.1000/182", "0028-0836", "978-0-306-40615-7", "arXiv:2401.01234", ] { assert!(parse(text).expect("an id").url.is_some(), "{text}"); } } #[test] fn what_is_not_an_identifier_is_not_one() { for text in ["", " ", "hello", "10.", "1234", "not/an/id"] { assert_eq!(parse(text), None, "{text:?}"); } } #[test] fn a_kind_can_be_insisted_on() { // Eight digits are an ISSN; asked for a DOI, they are nothing. assert!(parse_as("00280836", Some(Kind::Issn)).is_some()); assert_eq!(parse_as("00280836", Some(Kind::Doi)), None); } #[test] fn a_sentence_that_merely_contains_digits_is_not_an_isbn() { assert_eq!(parse("published in 9780306406157 by somebody"), None); } }