mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-11-14 08:05:40 +00:00
bookmeta, bibfetch, bindings: what a caller needs to trust a cover
bookmeta: Cover::guessed says the file left the choice open — a Kindle book's first image record, or an EPUB image picked because its name says cover — so a caller with a register to ask can prefer that. A cover the file names is never a guess. Cover::extension matches bibfetch's. bibfetch: Open Library answers 403 to more than a hundred covers by ISBN in five minutes from one address. That is now Error::Limited, not Unreachable: the other form of the ISBN is not asked, the cover id still is, since those are not limited, and the CLI exits 75. Covers asked by ISBN are kept three seconds apart across the process (Covers::pace), which stays under the limit however long a run goes. A deadline too long to add to the clock no longer panics; it is no deadline. bindings: fetch_cover and search_title for Python, and catena.Limited, which fetch_identifier raises too when a register refuses for being asked too often. corpus-core: exit 75, a service refused for now.
This commit is contained in:
14 files changed
+334
-16
No files matched your search
@@ -21,6 +21,8 @@ from ._catena import (
|
||||
compose_subjects,
|
||||
count_tokens,
|
||||
cue_windows,
|
||||
Limited,
|
||||
fetch_cover,
|
||||
fetch_identifier,
|
||||
find_citations,
|
||||
fuse_hits,
|
||||
@@ -41,6 +43,7 @@ from ._catena import (
|
||||
record_severity,
|
||||
same_work,
|
||||
scrub,
|
||||
search_title,
|
||||
segment_cues,
|
||||
series_of,
|
||||
shelve,
|
||||
@@ -53,6 +56,7 @@ from ._catena import (
|
||||
|
||||
__all__ = [
|
||||
"__version__",
|
||||
"Limited",
|
||||
"canonical_subject",
|
||||
"chunk_markdown",
|
||||
"citation_prompt",
|
||||
@@ -62,6 +66,7 @@ __all__ = [
|
||||
"compose_subjects",
|
||||
"count_tokens",
|
||||
"cue_windows",
|
||||
"fetch_cover",
|
||||
"fetch_identifier",
|
||||
"find_citations",
|
||||
"fuse_hits",
|
||||
@@ -82,6 +87,7 @@ __all__ = [
|
||||
"record_severity",
|
||||
"same_work",
|
||||
"scrub",
|
||||
"search_title",
|
||||
"segment_cues",
|
||||
"series_of",
|
||||
"shelve",
|
||||
|
||||
@@ -103,6 +103,40 @@ def fetch_identifier(
|
||||
"""
|
||||
...
|
||||
|
||||
class Limited(Exception):
|
||||
"""A register refused because it has been asked too often; ask again later."""
|
||||
|
||||
def fetch_cover(
|
||||
isbn: str,
|
||||
cover_id: int | None = ...,
|
||||
size: str = ...,
|
||||
polite: str | None = ...,
|
||||
timeout: int = ...,
|
||||
deadline: int = ...,
|
||||
) -> dict[str, Any] | None:
|
||||
"""The cover Open Library has for a book, by ISBN, then by cover id.
|
||||
|
||||
A dict with `bytes`, `media_type`, `extension`, `source` and `url`, or
|
||||
`None` when there is none. Raises `Limited` when Open Library refused for
|
||||
being asked too often. Blocks on the network.
|
||||
"""
|
||||
...
|
||||
|
||||
def search_title(
|
||||
title: str,
|
||||
author: str | None = ...,
|
||||
limit: int = ...,
|
||||
polite: str | None = ...,
|
||||
timeout: int = ...,
|
||||
attempts: int = ...,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""The editions the registers list for a title and author, in their order.
|
||||
|
||||
Records shaped like `fetch_identifier`'s. Use `same_work` to decide which
|
||||
one is the book. Blocks on the network.
|
||||
"""
|
||||
...
|
||||
|
||||
def colophon_prompt(
|
||||
record: dict[str, Any], text: str, pages: int = ...
|
||||
) -> str | None:
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
use pyo3::exceptions::PyValueError;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyList};
|
||||
use pyo3::types::{PyBytes, PyDict, PyList};
|
||||
|
||||
use crate::{ChunkBudgets, CueBudgets};
|
||||
|
||||
@@ -434,6 +434,9 @@ fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
module.add_function(wrap_pyfunction!(read_colophon, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(shelve, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(fetch_identifier, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(fetch_cover, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(search_title, module)?)?;
|
||||
module.add("Limited", module.py().get_type::<Limited>())?;
|
||||
module.add_function(wrap_pyfunction!(colophon_prompt, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(merge_colophon, module)?)?;
|
||||
Ok(())
|
||||
@@ -512,13 +515,112 @@ fn fetch_identifier<'py>(
|
||||
// would stop every other thread in the interpreter for the whole request.
|
||||
let found = py
|
||||
.detach(|| bibfetch::ask::lookup(&agent, id, kind, ask.attempts))
|
||||
.map_err(|e| PyValueError::new_err(e.to_string()))?;
|
||||
.map_err(register_error)?;
|
||||
match found {
|
||||
Some(record) => Ok(Some(to_py(py, &record)?)),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pyo3::create_exception!(
|
||||
_catena,
|
||||
Limited,
|
||||
pyo3::exceptions::PyException,
|
||||
"A register refused because it has been asked too often; ask again later."
|
||||
);
|
||||
|
||||
/// A register's error as the Python exception that says what to do about it.
|
||||
fn register_error(error: bibfetch::Error) -> PyErr {
|
||||
match error {
|
||||
bibfetch::Error::Limited(why) => Limited::new_err(why),
|
||||
other => PyValueError::new_err(other.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The cover Open Library has for a book, by its ISBN, then by the cover id
|
||||
/// a record named.
|
||||
///
|
||||
/// Returns a dict with `bytes`, `media_type` (`image/jpeg` or `image/png`),
|
||||
/// `extension`, `source` and `url`, or `None` when there is no cover. Covers
|
||||
/// asked by ISBN are kept three seconds apart across the process, and the
|
||||
/// whole fetch stops after `deadline` seconds. Raises `catena.Limited` when
|
||||
/// Open Library refused for being asked too often, and `ValueError` when the
|
||||
/// ISBN is not one or the store could not be reached. Blocks on the network.
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (isbn, cover_id = None, size = "large", polite = None, timeout = 20, deadline = 120))]
|
||||
fn fetch_cover<'py>(
|
||||
py: Python<'py>,
|
||||
isbn: &str,
|
||||
cover_id: Option<u64>,
|
||||
size: &str,
|
||||
polite: Option<String>,
|
||||
timeout: u64,
|
||||
deadline: u64,
|
||||
) -> PyResult<Option<Bound<'py, PyDict>>> {
|
||||
let size = match size {
|
||||
"small" => bibfetch::CoverSize::Small,
|
||||
"medium" => bibfetch::CoverSize::Medium,
|
||||
"large" => bibfetch::CoverSize::Large,
|
||||
other => {
|
||||
return Err(PyValueError::new_err(format!(
|
||||
"size is small, medium or large, not {other:?}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
let agent = bibfetch::ask::Ask {
|
||||
polite,
|
||||
timeout: std::time::Duration::from_secs(timeout.max(1)),
|
||||
..Default::default()
|
||||
}
|
||||
.agent();
|
||||
let covers = bibfetch::cover::Covers {
|
||||
deadline: std::time::Duration::from_secs(deadline.max(1)),
|
||||
..Default::default()
|
||||
};
|
||||
let found = py
|
||||
.detach(|| covers.fetch(&agent, isbn, cover_id, size))
|
||||
.map_err(register_error)?;
|
||||
let Some(cover) = found else {
|
||||
return Ok(None);
|
||||
};
|
||||
let out = PyDict::new(py);
|
||||
out.set_item("bytes", PyBytes::new(py, &cover.bytes))?;
|
||||
out.set_item("media_type", cover.media_type)?;
|
||||
out.set_item("extension", cover.extension())?;
|
||||
out.set_item("source", cover.source)?;
|
||||
out.set_item("url", cover.url)?;
|
||||
Ok(Some(out))
|
||||
}
|
||||
|
||||
/// The editions the registers list for a title, and an author if one is known.
|
||||
///
|
||||
/// Open Library is asked first and Google Books only when it has nothing.
|
||||
/// The candidates come back in the register's own order, as records shaped
|
||||
/// like `fetch_identifier`'s; nothing here decides which is the book, which is
|
||||
/// what `same_work` is for. Blocks on the network.
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (title, author = None, limit = 10, polite = None, timeout = 20, attempts = 3))]
|
||||
fn search_title<'py>(
|
||||
py: Python<'py>,
|
||||
title: &str,
|
||||
author: Option<&str>,
|
||||
limit: usize,
|
||||
polite: Option<String>,
|
||||
timeout: u64,
|
||||
attempts: u32,
|
||||
) -> PyResult<Bound<'py, PyList>> {
|
||||
let ask = bibfetch::ask::Ask {
|
||||
polite,
|
||||
timeout: std::time::Duration::from_secs(timeout.max(1)),
|
||||
attempts: attempts.max(1),
|
||||
};
|
||||
let agent = ask.agent();
|
||||
let found = py
|
||||
.detach(|| bibfetch::search(&agent, title, author, limit, ask.attempts))
|
||||
.map_err(register_error)?;
|
||||
list_to_py(py, &found)
|
||||
}
|
||||
|
||||
/// The question to put to a model about the fields a book's pages did not
|
||||
/// settle, or `None` when they settled everything.
|
||||
///
|
||||
|
||||
Reference in new issue
Block a user