Find books through CWA's OPDS feed and use its ingest folder

Bindery mounts no CWA database. It searches the OPDS feed by title and
reads the story URL inside each candidate epub to find the library book.
Updates go to the ingest folder with a .cwa.json sidecar, so CWA replaces
the epub of the existing book.
This commit is contained in:
tim
2026-09-27 23:38:28 +02:00
parent c30622e3a0
commit 312dc3efc5
6 changed files with 869 additions and 0 deletions
+83
View File
@@ -0,0 +1,83 @@
"""Optional, read-only access to Calibre-Web accounts and the calibre library."""
import sqlite3
from collections.abc import Iterator
from contextlib import closing, contextmanager
from pathlib import Path
from typing import Any
from werkzeug.security import check_password_hash
# Calibre-Web marks anonymous accounts with this role bit.
ROLE_ANONYMOUS = 1 << 5
class DatabaseUnavailable(Exception):
"""The database cannot answer. The caller can try CWA over HTTP."""
@contextmanager
def _connect(path: Path) -> Iterator[sqlite3.Connection]:
try:
# Do not use immutable=1. CWA can change the database and its WAL at any time.
with closing(sqlite3.connect(path.resolve().as_uri() + "?mode=ro", uri=True, timeout=1)) as db:
db.row_factory = sqlite3.Row
db.execute("PRAGMA query_only = ON")
db.execute("BEGIN")
yield db
except (OSError, sqlite3.Error) as e:
raise DatabaseUnavailable(f"Cannot read {path}: {e}") from e
def login_ok(path: Path, username: str, password: str) -> bool:
"""False means rejected credentials, not a reason to try another login source."""
with _connect(path) as db:
users = db.execute(
"SELECT name, password, role FROM user WHERE name = ? COLLATE NOCASE LIMIT 2", (username,)
).fetchall()
if len(users) != 1 or not password:
return False
user = users[0]
if not isinstance(user["role"], int):
raise DatabaseUnavailable("The Calibre-Web account has no valid role")
if user["name"].casefold() == "guest" or user["role"] & ROLE_ANONYMOUS or not user["password"]:
return False
if not isinstance(user["password"], str):
raise DatabaseUnavailable("The Calibre-Web password hash is not text")
try:
return check_password_hash(user["password"], password)
except ValueError as e:
raise DatabaseUnavailable("The Calibre-Web password hash is not supported") from e
def _book(db: sqlite3.Connection, rows: list[sqlite3.Row]) -> dict[str, Any] | None:
# A duplicate UUID or story identifier is not enough to choose an update target.
if len(rows) != 1 or not rows[0]["uuid"]:
return None
book = dict(rows[0])
book["authors"] = [
row[0]
for row in db.execute(
"SELECT a.name FROM authors a JOIN books_authors_link l ON l.author = a.id WHERE l.book = ? ORDER BY l.id",
(book["id"],),
)
]
book["formats"] = [row[0] for row in db.execute("SELECT format FROM data WHERE book = ?", (book["id"],))]
return book
def book_by_uuid(path: Path, book_uuid: str) -> dict[str, Any] | None:
with _connect(path) as db:
rows = db.execute("SELECT id, uuid, title FROM books WHERE uuid = ? LIMIT 2", (book_uuid,)).fetchall()
return _book(db, rows)
def book_by_url(path: Path, story_url: str) -> dict[str, Any] | None:
"""Match an exact FanFicFare URL identifier. Other URLs use the OPDS search."""
with _connect(path) as db:
rows = db.execute(
"SELECT DISTINCT b.id, b.uuid, b.title FROM books b JOIN identifiers i ON i.book = b.id "
"WHERE i.type = 'url' COLLATE NOCASE AND i.val = ? COLLATE BINARY LIMIT 2",
(story_url,),
).fetchall()
return _book(db, rows)
+309
View File
@@ -0,0 +1,309 @@
"""CWA's OPDS feed, over HTTP.
An optional read-only metadata.db supplies book IDs. Without a database match,
Bindery uses CWA over HTTP. OPDS does not show identifiers, so Bindery searches
by title, then reads the story URL inside each candidate epub.
"""
import logging
import re
import threading
import uuid
from collections.abc import Callable
from dataclasses import dataclass, field
from difflib import SequenceMatcher
from pathlib import Path
from typing import Any
from urllib.parse import urljoin
import requests
from defusedxml import ElementTree
from bindery import calibre_db, config
from bindery.epub import EpubError, epub_urls
logger = logging.getLogger(__name__)
ATOM = "{http://www.w3.org/2005/Atom}"
ACQUISITION = "http://opds-spec.org/acquisition"
# The most epubs that one search downloads to find a story.
MAX_DOWNLOADS = 5
MAX_PAGES = 100
class CwaError(Exception):
pass
class CwaLoginError(CwaError):
"""CWA refused the user name or password."""
@dataclass
class Book:
"""A book as the OPDS feed shows it."""
id: int
uuid: str
title: str
authors: list[str] = field(default_factory=list)
publisher: str = ""
formats: dict[str, str] = field(default_factory=dict) # "epub" -> download href
def to_dict(self) -> dict[str, Any]:
return {
"id": self.id,
"uuid": self.uuid,
"title": self.title,
"authors": " & ".join(self.authors),
"publisher": self.publisher,
"formats": sorted(self.formats),
}
Normalizer = Callable[[str], tuple[str, str] | None]
class CwaClient:
def __init__(self, base_url: str, username: str = "", password: str = "", timeout: int = 60) -> None:
self.base_url = base_url.strip().rstrip("/") + "/" if base_url and base_url.strip() else ""
self.auth = (username, password or "") if username else None
self.timeout = timeout
self.session = requests.Session()
def available(self) -> bool:
return bool(self.base_url)
def _get(self, href: str, auth: Any = None, **kwargs: Any) -> requests.Response:
if not self.available():
raise CwaError("CWA address is not set. Set CWA_URL.")
# hrefs from the feed are absolute paths, and include any path prefix of CWA.
url = urljoin(self.base_url, href)
try:
response = self.session.get(url, auth=auth or self.auth, timeout=self.timeout, **kwargs)
except requests.RequestException as e:
raise CwaError(f"Cannot reach CWA at {self.base_url}: {e}") from e
if response.status_code == 401:
raise CwaLoginError("CWA refused the login")
if response.status_code >= 400:
raise CwaError(f"CWA answered {response.status_code} for {url}")
return response
def check(self) -> str:
"""Log in to the OPDS feed. Returns the catalog title."""
root = ElementTree.fromstring(self._get("opds").content)
return (root.findtext(f"{ATOM}title") or "CWA").strip()
def login_ok(self, username: str, password: str) -> bool:
"""True if CWA accepts this user name and password."""
try:
self._get("opds", auth=(username, password))
except CwaLoginError:
return False
return True
def _feed(self, href: str, params: dict[str, str] | None = None) -> tuple[list[Book], list[tuple[str, str]]]:
"""Books and navigation entries (title, href) of an OPDS feed, with all its pages."""
books: list[Book] = []
navigation: list[tuple[str, str]] = []
for _ in range(MAX_PAGES):
try:
root = ElementTree.fromstring(self._get(href, params=params).content)
except ElementTree.ParseError as e:
raise CwaError(f"CWA sent invalid OPDS XML: {e}") from e
for entry in root.findall(f"{ATOM}entry"):
title = (entry.findtext(f"{ATOM}title") or "").strip()
formats: dict[str, str] = {}
book_id, subsection = None, None
for link in entry.findall(f"{ATOM}link"):
rel, link_href = link.get("rel", ""), link.get("href", "")
if rel == ACQUISITION:
match = re.search(r"/opds/download/(\d+)/(\w+)/?$", link_href)
if match:
book_id = int(match.group(1))
formats[match.group(2).lower()] = link_href
elif rel == "subsection":
subsection = link_href
entry_id = entry.findtext(f"{ATOM}id") or ""
if book_id is not None and entry_id.startswith("urn:uuid:"):
books.append(
Book(
id=book_id,
uuid=entry_id[len("urn:uuid:") :],
title=title,
authors=[(a.findtext(f"{ATOM}name") or "").strip() for a in entry.findall(f"{ATOM}author")],
publisher=(entry.findtext(f"{ATOM}publisher/{ATOM}name") or "").strip(),
formats=formats,
)
)
elif subsection:
navigation.append((title, subsection))
next_href = next(
(link.get("href") for link in root.findall(f"{ATOM}link") if link.get("rel") == "next"), None
)
if not next_href:
break
href, params = next_href, None
return books, navigation
def search(self, query: str) -> list[Book]:
"""CWA's search, which matches title, authors, tags, series and publisher."""
if not query.strip():
return []
return self._feed("opds/search", params={"query": query.strip()})[0]
def get_book(self, book_uuid: str) -> dict[str, Any] | None:
"""A book by its calibre uuid, or None if CWA has no such book."""
# The uuid can come from a request. Other characters could change the path.
if not re.fullmatch(r"[0-9A-Za-z-]+", book_uuid):
return None
if config.CALIBRE_DB_PATH:
try:
book = calibre_db.book_by_uuid(config.CALIBRE_DB_PATH, book_uuid)
if book:
return book
except calibre_db.DatabaseUnavailable as e:
logger.warning(f"Cannot read the calibre library database. Trying CWA: {e}")
response = self._get(f"ajax/book/{book_uuid}")
if not response.text.strip():
return None
try:
data = response.json()
except ValueError as e:
raise CwaError(f"CWA sent invalid book record: {e}") from e
if data.get("uuid") != book_uuid:
return None
return {
"id": int(data["application_id"]),
"uuid": book_uuid,
"title": data.get("title") or "",
"authors": data.get("authors") or [],
"formats": data.get("formats") or [],
}
def book_by_url(self, story_url: str) -> dict[str, Any] | None:
"""Try the library database. A miss leaves the OPDS search to the caller."""
if config.CALIBRE_DB_PATH:
try:
return calibre_db.book_by_url(config.CALIBRE_DB_PATH, story_url)
except calibre_db.DatabaseUnavailable as e:
logger.warning(f"Cannot read the calibre library database. Trying OPDS: {e}")
return None
def download_epub(self, book: Book, dest: Path) -> Path:
href = book.formats.get("epub")
if not href:
raise CwaError(f"Library book {book.id} has no epub")
dest.parent.mkdir(parents=True, exist_ok=True)
with self._get(href, stream=True) as response, open(dest, "wb") as f:
for chunk in response.iter_content(chunk_size=1 << 16):
f.write(chunk)
return dest
def site_books(self, is_site: Callable[[str], bool]) -> list[Book]:
"""Books whose publisher is a story site. FanFicFare sets the publisher to the site name."""
try:
_, publishers = self._feed("opds/publisher")
except CwaError as e:
raise CwaError(f'{e}. Turn on "Show Publisher Section" for Bindery in CWA.') from e
books: list[Book] = []
for name, href in publishers:
if is_site(name):
books.extend(b for b in self._feed(href)[0] if "epub" in b.formats)
return books
# A part of a title in parentheses or brackets, such as "[Tag A, Tag B]" or "(Book 1 Complete)".
BRACKETED = re.compile(r"\([^()]*\)|\[[^\[\]]*\]")
# Characters that a title can keep at an end after its bracketed parts go.
TITLE_EDGES = " -\u2013\u2014:|,."
def plain_title(title: str) -> str:
"""A title without its bracketed parts, in lower case, for comparisons."""
previous = None
while previous != title:
previous, title = title, BRACKETED.sub(" ", title)
return " ".join(title.split()).strip(TITLE_EDGES).casefold()
def search_text(title: str) -> str:
"""The text that CWA searches for. CWA finds titles that contain the text, so the text before the first
bracket finds the title with and without its bracketed parts."""
head = re.split(r"[(\[]", title, maxsplit=1)[0].strip(TITLE_EDGES)
return head or plain_title(title)
def title_rank(book_title: str, titles: list[str]) -> tuple[int, float] | None:
"""How close a library title is to one of the story titles. A lower rank is closer, and None is no match.
Rank 0 is the same title. Rank 1 is the same title without the bracketed parts. Rank 2 is a title that
starts with the other title without the bracketed parts, such as "Story A Part I" for "Story A". In a
rank, the more similar text comes first.
"""
full, plain = book_title.casefold(), plain_title(book_title)
ranks = []
for title in titles:
other = plain_title(title)
if full == title.casefold():
rank = 0
elif plain and plain == other:
rank = 1
elif plain and other and (plain.startswith(other + " ") or other.startswith(plain + " ")):
rank = 2
else:
continue
ranks.append((rank, -SequenceMatcher(None, full, title.casefold()).ratio()))
return min(ranks, default=None)
class BookFinder:
"""Finds the library book of a story, and remembers books that are not it."""
def __init__(self) -> None:
self._not_story: dict[str, set[int]] = {}
self._lock = threading.Lock()
def find(
self,
client: CwaClient,
story_url: str,
titles: list[str],
normalize: Normalizer,
work_dir: Path,
exclude_ids: set[int] = frozenset(),
) -> tuple[Book, Path] | None:
"""Return the book and its downloaded epub, or None. The caller deletes the epub.
Candidates are books with an epub and a close title (see title_rank). Bindery downloads
them, closest title first and then newest first, and compares the story URL in each epub.
"""
wanted = [t for t in titles if t]
candidates: dict[int, tuple[tuple[int, float], Book]] = {}
for text in dict.fromkeys(search_text(t) for t in wanted):
for book in client.search(text):
rank = title_rank(book.title, wanted)
if rank is not None and "epub" in book.formats:
candidates[book.id] = (rank, book)
with self._lock:
rejected = set(self._not_story.get(story_url, set()))
checked = 0
for _, book in sorted(candidates.values(), key=lambda c: (c[0], -c[1].id)):
if book.id in exclude_ids or book.id in rejected:
continue
if checked >= MAX_DOWNLOADS:
break
checked += 1
path = client.download_epub(book, work_dir / f"cwa-{book.id}-{uuid.uuid4().hex[:8]}.epub")
try:
urls = epub_urls(path)
except EpubError as e:
logger.warning(f"Cannot read the epub of library book {book.id}: {e}")
urls = []
if any((normalize(u) or (None,))[0] == story_url for u in urls):
return book, path
with self._lock:
self._not_story.setdefault(story_url, set()).add(book.id)
path.unlink(missing_ok=True)
return None
+73
View File
@@ -0,0 +1,73 @@
"""Read the OPF metadata of an epub that came from CWA."""
import re
import zipfile
from pathlib import Path
from typing import Any
from xml.etree.ElementTree import Element
from defusedxml import ElementTree
OPF = "{http://www.idpf.org/2007/opf}"
DC = "{http://purl.org/dc/elements/1.1/}"
CONTAINER = "{urn:oasis:names:tc:opendocument:xmlns:container}"
class EpubError(Exception):
pass
def _metadata(path: Path) -> Element:
try:
with zipfile.ZipFile(path) as z:
container = ElementTree.fromstring(z.read("META-INF/container.xml"))
rootfile = container.find(f".//{CONTAINER}rootfile")
if rootfile is None or not rootfile.get("full-path"):
raise EpubError(f"{path.name} has no OPF file")
opf = ElementTree.fromstring(z.read(rootfile.get("full-path")))
except (zipfile.BadZipFile, KeyError, ElementTree.ParseError) as e:
raise EpubError(f"{path.name} is not a readable epub: {e}") from e
metadata = opf.find(f"{OPF}metadata")
if metadata is None:
raise EpubError(f"{path.name} has no OPF metadata")
return metadata
def epub_urls(path: Path) -> list[str]:
"""Every URL in the identifiers and the source of an epub's OPF.
FanFicFare writes the story URL as dc:source and as a URL identifier. CWA
keeps the identifier when it writes its own metadata into a download.
"""
urls: list[str] = []
for element in _metadata(path):
if element.tag not in (f"{DC}identifier", f"{DC}source"):
continue
match = re.search(r"https?://\S+", element.text or "")
if match and match.group(0) not in urls:
urls.append(match.group(0))
return urls
def epub_metadata(path: Path) -> dict[str, Any]:
"""The metadata fields of an epub's OPF, in Bindery's field format."""
metadata = _metadata(path)
def texts(tag: str) -> list[str]:
return [(e.text or "").strip() for e in metadata.findall(f"{DC}{tag}") if (e.text or "").strip()]
meta = {m.get("name"): m.get("content") for m in metadata.findall(f"{OPF}meta") if m.get("name")}
try:
series_index = float(meta["calibre:series_index"]) if meta.get("calibre:series_index") else None
except ValueError:
series_index = None
return {
"title": (texts("title") or [""])[0],
"authors": texts("creator"),
"comments": (texts("description") or [""])[0],
"tags": texts("subject"),
"series": meta.get("calibre:series") or "",
"series_index": series_index,
"pubdate": (texts("date") or [""])[0],
"languages": (texts("language") or [""])[0],
}
+68
View File
@@ -0,0 +1,68 @@
"""Hand-off to CWA through its ingest folder. CWA's own calibredb does every write.
- A new story goes in as a plain epub, and CWA imports it as a new book.
- An update goes in with a sidecar file, <name>.epub.cwa.json, that holds
{"action": "add_format", "book_id": N}. CWA then replaces the epub of book N.
CWA's own web UI uses the same sidecar, in cps/editbooks.py.
"""
import json
import logging
import os
import re
import shutil
import uuid
from pathlib import Path
logger = logging.getLogger(__name__)
# CWA shortens ingest file names over 150 characters. That would break the
# match between an epub and its sidecar file, so names stay well under it.
MAX_NAME_LENGTH = 100
def sanitize_filename(name: str) -> str:
name = re.sub(r'[\\/:*?"<>|\x00-\x1f]', "_", name).strip(" .")
return name[:MAX_NAME_LENGTH].strip(" .") or "story"
class IngestFolder:
def __init__(self, path: Path) -> None:
self.path = Path(path)
def _place(self, epub: Path, final_path: Path) -> Path:
"""Copy an epub in under a temporary .part name, then rename it.
CWA ignores .part files, and reacts to the rename. A file without an
extension is not safe: CWA deletes files it cannot import.
"""
partial = final_path.with_name(final_path.name + ".part")
shutil.copyfile(epub, partial)
os.replace(partial, final_path)
return final_path
def ingest(self, epub: Path, name: str) -> Path:
"""Send a new book to CWA."""
self.path.mkdir(parents=True, exist_ok=True)
# A short unique suffix, so a file that CWA has not imported yet is never replaced.
stem = sanitize_filename(f"{name[:80]} {uuid.uuid4().hex[:6]}")
final_path = self._place(epub, self.path / f"{stem}.epub")
logger.info(f"Sent {final_path.name} to the ingest folder as a new book")
return final_path
def send_update(self, epub: Path, book_id: int, name: str) -> Path:
"""Send a new epub for an existing book. CWA replaces the book's epub."""
self.path.mkdir(parents=True, exist_ok=True)
# A unique name, so two updates of one book never share a sidecar file.
stem = sanitize_filename(f"{name[:60]} {book_id}-{uuid.uuid4().hex[:8]}")
final_path = self.path / f"{stem}.epub"
manifest = final_path.with_name(final_path.name + ".cwa.json")
# The sidecar must exist before the epub arrives. CWA skips .cwa.json files.
manifest.write_text(json.dumps({"action": "add_format", "book_id": int(book_id)}), encoding="utf-8")
try:
self._place(epub, final_path)
except Exception:
manifest.unlink(missing_ok=True)
raise
logger.info(f"Sent {final_path.name} to the ingest folder as an update of book {book_id}")
return final_path