From c30622e3a038a89fb040208e4d7a247f595d635e Mon Sep 17 00:00:00 2001 From: Timothy Allen Date: Sat, 26 Sep 2026 08:06:02 +0200 Subject: [PATCH] Download stories and build epubs with FanFicFare FanFicFare adapters fetch metadata and chapters, and its epub writer builds each book from the store. Environment variables configure the app, and a default personal.ini is copied to the config folder on first start. --- bindery/config.py | 88 +++++++++++ bindery/fff.py | 335 ++++++++++++++++++++++++++++++++++++++++ data/personal.ini | 28 ++++ tests/conftest.py | 18 +++ tests/test_build.py | 155 +++++++++++++++++++ tests/test_config.py | 112 ++++++++++++++ tests/test_fff_flows.py | 108 +++++++++++++ 7 files changed, 844 insertions(+) create mode 100644 bindery/config.py create mode 100644 bindery/fff.py create mode 100644 data/personal.ini create mode 100644 tests/test_build.py create mode 100644 tests/test_config.py create mode 100644 tests/test_fff_flows.py diff --git a/bindery/config.py b/bindery/config.py new file mode 100644 index 0000000..d58c8ba --- /dev/null +++ b/bindery/config.py @@ -0,0 +1,88 @@ +"""Configuration. Every setting comes from an environment variable.""" + +import logging +import os +import shutil +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def _bool(name: str, default: str = "false") -> bool: + return os.getenv(name, default).strip().lower() in ("true", "yes", "1", "y", "on") + + +def _list(name: str, default: str = "") -> list[str]: + return [v.strip() for v in os.getenv(name, default).split(",") if v.strip()] + + +# A relative path is relative to the working folder. Paths are made absolute at once, because Flask's send_file +# reads a relative path from the package folder. +def _path(name: str, default: str) -> Path: + return Path(os.getenv(name, default)).absolute() + + +def _optional_path(name: str) -> Path | None: + value = os.getenv(name, "").strip() + return Path(value).absolute() if value else None + + +def _base_path(value: str) -> str: + path = value.strip().strip("/") + return f"/{path}" if path else "" + + +PROJECT_ROOT = Path(__file__).resolve().parent.parent + +CONFIG_DIR = _path("CONFIG_DIR", "/config") +INGEST_DIR = _path("INGEST_DIR", "/cwa-book-ingest") +TMP_DIR = _path("TMP_DIR", "/tmp/bindery") +LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").strip().upper() +# A path prefix, such as /bindery, for a reverse proxy that forwards the full path. +URL_BASE = _base_path(os.getenv("URL_BASE", "")) + +# The CWA account that Bindery uses for the OPDS feed. +CWA_URL = os.getenv("CWA_URL", "").strip() +CWA_USERNAME = os.getenv("CWA_USERNAME", "").strip() +CWA_PASSWORD = os.getenv("CWA_PASSWORD", "") +# Independent, optional read-only databases. HTTP remains the fallback. +CWA_DB_PATH = _optional_path("CWA_DB_PATH") # Calibre-Web app.db, for logins +CALIBRE_DB_PATH = _optional_path("CALIBRE_DB_PATH") # calibre metadata.db, for book IDs + +UPDATE_INTERVAL_HOURS = float(os.getenv("UPDATE_INTERVAL_HOURS", "24")) +DEFAULT_LOCKED_FIELDS = _list("DEFAULT_LOCKED_FIELDS", "title") + +# Logins: none, cwa or proxy. An API key works in every mode. +AUTH_METHOD = os.getenv("AUTH_METHOD", "none").strip().lower() +CWA_ALLOWED_USERS = _list("CWA_ALLOWED_USERS") +PROXY_AUTH_USER_HEADER = os.getenv("PROXY_AUTH_USER_HEADER", "X-Remote-User").strip() +PROXY_AUTH_LOGOUT_URL = os.getenv("PROXY_AUTH_LOGOUT_URL", "").strip() +API_KEY = os.getenv("BINDERY_API_KEY", "").strip() +SESSION_COOKIE_SECURE = _bool("SESSION_COOKIE_SECURE") +# Take the client address from the last entry of X-Forwarded-For, which a reverse proxy adds. +TRUST_PROXY = _bool("TRUST_PROXY") + +# Tests turn off the job threads. +START_THREADS = _bool("BINDERY_START_THREADS", "true") + +SITE_DELAY = 5 # seconds between two jobs that go to a story site +JOB_HISTORY = 60 * 60 # seconds that finished jobs stay in the job list + +DB_PATH = CONFIG_DIR / "bindery.db" +UPLOAD_DIR = CONFIG_DIR / "uploads" # uploaded epubs wait here for their job, also over a restart +SECRET_KEY_FILE = CONFIG_DIR / ".flask_secret" +PERSONAL_INI = CONFIG_DIR / "personal.ini" +DEFAULT_PERSONAL_INI = PROJECT_ROOT / "data" / "personal.ini" +BACKUP_DIR = CONFIG_DIR / "backups" +BACKUP_COUNT = 7 # daily backups to keep + + +def prepare() -> None: + """Make the folders, and give the user a personal.ini to edit on first start.""" + for directory in (CONFIG_DIR, UPLOAD_DIR, TMP_DIR, INGEST_DIR): + directory.mkdir(parents=True, exist_ok=True) + if not PERSONAL_INI.exists(): + shutil.copy(DEFAULT_PERSONAL_INI, PERSONAL_INI) + logger.info(f"Created the default FanFicFare configuration at {PERSONAL_INI}") + if not CWA_URL: + logger.warning("CWA_URL is not set. OPDS imports and HTTP fallbacks are unavailable.") diff --git a/bindery/fff.py b/bindery/fff.py new file mode 100644 index 0000000..b87041a --- /dev/null +++ b/bindery/fff.py @@ -0,0 +1,335 @@ +"""Headless FanFicFare wrapper: site access, chapter fetches and epub builds. + +This module uses FanFicFare's adapters (the site scrapers) and its epub writer +without changes. The download and update flow of the FanFicFare CLI is not used. +Instead, chapters live in the store, and every epub is built from the store: + + 1. fetch_story() reads metadata and the chapter list from the site. + 2. fetch_chapter() downloads one chapter and returns its HTML with image + sources set back to their original URLs. + 3. build_epub() gives FanFicFare's writer the stored chapters as "old + chapters" (the same mechanism FanFicFare uses for epub updates), so the + writer never goes to the site. +""" + +import configparser +import html +import io +import logging +import re +from dataclasses import dataclass, field +from datetime import datetime +from functools import lru_cache +from pathlib import Path +from typing import Any + +import fanficfare +from bs4 import BeautifulSoup +from fanficfare import adapters, exceptions, writers +from fanficfare.configurable import Configuration +from fanficfare.epubutils import get_update_data +from fanficfare.story import imagetypes + +from bindery import config +from bindery.diffing import html_to_blocks +from bindery.rules import Step, apply_rules + +logger = logging.getLogger(__name__) + +# FanFicFare logs every request at DEBUG level. +logging.getLogger("fanficfare").setLevel(logging.DEBUG if config.LOG_LEVEL == "DEBUG" else logging.WARNING) + +DEFAULTS_INI = Path(fanficfare.__file__).parent / "defaults.ini" + +# Settings this application depends on. They go in [overrides], so +# personal.ini cannot change them. +FORCED_SETTINGS = { + "always_reload_first_chapter": "false", + "mark_new_chapters": "false", + "continue_on_chapter_error": "false", + "make_directories": "false", + "zip_output": "false", + # The epub is the only way metadata reaches CWA, so the series must be in the OPF. + "calibre_series_meta": "true", +} + + +class StoryError(Exception): + """A problem the user can fix, such as an unsupported URL.""" + + +@dataclass +class SiteSnapshot: + """What the site says about a story right now.""" + + url: str + site: str + title: str + author: str + metahtml: str + chapters: list[dict[str, str]] + calibre: dict[str, Any] + cover: dict[str, Any] | None = None + + +@dataclass +class EpubContents: + """Chapters and images read from an existing FanFicFare epub.""" + + url: str | None + chapters: list[dict[str, str]] = field(default_factory=list) + images: list[dict[str, Any]] = field(default_factory=list) + + +def escape_title(text: str) -> str: + """A chapter title in FanFicFare's form, which the store keeps. + + FanFicFare keeps &, < and > in chapter titles, because its epub + writer puts titles into XHTML as they are. + """ + return html.escape(text, quote=False) + + +def plain_title(title: str) -> str: + """A stored chapter title as text, for a message.""" + return html.unescape(title) + + +@lru_cache(maxsize=20000) +def normalize_url(url: str) -> tuple[str, str] | None: + """Return (story URL, site) for a supported URL, or None. No network access.""" + try: + return adapters.getNormalStoryURLSite(url.strip()) + except Exception: + return None + + +@lru_cache(maxsize=1) +def _site_names() -> frozenset: + return frozenset(name.lower() for name in adapters.getSiteSections()) + + +def is_supported_site(name: str) -> bool: + """True for a site name that FanFicFare supports, such as www.royalroad.com.""" + return name.strip().lower() in _site_names() + + +def get_configuration(url: str) -> Configuration: + try: + configuration = Configuration(adapters.getConfigSectionsFor(url), "epub") + except exceptions.UnknownSite as e: + raise StoryError(f"FanFicFare does not support site: {url}") from e + with open(DEFAULTS_INI, encoding="utf-8") as f: + configuration.read_file(f) + if config.PERSONAL_INI.exists(): + with open(config.PERSONAL_INI, encoding="utf-8") as f: + configuration.read_file(f) + try: + configuration.add_section("overrides") + except configparser.DuplicateSectionError: + pass + for key, value in FORCED_SETTINGS.items(): + configuration.set("overrides", key, value) + # FanFicFare downloads the cover only if include_images is on, and it is off by default. "coveronly" keeps the + # cover, but still leaves out the images in the story text. + if not configuration.getConfig("include_images"): + configuration.set("overrides", "include_images", "coveronly") + configuration.cached_config.clear() + return configuration + + +def check_personal_ini(text: str) -> list[str]: + """Check the text of a personal.ini. + + Raise ValueError for text that FanFicFare cannot read. Return the problems + that FanFicFare's own check finds, such as a section for an unknown site. + """ + configuration = Configuration(["defaults"], "epub") + try: + configuration.read_file(io.StringIO(text)) + except configparser.ParsingError as e: + lines = "; ".join(f"line {n}: {line.strip()}" for n, line in e.errors) + raise ValueError(f"FanFicFare cannot read this personal.ini. Check {lines}") from e + except configparser.Error as e: + raise ValueError(f"FanFicFare cannot read this personal.ini: {e}") from e + return [f"Line {n}: {message}" for n, message in configuration.test_config()] + + +def get_adapter(url: str): + try: + return adapters.getAdapter(get_configuration(url), url) + except exceptions.InvalidStoryURL as e: + raise StoryError(str(e)) from e + + +def parse_series(value: str) -> tuple[str, float | None]: + m = re.match(r"^(.*?)\s*\[([\d.]+)\]$", value or "") + if m: + return m.group(1), float(m.group(2)) + return value or "", None + + +def calibre_metadata(story) -> dict[str, Any]: + """Map FanFicFare story metadata to calibre fields.""" + series, series_index = parse_series(story.getMetadata("series", removeallentities=True)) + published = story.getMetadataRaw("datePublished") + return { + "title": story.getMetadata("title", removeallentities=True), + "authors": story.getList("author", removeallentities=True), + "comments": story.getMetadata("description"), + "tags": story.getSubjectTags(removeallentities=True), + "series": series, + "series_index": series_index, + "pubdate": published.isoformat() if hasattr(published, "isoformat") else "", + "languages": story.getMetadata("langcode"), + } + + +def fetch_story(adapter) -> SiteSnapshot: + """Read story metadata, the chapter list and the cover from the site.""" + try: + story = adapter.getStoryMetadataOnly(get_cover=True) + except (exceptions.StoryDoesNotExist, exceptions.AccessDenied, exceptions.FailedToLogin) as e: + raise StoryError(f"{type(e).__name__}: {e}") from e + except exceptions.AdultCheckRequired as e: + raise StoryError("Site asks for adult check. Set is_adult:true in personal.ini.") from e + cover = None + info = story.img_store.cover + if info and info.get("data"): + cover = {k: info[k] for k in ("url", "ext", "mime", "data")} + return SiteSnapshot( + url=story.getMetadata("storyUrl"), + site=adapter.getSiteDomain(), + title=story.getMetadata("title", removeallentities=True), + author=", ".join(story.getList("author", removeallentities=True)), + metahtml=story.dump_html_metadata(), + chapters=[{"url": c["url"], "title": c["title"]} for c in adapter.get_chapters()], + calibre=calibre_metadata(story), + cover=cover, + ) + + +def canonical_html(html: str) -> str: + """Point tags back at their original URLs, as FanFicFare's update code does.""" + if " str: + """Download one chapter. Images go into the adapter's image store.""" + return canonical_html(adapter.getChapterTextNum(url, index)) + + +def collected_images(adapter) -> list[dict[str, Any]]: + """Images downloaded so far by this adapter, without the cover.""" + images = [] + for info in adapter.story.img_store.infos: + if info.get("data") and info.get("newsrc") != "failedtoload": + images.append({k: info[k] for k in ("url", "ext", "mime", "data")}) + return images + + +def word_count(chapters: list[dict[str, Any]]) -> int: + return sum(len(block.split()) for c in chapters for block in html_to_blocks(c["html"])) + + +def apply_overrides(story, overrides: dict[str, Any]) -> None: + """Put the kept values of locked fields into FanFicFare's story metadata. + + The epub writer takes the OPF metadata from these entries, and CWA takes + the book record from the OPF on the first import. + """ + for key, value in overrides.items(): + if key == "title" and value: + story.setMetadata("title", value) + elif key == "authors" and value: + for n, author in enumerate(value): + story.addToList("author", author, clear=(n == 0)) + elif key == "comments": + story.setMetadata("description", value or "") + elif key == "series": + name, index = value if value else ("", None) + if name and index is not None: + story.setMetadata("series", f"{name} [{index:g}]") + else: + story.setMetadata("series", name or "") + elif key == "pubdate": + try: + story.setMetadata("datePublished", datetime.fromisoformat(value) if value else None) + except ValueError: + logger.warning(f"Ignoring a kept publication date that is not a date: {value!r}") + elif key == "languages" and value: + story.setMetadata("langcode", value) + elif key == "tags": + tags = list(value or []) + # getSubjectTags() builds the tags from configuration. The locked + # list replaces it for this build only. + story.getSubjectTags = lambda removeallentities=False, tags=tags: tags + + +def build_epub( + url: str, + metahtml: str, + chapters: list[dict[str, Any]], + images: list[dict[str, Any]], + cover: dict[str, Any] | None, + rules: list[Step], + overrides: dict[str, Any], + out_path: Path, +) -> dict[str, Any]: + """Write an epub from stored data. No network access. + + chapters: dicts with url, title and html, in book order. + overrides: calibre field values that replace the site values in the epub + (locked fields). + Returns the calibre metadata of the built book. + """ + if not chapters: + raise StoryError("Story has no chapters to put in a book") + adapter = get_adapter(url) + adapter.setStoryMetadata(metahtml) + story = adapter.story + apply_overrides(story, overrides) + + adapter.chapterUrls = [{"url": c["url"], "title": c["title"]} for c in chapters] + story.setMetadata("numChapters", len(chapters)) + story.setMetadata("numWords", word_count(chapters)) + adapter.oldchaptersmap = { + c["url"]: BeautifulSoup(apply_rules(c["html"] or "
", rules), "html5lib") for c in chapters + } + adapter.oldimgs = {img["url"]: (f"images/stored.{img['ext']}", img["data"]) for img in images} + if cover and adapter.getConfig("include_images"): + info = story.img_store.add_img(cover["url"], cover["ext"], cover["mime"], cover["data"], cover=True) + story.cover = info["newsrc"] + story.setMetadata("cover_image", "specific") + + writer = writers.getWriter("epub", adapter.get_configuration(), adapter) + out_path.parent.mkdir(parents=True, exist_ok=True) + writer.writeStory(outfilename=str(out_path), forceOverwrite=True) + return calibre_metadata(story) + + +def read_epub(path: Path) -> EpubContents: + """Read the chapters of an epub that FanFicFare wrote. + + Chapter HTML comes back with image sources set to the original URLs. + """ + (source, _filecount, _soups, images, _oldcover, _bookmark, _logfile, urlsoups, datamaps) = get_update_data( + str(path) + )[0:9] + contents = EpubContents(url=source) + for n, (chapter_url, bodysoup) in enumerate(urlsoups.items(), start=1): + data = datamaps.get(chapter_url, {}) + title = data.get("chapterorigtitle") or data.get("chaptertitle") or f"Chapter {n}" + text = re.sub(r"]*>", "", str(bodysoup)).strip() + contents.chapters.append({"url": chapter_url, "title": escape_title(title), "html": text}) + for img_url, (src, img_data) in images.items(): + ext = src.rsplit(".", 1)[-1].lower() + if ext in imagetypes: + contents.images.append({"url": img_url, "ext": ext, "mime": imagetypes[ext], "data": img_data}) + return contents diff --git a/data/personal.ini b/data/personal.ini new file mode 100644 index 0000000..2f88a09 --- /dev/null +++ b/data/personal.ini @@ -0,0 +1,28 @@ +## FanFicFare configuration for CWA Bindery. +## +## This file has the same format as personal.ini in the FanFicFare calibre +## plugin and CLI. You can copy settings from your plugin configuration. +## Settings here override FanFicFare's defaults.ini. Documentation: +## https://github.com/JimmXinu/FanFicFare/wiki +## +## Bindery always sets these, so changes to them have no effect: +## always_reload_first_chapter, mark_new_chapters, continue_on_chapter_error, +## make_directories, zip_output. + +[defaults] +## Confirm that you are an adult in your locale, for sites that ask. +is_adult:true + +## Put the images from chapters into the epub. +include_images:true + +[www.royalroad.com] +## Seconds to wait between requests, to stay under RoyalRoad's rate limit. +slow_down_sleep_time:2 + +## Uncomment to include author notes with the chapter text. +#include_author_notes:true + +## Uncomment to log in. RoyalRoad then marks downloaded chapters as read. +#username:YourName +#password:yourpassword diff --git a/tests/conftest.py b/tests/conftest.py index 7788b19..fd3400d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,3 +7,21 @@ from pathlib import Path _tmp = Path(tempfile.mkdtemp(prefix="bindery-tests-")) for name, sub in [("CONFIG_DIR", "config"), ("INGEST_DIR", "ingest"), ("TMP_DIR", "tmp")]: os.environ[name] = str(_tmp / sub) +os.environ["BINDERY_START_THREADS"] = "false" +os.environ["UPDATE_INTERVAL_HOURS"] = "0" +for name in ( + "CWA_URL", + "CWA_USERNAME", + "CWA_PASSWORD", + "CWA_DB_PATH", + "CALIBRE_DB_PATH", + "AUTH_METHOD", + "BINDERY_API_KEY", + "URL_BASE", + "CWA_ALLOWED_USERS", +): + os.environ.pop(name, None) + +from bindery import config # noqa: E402 + +config.prepare() diff --git a/tests/test_build.py b/tests/test_build.py new file mode 100644 index 0000000..31d953e --- /dev/null +++ b/tests/test_build.py @@ -0,0 +1,155 @@ +"""Build an epub from stored data without network access, then read it back.""" + +import base64 +import datetime +import re +import zipfile + +import pytest +from defusedxml import ElementTree + +from bindery import config, fff +from bindery.rules import compile_rules, normalize_rules + +URL = "https://www.royalroad.com/fiction/1234/test-story" +PNG = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" +) +IMG_URL = "https://www.royalroadcdn.com/public/img.png" +COVER = {"url": "https://www.royalroadcdn.com/public/covers/1234.png", "ext": "png", "mime": "image/png", "data": PNG} + + +def metahtml(url=URL): + adapter = fff.get_adapter(url) + story = adapter.story + story.setMetadata("title", "Test Story") + story.addToList("author", "Writer") + story.addToList("authorId", "42") + story.setMetadata("description", "

About the story

") + story.setMetadata("datePublished", datetime.datetime(2024, 1, 2)) + story.setMetadata("dateUpdated", datetime.datetime(2024, 3, 4)) + story.addToList("genre", "Fantasy") + return story.dump_html_metadata() + + +def chapters(): + base = "https://www.royalroad.com/fiction/1234/test-story/chapter/{0}/c{0}" + return [ + {"url": base.format(10), "title": "One", "html": '

Zorian wakes.

'}, + {"url": base.format(11), "title": "Two", "html": f'

Picture

'}, + {"url": base.format(15), "title": "Three", "html": "

Zorian again.

"}, + ] + + +def test_build_and_read_back(tmp_path): + rules = compile_rules(normalize_rules([{"pattern": "Zorian", "replacement": "Z"}])) + out = tmp_path / "book.epub" + images = [{"url": IMG_URL, "ext": "png", "mime": "image/png", "data": PNG}] + meta = fff.build_epub(URL, metahtml(), chapters(), images, None, rules, {"title": "Kept Title"}, out) + assert meta["title"] == "Kept Title" + assert meta["authors"] == ["Writer"] + assert meta["pubdate"].startswith("2024-01-02") + + with zipfile.ZipFile(out) as z: + names = z.namelist() + text = "".join(z.read(n).decode("utf-8", "ignore") for n in names if n.endswith(".xhtml")) + opf = z.read("content.opf").decode() + assert "Z wakes." in text and "Zorian" not in text + assert any(n.startswith("OEBPS/images/ffdl-") and n.endswith(".png") for n in names) + assert re.search(r"]*>Kept Title", opf) + + contents = fff.read_epub(out) + assert [c["title"] for c in contents.chapters] == ["One", "Two", "Three"] + assert [c["url"] for c in contents.chapters] == [c["url"] for c in chapters()] + assert IMG_URL in contents.chapters[1]["html"] + assert contents.images and contents.images[0]["url"] == IMG_URL + + +def test_a_title_with_entities_builds_and_imports_in_fanficfare_form(tmp_path): + # FanFicFare's form, as the site titles in the store have it. + title = "Folly & Magic <1>" + out = tmp_path / "book.epub" + fff.build_epub(URL, metahtml(), [dict(chapters()[0], title=title)], [], None, [], {}, out) + with zipfile.ZipFile(out) as z: + for name in z.namelist(): + if name.endswith((".xhtml", ".ncx", ".opf")): + ElementTree.fromstring(z.read(name)) + # An import stores the same title as a download, so the next update finds no rename. + assert fff.read_epub(out).chapters[0]["title"] == title + assert fff.plain_title(title) == "Folly & Magic <1>" + assert fff.escape_title("Folly & Magic <1>") == title + + +@pytest.mark.parametrize( + ("personal_ini", "expected"), + [ + ("", "coveronly"), + ("[www.royalroad.com]\ninclude_images:false\n", "coveronly"), + ("[defaults]\ninclude_images:true\n", "true"), + ("[defaults]\ninclude_images:coveronly\n", "coveronly"), + ], +) +def test_include_images_is_never_off(personal_ini, expected, tmp_path, monkeypatch): + path = tmp_path / "personal.ini" + path.write_text(personal_ini) + monkeypatch.setattr(config, "PERSONAL_INI", path) + assert fff.get_configuration(URL).getConfig("include_images") == expected + + +@pytest.mark.parametrize("personal_ini", ["", "[defaults]\ninclude_images:false\n"]) +def test_the_cover_goes_in_the_epub_without_include_images(personal_ini, tmp_path, monkeypatch): + path = tmp_path / "personal.ini" + path.write_text(personal_ini) + monkeypatch.setattr(config, "PERSONAL_INI", path) + out = tmp_path / "book.epub" + fff.build_epub(URL, metahtml(), chapters(), [], COVER, [], {}, out) + with zipfile.ZipFile(out) as z: + names = z.namelist() + opf = z.read("content.opf").decode() + assert "OEBPS/cover.xhtml" in names + assert any(n.startswith("OEBPS/images/cover.") for n in names) + assert re.search(r'Kept blurb

", + "series": ["Kept Saga", 3.0], + "pubdate": "2019-05-06T00:00:00+00:00", + "languages": "de", + "tags": ["mine", "reread"], + } + meta = fff.build_epub(URL, metahtml(), chapters(), [], None, [], overrides, out) + assert meta["authors"] == ["Pen Name", "Co Writer"] + assert meta["tags"] == ["mine", "reread"] + with zipfile.ZipFile(out) as z: + opf = z.read("content.opf").decode() + assert re.findall(r"]*>([^<]*)", opf) == ["Pen Name", "Co Writer"] + assert re.findall(r"([^<]*)", opf) == ["mine", "reread"] + assert 'name="calibre:series" content="Kept Saga"' in opf + assert 'name="calibre:series_index" content="3.00"' in opf + assert "de" in opf + assert "2019-05-06" in opf + assert "Kept blurb" in opf + + +def test_unlocked_fields_keep_site_metadata(tmp_path): + out = tmp_path / "book.epub" + meta = fff.build_epub(URL, metahtml(), chapters(), [], None, [], {}, out) + assert meta["title"] == "Test Story" + assert meta["authors"] == ["Writer"] + assert "Fantasy" in meta["tags"] + + +def test_supported_sites(): + assert fff.is_supported_site("www.royalroad.com") + assert fff.is_supported_site("WWW.RoyalRoad.com ") + assert not fff.is_supported_site("example.com") + + +def test_normalize_url(): + assert fff.normalize_url(URL) == ("https://www.royalroad.com/fiction/1234", "www.royalroad.com") + assert fff.normalize_url("https://example.com/nothing") is None diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..23c1b6c --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,112 @@ +"""Reading the configuration from environment variables.""" + +import importlib +from pathlib import Path + +import pytest + +from bindery import config + + +@pytest.fixture +def reload_config(monkeypatch): + """Load the config module again with a changed environment, and restore it afterwards.""" + yield lambda: importlib.reload(config) + monkeypatch.undo() + importlib.reload(config) + + +def test_bool(monkeypatch): + for value in ("true", "Yes", " 1 ", "on", "y"): + monkeypatch.setenv("X_TEST", value) + assert config._bool("X_TEST") + for value in ("false", "0", "no", ""): + monkeypatch.setenv("X_TEST", value) + assert not config._bool("X_TEST") + monkeypatch.delenv("X_TEST") + assert config._bool("X_TEST", "true") + + +def test_list(monkeypatch): + monkeypatch.setenv("X_TEST", " title, tags ,,series ") + assert config._list("X_TEST") == ["title", "tags", "series"] + monkeypatch.delenv("X_TEST") + assert config._list("X_TEST") == [] + assert config._list("X_TEST", "title") == ["title"] + + +def test_relative_paths_become_absolute(monkeypatch, tmp_path, reload_config): + monkeypatch.chdir(tmp_path) + for name, value in [("CONFIG_DIR", "dev/config"), ("INGEST_DIR", "dev/ingest"), ("TMP_DIR", "dev/tmp")]: + monkeypatch.setenv(name, value) + monkeypatch.setenv("CWA_DB_PATH", "cwa/app.db") + monkeypatch.setenv("CALIBRE_DB_PATH", "library/metadata.db") + reload_config() + assert config.CONFIG_DIR == Path.cwd() / "dev/config" + assert config.INGEST_DIR == Path.cwd() / "dev/ingest" + assert config.TMP_DIR == Path.cwd() / "dev/tmp" + assert config.CWA_DB_PATH == Path.cwd() / "cwa/app.db" + assert config.CALIBRE_DB_PATH == Path.cwd() / "library/metadata.db" + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("", ""), + ("/", ""), + ("bindery", "/bindery"), + ("/bindery/", "/bindery"), + (" /a/b ", "/a/b"), + ], +) +def test_base_path(value, expected): + assert config._base_path(value) == expected + + +def test_values_from_the_environment(monkeypatch, reload_config): + monkeypatch.setenv("URL_BASE", "bindery/") + monkeypatch.setenv("AUTH_METHOD", " CWA ") + monkeypatch.setenv("UPDATE_INTERVAL_HOURS", "1.5") + monkeypatch.setenv("DEFAULT_LOCKED_FIELDS", "title, authors") + monkeypatch.setenv("CWA_ALLOWED_USERS", "alice,bob") + monkeypatch.setenv("CONFIG_DIR", "/data/bindery") + monkeypatch.setenv("TRUST_PROXY", "yes") + monkeypatch.setenv("CWA_DB_PATH", " /cwa/app.db ") + monkeypatch.setenv("CALIBRE_DB_PATH", "/library/metadata.db") + reload_config() + assert config.URL_BASE == "/bindery" + assert config.AUTH_METHOD == "cwa" + assert config.UPDATE_INTERVAL_HOURS == 1.5 + assert config.DEFAULT_LOCKED_FIELDS == ["title", "authors"] + assert config.CWA_ALLOWED_USERS == ["alice", "bob"] + assert str(config.DB_PATH) == "/data/bindery/bindery.db" + assert config.TRUST_PROXY is True + assert str(config.CWA_DB_PATH) == "/cwa/app.db" + assert str(config.CALIBRE_DB_PATH) == "/library/metadata.db" + + +def test_defaults(monkeypatch, reload_config): + for name in ( + "URL_BASE", + "AUTH_METHOD", + "UPDATE_INTERVAL_HOURS", + "DEFAULT_LOCKED_FIELDS", + "LOG_LEVEL", + "PROXY_AUTH_USER_HEADER", + "SESSION_COOKIE_SECURE", + "TRUST_PROXY", + "CWA_DB_PATH", + "CALIBRE_DB_PATH", + ): + monkeypatch.delenv(name, raising=False) + reload_config() + assert config.URL_BASE == "" + assert config.AUTH_METHOD == "none" + assert config.UPDATE_INTERVAL_HOURS == 24 + assert config.DEFAULT_LOCKED_FIELDS == ["title"] + assert config.LOG_LEVEL == "INFO" + assert config.PROXY_AUTH_USER_HEADER == "X-Remote-User" + assert config.SESSION_COOKIE_SECURE is False + assert config.TRUST_PROXY is False + assert config.CWA_DB_PATH is None + assert config.CALIBRE_DB_PATH is None diff --git a/tests/test_fff_flows.py b/tests/test_fff_flows.py new file mode 100644 index 0000000..791d9d7 --- /dev/null +++ b/tests/test_fff_flows.py @@ -0,0 +1,108 @@ +"""Exercise the FanFicFare boundary without requests to a story site.""" + +import zipfile +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from test_build import IMG_URL, PNG, URL, chapters, metahtml + +from bindery import config, fff + + +@pytest.mark.parametrize("cover", [None, {}, {"url": IMG_URL, "ext": "png", "mime": "image/png", "data": PNG}]) +def test_metadata_snapshot_from_adapter(monkeypatch, cover): + adapter = fff.get_adapter(URL) + adapter.setStoryMetadata(metahtml()) + adapter.chapterUrls = chapters() + adapter.story.img_store.cover = cover + fetch = Mock(return_value=adapter.story) + monkeypatch.setattr(adapter, "getStoryMetadataOnly", fetch) + snap = fff.fetch_story(adapter) + fetch.assert_called_once_with(get_cover=True) + assert snap.title == "Test Story" + assert snap.author == "Writer" + assert snap.site == "www.royalroad.com" + assert snap.chapters == [{"url": c["url"], "title": c["title"]} for c in chapters()] + assert snap.calibre["authors"] == ["Writer"] + assert snap.cover == (cover or None) + assert "Test Story" in snap.metahtml + + +@pytest.mark.parametrize( + "error,message", + [ + (fff.exceptions.StoryDoesNotExist(URL), "StoryDoesNotExist"), + (fff.exceptions.AccessDenied("private"), "AccessDenied"), + (fff.exceptions.FailedToLogin(URL, "reader"), "FailedToLogin"), + (fff.exceptions.AdultCheckRequired(URL), "is_adult:true"), + ], +) +def test_adapter_errors_become_actionable_story_errors(error, message): + adapter = SimpleNamespace(getStoryMetadataOnly=Mock(side_effect=error)) + with pytest.raises(fff.StoryError, match=message): + fff.fetch_story(adapter) + + +def test_downloaded_chapter_restores_original_image_urls(): + html = '

Text

' + adapter = SimpleNamespace(getChapterTextNum=Mock(return_value=html)) + result = fff.fetch_chapter(adapter, "chapter", 3) + adapter.getChapterTextNum.assert_called_once_with("chapter", 3) + assert 'src="https://images.test/1.png"' in result + for src in ["data:image/png;base64,abc", "failedtoload", "https://images.test/direct.png"]: + markup = f'" + assert f'src="{src}"' in fff.canonical_html(markup) + assert fff.canonical_html("

plain

") == "

plain

" + + +def test_collected_images_skip_failed_and_empty_downloads(): + image = {"url": "image", "ext": "png", "mime": "image/png", "data": PNG} + adapter = SimpleNamespace( + story=SimpleNamespace( + img_store=SimpleNamespace( + infos=[{**image, "newsrc": "images/1.png"}, {**image, "data": b""}, {**image, "newsrc": "failedtoload"}] + ) + ) + ) + assert fff.collected_images(adapter) == [image] + + +def test_personal_ini_overrides_and_invalid_input(monkeypatch, tmp_path): + ini = tmp_path / "personal.ini" + ini.write_text("[overrides]\nis_adult: true\n", encoding="utf-8") + monkeypatch.setattr(config, "PERSONAL_INI", ini) + configuration = fff.get_configuration(URL) + assert configuration.getConfig("is_adult") + for text in ["[defaults]\nbroken line\n", "setting: before any section"]: + with pytest.raises(ValueError, match="cannot read"): + fff.check_personal_ini(text) + with pytest.raises(fff.StoryError, match="does not support"): + fff.get_configuration("https://unsupported.example/story") + + +def test_adapter_rejects_invalid_story_url(monkeypatch): + monkeypatch.setattr( + fff.adapters, "getAdapter", Mock(side_effect=fff.exceptions.InvalidStoryURL(URL, "site", "example")) + ) + with pytest.raises(fff.StoryError): + fff.get_adapter(URL) + + +def test_normalization_errors_are_not_site_requests(monkeypatch): + monkeypatch.setattr(fff.adapters, "getNormalStoryURLSite", Mock(side_effect=ValueError("bad URL"))) + fff.normalize_url.cache_clear() + try: + assert fff.normalize_url("bad-input") is None + finally: + fff.normalize_url.cache_clear() + + +def test_empty_chapters_fail_and_stored_cover_is_built(tmp_path): + with pytest.raises(fff.StoryError, match="no chapters"): + fff.build_epub(URL, metahtml(), [], [], None, [], {}, tmp_path / "empty.epub") + cover = {"url": IMG_URL, "ext": "png", "mime": "image/png", "data": PNG} + out = tmp_path / "cover.epub" + fff.build_epub(URL, metahtml(), chapters()[:1], [], cover, [], {}, out) + with zipfile.ZipFile(out) as book: + assert any(book.read(n) == PNG for n in book.namelist() if n.endswith(".png"))