Download stories and build epubs with FanFicFare

FanFicFare adapters fetch metadata and chapters, and its epub writer
builds each book from the store. Environment variables configure the
app, and a default personal.ini is copied to the config folder on first
start.

bindery/config.py

@@ -0,0 +1,88 @@
+"""Configuration. Every setting comes from an environment variable."""
+
+import logging
+import os
+import shutil
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+def _bool(name: str, default: str = "false") -> bool:
+    return os.getenv(name, default).strip().lower() in ("true", "yes", "1", "y", "on")
+
+
+def _list(name: str, default: str = "") -> list[str]:
+    return [v.strip() for v in os.getenv(name, default).split(",") if v.strip()]
+
+
+# A relative path is relative to the working folder. Paths are made absolute at once, because Flask's send_file
+# reads a relative path from the package folder.
+def _path(name: str, default: str) -> Path:
+    return Path(os.getenv(name, default)).absolute()
+
+
+def _optional_path(name: str) -> Path | None:
+    value = os.getenv(name, "").strip()
+    return Path(value).absolute() if value else None
+
+
+def _base_path(value: str) -> str:
+    path = value.strip().strip("/")
+    return f"/{path}" if path else ""
+
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+
+CONFIG_DIR = _path("CONFIG_DIR", "/config")
+INGEST_DIR = _path("INGEST_DIR", "/cwa-book-ingest")
+TMP_DIR = _path("TMP_DIR", "/tmp/bindery")
+LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").strip().upper()
+# A path prefix, such as /bindery, for a reverse proxy that forwards the full path.
+URL_BASE = _base_path(os.getenv("URL_BASE", ""))
+
+# The CWA account that Bindery uses for the OPDS feed.
+CWA_URL = os.getenv("CWA_URL", "").strip()
+CWA_USERNAME = os.getenv("CWA_USERNAME", "").strip()
+CWA_PASSWORD = os.getenv("CWA_PASSWORD", "")
+# Independent, optional read-only databases. HTTP remains the fallback.
+CWA_DB_PATH = _optional_path("CWA_DB_PATH")  # Calibre-Web app.db, for logins
+CALIBRE_DB_PATH = _optional_path("CALIBRE_DB_PATH")  # calibre metadata.db, for book IDs
+
+UPDATE_INTERVAL_HOURS = float(os.getenv("UPDATE_INTERVAL_HOURS", "24"))
+DEFAULT_LOCKED_FIELDS = _list("DEFAULT_LOCKED_FIELDS", "title")
+
+# Logins: none, cwa or proxy. An API key works in every mode.
+AUTH_METHOD = os.getenv("AUTH_METHOD", "none").strip().lower()
+CWA_ALLOWED_USERS = _list("CWA_ALLOWED_USERS")
+PROXY_AUTH_USER_HEADER = os.getenv("PROXY_AUTH_USER_HEADER", "X-Remote-User").strip()
+PROXY_AUTH_LOGOUT_URL = os.getenv("PROXY_AUTH_LOGOUT_URL", "").strip()
+API_KEY = os.getenv("BINDERY_API_KEY", "").strip()
+SESSION_COOKIE_SECURE = _bool("SESSION_COOKIE_SECURE")
+# Take the client address from the last entry of X-Forwarded-For, which a reverse proxy adds.
+TRUST_PROXY = _bool("TRUST_PROXY")
+
+# Tests turn off the job threads.
+START_THREADS = _bool("BINDERY_START_THREADS", "true")
+
+SITE_DELAY = 5  # seconds between two jobs that go to a story site
+JOB_HISTORY = 60 * 60  # seconds that finished jobs stay in the job list
+
+DB_PATH = CONFIG_DIR / "bindery.db"
+UPLOAD_DIR = CONFIG_DIR / "uploads"  # uploaded epubs wait here for their job, also over a restart
+SECRET_KEY_FILE = CONFIG_DIR / ".flask_secret"
+PERSONAL_INI = CONFIG_DIR / "personal.ini"
+DEFAULT_PERSONAL_INI = PROJECT_ROOT / "data" / "personal.ini"
+BACKUP_DIR = CONFIG_DIR / "backups"
+BACKUP_COUNT = 7  # daily backups to keep
+
+
+def prepare() -> None:
+    """Make the folders, and give the user a personal.ini to edit on first start."""
+    for directory in (CONFIG_DIR, UPLOAD_DIR, TMP_DIR, INGEST_DIR):
+        directory.mkdir(parents=True, exist_ok=True)
+    if not PERSONAL_INI.exists():
+        shutil.copy(DEFAULT_PERSONAL_INI, PERSONAL_INI)
+        logger.info(f"Created the default FanFicFare configuration at {PERSONAL_INI}")
+    if not CWA_URL:
+        logger.warning("CWA_URL is not set. OPDS imports and HTTP fallbacks are unavailable.")

bindery/fff.py

@@ -0,0 +1,335 @@
+"""Headless FanFicFare wrapper: site access, chapter fetches and epub builds.
+
+This module uses FanFicFare's adapters (the site scrapers) and its epub writer
+without changes. The download and update flow of the FanFicFare CLI is not used.
+Instead, chapters live in the store, and every epub is built from the store:
+
+  1. fetch_story() reads metadata and the chapter list from the site.
+  2. fetch_chapter() downloads one chapter and returns its HTML with image
+     sources set back to their original URLs.
+  3. build_epub() gives FanFicFare's writer the stored chapters as "old
+     chapters" (the same mechanism FanFicFare uses for epub updates), so the
+     writer never goes to the site.
+"""
+
+import configparser
+import html
+import io
+import logging
+import re
+from dataclasses import dataclass, field
+from datetime import datetime
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+import fanficfare
+from bs4 import BeautifulSoup
+from fanficfare import adapters, exceptions, writers
+from fanficfare.configurable import Configuration
+from fanficfare.epubutils import get_update_data
+from fanficfare.story import imagetypes
+
+from bindery import config
+from bindery.diffing import html_to_blocks
+from bindery.rules import Step, apply_rules
+
+logger = logging.getLogger(__name__)
+
+# FanFicFare logs every request at DEBUG level.
+logging.getLogger("fanficfare").setLevel(logging.DEBUG if config.LOG_LEVEL == "DEBUG" else logging.WARNING)
+
+DEFAULTS_INI = Path(fanficfare.__file__).parent / "defaults.ini"
+
+# Settings this application depends on. They go in [overrides], so
+# personal.ini cannot change them.
+FORCED_SETTINGS = {
+    "always_reload_first_chapter": "false",
+    "mark_new_chapters": "false",
+    "continue_on_chapter_error": "false",
+    "make_directories": "false",
+    "zip_output": "false",
+    # The epub is the only way metadata reaches CWA, so the series must be in the OPF.
+    "calibre_series_meta": "true",
+}
+
+
+class StoryError(Exception):
+    """A problem the user can fix, such as an unsupported URL."""
+
+
+@dataclass
+class SiteSnapshot:
+    """What the site says about a story right now."""
+
+    url: str
+    site: str
+    title: str
+    author: str
+    metahtml: str
+    chapters: list[dict[str, str]]
+    calibre: dict[str, Any]
+    cover: dict[str, Any] | None = None
+
+
+@dataclass
+class EpubContents:
+    """Chapters and images read from an existing FanFicFare epub."""
+
+    url: str | None
+    chapters: list[dict[str, str]] = field(default_factory=list)
+    images: list[dict[str, Any]] = field(default_factory=list)
+
+
+def escape_title(text: str) -> str:
+    """A chapter title in FanFicFare's form, which the store keeps.
+
+    FanFicFare keeps &, < and > in chapter titles, because its epub
+    writer puts titles into XHTML as they are.
+    """
+    return html.escape(text, quote=False)
+
+
+def plain_title(title: str) -> str:
+    """A stored chapter title as text, for a message."""
+    return html.unescape(title)
+
+
+@lru_cache(maxsize=20000)
+def normalize_url(url: str) -> tuple[str, str] | None:
+    """Return (story URL, site) for a supported URL, or None. No network access."""
+    try:
+        return adapters.getNormalStoryURLSite(url.strip())
+    except Exception:
+        return None
+
+
+@lru_cache(maxsize=1)
+def _site_names() -> frozenset:
+    return frozenset(name.lower() for name in adapters.getSiteSections())
+
+
+def is_supported_site(name: str) -> bool:
+    """True for a site name that FanFicFare supports, such as www.royalroad.com."""
+    return name.strip().lower() in _site_names()
+
+
+def get_configuration(url: str) -> Configuration:
+    try:
+        configuration = Configuration(adapters.getConfigSectionsFor(url), "epub")
+    except exceptions.UnknownSite as e:
+        raise StoryError(f"FanFicFare does not support site: {url}") from e
+    with open(DEFAULTS_INI, encoding="utf-8") as f:
+        configuration.read_file(f)
+    if config.PERSONAL_INI.exists():
+        with open(config.PERSONAL_INI, encoding="utf-8") as f:
+            configuration.read_file(f)
+    try:
+        configuration.add_section("overrides")
+    except configparser.DuplicateSectionError:
+        pass
+    for key, value in FORCED_SETTINGS.items():
+        configuration.set("overrides", key, value)
+    # FanFicFare downloads the cover only if include_images is on, and it is off by default. "coveronly" keeps the
+    # cover, but still leaves out the images in the story text.
+    if not configuration.getConfig("include_images"):
+        configuration.set("overrides", "include_images", "coveronly")
+        configuration.cached_config.clear()
+    return configuration
+
+
+def check_personal_ini(text: str) -> list[str]:
+    """Check the text of a personal.ini.
+
+    Raise ValueError for text that FanFicFare cannot read. Return the problems
+    that FanFicFare's own check finds, such as a section for an unknown site.
+    """
+    configuration = Configuration(["defaults"], "epub")
+    try:
+        configuration.read_file(io.StringIO(text))
+    except configparser.ParsingError as e:
+        lines = "; ".join(f"line {n}: {line.strip()}" for n, line in e.errors)
+        raise ValueError(f"FanFicFare cannot read this personal.ini. Check {lines}") from e
+    except configparser.Error as e:
+        raise ValueError(f"FanFicFare cannot read this personal.ini: {e}") from e
+    return [f"Line {n}: {message}" for n, message in configuration.test_config()]
+
+
+def get_adapter(url: str):
+    try:
+        return adapters.getAdapter(get_configuration(url), url)
+    except exceptions.InvalidStoryURL as e:
+        raise StoryError(str(e)) from e
+
+
+def parse_series(value: str) -> tuple[str, float | None]:
+    m = re.match(r"^(.*?)\s*\[([\d.]+)\]$", value or "")
+    if m:
+        return m.group(1), float(m.group(2))
+    return value or "", None
+
+
+def calibre_metadata(story) -> dict[str, Any]:
+    """Map FanFicFare story metadata to calibre fields."""
+    series, series_index = parse_series(story.getMetadata("series", removeallentities=True))
+    published = story.getMetadataRaw("datePublished")
+    return {
+        "title": story.getMetadata("title", removeallentities=True),
+        "authors": story.getList("author", removeallentities=True),
+        "comments": story.getMetadata("description"),
+        "tags": story.getSubjectTags(removeallentities=True),
+        "series": series,
+        "series_index": series_index,
+        "pubdate": published.isoformat() if hasattr(published, "isoformat") else "",
+        "languages": story.getMetadata("langcode"),
+    }
+
+
+def fetch_story(adapter) -> SiteSnapshot:
+    """Read story metadata, the chapter list and the cover from the site."""
+    try:
+        story = adapter.getStoryMetadataOnly(get_cover=True)
+    except (exceptions.StoryDoesNotExist, exceptions.AccessDenied, exceptions.FailedToLogin) as e:
+        raise StoryError(f"{type(e).__name__}: {e}") from e
+    except exceptions.AdultCheckRequired as e:
+        raise StoryError("Site asks for adult check. Set is_adult:true in personal.ini.") from e
+    cover = None
+    info = story.img_store.cover
+    if info and info.get("data"):
+        cover = {k: info[k] for k in ("url", "ext", "mime", "data")}
+    return SiteSnapshot(
+        url=story.getMetadata("storyUrl"),
+        site=adapter.getSiteDomain(),
+        title=story.getMetadata("title", removeallentities=True),
+        author=", ".join(story.getList("author", removeallentities=True)),
+        metahtml=story.dump_html_metadata(),
+        chapters=[{"url": c["url"], "title": c["title"]} for c in adapter.get_chapters()],
+        calibre=calibre_metadata(story),
+        cover=cover,
+    )
+
+
+def canonical_html(html: str) -> str:
+    """Point <img> tags back at their original URLs, as FanFicFare's update code does."""
+    if "<img" not in html:
+        return html
+    soup = BeautifulSoup(html, "html.parser")
+    for img in soup.find_all("img"):
+        if img.has_attr("longdesc") and not img["src"].startswith(("data:", "failedtoload")):
+            img["src"] = img["longdesc"]
+    return str(soup)
+
+
+def fetch_chapter(adapter, url: str, index: int) -> str:
+    """Download one chapter. Images go into the adapter's image store."""
+    return canonical_html(adapter.getChapterTextNum(url, index))
+
+
+def collected_images(adapter) -> list[dict[str, Any]]:
+    """Images downloaded so far by this adapter, without the cover."""
+    images = []
+    for info in adapter.story.img_store.infos:
+        if info.get("data") and info.get("newsrc") != "failedtoload":
+            images.append({k: info[k] for k in ("url", "ext", "mime", "data")})
+    return images
+
+
+def word_count(chapters: list[dict[str, Any]]) -> int:
+    return sum(len(block.split()) for c in chapters for block in html_to_blocks(c["html"]))
+
+
+def apply_overrides(story, overrides: dict[str, Any]) -> None:
+    """Put the kept values of locked fields into FanFicFare's story metadata.
+
+    The epub writer takes the OPF metadata from these entries, and CWA takes
+    the book record from the OPF on the first import.
+    """
+    for key, value in overrides.items():
+        if key == "title" and value:
+            story.setMetadata("title", value)
+        elif key == "authors" and value:
+            for n, author in enumerate(value):
+                story.addToList("author", author, clear=(n == 0))
+        elif key == "comments":
+            story.setMetadata("description", value or "")
+        elif key == "series":
+            name, index = value if value else ("", None)
+            if name and index is not None:
+                story.setMetadata("series", f"{name} [{index:g}]")
+            else:
+                story.setMetadata("series", name or "")
+        elif key == "pubdate":
+            try:
+                story.setMetadata("datePublished", datetime.fromisoformat(value) if value else None)
+            except ValueError:
+                logger.warning(f"Ignoring a kept publication date that is not a date: {value!r}")
+        elif key == "languages" and value:
+            story.setMetadata("langcode", value)
+        elif key == "tags":
+            tags = list(value or [])
+            # getSubjectTags() builds the tags from configuration. The locked
+            # list replaces it for this build only.
+            story.getSubjectTags = lambda removeallentities=False, tags=tags: tags
+
+
+def build_epub(
+    url: str,
+    metahtml: str,
+    chapters: list[dict[str, Any]],
+    images: list[dict[str, Any]],
+    cover: dict[str, Any] | None,
+    rules: list[Step],
+    overrides: dict[str, Any],
+    out_path: Path,
+) -> dict[str, Any]:
+    """Write an epub from stored data. No network access.
+
+    chapters: dicts with url, title and html, in book order.
+    overrides: calibre field values that replace the site values in the epub
+               (locked fields).
+    Returns the calibre metadata of the built book.
+    """
+    if not chapters:
+        raise StoryError("Story has no chapters to put in a book")
+    adapter = get_adapter(url)
+    adapter.setStoryMetadata(metahtml)
+    story = adapter.story
+    apply_overrides(story, overrides)
+
+    adapter.chapterUrls = [{"url": c["url"], "title": c["title"]} for c in chapters]
+    story.setMetadata("numChapters", len(chapters))
+    story.setMetadata("numWords", word_count(chapters))
+    adapter.oldchaptersmap = {
+        c["url"]: BeautifulSoup(apply_rules(c["html"] or "<div></div>", rules), "html5lib") for c in chapters
+    }
+    adapter.oldimgs = {img["url"]: (f"images/stored.{img['ext']}", img["data"]) for img in images}
+    if cover and adapter.getConfig("include_images"):
+        info = story.img_store.add_img(cover["url"], cover["ext"], cover["mime"], cover["data"], cover=True)
+        story.cover = info["newsrc"]
+        story.setMetadata("cover_image", "specific")
+
+    writer = writers.getWriter("epub", adapter.get_configuration(), adapter)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    writer.writeStory(outfilename=str(out_path), forceOverwrite=True)
+    return calibre_metadata(story)
+
+
+def read_epub(path: Path) -> EpubContents:
+    """Read the chapters of an epub that FanFicFare wrote.
+
+    Chapter HTML comes back with image sources set to the original URLs.
+    """
+    (source, _filecount, _soups, images, _oldcover, _bookmark, _logfile, urlsoups, datamaps) = get_update_data(
+        str(path)
+    )[0:9]
+    contents = EpubContents(url=source)
+    for n, (chapter_url, bodysoup) in enumerate(urlsoups.items(), start=1):
+        data = datamaps.get(chapter_url, {})
+        title = data.get("chapterorigtitle") or data.get("chaptertitle") or f"Chapter {n}"
+        text = re.sub(r"</?body[^>]*>", "", str(bodysoup)).strip()
+        contents.chapters.append({"url": chapter_url, "title": escape_title(title), "html": text})
+    for img_url, (src, img_data) in images.items():
+        ext = src.rsplit(".", 1)[-1].lower()
+        if ext in imagetypes:
+            contents.images.append({"url": img_url, "ext": ext, "mime": imagetypes[ext], "data": img_data})
+    return contents

data/personal.ini

@@ -0,0 +1,28 @@
+## FanFicFare configuration for CWA Bindery.
+##
+## This file has the same format as personal.ini in the FanFicFare calibre
+## plugin and CLI. You can copy settings from your plugin configuration.
+## Settings here override FanFicFare's defaults.ini. Documentation:
+## https://github.com/JimmXinu/FanFicFare/wiki
+##
+## Bindery always sets these, so changes to them have no effect:
+## always_reload_first_chapter, mark_new_chapters, continue_on_chapter_error,
+## make_directories, zip_output.
+
+[defaults]
+## Confirm that you are an adult in your locale, for sites that ask.
+is_adult:true
+
+## Put the images from chapters into the epub.
+include_images:true
+
+[www.royalroad.com]
+## Seconds to wait between requests, to stay under RoyalRoad's rate limit.
+slow_down_sleep_time:2
+
+## Uncomment to include author notes with the chapter text.
+#include_author_notes:true
+
+## Uncomment to log in. RoyalRoad then marks downloaded chapters as read.
+#username:YourName
+#password:yourpassword

tests/conftest.py

@@ -7,3 +7,21 @@ from pathlib import Path
 _tmp = Path(tempfile.mkdtemp(prefix="bindery-tests-"))
 for name, sub in [("CONFIG_DIR", "config"), ("INGEST_DIR", "ingest"), ("TMP_DIR", "tmp")]:
     os.environ[name] = str(_tmp / sub)
+os.environ["BINDERY_START_THREADS"] = "false"
+os.environ["UPDATE_INTERVAL_HOURS"] = "0"
+for name in (
+    "CWA_URL",
+    "CWA_USERNAME",
+    "CWA_PASSWORD",
+    "CWA_DB_PATH",
+    "CALIBRE_DB_PATH",
+    "AUTH_METHOD",
+    "BINDERY_API_KEY",
+    "URL_BASE",
+    "CWA_ALLOWED_USERS",
+):
+    os.environ.pop(name, None)
+
+from bindery import config  # noqa: E402
+
+config.prepare()

tests/test_build.py

@@ -0,0 +1,155 @@
+"""Build an epub from stored data without network access, then read it back."""
+
+import base64
+import datetime
+import re
+import zipfile
+
+import pytest
+from defusedxml import ElementTree
+
+from bindery import config, fff
+from bindery.rules import compile_rules, normalize_rules
+
+URL = "https://www.royalroad.com/fiction/1234/test-story"
+PNG = base64.b64decode(
+    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+)
+IMG_URL = "https://www.royalroadcdn.com/public/img.png"
+COVER = {"url": "https://www.royalroadcdn.com/public/covers/1234.png", "ext": "png", "mime": "image/png", "data": PNG}
+
+
+def metahtml(url=URL):
+    adapter = fff.get_adapter(url)
+    story = adapter.story
+    story.setMetadata("title", "Test Story")
+    story.addToList("author", "Writer")
+    story.addToList("authorId", "42")
+    story.setMetadata("description", "<p>About the story</p>")
+    story.setMetadata("datePublished", datetime.datetime(2024, 1, 2))
+    story.setMetadata("dateUpdated", datetime.datetime(2024, 3, 4))
+    story.addToList("genre", "Fantasy")
+    return story.dump_html_metadata()
+
+
+def chapters():
+    base = "https://www.royalroad.com/fiction/1234/test-story/chapter/{0}/c{0}"
+    return [
+        {"url": base.format(10), "title": "One", "html": '<div><p class="rnd1">Zorian wakes.</p></div>'},
+        {"url": base.format(11), "title": "Two", "html": f'<div><p>Picture</p><img src="{IMG_URL}"/></div>'},
+        {"url": base.format(15), "title": "Three", "html": "<div><p>Zorian again.</p></div>"},
+    ]
+
+
+def test_build_and_read_back(tmp_path):
+    rules = compile_rules(normalize_rules([{"pattern": "Zorian", "replacement": "Z"}]))
+    out = tmp_path / "book.epub"
+    images = [{"url": IMG_URL, "ext": "png", "mime": "image/png", "data": PNG}]
+    meta = fff.build_epub(URL, metahtml(), chapters(), images, None, rules, {"title": "Kept Title"}, out)
+    assert meta["title"] == "Kept Title"
+    assert meta["authors"] == ["Writer"]
+    assert meta["pubdate"].startswith("2024-01-02")
+
+    with zipfile.ZipFile(out) as z:
+        names = z.namelist()
+        text = "".join(z.read(n).decode("utf-8", "ignore") for n in names if n.endswith(".xhtml"))
+        opf = z.read("content.opf").decode()
+    assert "Z wakes." in text and "Zorian" not in text
+    assert any(n.startswith("OEBPS/images/ffdl-") and n.endswith(".png") for n in names)
+    assert re.search(r"<dc:title[^>]*>Kept Title</dc:title>", opf)
+
+    contents = fff.read_epub(out)
+    assert [c["title"] for c in contents.chapters] == ["One", "Two", "Three"]
+    assert [c["url"] for c in contents.chapters] == [c["url"] for c in chapters()]
+    assert IMG_URL in contents.chapters[1]["html"]
+    assert contents.images and contents.images[0]["url"] == IMG_URL
+
+
+def test_a_title_with_entities_builds_and_imports_in_fanficfare_form(tmp_path):
+    # FanFicFare's form, as the site titles in the store have it.
+    title = "Folly &amp; Magic &lt;1&gt;"
+    out = tmp_path / "book.epub"
+    fff.build_epub(URL, metahtml(), [dict(chapters()[0], title=title)], [], None, [], {}, out)
+    with zipfile.ZipFile(out) as z:
+        for name in z.namelist():
+            if name.endswith((".xhtml", ".ncx", ".opf")):
+                ElementTree.fromstring(z.read(name))
+    # An import stores the same title as a download, so the next update finds no rename.
+    assert fff.read_epub(out).chapters[0]["title"] == title
+    assert fff.plain_title(title) == "Folly & Magic <1>"
+    assert fff.escape_title("Folly & Magic <1>") == title
+
+
+@pytest.mark.parametrize(
+    ("personal_ini", "expected"),
+    [
+        ("", "coveronly"),
+        ("[www.royalroad.com]\ninclude_images:false\n", "coveronly"),
+        ("[defaults]\ninclude_images:true\n", "true"),
+        ("[defaults]\ninclude_images:coveronly\n", "coveronly"),
+    ],
+)
+def test_include_images_is_never_off(personal_ini, expected, tmp_path, monkeypatch):
+    path = tmp_path / "personal.ini"
+    path.write_text(personal_ini)
+    monkeypatch.setattr(config, "PERSONAL_INI", path)
+    assert fff.get_configuration(URL).getConfig("include_images") == expected
+
+
+@pytest.mark.parametrize("personal_ini", ["", "[defaults]\ninclude_images:false\n"])
+def test_the_cover_goes_in_the_epub_without_include_images(personal_ini, tmp_path, monkeypatch):
+    path = tmp_path / "personal.ini"
+    path.write_text(personal_ini)
+    monkeypatch.setattr(config, "PERSONAL_INI", path)
+    out = tmp_path / "book.epub"
+    fff.build_epub(URL, metahtml(), chapters(), [], COVER, [], {}, out)
+    with zipfile.ZipFile(out) as z:
+        names = z.namelist()
+        opf = z.read("content.opf").decode()
+    assert "OEBPS/cover.xhtml" in names
+    assert any(n.startswith("OEBPS/images/cover.") for n in names)
+    assert re.search(r'<meta name="cover" content="[^"]+"', opf)
+
+
+def test_locked_fields_replace_site_metadata(tmp_path):
+    out = tmp_path / "book.epub"
+    overrides = {
+        "title": "Kept Title",
+        "authors": ["Pen Name", "Co Writer"],
+        "comments": "<p>Kept blurb</p>",
+        "series": ["Kept Saga", 3.0],
+        "pubdate": "2019-05-06T00:00:00+00:00",
+        "languages": "de",
+        "tags": ["mine", "reread"],
+    }
+    meta = fff.build_epub(URL, metahtml(), chapters(), [], None, [], overrides, out)
+    assert meta["authors"] == ["Pen Name", "Co Writer"]
+    assert meta["tags"] == ["mine", "reread"]
+    with zipfile.ZipFile(out) as z:
+        opf = z.read("content.opf").decode()
+    assert re.findall(r"<dc:creator[^>]*>([^<]*)</dc:creator>", opf) == ["Pen Name", "Co Writer"]
+    assert re.findall(r"<dc:subject>([^<]*)</dc:subject>", opf) == ["mine", "reread"]
+    assert 'name="calibre:series" content="Kept Saga"' in opf
+    assert 'name="calibre:series_index" content="3.00"' in opf
+    assert "<dc:language>de</dc:language>" in opf
+    assert "2019-05-06" in opf
+    assert "Kept blurb" in opf
+
+
+def test_unlocked_fields_keep_site_metadata(tmp_path):
+    out = tmp_path / "book.epub"
+    meta = fff.build_epub(URL, metahtml(), chapters(), [], None, [], {}, out)
+    assert meta["title"] == "Test Story"
+    assert meta["authors"] == ["Writer"]
+    assert "Fantasy" in meta["tags"]
+
+
+def test_supported_sites():
+    assert fff.is_supported_site("www.royalroad.com")
+    assert fff.is_supported_site("WWW.RoyalRoad.com ")
+    assert not fff.is_supported_site("example.com")
+
+
+def test_normalize_url():
+    assert fff.normalize_url(URL) == ("https://www.royalroad.com/fiction/1234", "www.royalroad.com")
+    assert fff.normalize_url("https://example.com/nothing") is None

tests/test_config.py

@@ -0,0 +1,112 @@
+"""Reading the configuration from environment variables."""
+
+import importlib
+from pathlib import Path
+
+import pytest
+
+from bindery import config
+
+
+@pytest.fixture
+def reload_config(monkeypatch):
+    """Load the config module again with a changed environment, and restore it afterwards."""
+    yield lambda: importlib.reload(config)
+    monkeypatch.undo()
+    importlib.reload(config)
+
+
+def test_bool(monkeypatch):
+    for value in ("true", "Yes", " 1 ", "on", "y"):
+        monkeypatch.setenv("X_TEST", value)
+        assert config._bool("X_TEST")
+    for value in ("false", "0", "no", ""):
+        monkeypatch.setenv("X_TEST", value)
+        assert not config._bool("X_TEST")
+    monkeypatch.delenv("X_TEST")
+    assert config._bool("X_TEST", "true")
+
+
+def test_list(monkeypatch):
+    monkeypatch.setenv("X_TEST", " title, tags ,,series ")
+    assert config._list("X_TEST") == ["title", "tags", "series"]
+    monkeypatch.delenv("X_TEST")
+    assert config._list("X_TEST") == []
+    assert config._list("X_TEST", "title") == ["title"]
+
+
+def test_relative_paths_become_absolute(monkeypatch, tmp_path, reload_config):
+    monkeypatch.chdir(tmp_path)
+    for name, value in [("CONFIG_DIR", "dev/config"), ("INGEST_DIR", "dev/ingest"), ("TMP_DIR", "dev/tmp")]:
+        monkeypatch.setenv(name, value)
+    monkeypatch.setenv("CWA_DB_PATH", "cwa/app.db")
+    monkeypatch.setenv("CALIBRE_DB_PATH", "library/metadata.db")
+    reload_config()
+    assert config.CONFIG_DIR == Path.cwd() / "dev/config"
+    assert config.INGEST_DIR == Path.cwd() / "dev/ingest"
+    assert config.TMP_DIR == Path.cwd() / "dev/tmp"
+    assert config.CWA_DB_PATH == Path.cwd() / "cwa/app.db"
+    assert config.CALIBRE_DB_PATH == Path.cwd() / "library/metadata.db"
+
+
+@pytest.mark.parametrize(
+    ("value", "expected"),
+    [
+        ("", ""),
+        ("/", ""),
+        ("bindery", "/bindery"),
+        ("/bindery/", "/bindery"),
+        (" /a/b ", "/a/b"),
+    ],
+)
+def test_base_path(value, expected):
+    assert config._base_path(value) == expected
+
+
+def test_values_from_the_environment(monkeypatch, reload_config):
+    monkeypatch.setenv("URL_BASE", "bindery/")
+    monkeypatch.setenv("AUTH_METHOD", " CWA ")
+    monkeypatch.setenv("UPDATE_INTERVAL_HOURS", "1.5")
+    monkeypatch.setenv("DEFAULT_LOCKED_FIELDS", "title, authors")
+    monkeypatch.setenv("CWA_ALLOWED_USERS", "alice,bob")
+    monkeypatch.setenv("CONFIG_DIR", "/data/bindery")
+    monkeypatch.setenv("TRUST_PROXY", "yes")
+    monkeypatch.setenv("CWA_DB_PATH", " /cwa/app.db ")
+    monkeypatch.setenv("CALIBRE_DB_PATH", "/library/metadata.db")
+    reload_config()
+    assert config.URL_BASE == "/bindery"
+    assert config.AUTH_METHOD == "cwa"
+    assert config.UPDATE_INTERVAL_HOURS == 1.5
+    assert config.DEFAULT_LOCKED_FIELDS == ["title", "authors"]
+    assert config.CWA_ALLOWED_USERS == ["alice", "bob"]
+    assert str(config.DB_PATH) == "/data/bindery/bindery.db"
+    assert config.TRUST_PROXY is True
+    assert str(config.CWA_DB_PATH) == "/cwa/app.db"
+    assert str(config.CALIBRE_DB_PATH) == "/library/metadata.db"
+
+
+def test_defaults(monkeypatch, reload_config):
+    for name in (
+        "URL_BASE",
+        "AUTH_METHOD",
+        "UPDATE_INTERVAL_HOURS",
+        "DEFAULT_LOCKED_FIELDS",
+        "LOG_LEVEL",
+        "PROXY_AUTH_USER_HEADER",
+        "SESSION_COOKIE_SECURE",
+        "TRUST_PROXY",
+        "CWA_DB_PATH",
+        "CALIBRE_DB_PATH",
+    ):
+        monkeypatch.delenv(name, raising=False)
+    reload_config()
+    assert config.URL_BASE == ""
+    assert config.AUTH_METHOD == "none"
+    assert config.UPDATE_INTERVAL_HOURS == 24
+    assert config.DEFAULT_LOCKED_FIELDS == ["title"]
+    assert config.LOG_LEVEL == "INFO"
+    assert config.PROXY_AUTH_USER_HEADER == "X-Remote-User"
+    assert config.SESSION_COOKIE_SECURE is False
+    assert config.TRUST_PROXY is False
+    assert config.CWA_DB_PATH is None
+    assert config.CALIBRE_DB_PATH is None

tests/test_fff_flows.py

@@ -0,0 +1,108 @@
+"""Exercise the FanFicFare boundary without requests to a story site."""
+
+import zipfile
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+from test_build import IMG_URL, PNG, URL, chapters, metahtml
+
+from bindery import config, fff
+
+
+@pytest.mark.parametrize("cover", [None, {}, {"url": IMG_URL, "ext": "png", "mime": "image/png", "data": PNG}])
+def test_metadata_snapshot_from_adapter(monkeypatch, cover):
+    adapter = fff.get_adapter(URL)
+    adapter.setStoryMetadata(metahtml())
+    adapter.chapterUrls = chapters()
+    adapter.story.img_store.cover = cover
+    fetch = Mock(return_value=adapter.story)
+    monkeypatch.setattr(adapter, "getStoryMetadataOnly", fetch)
+    snap = fff.fetch_story(adapter)
+    fetch.assert_called_once_with(get_cover=True)
+    assert snap.title == "Test Story"
+    assert snap.author == "Writer"
+    assert snap.site == "www.royalroad.com"
+    assert snap.chapters == [{"url": c["url"], "title": c["title"]} for c in chapters()]
+    assert snap.calibre["authors"] == ["Writer"]
+    assert snap.cover == (cover or None)
+    assert "Test Story" in snap.metahtml
+
+
+@pytest.mark.parametrize(
+    "error,message",
+    [
+        (fff.exceptions.StoryDoesNotExist(URL), "StoryDoesNotExist"),
+        (fff.exceptions.AccessDenied("private"), "AccessDenied"),
+        (fff.exceptions.FailedToLogin(URL, "reader"), "FailedToLogin"),
+        (fff.exceptions.AdultCheckRequired(URL), "is_adult:true"),
+    ],
+)
+def test_adapter_errors_become_actionable_story_errors(error, message):
+    adapter = SimpleNamespace(getStoryMetadataOnly=Mock(side_effect=error))
+    with pytest.raises(fff.StoryError, match=message):
+        fff.fetch_story(adapter)
+
+
+def test_downloaded_chapter_restores_original_image_urls():
+    html = '<p>Text</p><img src="images/1.png" longdesc="https://images.test/1.png">'
+    adapter = SimpleNamespace(getChapterTextNum=Mock(return_value=html))
+    result = fff.fetch_chapter(adapter, "chapter", 3)
+    adapter.getChapterTextNum.assert_called_once_with("chapter", 3)
+    assert 'src="https://images.test/1.png"' in result
+    for src in ["data:image/png;base64,abc", "failedtoload", "https://images.test/direct.png"]:
+        markup = f'<img src="{src}"' + (' longdesc="original"' if src != "https://images.test/direct.png" else "") + ">"
+        assert f'src="{src}"' in fff.canonical_html(markup)
+    assert fff.canonical_html("<p>plain</p>") == "<p>plain</p>"
+
+
+def test_collected_images_skip_failed_and_empty_downloads():
+    image = {"url": "image", "ext": "png", "mime": "image/png", "data": PNG}
+    adapter = SimpleNamespace(
+        story=SimpleNamespace(
+            img_store=SimpleNamespace(
+                infos=[{**image, "newsrc": "images/1.png"}, {**image, "data": b""}, {**image, "newsrc": "failedtoload"}]
+            )
+        )
+    )
+    assert fff.collected_images(adapter) == [image]
+
+
+def test_personal_ini_overrides_and_invalid_input(monkeypatch, tmp_path):
+    ini = tmp_path / "personal.ini"
+    ini.write_text("[overrides]\nis_adult: true\n", encoding="utf-8")
+    monkeypatch.setattr(config, "PERSONAL_INI", ini)
+    configuration = fff.get_configuration(URL)
+    assert configuration.getConfig("is_adult")
+    for text in ["[defaults]\nbroken line\n", "setting: before any section"]:
+        with pytest.raises(ValueError, match="cannot read"):
+            fff.check_personal_ini(text)
+    with pytest.raises(fff.StoryError, match="does not support"):
+        fff.get_configuration("https://unsupported.example/story")
+
+
+def test_adapter_rejects_invalid_story_url(monkeypatch):
+    monkeypatch.setattr(
+        fff.adapters, "getAdapter", Mock(side_effect=fff.exceptions.InvalidStoryURL(URL, "site", "example"))
+    )
+    with pytest.raises(fff.StoryError):
+        fff.get_adapter(URL)
+
+
+def test_normalization_errors_are_not_site_requests(monkeypatch):
+    monkeypatch.setattr(fff.adapters, "getNormalStoryURLSite", Mock(side_effect=ValueError("bad URL")))
+    fff.normalize_url.cache_clear()
+    try:
+        assert fff.normalize_url("bad-input") is None
+    finally:
+        fff.normalize_url.cache_clear()
+
+
+def test_empty_chapters_fail_and_stored_cover_is_built(tmp_path):
+    with pytest.raises(fff.StoryError, match="no chapters"):
+        fff.build_epub(URL, metahtml(), [], [], None, [], {}, tmp_path / "empty.epub")
+    cover = {"url": IMG_URL, "ext": "png", "mime": "image/png", "data": PNG}
+    out = tmp_path / "cover.epub"
+    fff.build_epub(URL, metahtml(), chapters()[:1], [], cover, [], {}, out)
+    with zipfile.ZipFile(out) as book:
+        assert any(book.read(n) == PNG for n in book.namelist() if n.endswith(".png"))