diff --git a/bindery/config.py b/bindery/config.py
new file mode 100644
index 0000000..d58c8ba
--- /dev/null
+++ b/bindery/config.py
@@ -0,0 +1,88 @@
+"""Configuration. Every setting comes from an environment variable."""
+
+import logging
+import os
+import shutil
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+def _bool(name: str, default: str = "false") -> bool:
+ return os.getenv(name, default).strip().lower() in ("true", "yes", "1", "y", "on")
+
+
+def _list(name: str, default: str = "") -> list[str]:
+ return [v.strip() for v in os.getenv(name, default).split(",") if v.strip()]
+
+
+# A relative path is relative to the working folder. Paths are made absolute at once, because Flask's send_file
+# reads a relative path from the package folder.
+def _path(name: str, default: str) -> Path:
+ return Path(os.getenv(name, default)).absolute()
+
+
+def _optional_path(name: str) -> Path | None:
+ value = os.getenv(name, "").strip()
+ return Path(value).absolute() if value else None
+
+
+def _base_path(value: str) -> str:
+ path = value.strip().strip("/")
+ return f"/{path}" if path else ""
+
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+
+CONFIG_DIR = _path("CONFIG_DIR", "/config")
+INGEST_DIR = _path("INGEST_DIR", "/cwa-book-ingest")
+TMP_DIR = _path("TMP_DIR", "/tmp/bindery")
+LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").strip().upper()
+# A path prefix, such as /bindery, for a reverse proxy that forwards the full path.
+URL_BASE = _base_path(os.getenv("URL_BASE", ""))
+
+# The CWA account that Bindery uses for the OPDS feed.
+CWA_URL = os.getenv("CWA_URL", "").strip()
+CWA_USERNAME = os.getenv("CWA_USERNAME", "").strip()
+CWA_PASSWORD = os.getenv("CWA_PASSWORD", "")
+# Independent, optional read-only databases. HTTP remains the fallback.
+CWA_DB_PATH = _optional_path("CWA_DB_PATH") # Calibre-Web app.db, for logins
+CALIBRE_DB_PATH = _optional_path("CALIBRE_DB_PATH") # calibre metadata.db, for book IDs
+
+UPDATE_INTERVAL_HOURS = float(os.getenv("UPDATE_INTERVAL_HOURS", "24"))
+DEFAULT_LOCKED_FIELDS = _list("DEFAULT_LOCKED_FIELDS", "title")
+
+# Logins: none, cwa or proxy. An API key works in every mode.
+AUTH_METHOD = os.getenv("AUTH_METHOD", "none").strip().lower()
+CWA_ALLOWED_USERS = _list("CWA_ALLOWED_USERS")
+PROXY_AUTH_USER_HEADER = os.getenv("PROXY_AUTH_USER_HEADER", "X-Remote-User").strip()
+PROXY_AUTH_LOGOUT_URL = os.getenv("PROXY_AUTH_LOGOUT_URL", "").strip()
+API_KEY = os.getenv("BINDERY_API_KEY", "").strip()
+SESSION_COOKIE_SECURE = _bool("SESSION_COOKIE_SECURE")
+# Take the client address from the last entry of X-Forwarded-For, which a reverse proxy adds.
+TRUST_PROXY = _bool("TRUST_PROXY")
+
+# Tests turn off the job threads.
+START_THREADS = _bool("BINDERY_START_THREADS", "true")
+
+SITE_DELAY = 5 # seconds between two jobs that go to a story site
+JOB_HISTORY = 60 * 60 # seconds that finished jobs stay in the job list
+
+DB_PATH = CONFIG_DIR / "bindery.db"
+UPLOAD_DIR = CONFIG_DIR / "uploads" # uploaded epubs wait here for their job, also over a restart
+SECRET_KEY_FILE = CONFIG_DIR / ".flask_secret"
+PERSONAL_INI = CONFIG_DIR / "personal.ini"
+DEFAULT_PERSONAL_INI = PROJECT_ROOT / "data" / "personal.ini"
+BACKUP_DIR = CONFIG_DIR / "backups"
+BACKUP_COUNT = 7 # daily backups to keep
+
+
+def prepare() -> None:
+ """Make the folders, and give the user a personal.ini to edit on first start."""
+ for directory in (CONFIG_DIR, UPLOAD_DIR, TMP_DIR, INGEST_DIR):
+ directory.mkdir(parents=True, exist_ok=True)
+ if not PERSONAL_INI.exists():
+ shutil.copy(DEFAULT_PERSONAL_INI, PERSONAL_INI)
+ logger.info(f"Created the default FanFicFare configuration at {PERSONAL_INI}")
+ if not CWA_URL:
+ logger.warning("CWA_URL is not set. OPDS imports and HTTP fallbacks are unavailable.")
diff --git a/bindery/fff.py b/bindery/fff.py
new file mode 100644
index 0000000..b87041a
--- /dev/null
+++ b/bindery/fff.py
@@ -0,0 +1,335 @@
+"""Headless FanFicFare wrapper: site access, chapter fetches and epub builds.
+
+This module uses FanFicFare's adapters (the site scrapers) and its epub writer
+without changes. The download and update flow of the FanFicFare CLI is not used.
+Instead, chapters live in the store, and every epub is built from the store:
+
+ 1. fetch_story() reads metadata and the chapter list from the site.
+ 2. fetch_chapter() downloads one chapter and returns its HTML with image
+ sources set back to their original URLs.
+ 3. build_epub() gives FanFicFare's writer the stored chapters as "old
+ chapters" (the same mechanism FanFicFare uses for epub updates), so the
+ writer never goes to the site.
+"""
+
+import configparser
+import html
+import io
+import logging
+import re
+from dataclasses import dataclass, field
+from datetime import datetime
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+import fanficfare
+from bs4 import BeautifulSoup
+from fanficfare import adapters, exceptions, writers
+from fanficfare.configurable import Configuration
+from fanficfare.epubutils import get_update_data
+from fanficfare.story import imagetypes
+
+from bindery import config
+from bindery.diffing import html_to_blocks
+from bindery.rules import Step, apply_rules
+
+logger = logging.getLogger(__name__)
+
+# FanFicFare logs every request at DEBUG level.
+logging.getLogger("fanficfare").setLevel(logging.DEBUG if config.LOG_LEVEL == "DEBUG" else logging.WARNING)
+
+DEFAULTS_INI = Path(fanficfare.__file__).parent / "defaults.ini"
+
+# Settings this application depends on. They go in [overrides], so
+# personal.ini cannot change them.
+FORCED_SETTINGS = {
+ "always_reload_first_chapter": "false",
+ "mark_new_chapters": "false",
+ "continue_on_chapter_error": "false",
+ "make_directories": "false",
+ "zip_output": "false",
+ # The epub is the only way metadata reaches CWA, so the series must be in the OPF.
+ "calibre_series_meta": "true",
+}
+
+
+class StoryError(Exception):
+ """A problem the user can fix, such as an unsupported URL."""
+
+
+@dataclass
+class SiteSnapshot:
+ """What the site says about a story right now."""
+
+ url: str
+ site: str
+ title: str
+ author: str
+ metahtml: str
+ chapters: list[dict[str, str]]
+ calibre: dict[str, Any]
+ cover: dict[str, Any] | None = None
+
+
+@dataclass
+class EpubContents:
+ """Chapters and images read from an existing FanFicFare epub."""
+
+ url: str | None
+ chapters: list[dict[str, str]] = field(default_factory=list)
+ images: list[dict[str, Any]] = field(default_factory=list)
+
+
+def escape_title(text: str) -> str:
+ """A chapter title in FanFicFare's form, which the store keeps.
+
+ FanFicFare keeps &, < and > in chapter titles, because its epub
+ writer puts titles into XHTML as they are.
+ """
+ return html.escape(text, quote=False)
+
+
+def plain_title(title: str) -> str:
+ """A stored chapter title as text, for a message."""
+ return html.unescape(title)
+
+
+@lru_cache(maxsize=20000)
+def normalize_url(url: str) -> tuple[str, str] | None:
+ """Return (story URL, site) for a supported URL, or None. No network access."""
+ try:
+ return adapters.getNormalStoryURLSite(url.strip())
+ except Exception:
+ return None
+
+
+@lru_cache(maxsize=1)
+def _site_names() -> frozenset:
+ return frozenset(name.lower() for name in adapters.getSiteSections())
+
+
+def is_supported_site(name: str) -> bool:
+ """True for a site name that FanFicFare supports, such as www.royalroad.com."""
+ return name.strip().lower() in _site_names()
+
+
+def get_configuration(url: str) -> Configuration:
+ try:
+ configuration = Configuration(adapters.getConfigSectionsFor(url), "epub")
+ except exceptions.UnknownSite as e:
+ raise StoryError(f"FanFicFare does not support site: {url}") from e
+ with open(DEFAULTS_INI, encoding="utf-8") as f:
+ configuration.read_file(f)
+ if config.PERSONAL_INI.exists():
+ with open(config.PERSONAL_INI, encoding="utf-8") as f:
+ configuration.read_file(f)
+ try:
+ configuration.add_section("overrides")
+ except configparser.DuplicateSectionError:
+ pass
+ for key, value in FORCED_SETTINGS.items():
+ configuration.set("overrides", key, value)
+ # FanFicFare downloads the cover only if include_images is on, and it is off by default. "coveronly" keeps the
+ # cover, but still leaves out the images in the story text.
+ if not configuration.getConfig("include_images"):
+ configuration.set("overrides", "include_images", "coveronly")
+ configuration.cached_config.clear()
+ return configuration
+
+
+def check_personal_ini(text: str) -> list[str]:
+ """Check the text of a personal.ini.
+
+ Raise ValueError for text that FanFicFare cannot read. Return the problems
+ that FanFicFare's own check finds, such as a section for an unknown site.
+ """
+ configuration = Configuration(["defaults"], "epub")
+ try:
+ configuration.read_file(io.StringIO(text))
+ except configparser.ParsingError as e:
+ lines = "; ".join(f"line {n}: {line.strip()}" for n, line in e.errors)
+ raise ValueError(f"FanFicFare cannot read this personal.ini. Check {lines}") from e
+ except configparser.Error as e:
+ raise ValueError(f"FanFicFare cannot read this personal.ini: {e}") from e
+ return [f"Line {n}: {message}" for n, message in configuration.test_config()]
+
+
+def get_adapter(url: str):
+ try:
+ return adapters.getAdapter(get_configuration(url), url)
+ except exceptions.InvalidStoryURL as e:
+ raise StoryError(str(e)) from e
+
+
+def parse_series(value: str) -> tuple[str, float | None]:
+ m = re.match(r"^(.*?)\s*\[([\d.]+)\]$", value or "")
+ if m:
+ return m.group(1), float(m.group(2))
+ return value or "", None
+
+
+def calibre_metadata(story) -> dict[str, Any]:
+ """Map FanFicFare story metadata to calibre fields."""
+ series, series_index = parse_series(story.getMetadata("series", removeallentities=True))
+ published = story.getMetadataRaw("datePublished")
+ return {
+ "title": story.getMetadata("title", removeallentities=True),
+ "authors": story.getList("author", removeallentities=True),
+ "comments": story.getMetadata("description"),
+ "tags": story.getSubjectTags(removeallentities=True),
+ "series": series,
+ "series_index": series_index,
+ "pubdate": published.isoformat() if hasattr(published, "isoformat") else "",
+ "languages": story.getMetadata("langcode"),
+ }
+
+
+def fetch_story(adapter) -> SiteSnapshot:
+ """Read story metadata, the chapter list and the cover from the site."""
+ try:
+ story = adapter.getStoryMetadataOnly(get_cover=True)
+ except (exceptions.StoryDoesNotExist, exceptions.AccessDenied, exceptions.FailedToLogin) as e:
+ raise StoryError(f"{type(e).__name__}: {e}") from e
+ except exceptions.AdultCheckRequired as e:
+ raise StoryError("Site asks for adult check. Set is_adult:true in personal.ini.") from e
+ cover = None
+ info = story.img_store.cover
+ if info and info.get("data"):
+ cover = {k: info[k] for k in ("url", "ext", "mime", "data")}
+ return SiteSnapshot(
+ url=story.getMetadata("storyUrl"),
+ site=adapter.getSiteDomain(),
+ title=story.getMetadata("title", removeallentities=True),
+ author=", ".join(story.getList("author", removeallentities=True)),
+ metahtml=story.dump_html_metadata(),
+ chapters=[{"url": c["url"], "title": c["title"]} for c in adapter.get_chapters()],
+ calibre=calibre_metadata(story),
+ cover=cover,
+ )
+
+
+def canonical_html(html: str) -> str:
+ """Point ![]()
tags back at their original URLs, as FanFicFare's update code does."""
+ if " str:
+ """Download one chapter. Images go into the adapter's image store."""
+ return canonical_html(adapter.getChapterTextNum(url, index))
+
+
+def collected_images(adapter) -> list[dict[str, Any]]:
+ """Images downloaded so far by this adapter, without the cover."""
+ images = []
+ for info in adapter.story.img_store.infos:
+ if info.get("data") and info.get("newsrc") != "failedtoload":
+ images.append({k: info[k] for k in ("url", "ext", "mime", "data")})
+ return images
+
+
+def word_count(chapters: list[dict[str, Any]]) -> int:
+ return sum(len(block.split()) for c in chapters for block in html_to_blocks(c["html"]))
+
+
+def apply_overrides(story, overrides: dict[str, Any]) -> None:
+ """Put the kept values of locked fields into FanFicFare's story metadata.
+
+ The epub writer takes the OPF metadata from these entries, and CWA takes
+ the book record from the OPF on the first import.
+ """
+ for key, value in overrides.items():
+ if key == "title" and value:
+ story.setMetadata("title", value)
+ elif key == "authors" and value:
+ for n, author in enumerate(value):
+ story.addToList("author", author, clear=(n == 0))
+ elif key == "comments":
+ story.setMetadata("description", value or "")
+ elif key == "series":
+ name, index = value if value else ("", None)
+ if name and index is not None:
+ story.setMetadata("series", f"{name} [{index:g}]")
+ else:
+ story.setMetadata("series", name or "")
+ elif key == "pubdate":
+ try:
+ story.setMetadata("datePublished", datetime.fromisoformat(value) if value else None)
+ except ValueError:
+ logger.warning(f"Ignoring a kept publication date that is not a date: {value!r}")
+ elif key == "languages" and value:
+ story.setMetadata("langcode", value)
+ elif key == "tags":
+ tags = list(value or [])
+ # getSubjectTags() builds the tags from configuration. The locked
+ # list replaces it for this build only.
+ story.getSubjectTags = lambda removeallentities=False, tags=tags: tags
+
+
+def build_epub(
+ url: str,
+ metahtml: str,
+ chapters: list[dict[str, Any]],
+ images: list[dict[str, Any]],
+ cover: dict[str, Any] | None,
+ rules: list[Step],
+ overrides: dict[str, Any],
+ out_path: Path,
+) -> dict[str, Any]:
+ """Write an epub from stored data. No network access.
+
+ chapters: dicts with url, title and html, in book order.
+ overrides: calibre field values that replace the site values in the epub
+ (locked fields).
+ Returns the calibre metadata of the built book.
+ """
+ if not chapters:
+ raise StoryError("Story has no chapters to put in a book")
+ adapter = get_adapter(url)
+ adapter.setStoryMetadata(metahtml)
+ story = adapter.story
+ apply_overrides(story, overrides)
+
+ adapter.chapterUrls = [{"url": c["url"], "title": c["title"]} for c in chapters]
+ story.setMetadata("numChapters", len(chapters))
+ story.setMetadata("numWords", word_count(chapters))
+ adapter.oldchaptersmap = {
+ c["url"]: BeautifulSoup(apply_rules(c["html"] or "", rules), "html5lib") for c in chapters
+ }
+ adapter.oldimgs = {img["url"]: (f"images/stored.{img['ext']}", img["data"]) for img in images}
+ if cover and adapter.getConfig("include_images"):
+ info = story.img_store.add_img(cover["url"], cover["ext"], cover["mime"], cover["data"], cover=True)
+ story.cover = info["newsrc"]
+ story.setMetadata("cover_image", "specific")
+
+ writer = writers.getWriter("epub", adapter.get_configuration(), adapter)
+ out_path.parent.mkdir(parents=True, exist_ok=True)
+ writer.writeStory(outfilename=str(out_path), forceOverwrite=True)
+ return calibre_metadata(story)
+
+
+def read_epub(path: Path) -> EpubContents:
+ """Read the chapters of an epub that FanFicFare wrote.
+
+ Chapter HTML comes back with image sources set to the original URLs.
+ """
+ (source, _filecount, _soups, images, _oldcover, _bookmark, _logfile, urlsoups, datamaps) = get_update_data(
+ str(path)
+ )[0:9]
+ contents = EpubContents(url=source)
+ for n, (chapter_url, bodysoup) in enumerate(urlsoups.items(), start=1):
+ data = datamaps.get(chapter_url, {})
+ title = data.get("chapterorigtitle") or data.get("chaptertitle") or f"Chapter {n}"
+ text = re.sub(r"]*>", "", str(bodysoup)).strip()
+ contents.chapters.append({"url": chapter_url, "title": escape_title(title), "html": text})
+ for img_url, (src, img_data) in images.items():
+ ext = src.rsplit(".", 1)[-1].lower()
+ if ext in imagetypes:
+ contents.images.append({"url": img_url, "ext": ext, "mime": imagetypes[ext], "data": img_data})
+ return contents
diff --git a/data/personal.ini b/data/personal.ini
new file mode 100644
index 0000000..2f88a09
--- /dev/null
+++ b/data/personal.ini
@@ -0,0 +1,28 @@
+## FanFicFare configuration for CWA Bindery.
+##
+## This file has the same format as personal.ini in the FanFicFare calibre
+## plugin and CLI. You can copy settings from your plugin configuration.
+## Settings here override FanFicFare's defaults.ini. Documentation:
+## https://github.com/JimmXinu/FanFicFare/wiki
+##
+## Bindery always sets these, so changes to them have no effect:
+## always_reload_first_chapter, mark_new_chapters, continue_on_chapter_error,
+## make_directories, zip_output.
+
+[defaults]
+## Confirm that you are an adult in your locale, for sites that ask.
+is_adult:true
+
+## Put the images from chapters into the epub.
+include_images:true
+
+[www.royalroad.com]
+## Seconds to wait between requests, to stay under RoyalRoad's rate limit.
+slow_down_sleep_time:2
+
+## Uncomment to include author notes with the chapter text.
+#include_author_notes:true
+
+## Uncomment to log in. RoyalRoad then marks downloaded chapters as read.
+#username:YourName
+#password:yourpassword
diff --git a/tests/conftest.py b/tests/conftest.py
index 7788b19..fd3400d 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -7,3 +7,21 @@ from pathlib import Path
_tmp = Path(tempfile.mkdtemp(prefix="bindery-tests-"))
for name, sub in [("CONFIG_DIR", "config"), ("INGEST_DIR", "ingest"), ("TMP_DIR", "tmp")]:
os.environ[name] = str(_tmp / sub)
+os.environ["BINDERY_START_THREADS"] = "false"
+os.environ["UPDATE_INTERVAL_HOURS"] = "0"
+for name in (
+ "CWA_URL",
+ "CWA_USERNAME",
+ "CWA_PASSWORD",
+ "CWA_DB_PATH",
+ "CALIBRE_DB_PATH",
+ "AUTH_METHOD",
+ "BINDERY_API_KEY",
+ "URL_BASE",
+ "CWA_ALLOWED_USERS",
+):
+ os.environ.pop(name, None)
+
+from bindery import config # noqa: E402
+
+config.prepare()
diff --git a/tests/test_build.py b/tests/test_build.py
new file mode 100644
index 0000000..31d953e
--- /dev/null
+++ b/tests/test_build.py
@@ -0,0 +1,155 @@
+"""Build an epub from stored data without network access, then read it back."""
+
+import base64
+import datetime
+import re
+import zipfile
+
+import pytest
+from defusedxml import ElementTree
+
+from bindery import config, fff
+from bindery.rules import compile_rules, normalize_rules
+
+URL = "https://www.royalroad.com/fiction/1234/test-story"
+PNG = base64.b64decode(
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+)
+IMG_URL = "https://www.royalroadcdn.com/public/img.png"
+COVER = {"url": "https://www.royalroadcdn.com/public/covers/1234.png", "ext": "png", "mime": "image/png", "data": PNG}
+
+
+def metahtml(url=URL):
+ adapter = fff.get_adapter(url)
+ story = adapter.story
+ story.setMetadata("title", "Test Story")
+ story.addToList("author", "Writer")
+ story.addToList("authorId", "42")
+ story.setMetadata("description", "About the story
")
+ story.setMetadata("datePublished", datetime.datetime(2024, 1, 2))
+ story.setMetadata("dateUpdated", datetime.datetime(2024, 3, 4))
+ story.addToList("genre", "Fantasy")
+ return story.dump_html_metadata()
+
+
+def chapters():
+ base = "https://www.royalroad.com/fiction/1234/test-story/chapter/{0}/c{0}"
+ return [
+ {"url": base.format(10), "title": "One", "html": ''},
+ {"url": base.format(11), "title": "Two", "html": f'Picture
Text
'
+ adapter = SimpleNamespace(getChapterTextNum=Mock(return_value=html))
+ result = fff.fetch_chapter(adapter, "chapter", 3)
+ adapter.getChapterTextNum.assert_called_once_with("chapter", 3)
+ assert 'src="https://images.test/1.png"' in result
+ for src in ["data:image/png;base64,abc", "failedtoload", "https://images.test/direct.png"]:
+ markup = f'
"
+ assert f'src="{src}"' in fff.canonical_html(markup)
+ assert fff.canonical_html("plain
") == "plain
"
+
+
+def test_collected_images_skip_failed_and_empty_downloads():
+ image = {"url": "image", "ext": "png", "mime": "image/png", "data": PNG}
+ adapter = SimpleNamespace(
+ story=SimpleNamespace(
+ img_store=SimpleNamespace(
+ infos=[{**image, "newsrc": "images/1.png"}, {**image, "data": b""}, {**image, "newsrc": "failedtoload"}]
+ )
+ )
+ )
+ assert fff.collected_images(adapter) == [image]
+
+
+def test_personal_ini_overrides_and_invalid_input(monkeypatch, tmp_path):
+ ini = tmp_path / "personal.ini"
+ ini.write_text("[overrides]\nis_adult: true\n", encoding="utf-8")
+ monkeypatch.setattr(config, "PERSONAL_INI", ini)
+ configuration = fff.get_configuration(URL)
+ assert configuration.getConfig("is_adult")
+ for text in ["[defaults]\nbroken line\n", "setting: before any section"]:
+ with pytest.raises(ValueError, match="cannot read"):
+ fff.check_personal_ini(text)
+ with pytest.raises(fff.StoryError, match="does not support"):
+ fff.get_configuration("https://unsupported.example/story")
+
+
+def test_adapter_rejects_invalid_story_url(monkeypatch):
+ monkeypatch.setattr(
+ fff.adapters, "getAdapter", Mock(side_effect=fff.exceptions.InvalidStoryURL(URL, "site", "example"))
+ )
+ with pytest.raises(fff.StoryError):
+ fff.get_adapter(URL)
+
+
+def test_normalization_errors_are_not_site_requests(monkeypatch):
+ monkeypatch.setattr(fff.adapters, "getNormalStoryURLSite", Mock(side_effect=ValueError("bad URL")))
+ fff.normalize_url.cache_clear()
+ try:
+ assert fff.normalize_url("bad-input") is None
+ finally:
+ fff.normalize_url.cache_clear()
+
+
+def test_empty_chapters_fail_and_stored_cover_is_built(tmp_path):
+ with pytest.raises(fff.StoryError, match="no chapters"):
+ fff.build_epub(URL, metahtml(), [], [], None, [], {}, tmp_path / "empty.epub")
+ cover = {"url": IMG_URL, "ext": "png", "mime": "image/png", "data": PNG}
+ out = tmp_path / "cover.epub"
+ fff.build_epub(URL, metahtml(), chapters()[:1], [], cover, [], {}, out)
+ with zipfile.ZipFile(out) as book:
+ assert any(book.read(n) == PNG for n in book.namelist() if n.endswith(".png"))