Download stories and build epubs with FanFicFare
FanFicFare adapters fetch metadata and chapters, and its epub writer builds each book from the store. Environment variables configure the app, and a default personal.ini is copied to the config folder on first start.
This commit is contained in:
@@ -0,0 +1,88 @@
|
||||
"""Configuration. Every setting comes from an environment variable."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _bool(name: str, default: str = "false") -> bool:
|
||||
return os.getenv(name, default).strip().lower() in ("true", "yes", "1", "y", "on")
|
||||
|
||||
|
||||
def _list(name: str, default: str = "") -> list[str]:
|
||||
return [v.strip() for v in os.getenv(name, default).split(",") if v.strip()]
|
||||
|
||||
|
||||
# A relative path is relative to the working folder. Paths are made absolute at once, because Flask's send_file
|
||||
# reads a relative path from the package folder.
|
||||
def _path(name: str, default: str) -> Path:
|
||||
return Path(os.getenv(name, default)).absolute()
|
||||
|
||||
|
||||
def _optional_path(name: str) -> Path | None:
|
||||
value = os.getenv(name, "").strip()
|
||||
return Path(value).absolute() if value else None
|
||||
|
||||
|
||||
def _base_path(value: str) -> str:
|
||||
path = value.strip().strip("/")
|
||||
return f"/{path}" if path else ""
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
CONFIG_DIR = _path("CONFIG_DIR", "/config")
|
||||
INGEST_DIR = _path("INGEST_DIR", "/cwa-book-ingest")
|
||||
TMP_DIR = _path("TMP_DIR", "/tmp/bindery")
|
||||
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").strip().upper()
|
||||
# A path prefix, such as /bindery, for a reverse proxy that forwards the full path.
|
||||
URL_BASE = _base_path(os.getenv("URL_BASE", ""))
|
||||
|
||||
# The CWA account that Bindery uses for the OPDS feed.
|
||||
CWA_URL = os.getenv("CWA_URL", "").strip()
|
||||
CWA_USERNAME = os.getenv("CWA_USERNAME", "").strip()
|
||||
CWA_PASSWORD = os.getenv("CWA_PASSWORD", "")
|
||||
# Independent, optional read-only databases. HTTP remains the fallback.
|
||||
CWA_DB_PATH = _optional_path("CWA_DB_PATH") # Calibre-Web app.db, for logins
|
||||
CALIBRE_DB_PATH = _optional_path("CALIBRE_DB_PATH") # calibre metadata.db, for book IDs
|
||||
|
||||
UPDATE_INTERVAL_HOURS = float(os.getenv("UPDATE_INTERVAL_HOURS", "24"))
|
||||
DEFAULT_LOCKED_FIELDS = _list("DEFAULT_LOCKED_FIELDS", "title")
|
||||
|
||||
# Logins: none, cwa or proxy. An API key works in every mode.
|
||||
AUTH_METHOD = os.getenv("AUTH_METHOD", "none").strip().lower()
|
||||
CWA_ALLOWED_USERS = _list("CWA_ALLOWED_USERS")
|
||||
PROXY_AUTH_USER_HEADER = os.getenv("PROXY_AUTH_USER_HEADER", "X-Remote-User").strip()
|
||||
PROXY_AUTH_LOGOUT_URL = os.getenv("PROXY_AUTH_LOGOUT_URL", "").strip()
|
||||
API_KEY = os.getenv("BINDERY_API_KEY", "").strip()
|
||||
SESSION_COOKIE_SECURE = _bool("SESSION_COOKIE_SECURE")
|
||||
# Take the client address from the last entry of X-Forwarded-For, which a reverse proxy adds.
|
||||
TRUST_PROXY = _bool("TRUST_PROXY")
|
||||
|
||||
# Tests turn off the job threads.
|
||||
START_THREADS = _bool("BINDERY_START_THREADS", "true")
|
||||
|
||||
SITE_DELAY = 5 # seconds between two jobs that go to a story site
|
||||
JOB_HISTORY = 60 * 60 # seconds that finished jobs stay in the job list
|
||||
|
||||
DB_PATH = CONFIG_DIR / "bindery.db"
|
||||
UPLOAD_DIR = CONFIG_DIR / "uploads" # uploaded epubs wait here for their job, also over a restart
|
||||
SECRET_KEY_FILE = CONFIG_DIR / ".flask_secret"
|
||||
PERSONAL_INI = CONFIG_DIR / "personal.ini"
|
||||
DEFAULT_PERSONAL_INI = PROJECT_ROOT / "data" / "personal.ini"
|
||||
BACKUP_DIR = CONFIG_DIR / "backups"
|
||||
BACKUP_COUNT = 7 # daily backups to keep
|
||||
|
||||
|
||||
def prepare() -> None:
|
||||
"""Make the folders, and give the user a personal.ini to edit on first start."""
|
||||
for directory in (CONFIG_DIR, UPLOAD_DIR, TMP_DIR, INGEST_DIR):
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
if not PERSONAL_INI.exists():
|
||||
shutil.copy(DEFAULT_PERSONAL_INI, PERSONAL_INI)
|
||||
logger.info(f"Created the default FanFicFare configuration at {PERSONAL_INI}")
|
||||
if not CWA_URL:
|
||||
logger.warning("CWA_URL is not set. OPDS imports and HTTP fallbacks are unavailable.")
|
||||
+335
@@ -0,0 +1,335 @@
|
||||
"""Headless FanFicFare wrapper: site access, chapter fetches and epub builds.
|
||||
|
||||
This module uses FanFicFare's adapters (the site scrapers) and its epub writer
|
||||
without changes. The download and update flow of the FanFicFare CLI is not used.
|
||||
Instead, chapters live in the store, and every epub is built from the store:
|
||||
|
||||
1. fetch_story() reads metadata and the chapter list from the site.
|
||||
2. fetch_chapter() downloads one chapter and returns its HTML with image
|
||||
sources set back to their original URLs.
|
||||
3. build_epub() gives FanFicFare's writer the stored chapters as "old
|
||||
chapters" (the same mechanism FanFicFare uses for epub updates), so the
|
||||
writer never goes to the site.
|
||||
"""
|
||||
|
||||
import configparser
|
||||
import html
|
||||
import io
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import fanficfare
|
||||
from bs4 import BeautifulSoup
|
||||
from fanficfare import adapters, exceptions, writers
|
||||
from fanficfare.configurable import Configuration
|
||||
from fanficfare.epubutils import get_update_data
|
||||
from fanficfare.story import imagetypes
|
||||
|
||||
from bindery import config
|
||||
from bindery.diffing import html_to_blocks
|
||||
from bindery.rules import Step, apply_rules
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# FanFicFare logs every request at DEBUG level.
|
||||
logging.getLogger("fanficfare").setLevel(logging.DEBUG if config.LOG_LEVEL == "DEBUG" else logging.WARNING)
|
||||
|
||||
DEFAULTS_INI = Path(fanficfare.__file__).parent / "defaults.ini"
|
||||
|
||||
# Settings this application depends on. They go in [overrides], so
|
||||
# personal.ini cannot change them.
|
||||
FORCED_SETTINGS = {
|
||||
"always_reload_first_chapter": "false",
|
||||
"mark_new_chapters": "false",
|
||||
"continue_on_chapter_error": "false",
|
||||
"make_directories": "false",
|
||||
"zip_output": "false",
|
||||
# The epub is the only way metadata reaches CWA, so the series must be in the OPF.
|
||||
"calibre_series_meta": "true",
|
||||
}
|
||||
|
||||
|
||||
class StoryError(Exception):
|
||||
"""A problem the user can fix, such as an unsupported URL."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SiteSnapshot:
|
||||
"""What the site says about a story right now."""
|
||||
|
||||
url: str
|
||||
site: str
|
||||
title: str
|
||||
author: str
|
||||
metahtml: str
|
||||
chapters: list[dict[str, str]]
|
||||
calibre: dict[str, Any]
|
||||
cover: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class EpubContents:
|
||||
"""Chapters and images read from an existing FanFicFare epub."""
|
||||
|
||||
url: str | None
|
||||
chapters: list[dict[str, str]] = field(default_factory=list)
|
||||
images: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
def escape_title(text: str) -> str:
|
||||
"""A chapter title in FanFicFare's form, which the store keeps.
|
||||
|
||||
FanFicFare keeps &, < and > in chapter titles, because its epub
|
||||
writer puts titles into XHTML as they are.
|
||||
"""
|
||||
return html.escape(text, quote=False)
|
||||
|
||||
|
||||
def plain_title(title: str) -> str:
|
||||
"""A stored chapter title as text, for a message."""
|
||||
return html.unescape(title)
|
||||
|
||||
|
||||
@lru_cache(maxsize=20000)
|
||||
def normalize_url(url: str) -> tuple[str, str] | None:
|
||||
"""Return (story URL, site) for a supported URL, or None. No network access."""
|
||||
try:
|
||||
return adapters.getNormalStoryURLSite(url.strip())
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _site_names() -> frozenset:
|
||||
return frozenset(name.lower() for name in adapters.getSiteSections())
|
||||
|
||||
|
||||
def is_supported_site(name: str) -> bool:
|
||||
"""True for a site name that FanFicFare supports, such as www.royalroad.com."""
|
||||
return name.strip().lower() in _site_names()
|
||||
|
||||
|
||||
def get_configuration(url: str) -> Configuration:
|
||||
try:
|
||||
configuration = Configuration(adapters.getConfigSectionsFor(url), "epub")
|
||||
except exceptions.UnknownSite as e:
|
||||
raise StoryError(f"FanFicFare does not support site: {url}") from e
|
||||
with open(DEFAULTS_INI, encoding="utf-8") as f:
|
||||
configuration.read_file(f)
|
||||
if config.PERSONAL_INI.exists():
|
||||
with open(config.PERSONAL_INI, encoding="utf-8") as f:
|
||||
configuration.read_file(f)
|
||||
try:
|
||||
configuration.add_section("overrides")
|
||||
except configparser.DuplicateSectionError:
|
||||
pass
|
||||
for key, value in FORCED_SETTINGS.items():
|
||||
configuration.set("overrides", key, value)
|
||||
# FanFicFare downloads the cover only if include_images is on, and it is off by default. "coveronly" keeps the
|
||||
# cover, but still leaves out the images in the story text.
|
||||
if not configuration.getConfig("include_images"):
|
||||
configuration.set("overrides", "include_images", "coveronly")
|
||||
configuration.cached_config.clear()
|
||||
return configuration
|
||||
|
||||
|
||||
def check_personal_ini(text: str) -> list[str]:
|
||||
"""Check the text of a personal.ini.
|
||||
|
||||
Raise ValueError for text that FanFicFare cannot read. Return the problems
|
||||
that FanFicFare's own check finds, such as a section for an unknown site.
|
||||
"""
|
||||
configuration = Configuration(["defaults"], "epub")
|
||||
try:
|
||||
configuration.read_file(io.StringIO(text))
|
||||
except configparser.ParsingError as e:
|
||||
lines = "; ".join(f"line {n}: {line.strip()}" for n, line in e.errors)
|
||||
raise ValueError(f"FanFicFare cannot read this personal.ini. Check {lines}") from e
|
||||
except configparser.Error as e:
|
||||
raise ValueError(f"FanFicFare cannot read this personal.ini: {e}") from e
|
||||
return [f"Line {n}: {message}" for n, message in configuration.test_config()]
|
||||
|
||||
|
||||
def get_adapter(url: str):
|
||||
try:
|
||||
return adapters.getAdapter(get_configuration(url), url)
|
||||
except exceptions.InvalidStoryURL as e:
|
||||
raise StoryError(str(e)) from e
|
||||
|
||||
|
||||
def parse_series(value: str) -> tuple[str, float | None]:
|
||||
m = re.match(r"^(.*?)\s*\[([\d.]+)\]$", value or "")
|
||||
if m:
|
||||
return m.group(1), float(m.group(2))
|
||||
return value or "", None
|
||||
|
||||
|
||||
def calibre_metadata(story) -> dict[str, Any]:
|
||||
"""Map FanFicFare story metadata to calibre fields."""
|
||||
series, series_index = parse_series(story.getMetadata("series", removeallentities=True))
|
||||
published = story.getMetadataRaw("datePublished")
|
||||
return {
|
||||
"title": story.getMetadata("title", removeallentities=True),
|
||||
"authors": story.getList("author", removeallentities=True),
|
||||
"comments": story.getMetadata("description"),
|
||||
"tags": story.getSubjectTags(removeallentities=True),
|
||||
"series": series,
|
||||
"series_index": series_index,
|
||||
"pubdate": published.isoformat() if hasattr(published, "isoformat") else "",
|
||||
"languages": story.getMetadata("langcode"),
|
||||
}
|
||||
|
||||
|
||||
def fetch_story(adapter) -> SiteSnapshot:
|
||||
"""Read story metadata, the chapter list and the cover from the site."""
|
||||
try:
|
||||
story = adapter.getStoryMetadataOnly(get_cover=True)
|
||||
except (exceptions.StoryDoesNotExist, exceptions.AccessDenied, exceptions.FailedToLogin) as e:
|
||||
raise StoryError(f"{type(e).__name__}: {e}") from e
|
||||
except exceptions.AdultCheckRequired as e:
|
||||
raise StoryError("Site asks for adult check. Set is_adult:true in personal.ini.") from e
|
||||
cover = None
|
||||
info = story.img_store.cover
|
||||
if info and info.get("data"):
|
||||
cover = {k: info[k] for k in ("url", "ext", "mime", "data")}
|
||||
return SiteSnapshot(
|
||||
url=story.getMetadata("storyUrl"),
|
||||
site=adapter.getSiteDomain(),
|
||||
title=story.getMetadata("title", removeallentities=True),
|
||||
author=", ".join(story.getList("author", removeallentities=True)),
|
||||
metahtml=story.dump_html_metadata(),
|
||||
chapters=[{"url": c["url"], "title": c["title"]} for c in adapter.get_chapters()],
|
||||
calibre=calibre_metadata(story),
|
||||
cover=cover,
|
||||
)
|
||||
|
||||
|
||||
def canonical_html(html: str) -> str:
|
||||
"""Point <img> tags back at their original URLs, as FanFicFare's update code does."""
|
||||
if "<img" not in html:
|
||||
return html
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
for img in soup.find_all("img"):
|
||||
if img.has_attr("longdesc") and not img["src"].startswith(("data:", "failedtoload")):
|
||||
img["src"] = img["longdesc"]
|
||||
return str(soup)
|
||||
|
||||
|
||||
def fetch_chapter(adapter, url: str, index: int) -> str:
|
||||
"""Download one chapter. Images go into the adapter's image store."""
|
||||
return canonical_html(adapter.getChapterTextNum(url, index))
|
||||
|
||||
|
||||
def collected_images(adapter) -> list[dict[str, Any]]:
|
||||
"""Images downloaded so far by this adapter, without the cover."""
|
||||
images = []
|
||||
for info in adapter.story.img_store.infos:
|
||||
if info.get("data") and info.get("newsrc") != "failedtoload":
|
||||
images.append({k: info[k] for k in ("url", "ext", "mime", "data")})
|
||||
return images
|
||||
|
||||
|
||||
def word_count(chapters: list[dict[str, Any]]) -> int:
|
||||
return sum(len(block.split()) for c in chapters for block in html_to_blocks(c["html"]))
|
||||
|
||||
|
||||
def apply_overrides(story, overrides: dict[str, Any]) -> None:
|
||||
"""Put the kept values of locked fields into FanFicFare's story metadata.
|
||||
|
||||
The epub writer takes the OPF metadata from these entries, and CWA takes
|
||||
the book record from the OPF on the first import.
|
||||
"""
|
||||
for key, value in overrides.items():
|
||||
if key == "title" and value:
|
||||
story.setMetadata("title", value)
|
||||
elif key == "authors" and value:
|
||||
for n, author in enumerate(value):
|
||||
story.addToList("author", author, clear=(n == 0))
|
||||
elif key == "comments":
|
||||
story.setMetadata("description", value or "")
|
||||
elif key == "series":
|
||||
name, index = value if value else ("", None)
|
||||
if name and index is not None:
|
||||
story.setMetadata("series", f"{name} [{index:g}]")
|
||||
else:
|
||||
story.setMetadata("series", name or "")
|
||||
elif key == "pubdate":
|
||||
try:
|
||||
story.setMetadata("datePublished", datetime.fromisoformat(value) if value else None)
|
||||
except ValueError:
|
||||
logger.warning(f"Ignoring a kept publication date that is not a date: {value!r}")
|
||||
elif key == "languages" and value:
|
||||
story.setMetadata("langcode", value)
|
||||
elif key == "tags":
|
||||
tags = list(value or [])
|
||||
# getSubjectTags() builds the tags from configuration. The locked
|
||||
# list replaces it for this build only.
|
||||
story.getSubjectTags = lambda removeallentities=False, tags=tags: tags
|
||||
|
||||
|
||||
def build_epub(
|
||||
url: str,
|
||||
metahtml: str,
|
||||
chapters: list[dict[str, Any]],
|
||||
images: list[dict[str, Any]],
|
||||
cover: dict[str, Any] | None,
|
||||
rules: list[Step],
|
||||
overrides: dict[str, Any],
|
||||
out_path: Path,
|
||||
) -> dict[str, Any]:
|
||||
"""Write an epub from stored data. No network access.
|
||||
|
||||
chapters: dicts with url, title and html, in book order.
|
||||
overrides: calibre field values that replace the site values in the epub
|
||||
(locked fields).
|
||||
Returns the calibre metadata of the built book.
|
||||
"""
|
||||
if not chapters:
|
||||
raise StoryError("Story has no chapters to put in a book")
|
||||
adapter = get_adapter(url)
|
||||
adapter.setStoryMetadata(metahtml)
|
||||
story = adapter.story
|
||||
apply_overrides(story, overrides)
|
||||
|
||||
adapter.chapterUrls = [{"url": c["url"], "title": c["title"]} for c in chapters]
|
||||
story.setMetadata("numChapters", len(chapters))
|
||||
story.setMetadata("numWords", word_count(chapters))
|
||||
adapter.oldchaptersmap = {
|
||||
c["url"]: BeautifulSoup(apply_rules(c["html"] or "<div></div>", rules), "html5lib") for c in chapters
|
||||
}
|
||||
adapter.oldimgs = {img["url"]: (f"images/stored.{img['ext']}", img["data"]) for img in images}
|
||||
if cover and adapter.getConfig("include_images"):
|
||||
info = story.img_store.add_img(cover["url"], cover["ext"], cover["mime"], cover["data"], cover=True)
|
||||
story.cover = info["newsrc"]
|
||||
story.setMetadata("cover_image", "specific")
|
||||
|
||||
writer = writers.getWriter("epub", adapter.get_configuration(), adapter)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
writer.writeStory(outfilename=str(out_path), forceOverwrite=True)
|
||||
return calibre_metadata(story)
|
||||
|
||||
|
||||
def read_epub(path: Path) -> EpubContents:
|
||||
"""Read the chapters of an epub that FanFicFare wrote.
|
||||
|
||||
Chapter HTML comes back with image sources set to the original URLs.
|
||||
"""
|
||||
(source, _filecount, _soups, images, _oldcover, _bookmark, _logfile, urlsoups, datamaps) = get_update_data(
|
||||
str(path)
|
||||
)[0:9]
|
||||
contents = EpubContents(url=source)
|
||||
for n, (chapter_url, bodysoup) in enumerate(urlsoups.items(), start=1):
|
||||
data = datamaps.get(chapter_url, {})
|
||||
title = data.get("chapterorigtitle") or data.get("chaptertitle") or f"Chapter {n}"
|
||||
text = re.sub(r"</?body[^>]*>", "", str(bodysoup)).strip()
|
||||
contents.chapters.append({"url": chapter_url, "title": escape_title(title), "html": text})
|
||||
for img_url, (src, img_data) in images.items():
|
||||
ext = src.rsplit(".", 1)[-1].lower()
|
||||
if ext in imagetypes:
|
||||
contents.images.append({"url": img_url, "ext": ext, "mime": imagetypes[ext], "data": img_data})
|
||||
return contents
|
||||
@@ -0,0 +1,28 @@
|
||||
## FanFicFare configuration for CWA Bindery.
|
||||
##
|
||||
## This file has the same format as personal.ini in the FanFicFare calibre
|
||||
## plugin and CLI. You can copy settings from your plugin configuration.
|
||||
## Settings here override FanFicFare's defaults.ini. Documentation:
|
||||
## https://github.com/JimmXinu/FanFicFare/wiki
|
||||
##
|
||||
## Bindery always sets these, so changes to them have no effect:
|
||||
## always_reload_first_chapter, mark_new_chapters, continue_on_chapter_error,
|
||||
## make_directories, zip_output.
|
||||
|
||||
[defaults]
|
||||
## Confirm that you are an adult in your locale, for sites that ask.
|
||||
is_adult:true
|
||||
|
||||
## Put the images from chapters into the epub.
|
||||
include_images:true
|
||||
|
||||
[www.royalroad.com]
|
||||
## Seconds to wait between requests, to stay under RoyalRoad's rate limit.
|
||||
slow_down_sleep_time:2
|
||||
|
||||
## Uncomment to include author notes with the chapter text.
|
||||
#include_author_notes:true
|
||||
|
||||
## Uncomment to log in. RoyalRoad then marks downloaded chapters as read.
|
||||
#username:YourName
|
||||
#password:yourpassword
|
||||
@@ -7,3 +7,21 @@ from pathlib import Path
|
||||
_tmp = Path(tempfile.mkdtemp(prefix="bindery-tests-"))
|
||||
for name, sub in [("CONFIG_DIR", "config"), ("INGEST_DIR", "ingest"), ("TMP_DIR", "tmp")]:
|
||||
os.environ[name] = str(_tmp / sub)
|
||||
os.environ["BINDERY_START_THREADS"] = "false"
|
||||
os.environ["UPDATE_INTERVAL_HOURS"] = "0"
|
||||
for name in (
|
||||
"CWA_URL",
|
||||
"CWA_USERNAME",
|
||||
"CWA_PASSWORD",
|
||||
"CWA_DB_PATH",
|
||||
"CALIBRE_DB_PATH",
|
||||
"AUTH_METHOD",
|
||||
"BINDERY_API_KEY",
|
||||
"URL_BASE",
|
||||
"CWA_ALLOWED_USERS",
|
||||
):
|
||||
os.environ.pop(name, None)
|
||||
|
||||
from bindery import config # noqa: E402
|
||||
|
||||
config.prepare()
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Build an epub from stored data without network access, then read it back."""
|
||||
|
||||
import base64
|
||||
import datetime
|
||||
import re
|
||||
import zipfile
|
||||
|
||||
import pytest
|
||||
from defusedxml import ElementTree
|
||||
|
||||
from bindery import config, fff
|
||||
from bindery.rules import compile_rules, normalize_rules
|
||||
|
||||
URL = "https://www.royalroad.com/fiction/1234/test-story"
|
||||
PNG = base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
|
||||
)
|
||||
IMG_URL = "https://www.royalroadcdn.com/public/img.png"
|
||||
COVER = {"url": "https://www.royalroadcdn.com/public/covers/1234.png", "ext": "png", "mime": "image/png", "data": PNG}
|
||||
|
||||
|
||||
def metahtml(url=URL):
|
||||
adapter = fff.get_adapter(url)
|
||||
story = adapter.story
|
||||
story.setMetadata("title", "Test Story")
|
||||
story.addToList("author", "Writer")
|
||||
story.addToList("authorId", "42")
|
||||
story.setMetadata("description", "<p>About the story</p>")
|
||||
story.setMetadata("datePublished", datetime.datetime(2024, 1, 2))
|
||||
story.setMetadata("dateUpdated", datetime.datetime(2024, 3, 4))
|
||||
story.addToList("genre", "Fantasy")
|
||||
return story.dump_html_metadata()
|
||||
|
||||
|
||||
def chapters():
|
||||
base = "https://www.royalroad.com/fiction/1234/test-story/chapter/{0}/c{0}"
|
||||
return [
|
||||
{"url": base.format(10), "title": "One", "html": '<div><p class="rnd1">Zorian wakes.</p></div>'},
|
||||
{"url": base.format(11), "title": "Two", "html": f'<div><p>Picture</p><img src="{IMG_URL}"/></div>'},
|
||||
{"url": base.format(15), "title": "Three", "html": "<div><p>Zorian again.</p></div>"},
|
||||
]
|
||||
|
||||
|
||||
def test_build_and_read_back(tmp_path):
|
||||
rules = compile_rules(normalize_rules([{"pattern": "Zorian", "replacement": "Z"}]))
|
||||
out = tmp_path / "book.epub"
|
||||
images = [{"url": IMG_URL, "ext": "png", "mime": "image/png", "data": PNG}]
|
||||
meta = fff.build_epub(URL, metahtml(), chapters(), images, None, rules, {"title": "Kept Title"}, out)
|
||||
assert meta["title"] == "Kept Title"
|
||||
assert meta["authors"] == ["Writer"]
|
||||
assert meta["pubdate"].startswith("2024-01-02")
|
||||
|
||||
with zipfile.ZipFile(out) as z:
|
||||
names = z.namelist()
|
||||
text = "".join(z.read(n).decode("utf-8", "ignore") for n in names if n.endswith(".xhtml"))
|
||||
opf = z.read("content.opf").decode()
|
||||
assert "Z wakes." in text and "Zorian" not in text
|
||||
assert any(n.startswith("OEBPS/images/ffdl-") and n.endswith(".png") for n in names)
|
||||
assert re.search(r"<dc:title[^>]*>Kept Title</dc:title>", opf)
|
||||
|
||||
contents = fff.read_epub(out)
|
||||
assert [c["title"] for c in contents.chapters] == ["One", "Two", "Three"]
|
||||
assert [c["url"] for c in contents.chapters] == [c["url"] for c in chapters()]
|
||||
assert IMG_URL in contents.chapters[1]["html"]
|
||||
assert contents.images and contents.images[0]["url"] == IMG_URL
|
||||
|
||||
|
||||
def test_a_title_with_entities_builds_and_imports_in_fanficfare_form(tmp_path):
|
||||
# FanFicFare's form, as the site titles in the store have it.
|
||||
title = "Folly & Magic <1>"
|
||||
out = tmp_path / "book.epub"
|
||||
fff.build_epub(URL, metahtml(), [dict(chapters()[0], title=title)], [], None, [], {}, out)
|
||||
with zipfile.ZipFile(out) as z:
|
||||
for name in z.namelist():
|
||||
if name.endswith((".xhtml", ".ncx", ".opf")):
|
||||
ElementTree.fromstring(z.read(name))
|
||||
# An import stores the same title as a download, so the next update finds no rename.
|
||||
assert fff.read_epub(out).chapters[0]["title"] == title
|
||||
assert fff.plain_title(title) == "Folly & Magic <1>"
|
||||
assert fff.escape_title("Folly & Magic <1>") == title
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("personal_ini", "expected"),
|
||||
[
|
||||
("", "coveronly"),
|
||||
("[www.royalroad.com]\ninclude_images:false\n", "coveronly"),
|
||||
("[defaults]\ninclude_images:true\n", "true"),
|
||||
("[defaults]\ninclude_images:coveronly\n", "coveronly"),
|
||||
],
|
||||
)
|
||||
def test_include_images_is_never_off(personal_ini, expected, tmp_path, monkeypatch):
|
||||
path = tmp_path / "personal.ini"
|
||||
path.write_text(personal_ini)
|
||||
monkeypatch.setattr(config, "PERSONAL_INI", path)
|
||||
assert fff.get_configuration(URL).getConfig("include_images") == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("personal_ini", ["", "[defaults]\ninclude_images:false\n"])
|
||||
def test_the_cover_goes_in_the_epub_without_include_images(personal_ini, tmp_path, monkeypatch):
|
||||
path = tmp_path / "personal.ini"
|
||||
path.write_text(personal_ini)
|
||||
monkeypatch.setattr(config, "PERSONAL_INI", path)
|
||||
out = tmp_path / "book.epub"
|
||||
fff.build_epub(URL, metahtml(), chapters(), [], COVER, [], {}, out)
|
||||
with zipfile.ZipFile(out) as z:
|
||||
names = z.namelist()
|
||||
opf = z.read("content.opf").decode()
|
||||
assert "OEBPS/cover.xhtml" in names
|
||||
assert any(n.startswith("OEBPS/images/cover.") for n in names)
|
||||
assert re.search(r'<meta name="cover" content="[^"]+"', opf)
|
||||
|
||||
|
||||
def test_locked_fields_replace_site_metadata(tmp_path):
|
||||
out = tmp_path / "book.epub"
|
||||
overrides = {
|
||||
"title": "Kept Title",
|
||||
"authors": ["Pen Name", "Co Writer"],
|
||||
"comments": "<p>Kept blurb</p>",
|
||||
"series": ["Kept Saga", 3.0],
|
||||
"pubdate": "2019-05-06T00:00:00+00:00",
|
||||
"languages": "de",
|
||||
"tags": ["mine", "reread"],
|
||||
}
|
||||
meta = fff.build_epub(URL, metahtml(), chapters(), [], None, [], overrides, out)
|
||||
assert meta["authors"] == ["Pen Name", "Co Writer"]
|
||||
assert meta["tags"] == ["mine", "reread"]
|
||||
with zipfile.ZipFile(out) as z:
|
||||
opf = z.read("content.opf").decode()
|
||||
assert re.findall(r"<dc:creator[^>]*>([^<]*)</dc:creator>", opf) == ["Pen Name", "Co Writer"]
|
||||
assert re.findall(r"<dc:subject>([^<]*)</dc:subject>", opf) == ["mine", "reread"]
|
||||
assert 'name="calibre:series" content="Kept Saga"' in opf
|
||||
assert 'name="calibre:series_index" content="3.00"' in opf
|
||||
assert "<dc:language>de</dc:language>" in opf
|
||||
assert "2019-05-06" in opf
|
||||
assert "Kept blurb" in opf
|
||||
|
||||
|
||||
def test_unlocked_fields_keep_site_metadata(tmp_path):
|
||||
out = tmp_path / "book.epub"
|
||||
meta = fff.build_epub(URL, metahtml(), chapters(), [], None, [], {}, out)
|
||||
assert meta["title"] == "Test Story"
|
||||
assert meta["authors"] == ["Writer"]
|
||||
assert "Fantasy" in meta["tags"]
|
||||
|
||||
|
||||
def test_supported_sites():
|
||||
assert fff.is_supported_site("www.royalroad.com")
|
||||
assert fff.is_supported_site("WWW.RoyalRoad.com ")
|
||||
assert not fff.is_supported_site("example.com")
|
||||
|
||||
|
||||
def test_normalize_url():
|
||||
assert fff.normalize_url(URL) == ("https://www.royalroad.com/fiction/1234", "www.royalroad.com")
|
||||
assert fff.normalize_url("https://example.com/nothing") is None
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Reading the configuration from environment variables."""
|
||||
|
||||
import importlib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from bindery import config
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def reload_config(monkeypatch):
|
||||
"""Load the config module again with a changed environment, and restore it afterwards."""
|
||||
yield lambda: importlib.reload(config)
|
||||
monkeypatch.undo()
|
||||
importlib.reload(config)
|
||||
|
||||
|
||||
def test_bool(monkeypatch):
|
||||
for value in ("true", "Yes", " 1 ", "on", "y"):
|
||||
monkeypatch.setenv("X_TEST", value)
|
||||
assert config._bool("X_TEST")
|
||||
for value in ("false", "0", "no", ""):
|
||||
monkeypatch.setenv("X_TEST", value)
|
||||
assert not config._bool("X_TEST")
|
||||
monkeypatch.delenv("X_TEST")
|
||||
assert config._bool("X_TEST", "true")
|
||||
|
||||
|
||||
def test_list(monkeypatch):
|
||||
monkeypatch.setenv("X_TEST", " title, tags ,,series ")
|
||||
assert config._list("X_TEST") == ["title", "tags", "series"]
|
||||
monkeypatch.delenv("X_TEST")
|
||||
assert config._list("X_TEST") == []
|
||||
assert config._list("X_TEST", "title") == ["title"]
|
||||
|
||||
|
||||
def test_relative_paths_become_absolute(monkeypatch, tmp_path, reload_config):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
for name, value in [("CONFIG_DIR", "dev/config"), ("INGEST_DIR", "dev/ingest"), ("TMP_DIR", "dev/tmp")]:
|
||||
monkeypatch.setenv(name, value)
|
||||
monkeypatch.setenv("CWA_DB_PATH", "cwa/app.db")
|
||||
monkeypatch.setenv("CALIBRE_DB_PATH", "library/metadata.db")
|
||||
reload_config()
|
||||
assert config.CONFIG_DIR == Path.cwd() / "dev/config"
|
||||
assert config.INGEST_DIR == Path.cwd() / "dev/ingest"
|
||||
assert config.TMP_DIR == Path.cwd() / "dev/tmp"
|
||||
assert config.CWA_DB_PATH == Path.cwd() / "cwa/app.db"
|
||||
assert config.CALIBRE_DB_PATH == Path.cwd() / "library/metadata.db"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[
|
||||
("", ""),
|
||||
("/", ""),
|
||||
("bindery", "/bindery"),
|
||||
("/bindery/", "/bindery"),
|
||||
(" /a/b ", "/a/b"),
|
||||
],
|
||||
)
|
||||
def test_base_path(value, expected):
|
||||
assert config._base_path(value) == expected
|
||||
|
||||
|
||||
def test_values_from_the_environment(monkeypatch, reload_config):
|
||||
monkeypatch.setenv("URL_BASE", "bindery/")
|
||||
monkeypatch.setenv("AUTH_METHOD", " CWA ")
|
||||
monkeypatch.setenv("UPDATE_INTERVAL_HOURS", "1.5")
|
||||
monkeypatch.setenv("DEFAULT_LOCKED_FIELDS", "title, authors")
|
||||
monkeypatch.setenv("CWA_ALLOWED_USERS", "alice,bob")
|
||||
monkeypatch.setenv("CONFIG_DIR", "/data/bindery")
|
||||
monkeypatch.setenv("TRUST_PROXY", "yes")
|
||||
monkeypatch.setenv("CWA_DB_PATH", " /cwa/app.db ")
|
||||
monkeypatch.setenv("CALIBRE_DB_PATH", "/library/metadata.db")
|
||||
reload_config()
|
||||
assert config.URL_BASE == "/bindery"
|
||||
assert config.AUTH_METHOD == "cwa"
|
||||
assert config.UPDATE_INTERVAL_HOURS == 1.5
|
||||
assert config.DEFAULT_LOCKED_FIELDS == ["title", "authors"]
|
||||
assert config.CWA_ALLOWED_USERS == ["alice", "bob"]
|
||||
assert str(config.DB_PATH) == "/data/bindery/bindery.db"
|
||||
assert config.TRUST_PROXY is True
|
||||
assert str(config.CWA_DB_PATH) == "/cwa/app.db"
|
||||
assert str(config.CALIBRE_DB_PATH) == "/library/metadata.db"
|
||||
|
||||
|
||||
def test_defaults(monkeypatch, reload_config):
|
||||
for name in (
|
||||
"URL_BASE",
|
||||
"AUTH_METHOD",
|
||||
"UPDATE_INTERVAL_HOURS",
|
||||
"DEFAULT_LOCKED_FIELDS",
|
||||
"LOG_LEVEL",
|
||||
"PROXY_AUTH_USER_HEADER",
|
||||
"SESSION_COOKIE_SECURE",
|
||||
"TRUST_PROXY",
|
||||
"CWA_DB_PATH",
|
||||
"CALIBRE_DB_PATH",
|
||||
):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
reload_config()
|
||||
assert config.URL_BASE == ""
|
||||
assert config.AUTH_METHOD == "none"
|
||||
assert config.UPDATE_INTERVAL_HOURS == 24
|
||||
assert config.DEFAULT_LOCKED_FIELDS == ["title"]
|
||||
assert config.LOG_LEVEL == "INFO"
|
||||
assert config.PROXY_AUTH_USER_HEADER == "X-Remote-User"
|
||||
assert config.SESSION_COOKIE_SECURE is False
|
||||
assert config.TRUST_PROXY is False
|
||||
assert config.CWA_DB_PATH is None
|
||||
assert config.CALIBRE_DB_PATH is None
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Exercise the FanFicFare boundary without requests to a story site."""
|
||||
|
||||
import zipfile
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from test_build import IMG_URL, PNG, URL, chapters, metahtml
|
||||
|
||||
from bindery import config, fff
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cover", [None, {}, {"url": IMG_URL, "ext": "png", "mime": "image/png", "data": PNG}])
|
||||
def test_metadata_snapshot_from_adapter(monkeypatch, cover):
|
||||
adapter = fff.get_adapter(URL)
|
||||
adapter.setStoryMetadata(metahtml())
|
||||
adapter.chapterUrls = chapters()
|
||||
adapter.story.img_store.cover = cover
|
||||
fetch = Mock(return_value=adapter.story)
|
||||
monkeypatch.setattr(adapter, "getStoryMetadataOnly", fetch)
|
||||
snap = fff.fetch_story(adapter)
|
||||
fetch.assert_called_once_with(get_cover=True)
|
||||
assert snap.title == "Test Story"
|
||||
assert snap.author == "Writer"
|
||||
assert snap.site == "www.royalroad.com"
|
||||
assert snap.chapters == [{"url": c["url"], "title": c["title"]} for c in chapters()]
|
||||
assert snap.calibre["authors"] == ["Writer"]
|
||||
assert snap.cover == (cover or None)
|
||||
assert "Test Story" in snap.metahtml
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error,message",
|
||||
[
|
||||
(fff.exceptions.StoryDoesNotExist(URL), "StoryDoesNotExist"),
|
||||
(fff.exceptions.AccessDenied("private"), "AccessDenied"),
|
||||
(fff.exceptions.FailedToLogin(URL, "reader"), "FailedToLogin"),
|
||||
(fff.exceptions.AdultCheckRequired(URL), "is_adult:true"),
|
||||
],
|
||||
)
|
||||
def test_adapter_errors_become_actionable_story_errors(error, message):
|
||||
adapter = SimpleNamespace(getStoryMetadataOnly=Mock(side_effect=error))
|
||||
with pytest.raises(fff.StoryError, match=message):
|
||||
fff.fetch_story(adapter)
|
||||
|
||||
|
||||
def test_downloaded_chapter_restores_original_image_urls():
|
||||
html = '<p>Text</p><img src="images/1.png" longdesc="https://images.test/1.png">'
|
||||
adapter = SimpleNamespace(getChapterTextNum=Mock(return_value=html))
|
||||
result = fff.fetch_chapter(adapter, "chapter", 3)
|
||||
adapter.getChapterTextNum.assert_called_once_with("chapter", 3)
|
||||
assert 'src="https://images.test/1.png"' in result
|
||||
for src in ["data:image/png;base64,abc", "failedtoload", "https://images.test/direct.png"]:
|
||||
markup = f'<img src="{src}"' + (' longdesc="original"' if src != "https://images.test/direct.png" else "") + ">"
|
||||
assert f'src="{src}"' in fff.canonical_html(markup)
|
||||
assert fff.canonical_html("<p>plain</p>") == "<p>plain</p>"
|
||||
|
||||
|
||||
def test_collected_images_skip_failed_and_empty_downloads():
|
||||
image = {"url": "image", "ext": "png", "mime": "image/png", "data": PNG}
|
||||
adapter = SimpleNamespace(
|
||||
story=SimpleNamespace(
|
||||
img_store=SimpleNamespace(
|
||||
infos=[{**image, "newsrc": "images/1.png"}, {**image, "data": b""}, {**image, "newsrc": "failedtoload"}]
|
||||
)
|
||||
)
|
||||
)
|
||||
assert fff.collected_images(adapter) == [image]
|
||||
|
||||
|
||||
def test_personal_ini_overrides_and_invalid_input(monkeypatch, tmp_path):
|
||||
ini = tmp_path / "personal.ini"
|
||||
ini.write_text("[overrides]\nis_adult: true\n", encoding="utf-8")
|
||||
monkeypatch.setattr(config, "PERSONAL_INI", ini)
|
||||
configuration = fff.get_configuration(URL)
|
||||
assert configuration.getConfig("is_adult")
|
||||
for text in ["[defaults]\nbroken line\n", "setting: before any section"]:
|
||||
with pytest.raises(ValueError, match="cannot read"):
|
||||
fff.check_personal_ini(text)
|
||||
with pytest.raises(fff.StoryError, match="does not support"):
|
||||
fff.get_configuration("https://unsupported.example/story")
|
||||
|
||||
|
||||
def test_adapter_rejects_invalid_story_url(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
fff.adapters, "getAdapter", Mock(side_effect=fff.exceptions.InvalidStoryURL(URL, "site", "example"))
|
||||
)
|
||||
with pytest.raises(fff.StoryError):
|
||||
fff.get_adapter(URL)
|
||||
|
||||
|
||||
def test_normalization_errors_are_not_site_requests(monkeypatch):
|
||||
monkeypatch.setattr(fff.adapters, "getNormalStoryURLSite", Mock(side_effect=ValueError("bad URL")))
|
||||
fff.normalize_url.cache_clear()
|
||||
try:
|
||||
assert fff.normalize_url("bad-input") is None
|
||||
finally:
|
||||
fff.normalize_url.cache_clear()
|
||||
|
||||
|
||||
def test_empty_chapters_fail_and_stored_cover_is_built(tmp_path):
|
||||
with pytest.raises(fff.StoryError, match="no chapters"):
|
||||
fff.build_epub(URL, metahtml(), [], [], None, [], {}, tmp_path / "empty.epub")
|
||||
cover = {"url": IMG_URL, "ext": "png", "mime": "image/png", "data": PNG}
|
||||
out = tmp_path / "cover.epub"
|
||||
fff.build_epub(URL, metahtml(), chapters()[:1], [], cover, [], {}, out)
|
||||
with zipfile.ZipFile(out) as book:
|
||||
assert any(book.read(n) == PNG for n in book.namelist() if n.endswith(".png"))
|
||||
Reference in New Issue
Block a user