tim
c30622e3a0
FanFicFare adapters fetch metadata and chapters, and its epub writer builds each book from the store. Environment variables configure the app, and a default personal.ini is copied to the config folder on first start.
336 lines
13 KiB
Python
336 lines
13 KiB
Python
"""Headless FanFicFare wrapper: site access, chapter fetches and epub builds.
|
|
|
|
This module uses FanFicFare's adapters (the site scrapers) and its epub writer
|
|
without changes. The download and update flow of the FanFicFare CLI is not used.
|
|
Instead, chapters live in the store, and every epub is built from the store:
|
|
|
|
1. fetch_story() reads metadata and the chapter list from the site.
|
|
2. fetch_chapter() downloads one chapter and returns its HTML with image
|
|
sources set back to their original URLs.
|
|
3. build_epub() gives FanFicFare's writer the stored chapters as "old
|
|
chapters" (the same mechanism FanFicFare uses for epub updates), so the
|
|
writer never goes to the site.
|
|
"""
|
|
|
|
import configparser
|
|
import html
|
|
import io
|
|
import logging
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
from functools import lru_cache
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import fanficfare
|
|
from bs4 import BeautifulSoup
|
|
from fanficfare import adapters, exceptions, writers
|
|
from fanficfare.configurable import Configuration
|
|
from fanficfare.epubutils import get_update_data
|
|
from fanficfare.story import imagetypes
|
|
|
|
from bindery import config
|
|
from bindery.diffing import html_to_blocks
|
|
from bindery.rules import Step, apply_rules
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# FanFicFare logs every request at DEBUG level.
|
|
logging.getLogger("fanficfare").setLevel(logging.DEBUG if config.LOG_LEVEL == "DEBUG" else logging.WARNING)
|
|
|
|
DEFAULTS_INI = Path(fanficfare.__file__).parent / "defaults.ini"
|
|
|
|
# Settings this application depends on. They go in [overrides], so
|
|
# personal.ini cannot change them.
|
|
FORCED_SETTINGS = {
|
|
"always_reload_first_chapter": "false",
|
|
"mark_new_chapters": "false",
|
|
"continue_on_chapter_error": "false",
|
|
"make_directories": "false",
|
|
"zip_output": "false",
|
|
# The epub is the only way metadata reaches CWA, so the series must be in the OPF.
|
|
"calibre_series_meta": "true",
|
|
}
|
|
|
|
|
|
class StoryError(Exception):
|
|
"""A problem the user can fix, such as an unsupported URL."""
|
|
|
|
|
|
@dataclass
|
|
class SiteSnapshot:
|
|
"""What the site says about a story right now."""
|
|
|
|
url: str
|
|
site: str
|
|
title: str
|
|
author: str
|
|
metahtml: str
|
|
chapters: list[dict[str, str]]
|
|
calibre: dict[str, Any]
|
|
cover: dict[str, Any] | None = None
|
|
|
|
|
|
@dataclass
|
|
class EpubContents:
|
|
"""Chapters and images read from an existing FanFicFare epub."""
|
|
|
|
url: str | None
|
|
chapters: list[dict[str, str]] = field(default_factory=list)
|
|
images: list[dict[str, Any]] = field(default_factory=list)
|
|
|
|
|
|
def escape_title(text: str) -> str:
|
|
"""A chapter title in FanFicFare's form, which the store keeps.
|
|
|
|
FanFicFare keeps &, < and > in chapter titles, because its epub
|
|
writer puts titles into XHTML as they are.
|
|
"""
|
|
return html.escape(text, quote=False)
|
|
|
|
|
|
def plain_title(title: str) -> str:
|
|
"""A stored chapter title as text, for a message."""
|
|
return html.unescape(title)
|
|
|
|
|
|
@lru_cache(maxsize=20000)
|
|
def normalize_url(url: str) -> tuple[str, str] | None:
|
|
"""Return (story URL, site) for a supported URL, or None. No network access."""
|
|
try:
|
|
return adapters.getNormalStoryURLSite(url.strip())
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def _site_names() -> frozenset:
|
|
return frozenset(name.lower() for name in adapters.getSiteSections())
|
|
|
|
|
|
def is_supported_site(name: str) -> bool:
|
|
"""True for a site name that FanFicFare supports, such as www.royalroad.com."""
|
|
return name.strip().lower() in _site_names()
|
|
|
|
|
|
def get_configuration(url: str) -> Configuration:
|
|
try:
|
|
configuration = Configuration(adapters.getConfigSectionsFor(url), "epub")
|
|
except exceptions.UnknownSite as e:
|
|
raise StoryError(f"FanFicFare does not support site: {url}") from e
|
|
with open(DEFAULTS_INI, encoding="utf-8") as f:
|
|
configuration.read_file(f)
|
|
if config.PERSONAL_INI.exists():
|
|
with open(config.PERSONAL_INI, encoding="utf-8") as f:
|
|
configuration.read_file(f)
|
|
try:
|
|
configuration.add_section("overrides")
|
|
except configparser.DuplicateSectionError:
|
|
pass
|
|
for key, value in FORCED_SETTINGS.items():
|
|
configuration.set("overrides", key, value)
|
|
# FanFicFare downloads the cover only if include_images is on, and it is off by default. "coveronly" keeps the
|
|
# cover, but still leaves out the images in the story text.
|
|
if not configuration.getConfig("include_images"):
|
|
configuration.set("overrides", "include_images", "coveronly")
|
|
configuration.cached_config.clear()
|
|
return configuration
|
|
|
|
|
|
def check_personal_ini(text: str) -> list[str]:
|
|
"""Check the text of a personal.ini.
|
|
|
|
Raise ValueError for text that FanFicFare cannot read. Return the problems
|
|
that FanFicFare's own check finds, such as a section for an unknown site.
|
|
"""
|
|
configuration = Configuration(["defaults"], "epub")
|
|
try:
|
|
configuration.read_file(io.StringIO(text))
|
|
except configparser.ParsingError as e:
|
|
lines = "; ".join(f"line {n}: {line.strip()}" for n, line in e.errors)
|
|
raise ValueError(f"FanFicFare cannot read this personal.ini. Check {lines}") from e
|
|
except configparser.Error as e:
|
|
raise ValueError(f"FanFicFare cannot read this personal.ini: {e}") from e
|
|
return [f"Line {n}: {message}" for n, message in configuration.test_config()]
|
|
|
|
|
|
def get_adapter(url: str):
|
|
try:
|
|
return adapters.getAdapter(get_configuration(url), url)
|
|
except exceptions.InvalidStoryURL as e:
|
|
raise StoryError(str(e)) from e
|
|
|
|
|
|
def parse_series(value: str) -> tuple[str, float | None]:
|
|
m = re.match(r"^(.*?)\s*\[([\d.]+)\]$", value or "")
|
|
if m:
|
|
return m.group(1), float(m.group(2))
|
|
return value or "", None
|
|
|
|
|
|
def calibre_metadata(story) -> dict[str, Any]:
|
|
"""Map FanFicFare story metadata to calibre fields."""
|
|
series, series_index = parse_series(story.getMetadata("series", removeallentities=True))
|
|
published = story.getMetadataRaw("datePublished")
|
|
return {
|
|
"title": story.getMetadata("title", removeallentities=True),
|
|
"authors": story.getList("author", removeallentities=True),
|
|
"comments": story.getMetadata("description"),
|
|
"tags": story.getSubjectTags(removeallentities=True),
|
|
"series": series,
|
|
"series_index": series_index,
|
|
"pubdate": published.isoformat() if hasattr(published, "isoformat") else "",
|
|
"languages": story.getMetadata("langcode"),
|
|
}
|
|
|
|
|
|
def fetch_story(adapter) -> SiteSnapshot:
|
|
"""Read story metadata, the chapter list and the cover from the site."""
|
|
try:
|
|
story = adapter.getStoryMetadataOnly(get_cover=True)
|
|
except (exceptions.StoryDoesNotExist, exceptions.AccessDenied, exceptions.FailedToLogin) as e:
|
|
raise StoryError(f"{type(e).__name__}: {e}") from e
|
|
except exceptions.AdultCheckRequired as e:
|
|
raise StoryError("Site asks for adult check. Set is_adult:true in personal.ini.") from e
|
|
cover = None
|
|
info = story.img_store.cover
|
|
if info and info.get("data"):
|
|
cover = {k: info[k] for k in ("url", "ext", "mime", "data")}
|
|
return SiteSnapshot(
|
|
url=story.getMetadata("storyUrl"),
|
|
site=adapter.getSiteDomain(),
|
|
title=story.getMetadata("title", removeallentities=True),
|
|
author=", ".join(story.getList("author", removeallentities=True)),
|
|
metahtml=story.dump_html_metadata(),
|
|
chapters=[{"url": c["url"], "title": c["title"]} for c in adapter.get_chapters()],
|
|
calibre=calibre_metadata(story),
|
|
cover=cover,
|
|
)
|
|
|
|
|
|
def canonical_html(html: str) -> str:
|
|
"""Point <img> tags back at their original URLs, as FanFicFare's update code does."""
|
|
if "<img" not in html:
|
|
return html
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
for img in soup.find_all("img"):
|
|
if img.has_attr("longdesc") and not img["src"].startswith(("data:", "failedtoload")):
|
|
img["src"] = img["longdesc"]
|
|
return str(soup)
|
|
|
|
|
|
def fetch_chapter(adapter, url: str, index: int) -> str:
|
|
"""Download one chapter. Images go into the adapter's image store."""
|
|
return canonical_html(adapter.getChapterTextNum(url, index))
|
|
|
|
|
|
def collected_images(adapter) -> list[dict[str, Any]]:
|
|
"""Images downloaded so far by this adapter, without the cover."""
|
|
images = []
|
|
for info in adapter.story.img_store.infos:
|
|
if info.get("data") and info.get("newsrc") != "failedtoload":
|
|
images.append({k: info[k] for k in ("url", "ext", "mime", "data")})
|
|
return images
|
|
|
|
|
|
def word_count(chapters: list[dict[str, Any]]) -> int:
|
|
return sum(len(block.split()) for c in chapters for block in html_to_blocks(c["html"]))
|
|
|
|
|
|
def apply_overrides(story, overrides: dict[str, Any]) -> None:
|
|
"""Put the kept values of locked fields into FanFicFare's story metadata.
|
|
|
|
The epub writer takes the OPF metadata from these entries, and CWA takes
|
|
the book record from the OPF on the first import.
|
|
"""
|
|
for key, value in overrides.items():
|
|
if key == "title" and value:
|
|
story.setMetadata("title", value)
|
|
elif key == "authors" and value:
|
|
for n, author in enumerate(value):
|
|
story.addToList("author", author, clear=(n == 0))
|
|
elif key == "comments":
|
|
story.setMetadata("description", value or "")
|
|
elif key == "series":
|
|
name, index = value if value else ("", None)
|
|
if name and index is not None:
|
|
story.setMetadata("series", f"{name} [{index:g}]")
|
|
else:
|
|
story.setMetadata("series", name or "")
|
|
elif key == "pubdate":
|
|
try:
|
|
story.setMetadata("datePublished", datetime.fromisoformat(value) if value else None)
|
|
except ValueError:
|
|
logger.warning(f"Ignoring a kept publication date that is not a date: {value!r}")
|
|
elif key == "languages" and value:
|
|
story.setMetadata("langcode", value)
|
|
elif key == "tags":
|
|
tags = list(value or [])
|
|
# getSubjectTags() builds the tags from configuration. The locked
|
|
# list replaces it for this build only.
|
|
story.getSubjectTags = lambda removeallentities=False, tags=tags: tags
|
|
|
|
|
|
def build_epub(
|
|
url: str,
|
|
metahtml: str,
|
|
chapters: list[dict[str, Any]],
|
|
images: list[dict[str, Any]],
|
|
cover: dict[str, Any] | None,
|
|
rules: list[Step],
|
|
overrides: dict[str, Any],
|
|
out_path: Path,
|
|
) -> dict[str, Any]:
|
|
"""Write an epub from stored data. No network access.
|
|
|
|
chapters: dicts with url, title and html, in book order.
|
|
overrides: calibre field values that replace the site values in the epub
|
|
(locked fields).
|
|
Returns the calibre metadata of the built book.
|
|
"""
|
|
if not chapters:
|
|
raise StoryError("Story has no chapters to put in a book")
|
|
adapter = get_adapter(url)
|
|
adapter.setStoryMetadata(metahtml)
|
|
story = adapter.story
|
|
apply_overrides(story, overrides)
|
|
|
|
adapter.chapterUrls = [{"url": c["url"], "title": c["title"]} for c in chapters]
|
|
story.setMetadata("numChapters", len(chapters))
|
|
story.setMetadata("numWords", word_count(chapters))
|
|
adapter.oldchaptersmap = {
|
|
c["url"]: BeautifulSoup(apply_rules(c["html"] or "<div></div>", rules), "html5lib") for c in chapters
|
|
}
|
|
adapter.oldimgs = {img["url"]: (f"images/stored.{img['ext']}", img["data"]) for img in images}
|
|
if cover and adapter.getConfig("include_images"):
|
|
info = story.img_store.add_img(cover["url"], cover["ext"], cover["mime"], cover["data"], cover=True)
|
|
story.cover = info["newsrc"]
|
|
story.setMetadata("cover_image", "specific")
|
|
|
|
writer = writers.getWriter("epub", adapter.get_configuration(), adapter)
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
writer.writeStory(outfilename=str(out_path), forceOverwrite=True)
|
|
return calibre_metadata(story)
|
|
|
|
|
|
def read_epub(path: Path) -> EpubContents:
|
|
"""Read the chapters of an epub that FanFicFare wrote.
|
|
|
|
Chapter HTML comes back with image sources set to the original URLs.
|
|
"""
|
|
(source, _filecount, _soups, images, _oldcover, _bookmark, _logfile, urlsoups, datamaps) = get_update_data(
|
|
str(path)
|
|
)[0:9]
|
|
contents = EpubContents(url=source)
|
|
for n, (chapter_url, bodysoup) in enumerate(urlsoups.items(), start=1):
|
|
data = datamaps.get(chapter_url, {})
|
|
title = data.get("chapterorigtitle") or data.get("chaptertitle") or f"Chapter {n}"
|
|
text = re.sub(r"</?body[^>]*>", "", str(bodysoup)).strip()
|
|
contents.chapters.append({"url": chapter_url, "title": escape_title(title), "html": text})
|
|
for img_url, (src, img_data) in images.items():
|
|
ext = src.rsplit(".", 1)[-1].lower()
|
|
if ext in imagetypes:
|
|
contents.images.append({"url": img_url, "ext": ext, "mime": imagetypes[ext], "data": img_data})
|
|
return contents
|