From 49cd18a09f82d15e4a1fc4d24fd8e319fbfd46de Mon Sep 17 00:00:00 2001 From: Timothy Allen Date: Sat, 26 Sep 2026 08:06:02 +0200 Subject: [PATCH] Run download jobs and deliver books to CWA Jobs download new chapters, recheck chapters, replace the book, update metadata or rebuild from stored chapters. Locked metadata fields keep their values. A scheduler queues updates. An update finds the library book of a story through CWA, and fails with an error if it cannot, so that it never makes a duplicate book. --- bindery/backup.py | 40 ++ bindery/jobs.py | 239 ++++++++ bindery/orchestrator.py | 1163 ++++++++++++++++++++++++++++++++++++ tests/test_backup.py | 25 + tests/test_deliver.py | 610 +++++++++++++++++++ tests/test_jobs.py | 163 +++++ tests/test_metadata.py | 74 +++ tests/test_worker_flows.py | 267 +++++++++ 8 files changed, 2581 insertions(+) create mode 100644 bindery/backup.py create mode 100644 bindery/jobs.py create mode 100644 bindery/orchestrator.py create mode 100644 tests/test_backup.py create mode 100644 tests/test_deliver.py create mode 100644 tests/test_jobs.py create mode 100644 tests/test_metadata.py create mode 100644 tests/test_worker_flows.py diff --git a/bindery/backup.py b/bindery/backup.py new file mode 100644 index 0000000..5542f8f --- /dev/null +++ b/bindery/backup.py @@ -0,0 +1,40 @@ +"""Daily copies of the database and personal.ini. + +The database holds chapters that the sites removed, so it is the only copy of +them. A backup a day, kept for a week, limits a bad change or a broken disk to +the loss of one day. +""" + +import logging +import shutil +from datetime import date +from pathlib import Path + +from bindery.store import Store + +logger = logging.getLogger(__name__) + + +def daily_backup(store: Store, folder: Path, personal_ini: Path, keep: int, today: date | None = None) -> Path | None: + """Make the backup of today if there is none yet, then delete all but the newest `keep` backups. + + Return the path of the new database copy, or None if today had one already. + """ + today = today or date.today() + folder.mkdir(parents=True, exist_ok=True) + target = folder / f"bindery-{today.isoformat()}.db" + made = None + if not target.exists(): + # A copy under another name first, so that a stop never leaves half a backup. + partial = folder / f"{target.name}.part" + partial.unlink(missing_ok=True) + store.backup(partial) + partial.replace(target) + if personal_ini.exists(): + shutil.copy2(personal_ini, folder / f"personal-{today.isoformat()}.ini") + logger.info(f"Backed up the database to {target}") + made = target + for pattern in ("bindery-*.db", "personal-*.ini"): + for old in sorted(folder.glob(pattern))[:-keep]: + old.unlink() + return made diff --git a/bindery/jobs.py b/bindery/jobs.py new file mode 100644 index 0000000..d6dc66e --- /dev/null +++ b/bindery/jobs.py @@ -0,0 +1,239 @@ +"""Download modes, jobs and the job queue.""" + +import json +import sqlite3 +import uuid +from collections import deque +from dataclasses import asdict, dataclass, field +from datetime import datetime, timedelta +from enum import StrEnum +from pathlib import Path +from threading import Lock +from typing import Any + +from bindery import config + +# Calibre fields that a story can lock. A locked field keeps its current +# value until a download explicitly forces an update of that field. +METADATA_FIELDS: dict[str, str] = { + "title": "Title", + "authors": "Authors", + "comments": "Description", + "tags": "Tags", + "series": "Series", + "pubdate": "Published date", + "languages": "Language", +} + + +class QueueStatus(StrEnum): + """Enum for possible job statuses.""" + + QUEUED = "queued" + RUNNING = "running" + DONE = "done" + ERROR = "error" + CANCELLED = "cancelled" + + +class DownloadMode(StrEnum): + """What a job fetches from the site and how it changes the book.""" + + NEW = "new" # new chapters only, appended after the chapters we have + RECHECK = "recheck" # download selected chapters again and keep changes + RECHECK_ALL = "recheck_all" # download every chapter still on the site again + FULL = "full" # replace the whole book with what is on the site now + METADATA = "metadata" # update metadata only, no chapter changes + REBUILD = "rebuild" # no site access: rebuild the epub from stored chapters + IMPORT = "import" # start tracking a book that is already in the library + IMPORT_FILE = "import_file" # add the chapters of an uploaded epub to a story + + +MODE_LABELS: dict[str, str] = { + DownloadMode.NEW.value: "Download new chapters (default)", + DownloadMode.RECHECK.value: "Recheck selected chapters", + DownloadMode.RECHECK_ALL.value: "Recheck all chapters", + DownloadMode.FULL.value: "Download and replace entire book", + DownloadMode.METADATA.value: "Update metadata from site", + DownloadMode.REBUILD.value: "Rebuild book from stored chapters", + DownloadMode.IMPORT.value: "Import from library", + DownloadMode.IMPORT_FILE.value: "Import chapters from an epub file", +} + +# A job that was running when Bindery stopped this many times is not started again. +MAX_RESTARTS = 3 + + +@dataclass +class Job: + """One unit of work for the download thread.""" + + mode: DownloadMode + story_id: int | None = None + url: str | None = None + calibre_id: int | None = None + chapter_ids: list[int] = field(default_factory=list) + force_fields: list[str] = field(default_factory=list) + options: dict[str, Any] = field(default_factory=dict) + id: str = field(default_factory=lambda: uuid.uuid4().hex[:12]) + status: QueueStatus = QueueStatus.QUEUED + title: str = "" + message: str = "" + progress: float = 0.0 + created: datetime = field(default_factory=datetime.now) + updated: datetime = field(default_factory=datetime.now) + result: dict[str, Any] = field(default_factory=dict) + restarts: int = 0 + + def to_dict(self) -> dict[str, Any]: + d = asdict(self) + d["mode"] = self.mode.value + d["mode_label"] = MODE_LABELS[self.mode.value] + d["status"] = self.status.value + d["created"] = self.created.isoformat(timespec="seconds") + d["updated"] = self.updated.isoformat(timespec="seconds") + return d + + def to_record(self) -> str: + d = asdict(self) + d["created"] = self.created.isoformat() + d["updated"] = self.updated.isoformat() + return json.dumps(d) + + @classmethod + def from_record(cls, record: str) -> "Job": + d = json.loads(record) + d["mode"] = DownloadMode(d["mode"]) + d["status"] = QueueStatus(d["status"]) + d["created"] = datetime.fromisoformat(d["created"]) + d["updated"] = datetime.fromisoformat(d["updated"]) + return cls(**d) + + +class JobQueue: + """Thread-safe FIFO job queue that also keeps recent job history. + + With a database path, the queue keeps each job in SQLite, so the jobs outlive + a restart. A job that was running goes back to the front of the queue. + """ + + def __init__(self, path: Path | None = None) -> None: + self._lock = Lock() + self._pending: deque[str] = deque() + self._jobs: dict[str, Job] = {} + self._history = timedelta(seconds=config.JOB_HISTORY) + self._db: sqlite3.Connection | None = None + if path: + self._load(Path(path)) + + def _load(self, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + # One connection, used only while the lock is held. Each statement commits. + self._db = sqlite3.connect(path, timeout=30, check_same_thread=False, isolation_level=None) + self._db.execute("PRAGMA journal_mode=WAL") + self._db.execute("CREATE TABLE IF NOT EXISTS jobs (id TEXT PRIMARY KEY, record TEXT NOT NULL)") + jobs = [Job.from_record(r[0]) for r in self._db.execute("SELECT record FROM jobs ORDER BY rowid")] + interrupted = [j for j in jobs if j.status == QueueStatus.RUNNING] + for job in interrupted: + job.restarts += 1 + if job.restarts >= MAX_RESTARTS: + job.status = QueueStatus.ERROR + job.message = f"Stopped {job.restarts} times, giving up" + else: + job.status = QueueStatus.QUEUED + job.message = "Resumed" + job.updated = datetime.now() + self._save(job) + self._jobs = {j.id: j for j in jobs} + waiting = [ + j.id for j in interrupted + [j for j in jobs if j not in interrupted] if j.status == QueueStatus.QUEUED + ] + self._pending.extend(waiting) + self._expire() + + def _save(self, job: Job) -> None: + if self._db: + self._db.execute( + "INSERT INTO jobs (id, record) VALUES (?, ?) ON CONFLICT (id) DO UPDATE SET record = excluded.record", + (job.id, job.to_record()), + ) + + def _expire(self) -> None: + """Forget finished jobs after JOB_HISTORY seconds.""" + now = datetime.now() + expired = [ + k + for k, j in self._jobs.items() + if j.status not in (QueueStatus.QUEUED, QueueStatus.RUNNING) and now - j.updated > self._history + ] + for job_id in expired: + del self._jobs[job_id] + if self._db and expired: + self._db.executemany("DELETE FROM jobs WHERE id = ?", [(k,) for k in expired]) + + def add(self, job: Job) -> Job: + """Queue a job. If the same story already waits for the same mode, return that job.""" + with self._lock: + for job_id in self._pending: + other = self._jobs[job_id] + if ( + other.mode == job.mode + and other.chapter_ids == job.chapter_ids + and other.force_fields == job.force_fields + and other.options == job.options + and ( + (job.story_id and other.story_id == job.story_id) + or (job.url and other.url == job.url) + or (job.calibre_id and other.calibre_id == job.calibre_id) + ) + ): + return other + self._expire() + self._jobs[job.id] = job + self._pending.append(job.id) + self._save(job) + return job + + def get_next(self) -> Job | None: + with self._lock: + while self._pending: + job = self._jobs[self._pending.popleft()] + if job.status == QueueStatus.QUEUED: + return job + return None + + def update(self, job: Job, **changes: Any) -> None: + with self._lock: + for key, value in changes.items(): + setattr(job, key, value) + job.updated = datetime.now() + self._save(job) + + def cancel(self, job_id: str) -> bool: + with self._lock: + job = self._jobs.get(job_id) + if not job or job.status != QueueStatus.QUEUED: + return False + job.status = QueueStatus.CANCELLED + job.updated = datetime.now() + # A cancelled job must not be shared, and it can expire before the download thread reaches it. + self._pending.remove(job_id) + self._save(job) + return True + + def has_active(self, story_id: int) -> bool: + with self._lock: + return any( + j.story_id == story_id and j.status in (QueueStatus.QUEUED, QueueStatus.RUNNING) + for j in self._jobs.values() + ) + + def get_status(self, story_id: int | None = None) -> list[dict[str, Any]]: + """Return recent jobs, newest first.""" + with self._lock: + self._expire() + jobs = [j for j in self._jobs.values() if story_id is None or j.story_id == story_id] + return [j.to_dict() for j in sorted(jobs, key=lambda j: j.created, reverse=True)] + + +job_queue = JobQueue(config.DB_PATH) diff --git a/bindery/orchestrator.py b/bindery/orchestrator.py new file mode 100644 index 0000000..cce27d8 --- /dev/null +++ b/bindery/orchestrator.py @@ -0,0 +1,1163 @@ +"""Download modes, metadata locks, delivery to CWA, and the job threads.""" + +import hashlib +import logging +import re +import threading +import time +import uuid +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any + +from bindery import config, fff +from bindery.backup import daily_backup +from bindery.cwa import Book, BookFinder, CwaClient, CwaError +from bindery.diffing import diff_html, text_hash +from bindery.epub import EpubError, epub_metadata, epub_urls +from bindery.fff import StoryError +from bindery.ingest import IngestFolder +from bindery.jobs import METADATA_FIELDS, DownloadMode, Job, QueueStatus, job_queue +from bindery.merge import merge_chapter_lists +from bindery.rules import RuleError, Step, apply_rules, compile_rules, normalize_rules +from bindery.store import Store, now + +logger = logging.getLogger(__name__) + +store = Store(config.DB_PATH) +ingest_folder = IngestFolder(config.INGEST_DIR) +finder = BookFinder() + +SITE_MODES = ( + DownloadMode.NEW, + DownloadMode.RECHECK, + DownloadMode.RECHECK_ALL, + DownloadMode.FULL, + DownloadMode.METADATA, +) + +_client_lock = threading.Lock() +_client: tuple[tuple[str, str, str], CwaClient | None] = (("", "", ""), None) + + +def cwa() -> CwaClient: + """The CWA client for the configured account.""" + global _client + key = (config.CWA_URL, config.CWA_USERNAME, config.CWA_PASSWORD) + with _client_lock: + if _client[0] != key or _client[1] is None: + _client = (key, CwaClient(*key)) + return _client[1] + + +def library_available() -> bool: + return bool(config.CWA_URL) + + +def default_locked_fields() -> list[str]: + return [f for f in config.DEFAULT_LOCKED_FIELDS if f in METADATA_FIELDS] + + +# Queueing + + +def queue_urls(urls: list[str]) -> tuple[list[dict[str, Any]], list[str]]: + """Queue a "new chapters" download for each URL. Returns (jobs, errors).""" + jobs, errors = [], [] + for url in (u.strip() for u in urls): + if not url: + continue + normal = fff.normalize_url(url) + if not normal: + errors.append(f"Not a supported story URL: {url}") + continue + story = store.get_story_by_url(normal[0]) + job = Job( + mode=DownloadMode.NEW, + story_id=story["id"] if story else None, + url=normal[0], + title=story["title"] if story else normal[0], + ) + jobs.append(job_queue.add(job).to_dict()) + return jobs, errors + + +def queue_story( + story_id: int, mode: str, chapter_ids: list[int] | None = None, force_fields: list[str] | None = None +) -> dict[str, Any]: + story = _require_story(story_id) + mode_enum = DownloadMode(mode) + if mode_enum == DownloadMode.IMPORT: + raise ValueError("Use the library import for this mode") + if mode_enum == DownloadMode.RECHECK and not chapter_ids: + raise ValueError("Select at least one chapter to recheck") + bad = [f for f in force_fields or [] if f not in METADATA_FIELDS] + if bad: + raise ValueError(f"Unknown metadata fields: {bad}") + job = Job( + mode=mode_enum, + story_id=story_id, + title=story["title"], + chapter_ids=sorted(int(c) for c in chapter_ids or []), + force_fields=sorted(force_fields or []), + ) + return job_queue.add(job).to_dict() + + +def queue_import(books: list[dict[str, Any]], check: bool = True) -> list[dict[str, Any]]: + """Queue imports of library books, given as {"id": ..., "uuid": ..., "title": ...}.""" + jobs = [] + for b in books: + book_id = int(b["id"]) + jobs.append( + job_queue.add( + Job( + mode=DownloadMode.IMPORT, + calibre_id=book_id, + title=b.get("title") or f"Library book {book_id}", + options={"check": check, "uuid": str(b.get("uuid") or "")}, + ) + ).to_dict() + ) + return jobs + + +def queue_epub_file(path: Path, filename: str, story_id: int | None = None, check: bool = True) -> dict[str, Any]: + """Queue an import of an uploaded epub. A file without a story URL, or of another story, fails at once.""" + try: + normal = next((n for n in map(fff.normalize_url, epub_urls(path)) if n), None) + if not normal: + raise StoryError(f"{filename} has no supported story URL. Only epubs from FanFicFare can be imported.") + story = store.get_story_by_url(normal[0]) + if story_id is not None: + story = _require_story(story_id) + if story["url"] != normal[0]: + raise StoryError(f"{filename} is an epub of another story: {normal[0]}") + except EpubError as e: + path.unlink(missing_ok=True) + raise EpubError(str(e).replace(path.name, filename)) from e + except Exception: + path.unlink(missing_ok=True) + raise + job = Job( + mode=DownloadMode.IMPORT_FILE, + story_id=story["id"] if story else None, + url=normal[0], + title=story["title"] if story else filename, + options={"path": str(path), "name": filename, "check": check}, + ) + return job_queue.add(job).to_dict() + + +def queue_status(story_id: int | None = None) -> list[dict[str, Any]]: + return job_queue.get_status(story_id) + + +def cancel_job(job_id: str) -> bool: + return job_queue.cancel(job_id) + + +# Story, chapter and rule management for the UI + + +def _require_story(story_id: int) -> dict[str, Any]: + story = store.get_story(story_id) + if not story: + raise KeyError(f"No story with id {story_id}") + return story + + +def list_stories() -> list[dict[str, Any]]: + stories = store.list_stories() + held = store.story_settings("upload_held") + for s in stories: + s["busy"] = job_queue.has_active(s["id"]) + s["upload_held"] = bool(held.get(s["id"])) + return stories + + +# Only raster images. An SVG cover could hold a script that runs on Bindery's origin. +COVER_TYPES = frozenset({"image/jpeg", "image/png", "image/gif", "image/webp"}) + + +def story_cover(story_id: int) -> dict[str, Any] | None: + """The site cover of a story, if a browser can show it safely.""" + _require_story(story_id) + cover = store.cover(story_id) + return cover if cover and cover["mime"] in COVER_TYPES else None + + +def story_details(story_id: int) -> dict[str, Any]: + story = _require_story(story_id) + story.pop("metahtml", None) + cover = story_cover(story_id) + # The version changes with the image, so a browser can keep a cover until the site changes it. + story["cover"] = hashlib.sha256(cover["data"]).hexdigest()[:16] if cover else None + book, library_error = None, None + if story["calibre_uuid"] and (library_available() or config.CALIBRE_DB_PATH): + try: + book = cwa().get_book(story["calibre_uuid"]) + except CwaError as e: + library_error = str(e) + story["recheck_days"] = store.get_setting(story_id, "recheck_days") + story["last_full_recheck"] = store.get_setting(story_id, "last_full_recheck") + story["upload_held"] = bool(store.get_setting(story_id, "upload_held")) + story["needs_approval"] = _needs_approval(story_id) + return { + "recent_chapters": _setting_recent_chapters(), + "uploads_paused": _uploads_paused(), + "story": story, + "chapters": store.list_chapters(story_id), + "rules": store.rules(story_id), + "book": book, + "library_available": library_available(), + "library_error": library_error, + "jobs": job_queue.get_status(story_id), + } + + +def update_story_settings( + story_id: int, + locked_fields: list[str] | None = None, + auto_update: bool | None = None, + recheck_days: int | None = None, +) -> None: + """Change the settings that are given. A recheck_days of 0 turns the full rechecks off.""" + story = _require_story(story_id) + if recheck_days is not None: + days = int(recheck_days) + if days < 0: + raise ValueError("The number of days cannot be negative") + store.set_setting(story_id, "recheck_days", days or None) + changes: dict[str, Any] = {} + if locked_fields is not None: + bad = [f for f in locked_fields if f not in METADATA_FIELDS] + if bad: + raise ValueError(f"Unknown metadata fields: {bad}") + changes["locked_fields"] = sorted(set(locked_fields)) + # An unlocked field takes the site value again. + unlocked = set(story["locked_fields"]) - set(locked_fields) + changes["pinned"] = {f: v for f, v in story["pinned"].items() if f not in unlocked} + if auto_update is not None: + changes["auto_update"] = int(bool(auto_update)) + store.update_story(story_id, **changes) + + +def _edited_value(field: str, value: Any) -> Any: + """Check an edited metadata value and put it in the stored form.""" + if field in ("authors", "tags"): + items = value if isinstance(value, list) else re.split("&" if field == "authors" else ",", str(value)) + value = [str(i).strip() for i in items if str(i).strip()] + if field == "authors" and not value: + raise ValueError("A story needs at least one author") + return value + if field == "series": + if isinstance(value, list): + name, index = (value + [None, None])[:2] + name, index = str(name or "").strip(), float(index) if index not in (None, "") else None + else: + name, index = fff.parse_series(str(value).strip()) + return [name, index if name else None] + value = str(value).strip() + if field == "title" and not value: + raise ValueError("A story needs a title") + if field == "pubdate" and value: + try: + datetime.fromisoformat(value) + except ValueError as e: + raise ValueError(f"The published date must be a date, for example 2024-01-31: {value}") from e + return value + + +def edit_metadata(story_id: int, values: dict[str, Any]) -> dict[str, Any]: + """Keep edited metadata values. A value of None removes the edit, so the site value comes back. + + The book is rebuilt when Bindery sent it to CWA before. Otherwise the next + build uses the values. + """ + story = _require_story(story_id) + bad = [f for f in values if f not in METADATA_FIELDS] + if bad: + raise ValueError(f"Unknown metadata fields: {bad}") + pinned = dict(story["pinned"]) + for field, value in values.items(): + if value is None: + pinned.pop(field, None) + else: + pinned[field] = _edited_value(field, value) + title = _display_title(story, pinned, story["site_values"].get("title") or story["title"]) + store.update_story(story_id, pinned=pinned, title=title) + queued = None + if story["metahtml"] and (story["calibre_id"] or story["ingested_at"]): + queued = job_queue.add(Job(mode=DownloadMode.REBUILD, story_id=story_id, title=title)).to_dict() + return {"story": story_details(story_id)["story"], "job": queued} + + +def get_settings() -> dict[str, Any]: + ini = config.PERSONAL_INI + return { + "recent_chapters": _setting_recent_chapters(), + "hold_new_links": bool(store.get_setting(None, "hold_new_links")), + "pause_uploads": _uploads_paused(), + "personal_ini": ini.read_text(encoding="utf-8") if ini.exists() else "", + } + + +def save_settings( + recent_chapters: Any = None, + personal_ini: str | None = None, + hold_new_links: bool | None = None, + pause_uploads: bool | None = None, +) -> dict[str, Any]: + """Change the settings that are given. Return all settings, and FanFicFare's warnings about personal.ini.""" + warnings: list[str] = [] + released = False + for name, value in (("hold_new_links", hold_new_links), ("pause_uploads", pause_uploads)): + if value is None: + continue + released = released or (bool(store.get_setting(None, name)) and not value) + store.set_setting(None, name, True if value else None) + if released: + _release_held_uploads() + if recent_chapters is not None: + try: + count = int(recent_chapters) + except (TypeError, ValueError) as e: + raise ValueError("The number of recent chapters must be a whole number") from e + if count < 0: + raise ValueError("The number of recent chapters cannot be negative") + store.set_setting(None, "recent_chapters", count) + if personal_ini is not None: + warnings = fff.check_personal_ini(personal_ini) + # Write a new file, then replace the old one, so a job never reads half a file. + partial = config.PERSONAL_INI.with_name("personal.ini.part") + partial.write_text(personal_ini, encoding="utf-8") + partial.replace(config.PERSONAL_INI) + return {**get_settings(), "warnings": warnings} + + +def get_rules(story_id: int | None) -> dict[str, Any]: + """The rules of a story, or the global rules for None, with the paragraph setting. + + For a story, `paragraphs` is None when the story follows the global setting. + """ + if story_id is not None: + _require_story(story_id) + return { + "rules": store.rules(story_id), + "paragraphs": store.get_setting(story_id, "paragraphs"), + "global_paragraphs": bool(store.get_setting(None, "paragraphs")), + } + + +def save_rules(story_id: int | None, rules: list[dict[str, Any]], paragraphs: Any = ...) -> dict[str, Any]: + """Store the rules. `paragraphs` is True, False, None (a story follows the global setting), or left out.""" + if story_id is not None: + _require_story(story_id) + clean = normalize_rules(rules) + store.set_rules(story_id, clean) + if paragraphs is not ...: + if paragraphs is None and story_id is None: + paragraphs = False + store.set_setting(story_id, "paragraphs", None if paragraphs is None else bool(paragraphs)) + return get_rules(story_id) + + +def preview_rules( + story_id: int | None, + rules: list[dict[str, Any]], + chapter_id: int, + source: bool = False, + paragraphs: bool | None = None, +) -> dict[str, Any]: + """Diff one chapter before and after the given rules (plus the global rules for a story). + + `paragraphs` is the setting in the editor. None follows the global setting. + """ + chapter = store.get_chapter(chapter_id) + if not chapter or (story_id is not None and chapter["story_id"] != story_id): + raise KeyError(f"No chapter with id {chapter_id}") + clean = normalize_rules(rules) + lists = [clean] if story_id is None else [store.rules(None), clean] + if paragraphs is None: + paragraphs = bool(store.get_setting(None, "paragraphs")) + html = store.revision_html(chapter_id, chapter["current_rev"]) or "" + result = diff_html(html, apply_rules(html, compile_rules(*lists, paragraphs=bool(paragraphs))), source=source) + result["chapter"] = chapter + return result + + +def chapter_diff(chapter_id: int, a: int | None = None, b: int | None = None, source: bool = False) -> dict[str, Any]: + chapter = store.get_chapter(chapter_id) + if not chapter: + raise KeyError(f"No chapter with id {chapter_id}") + revisions = store.list_revisions(chapter_id) + b = b or chapter["current_rev"] + if not a: + # The revision before b. Deleted revisions leave gaps in the numbers. + others = [r["rev"] for r in revisions if r["rev"] != b] + a = max((r for r in others if r < b), default=min(others, default=b)) + a, b = min(a, b), max(a, b) + old, new = store.revision_html(chapter_id, a), store.revision_html(chapter_id, b) + if old is None or new is None: + raise KeyError("No such revision") + result = diff_html(old, new, source=source) + story = store.get_story(chapter["story_id"]) + result.update(chapter=chapter, revisions=revisions, a=a, b=b, story_title=story["title"] if story else "") + return result + + +def set_chapter_revision(chapter_id: int, rev: int) -> dict[str, Any]: + chapter = store.get_chapter(chapter_id) + if not chapter or store.revision_html(chapter_id, rev) is None: + raise KeyError("No such chapter revision") + store.update_chapter(chapter_id, current_rev=rev, changed_at=now()) + return queue_story(chapter["story_id"], DownloadMode.REBUILD.value) + + +def delete_revision(chapter_id: int, rev: int) -> list[dict[str, Any]]: + """Delete a revision that is not in the book and not the newest, and return the revisions that are left.""" + chapter = store.get_chapter(chapter_id) + if not chapter or store.revision_html(chapter_id, rev) is None: + raise KeyError("No such chapter revision") + if rev == chapter["current_rev"]: + raise ValueError("This revision is in the book. Put another revision in the book first.") + if rev == store.list_revisions(chapter_id)[-1]["rev"]: + raise ValueError( + "This revision is the newest text from the site. " + "The next recheck compares with it, so you cannot delete it." + ) + store.delete_revision(chapter_id, rev) + return store.list_revisions(chapter_id) + + +def set_chapters_included(story_id: int, chapter_ids: list[int], included: bool) -> dict[str, Any]: + for cid in chapter_ids: + chapter = store.get_chapter(cid) + if chapter and chapter["story_id"] == story_id: + store.update_chapter(cid, included=int(included)) + return queue_story(story_id, DownloadMode.REBUILD.value) + + +def link_story(story_id: int, book_uuid: str | None) -> None: + """Link a story to a library book by its uuid, or unlink it with None.""" + story = _require_story(story_id) + if not book_uuid: + store.update_story(story_id, calibre_id=None, calibre_uuid=None, ingested_at=None) + _link_changed(story_id) + return + book = cwa().get_book(book_uuid) + if not book: + raise KeyError(f"No library book with uuid {book_uuid}") + other = store.get_story_by_calibre_id(book["id"]) + if other and other["id"] != story_id: + raise ValueError(f"Library book {book['id']} is already linked to {other['title']}") + store.update_story( + story_id, calibre_id=book["id"], calibre_uuid=book_uuid, ingested_at=story["ingested_at"] or now() + ) + if book_uuid != story["calibre_uuid"]: + _link_changed(story_id) + + +def approve_upload(story_id: int, book_uuid: str | None) -> dict[str, Any] | None: + """Approve the library link of a story: the uuid of its book, or None for a new book. + + Returns the queued rebuild when a built book waits for the approval. + """ + story = _require_story(story_id) + if (book_uuid or None) != story["calibre_uuid"]: + raise ValueError("The library link of this story changed. Reload the page, and check the link again.") + store.set_setting(story_id, "unapproved_link", None) + if store.get_setting(story_id, "upload_held") and story["metahtml"]: + return job_queue.add(Job(mode=DownloadMode.REBUILD, story_id=story_id, title=story["title"])).to_dict() + return None + + +def delete_story(story_id: int) -> None: + store.delete_story(story_id) + + +def search_library(query: str) -> list[dict[str, Any]]: + linked = {s["calibre_id"]: s["title"] for s in store.list_stories() if s["calibre_id"]} + results = [] + for book in cwa().search(query)[:50]: + item = book.to_dict() + item["tracked_as"] = linked.get(book.id) + results.append(item) + return results + + +def library_candidates() -> list[dict[str, Any]]: + """Library books from a story site that are not tracked yet.""" + tracked_ids = {s["calibre_id"] for s in store.list_stories() if s["calibre_id"]} + return [ + {"calibre_id": b.id, "uuid": b.uuid, "title": b.title, "authors": " & ".join(b.authors), "site": b.publisher} + for b in cwa().site_books(fff.is_supported_site) + if b.id not in tracked_ids + ] + + +def build_for_download(story_id: int) -> tuple[Path, str]: + """Build the epub from stored data, for a browser download.""" + story = _require_story(story_id) + path = config.TMP_DIR / f"download-{story_id}.epub" + _build(story, _epub_overrides(story, story["pinned"]), path) + return path, story["title"] + + +# Metadata locks + + +def _field_value(field: str, meta: dict[str, Any]) -> Any: + if field == "series": + return [meta.get("series") or "", meta.get("series_index")] + return meta.get(field) + + +def _resolve_metadata(story: dict[str, Any], site: dict[str, Any], force_fields: list[str]) -> dict[str, Any]: + """Return the values that replace the site values in this build. + + A locked field keeps its stored value. The first download, a library + import, an edit on the story page, or a forced update sets that value. + An unlocked field keeps an edited value until the site changes that field, + or until a forced update. The newer value wins. + """ + locked = set(story["locked_fields"]) + before = story["site_values"] + pinned = {} + for f, value in story["pinned"].items(): + site_changed = bool(before) and _field_value(f, site) != _field_value(f, before) + if f in locked or (f not in force_fields and not site_changed): + pinned[f] = value + for f in locked: + if f in force_fields or f not in pinned: + pinned[f] = _field_value(f, site) + return pinned + + +def _epub_overrides(story: dict[str, Any], pinned: dict[str, Any]) -> dict[str, Any]: + return {f: v for f, v in pinned.items() if f in METADATA_FIELDS} + + +def _display_title(story: dict[str, Any], pinned: dict[str, Any], site_title: str) -> str: + return pinned.get("title") or site_title + + +# Building and delivery + + +def _paragraphs(story_id: int | None) -> bool: + """Whether line breaks become paragraphs before the rules run. A story setting wins over the global one.""" + value = store.get_setting(story_id, "paragraphs") if story_id is not None else None + return bool(store.get_setting(None, "paragraphs") if value is None else value) + + +def _steps(story_id: int) -> list[Step]: + return compile_rules(store.rules(None), store.rules(story_id), paragraphs=_paragraphs(story_id)) + + +def _build(story: dict[str, Any], overrides: dict[str, Any], out_path: Path) -> dict[str, Any]: + if not story.get("metahtml"): + raise StoryError("Story has no site metadata yet. Download new chapters first.") + return fff.build_epub( + story["url"], + story["metahtml"], + store.chapters_for_build(story["id"]), + store.images(story["id"]), + store.cover(story["id"]), + _steps(story["id"]), + overrides, + out_path, + ) + + +def _find_book(story_url: str, titles: list[str | None]) -> tuple[Book, Path] | None: + """Search CWA for the book of a story. Returns the book and its downloaded epub.""" + linked = {s["calibre_id"] for s in store.list_stories() if s["calibre_id"]} + return finder.find(cwa(), story_url, [t for t in titles if t], fff.normalize_url, config.TMP_DIR, linked) + + +def _link(story: dict[str, Any], book_id: int, book_uuid: str) -> None: + # A book that CWA imported from the first epub of the story is the book that Bindery sent. Other links need + # an approval when the setting asks for one. + if book_uuid != story["calibre_uuid"] and (story["calibre_uuid"] or not story["ingested_at"]): + _link_changed(story["id"]) + story.update(calibre_id=book_id, calibre_uuid=book_uuid, ingested_at=story["ingested_at"] or now()) + store.update_story(story["id"], calibre_id=book_id, calibre_uuid=book_uuid, ingested_at=story["ingested_at"]) + + +def _linked_book(story: dict[str, Any]) -> dict[str, Any] | None: + """The library book for a story. Links the story if CWA has a book with its URL.""" + if not library_available() and not config.CALIBRE_DB_PATH: + return None + if story["calibre_uuid"]: + try: + book = cwa().get_book(story["calibre_uuid"]) + except CwaError as e: + # CWA checks the book id itself, so an update can still go out. + logger.warning(f"Cannot check library book {story['calibre_id']}: {e}") + return {"id": story["calibre_id"], "uuid": story["calibre_uuid"], "title": story["title"]} + if book: + if book["id"] != story["calibre_id"]: + _link(story, book["id"], book["uuid"]) + return book + logger.warning(f"Library book {story['calibre_id']} for {story['url']} is gone") + store.update_story(story["id"], calibre_id=None, calibre_uuid=None) + _link_changed(story["id"]) + story.update(calibre_id=None, calibre_uuid=None) + if config.CALIBRE_DB_PATH: + book = cwa().book_by_url(story["url"]) + if book: + other = store.get_story_by_calibre_id(book["id"]) + if not other or other["id"] == story["id"]: + _link(story, book["id"], book["uuid"]) + return book + if not library_available(): + return None + # CWA keeps the title of its first import, so an edited title can differ from it. + titles = [story["title"], story["pinned"].get("title"), story["site_values"].get("title")] + found = _find_book(story["url"], list(dict.fromkeys(titles))) + if not found: + return None + book, path = found + path.unlink(missing_ok=True) + _link(story, book.id, book.uuid) + logger.info(f"Linked {story['title']} to library book {book.id}") + return book.to_dict() + + +NOT_FOUND = "Cannot find this story in CWA. Link to library book, or send as new book." + + +# Upload holds. A story gets the per-story setting "unapproved_link" when its library link changes, and loses it +# with the next upload or an approval. "upload_held" marks a built book that did not go to CWA. + + +def _uploads_paused() -> bool: + return bool(store.get_setting(None, "pause_uploads")) + + +def _link_changed(story_id: int) -> None: + store.set_setting(story_id, "unapproved_link", True) + + +def _needs_approval(story_id: int) -> bool: + return bool(store.get_setting(None, "hold_new_links") and store.get_setting(story_id, "unapproved_link")) + + +def _release_held_uploads() -> None: + """Rebuild the held books, so that each one goes to CWA or waits for its own reason.""" + for story_id in store.story_settings("upload_held"): + story = store.get_story(story_id) + if story and story["metahtml"]: + job_queue.add(Job(mode=DownloadMode.REBUILD, story_id=story_id, title=story["title"])) + + +def _hold_reason(story: dict[str, Any], book: dict[str, Any] | None) -> str | None: + if _uploads_paused(): + return "Not sent. Turn off pause to send." + if not _needs_approval(story["id"]): + return None + if book: + return f"Not sent. Approve link to library book {book['id']}." + return "Not sent. Approve as new book." + + +def _deliver(story: dict[str, Any], epub: Path, book: dict[str, Any] | None, title: str, author: str) -> str: + # A second plain epub would make a duplicate book, so an update without a library book fails. + if not book and story["ingested_at"] and not library_available() and not config.CALIBRE_DB_PATH: + raise StoryError("Bindery cannot find the library book to update, because CWA_URL is not set.") + if not book and story["ingested_at"]: + raise StoryError(NOT_FOUND) + held = _hold_reason(story, book) + if held: + store.set_setting(story["id"], "upload_held", True) + return held + if book: + ingest_folder.send_update(epub, book["id"], title) + message = f"Updated library book {book['id']}" + else: + ingest_folder.ingest(epub, f"{title} - {author}" if author else title) + store.update_story(story["id"], ingested_at=now()) + story["ingested_at"] = now() + message = "Sent to CWA" + store.set_setting(story["id"], "upload_held", None) + store.set_setting(story["id"], "unapproved_link", None) + return message + + +# Jobs + + +def _progress(job: Job, message: str, progress: float | None = None) -> None: + changes: dict[str, Any] = {"message": message} + if progress is not None: + changes["progress"] = progress + job_queue.update(job, **changes) + + +def _read_epub(epub: Path, name: str) -> fff.EpubContents: + try: + contents = fff.read_epub(epub) + except Exception as e: + raise StoryError(f"Cannot read chapters of {name}: {e}") from e + if not contents.chapters: + raise StoryError(f"{name} has no FanFicFare chapter data") + return contents + + +def _import_epub(story_id: int, contents: fff.EpubContents) -> int: + """Add the chapters and images of a FanFicFare epub that the story does not have yet. + + Each chapter is stored as if it had been downloaded. Chapters that the story + has already keep their stored text. The chapter order of the epub places the + new chapters, in the same way as a site update, so chapters that the site + removed go back to their place. The chapters go in with one transaction, so + an import that stops leaves no part of the epub behind. + """ + # FanFicFare maps old forms of a chapter URL to the current one. + normalize = fff.get_adapter(_require_story(story_id)["url"]).normalize_chapterurl + stored = {normalize(c["url"]): c["url"] for c in store.list_chapters(story_id)} + epub_order = [] + new = [] + for c in contents.chapters: + url = normalize(c["url"]) + if url not in stored and url not in epub_order: + new.append({"url": url, "title": c["title"], "html": c["html"], "text_hash": text_hash(c["html"])}) + epub_order.append(url) + for img in contents.images: + store.put_image(story_id, img["url"], img["ext"], img["mime"], img["data"]) + if new: + order = merge_chapter_lists(list(stored), epub_order, newest_last=False) + store.add_chapters(story_id, new, "import", [stored.get(u, u) for u in order]) + return len(new) + + +def _run_import(job: Job) -> str: + tracked = store.get_story_by_calibre_id(job.calibre_id) + # A tracked story without chapters is left from an import that stopped, so the import goes on. + if tracked and store.list_chapters(tracked["id"]): + return "Book already tracked" + book_uuid = job.options.get("uuid") or "" + source = Book( + id=job.calibre_id, uuid=book_uuid, title=job.title, formats={"epub": f"opds/download/{job.calibre_id}/epub/"} + ) + _progress(job, "Downloading epub from CWA") + epub = cwa().download_epub(source, config.TMP_DIR / f"import-{job.calibre_id}-{uuid.uuid4().hex[:8]}.epub") + try: + normal = next((n for n in map(fff.normalize_url, epub_urls(epub)) if n), None) + if not normal: + raise StoryError(f"{job.title} has no supported story URL") + meta = epub_metadata(epub) + title = meta["title"] or job.title + job_queue.update(job, title=title) + story = store.get_story_by_url(normal[0]) + if story and story["calibre_id"] != job.calibre_id: + _link(story, job.calibre_id, book_uuid) + return f"Linked to tracked story {story['title']}" + contents = _read_epub(epub, title) + if story is None: + locked = default_locked_fields() + story_id = store.create_story( + normal[0], normal[1], title, " & ".join(meta["authors"]), locked, job.calibre_id, book_uuid or None + ) + pinned = {f: _field_value(f, meta) for f in locked} + store.update_story(story_id, pinned=pinned, ingested_at=now(), status="ok") + story = _require_story(story_id) + story_id = story["id"] + job_queue.update(job, story_id=story_id) + added = _import_epub(story_id, contents) + finally: + epub.unlink(missing_ok=True) + if job.options.get("check", True): + job_queue.add(Job(mode=DownloadMode.NEW, story_id=story_id, title=title)) + return f"Imported {added} chapters" + + +def _run_import_file(job: Job) -> str: + path, name = Path(job.options["path"]), job.options["name"] + if not path.exists(): + raise StoryError(f"Uploaded file {name} is gone. Upload it again.") + try: + contents = _read_epub(path, name) + story = store.get_story(job.story_id) if job.story_id else store.get_story_by_url(job.url) + if story is None: + meta = epub_metadata(path) + locked = default_locked_fields() + story_id = store.create_story( + job.url, fff.normalize_url(job.url)[1], meta["title"] or job.url, " & ".join(meta["authors"]), locked + ) + store.update_story(story_id, pinned={f: _field_value(f, meta) for f in locked}, status="ok") + _link_changed(story_id) + story = _require_story(story_id) + job_queue.update(job, story_id=story["id"], title=story["title"]) + added = _import_epub(story["id"], contents) + finally: + path.unlink(missing_ok=True) + message = f"Imported {added} chapter{'s' if added != 1 else ''} from {name}" + if not added: + message += " Already had every chapter." + store.update_story(story["id"], message=message) + # The book gets the imported chapters on the next build. A story without site + # metadata cannot be built before its first download. + if job.options.get("check", True): + job_queue.add( + Job(mode=DownloadMode.NEW, story_id=story["id"], title=story["title"], options={"build": bool(added)}) + ) + elif added and story["metahtml"]: + job_queue.add(Job(mode=DownloadMode.REBUILD, story_id=story["id"], title=story["title"])) + return message + + +def _run_rebuild(job: Job) -> str: + story = _require_story(job.story_id) + job_queue.update(job, title=story["title"]) + book = _linked_book(story) + pinned = _resolve_metadata(story, story["site_values"], []) + epub = config.TMP_DIR / f"story-{story['id']}.epub" + _build(story, _epub_overrides(story, pinned), epub) + message = _deliver(story, epub, book, story["title"], story["author"]) + store.update_story(story["id"], pinned=pinned, status="ok", message=message, last_changed=now()) + return message + + +def _rules_explain_change(story_id: int, newest: dict[str, Any], html: str) -> bool: + """Whether an imported chapter differs from the site text only by the rules. + + An imported epub has the rules of its build in its text, and the site text + has none. If the rules turn the site text into the imported text, the + chapter did not change. + """ + if newest["source"] != "import": + return False + return text_hash(apply_rules(html, _steps(story_id))) == newest["text_hash"] + + +def _setting_recent_chapters() -> int: + return int(store.get_setting(None, "recent_chapters") or 0) + + +def _recent_chapters(story_id: int) -> int: + """How many of the last chapters a new-chapter download checks again. + + A story with its own interval for full rechecks checks none. + """ + if store.get_setting(story_id, "recheck_days"): + return 0 + return _setting_recent_chapters() + + +def _run_site_job(job: Job) -> str: + story = store.get_story(job.story_id) if job.story_id else None + url = story["url"] if story else job.url + adapter = fff.get_adapter(url) + _progress(job, "Reading story page") + snap = fff.fetch_story(adapter) + if story is None: + story = store.get_story_by_url(snap.url) + if story is None: + # A new story. If the library already has it (for example from the + # FanFicFare calibre plugin), link it and keep its chapters. + found = _find_book(snap.url, [snap.title]) if library_available() else None + book_id, book_uuid = (found[0].id, found[0].uuid) if found else (None, None) + contents = None + if found: + _progress(job, "Reading chapters of library book") + try: + contents = _read_epub(found[1], found[0].title) + except StoryError as e: + logger.warning(str(e)) + finally: + found[1].unlink(missing_ok=True) + story_id = store.create_story( + snap.url, snap.site, snap.title, snap.author, default_locked_fields(), book_id, book_uuid + ) + _link_changed(story_id) + if found: + store.update_story(story_id, ingested_at=now()) + if contents: + _import_epub(story_id, contents) + story = _require_story(story_id) + job.story_id = story["id"] + job_queue.update(job, title=snap.title) + story_id = story["id"] + + store.update_story(story_id, metahtml=snap.metahtml, site=snap.site, author=snap.author) + story["metahtml"] = snap.metahtml + if snap.cover: + store.put_image( + story_id, snap.cover["url"], snap.cover["ext"], snap.cover["mime"], snap.cover["data"], is_cover=True + ) + + # Chapter URLs can change (RoyalRoad changes the slug when a title + # changes). FanFicFare maps old forms to the current one. + chapters = store.list_chapters(story_id) + known = {c["url"] for c in chapters} + for c in chapters: + normal = adapter.normalize_chapterurl(c["url"]) + if normal != c["url"] and normal not in known: + store.update_chapter(c["id"], url=normal) + known.discard(c["url"]) + known.add(normal) + c["url"] = normal + + site_titles = {c["url"]: c["title"] for c in snap.chapters} + site_index = {url: i for i, url in enumerate(site_titles)} + by_url = {c["url"]: c for c in chapters} + dropped = 0 + renamed: dict[int, str] = {} # chapter id: new title on the site + for c in chapters: + on_site = int(c["url"] in site_titles) + changes = {} + if on_site != c["on_site"]: + changes["on_site"] = on_site + if on_site and site_titles[c["url"]] != c["title"]: + renamed[c["id"]] = site_titles[c["url"]] + if job.mode == DownloadMode.FULL and c["included"] != on_site: + changes["included"] = on_site + dropped += 1 - on_site + if changes: + store.update_chapter(c["id"], **changes) + c.update(changes) + + mode = job.mode + new_urls = [u for u in site_titles if u not in by_url] + if mode == DownloadMode.METADATA: + new_urls = [] + refetch: list[dict[str, Any]] = [] + skipped: list[str] = [] + if mode in (DownloadMode.RECHECK_ALL, DownloadMode.FULL): + refetch = [c for c in chapters if c["on_site"]] + elif mode == DownloadMode.RECHECK: + new_urls = [] + for c in chapters: + if c["id"] not in job.chapter_ids: + continue + if c["on_site"]: + refetch.append(c) + else: + skipped.append(fff.plain_title(c["title"])) + # A new title can come with new text. On a site that numbers its chapter + # URLs, it can also mean that the chapters moved up. So a renamed chapter is + # checked again, and it takes the new title only when that works. A metadata + # update downloads no chapters, so it only takes the new titles. + renamed_count = job.result.get("renamed", 0) + if mode == DownloadMode.METADATA: + for chapter_id, title in renamed.items(): + store.update_chapter(chapter_id, title=title) + renamed_count += len(renamed) + else: + refetch += [c for c in chapters if c["id"] in renamed and c not in refetch] + # Authors most often fix their latest chapters, so a new-chapter download can check those again. + recent = _recent_chapters(story_id) if mode == DownloadMode.NEW else 0 + if recent: + latest = [c for c in chapters if c["on_site"] and c["included"]][-recent:] + refetch += [c for c in latest if c not in refetch] + + # Each chapter is stored as soon as it arrives, after its images. The job + # result keeps what this job did, so a job that resumes after a restart + # does not recheck the same chapters again, and its summary counts them. + new_count = job.result.get("new", 0) + changed: list[dict[str, Any]] = job.result.get("changed", []) + checked: list[int] = job.result.get("checked", []) + rechecked = len(refetch) + refetch = [c for c in refetch if c["id"] not in checked] + total = len(new_urls) + len(refetch) + done = 0 + failed: list[str] = [] + order = merge_chapter_lists([c["url"] for c in chapters], list(site_titles)) + stored_images = store.image_urls(story_id) + + def store_images() -> None: + for img in fff.collected_images(adapter): + if img["url"] not in stored_images: + store.put_image(story_id, img["url"], img["ext"], img["mime"], img["data"]) + stored_images.add(img["url"]) + + def save_progress() -> None: + job_queue.update( + job, result={"new": new_count, "changed": changed, "checked": checked, "renamed": renamed_count} + ) + + for url in new_urls: + _progress(job, f"New chapter {done + 1} of {total}: {fff.plain_title(site_titles[url])}", done / max(total, 1)) + done += 1 + try: + html = fff.fetch_chapter(adapter, url, site_index[url]) + except Exception as e: + logger.warning(f"Chapter {url} failed: {e}") + failed.append(f"{fff.plain_title(site_titles[url])}: {e}") + continue + store_images() + chapter = {"url": url, "title": site_titles[url], "html": html, "text_hash": text_hash(html)} + by_url[url] = chapter + store.add_chapters(story_id, [chapter], mode.value, [u for u in order if u in by_url]) + new_count += 1 + save_progress() + + for c in refetch: + _progress(job, f"Rechecking chapter {done + 1} of {total}: {fff.plain_title(c['title'])}", done / max(total, 1)) + try: + html = fff.fetch_chapter(adapter, c["url"], site_index[c["url"]]) + except Exception as e: + logger.warning(f"Chapter {c['url']} failed: {e}") + failed.append(f"{fff.plain_title(c['title'])}: {e}") + done += 1 + continue + store_images() + if c["id"] in renamed: + c["title"] = renamed[c["id"]] + store.update_chapter(c["id"], title=c["title"]) + renamed_count += 1 + new_hash = text_hash(html) + # The site changed when its text differs from the text that it sent last. Until then, the book keeps + # a revision that the user chose. + newest = store.list_revisions(c["id"])[-1] + if new_hash != newest["text_hash"]: + if _rules_explain_change(story_id, newest, html): + store.replace_revision(c["id"], newest["rev"], html, new_hash) + else: + rev = store.add_revision(c["id"], html, new_hash, mode.value) + changed.append({"chapter_id": c["id"], "title": c["title"], "rev": rev}) + checked.append(c["id"]) + save_progress() + done += 1 + + removed = sum(1 for c in chapters if not c["on_site"] and c["included"]) + summary = [f"{new_count} new chapter{'s' if new_count != 1 else ''}"] + if renamed_count: + summary.append(f"{renamed_count} renamed") + if rechecked: + summary.append(f"{len(changed)} of {rechecked} rechecked chapters changed") + if removed: + summary.append(f"{removed} kept after removal from site") + if dropped: + summary.append(f"{dropped} taken out because site removed them") + if skipped: + summary.append(f"{len(skipped)} skipped, no longer on site") + if failed: + summary.append(f"{len(failed)} failed") + job_queue.update( + job, result={"new": new_count, "changed": changed, "failed": failed, "skipped": skipped, "removed": removed} + ) + + metadata_changed = snap.calibre != story["site_values"] + delivered = bool(story["calibre_id"] or story["ingested_at"]) + # A resumed job builds the book, because the run before the restart can have stored chapters. + needs_build = ( + new_count + or changed + or renamed_count + or job.force_fields + or metadata_changed + or not delivered + or job.options.get("build") + or job.restarts + or mode in (DownloadMode.FULL, DownloadMode.METADATA) + or (mode == DownloadMode.NEW and story["status"] == "error" and not failed) + ) + if mode in (DownloadMode.RECHECK_ALL, DownloadMode.FULL): + store.set_setting(story_id, "last_full_recheck", now()) + if not store.chapters_for_build(story_id): + raise StoryError("; ".join(summary) + ". No chapters downloaded.") + if not needs_build: + store.update_story(story_id, last_checked=now(), status="ok", message="No changes") + return "No changes. " + ", ".join(summary) + + book = _linked_book(story) + pinned = _resolve_metadata(story, snap.calibre, job.force_fields) + title = _display_title(story, pinned, snap.title) + _progress(job, "Building epub", 1.0) + epub = config.TMP_DIR / f"story-{story_id}.epub" + _build(story, _epub_overrides(story, pinned), epub) + message = _deliver(story, epub, book, title, snap.author) + store.update_story( + story_id, + pinned=pinned, + site_values=snap.calibre, + title=title, + last_checked=now(), + last_changed=now(), + message=message, + status="error" if failed else "ok", + ) + return ", ".join(summary) + ". " + message + + +def _run_job(job: Job) -> str: + if job.mode == DownloadMode.IMPORT: + return _run_import(job) + if job.mode == DownloadMode.REBUILD: + return _run_rebuild(job) + if job.mode == DownloadMode.IMPORT_FILE: + return _run_import_file(job) + return _run_site_job(job) + + +def download_loop() -> None: + """Background thread for processing the job queue.""" + logger.info("Starting download loop") + while True: + job = job_queue.get_next() + if not job: + time.sleep(2) + continue + job_queue.update(job, status=QueueStatus.RUNNING, message="Starting") + try: + message = _run_job(job) + job_queue.update(job, status=QueueStatus.DONE, message=message, progress=1.0) + logger.info(f"Job {job.id} ({job.mode.value}) for {job.title}: {message}") + except (StoryError, CwaError, EpubError, RuleError, KeyError, ValueError) as e: + _fail(job, str(e)) + except Exception as e: + logger.exception(f"Job {job.id} failed: {e}") + _fail(job, f"{type(e).__name__}: {e}") + if job.mode in SITE_MODES: + time.sleep(config.SITE_DELAY) + + +def _fail(job: Job, message: str) -> None: + logger.warning(f"Job {job.id} ({job.mode.value}) for {job.title} failed: {message}") + job_queue.update(job, status=QueueStatus.ERROR, message=message) + if job.story_id: + store.update_story(job.story_id, status="error", message=message, last_checked=now()) + + +def _queue_due_updates() -> None: + interval = config.UPDATE_INTERVAL_HOURS + if interval <= 0: + return + due = datetime.now() - timedelta(hours=interval) + recheck_days = store.story_settings("recheck_days") + last_full = store.story_settings("last_full_recheck") + for s in store.list_stories(): + if not s["auto_update"] or job_queue.has_active(s["id"]): + continue + if s["last_checked"] and datetime.fromisoformat(s["last_checked"]) > due: + continue + # A story with a recheck interval gets a full recheck in place of its update when one is due. + mode = DownloadMode.NEW + days = recheck_days.get(s["id"]) + last = last_full.get(s["id"]) + if days and (not last or datetime.fromisoformat(last) <= datetime.now() - timedelta(days=days)): + mode = DownloadMode.RECHECK_ALL + job_queue.add(Job(mode=mode, story_id=s["id"], title=s["title"])) + + +def scheduler_loop() -> None: + logger.info(f"Starting scheduler, update interval {config.UPDATE_INTERVAL_HOURS} hours") + while True: + try: + daily_backup(store, config.BACKUP_DIR, config.PERSONAL_INI, config.BACKUP_COUNT) + except Exception as e: + logger.exception(f"Backup error: {e}") + try: + _queue_due_updates() + except Exception as e: + logger.exception(f"Scheduler error: {e}") + time.sleep(60) + + +def start_threads() -> None: + for target in (download_loop, scheduler_loop): + threading.Thread(target=target, daemon=True, name=target.__name__).start() diff --git a/tests/test_backup.py b/tests/test_backup.py new file mode 100644 index 0000000..c494c15 --- /dev/null +++ b/tests/test_backup.py @@ -0,0 +1,25 @@ +"""Daily backups of the database and personal.ini.""" + +import sqlite3 +from datetime import date + +from bindery.backup import daily_backup +from bindery.store import Store + + +def test_one_backup_a_day_and_only_the_newest_are_kept(tmp_path): + store = Store(tmp_path / "bindery.db") + store.create_story("https://example.com/s/1", "example.com", "T", "A", []) + ini = tmp_path / "personal.ini" + ini.write_text("[defaults]\n") + folder = tmp_path / "backups" + for day in range(1, 11): + daily_backup(store, folder, ini, 7, date(2026, 9, day)) + assert daily_backup(store, folder, ini, 7, date(2026, 9, 10)) is None + names = sorted(p.name for p in folder.iterdir()) + assert names == [f"bindery-2026-09-{d:02}.db" for d in range(4, 11)] + [ + f"personal-2026-09-{d:02}.ini" for d in range(4, 11) + ] + copy = sqlite3.connect(folder / "bindery-2026-09-10.db") + assert copy.execute("SELECT title FROM stories").fetchall() == [("T",)] + copy.close() diff --git a/tests/test_deliver.py b/tests/test_deliver.py new file mode 100644 index 0000000..0cbf0b5 --- /dev/null +++ b/tests/test_deliver.py @@ -0,0 +1,610 @@ +"""Rebuild stored stories, import epub files and hand books to CWA, against a fake CWA. No story site access.""" + +import dataclasses +import json +import re +import zipfile + +import pytest +from fake_cwa import FakeBook, FakeCwa +from test_build import COVER, IMG_URL, PNG, URL, chapters, metahtml + +from bindery import config, fff, orchestrator +from bindery.diffing import text_hash +from bindery.epub import EpubError +from bindery.fff import SiteSnapshot, StoryError +from bindery.ingest import IngestFolder +from bindery.jobs import DownloadMode, Job, JobQueue, QueueStatus +from bindery.store import Store + +STORY_URL = "https://www.royalroad.com/fiction/1234" + + +@pytest.fixture(scope="module") +def story_epub(tmp_path_factory): + path = tmp_path_factory.mktemp("epub") / "story.epub" + fff.build_epub(URL, metahtml(), chapters(), [], None, [], {"title": "Library Title"}, path) + return path.read_bytes() + + +@pytest.fixture +def fake(story_epub, monkeypatch, tmp_path): + server = FakeCwa() + server.books = [FakeBook(1, "uuid-1", "Test Story", publisher="www.royalroad.com", epub=story_epub)] + monkeypatch.setattr(config, "CWA_URL", server.url) + monkeypatch.setattr(config, "CWA_USERNAME", "bindery") + monkeypatch.setattr(config, "CWA_PASSWORD", "secret") + monkeypatch.setattr(orchestrator, "store", Store(tmp_path / "bindery.db")) + monkeypatch.setattr(orchestrator, "ingest_folder", IngestFolder(tmp_path / "ingest")) + monkeypatch.setattr(orchestrator, "finder", orchestrator.BookFinder()) + yield server + server.close() + + +def add_story(**changes): + store = orchestrator.store + story_id = store.create_story(STORY_URL, "www.royalroad.com", "Test Story", "Writer", ["title"]) + store.update_story(story_id, metahtml=metahtml(), pinned={"title": "My Title"}, **changes) + for n, c in enumerate(chapters()): + store.add_chapter(story_id, c["url"], c["title"], c["html"], "h", "site", n) + return story_id + + +def ingest_files(): + folder = orchestrator.ingest_folder.path + return sorted(p.name for p in folder.iterdir()) if folder.exists() else [] + + +def opf(name): + with zipfile.ZipFile(orchestrator.ingest_folder.path / name) as z: + return z.read("content.opf").decode() + + +def rebuild(story_id): + return orchestrator._run_rebuild(Job(mode=DownloadMode.REBUILD, story_id=story_id)) + + +def test_first_version_goes_in_as_a_new_book(fake): + fake.books = [] + story_id = add_story() + assert rebuild(story_id) == "Sent to CWA" + (name,) = ingest_files() + assert re.search(r"]*>My Title", opf(name)) + assert orchestrator.store.get_story(story_id)["ingested_at"] + + +def test_ingested_story_is_found_and_updated_with_a_sidecar(fake): + story_id = add_story(ingested_at="2026-01-01T00:00:00") + assert "library book 1" in rebuild(story_id) + story = orchestrator.store.get_story(story_id) + assert (story["calibre_id"], story["calibre_uuid"]) == (1, "uuid-1") + epub, manifest = ingest_files() + assert manifest == epub + ".cwa.json" + assert json.loads((orchestrator.ingest_folder.path / manifest).read_text()) == { + "action": "add_format", + "book_id": 1, + } + + +def test_update_fails_while_cwa_has_no_such_book(fake): + fake.books = [] + story_id = add_story(ingested_at="2026-01-01T00:00:00") + with pytest.raises(orchestrator.StoryError, match="Cannot find this story in CWA"): + rebuild(story_id) + assert ingest_files() == [] # a second plain epub would make a duplicate book + + +def test_update_fails_without_cwa_url(fake, monkeypatch): + monkeypatch.setattr(config, "CWA_URL", "") + story_id = add_story(ingested_at="2026-01-01T00:00:00") + with pytest.raises(orchestrator.StoryError, match="CWA_URL is not set"): + rebuild(story_id) + assert ingest_files() == [] + + +def test_linked_story_still_updates_when_cwa_is_down(fake): + story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=1, calibre_uuid="uuid-1") + fake.close() + assert "library book 1" in rebuild(story_id) + assert len(ingest_files()) == 2 + + +def test_deleted_book_is_unlinked_and_the_update_fails(fake): + story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=5, calibre_uuid="uuid-gone") + fake.books = [] + with pytest.raises(orchestrator.StoryError, match="Cannot find this story in CWA"): + rebuild(story_id) + assert orchestrator.store.get_story(story_id)["calibre_id"] is None + + +def test_unlinked_story_can_go_to_cwa_again_as_a_new_book(fake): + fake.books = [] + story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=5, calibre_uuid="uuid-gone") + orchestrator.link_story(story_id, None) + assert rebuild(story_id) == "Sent to CWA" + assert len(ingest_files()) == 1 + + +def test_import_from_the_library(fake): + job = Job(mode=DownloadMode.IMPORT, calibre_id=1, title="Test Story", options={"uuid": "uuid-1", "check": False}) + assert orchestrator._run_import(job) == "Imported 3 chapters" + story = orchestrator.store.get_story(job.story_id) + assert story["url"] == STORY_URL + assert (story["calibre_id"], story["calibre_uuid"]) == (1, "uuid-1") + assert story["pinned"] == {"title": "Library Title"} + assert [c["title"] for c in orchestrator.store.list_chapters(job.story_id)] == ["One", "Two", "Three"] + assert orchestrator._run_import(job) == "Book already tracked" + assert not list(config.TMP_DIR.glob("import-*")) + + +def test_library_candidates_and_search(fake): + candidates = orchestrator.library_candidates() + assert candidates == [ + {"calibre_id": 1, "uuid": "uuid-1", "title": "Test Story", "authors": "Writer", "site": "www.royalroad.com"} + ] + story_id = add_story() + orchestrator.link_story(story_id, "uuid-1") + assert orchestrator.library_candidates() == [] + assert orchestrator.search_library("test")[0]["tracked_as"] == "Test Story" + with pytest.raises(KeyError): + orchestrator.link_story(story_id, "uuid-nope") + + +# Import chapters from an epub file + + +@pytest.fixture +def jobs(monkeypatch): + queue = JobQueue() + monkeypatch.setattr(orchestrator, "job_queue", queue) + return queue + + +def upload(tmp_path, data): + path = tmp_path / "upload-1.epub" + path.write_bytes(data) + return path + + +def import_file(jobs, path, story_id=None, check=True): + queued = orchestrator.queue_epub_file(path, "story.epub", story_id, check) + job = jobs.get_next() + assert job.id == queued["id"] + return orchestrator._run_import_file(job) + + +def titles(story_id): + return [c["title"] for c in orchestrator.store.list_chapters(story_id)] + + +def test_epub_file_starts_a_new_story(fake, jobs, story_epub, tmp_path): + path = upload(tmp_path, story_epub) + assert import_file(jobs, path) == "Imported 3 chapters from story.epub" + assert not path.exists() + story = orchestrator.store.get_story_by_url(STORY_URL) + assert story["title"] == "Library Title" + assert story["pinned"] == {"title": "Library Title"} + assert titles(story["id"]) == ["One", "Two", "Three"] + check = jobs.get_next() + assert (check.mode, check.story_id, check.options) == (DownloadMode.NEW, story["id"], {"build": True}) + + +def test_epub_file_fills_in_a_stubbed_story(fake, jobs, story_epub, tmp_path): + store = orchestrator.store + story_id = store.create_story(STORY_URL, "www.royalroad.com", "Test Story", "Writer", ["title"]) + one, _, three = chapters() + store.add_chapter(story_id, one["url"], "One", "

Kept

", "h", "site", 0) + store.add_chapter(story_id, three["url"], "Three", "

Kept

", "h", "site", 1) + assert import_file(jobs, upload(tmp_path, story_epub), story_id) == "Imported 1 chapter from story.epub" + assert titles(story_id) == ["One", "Two", "Three"] + assert store.chapters_for_build(story_id)[0]["html"] == "

Kept

" + + +def test_next_update_adds_newer_chapters_after_the_imported_ones(fake, jobs, story_epub, tmp_path, monkeypatch): + # The site stubbed the first two chapters and added a fourth after the epub was made. + fake.books = [] + import_file(jobs, upload(tmp_path, story_epub)) + four = chapters()[2]["url"].replace("15/c15", "16/c16") + site = [{"url": chapters()[2]["url"], "title": "Three"}, {"url": four, "title": "Four"}] + monkeypatch.setattr( + fff, + "fetch_story", + lambda adapter: SiteSnapshot(STORY_URL, "www.royalroad.com", "Test Story", "Writer", metahtml(), site, {}), + ) + monkeypatch.setattr(fff, "fetch_chapter", lambda adapter, url, index: "

Four

") + monkeypatch.setattr(fff, "collected_images", lambda adapter: []) + check = jobs.get_next() + assert orchestrator._run_site_job(check).startswith("1 new chapter") + assert titles(check.story_id) == ["One", "Two", "Three", "Four"] + assert len(ingest_files()) == 1 + + +def test_epub_file_without_a_check_rebuilds_a_downloaded_story(fake, jobs, story_epub, tmp_path): + store = orchestrator.store + story_id = store.create_story(STORY_URL, "www.royalroad.com", "Test Story", "Writer", ["title"]) + store.update_story(story_id, metahtml=metahtml(), ingested_at="2026-01-01T00:00:00") + store.add_chapter(story_id, chapters()[0]["url"], "One", "

Kept

", "h", "site", 0) + assert import_file(jobs, upload(tmp_path, story_epub), story_id, check=False).startswith("Imported 2") + rebuild_job = jobs.get_next() + assert (rebuild_job.mode, rebuild_job.story_id) == (DownloadMode.REBUILD, story_id) + assert "library book 1" in orchestrator._run_rebuild(rebuild_job) + + +def test_epub_of_another_story_is_refused(fake, jobs, story_epub, tmp_path): + other = orchestrator.store.create_story( + "https://www.royalroad.com/fiction/99", "www.royalroad.com", "Other", "Writer", [] + ) + path = upload(tmp_path, story_epub) + with pytest.raises(StoryError, match="story.epub is an epub of another story"): + orchestrator.queue_epub_file(path, "story.epub", other) + assert not path.exists() + assert jobs.get_next() is None + + +def test_file_that_is_not_an_epub_is_refused(fake, jobs, tmp_path): + path = upload(tmp_path, b"not a zip file") + with pytest.raises(EpubError, match="story.epub is not a readable epub"): + orchestrator.queue_epub_file(path, "story.epub") + assert not path.exists() + + +# Updates from the site + +CHAPTER = "https://www.royalroad.com/fiction/1234/test-story/chapter/{0}/c{0}" + + +def linked_story(chapter_list): + """A story in CWA with the chapters (number, title, text).""" + store = orchestrator.store + story_id = store.create_story(STORY_URL, "www.royalroad.com", "Test Story", "Writer", ["title"], 1, "uuid-1") + store.update_story(story_id, metahtml=metahtml(), ingested_at="2026-01-01T00:00:00") + for n, (number, title, text, *source) in enumerate(chapter_list): + html = f"

{text}

" + store.add_chapter( + story_id, CHAPTER.format(number), title, html, text_hash(html), source[0] if source else "new", n + ) + return story_id + + +def site(monkeypatch, chapter_list, broken=()): + """The site lists these chapters (number, title, text). Chapters in `broken` fail to download.""" + texts = {CHAPTER.format(n): text for n, _, text in chapter_list} + listed = [{"url": CHAPTER.format(n), "title": title} for n, title, _ in chapter_list] + monkeypatch.setattr( + fff, + "fetch_story", + lambda adapter: SiteSnapshot(STORY_URL, "www.royalroad.com", "Test Story", "Writer", metahtml(), listed, {}), + ) + monkeypatch.setattr(fff, "collected_images", lambda adapter: []) + + def fetch(adapter, url, index): + if url in {CHAPTER.format(n) for n in broken}: + raise OSError("timeout") + return f"

{texts[url]}

" + + monkeypatch.setattr(fff, "fetch_chapter", fetch) + + +def update(story_id, mode=DownloadMode.NEW, **fields): + return orchestrator._run_site_job(Job(mode=mode, story_id=story_id, **fields)) + + +def book(story_id): + """The title and text of each chapter in the book.""" + return [(c["title"], re.sub(r"<[^>]+>", "", c["html"])) for c in orchestrator.store.chapters_for_build(story_id)] + + +def test_the_site_cover_goes_in_the_book(fake, jobs, monkeypatch, tmp_path): + # A personal.ini without include_images, as the calibre plugin often has. + empty = tmp_path / "personal.ini" + empty.write_text("") + monkeypatch.setattr(config, "PERSONAL_INI", empty) + story_id = linked_story([(1, "One", "a")]) + site(monkeypatch, [(1, "One", "a"), (2, "Two", "b")]) + snapshot = fff.fetch_story + monkeypatch.setattr(fff, "fetch_story", lambda adapter: dataclasses.replace(snapshot(adapter), cover=COVER)) + assert update(story_id).startswith("1 new chapter") + assert orchestrator.store.cover(story_id)["data"] == PNG + (epub,) = [n for n in ingest_files() if n.endswith(".epub")] + with zipfile.ZipFile(orchestrator.ingest_folder.path / epub) as z: + assert "OEBPS/cover.xhtml" in z.namelist() + + +def test_new_chapter_after_stubbed_chapters_goes_last(fake, jobs, monkeypatch): + story_id = linked_story([(1, "One", "a"), (2, "Two", "b"), (3, "Three", "c")]) + site(monkeypatch, [(1, "One", "a"), (4, "Four", "d")]) + assert update(story_id).startswith("1 new chapter") + assert book(story_id) == [("One", "a"), ("Two", "b"), ("Three", "c"), ("Four", "d")] + + +def test_renamed_chapter_is_rechecked_and_rebuilt(fake, jobs, monkeypatch): + story_id = linked_story([(1, "One", "a"), (2, "Two", "b")]) + site(monkeypatch, [(1, "One", "a"), (2, "Two (rewritten)", "b2")]) + message = update(story_id) + assert "1 renamed, 1 of 1 rechecked chapters changed" in message and "library book 1" in message + assert book(story_id) == [("One", "a"), ("Two (rewritten)", "b2")] + + +def test_renamed_chapter_with_the_same_text_is_rebuilt(fake, jobs, monkeypatch): + story_id = linked_story([(1, "One", "a")]) + site(monkeypatch, [(1, "Chapter One", "a")]) + message = update(story_id) + assert "1 renamed, 0 of 1 rechecked chapters changed" in message and "library book 1" in message + assert len(orchestrator.store.list_revisions(orchestrator.store.list_chapters(story_id)[0]["id"])) == 1 + + +def test_job_results_show_chapter_titles_as_text(fake, jobs, monkeypatch): + story_id = linked_story([(1, "One", "a")]) + site(monkeypatch, [(1, "One", "a"), (2, "Folly & Magic", "b")], broken=[2]) + job = Job(mode=DownloadMode.NEW, story_id=story_id) + orchestrator._run_site_job(job) + assert job.result["failed"] == ["Folly & Magic: timeout"] + site(monkeypatch, [(1, "One", "a"), (2, "Folly & Magic", "b")]) + orchestrator._run_site_job(Job(mode=DownloadMode.NEW, story_id=story_id)) + # The store keeps FanFicFare's form, for the epub writer. + assert [c["title"] for c in orchestrator.store.list_chapters(story_id)] == ["One", "Folly & Magic"] + + +def test_renamed_chapter_keeps_its_title_until_its_recheck_works(fake, jobs, monkeypatch): + story_id = linked_story([(1, "One", "a")]) + site(monkeypatch, [(1, "Chapter One", "a2")], broken=[1]) + update(story_id) + assert book(story_id) == [("One", "a")] + site(monkeypatch, [(1, "Chapter One", "a2")]) + update(story_id) + assert book(story_id) == [("Chapter One", "a2")] + + +def test_insert_on_a_site_that_numbers_its_chapter_urls(fake, jobs, monkeypatch): + # FanFiction.net and similar sites. A chapter inserted at 2 moves B and C up one URL. + story_id = linked_story([(1, "1. A", "a"), (2, "2. B", "b"), (3, "3. C", "c")]) + site(monkeypatch, [(1, "1. A", "a"), (2, "2. New", "new"), (3, "3. B", "b"), (4, "4. C", "c")]) + update(story_id) + assert book(story_id) == [("1. A", "a"), ("2. New", "new"), ("3. B", "b"), ("4. C", "c")] + + +def test_imported_chapter_that_differs_only_by_the_rules_is_unchanged(fake, jobs, monkeypatch): + orchestrator.save_rules(None, [{"pattern": "Zorian", "replacement": "Z"}]) + story_id = linked_story([(1, "One", "Z wakes.", "import")]) + site(monkeypatch, [(1, "One", "Zorian wakes.")]) + assert "0 of 1 rechecked chapters changed" in update(story_id, DownloadMode.RECHECK_ALL) + # The stored text is now the site text, so the rules run on it once. + (chapter,) = orchestrator.store.list_chapters(story_id) + assert orchestrator.store.revision_html(chapter["id"], 1) == "

Zorian wakes.

" + # A real change is still a change. + site(monkeypatch, [(1, "One", "Zorian sleeps.")]) + assert "1 of 1 rechecked chapters changed" in update(story_id, DownloadMode.RECHECK_ALL) + + +def test_a_chosen_revision_stays_until_the_site_changes_again(fake, jobs, monkeypatch): + story_id = linked_story([(1, "One", "a")]) + (chapter,) = orchestrator.store.list_chapters(story_id) + site(monkeypatch, [(1, "One", "b")]) + assert "1 of 1 rechecked chapters changed" in update(story_id, DownloadMode.RECHECK_ALL) + orchestrator.store.update_chapter(chapter["id"], current_rev=1) + assert "0 of 1 rechecked chapters changed" in update(story_id, DownloadMode.RECHECK_ALL) + assert book(story_id) == [("One", "a")] + assert len(orchestrator.store.list_revisions(chapter["id"])) == 2 + site(monkeypatch, [(1, "One", "c")]) + assert "1 of 1 rechecked chapters changed" in update(story_id, DownloadMode.RECHECK_ALL) + assert book(story_id) == [("One", "c")] + + +def test_a_site_that_goes_back_to_an_older_text_changes_the_book(fake, jobs, monkeypatch): + story_id = linked_story([(1, "One", "a")]) + site(monkeypatch, [(1, "One", "b")]) + update(story_id, DownloadMode.RECHECK_ALL) + site(monkeypatch, [(1, "One", "a")]) + assert "1 of 1 rechecked chapters changed" in update(story_id, DownloadMode.RECHECK_ALL) + assert book(story_id) == [("One", "a")] + + +def test_new_chapter_download_rechecks_the_latest_chapters(fake, jobs, monkeypatch): + orchestrator.store.set_setting(None, "recent_chapters", 2) + story_id = linked_story([(1, "One", "a"), (2, "Two", "b"), (3, "Three", "c")]) + site(monkeypatch, [(1, "One", "a (edited)"), (2, "Two", "b"), (3, "Three", "c (edited)")]) + assert "1 of 2 rechecked chapters changed" in update(story_id) + assert book(story_id) == [("One", "a"), ("Two", "b"), ("Three", "c (edited)")] + # A story with its own full recheck interval does not check its latest chapters. + orchestrator.update_story_settings(story_id, recheck_days=7) + site(monkeypatch, [(1, "One", "a"), (2, "Two", "b"), (3, "Three", "c (again)")]) + assert update(story_id).startswith("No changes") + + +def test_scheduler_runs_a_full_recheck_when_one_is_due(fake, jobs, monkeypatch): + monkeypatch.setattr(config, "UPDATE_INTERVAL_HOURS", 12) + story_id = linked_story([(1, "One", "a")]) + orchestrator.update_story_settings(story_id, recheck_days=7) + orchestrator._queue_due_updates() + job = jobs.get_next() + assert job.mode == DownloadMode.RECHECK_ALL + site(monkeypatch, [(1, "One", "a")]) + orchestrator._run_site_job(job) + jobs.update(job, status=QueueStatus.DONE) + assert orchestrator.story_details(story_id)["story"]["last_full_recheck"] + orchestrator.store.update_story(story_id, last_checked="2020-01-01T00:00:00") + orchestrator._queue_due_updates() + assert jobs.get_next().mode == DownloadMode.NEW + with pytest.raises(ValueError): + orchestrator.update_story_settings(story_id, recheck_days=-1) + orchestrator.update_story_settings(story_id, recheck_days=0) + assert orchestrator.story_details(story_id)["story"]["recheck_days"] is None + + +# Metadata edits + + +def test_metadata_edit_rebuilds_the_book(fake, jobs): + story_id = add_story(calibre_id=1, calibre_uuid="uuid-1", ingested_at="2026-01-01T00:00:00") + result = orchestrator.edit_metadata(story_id, {"title": "Edited", "tags": "Mine, Yours"}) + assert result["story"]["title"] == "Edited" + assert result["story"]["pinned"] == {"title": "Edited", "tags": ["Mine", "Yours"]} + job = jobs.get_next() + assert result["job"]["id"] == job.id and job.mode == DownloadMode.REBUILD + orchestrator._run_rebuild(job) + epub, _ = ingest_files() + assert re.search(r"]*>Edited", opf(epub)) + assert re.findall(r"([^<]*)", opf(epub)) == ["Mine", "Yours"] + # Without the edit, the unlocked tags come from the site again. + orchestrator.edit_metadata(story_id, {"tags": None}) + assert orchestrator.store.get_story(story_id)["pinned"] == {"title": "Edited"} + + +def test_unlocking_a_field_drops_its_kept_value(fake, jobs): + story_id = add_story() + orchestrator.update_story_settings(story_id, locked_fields=[]) + assert orchestrator.store.get_story(story_id)["pinned"] == {} + + +def test_metadata_edit_of_a_story_that_cwa_does_not_have_waits_for_the_next_build(fake, jobs): + story_id = add_story() + assert orchestrator.edit_metadata(story_id, {"series": "Saga [2]"})["job"] is None + assert jobs.get_next() is None + with pytest.raises(ValueError, match="Unknown metadata fields"): + orchestrator.edit_metadata(story_id, {"publisher": "x"}) + + +# Restarts + + +class Crash(BaseException): + """Bindery stops in the middle of a job.""" + + +def test_a_download_that_stops_keeps_its_chapters_and_resumes(fake, story_epub, tmp_path, monkeypatch): + store = orchestrator.store + story_id = store.create_story(STORY_URL, "www.royalroad.com", "Test Story", "Writer", ["title"]) + store.update_story(story_id, metahtml=metahtml(), ingested_at="2026-01-01T00:00:00") + one, two, three = chapters() + store.add_chapter(story_id, one["url"], "One", "

1

", "h", "site", 0) + four = three["url"].replace("15/c15", "16/c16") + site = [one, two, three, {"url": four, "title": "Four"}] + image = {"url": IMG_URL, "ext": "png", "mime": "image/png", "data": PNG} + monkeypatch.setattr( + fff, + "fetch_story", + lambda adapter: SiteSnapshot(STORY_URL, "www.royalroad.com", "Test Story", "Writer", metahtml(), site, {}), + ) + monkeypatch.setattr(fff, "collected_images", lambda adapter: [image]) + crash_at = [three["url"]] + + def fetch(adapter, url, index): + if url in crash_at: + raise Crash + return f"

{url}

" + + monkeypatch.setattr(fff, "fetch_chapter", fetch) + + queue = JobQueue(tmp_path / "jobs.db") + monkeypatch.setattr(orchestrator, "job_queue", queue) + job = queue.add(Job(mode=DownloadMode.NEW, story_id=story_id)) + queue.update(queue.get_next(), status=QueueStatus.RUNNING) + with pytest.raises(Crash): + orchestrator._run_site_job(job) + # Chapter Two and its image are stored, in their place. + assert titles(story_id) == ["One", "Two"] + assert store.image_urls(story_id) == {IMG_URL} + + queue = JobQueue(tmp_path / "jobs.db") + monkeypatch.setattr(orchestrator, "job_queue", queue) + crash_at.clear() + resumed = queue.get_next() + assert (resumed.id, resumed.restarts, resumed.result["new"]) == (job.id, 1, 1) + message = orchestrator._run_site_job(resumed) + assert message.startswith("3 new chapters") and "library book 1" in message + assert titles(story_id) == ["One", "Two", "Three", "Four"] + + +def test_an_upload_that_is_gone_fails_clearly(fake, jobs, story_epub, tmp_path): + path = upload(tmp_path, story_epub) + orchestrator.queue_epub_file(path, "story.epub") + path.unlink() + with pytest.raises(StoryError, match="Uploaded file story.epub is gone. Upload it again."): + orchestrator._run_import_file(jobs.get_next()) + + +# Upload holds + + +def test_a_new_story_waits_for_approval_of_its_first_upload(fake, jobs, monkeypatch): + orchestrator.save_settings(hold_new_links=True) + fake.books = [] + site(monkeypatch, [(1, "One", "a")]) + job = Job(mode=DownloadMode.NEW, url=STORY_URL) + assert "Approve as new book" in orchestrator._run_site_job(job) + assert ingest_files() == [] + story = orchestrator.story_details(job.story_id)["story"] + assert (story["upload_held"], story["needs_approval"], story["ingested_at"]) == (True, True, None) + assert orchestrator.list_stories()[0]["upload_held"] + queued = orchestrator.approve_upload(job.story_id, None) + assert jobs.get_next().id == queued["id"] + assert rebuild(job.story_id) == "Sent to CWA" + story = orchestrator.story_details(job.story_id)["story"] + assert (story["upload_held"], story["needs_approval"]) == (False, False) + + +def test_a_new_story_linked_to_a_library_book_waits_for_approval(fake, jobs, monkeypatch): + orchestrator.save_settings(hold_new_links=True) + site(monkeypatch, [(1, "One", "a")]) + job = Job(mode=DownloadMode.NEW, url=STORY_URL) + assert "Approve link to library book 1" in orchestrator._run_site_job(job) + assert ingest_files() == [] + with pytest.raises(ValueError, match="link of this story changed"): + orchestrator.approve_upload(job.story_id, None) + orchestrator.approve_upload(job.story_id, "uuid-1") + assert "library book 1" in rebuild(job.story_id) + assert len(ingest_files()) == 2 + + +def test_a_changed_link_waits_for_approval(fake, jobs): + orchestrator.save_settings(hold_new_links=True) + fake.books.append(FakeBook(2, "uuid-2", "Other Story", epub=b"x")) + story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=1, calibre_uuid="uuid-1") + assert "library book 1" in rebuild(story_id) + orchestrator.link_story(story_id, "uuid-2") + assert "Approve link to library book 2" in rebuild(story_id) + assert len(ingest_files()) == 2 + orchestrator.approve_upload(story_id, "uuid-2") + assert "library book 2" in rebuild(story_id) + + +def test_a_book_found_after_the_linked_book_is_gone_waits_for_approval(fake, jobs): + orchestrator.save_settings(hold_new_links=True) + story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=5, calibre_uuid="uuid-gone") + assert "Approve link to library book 1" in rebuild(story_id) + assert ingest_files() == [] + + +def test_the_book_that_cwa_imports_from_the_first_upload_needs_no_approval(fake, jobs): + orchestrator.save_settings(hold_new_links=True) + books, fake.books = fake.books, [] + story_id = add_story() + assert rebuild(story_id) == "Sent to CWA" + fake.books = books + assert "library book 1" in rebuild(story_id) + + +def test_a_link_that_went_to_cwa_before_the_setting_needs_no_approval(fake, jobs): + story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=1, calibre_uuid="uuid-1") + orchestrator.link_story(story_id, None) + orchestrator.link_story(story_id, "uuid-1") + assert "library book 1" in rebuild(story_id) + orchestrator.save_settings(hold_new_links=True) + assert "library book 1" in rebuild(story_id) + + +def test_paused_uploads_go_to_cwa_when_the_pause_ends(fake, jobs): + story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=1, calibre_uuid="uuid-1") + orchestrator.save_settings(pause_uploads=True) + assert "pause" in rebuild(story_id) + assert ingest_files() == [] + assert orchestrator.approve_upload(story_id, "uuid-1")["mode"] == DownloadMode.REBUILD.value + assert "pause" in orchestrator._run_rebuild(jobs.get_next()) + orchestrator.save_settings(pause_uploads=True) + assert jobs.get_next() is None + orchestrator.save_settings(pause_uploads=False) + assert "library book 1" in orchestrator._run_rebuild(jobs.get_next()) + assert len(ingest_files()) == 2 + assert not orchestrator.story_details(story_id)["story"]["upload_held"] + orchestrator.save_settings(pause_uploads=False) + assert jobs.get_next() is None diff --git a/tests/test_jobs.py b/tests/test_jobs.py new file mode 100644 index 0000000..fb39524 --- /dev/null +++ b/tests/test_jobs.py @@ -0,0 +1,163 @@ +"""The job queue: shared jobs, order, cancelling and the job history.""" + +from datetime import datetime, timedelta + +from bindery import config +from bindery.jobs import DownloadMode, Job, JobQueue, QueueStatus + + +def job(mode=DownloadMode.NEW, **fields): + return Job(mode=mode, **fields) + + +def test_a_waiting_job_is_shared(): + queue = JobQueue() + first = queue.add(job(story_id=1)) + assert queue.add(job(story_id=1)) is first + assert queue.add(job(url="https://example.com/s/1")) is not first + assert queue.add(job(url="https://example.com/s/1")).url == "https://example.com/s/1" + assert queue.add(job(DownloadMode.REBUILD, story_id=1)) is not first + assert queue.add(job(story_id=2)) is not first + assert queue.add(job(story_id=1, force_fields=["title"])) is not first + + +def test_rechecks_of_other_chapters_are_separate_jobs(): + queue = JobQueue() + one = queue.add(job(DownloadMode.RECHECK, story_id=1, chapter_ids=[1])) + assert queue.add(job(DownloadMode.RECHECK, story_id=1, chapter_ids=[2])) is not one + assert queue.add(job(DownloadMode.RECHECK, story_id=1, chapter_ids=[1])) is one + + +def test_uploads_are_separate_jobs(): + queue = JobQueue() + one = queue.add(job(DownloadMode.IMPORT_FILE, story_id=1, options={"path": "a.epub"})) + assert queue.add(job(DownloadMode.IMPORT_FILE, story_id=1, options={"path": "b.epub"})) is not one + + +def test_a_started_job_is_not_shared(): + queue = JobQueue() + first = queue.add(job(story_id=1)) + assert queue.get_next() is first + assert queue.add(job(story_id=1)) is not first + + +def test_jobs_run_in_order_and_cancelled_jobs_are_skipped(): + queue = JobQueue() + a, b, c = (queue.add(job(story_id=n)) for n in (1, 2, 3)) + assert queue.cancel(b.id) + assert b.status == QueueStatus.CANCELLED + assert queue.get_next() is a + assert queue.get_next() is c + assert queue.get_next() is None + + +def test_a_cancelled_job_is_not_shared(): + queue = JobQueue() + cancelled = queue.add(job(story_id=1)) + queue.cancel(cancelled.id) + again = queue.add(job(story_id=1)) + assert again is not cancelled and again.status == QueueStatus.QUEUED + assert queue.get_next() is again + + +def test_a_cancelled_job_can_expire_before_the_queue_reaches_it(): + queue = JobQueue() + queue.add(job(story_id=1)) + running = queue.get_next() + cancelled = queue.add(job(story_id=2)) + queue.cancel(cancelled.id) + cancelled.updated = datetime.now() - timedelta(seconds=config.JOB_HISTORY + 1) + assert cancelled.id not in [j["id"] for j in queue.get_status()] + queue.update(running, status=QueueStatus.DONE) + later = queue.add(job(story_id=3)) + assert queue.get_next() is later + + +def test_only_waiting_jobs_can_be_cancelled(): + queue = JobQueue() + running = queue.add(job(story_id=1)) + queue.update(queue.get_next(), status=QueueStatus.RUNNING) + assert not queue.cancel(running.id) + assert not queue.cancel("no-such-job") + + +def test_has_active(): + queue = JobQueue() + first = queue.add(job(story_id=1)) + assert queue.has_active(1) and not queue.has_active(2) + queue.update(first, status=QueueStatus.RUNNING) + assert queue.has_active(1) + queue.update(first, status=QueueStatus.DONE) + assert not queue.has_active(1) + + +def test_status_is_newest_first_and_filters_by_story(): + queue = JobQueue() + old = queue.add(job(story_id=1, created=datetime.now() - timedelta(minutes=5))) + new = queue.add(job(DownloadMode.REBUILD, story_id=1)) + other = queue.add(job(story_id=2)) + assert [j["id"] for j in queue.get_status(1)] == [new.id, old.id] + assert {j["id"] for j in queue.get_status()} == {old.id, new.id, other.id} + data = queue.get_status(2)[0] + assert data["mode"] == "new" and data["status"] == "queued" + assert data["mode_label"] == "Download new chapters (default)" + assert datetime.fromisoformat(data["created"]) + + +def test_finished_jobs_expire(): + queue = JobQueue() + done = queue.add(job(story_id=1)) + waiting = queue.add(job(story_id=2)) + long_ago = datetime.now() - timedelta(seconds=config.JOB_HISTORY + 1) + queue.update(done, status=QueueStatus.DONE) + done.updated = long_ago + waiting.updated = long_ago + assert [j["id"] for j in queue.get_status()] == [waiting.id] + + +def test_jobs_outlive_a_restart(tmp_path): + queue = JobQueue(tmp_path / "db.sqlite") + running = queue.add(job(story_id=1)) + waiting = queue.add(job(story_id=2, options={"x": [1]})) + done = queue.add(job(story_id=3)) + cancelled = queue.add(job(story_id=4)) + queue.update(queue.get_next(), status=QueueStatus.RUNNING, result={"new": 2}) + queue.update(done, status=QueueStatus.DONE) + queue.cancel(cancelled.id) + + again = JobQueue(tmp_path / "db.sqlite") + status = {j["id"]: j for j in again.get_status()} + assert (status[running.id]["status"], status[running.id]["restarts"]) == ("queued", 1) + assert status[running.id]["message"] == "Resumed" + assert status[running.id]["result"] == {"new": 2} + assert status[done.id]["status"] == "done" and status[cancelled.id]["status"] == "cancelled" + # The interrupted job goes first. + assert again.get_next().id == running.id + nxt = again.get_next() + assert (nxt.id, nxt.options, nxt.mode) == (waiting.id, {"x": [1]}, DownloadMode.NEW) + assert again.get_next() is None + + +def test_a_job_that_stops_bindery_again_and_again_is_given_up(tmp_path): + path = tmp_path / "db.sqlite" + queue = JobQueue(path) + first = queue.add(job(story_id=1)) + queue.update(queue.get_next(), status=QueueStatus.RUNNING) + for restarts in (1, 2): + queue = JobQueue(path) + started = queue.get_next() + assert (started.id, started.restarts) == (first.id, restarts) + queue.update(started, status=QueueStatus.RUNNING) + queue = JobQueue(path) + assert queue.get_next() is None + (gone,) = queue.get_status() + assert gone["status"] == "error" and "3 times" in gone["message"] + + +def test_finished_jobs_leave_the_database(tmp_path, monkeypatch): + queue = JobQueue(tmp_path / "db.sqlite") + old = queue.add(job(story_id=1)) + queue.update(old, status=QueueStatus.DONE) + old.updated = datetime.now() - timedelta(seconds=config.JOB_HISTORY + 1) + queue.get_status() + assert JobQueue(tmp_path / "db.sqlite").get_status() == [] diff --git a/tests/test_metadata.py b/tests/test_metadata.py new file mode 100644 index 0000000..f86ef4b --- /dev/null +++ b/tests/test_metadata.py @@ -0,0 +1,74 @@ +import pytest + +from bindery import orchestrator as backend + +SITE = { + "title": "Site Title", + "authors": ["Writer"], + "comments": "

New blurb

", + "tags": ["Fantasy", "Completed"], + "series": "Saga", + "series_index": 2.0, + "pubdate": "2020-01-01T00:00:00", + "languages": "en", +} + + +def story(**extra): + base = {"locked_fields": ["title"], "pinned": {}, "site_values": {}} + base.update(extra) + return base + + +def test_first_download_keeps_site_value(): + assert backend._resolve_metadata(story(), SITE, []) == {"title": "Site Title"} + + +def test_locked_field_keeps_its_value(): + assert backend._resolve_metadata(story(pinned={"title": "Old"}), SITE, []) == {"title": "Old"} + + +def test_forced_field_takes_site_value_and_stays_locked(): + s = story(pinned={"title": "Old"}) + assert backend._resolve_metadata(s, SITE, ["title"]) == {"title": "Site Title"} + + +def test_edited_unlocked_field_stays_until_the_site_changes_it(): + s = story(pinned={"title": "Old", "tags": ["mine"]}, site_values=SITE) + assert backend._resolve_metadata(s, SITE, []) == {"title": "Old", "tags": ["mine"]} + changed = {**SITE, "tags": ["Fantasy", "Completed", "LitRPG"]} + assert backend._resolve_metadata(s, changed, []) == {"title": "Old"} + # A change to another field keeps the edit. + assert backend._resolve_metadata(s, {**SITE, "comments": "

x

"}, []) == {"title": "Old", "tags": ["mine"]} + + +def test_forced_update_replaces_an_edited_unlocked_field(): + s = story(pinned={"tags": ["mine"]}, site_values=SITE) + assert backend._resolve_metadata(s, SITE, ["tags"]) == {"title": "Site Title"} + + +def test_edit_before_the_first_site_values_stays(): + s = story(pinned={"tags": ["mine"]}) + assert backend._resolve_metadata(s, SITE, []) == {"title": "Site Title", "tags": ["mine"]} + + +def test_edited_values_are_checked_and_normalized(): + assert backend._edited_value("authors", "A & B ") == ["A", "B"] + assert backend._edited_value("tags", "x, y,,") == ["x", "y"] + assert backend._edited_value("series", "Saga [2]") == ["Saga", 2.0] + assert backend._edited_value("series", ["Saga", "3"]) == ["Saga", 3.0] + assert backend._edited_value("series", "") == ["", None] + assert backend._edited_value("pubdate", "2024-01-31") == "2024-01-31" + assert backend._edited_value("title", " New ") == "New" + for field, value in (("title", " "), ("authors", "&"), ("pubdate", "soon")): + with pytest.raises(ValueError): + backend._edited_value(field, value) + + +def test_series_keeps_name_and_index(): + s = story(locked_fields=["series"]) + assert backend._resolve_metadata(s, SITE, []) == {"series": ["Saga", 2.0]} + + +def test_resolve_without_site_values_does_not_fail(): + assert backend._resolve_metadata(story(pinned={"title": "Old"}), {}, []) == {"title": "Old"} diff --git a/tests/test_worker_flows.py b/tests/test_worker_flows.py new file mode 100644 index 0000000..ec1c66b --- /dev/null +++ b/tests/test_worker_flows.py @@ -0,0 +1,267 @@ +"""Worker and scheduler transitions, with real temporary stores and no background threads.""" + +from datetime import datetime, timedelta +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from bindery import config, fff +from bindery import orchestrator as o +from bindery.diffing import text_hash +from bindery.jobs import DownloadMode, Job, JobQueue, QueueStatus +from bindery.store import Store + + +class StopLoop(BaseException): + """Stop an infinite loop without entering its application error handler.""" + + +@pytest.fixture +def state(monkeypatch, tmp_path): + store = Store(tmp_path / "stories.db") + queue = JobQueue(tmp_path / "jobs.db") + monkeypatch.setattr(o, "store", store) + monkeypatch.setattr(o, "job_queue", queue) + monkeypatch.setattr(config, "CWA_URL", "") + monkeypatch.setattr(config, "SITE_DELAY", 7) + return store, queue + + +def add_story(store, url="https://www.royalroad.com/fiction/1234", **values): + sid = store.create_story(url, "www.royalroad.com", "Title", "Writer", []) + store.update_story(sid, **values) + return sid + + +@pytest.mark.parametrize("mode", list(DownloadMode)) +def test_dispatches_each_download_mode(state, monkeypatch, mode): + handlers = { + name: Mock(return_value=name) for name in ("_run_import", "_run_import_file", "_run_rebuild", "_run_site_job") + } + for name, handler in handlers.items(): + monkeypatch.setattr(o, name, handler) + expected = { + DownloadMode.IMPORT: "_run_import", + DownloadMode.IMPORT_FILE: "_run_import_file", + DownloadMode.REBUILD: "_run_rebuild", + }.get(mode, "_run_site_job") + job = Job(mode=mode) + assert o._run_job(job) == expected + handlers[expected].assert_called_once_with(job) + assert sum(h.call_count for h in handlers.values()) == 1 + + +@pytest.mark.parametrize("error", [None, o.StoryError("site refused"), RuntimeError("broken")]) +@pytest.mark.parametrize("with_story", [False, True]) +def test_worker_persists_outcomes_and_continues_after_failure(state, monkeypatch, tmp_path, error, with_story): + store, queue = state + sid = add_story(store) if with_story else None + first = queue.add(Job(mode=DownloadMode.NEW, story_id=sid, url="first")) + second = queue.add(Job(mode=DownloadMode.REBUILD, story_id=sid)) + monkeypatch.setattr(queue, "get_next", Mock(side_effect=[None, first, second, StopLoop()])) + sleeps = Mock() + monkeypatch.setattr(o.time, "sleep", sleeps) + + def run(job): + assert job.status == QueueStatus.RUNNING + if job is first and error: + raise error + return "Sent" + + monkeypatch.setattr(o, "_run_job", run) + with pytest.raises(StopLoop): + o.download_loop() + saved = {j["id"]: j for j in JobQueue(tmp_path / "jobs.db").get_status()} + assert saved[second.id]["status"] == "done" + assert saved[second.id]["progress"] == 1.0 + assert saved[first.id]["status"] == ("error" if error else "done") + expected = "RuntimeError: broken" if isinstance(error, RuntimeError) else str(error) if error else "Sent" + assert saved[first.id]["message"] == expected + if error and sid: + story = store.get_story(sid) + assert (story["status"], story["message"]) == ("error", expected) + assert story["last_checked"] + assert [c.args[0] for c in sleeps.call_args_list] == [2, 7] + + +def test_scheduler_skips_disabled_busy_and_recent_stories(state, monkeypatch): + store, queue = state + now = datetime(2026, 6, 1, 12) + + class Clock(datetime): + @classmethod + def now(cls): + return now + + monkeypatch.setattr(o, "datetime", Clock) + monkeypatch.setattr(config, "UPDATE_INTERVAL_HOURS", 2) + ids = [add_story(store, f"url-{n}") for n in range(7)] + store.update_story(ids[0], auto_update=0) + queue.add(Job(mode=DownloadMode.NEW, story_id=ids[1])) + store.update_story(ids[2], last_checked=(now - timedelta(hours=1)).isoformat()) + store.update_story(ids[3], last_checked=(now - timedelta(hours=2)).isoformat()) + for sid in ids[4:]: + store.set_setting(sid, "recheck_days", 3) + store.set_setting(ids[5], "last_full_recheck", (now - timedelta(days=3)).isoformat()) + store.set_setting(ids[6], "last_full_recheck", (now - timedelta(days=1)).isoformat()) + o._queue_due_updates() + queued = {j["story_id"]: j["mode"] for j in queue.get_status()} + assert queued == {ids[1]: "new", ids[3]: "new", ids[4]: "recheck_all", ids[5]: "recheck_all", ids[6]: "new"} + o._queue_due_updates() + assert len(queue.get_status()) == 5 + monkeypatch.setattr(config, "UPDATE_INTERVAL_HOURS", 0) + extra = add_story(store, "extra") + o._queue_due_updates() + assert not queue.has_active(extra) + + +def test_scheduler_recovers_from_backup_and_queue_errors(state, monkeypatch): + backup = Mock(side_effect=[OSError("disk full"), None]) + due = Mock(side_effect=[ValueError("bad date"), None]) + sleep = Mock(side_effect=[None, StopLoop()]) + monkeypatch.setattr(o, "daily_backup", backup) + monkeypatch.setattr(o, "_queue_due_updates", due) + monkeypatch.setattr(o.time, "sleep", sleep) + with pytest.raises(StopLoop): + o.scheduler_loop() + assert backup.call_count == due.call_count == 2 + assert [c.args for c in sleep.call_args_list] == [(60,), (60,)] + + +@pytest.fixture +def site(state, monkeypatch): + store, queue = state + sid = add_story(store, ingested_at="2026-01-01", site_values={"title": "Title"}) + for i in range(1, 4): + html = f"

old {i}

" + store.add_chapter(sid, f"chapter-{i}", f"Chapter {i}", html, text_hash(html), "new", i) + adapter = SimpleNamespace(normalize_chapterurl=lambda url: url) + snap = fff.SiteSnapshot( + url=store.get_story(sid)["url"], + site="www.royalroad.com", + title="Title", + author="Writer", + metahtml="metadata", + chapters=[ + {"url": "chapter-1", "title": "Renamed"}, + {"url": "chapter-2", "title": "Chapter 2"}, + {"url": "chapter-4", "title": "New"}, + ], + calibre={"title": "Title"}, + ) + fetch = Mock(side_effect=lambda a, url, i: f"

fresh {url}

") + build, deliver = Mock(), Mock(return_value="Sent") + monkeypatch.setattr(fff, "get_adapter", lambda url: adapter) + monkeypatch.setattr(fff, "fetch_story", lambda a: snap) + monkeypatch.setattr(fff, "fetch_chapter", fetch) + monkeypatch.setattr(fff, "collected_images", lambda a: []) + monkeypatch.setattr(o, "_build", build) + monkeypatch.setattr(o, "_linked_book", lambda story: None) + monkeypatch.setattr(o, "_deliver", deliver) + return SimpleNamespace( + store=store, queue=queue, id=sid, snap=snap, adapter=adapter, fetch=fetch, build=build, deliver=deliver + ) + + +@pytest.mark.parametrize("mode", [DownloadMode.METADATA, DownloadMode.FULL, DownloadMode.RECHECK]) +def test_download_modes_preserve_history_and_select_the_right_chapters(site, mode): + original = site.store.list_chapters(site.id) + job = Job(mode=mode, story_id=site.id, chapter_ids=[original[1]["id"], original[2]["id"]]) + result = o._run_site_job(job) + chapters = site.store.list_chapters(site.id) + assert chapters[0]["title"] == "Renamed" + assert all(site.store.revision_html(c["id"], 1) == f"

old {n}

" for n, c in enumerate(original, 1)) + fetched = [call.args[1] for call in site.fetch.call_args_list] + removed = site.store.get_chapter(original[2]["id"]) + assert not removed["on_site"] + if mode == DownloadMode.METADATA: + assert fetched == [] + assert len(chapters) == 3 + assert removed["included"] + elif mode == DownloadMode.FULL: + assert set(fetched) == {"chapter-1", "chapter-2", "chapter-4"} + assert not removed["included"] + assert "1 taken out" in result + assert site.store.get_setting(site.id, "last_full_recheck") + else: + assert set(fetched) == {"chapter-1", "chapter-2"} + assert job.result["skipped"] == ["Chapter 3"] + assert removed["included"] + site.build.assert_called_once() + site.deliver.assert_called_once() + + +def test_failed_new_chapter_keeps_successful_chapters_and_images(site, monkeypatch): + site.snap.chapters.append({"url": "chapter-5", "title": "Broken"}) + + def fetch(adapter, url, index): + if url == "chapter-5": + raise OSError("timeout") + return "

ok

" + + site.fetch.side_effect = fetch + image = {"url": "image", "ext": "png", "mime": "image/png", "data": b"image"} + site.snap.cover = {**image, "url": "cover"} + monkeypatch.setattr(fff, "collected_images", lambda a: [image, image]) + job = Job(mode=DownloadMode.NEW, story_id=site.id) + assert "1 failed" in o._run_site_job(job) + assert job.result["failed"] == ["Broken: timeout"] + assert len(site.store.list_chapters(site.id)) == 4 + assert site.store.get_story(site.id)["status"] == "error" + assert site.store.image_urls(site.id) == {"image"} + assert site.store.cover(site.id)["url"] == "cover" + + +def test_new_story_with_no_downloaded_chapters_fails_without_delivery(site): + site.fetch.side_effect = OSError("offline") + job = Job(mode=DownloadMode.NEW, url="new-url") + site.snap.url = "new-url" + with pytest.raises(o.StoryError, match="No chapters downloaded"): + o._run_site_job(job) + assert job.story_id != site.id + assert site.store.list_chapters(job.story_id) == [] + site.deliver.assert_not_called() + + +def test_normalized_chapter_urls_keep_the_existing_identity(site): + original = site.store.list_chapters(site.id)[0] + site.adapter.normalize_chapterurl = lambda url: "renamed-url" if url == "chapter-1" else url + site.snap.chapters[0]["url"] = "renamed-url" + o._run_site_job(Job(mode=DownloadMode.RECHECK_ALL, story_id=site.id)) + updated = site.store.get_chapter(original["id"]) + assert updated["url"] == "renamed-url" + assert updated["current_rev"] == 2 + assert len(site.store.list_chapters(site.id)) == 4 + + +def test_inclusion_changes_only_the_requested_story_and_queues_a_rebuild(state): + store, queue = state + sid, other = add_story(store), add_story(store, "other") + cid = store.add_chapter(sid, "chapter", "One", "

text

", "h", "new", 0) + foreign = store.add_chapter(other, "foreign", "Other", "

other

", "h", "new", 0) + result = o.set_chapters_included(sid, [cid, foreign, 999], False) + assert result["mode"] == "rebuild" + assert not store.get_chapter(cid)["included"] + assert store.get_chapter(foreign)["included"] + assert store.revision_html(cid, 1) == "

text

" + o.set_chapters_included(sid, [cid], True) + assert store.get_chapter(cid)["included"] + assert len(queue.get_status()) == 1 + + +@pytest.mark.parametrize( + "mode,chapters,fields,message", + [ + ("import", [], [], "library import"), + ("recheck", [], [], "at least one chapter"), + ("new", [], ["unknown"], "Unknown metadata"), + ("invalid", [], [], "not a valid"), + ], +) +def test_invalid_job_requests_do_not_enter_the_queue(state, mode, chapters, fields, message): + store, queue = state + sid = add_story(store) + with pytest.raises(ValueError, match=message): + o.queue_story(sid, mode, chapters, fields) + assert queue.get_status() == []