diff --git a/bindery/backup.py b/bindery/backup.py
new file mode 100644
index 0000000..5542f8f
--- /dev/null
+++ b/bindery/backup.py
@@ -0,0 +1,40 @@
+"""Daily copies of the database and personal.ini.
+
+The database holds chapters that the sites removed, so it is the only copy of
+them. A backup a day, kept for a week, limits a bad change or a broken disk to
+the loss of one day.
+"""
+
+import logging
+import shutil
+from datetime import date
+from pathlib import Path
+
+from bindery.store import Store
+
+logger = logging.getLogger(__name__)
+
+
+def daily_backup(store: Store, folder: Path, personal_ini: Path, keep: int, today: date | None = None) -> Path | None:
+ """Make the backup of today if there is none yet, then delete all but the newest `keep` backups.
+
+ Return the path of the new database copy, or None if today had one already.
+ """
+ today = today or date.today()
+ folder.mkdir(parents=True, exist_ok=True)
+ target = folder / f"bindery-{today.isoformat()}.db"
+ made = None
+ if not target.exists():
+ # A copy under another name first, so that a stop never leaves half a backup.
+ partial = folder / f"{target.name}.part"
+ partial.unlink(missing_ok=True)
+ store.backup(partial)
+ partial.replace(target)
+ if personal_ini.exists():
+ shutil.copy2(personal_ini, folder / f"personal-{today.isoformat()}.ini")
+ logger.info(f"Backed up the database to {target}")
+ made = target
+ for pattern in ("bindery-*.db", "personal-*.ini"):
+ for old in sorted(folder.glob(pattern))[:-keep]:
+ old.unlink()
+ return made
diff --git a/bindery/jobs.py b/bindery/jobs.py
new file mode 100644
index 0000000..d6dc66e
--- /dev/null
+++ b/bindery/jobs.py
@@ -0,0 +1,239 @@
+"""Download modes, jobs and the job queue."""
+
+import json
+import sqlite3
+import uuid
+from collections import deque
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timedelta
+from enum import StrEnum
+from pathlib import Path
+from threading import Lock
+from typing import Any
+
+from bindery import config
+
+# Calibre fields that a story can lock. A locked field keeps its current
+# value until a download explicitly forces an update of that field.
+METADATA_FIELDS: dict[str, str] = {
+ "title": "Title",
+ "authors": "Authors",
+ "comments": "Description",
+ "tags": "Tags",
+ "series": "Series",
+ "pubdate": "Published date",
+ "languages": "Language",
+}
+
+
+class QueueStatus(StrEnum):
+ """Enum for possible job statuses."""
+
+ QUEUED = "queued"
+ RUNNING = "running"
+ DONE = "done"
+ ERROR = "error"
+ CANCELLED = "cancelled"
+
+
+class DownloadMode(StrEnum):
+ """What a job fetches from the site and how it changes the book."""
+
+ NEW = "new" # new chapters only, appended after the chapters we have
+ RECHECK = "recheck" # download selected chapters again and keep changes
+ RECHECK_ALL = "recheck_all" # download every chapter still on the site again
+ FULL = "full" # replace the whole book with what is on the site now
+ METADATA = "metadata" # update metadata only, no chapter changes
+ REBUILD = "rebuild" # no site access: rebuild the epub from stored chapters
+ IMPORT = "import" # start tracking a book that is already in the library
+ IMPORT_FILE = "import_file" # add the chapters of an uploaded epub to a story
+
+
+MODE_LABELS: dict[str, str] = {
+ DownloadMode.NEW.value: "Download new chapters (default)",
+ DownloadMode.RECHECK.value: "Recheck selected chapters",
+ DownloadMode.RECHECK_ALL.value: "Recheck all chapters",
+ DownloadMode.FULL.value: "Download and replace entire book",
+ DownloadMode.METADATA.value: "Update metadata from site",
+ DownloadMode.REBUILD.value: "Rebuild book from stored chapters",
+ DownloadMode.IMPORT.value: "Import from library",
+ DownloadMode.IMPORT_FILE.value: "Import chapters from an epub file",
+}
+
+# A job that was running when Bindery stopped this many times is not started again.
+MAX_RESTARTS = 3
+
+
+@dataclass
+class Job:
+ """One unit of work for the download thread."""
+
+ mode: DownloadMode
+ story_id: int | None = None
+ url: str | None = None
+ calibre_id: int | None = None
+ chapter_ids: list[int] = field(default_factory=list)
+ force_fields: list[str] = field(default_factory=list)
+ options: dict[str, Any] = field(default_factory=dict)
+ id: str = field(default_factory=lambda: uuid.uuid4().hex[:12])
+ status: QueueStatus = QueueStatus.QUEUED
+ title: str = ""
+ message: str = ""
+ progress: float = 0.0
+ created: datetime = field(default_factory=datetime.now)
+ updated: datetime = field(default_factory=datetime.now)
+ result: dict[str, Any] = field(default_factory=dict)
+ restarts: int = 0
+
+ def to_dict(self) -> dict[str, Any]:
+ d = asdict(self)
+ d["mode"] = self.mode.value
+ d["mode_label"] = MODE_LABELS[self.mode.value]
+ d["status"] = self.status.value
+ d["created"] = self.created.isoformat(timespec="seconds")
+ d["updated"] = self.updated.isoformat(timespec="seconds")
+ return d
+
+ def to_record(self) -> str:
+ d = asdict(self)
+ d["created"] = self.created.isoformat()
+ d["updated"] = self.updated.isoformat()
+ return json.dumps(d)
+
+ @classmethod
+ def from_record(cls, record: str) -> "Job":
+ d = json.loads(record)
+ d["mode"] = DownloadMode(d["mode"])
+ d["status"] = QueueStatus(d["status"])
+ d["created"] = datetime.fromisoformat(d["created"])
+ d["updated"] = datetime.fromisoformat(d["updated"])
+ return cls(**d)
+
+
+class JobQueue:
+ """Thread-safe FIFO job queue that also keeps recent job history.
+
+ With a database path, the queue keeps each job in SQLite, so the jobs outlive
+ a restart. A job that was running goes back to the front of the queue.
+ """
+
+ def __init__(self, path: Path | None = None) -> None:
+ self._lock = Lock()
+ self._pending: deque[str] = deque()
+ self._jobs: dict[str, Job] = {}
+ self._history = timedelta(seconds=config.JOB_HISTORY)
+ self._db: sqlite3.Connection | None = None
+ if path:
+ self._load(Path(path))
+
+ def _load(self, path: Path) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ # One connection, used only while the lock is held. Each statement commits.
+ self._db = sqlite3.connect(path, timeout=30, check_same_thread=False, isolation_level=None)
+ self._db.execute("PRAGMA journal_mode=WAL")
+ self._db.execute("CREATE TABLE IF NOT EXISTS jobs (id TEXT PRIMARY KEY, record TEXT NOT NULL)")
+ jobs = [Job.from_record(r[0]) for r in self._db.execute("SELECT record FROM jobs ORDER BY rowid")]
+ interrupted = [j for j in jobs if j.status == QueueStatus.RUNNING]
+ for job in interrupted:
+ job.restarts += 1
+ if job.restarts >= MAX_RESTARTS:
+ job.status = QueueStatus.ERROR
+ job.message = f"Stopped {job.restarts} times, giving up"
+ else:
+ job.status = QueueStatus.QUEUED
+ job.message = "Resumed"
+ job.updated = datetime.now()
+ self._save(job)
+ self._jobs = {j.id: j for j in jobs}
+ waiting = [
+ j.id for j in interrupted + [j for j in jobs if j not in interrupted] if j.status == QueueStatus.QUEUED
+ ]
+ self._pending.extend(waiting)
+ self._expire()
+
+ def _save(self, job: Job) -> None:
+ if self._db:
+ self._db.execute(
+ "INSERT INTO jobs (id, record) VALUES (?, ?) ON CONFLICT (id) DO UPDATE SET record = excluded.record",
+ (job.id, job.to_record()),
+ )
+
+ def _expire(self) -> None:
+ """Forget finished jobs after JOB_HISTORY seconds."""
+ now = datetime.now()
+ expired = [
+ k
+ for k, j in self._jobs.items()
+ if j.status not in (QueueStatus.QUEUED, QueueStatus.RUNNING) and now - j.updated > self._history
+ ]
+ for job_id in expired:
+ del self._jobs[job_id]
+ if self._db and expired:
+ self._db.executemany("DELETE FROM jobs WHERE id = ?", [(k,) for k in expired])
+
+ def add(self, job: Job) -> Job:
+ """Queue a job. If the same story already waits for the same mode, return that job."""
+ with self._lock:
+ for job_id in self._pending:
+ other = self._jobs[job_id]
+ if (
+ other.mode == job.mode
+ and other.chapter_ids == job.chapter_ids
+ and other.force_fields == job.force_fields
+ and other.options == job.options
+ and (
+ (job.story_id and other.story_id == job.story_id)
+ or (job.url and other.url == job.url)
+ or (job.calibre_id and other.calibre_id == job.calibre_id)
+ )
+ ):
+ return other
+ self._expire()
+ self._jobs[job.id] = job
+ self._pending.append(job.id)
+ self._save(job)
+ return job
+
+ def get_next(self) -> Job | None:
+ with self._lock:
+ while self._pending:
+ job = self._jobs[self._pending.popleft()]
+ if job.status == QueueStatus.QUEUED:
+ return job
+ return None
+
+ def update(self, job: Job, **changes: Any) -> None:
+ with self._lock:
+ for key, value in changes.items():
+ setattr(job, key, value)
+ job.updated = datetime.now()
+ self._save(job)
+
+ def cancel(self, job_id: str) -> bool:
+ with self._lock:
+ job = self._jobs.get(job_id)
+ if not job or job.status != QueueStatus.QUEUED:
+ return False
+ job.status = QueueStatus.CANCELLED
+ job.updated = datetime.now()
+ # A cancelled job must not be shared, and it can expire before the download thread reaches it.
+ self._pending.remove(job_id)
+ self._save(job)
+ return True
+
+ def has_active(self, story_id: int) -> bool:
+ with self._lock:
+ return any(
+ j.story_id == story_id and j.status in (QueueStatus.QUEUED, QueueStatus.RUNNING)
+ for j in self._jobs.values()
+ )
+
+ def get_status(self, story_id: int | None = None) -> list[dict[str, Any]]:
+ """Return recent jobs, newest first."""
+ with self._lock:
+ self._expire()
+ jobs = [j for j in self._jobs.values() if story_id is None or j.story_id == story_id]
+ return [j.to_dict() for j in sorted(jobs, key=lambda j: j.created, reverse=True)]
+
+
+job_queue = JobQueue(config.DB_PATH)
diff --git a/bindery/orchestrator.py b/bindery/orchestrator.py
new file mode 100644
index 0000000..cce27d8
--- /dev/null
+++ b/bindery/orchestrator.py
@@ -0,0 +1,1163 @@
+"""Download modes, metadata locks, delivery to CWA, and the job threads."""
+
+import hashlib
+import logging
+import re
+import threading
+import time
+import uuid
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+from bindery import config, fff
+from bindery.backup import daily_backup
+from bindery.cwa import Book, BookFinder, CwaClient, CwaError
+from bindery.diffing import diff_html, text_hash
+from bindery.epub import EpubError, epub_metadata, epub_urls
+from bindery.fff import StoryError
+from bindery.ingest import IngestFolder
+from bindery.jobs import METADATA_FIELDS, DownloadMode, Job, QueueStatus, job_queue
+from bindery.merge import merge_chapter_lists
+from bindery.rules import RuleError, Step, apply_rules, compile_rules, normalize_rules
+from bindery.store import Store, now
+
+logger = logging.getLogger(__name__)
+
+store = Store(config.DB_PATH)
+ingest_folder = IngestFolder(config.INGEST_DIR)
+finder = BookFinder()
+
+SITE_MODES = (
+ DownloadMode.NEW,
+ DownloadMode.RECHECK,
+ DownloadMode.RECHECK_ALL,
+ DownloadMode.FULL,
+ DownloadMode.METADATA,
+)
+
+_client_lock = threading.Lock()
+_client: tuple[tuple[str, str, str], CwaClient | None] = (("", "", ""), None)
+
+
+def cwa() -> CwaClient:
+ """The CWA client for the configured account."""
+ global _client
+ key = (config.CWA_URL, config.CWA_USERNAME, config.CWA_PASSWORD)
+ with _client_lock:
+ if _client[0] != key or _client[1] is None:
+ _client = (key, CwaClient(*key))
+ return _client[1]
+
+
+def library_available() -> bool:
+ return bool(config.CWA_URL)
+
+
+def default_locked_fields() -> list[str]:
+ return [f for f in config.DEFAULT_LOCKED_FIELDS if f in METADATA_FIELDS]
+
+
+# Queueing
+
+
+def queue_urls(urls: list[str]) -> tuple[list[dict[str, Any]], list[str]]:
+ """Queue a "new chapters" download for each URL. Returns (jobs, errors)."""
+ jobs, errors = [], []
+ for url in (u.strip() for u in urls):
+ if not url:
+ continue
+ normal = fff.normalize_url(url)
+ if not normal:
+ errors.append(f"Not a supported story URL: {url}")
+ continue
+ story = store.get_story_by_url(normal[0])
+ job = Job(
+ mode=DownloadMode.NEW,
+ story_id=story["id"] if story else None,
+ url=normal[0],
+ title=story["title"] if story else normal[0],
+ )
+ jobs.append(job_queue.add(job).to_dict())
+ return jobs, errors
+
+
+def queue_story(
+ story_id: int, mode: str, chapter_ids: list[int] | None = None, force_fields: list[str] | None = None
+) -> dict[str, Any]:
+ story = _require_story(story_id)
+ mode_enum = DownloadMode(mode)
+ if mode_enum == DownloadMode.IMPORT:
+ raise ValueError("Use the library import for this mode")
+ if mode_enum == DownloadMode.RECHECK and not chapter_ids:
+ raise ValueError("Select at least one chapter to recheck")
+ bad = [f for f in force_fields or [] if f not in METADATA_FIELDS]
+ if bad:
+ raise ValueError(f"Unknown metadata fields: {bad}")
+ job = Job(
+ mode=mode_enum,
+ story_id=story_id,
+ title=story["title"],
+ chapter_ids=sorted(int(c) for c in chapter_ids or []),
+ force_fields=sorted(force_fields or []),
+ )
+ return job_queue.add(job).to_dict()
+
+
+def queue_import(books: list[dict[str, Any]], check: bool = True) -> list[dict[str, Any]]:
+ """Queue imports of library books, given as {"id": ..., "uuid": ..., "title": ...}."""
+ jobs = []
+ for b in books:
+ book_id = int(b["id"])
+ jobs.append(
+ job_queue.add(
+ Job(
+ mode=DownloadMode.IMPORT,
+ calibre_id=book_id,
+ title=b.get("title") or f"Library book {book_id}",
+ options={"check": check, "uuid": str(b.get("uuid") or "")},
+ )
+ ).to_dict()
+ )
+ return jobs
+
+
+def queue_epub_file(path: Path, filename: str, story_id: int | None = None, check: bool = True) -> dict[str, Any]:
+ """Queue an import of an uploaded epub. A file without a story URL, or of another story, fails at once."""
+ try:
+ normal = next((n for n in map(fff.normalize_url, epub_urls(path)) if n), None)
+ if not normal:
+ raise StoryError(f"{filename} has no supported story URL. Only epubs from FanFicFare can be imported.")
+ story = store.get_story_by_url(normal[0])
+ if story_id is not None:
+ story = _require_story(story_id)
+ if story["url"] != normal[0]:
+ raise StoryError(f"{filename} is an epub of another story: {normal[0]}")
+ except EpubError as e:
+ path.unlink(missing_ok=True)
+ raise EpubError(str(e).replace(path.name, filename)) from e
+ except Exception:
+ path.unlink(missing_ok=True)
+ raise
+ job = Job(
+ mode=DownloadMode.IMPORT_FILE,
+ story_id=story["id"] if story else None,
+ url=normal[0],
+ title=story["title"] if story else filename,
+ options={"path": str(path), "name": filename, "check": check},
+ )
+ return job_queue.add(job).to_dict()
+
+
+def queue_status(story_id: int | None = None) -> list[dict[str, Any]]:
+ return job_queue.get_status(story_id)
+
+
+def cancel_job(job_id: str) -> bool:
+ return job_queue.cancel(job_id)
+
+
+# Story, chapter and rule management for the UI
+
+
+def _require_story(story_id: int) -> dict[str, Any]:
+ story = store.get_story(story_id)
+ if not story:
+ raise KeyError(f"No story with id {story_id}")
+ return story
+
+
+def list_stories() -> list[dict[str, Any]]:
+ stories = store.list_stories()
+ held = store.story_settings("upload_held")
+ for s in stories:
+ s["busy"] = job_queue.has_active(s["id"])
+ s["upload_held"] = bool(held.get(s["id"]))
+ return stories
+
+
+# Only raster images. An SVG cover could hold a script that runs on Bindery's origin.
+COVER_TYPES = frozenset({"image/jpeg", "image/png", "image/gif", "image/webp"})
+
+
+def story_cover(story_id: int) -> dict[str, Any] | None:
+ """The site cover of a story, if a browser can show it safely."""
+ _require_story(story_id)
+ cover = store.cover(story_id)
+ return cover if cover and cover["mime"] in COVER_TYPES else None
+
+
+def story_details(story_id: int) -> dict[str, Any]:
+ story = _require_story(story_id)
+ story.pop("metahtml", None)
+ cover = story_cover(story_id)
+ # The version changes with the image, so a browser can keep a cover until the site changes it.
+ story["cover"] = hashlib.sha256(cover["data"]).hexdigest()[:16] if cover else None
+ book, library_error = None, None
+ if story["calibre_uuid"] and (library_available() or config.CALIBRE_DB_PATH):
+ try:
+ book = cwa().get_book(story["calibre_uuid"])
+ except CwaError as e:
+ library_error = str(e)
+ story["recheck_days"] = store.get_setting(story_id, "recheck_days")
+ story["last_full_recheck"] = store.get_setting(story_id, "last_full_recheck")
+ story["upload_held"] = bool(store.get_setting(story_id, "upload_held"))
+ story["needs_approval"] = _needs_approval(story_id)
+ return {
+ "recent_chapters": _setting_recent_chapters(),
+ "uploads_paused": _uploads_paused(),
+ "story": story,
+ "chapters": store.list_chapters(story_id),
+ "rules": store.rules(story_id),
+ "book": book,
+ "library_available": library_available(),
+ "library_error": library_error,
+ "jobs": job_queue.get_status(story_id),
+ }
+
+
+def update_story_settings(
+ story_id: int,
+ locked_fields: list[str] | None = None,
+ auto_update: bool | None = None,
+ recheck_days: int | None = None,
+) -> None:
+ """Change the settings that are given. A recheck_days of 0 turns the full rechecks off."""
+ story = _require_story(story_id)
+ if recheck_days is not None:
+ days = int(recheck_days)
+ if days < 0:
+ raise ValueError("The number of days cannot be negative")
+ store.set_setting(story_id, "recheck_days", days or None)
+ changes: dict[str, Any] = {}
+ if locked_fields is not None:
+ bad = [f for f in locked_fields if f not in METADATA_FIELDS]
+ if bad:
+ raise ValueError(f"Unknown metadata fields: {bad}")
+ changes["locked_fields"] = sorted(set(locked_fields))
+ # An unlocked field takes the site value again.
+ unlocked = set(story["locked_fields"]) - set(locked_fields)
+ changes["pinned"] = {f: v for f, v in story["pinned"].items() if f not in unlocked}
+ if auto_update is not None:
+ changes["auto_update"] = int(bool(auto_update))
+ store.update_story(story_id, **changes)
+
+
+def _edited_value(field: str, value: Any) -> Any:
+ """Check an edited metadata value and put it in the stored form."""
+ if field in ("authors", "tags"):
+ items = value if isinstance(value, list) else re.split("&" if field == "authors" else ",", str(value))
+ value = [str(i).strip() for i in items if str(i).strip()]
+ if field == "authors" and not value:
+ raise ValueError("A story needs at least one author")
+ return value
+ if field == "series":
+ if isinstance(value, list):
+ name, index = (value + [None, None])[:2]
+ name, index = str(name or "").strip(), float(index) if index not in (None, "") else None
+ else:
+ name, index = fff.parse_series(str(value).strip())
+ return [name, index if name else None]
+ value = str(value).strip()
+ if field == "title" and not value:
+ raise ValueError("A story needs a title")
+ if field == "pubdate" and value:
+ try:
+ datetime.fromisoformat(value)
+ except ValueError as e:
+ raise ValueError(f"The published date must be a date, for example 2024-01-31: {value}") from e
+ return value
+
+
+def edit_metadata(story_id: int, values: dict[str, Any]) -> dict[str, Any]:
+ """Keep edited metadata values. A value of None removes the edit, so the site value comes back.
+
+ The book is rebuilt when Bindery sent it to CWA before. Otherwise the next
+ build uses the values.
+ """
+ story = _require_story(story_id)
+ bad = [f for f in values if f not in METADATA_FIELDS]
+ if bad:
+ raise ValueError(f"Unknown metadata fields: {bad}")
+ pinned = dict(story["pinned"])
+ for field, value in values.items():
+ if value is None:
+ pinned.pop(field, None)
+ else:
+ pinned[field] = _edited_value(field, value)
+ title = _display_title(story, pinned, story["site_values"].get("title") or story["title"])
+ store.update_story(story_id, pinned=pinned, title=title)
+ queued = None
+ if story["metahtml"] and (story["calibre_id"] or story["ingested_at"]):
+ queued = job_queue.add(Job(mode=DownloadMode.REBUILD, story_id=story_id, title=title)).to_dict()
+ return {"story": story_details(story_id)["story"], "job": queued}
+
+
+def get_settings() -> dict[str, Any]:
+ ini = config.PERSONAL_INI
+ return {
+ "recent_chapters": _setting_recent_chapters(),
+ "hold_new_links": bool(store.get_setting(None, "hold_new_links")),
+ "pause_uploads": _uploads_paused(),
+ "personal_ini": ini.read_text(encoding="utf-8") if ini.exists() else "",
+ }
+
+
+def save_settings(
+ recent_chapters: Any = None,
+ personal_ini: str | None = None,
+ hold_new_links: bool | None = None,
+ pause_uploads: bool | None = None,
+) -> dict[str, Any]:
+ """Change the settings that are given. Return all settings, and FanFicFare's warnings about personal.ini."""
+ warnings: list[str] = []
+ released = False
+ for name, value in (("hold_new_links", hold_new_links), ("pause_uploads", pause_uploads)):
+ if value is None:
+ continue
+ released = released or (bool(store.get_setting(None, name)) and not value)
+ store.set_setting(None, name, True if value else None)
+ if released:
+ _release_held_uploads()
+ if recent_chapters is not None:
+ try:
+ count = int(recent_chapters)
+ except (TypeError, ValueError) as e:
+ raise ValueError("The number of recent chapters must be a whole number") from e
+ if count < 0:
+ raise ValueError("The number of recent chapters cannot be negative")
+ store.set_setting(None, "recent_chapters", count)
+ if personal_ini is not None:
+ warnings = fff.check_personal_ini(personal_ini)
+ # Write a new file, then replace the old one, so a job never reads half a file.
+ partial = config.PERSONAL_INI.with_name("personal.ini.part")
+ partial.write_text(personal_ini, encoding="utf-8")
+ partial.replace(config.PERSONAL_INI)
+ return {**get_settings(), "warnings": warnings}
+
+
+def get_rules(story_id: int | None) -> dict[str, Any]:
+ """The rules of a story, or the global rules for None, with the paragraph setting.
+
+ For a story, `paragraphs` is None when the story follows the global setting.
+ """
+ if story_id is not None:
+ _require_story(story_id)
+ return {
+ "rules": store.rules(story_id),
+ "paragraphs": store.get_setting(story_id, "paragraphs"),
+ "global_paragraphs": bool(store.get_setting(None, "paragraphs")),
+ }
+
+
+def save_rules(story_id: int | None, rules: list[dict[str, Any]], paragraphs: Any = ...) -> dict[str, Any]:
+ """Store the rules. `paragraphs` is True, False, None (a story follows the global setting), or left out."""
+ if story_id is not None:
+ _require_story(story_id)
+ clean = normalize_rules(rules)
+ store.set_rules(story_id, clean)
+ if paragraphs is not ...:
+ if paragraphs is None and story_id is None:
+ paragraphs = False
+ store.set_setting(story_id, "paragraphs", None if paragraphs is None else bool(paragraphs))
+ return get_rules(story_id)
+
+
+def preview_rules(
+ story_id: int | None,
+ rules: list[dict[str, Any]],
+ chapter_id: int,
+ source: bool = False,
+ paragraphs: bool | None = None,
+) -> dict[str, Any]:
+ """Diff one chapter before and after the given rules (plus the global rules for a story).
+
+ `paragraphs` is the setting in the editor. None follows the global setting.
+ """
+ chapter = store.get_chapter(chapter_id)
+ if not chapter or (story_id is not None and chapter["story_id"] != story_id):
+ raise KeyError(f"No chapter with id {chapter_id}")
+ clean = normalize_rules(rules)
+ lists = [clean] if story_id is None else [store.rules(None), clean]
+ if paragraphs is None:
+ paragraphs = bool(store.get_setting(None, "paragraphs"))
+ html = store.revision_html(chapter_id, chapter["current_rev"]) or ""
+ result = diff_html(html, apply_rules(html, compile_rules(*lists, paragraphs=bool(paragraphs))), source=source)
+ result["chapter"] = chapter
+ return result
+
+
+def chapter_diff(chapter_id: int, a: int | None = None, b: int | None = None, source: bool = False) -> dict[str, Any]:
+ chapter = store.get_chapter(chapter_id)
+ if not chapter:
+ raise KeyError(f"No chapter with id {chapter_id}")
+ revisions = store.list_revisions(chapter_id)
+ b = b or chapter["current_rev"]
+ if not a:
+ # The revision before b. Deleted revisions leave gaps in the numbers.
+ others = [r["rev"] for r in revisions if r["rev"] != b]
+ a = max((r for r in others if r < b), default=min(others, default=b))
+ a, b = min(a, b), max(a, b)
+ old, new = store.revision_html(chapter_id, a), store.revision_html(chapter_id, b)
+ if old is None or new is None:
+ raise KeyError("No such revision")
+ result = diff_html(old, new, source=source)
+ story = store.get_story(chapter["story_id"])
+ result.update(chapter=chapter, revisions=revisions, a=a, b=b, story_title=story["title"] if story else "")
+ return result
+
+
+def set_chapter_revision(chapter_id: int, rev: int) -> dict[str, Any]:
+ chapter = store.get_chapter(chapter_id)
+ if not chapter or store.revision_html(chapter_id, rev) is None:
+ raise KeyError("No such chapter revision")
+ store.update_chapter(chapter_id, current_rev=rev, changed_at=now())
+ return queue_story(chapter["story_id"], DownloadMode.REBUILD.value)
+
+
+def delete_revision(chapter_id: int, rev: int) -> list[dict[str, Any]]:
+ """Delete a revision that is not in the book and not the newest, and return the revisions that are left."""
+ chapter = store.get_chapter(chapter_id)
+ if not chapter or store.revision_html(chapter_id, rev) is None:
+ raise KeyError("No such chapter revision")
+ if rev == chapter["current_rev"]:
+ raise ValueError("This revision is in the book. Put another revision in the book first.")
+ if rev == store.list_revisions(chapter_id)[-1]["rev"]:
+ raise ValueError(
+ "This revision is the newest text from the site. "
+ "The next recheck compares with it, so you cannot delete it."
+ )
+ store.delete_revision(chapter_id, rev)
+ return store.list_revisions(chapter_id)
+
+
+def set_chapters_included(story_id: int, chapter_ids: list[int], included: bool) -> dict[str, Any]:
+ for cid in chapter_ids:
+ chapter = store.get_chapter(cid)
+ if chapter and chapter["story_id"] == story_id:
+ store.update_chapter(cid, included=int(included))
+ return queue_story(story_id, DownloadMode.REBUILD.value)
+
+
+def link_story(story_id: int, book_uuid: str | None) -> None:
+ """Link a story to a library book by its uuid, or unlink it with None."""
+ story = _require_story(story_id)
+ if not book_uuid:
+ store.update_story(story_id, calibre_id=None, calibre_uuid=None, ingested_at=None)
+ _link_changed(story_id)
+ return
+ book = cwa().get_book(book_uuid)
+ if not book:
+ raise KeyError(f"No library book with uuid {book_uuid}")
+ other = store.get_story_by_calibre_id(book["id"])
+ if other and other["id"] != story_id:
+ raise ValueError(f"Library book {book['id']} is already linked to {other['title']}")
+ store.update_story(
+ story_id, calibre_id=book["id"], calibre_uuid=book_uuid, ingested_at=story["ingested_at"] or now()
+ )
+ if book_uuid != story["calibre_uuid"]:
+ _link_changed(story_id)
+
+
+def approve_upload(story_id: int, book_uuid: str | None) -> dict[str, Any] | None:
+ """Approve the library link of a story: the uuid of its book, or None for a new book.
+
+ Returns the queued rebuild when a built book waits for the approval.
+ """
+ story = _require_story(story_id)
+ if (book_uuid or None) != story["calibre_uuid"]:
+ raise ValueError("The library link of this story changed. Reload the page, and check the link again.")
+ store.set_setting(story_id, "unapproved_link", None)
+ if store.get_setting(story_id, "upload_held") and story["metahtml"]:
+ return job_queue.add(Job(mode=DownloadMode.REBUILD, story_id=story_id, title=story["title"])).to_dict()
+ return None
+
+
+def delete_story(story_id: int) -> None:
+ store.delete_story(story_id)
+
+
+def search_library(query: str) -> list[dict[str, Any]]:
+ linked = {s["calibre_id"]: s["title"] for s in store.list_stories() if s["calibre_id"]}
+ results = []
+ for book in cwa().search(query)[:50]:
+ item = book.to_dict()
+ item["tracked_as"] = linked.get(book.id)
+ results.append(item)
+ return results
+
+
+def library_candidates() -> list[dict[str, Any]]:
+ """Library books from a story site that are not tracked yet."""
+ tracked_ids = {s["calibre_id"] for s in store.list_stories() if s["calibre_id"]}
+ return [
+ {"calibre_id": b.id, "uuid": b.uuid, "title": b.title, "authors": " & ".join(b.authors), "site": b.publisher}
+ for b in cwa().site_books(fff.is_supported_site)
+ if b.id not in tracked_ids
+ ]
+
+
+def build_for_download(story_id: int) -> tuple[Path, str]:
+ """Build the epub from stored data, for a browser download."""
+ story = _require_story(story_id)
+ path = config.TMP_DIR / f"download-{story_id}.epub"
+ _build(story, _epub_overrides(story, story["pinned"]), path)
+ return path, story["title"]
+
+
+# Metadata locks
+
+
+def _field_value(field: str, meta: dict[str, Any]) -> Any:
+ if field == "series":
+ return [meta.get("series") or "", meta.get("series_index")]
+ return meta.get(field)
+
+
+def _resolve_metadata(story: dict[str, Any], site: dict[str, Any], force_fields: list[str]) -> dict[str, Any]:
+ """Return the values that replace the site values in this build.
+
+ A locked field keeps its stored value. The first download, a library
+ import, an edit on the story page, or a forced update sets that value.
+ An unlocked field keeps an edited value until the site changes that field,
+ or until a forced update. The newer value wins.
+ """
+ locked = set(story["locked_fields"])
+ before = story["site_values"]
+ pinned = {}
+ for f, value in story["pinned"].items():
+ site_changed = bool(before) and _field_value(f, site) != _field_value(f, before)
+ if f in locked or (f not in force_fields and not site_changed):
+ pinned[f] = value
+ for f in locked:
+ if f in force_fields or f not in pinned:
+ pinned[f] = _field_value(f, site)
+ return pinned
+
+
+def _epub_overrides(story: dict[str, Any], pinned: dict[str, Any]) -> dict[str, Any]:
+ return {f: v for f, v in pinned.items() if f in METADATA_FIELDS}
+
+
+def _display_title(story: dict[str, Any], pinned: dict[str, Any], site_title: str) -> str:
+ return pinned.get("title") or site_title
+
+
+# Building and delivery
+
+
+def _paragraphs(story_id: int | None) -> bool:
+ """Whether line breaks become paragraphs before the rules run. A story setting wins over the global one."""
+ value = store.get_setting(story_id, "paragraphs") if story_id is not None else None
+ return bool(store.get_setting(None, "paragraphs") if value is None else value)
+
+
+def _steps(story_id: int) -> list[Step]:
+ return compile_rules(store.rules(None), store.rules(story_id), paragraphs=_paragraphs(story_id))
+
+
+def _build(story: dict[str, Any], overrides: dict[str, Any], out_path: Path) -> dict[str, Any]:
+ if not story.get("metahtml"):
+ raise StoryError("Story has no site metadata yet. Download new chapters first.")
+ return fff.build_epub(
+ story["url"],
+ story["metahtml"],
+ store.chapters_for_build(story["id"]),
+ store.images(story["id"]),
+ store.cover(story["id"]),
+ _steps(story["id"]),
+ overrides,
+ out_path,
+ )
+
+
+def _find_book(story_url: str, titles: list[str | None]) -> tuple[Book, Path] | None:
+ """Search CWA for the book of a story. Returns the book and its downloaded epub."""
+ linked = {s["calibre_id"] for s in store.list_stories() if s["calibre_id"]}
+ return finder.find(cwa(), story_url, [t for t in titles if t], fff.normalize_url, config.TMP_DIR, linked)
+
+
+def _link(story: dict[str, Any], book_id: int, book_uuid: str) -> None:
+ # A book that CWA imported from the first epub of the story is the book that Bindery sent. Other links need
+ # an approval when the setting asks for one.
+ if book_uuid != story["calibre_uuid"] and (story["calibre_uuid"] or not story["ingested_at"]):
+ _link_changed(story["id"])
+ story.update(calibre_id=book_id, calibre_uuid=book_uuid, ingested_at=story["ingested_at"] or now())
+ store.update_story(story["id"], calibre_id=book_id, calibre_uuid=book_uuid, ingested_at=story["ingested_at"])
+
+
+def _linked_book(story: dict[str, Any]) -> dict[str, Any] | None:
+ """The library book for a story. Links the story if CWA has a book with its URL."""
+ if not library_available() and not config.CALIBRE_DB_PATH:
+ return None
+ if story["calibre_uuid"]:
+ try:
+ book = cwa().get_book(story["calibre_uuid"])
+ except CwaError as e:
+ # CWA checks the book id itself, so an update can still go out.
+ logger.warning(f"Cannot check library book {story['calibre_id']}: {e}")
+ return {"id": story["calibre_id"], "uuid": story["calibre_uuid"], "title": story["title"]}
+ if book:
+ if book["id"] != story["calibre_id"]:
+ _link(story, book["id"], book["uuid"])
+ return book
+ logger.warning(f"Library book {story['calibre_id']} for {story['url']} is gone")
+ store.update_story(story["id"], calibre_id=None, calibre_uuid=None)
+ _link_changed(story["id"])
+ story.update(calibre_id=None, calibre_uuid=None)
+ if config.CALIBRE_DB_PATH:
+ book = cwa().book_by_url(story["url"])
+ if book:
+ other = store.get_story_by_calibre_id(book["id"])
+ if not other or other["id"] == story["id"]:
+ _link(story, book["id"], book["uuid"])
+ return book
+ if not library_available():
+ return None
+ # CWA keeps the title of its first import, so an edited title can differ from it.
+ titles = [story["title"], story["pinned"].get("title"), story["site_values"].get("title")]
+ found = _find_book(story["url"], list(dict.fromkeys(titles)))
+ if not found:
+ return None
+ book, path = found
+ path.unlink(missing_ok=True)
+ _link(story, book.id, book.uuid)
+ logger.info(f"Linked {story['title']} to library book {book.id}")
+ return book.to_dict()
+
+
+NOT_FOUND = "Cannot find this story in CWA. Link to library book, or send as new book."
+
+
+# Upload holds. A story gets the per-story setting "unapproved_link" when its library link changes, and loses it
+# with the next upload or an approval. "upload_held" marks a built book that did not go to CWA.
+
+
+def _uploads_paused() -> bool:
+ return bool(store.get_setting(None, "pause_uploads"))
+
+
+def _link_changed(story_id: int) -> None:
+ store.set_setting(story_id, "unapproved_link", True)
+
+
+def _needs_approval(story_id: int) -> bool:
+ return bool(store.get_setting(None, "hold_new_links") and store.get_setting(story_id, "unapproved_link"))
+
+
+def _release_held_uploads() -> None:
+ """Rebuild the held books, so that each one goes to CWA or waits for its own reason."""
+ for story_id in store.story_settings("upload_held"):
+ story = store.get_story(story_id)
+ if story and story["metahtml"]:
+ job_queue.add(Job(mode=DownloadMode.REBUILD, story_id=story_id, title=story["title"]))
+
+
+def _hold_reason(story: dict[str, Any], book: dict[str, Any] | None) -> str | None:
+ if _uploads_paused():
+ return "Not sent. Turn off pause to send."
+ if not _needs_approval(story["id"]):
+ return None
+ if book:
+ return f"Not sent. Approve link to library book {book['id']}."
+ return "Not sent. Approve as new book."
+
+
+def _deliver(story: dict[str, Any], epub: Path, book: dict[str, Any] | None, title: str, author: str) -> str:
+ # A second plain epub would make a duplicate book, so an update without a library book fails.
+ if not book and story["ingested_at"] and not library_available() and not config.CALIBRE_DB_PATH:
+ raise StoryError("Bindery cannot find the library book to update, because CWA_URL is not set.")
+ if not book and story["ingested_at"]:
+ raise StoryError(NOT_FOUND)
+ held = _hold_reason(story, book)
+ if held:
+ store.set_setting(story["id"], "upload_held", True)
+ return held
+ if book:
+ ingest_folder.send_update(epub, book["id"], title)
+ message = f"Updated library book {book['id']}"
+ else:
+ ingest_folder.ingest(epub, f"{title} - {author}" if author else title)
+ store.update_story(story["id"], ingested_at=now())
+ story["ingested_at"] = now()
+ message = "Sent to CWA"
+ store.set_setting(story["id"], "upload_held", None)
+ store.set_setting(story["id"], "unapproved_link", None)
+ return message
+
+
+# Jobs
+
+
+def _progress(job: Job, message: str, progress: float | None = None) -> None:
+ changes: dict[str, Any] = {"message": message}
+ if progress is not None:
+ changes["progress"] = progress
+ job_queue.update(job, **changes)
+
+
+def _read_epub(epub: Path, name: str) -> fff.EpubContents:
+ try:
+ contents = fff.read_epub(epub)
+ except Exception as e:
+ raise StoryError(f"Cannot read chapters of {name}: {e}") from e
+ if not contents.chapters:
+ raise StoryError(f"{name} has no FanFicFare chapter data")
+ return contents
+
+
+def _import_epub(story_id: int, contents: fff.EpubContents) -> int:
+ """Add the chapters and images of a FanFicFare epub that the story does not have yet.
+
+ Each chapter is stored as if it had been downloaded. Chapters that the story
+ has already keep their stored text. The chapter order of the epub places the
+ new chapters, in the same way as a site update, so chapters that the site
+ removed go back to their place. The chapters go in with one transaction, so
+ an import that stops leaves no part of the epub behind.
+ """
+ # FanFicFare maps old forms of a chapter URL to the current one.
+ normalize = fff.get_adapter(_require_story(story_id)["url"]).normalize_chapterurl
+ stored = {normalize(c["url"]): c["url"] for c in store.list_chapters(story_id)}
+ epub_order = []
+ new = []
+ for c in contents.chapters:
+ url = normalize(c["url"])
+ if url not in stored and url not in epub_order:
+ new.append({"url": url, "title": c["title"], "html": c["html"], "text_hash": text_hash(c["html"])})
+ epub_order.append(url)
+ for img in contents.images:
+ store.put_image(story_id, img["url"], img["ext"], img["mime"], img["data"])
+ if new:
+ order = merge_chapter_lists(list(stored), epub_order, newest_last=False)
+ store.add_chapters(story_id, new, "import", [stored.get(u, u) for u in order])
+ return len(new)
+
+
+def _run_import(job: Job) -> str:
+ tracked = store.get_story_by_calibre_id(job.calibre_id)
+ # A tracked story without chapters is left from an import that stopped, so the import goes on.
+ if tracked and store.list_chapters(tracked["id"]):
+ return "Book already tracked"
+ book_uuid = job.options.get("uuid") or ""
+ source = Book(
+ id=job.calibre_id, uuid=book_uuid, title=job.title, formats={"epub": f"opds/download/{job.calibre_id}/epub/"}
+ )
+ _progress(job, "Downloading epub from CWA")
+ epub = cwa().download_epub(source, config.TMP_DIR / f"import-{job.calibre_id}-{uuid.uuid4().hex[:8]}.epub")
+ try:
+ normal = next((n for n in map(fff.normalize_url, epub_urls(epub)) if n), None)
+ if not normal:
+ raise StoryError(f"{job.title} has no supported story URL")
+ meta = epub_metadata(epub)
+ title = meta["title"] or job.title
+ job_queue.update(job, title=title)
+ story = store.get_story_by_url(normal[0])
+ if story and story["calibre_id"] != job.calibre_id:
+ _link(story, job.calibre_id, book_uuid)
+ return f"Linked to tracked story {story['title']}"
+ contents = _read_epub(epub, title)
+ if story is None:
+ locked = default_locked_fields()
+ story_id = store.create_story(
+ normal[0], normal[1], title, " & ".join(meta["authors"]), locked, job.calibre_id, book_uuid or None
+ )
+ pinned = {f: _field_value(f, meta) for f in locked}
+ store.update_story(story_id, pinned=pinned, ingested_at=now(), status="ok")
+ story = _require_story(story_id)
+ story_id = story["id"]
+ job_queue.update(job, story_id=story_id)
+ added = _import_epub(story_id, contents)
+ finally:
+ epub.unlink(missing_ok=True)
+ if job.options.get("check", True):
+ job_queue.add(Job(mode=DownloadMode.NEW, story_id=story_id, title=title))
+ return f"Imported {added} chapters"
+
+
+def _run_import_file(job: Job) -> str:
+ path, name = Path(job.options["path"]), job.options["name"]
+ if not path.exists():
+ raise StoryError(f"Uploaded file {name} is gone. Upload it again.")
+ try:
+ contents = _read_epub(path, name)
+ story = store.get_story(job.story_id) if job.story_id else store.get_story_by_url(job.url)
+ if story is None:
+ meta = epub_metadata(path)
+ locked = default_locked_fields()
+ story_id = store.create_story(
+ job.url, fff.normalize_url(job.url)[1], meta["title"] or job.url, " & ".join(meta["authors"]), locked
+ )
+ store.update_story(story_id, pinned={f: _field_value(f, meta) for f in locked}, status="ok")
+ _link_changed(story_id)
+ story = _require_story(story_id)
+ job_queue.update(job, story_id=story["id"], title=story["title"])
+ added = _import_epub(story["id"], contents)
+ finally:
+ path.unlink(missing_ok=True)
+ message = f"Imported {added} chapter{'s' if added != 1 else ''} from {name}"
+ if not added:
+ message += " Already had every chapter."
+ store.update_story(story["id"], message=message)
+ # The book gets the imported chapters on the next build. A story without site
+ # metadata cannot be built before its first download.
+ if job.options.get("check", True):
+ job_queue.add(
+ Job(mode=DownloadMode.NEW, story_id=story["id"], title=story["title"], options={"build": bool(added)})
+ )
+ elif added and story["metahtml"]:
+ job_queue.add(Job(mode=DownloadMode.REBUILD, story_id=story["id"], title=story["title"]))
+ return message
+
+
+def _run_rebuild(job: Job) -> str:
+ story = _require_story(job.story_id)
+ job_queue.update(job, title=story["title"])
+ book = _linked_book(story)
+ pinned = _resolve_metadata(story, story["site_values"], [])
+ epub = config.TMP_DIR / f"story-{story['id']}.epub"
+ _build(story, _epub_overrides(story, pinned), epub)
+ message = _deliver(story, epub, book, story["title"], story["author"])
+ store.update_story(story["id"], pinned=pinned, status="ok", message=message, last_changed=now())
+ return message
+
+
+def _rules_explain_change(story_id: int, newest: dict[str, Any], html: str) -> bool:
+ """Whether an imported chapter differs from the site text only by the rules.
+
+ An imported epub has the rules of its build in its text, and the site text
+ has none. If the rules turn the site text into the imported text, the
+ chapter did not change.
+ """
+ if newest["source"] != "import":
+ return False
+ return text_hash(apply_rules(html, _steps(story_id))) == newest["text_hash"]
+
+
+def _setting_recent_chapters() -> int:
+ return int(store.get_setting(None, "recent_chapters") or 0)
+
+
+def _recent_chapters(story_id: int) -> int:
+ """How many of the last chapters a new-chapter download checks again.
+
+ A story with its own interval for full rechecks checks none.
+ """
+ if store.get_setting(story_id, "recheck_days"):
+ return 0
+ return _setting_recent_chapters()
+
+
+def _run_site_job(job: Job) -> str:
+ story = store.get_story(job.story_id) if job.story_id else None
+ url = story["url"] if story else job.url
+ adapter = fff.get_adapter(url)
+ _progress(job, "Reading story page")
+ snap = fff.fetch_story(adapter)
+ if story is None:
+ story = store.get_story_by_url(snap.url)
+ if story is None:
+ # A new story. If the library already has it (for example from the
+ # FanFicFare calibre plugin), link it and keep its chapters.
+ found = _find_book(snap.url, [snap.title]) if library_available() else None
+ book_id, book_uuid = (found[0].id, found[0].uuid) if found else (None, None)
+ contents = None
+ if found:
+ _progress(job, "Reading chapters of library book")
+ try:
+ contents = _read_epub(found[1], found[0].title)
+ except StoryError as e:
+ logger.warning(str(e))
+ finally:
+ found[1].unlink(missing_ok=True)
+ story_id = store.create_story(
+ snap.url, snap.site, snap.title, snap.author, default_locked_fields(), book_id, book_uuid
+ )
+ _link_changed(story_id)
+ if found:
+ store.update_story(story_id, ingested_at=now())
+ if contents:
+ _import_epub(story_id, contents)
+ story = _require_story(story_id)
+ job.story_id = story["id"]
+ job_queue.update(job, title=snap.title)
+ story_id = story["id"]
+
+ store.update_story(story_id, metahtml=snap.metahtml, site=snap.site, author=snap.author)
+ story["metahtml"] = snap.metahtml
+ if snap.cover:
+ store.put_image(
+ story_id, snap.cover["url"], snap.cover["ext"], snap.cover["mime"], snap.cover["data"], is_cover=True
+ )
+
+ # Chapter URLs can change (RoyalRoad changes the slug when a title
+ # changes). FanFicFare maps old forms to the current one.
+ chapters = store.list_chapters(story_id)
+ known = {c["url"] for c in chapters}
+ for c in chapters:
+ normal = adapter.normalize_chapterurl(c["url"])
+ if normal != c["url"] and normal not in known:
+ store.update_chapter(c["id"], url=normal)
+ known.discard(c["url"])
+ known.add(normal)
+ c["url"] = normal
+
+ site_titles = {c["url"]: c["title"] for c in snap.chapters}
+ site_index = {url: i for i, url in enumerate(site_titles)}
+ by_url = {c["url"]: c for c in chapters}
+ dropped = 0
+ renamed: dict[int, str] = {} # chapter id: new title on the site
+ for c in chapters:
+ on_site = int(c["url"] in site_titles)
+ changes = {}
+ if on_site != c["on_site"]:
+ changes["on_site"] = on_site
+ if on_site and site_titles[c["url"]] != c["title"]:
+ renamed[c["id"]] = site_titles[c["url"]]
+ if job.mode == DownloadMode.FULL and c["included"] != on_site:
+ changes["included"] = on_site
+ dropped += 1 - on_site
+ if changes:
+ store.update_chapter(c["id"], **changes)
+ c.update(changes)
+
+ mode = job.mode
+ new_urls = [u for u in site_titles if u not in by_url]
+ if mode == DownloadMode.METADATA:
+ new_urls = []
+ refetch: list[dict[str, Any]] = []
+ skipped: list[str] = []
+ if mode in (DownloadMode.RECHECK_ALL, DownloadMode.FULL):
+ refetch = [c for c in chapters if c["on_site"]]
+ elif mode == DownloadMode.RECHECK:
+ new_urls = []
+ for c in chapters:
+ if c["id"] not in job.chapter_ids:
+ continue
+ if c["on_site"]:
+ refetch.append(c)
+ else:
+ skipped.append(fff.plain_title(c["title"]))
+ # A new title can come with new text. On a site that numbers its chapter
+ # URLs, it can also mean that the chapters moved up. So a renamed chapter is
+ # checked again, and it takes the new title only when that works. A metadata
+ # update downloads no chapters, so it only takes the new titles.
+ renamed_count = job.result.get("renamed", 0)
+ if mode == DownloadMode.METADATA:
+ for chapter_id, title in renamed.items():
+ store.update_chapter(chapter_id, title=title)
+ renamed_count += len(renamed)
+ else:
+ refetch += [c for c in chapters if c["id"] in renamed and c not in refetch]
+ # Authors most often fix their latest chapters, so a new-chapter download can check those again.
+ recent = _recent_chapters(story_id) if mode == DownloadMode.NEW else 0
+ if recent:
+ latest = [c for c in chapters if c["on_site"] and c["included"]][-recent:]
+ refetch += [c for c in latest if c not in refetch]
+
+ # Each chapter is stored as soon as it arrives, after its images. The job
+ # result keeps what this job did, so a job that resumes after a restart
+ # does not recheck the same chapters again, and its summary counts them.
+ new_count = job.result.get("new", 0)
+ changed: list[dict[str, Any]] = job.result.get("changed", [])
+ checked: list[int] = job.result.get("checked", [])
+ rechecked = len(refetch)
+ refetch = [c for c in refetch if c["id"] not in checked]
+ total = len(new_urls) + len(refetch)
+ done = 0
+ failed: list[str] = []
+ order = merge_chapter_lists([c["url"] for c in chapters], list(site_titles))
+ stored_images = store.image_urls(story_id)
+
+ def store_images() -> None:
+ for img in fff.collected_images(adapter):
+ if img["url"] not in stored_images:
+ store.put_image(story_id, img["url"], img["ext"], img["mime"], img["data"])
+ stored_images.add(img["url"])
+
+ def save_progress() -> None:
+ job_queue.update(
+ job, result={"new": new_count, "changed": changed, "checked": checked, "renamed": renamed_count}
+ )
+
+ for url in new_urls:
+ _progress(job, f"New chapter {done + 1} of {total}: {fff.plain_title(site_titles[url])}", done / max(total, 1))
+ done += 1
+ try:
+ html = fff.fetch_chapter(adapter, url, site_index[url])
+ except Exception as e:
+ logger.warning(f"Chapter {url} failed: {e}")
+ failed.append(f"{fff.plain_title(site_titles[url])}: {e}")
+ continue
+ store_images()
+ chapter = {"url": url, "title": site_titles[url], "html": html, "text_hash": text_hash(html)}
+ by_url[url] = chapter
+ store.add_chapters(story_id, [chapter], mode.value, [u for u in order if u in by_url])
+ new_count += 1
+ save_progress()
+
+ for c in refetch:
+ _progress(job, f"Rechecking chapter {done + 1} of {total}: {fff.plain_title(c['title'])}", done / max(total, 1))
+ try:
+ html = fff.fetch_chapter(adapter, c["url"], site_index[c["url"]])
+ except Exception as e:
+ logger.warning(f"Chapter {c['url']} failed: {e}")
+ failed.append(f"{fff.plain_title(c['title'])}: {e}")
+ done += 1
+ continue
+ store_images()
+ if c["id"] in renamed:
+ c["title"] = renamed[c["id"]]
+ store.update_chapter(c["id"], title=c["title"])
+ renamed_count += 1
+ new_hash = text_hash(html)
+ # The site changed when its text differs from the text that it sent last. Until then, the book keeps
+ # a revision that the user chose.
+ newest = store.list_revisions(c["id"])[-1]
+ if new_hash != newest["text_hash"]:
+ if _rules_explain_change(story_id, newest, html):
+ store.replace_revision(c["id"], newest["rev"], html, new_hash)
+ else:
+ rev = store.add_revision(c["id"], html, new_hash, mode.value)
+ changed.append({"chapter_id": c["id"], "title": c["title"], "rev": rev})
+ checked.append(c["id"])
+ save_progress()
+ done += 1
+
+ removed = sum(1 for c in chapters if not c["on_site"] and c["included"])
+ summary = [f"{new_count} new chapter{'s' if new_count != 1 else ''}"]
+ if renamed_count:
+ summary.append(f"{renamed_count} renamed")
+ if rechecked:
+ summary.append(f"{len(changed)} of {rechecked} rechecked chapters changed")
+ if removed:
+ summary.append(f"{removed} kept after removal from site")
+ if dropped:
+ summary.append(f"{dropped} taken out because site removed them")
+ if skipped:
+ summary.append(f"{len(skipped)} skipped, no longer on site")
+ if failed:
+ summary.append(f"{len(failed)} failed")
+ job_queue.update(
+ job, result={"new": new_count, "changed": changed, "failed": failed, "skipped": skipped, "removed": removed}
+ )
+
+ metadata_changed = snap.calibre != story["site_values"]
+ delivered = bool(story["calibre_id"] or story["ingested_at"])
+ # A resumed job builds the book, because the run before the restart can have stored chapters.
+ needs_build = (
+ new_count
+ or changed
+ or renamed_count
+ or job.force_fields
+ or metadata_changed
+ or not delivered
+ or job.options.get("build")
+ or job.restarts
+ or mode in (DownloadMode.FULL, DownloadMode.METADATA)
+ or (mode == DownloadMode.NEW and story["status"] == "error" and not failed)
+ )
+ if mode in (DownloadMode.RECHECK_ALL, DownloadMode.FULL):
+ store.set_setting(story_id, "last_full_recheck", now())
+ if not store.chapters_for_build(story_id):
+ raise StoryError("; ".join(summary) + ". No chapters downloaded.")
+ if not needs_build:
+ store.update_story(story_id, last_checked=now(), status="ok", message="No changes")
+ return "No changes. " + ", ".join(summary)
+
+ book = _linked_book(story)
+ pinned = _resolve_metadata(story, snap.calibre, job.force_fields)
+ title = _display_title(story, pinned, snap.title)
+ _progress(job, "Building epub", 1.0)
+ epub = config.TMP_DIR / f"story-{story_id}.epub"
+ _build(story, _epub_overrides(story, pinned), epub)
+ message = _deliver(story, epub, book, title, snap.author)
+ store.update_story(
+ story_id,
+ pinned=pinned,
+ site_values=snap.calibre,
+ title=title,
+ last_checked=now(),
+ last_changed=now(),
+ message=message,
+ status="error" if failed else "ok",
+ )
+ return ", ".join(summary) + ". " + message
+
+
+def _run_job(job: Job) -> str:
+ if job.mode == DownloadMode.IMPORT:
+ return _run_import(job)
+ if job.mode == DownloadMode.REBUILD:
+ return _run_rebuild(job)
+ if job.mode == DownloadMode.IMPORT_FILE:
+ return _run_import_file(job)
+ return _run_site_job(job)
+
+
+def download_loop() -> None:
+ """Background thread for processing the job queue."""
+ logger.info("Starting download loop")
+ while True:
+ job = job_queue.get_next()
+ if not job:
+ time.sleep(2)
+ continue
+ job_queue.update(job, status=QueueStatus.RUNNING, message="Starting")
+ try:
+ message = _run_job(job)
+ job_queue.update(job, status=QueueStatus.DONE, message=message, progress=1.0)
+ logger.info(f"Job {job.id} ({job.mode.value}) for {job.title}: {message}")
+ except (StoryError, CwaError, EpubError, RuleError, KeyError, ValueError) as e:
+ _fail(job, str(e))
+ except Exception as e:
+ logger.exception(f"Job {job.id} failed: {e}")
+ _fail(job, f"{type(e).__name__}: {e}")
+ if job.mode in SITE_MODES:
+ time.sleep(config.SITE_DELAY)
+
+
+def _fail(job: Job, message: str) -> None:
+ logger.warning(f"Job {job.id} ({job.mode.value}) for {job.title} failed: {message}")
+ job_queue.update(job, status=QueueStatus.ERROR, message=message)
+ if job.story_id:
+ store.update_story(job.story_id, status="error", message=message, last_checked=now())
+
+
+def _queue_due_updates() -> None:
+ interval = config.UPDATE_INTERVAL_HOURS
+ if interval <= 0:
+ return
+ due = datetime.now() - timedelta(hours=interval)
+ recheck_days = store.story_settings("recheck_days")
+ last_full = store.story_settings("last_full_recheck")
+ for s in store.list_stories():
+ if not s["auto_update"] or job_queue.has_active(s["id"]):
+ continue
+ if s["last_checked"] and datetime.fromisoformat(s["last_checked"]) > due:
+ continue
+ # A story with a recheck interval gets a full recheck in place of its update when one is due.
+ mode = DownloadMode.NEW
+ days = recheck_days.get(s["id"])
+ last = last_full.get(s["id"])
+ if days and (not last or datetime.fromisoformat(last) <= datetime.now() - timedelta(days=days)):
+ mode = DownloadMode.RECHECK_ALL
+ job_queue.add(Job(mode=mode, story_id=s["id"], title=s["title"]))
+
+
+def scheduler_loop() -> None:
+ logger.info(f"Starting scheduler, update interval {config.UPDATE_INTERVAL_HOURS} hours")
+ while True:
+ try:
+ daily_backup(store, config.BACKUP_DIR, config.PERSONAL_INI, config.BACKUP_COUNT)
+ except Exception as e:
+ logger.exception(f"Backup error: {e}")
+ try:
+ _queue_due_updates()
+ except Exception as e:
+ logger.exception(f"Scheduler error: {e}")
+ time.sleep(60)
+
+
+def start_threads() -> None:
+ for target in (download_loop, scheduler_loop):
+ threading.Thread(target=target, daemon=True, name=target.__name__).start()
diff --git a/tests/test_backup.py b/tests/test_backup.py
new file mode 100644
index 0000000..c494c15
--- /dev/null
+++ b/tests/test_backup.py
@@ -0,0 +1,25 @@
+"""Daily backups of the database and personal.ini."""
+
+import sqlite3
+from datetime import date
+
+from bindery.backup import daily_backup
+from bindery.store import Store
+
+
+def test_one_backup_a_day_and_only_the_newest_are_kept(tmp_path):
+ store = Store(tmp_path / "bindery.db")
+ store.create_story("https://example.com/s/1", "example.com", "T", "A", [])
+ ini = tmp_path / "personal.ini"
+ ini.write_text("[defaults]\n")
+ folder = tmp_path / "backups"
+ for day in range(1, 11):
+ daily_backup(store, folder, ini, 7, date(2026, 9, day))
+ assert daily_backup(store, folder, ini, 7, date(2026, 9, 10)) is None
+ names = sorted(p.name for p in folder.iterdir())
+ assert names == [f"bindery-2026-09-{d:02}.db" for d in range(4, 11)] + [
+ f"personal-2026-09-{d:02}.ini" for d in range(4, 11)
+ ]
+ copy = sqlite3.connect(folder / "bindery-2026-09-10.db")
+ assert copy.execute("SELECT title FROM stories").fetchall() == [("T",)]
+ copy.close()
diff --git a/tests/test_deliver.py b/tests/test_deliver.py
new file mode 100644
index 0000000..0cbf0b5
--- /dev/null
+++ b/tests/test_deliver.py
@@ -0,0 +1,610 @@
+"""Rebuild stored stories, import epub files and hand books to CWA, against a fake CWA. No story site access."""
+
+import dataclasses
+import json
+import re
+import zipfile
+
+import pytest
+from fake_cwa import FakeBook, FakeCwa
+from test_build import COVER, IMG_URL, PNG, URL, chapters, metahtml
+
+from bindery import config, fff, orchestrator
+from bindery.diffing import text_hash
+from bindery.epub import EpubError
+from bindery.fff import SiteSnapshot, StoryError
+from bindery.ingest import IngestFolder
+from bindery.jobs import DownloadMode, Job, JobQueue, QueueStatus
+from bindery.store import Store
+
+STORY_URL = "https://www.royalroad.com/fiction/1234"
+
+
+@pytest.fixture(scope="module")
+def story_epub(tmp_path_factory):
+ path = tmp_path_factory.mktemp("epub") / "story.epub"
+ fff.build_epub(URL, metahtml(), chapters(), [], None, [], {"title": "Library Title"}, path)
+ return path.read_bytes()
+
+
+@pytest.fixture
+def fake(story_epub, monkeypatch, tmp_path):
+ server = FakeCwa()
+ server.books = [FakeBook(1, "uuid-1", "Test Story", publisher="www.royalroad.com", epub=story_epub)]
+ monkeypatch.setattr(config, "CWA_URL", server.url)
+ monkeypatch.setattr(config, "CWA_USERNAME", "bindery")
+ monkeypatch.setattr(config, "CWA_PASSWORD", "secret")
+ monkeypatch.setattr(orchestrator, "store", Store(tmp_path / "bindery.db"))
+ monkeypatch.setattr(orchestrator, "ingest_folder", IngestFolder(tmp_path / "ingest"))
+ monkeypatch.setattr(orchestrator, "finder", orchestrator.BookFinder())
+ yield server
+ server.close()
+
+
+def add_story(**changes):
+ store = orchestrator.store
+ story_id = store.create_story(STORY_URL, "www.royalroad.com", "Test Story", "Writer", ["title"])
+ store.update_story(story_id, metahtml=metahtml(), pinned={"title": "My Title"}, **changes)
+ for n, c in enumerate(chapters()):
+ store.add_chapter(story_id, c["url"], c["title"], c["html"], "h", "site", n)
+ return story_id
+
+
+def ingest_files():
+ folder = orchestrator.ingest_folder.path
+ return sorted(p.name for p in folder.iterdir()) if folder.exists() else []
+
+
+def opf(name):
+ with zipfile.ZipFile(orchestrator.ingest_folder.path / name) as z:
+ return z.read("content.opf").decode()
+
+
+def rebuild(story_id):
+ return orchestrator._run_rebuild(Job(mode=DownloadMode.REBUILD, story_id=story_id))
+
+
+def test_first_version_goes_in_as_a_new_book(fake):
+ fake.books = []
+ story_id = add_story()
+ assert rebuild(story_id) == "Sent to CWA"
+ (name,) = ingest_files()
+ assert re.search(r"
Kept
", "h", "site", 0) + store.add_chapter(story_id, three["url"], "Three", "Kept
", "h", "site", 1) + assert import_file(jobs, upload(tmp_path, story_epub), story_id) == "Imported 1 chapter from story.epub" + assert titles(story_id) == ["One", "Two", "Three"] + assert store.chapters_for_build(story_id)[0]["html"] == "Kept
" + + +def test_next_update_adds_newer_chapters_after_the_imported_ones(fake, jobs, story_epub, tmp_path, monkeypatch): + # The site stubbed the first two chapters and added a fourth after the epub was made. + fake.books = [] + import_file(jobs, upload(tmp_path, story_epub)) + four = chapters()[2]["url"].replace("15/c15", "16/c16") + site = [{"url": chapters()[2]["url"], "title": "Three"}, {"url": four, "title": "Four"}] + monkeypatch.setattr( + fff, + "fetch_story", + lambda adapter: SiteSnapshot(STORY_URL, "www.royalroad.com", "Test Story", "Writer", metahtml(), site, {}), + ) + monkeypatch.setattr(fff, "fetch_chapter", lambda adapter, url, index: "Four
Kept
", "h", "site", 0) + assert import_file(jobs, upload(tmp_path, story_epub), story_id, check=False).startswith("Imported 2") + rebuild_job = jobs.get_next() + assert (rebuild_job.mode, rebuild_job.story_id) == (DownloadMode.REBUILD, story_id) + assert "library book 1" in orchestrator._run_rebuild(rebuild_job) + + +def test_epub_of_another_story_is_refused(fake, jobs, story_epub, tmp_path): + other = orchestrator.store.create_story( + "https://www.royalroad.com/fiction/99", "www.royalroad.com", "Other", "Writer", [] + ) + path = upload(tmp_path, story_epub) + with pytest.raises(StoryError, match="story.epub is an epub of another story"): + orchestrator.queue_epub_file(path, "story.epub", other) + assert not path.exists() + assert jobs.get_next() is None + + +def test_file_that_is_not_an_epub_is_refused(fake, jobs, tmp_path): + path = upload(tmp_path, b"not a zip file") + with pytest.raises(EpubError, match="story.epub is not a readable epub"): + orchestrator.queue_epub_file(path, "story.epub") + assert not path.exists() + + +# Updates from the site + +CHAPTER = "https://www.royalroad.com/fiction/1234/test-story/chapter/{0}/c{0}" + + +def linked_story(chapter_list): + """A story in CWA with the chapters (number, title, text).""" + store = orchestrator.store + story_id = store.create_story(STORY_URL, "www.royalroad.com", "Test Story", "Writer", ["title"], 1, "uuid-1") + store.update_story(story_id, metahtml=metahtml(), ingested_at="2026-01-01T00:00:00") + for n, (number, title, text, *source) in enumerate(chapter_list): + html = f"{text}
{texts[url]}
Zorian wakes.
1
", "h", "site", 0) + four = three["url"].replace("15/c15", "16/c16") + site = [one, two, three, {"url": four, "title": "Four"}] + image = {"url": IMG_URL, "ext": "png", "mime": "image/png", "data": PNG} + monkeypatch.setattr( + fff, + "fetch_story", + lambda adapter: SiteSnapshot(STORY_URL, "www.royalroad.com", "Test Story", "Writer", metahtml(), site, {}), + ) + monkeypatch.setattr(fff, "collected_images", lambda adapter: [image]) + crash_at = [three["url"]] + + def fetch(adapter, url, index): + if url in crash_at: + raise Crash + return f"{url}
New blurb
", + "tags": ["Fantasy", "Completed"], + "series": "Saga", + "series_index": 2.0, + "pubdate": "2020-01-01T00:00:00", + "languages": "en", +} + + +def story(**extra): + base = {"locked_fields": ["title"], "pinned": {}, "site_values": {}} + base.update(extra) + return base + + +def test_first_download_keeps_site_value(): + assert backend._resolve_metadata(story(), SITE, []) == {"title": "Site Title"} + + +def test_locked_field_keeps_its_value(): + assert backend._resolve_metadata(story(pinned={"title": "Old"}), SITE, []) == {"title": "Old"} + + +def test_forced_field_takes_site_value_and_stays_locked(): + s = story(pinned={"title": "Old"}) + assert backend._resolve_metadata(s, SITE, ["title"]) == {"title": "Site Title"} + + +def test_edited_unlocked_field_stays_until_the_site_changes_it(): + s = story(pinned={"title": "Old", "tags": ["mine"]}, site_values=SITE) + assert backend._resolve_metadata(s, SITE, []) == {"title": "Old", "tags": ["mine"]} + changed = {**SITE, "tags": ["Fantasy", "Completed", "LitRPG"]} + assert backend._resolve_metadata(s, changed, []) == {"title": "Old"} + # A change to another field keeps the edit. + assert backend._resolve_metadata(s, {**SITE, "comments": "x
"}, []) == {"title": "Old", "tags": ["mine"]} + + +def test_forced_update_replaces_an_edited_unlocked_field(): + s = story(pinned={"tags": ["mine"]}, site_values=SITE) + assert backend._resolve_metadata(s, SITE, ["tags"]) == {"title": "Site Title"} + + +def test_edit_before_the_first_site_values_stays(): + s = story(pinned={"tags": ["mine"]}) + assert backend._resolve_metadata(s, SITE, []) == {"title": "Site Title", "tags": ["mine"]} + + +def test_edited_values_are_checked_and_normalized(): + assert backend._edited_value("authors", "A & B ") == ["A", "B"] + assert backend._edited_value("tags", "x, y,,") == ["x", "y"] + assert backend._edited_value("series", "Saga [2]") == ["Saga", 2.0] + assert backend._edited_value("series", ["Saga", "3"]) == ["Saga", 3.0] + assert backend._edited_value("series", "") == ["", None] + assert backend._edited_value("pubdate", "2024-01-31") == "2024-01-31" + assert backend._edited_value("title", " New ") == "New" + for field, value in (("title", " "), ("authors", "&"), ("pubdate", "soon")): + with pytest.raises(ValueError): + backend._edited_value(field, value) + + +def test_series_keeps_name_and_index(): + s = story(locked_fields=["series"]) + assert backend._resolve_metadata(s, SITE, []) == {"series": ["Saga", 2.0]} + + +def test_resolve_without_site_values_does_not_fail(): + assert backend._resolve_metadata(story(pinned={"title": "Old"}), {}, []) == {"title": "Old"} diff --git a/tests/test_worker_flows.py b/tests/test_worker_flows.py new file mode 100644 index 0000000..ec1c66b --- /dev/null +++ b/tests/test_worker_flows.py @@ -0,0 +1,267 @@ +"""Worker and scheduler transitions, with real temporary stores and no background threads.""" + +from datetime import datetime, timedelta +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from bindery import config, fff +from bindery import orchestrator as o +from bindery.diffing import text_hash +from bindery.jobs import DownloadMode, Job, JobQueue, QueueStatus +from bindery.store import Store + + +class StopLoop(BaseException): + """Stop an infinite loop without entering its application error handler.""" + + +@pytest.fixture +def state(monkeypatch, tmp_path): + store = Store(tmp_path / "stories.db") + queue = JobQueue(tmp_path / "jobs.db") + monkeypatch.setattr(o, "store", store) + monkeypatch.setattr(o, "job_queue", queue) + monkeypatch.setattr(config, "CWA_URL", "") + monkeypatch.setattr(config, "SITE_DELAY", 7) + return store, queue + + +def add_story(store, url="https://www.royalroad.com/fiction/1234", **values): + sid = store.create_story(url, "www.royalroad.com", "Title", "Writer", []) + store.update_story(sid, **values) + return sid + + +@pytest.mark.parametrize("mode", list(DownloadMode)) +def test_dispatches_each_download_mode(state, monkeypatch, mode): + handlers = { + name: Mock(return_value=name) for name in ("_run_import", "_run_import_file", "_run_rebuild", "_run_site_job") + } + for name, handler in handlers.items(): + monkeypatch.setattr(o, name, handler) + expected = { + DownloadMode.IMPORT: "_run_import", + DownloadMode.IMPORT_FILE: "_run_import_file", + DownloadMode.REBUILD: "_run_rebuild", + }.get(mode, "_run_site_job") + job = Job(mode=mode) + assert o._run_job(job) == expected + handlers[expected].assert_called_once_with(job) + assert sum(h.call_count for h in handlers.values()) == 1 + + +@pytest.mark.parametrize("error", [None, o.StoryError("site refused"), RuntimeError("broken")]) +@pytest.mark.parametrize("with_story", [False, True]) +def test_worker_persists_outcomes_and_continues_after_failure(state, monkeypatch, tmp_path, error, with_story): + store, queue = state + sid = add_story(store) if with_story else None + first = queue.add(Job(mode=DownloadMode.NEW, story_id=sid, url="first")) + second = queue.add(Job(mode=DownloadMode.REBUILD, story_id=sid)) + monkeypatch.setattr(queue, "get_next", Mock(side_effect=[None, first, second, StopLoop()])) + sleeps = Mock() + monkeypatch.setattr(o.time, "sleep", sleeps) + + def run(job): + assert job.status == QueueStatus.RUNNING + if job is first and error: + raise error + return "Sent" + + monkeypatch.setattr(o, "_run_job", run) + with pytest.raises(StopLoop): + o.download_loop() + saved = {j["id"]: j for j in JobQueue(tmp_path / "jobs.db").get_status()} + assert saved[second.id]["status"] == "done" + assert saved[second.id]["progress"] == 1.0 + assert saved[first.id]["status"] == ("error" if error else "done") + expected = "RuntimeError: broken" if isinstance(error, RuntimeError) else str(error) if error else "Sent" + assert saved[first.id]["message"] == expected + if error and sid: + story = store.get_story(sid) + assert (story["status"], story["message"]) == ("error", expected) + assert story["last_checked"] + assert [c.args[0] for c in sleeps.call_args_list] == [2, 7] + + +def test_scheduler_skips_disabled_busy_and_recent_stories(state, monkeypatch): + store, queue = state + now = datetime(2026, 6, 1, 12) + + class Clock(datetime): + @classmethod + def now(cls): + return now + + monkeypatch.setattr(o, "datetime", Clock) + monkeypatch.setattr(config, "UPDATE_INTERVAL_HOURS", 2) + ids = [add_story(store, f"url-{n}") for n in range(7)] + store.update_story(ids[0], auto_update=0) + queue.add(Job(mode=DownloadMode.NEW, story_id=ids[1])) + store.update_story(ids[2], last_checked=(now - timedelta(hours=1)).isoformat()) + store.update_story(ids[3], last_checked=(now - timedelta(hours=2)).isoformat()) + for sid in ids[4:]: + store.set_setting(sid, "recheck_days", 3) + store.set_setting(ids[5], "last_full_recheck", (now - timedelta(days=3)).isoformat()) + store.set_setting(ids[6], "last_full_recheck", (now - timedelta(days=1)).isoformat()) + o._queue_due_updates() + queued = {j["story_id"]: j["mode"] for j in queue.get_status()} + assert queued == {ids[1]: "new", ids[3]: "new", ids[4]: "recheck_all", ids[5]: "recheck_all", ids[6]: "new"} + o._queue_due_updates() + assert len(queue.get_status()) == 5 + monkeypatch.setattr(config, "UPDATE_INTERVAL_HOURS", 0) + extra = add_story(store, "extra") + o._queue_due_updates() + assert not queue.has_active(extra) + + +def test_scheduler_recovers_from_backup_and_queue_errors(state, monkeypatch): + backup = Mock(side_effect=[OSError("disk full"), None]) + due = Mock(side_effect=[ValueError("bad date"), None]) + sleep = Mock(side_effect=[None, StopLoop()]) + monkeypatch.setattr(o, "daily_backup", backup) + monkeypatch.setattr(o, "_queue_due_updates", due) + monkeypatch.setattr(o.time, "sleep", sleep) + with pytest.raises(StopLoop): + o.scheduler_loop() + assert backup.call_count == due.call_count == 2 + assert [c.args for c in sleep.call_args_list] == [(60,), (60,)] + + +@pytest.fixture +def site(state, monkeypatch): + store, queue = state + sid = add_story(store, ingested_at="2026-01-01", site_values={"title": "Title"}) + for i in range(1, 4): + html = f"old {i}
" + store.add_chapter(sid, f"chapter-{i}", f"Chapter {i}", html, text_hash(html), "new", i) + adapter = SimpleNamespace(normalize_chapterurl=lambda url: url) + snap = fff.SiteSnapshot( + url=store.get_story(sid)["url"], + site="www.royalroad.com", + title="Title", + author="Writer", + metahtml="metadata", + chapters=[ + {"url": "chapter-1", "title": "Renamed"}, + {"url": "chapter-2", "title": "Chapter 2"}, + {"url": "chapter-4", "title": "New"}, + ], + calibre={"title": "Title"}, + ) + fetch = Mock(side_effect=lambda a, url, i: f"fresh {url}
") + build, deliver = Mock(), Mock(return_value="Sent") + monkeypatch.setattr(fff, "get_adapter", lambda url: adapter) + monkeypatch.setattr(fff, "fetch_story", lambda a: snap) + monkeypatch.setattr(fff, "fetch_chapter", fetch) + monkeypatch.setattr(fff, "collected_images", lambda a: []) + monkeypatch.setattr(o, "_build", build) + monkeypatch.setattr(o, "_linked_book", lambda story: None) + monkeypatch.setattr(o, "_deliver", deliver) + return SimpleNamespace( + store=store, queue=queue, id=sid, snap=snap, adapter=adapter, fetch=fetch, build=build, deliver=deliver + ) + + +@pytest.mark.parametrize("mode", [DownloadMode.METADATA, DownloadMode.FULL, DownloadMode.RECHECK]) +def test_download_modes_preserve_history_and_select_the_right_chapters(site, mode): + original = site.store.list_chapters(site.id) + job = Job(mode=mode, story_id=site.id, chapter_ids=[original[1]["id"], original[2]["id"]]) + result = o._run_site_job(job) + chapters = site.store.list_chapters(site.id) + assert chapters[0]["title"] == "Renamed" + assert all(site.store.revision_html(c["id"], 1) == f"old {n}
" for n, c in enumerate(original, 1)) + fetched = [call.args[1] for call in site.fetch.call_args_list] + removed = site.store.get_chapter(original[2]["id"]) + assert not removed["on_site"] + if mode == DownloadMode.METADATA: + assert fetched == [] + assert len(chapters) == 3 + assert removed["included"] + elif mode == DownloadMode.FULL: + assert set(fetched) == {"chapter-1", "chapter-2", "chapter-4"} + assert not removed["included"] + assert "1 taken out" in result + assert site.store.get_setting(site.id, "last_full_recheck") + else: + assert set(fetched) == {"chapter-1", "chapter-2"} + assert job.result["skipped"] == ["Chapter 3"] + assert removed["included"] + site.build.assert_called_once() + site.deliver.assert_called_once() + + +def test_failed_new_chapter_keeps_successful_chapters_and_images(site, monkeypatch): + site.snap.chapters.append({"url": "chapter-5", "title": "Broken"}) + + def fetch(adapter, url, index): + if url == "chapter-5": + raise OSError("timeout") + return "ok
" + + site.fetch.side_effect = fetch + image = {"url": "image", "ext": "png", "mime": "image/png", "data": b"image"} + site.snap.cover = {**image, "url": "cover"} + monkeypatch.setattr(fff, "collected_images", lambda a: [image, image]) + job = Job(mode=DownloadMode.NEW, story_id=site.id) + assert "1 failed" in o._run_site_job(job) + assert job.result["failed"] == ["Broken: timeout"] + assert len(site.store.list_chapters(site.id)) == 4 + assert site.store.get_story(site.id)["status"] == "error" + assert site.store.image_urls(site.id) == {"image"} + assert site.store.cover(site.id)["url"] == "cover" + + +def test_new_story_with_no_downloaded_chapters_fails_without_delivery(site): + site.fetch.side_effect = OSError("offline") + job = Job(mode=DownloadMode.NEW, url="new-url") + site.snap.url = "new-url" + with pytest.raises(o.StoryError, match="No chapters downloaded"): + o._run_site_job(job) + assert job.story_id != site.id + assert site.store.list_chapters(job.story_id) == [] + site.deliver.assert_not_called() + + +def test_normalized_chapter_urls_keep_the_existing_identity(site): + original = site.store.list_chapters(site.id)[0] + site.adapter.normalize_chapterurl = lambda url: "renamed-url" if url == "chapter-1" else url + site.snap.chapters[0]["url"] = "renamed-url" + o._run_site_job(Job(mode=DownloadMode.RECHECK_ALL, story_id=site.id)) + updated = site.store.get_chapter(original["id"]) + assert updated["url"] == "renamed-url" + assert updated["current_rev"] == 2 + assert len(site.store.list_chapters(site.id)) == 4 + + +def test_inclusion_changes_only_the_requested_story_and_queues_a_rebuild(state): + store, queue = state + sid, other = add_story(store), add_story(store, "other") + cid = store.add_chapter(sid, "chapter", "One", "text
", "h", "new", 0) + foreign = store.add_chapter(other, "foreign", "Other", "other
", "h", "new", 0) + result = o.set_chapters_included(sid, [cid, foreign, 999], False) + assert result["mode"] == "rebuild" + assert not store.get_chapter(cid)["included"] + assert store.get_chapter(foreign)["included"] + assert store.revision_html(cid, 1) == "text
" + o.set_chapters_included(sid, [cid], True) + assert store.get_chapter(cid)["included"] + assert len(queue.get_status()) == 1 + + +@pytest.mark.parametrize( + "mode,chapters,fields,message", + [ + ("import", [], [], "library import"), + ("recheck", [], [], "at least one chapter"), + ("new", [], ["unknown"], "Unknown metadata"), + ("invalid", [], [], "not a valid"), + ], +) +def test_invalid_job_requests_do_not_enter_the_queue(state, mode, chapters, fields, message): + store, queue = state + sid = add_story(store) + with pytest.raises(ValueError, match=message): + o.queue_story(sid, mode, chapters, fields) + assert queue.get_status() == []