tim/bindery
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Run download jobs and deliver books to CWA

Browse Source 
Jobs download new chapters, recheck chapters, replace the book, update
metadata or rebuild from stored chapters. Locked metadata fields keep
their values. A scheduler queues updates. An update finds the library
book of a story through CWA, and fails with an error if it cannot, so
that it never makes a duplicate book.
This commit is contained in:
tim
2026-09-27 23:38:28 +02:00
parent 312dc3efc5
commit 49cd18a09f
8 changed files with 2581 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bindery/backup.py
+40
View File
@@ -0,0 +1,40 @@
"""Daily copies of the database and personal.ini.
The database holds chapters that the sites removed, so it is the only copy of
them. A backup a day, kept for a week, limits a bad change or a broken disk to
the loss of one day.
"""
import logging
import shutil
from datetime import date
from pathlib import Path
from bindery.store import Store
logger = logging.getLogger(__name__)
def daily_backup(store: Store, folder: Path, personal_ini: Path, keep: int, today: date | None = None) -> Path | None:
"""Make the backup of today if there is none yet, then delete all but the newest `keep` backups.
Return the path of the new database copy, or None if today had one already.
"""
today = today or date.today()
folder.mkdir(parents=True, exist_ok=True)
target = folder / f"bindery-{today.isoformat()}.db"
made = None
if not target.exists():
# A copy under another name first, so that a stop never leaves half a backup.
partial = folder / f"{target.name}.part"
partial.unlink(missing_ok=True)
store.backup(partial)
partial.replace(target)
if personal_ini.exists():
shutil.copy2(personal_ini, folder / f"personal-{today.isoformat()}.ini")
logger.info(f"Backed up the database to {target}")
made = target
for pattern in ("bindery-*.db", "personal-*.ini"):
for old in sorted(folder.glob(pattern))[:-keep]:
old.unlink()
return made
bindery/jobs.py
+239
View File
@@ -0,0 +1,239 @@
"""Download modes, jobs and the job queue."""
import json
import sqlite3
import uuid
from collections import deque
from dataclasses import asdict, dataclass, field
from datetime import datetime, timedelta
from enum import StrEnum
from pathlib import Path
from threading import Lock
from typing import Any
from bindery import config
# Calibre fields that a story can lock. A locked field keeps its current
# value until a download explicitly forces an update of that field.
METADATA_FIELDS: dict[str, str] = {
"title": "Title",
"authors": "Authors",
"comments": "Description",
"tags": "Tags",
"series": "Series",
"pubdate": "Published date",
"languages": "Language",
}
class QueueStatus(StrEnum):
"""Enum for possible job statuses."""
QUEUED = "queued"
RUNNING = "running"
DONE = "done"
ERROR = "error"
CANCELLED = "cancelled"
class DownloadMode(StrEnum):
"""What a job fetches from the site and how it changes the book."""
NEW = "new" # new chapters only, appended after the chapters we have
RECHECK = "recheck" # download selected chapters again and keep changes
RECHECK_ALL = "recheck_all" # download every chapter still on the site again
FULL = "full" # replace the whole book with what is on the site now
METADATA = "metadata" # update metadata only, no chapter changes
REBUILD = "rebuild" # no site access: rebuild the epub from stored chapters
IMPORT = "import" # start tracking a book that is already in the library
IMPORT_FILE = "import_file" # add the chapters of an uploaded epub to a story
MODE_LABELS: dict[str, str] = {
DownloadMode.NEW.value: "Download new chapters (default)",
DownloadMode.RECHECK.value: "Recheck selected chapters",
DownloadMode.RECHECK_ALL.value: "Recheck all chapters",
DownloadMode.FULL.value: "Download and replace entire book",
DownloadMode.METADATA.value: "Update metadata from site",
DownloadMode.REBUILD.value: "Rebuild book from stored chapters",
DownloadMode.IMPORT.value: "Import from library",
DownloadMode.IMPORT_FILE.value: "Import chapters from an epub file",
}
# A job that was running when Bindery stopped this many times is not started again.
MAX_RESTARTS = 3
@dataclass
class Job:
"""One unit of work for the download thread."""
mode: DownloadMode
story_id: int | None = None
url: str | None = None
calibre_id: int | None = None
chapter_ids: list[int] = field(default_factory=list)
force_fields: list[str] = field(default_factory=list)
options: dict[str, Any] = field(default_factory=dict)
id: str = field(default_factory=lambda: uuid.uuid4().hex[:12])
status: QueueStatus = QueueStatus.QUEUED
title: str = ""
message: str = ""
progress: float = 0.0
created: datetime = field(default_factory=datetime.now)
updated: datetime = field(default_factory=datetime.now)
result: dict[str, Any] = field(default_factory=dict)
restarts: int = 0
def to_dict(self) -> dict[str, Any]:
d = asdict(self)
d["mode"] = self.mode.value
d["mode_label"] = MODE_LABELS[self.mode.value]
d["status"] = self.status.value
d["created"] = self.created.isoformat(timespec="seconds")
d["updated"] = self.updated.isoformat(timespec="seconds")
return d
def to_record(self) -> str:
d = asdict(self)
d["created"] = self.created.isoformat()
d["updated"] = self.updated.isoformat()
return json.dumps(d)
@classmethod
def from_record(cls, record: str) -> "Job":
d = json.loads(record)
d["mode"] = DownloadMode(d["mode"])
d["status"] = QueueStatus(d["status"])
d["created"] = datetime.fromisoformat(d["created"])
d["updated"] = datetime.fromisoformat(d["updated"])
return cls(**d)
class JobQueue:
"""Thread-safe FIFO job queue that also keeps recent job history.
With a database path, the queue keeps each job in SQLite, so the jobs outlive
a restart. A job that was running goes back to the front of the queue.
"""
def __init__(self, path: Path | None = None) -> None:
self._lock = Lock()
self._pending: deque[str] = deque()
self._jobs: dict[str, Job] = {}
self._history = timedelta(seconds=config.JOB_HISTORY)
self._db: sqlite3.Connection | None = None
if path:
self._load(Path(path))
def _load(self, path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
# One connection, used only while the lock is held. Each statement commits.
self._db = sqlite3.connect(path, timeout=30, check_same_thread=False, isolation_level=None)
self._db.execute("PRAGMA journal_mode=WAL")
self._db.execute("CREATE TABLE IF NOT EXISTS jobs (id TEXT PRIMARY KEY, record TEXT NOT NULL)")
jobs = [Job.from_record(r[0]) for r in self._db.execute("SELECT record FROM jobs ORDER BY rowid")]
interrupted = [j for j in jobs if j.status == QueueStatus.RUNNING]
for job in interrupted:
job.restarts += 1
if job.restarts >= MAX_RESTARTS:
job.status = QueueStatus.ERROR
job.message = f"Stopped {job.restarts} times, giving up"
else:
job.status = QueueStatus.QUEUED
job.message = "Resumed"
job.updated = datetime.now()
self._save(job)
self._jobs = {j.id: j for j in jobs}
waiting = [
j.id for j in interrupted + [j for j in jobs if j not in interrupted] if j.status == QueueStatus.QUEUED
]
self._pending.extend(waiting)
self._expire()
def _save(self, job: Job) -> None:
if self._db:
self._db.execute(
"INSERT INTO jobs (id, record) VALUES (?, ?) ON CONFLICT (id) DO UPDATE SET record = excluded.record",
(job.id, job.to_record()),
)
def _expire(self) -> None:
"""Forget finished jobs after JOB_HISTORY seconds."""
now = datetime.now()
expired = [
k
for k, j in self._jobs.items()
if j.status not in (QueueStatus.QUEUED, QueueStatus.RUNNING) and now - j.updated > self._history
]
for job_id in expired:
del self._jobs[job_id]
if self._db and expired:
self._db.executemany("DELETE FROM jobs WHERE id = ?", [(k,) for k in expired])
def add(self, job: Job) -> Job:
"""Queue a job. If the same story already waits for the same mode, return that job."""
with self._lock:
for job_id in self._pending:
other = self._jobs[job_id]
if (
other.mode == job.mode
and other.chapter_ids == job.chapter_ids
and other.force_fields == job.force_fields
and other.options == job.options
and (
(job.story_id and other.story_id == job.story_id)
or (job.url and other.url == job.url)
or (job.calibre_id and other.calibre_id == job.calibre_id)
)
):
return other
self._expire()
self._jobs[job.id] = job
self._pending.append(job.id)
self._save(job)
return job
def get_next(self) -> Job | None:
with self._lock:
while self._pending:
job = self._jobs[self._pending.popleft()]
if job.status == QueueStatus.QUEUED:
return job
return None
def update(self, job: Job, **changes: Any) -> None:
with self._lock:
for key, value in changes.items():
setattr(job, key, value)
job.updated = datetime.now()
self._save(job)
def cancel(self, job_id: str) -> bool:
with self._lock:
job = self._jobs.get(job_id)
if not job or job.status != QueueStatus.QUEUED:
return False
job.status = QueueStatus.CANCELLED
job.updated = datetime.now()
# A cancelled job must not be shared, and it can expire before the download thread reaches it.
self._pending.remove(job_id)
self._save(job)
return True
def has_active(self, story_id: int) -> bool:
with self._lock:
return any(
j.story_id == story_id and j.status in (QueueStatus.QUEUED, QueueStatus.RUNNING)
for j in self._jobs.values()
)
def get_status(self, story_id: int | None = None) -> list[dict[str, Any]]:
"""Return recent jobs, newest first."""
with self._lock:
self._expire()
jobs = [j for j in self._jobs.values() if story_id is None or j.story_id == story_id]
return [j.to_dict() for j in sorted(jobs, key=lambda j: j.created, reverse=True)]
job_queue = JobQueue(config.DB_PATH)
bindery/orchestrator.py
+1163
View File
File diff suppressed because it is too large Load Diff
tests/test_backup.py
+25
View File
@@ -0,0 +1,25 @@
"""Daily backups of the database and personal.ini."""
import sqlite3
from datetime import date
from bindery.backup import daily_backup
from bindery.store import Store
def test_one_backup_a_day_and_only_the_newest_are_kept(tmp_path):
store = Store(tmp_path / "bindery.db")
store.create_story("https://example.com/s/1", "example.com", "T", "A", [])
ini = tmp_path / "personal.ini"
ini.write_text("[defaults]\n")
folder = tmp_path / "backups"
for day in range(1, 11):
daily_backup(store, folder, ini, 7, date(2026, 9, day))
assert daily_backup(store, folder, ini, 7, date(2026, 9, 10)) is None
names = sorted(p.name for p in folder.iterdir())
assert names == [f"bindery-2026-09-{d:02}.db" for d in range(4, 11)] + [
f"personal-2026-09-{d:02}.ini" for d in range(4, 11)
]
copy = sqlite3.connect(folder / "bindery-2026-09-10.db")
assert copy.execute("SELECT title FROM stories").fetchall() == [("T",)]
copy.close()
tests/test_deliver.py
+610
View File
@@ -0,0 +1,610 @@
"""Rebuild stored stories, import epub files and hand books to CWA, against a fake CWA. No story site access."""
import dataclasses
import json
import re
import zipfile
import pytest
from fake_cwa import FakeBook, FakeCwa
from test_build import COVER, IMG_URL, PNG, URL, chapters, metahtml
from bindery import config, fff, orchestrator
from bindery.diffing import text_hash
from bindery.epub import EpubError
from bindery.fff import SiteSnapshot, StoryError
from bindery.ingest import IngestFolder
from bindery.jobs import DownloadMode, Job, JobQueue, QueueStatus
from bindery.store import Store
STORY_URL = "https://www.royalroad.com/fiction/1234"
@pytest.fixture(scope="module")
def story_epub(tmp_path_factory):
path = tmp_path_factory.mktemp("epub") / "story.epub"
fff.build_epub(URL, metahtml(), chapters(), [], None, [], {"title": "Library Title"}, path)
return path.read_bytes()
@pytest.fixture
def fake(story_epub, monkeypatch, tmp_path):
server = FakeCwa()
server.books = [FakeBook(1, "uuid-1", "Test Story", publisher="www.royalroad.com", epub=story_epub)]
monkeypatch.setattr(config, "CWA_URL", server.url)
monkeypatch.setattr(config, "CWA_USERNAME", "bindery")
monkeypatch.setattr(config, "CWA_PASSWORD", "secret")
monkeypatch.setattr(orchestrator, "store", Store(tmp_path / "bindery.db"))
monkeypatch.setattr(orchestrator, "ingest_folder", IngestFolder(tmp_path / "ingest"))
monkeypatch.setattr(orchestrator, "finder", orchestrator.BookFinder())
yield server
server.close()
def add_story(**changes):
store = orchestrator.store
story_id = store.create_story(STORY_URL, "www.royalroad.com", "Test Story", "Writer", ["title"])
store.update_story(story_id, metahtml=metahtml(), pinned={"title": "My Title"}, **changes)
for n, c in enumerate(chapters()):
store.add_chapter(story_id, c["url"], c["title"], c["html"], "h", "site", n)
return story_id
def ingest_files():
folder = orchestrator.ingest_folder.path
return sorted(p.name for p in folder.iterdir()) if folder.exists() else []
def opf(name):
with zipfile.ZipFile(orchestrator.ingest_folder.path / name) as z:
return z.read("content.opf").decode()
def rebuild(story_id):
return orchestrator._run_rebuild(Job(mode=DownloadMode.REBUILD, story_id=story_id))
def test_first_version_goes_in_as_a_new_book(fake):
fake.books = []
story_id = add_story()
assert rebuild(story_id) == "Sent to CWA"
(name,) = ingest_files()
assert re.search(r"<dc:title[^>]*>My Title</dc:title>", opf(name))
assert orchestrator.store.get_story(story_id)["ingested_at"]
def test_ingested_story_is_found_and_updated_with_a_sidecar(fake):
story_id = add_story(ingested_at="2026-01-01T00:00:00")
assert "library book 1" in rebuild(story_id)
story = orchestrator.store.get_story(story_id)
assert (story["calibre_id"], story["calibre_uuid"]) == (1, "uuid-1")
epub, manifest = ingest_files()
assert manifest == epub + ".cwa.json"
assert json.loads((orchestrator.ingest_folder.path / manifest).read_text()) == {
"action": "add_format",
"book_id": 1,
}
def test_update_fails_while_cwa_has_no_such_book(fake):
fake.books = []
story_id = add_story(ingested_at="2026-01-01T00:00:00")
with pytest.raises(orchestrator.StoryError, match="Cannot find this story in CWA"):
rebuild(story_id)
assert ingest_files() == [] # a second plain epub would make a duplicate book
def test_update_fails_without_cwa_url(fake, monkeypatch):
monkeypatch.setattr(config, "CWA_URL", "")
story_id = add_story(ingested_at="2026-01-01T00:00:00")
with pytest.raises(orchestrator.StoryError, match="CWA_URL is not set"):
rebuild(story_id)
assert ingest_files() == []
def test_linked_story_still_updates_when_cwa_is_down(fake):
story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=1, calibre_uuid="uuid-1")
fake.close()
assert "library book 1" in rebuild(story_id)
assert len(ingest_files()) == 2
def test_deleted_book_is_unlinked_and_the_update_fails(fake):
story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=5, calibre_uuid="uuid-gone")
fake.books = []
with pytest.raises(orchestrator.StoryError, match="Cannot find this story in CWA"):
rebuild(story_id)
assert orchestrator.store.get_story(story_id)["calibre_id"] is None
def test_unlinked_story_can_go_to_cwa_again_as_a_new_book(fake):
fake.books = []
story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=5, calibre_uuid="uuid-gone")
orchestrator.link_story(story_id, None)
assert rebuild(story_id) == "Sent to CWA"
assert len(ingest_files()) == 1
def test_import_from_the_library(fake):
job = Job(mode=DownloadMode.IMPORT, calibre_id=1, title="Test Story", options={"uuid": "uuid-1", "check": False})
assert orchestrator._run_import(job) == "Imported 3 chapters"
story = orchestrator.store.get_story(job.story_id)
assert story["url"] == STORY_URL
assert (story["calibre_id"], story["calibre_uuid"]) == (1, "uuid-1")
assert story["pinned"] == {"title": "Library Title"}
assert [c["title"] for c in orchestrator.store.list_chapters(job.story_id)] == ["One", "Two", "Three"]
assert orchestrator._run_import(job) == "Book already tracked"
assert not list(config.TMP_DIR.glob("import-*"))
def test_library_candidates_and_search(fake):
candidates = orchestrator.library_candidates()
assert candidates == [
{"calibre_id": 1, "uuid": "uuid-1", "title": "Test Story", "authors": "Writer", "site": "www.royalroad.com"}
]
story_id = add_story()
orchestrator.link_story(story_id, "uuid-1")
assert orchestrator.library_candidates() == []
assert orchestrator.search_library("test")[0]["tracked_as"] == "Test Story"
with pytest.raises(KeyError):
orchestrator.link_story(story_id, "uuid-nope")
# Import chapters from an epub file
@pytest.fixture
def jobs(monkeypatch):
queue = JobQueue()
monkeypatch.setattr(orchestrator, "job_queue", queue)
return queue
def upload(tmp_path, data):
path = tmp_path / "upload-1.epub"
path.write_bytes(data)
return path
def import_file(jobs, path, story_id=None, check=True):
queued = orchestrator.queue_epub_file(path, "story.epub", story_id, check)
job = jobs.get_next()
assert job.id == queued["id"]
return orchestrator._run_import_file(job)
def titles(story_id):
return [c["title"] for c in orchestrator.store.list_chapters(story_id)]
def test_epub_file_starts_a_new_story(fake, jobs, story_epub, tmp_path):
path = upload(tmp_path, story_epub)
assert import_file(jobs, path) == "Imported 3 chapters from story.epub"
assert not path.exists()
story = orchestrator.store.get_story_by_url(STORY_URL)
assert story["title"] == "Library Title"
assert story["pinned"] == {"title": "Library Title"}
assert titles(story["id"]) == ["One", "Two", "Three"]
check = jobs.get_next()
assert (check.mode, check.story_id, check.options) == (DownloadMode.NEW, story["id"], {"build": True})
def test_epub_file_fills_in_a_stubbed_story(fake, jobs, story_epub, tmp_path):
store = orchestrator.store
story_id = store.create_story(STORY_URL, "www.royalroad.com", "Test Story", "Writer", ["title"])
one, _, three = chapters()
store.add_chapter(story_id, one["url"], "One", "<p>Kept</p>", "h", "site", 0)
store.add_chapter(story_id, three["url"], "Three", "<p>Kept</p>", "h", "site", 1)
assert import_file(jobs, upload(tmp_path, story_epub), story_id) == "Imported 1 chapter from story.epub"
assert titles(story_id) == ["One", "Two", "Three"]
assert store.chapters_for_build(story_id)[0]["html"] == "<p>Kept</p>"
def test_next_update_adds_newer_chapters_after_the_imported_ones(fake, jobs, story_epub, tmp_path, monkeypatch):
# The site stubbed the first two chapters and added a fourth after the epub was made.
fake.books = []
import_file(jobs, upload(tmp_path, story_epub))
four = chapters()[2]["url"].replace("15/c15", "16/c16")
site = [{"url": chapters()[2]["url"], "title": "Three"}, {"url": four, "title": "Four"}]
monkeypatch.setattr(
fff,
"fetch_story",
lambda adapter: SiteSnapshot(STORY_URL, "www.royalroad.com", "Test Story", "Writer", metahtml(), site, {}),
)
monkeypatch.setattr(fff, "fetch_chapter", lambda adapter, url, index: "<div><p>Four</p></div>")
monkeypatch.setattr(fff, "collected_images", lambda adapter: [])
check = jobs.get_next()
assert orchestrator._run_site_job(check).startswith("1 new chapter")
assert titles(check.story_id) == ["One", "Two", "Three", "Four"]
assert len(ingest_files()) == 1
def test_epub_file_without_a_check_rebuilds_a_downloaded_story(fake, jobs, story_epub, tmp_path):
store = orchestrator.store
story_id = store.create_story(STORY_URL, "www.royalroad.com", "Test Story", "Writer", ["title"])
store.update_story(story_id, metahtml=metahtml(), ingested_at="2026-01-01T00:00:00")
store.add_chapter(story_id, chapters()[0]["url"], "One", "<p>Kept</p>", "h", "site", 0)
assert import_file(jobs, upload(tmp_path, story_epub), story_id, check=False).startswith("Imported 2")
rebuild_job = jobs.get_next()
assert (rebuild_job.mode, rebuild_job.story_id) == (DownloadMode.REBUILD, story_id)
assert "library book 1" in orchestrator._run_rebuild(rebuild_job)
def test_epub_of_another_story_is_refused(fake, jobs, story_epub, tmp_path):
other = orchestrator.store.create_story(
"https://www.royalroad.com/fiction/99", "www.royalroad.com", "Other", "Writer", []
)
path = upload(tmp_path, story_epub)
with pytest.raises(StoryError, match="story.epub is an epub of another story"):
orchestrator.queue_epub_file(path, "story.epub", other)
assert not path.exists()
assert jobs.get_next() is None
def test_file_that_is_not_an_epub_is_refused(fake, jobs, tmp_path):
path = upload(tmp_path, b"not a zip file")
with pytest.raises(EpubError, match="story.epub is not a readable epub"):
orchestrator.queue_epub_file(path, "story.epub")
assert not path.exists()
# Updates from the site
CHAPTER = "https://www.royalroad.com/fiction/1234/test-story/chapter/{0}/c{0}"
def linked_story(chapter_list):
"""A story in CWA with the chapters (number, title, text)."""
store = orchestrator.store
story_id = store.create_story(STORY_URL, "www.royalroad.com", "Test Story", "Writer", ["title"], 1, "uuid-1")
store.update_story(story_id, metahtml=metahtml(), ingested_at="2026-01-01T00:00:00")
for n, (number, title, text, *source) in enumerate(chapter_list):
html = f"<div><p>{text}</p></div>"
store.add_chapter(
story_id, CHAPTER.format(number), title, html, text_hash(html), source[0] if source else "new", n
)
return story_id
def site(monkeypatch, chapter_list, broken=()):
"""The site lists these chapters (number, title, text). Chapters in `broken` fail to download."""
texts = {CHAPTER.format(n): text for n, _, text in chapter_list}
listed = [{"url": CHAPTER.format(n), "title": title} for n, title, _ in chapter_list]
monkeypatch.setattr(
fff,
"fetch_story",
lambda adapter: SiteSnapshot(STORY_URL, "www.royalroad.com", "Test Story", "Writer", metahtml(), listed, {}),
)
monkeypatch.setattr(fff, "collected_images", lambda adapter: [])
def fetch(adapter, url, index):
if url in {CHAPTER.format(n) for n in broken}:
raise OSError("timeout")
return f"<div><p>{texts[url]}</p></div>"
monkeypatch.setattr(fff, "fetch_chapter", fetch)
def update(story_id, mode=DownloadMode.NEW, **fields):
return orchestrator._run_site_job(Job(mode=mode, story_id=story_id, **fields))
def book(story_id):
"""The title and text of each chapter in the book."""
return [(c["title"], re.sub(r"<[^>]+>", "", c["html"])) for c in orchestrator.store.chapters_for_build(story_id)]
def test_the_site_cover_goes_in_the_book(fake, jobs, monkeypatch, tmp_path):
# A personal.ini without include_images, as the calibre plugin often has.
empty = tmp_path / "personal.ini"
empty.write_text("")
monkeypatch.setattr(config, "PERSONAL_INI", empty)
story_id = linked_story([(1, "One", "a")])
site(monkeypatch, [(1, "One", "a"), (2, "Two", "b")])
snapshot = fff.fetch_story
monkeypatch.setattr(fff, "fetch_story", lambda adapter: dataclasses.replace(snapshot(adapter), cover=COVER))
assert update(story_id).startswith("1 new chapter")
assert orchestrator.store.cover(story_id)["data"] == PNG
(epub,) = [n for n in ingest_files() if n.endswith(".epub")]
with zipfile.ZipFile(orchestrator.ingest_folder.path / epub) as z:
assert "OEBPS/cover.xhtml" in z.namelist()
def test_new_chapter_after_stubbed_chapters_goes_last(fake, jobs, monkeypatch):
story_id = linked_story([(1, "One", "a"), (2, "Two", "b"), (3, "Three", "c")])
site(monkeypatch, [(1, "One", "a"), (4, "Four", "d")])
assert update(story_id).startswith("1 new chapter")
assert book(story_id) == [("One", "a"), ("Two", "b"), ("Three", "c"), ("Four", "d")]
def test_renamed_chapter_is_rechecked_and_rebuilt(fake, jobs, monkeypatch):
story_id = linked_story([(1, "One", "a"), (2, "Two", "b")])
site(monkeypatch, [(1, "One", "a"), (2, "Two (rewritten)", "b2")])
message = update(story_id)
assert "1 renamed, 1 of 1 rechecked chapters changed" in message and "library book 1" in message
assert book(story_id) == [("One", "a"), ("Two (rewritten)", "b2")]
def test_renamed_chapter_with_the_same_text_is_rebuilt(fake, jobs, monkeypatch):
story_id = linked_story([(1, "One", "a")])
site(monkeypatch, [(1, "Chapter One", "a")])
message = update(story_id)
assert "1 renamed, 0 of 1 rechecked chapters changed" in message and "library book 1" in message
assert len(orchestrator.store.list_revisions(orchestrator.store.list_chapters(story_id)[0]["id"])) == 1
def test_job_results_show_chapter_titles_as_text(fake, jobs, monkeypatch):
story_id = linked_story([(1, "One", "a")])
site(monkeypatch, [(1, "One", "a"), (2, "Folly &amp; Magic", "b")], broken=[2])
job = Job(mode=DownloadMode.NEW, story_id=story_id)
orchestrator._run_site_job(job)
assert job.result["failed"] == ["Folly & Magic: timeout"]
site(monkeypatch, [(1, "One", "a"), (2, "Folly &amp; Magic", "b")])
orchestrator._run_site_job(Job(mode=DownloadMode.NEW, story_id=story_id))
# The store keeps FanFicFare's form, for the epub writer.
assert [c["title"] for c in orchestrator.store.list_chapters(story_id)] == ["One", "Folly &amp; Magic"]
def test_renamed_chapter_keeps_its_title_until_its_recheck_works(fake, jobs, monkeypatch):
story_id = linked_story([(1, "One", "a")])
site(monkeypatch, [(1, "Chapter One", "a2")], broken=[1])
update(story_id)
assert book(story_id) == [("One", "a")]
site(monkeypatch, [(1, "Chapter One", "a2")])
update(story_id)
assert book(story_id) == [("Chapter One", "a2")]
def test_insert_on_a_site_that_numbers_its_chapter_urls(fake, jobs, monkeypatch):
# FanFiction.net and similar sites. A chapter inserted at 2 moves B and C up one URL.
story_id = linked_story([(1, "1. A", "a"), (2, "2. B", "b"), (3, "3. C", "c")])
site(monkeypatch, [(1, "1. A", "a"), (2, "2. New", "new"), (3, "3. B", "b"), (4, "4. C", "c")])
update(story_id)
assert book(story_id) == [("1. A", "a"), ("2. New", "new"), ("3. B", "b"), ("4. C", "c")]
def test_imported_chapter_that_differs_only_by_the_rules_is_unchanged(fake, jobs, monkeypatch):
orchestrator.save_rules(None, [{"pattern": "Zorian", "replacement": "Z"}])
story_id = linked_story([(1, "One", "Z wakes.", "import")])
site(monkeypatch, [(1, "One", "Zorian wakes.")])
assert "0 of 1 rechecked chapters changed" in update(story_id, DownloadMode.RECHECK_ALL)
# The stored text is now the site text, so the rules run on it once.
(chapter,) = orchestrator.store.list_chapters(story_id)
assert orchestrator.store.revision_html(chapter["id"], 1) == "<div><p>Zorian wakes.</p></div>"
# A real change is still a change.
site(monkeypatch, [(1, "One", "Zorian sleeps.")])
assert "1 of 1 rechecked chapters changed" in update(story_id, DownloadMode.RECHECK_ALL)
def test_a_chosen_revision_stays_until_the_site_changes_again(fake, jobs, monkeypatch):
story_id = linked_story([(1, "One", "a")])
(chapter,) = orchestrator.store.list_chapters(story_id)
site(monkeypatch, [(1, "One", "b")])
assert "1 of 1 rechecked chapters changed" in update(story_id, DownloadMode.RECHECK_ALL)
orchestrator.store.update_chapter(chapter["id"], current_rev=1)
assert "0 of 1 rechecked chapters changed" in update(story_id, DownloadMode.RECHECK_ALL)
assert book(story_id) == [("One", "a")]
assert len(orchestrator.store.list_revisions(chapter["id"])) == 2
site(monkeypatch, [(1, "One", "c")])
assert "1 of 1 rechecked chapters changed" in update(story_id, DownloadMode.RECHECK_ALL)
assert book(story_id) == [("One", "c")]
def test_a_site_that_goes_back_to_an_older_text_changes_the_book(fake, jobs, monkeypatch):
story_id = linked_story([(1, "One", "a")])
site(monkeypatch, [(1, "One", "b")])
update(story_id, DownloadMode.RECHECK_ALL)
site(monkeypatch, [(1, "One", "a")])
assert "1 of 1 rechecked chapters changed" in update(story_id, DownloadMode.RECHECK_ALL)
assert book(story_id) == [("One", "a")]
def test_new_chapter_download_rechecks_the_latest_chapters(fake, jobs, monkeypatch):
orchestrator.store.set_setting(None, "recent_chapters", 2)
story_id = linked_story([(1, "One", "a"), (2, "Two", "b"), (3, "Three", "c")])
site(monkeypatch, [(1, "One", "a (edited)"), (2, "Two", "b"), (3, "Three", "c (edited)")])
assert "1 of 2 rechecked chapters changed" in update(story_id)
assert book(story_id) == [("One", "a"), ("Two", "b"), ("Three", "c (edited)")]
# A story with its own full recheck interval does not check its latest chapters.
orchestrator.update_story_settings(story_id, recheck_days=7)
site(monkeypatch, [(1, "One", "a"), (2, "Two", "b"), (3, "Three", "c (again)")])
assert update(story_id).startswith("No changes")
def test_scheduler_runs_a_full_recheck_when_one_is_due(fake, jobs, monkeypatch):
monkeypatch.setattr(config, "UPDATE_INTERVAL_HOURS", 12)
story_id = linked_story([(1, "One", "a")])
orchestrator.update_story_settings(story_id, recheck_days=7)
orchestrator._queue_due_updates()
job = jobs.get_next()
assert job.mode == DownloadMode.RECHECK_ALL
site(monkeypatch, [(1, "One", "a")])
orchestrator._run_site_job(job)
jobs.update(job, status=QueueStatus.DONE)
assert orchestrator.story_details(story_id)["story"]["last_full_recheck"]
orchestrator.store.update_story(story_id, last_checked="2020-01-01T00:00:00")
orchestrator._queue_due_updates()
assert jobs.get_next().mode == DownloadMode.NEW
with pytest.raises(ValueError):
orchestrator.update_story_settings(story_id, recheck_days=-1)
orchestrator.update_story_settings(story_id, recheck_days=0)
assert orchestrator.story_details(story_id)["story"]["recheck_days"] is None
# Metadata edits
def test_metadata_edit_rebuilds_the_book(fake, jobs):
story_id = add_story(calibre_id=1, calibre_uuid="uuid-1", ingested_at="2026-01-01T00:00:00")
result = orchestrator.edit_metadata(story_id, {"title": "Edited", "tags": "Mine, Yours"})
assert result["story"]["title"] == "Edited"
assert result["story"]["pinned"] == {"title": "Edited", "tags": ["Mine", "Yours"]}
job = jobs.get_next()
assert result["job"]["id"] == job.id and job.mode == DownloadMode.REBUILD
orchestrator._run_rebuild(job)
epub, _ = ingest_files()
assert re.search(r"<dc:title[^>]*>Edited</dc:title>", opf(epub))
assert re.findall(r"<dc:subject>([^<]*)</dc:subject>", opf(epub)) == ["Mine", "Yours"]
# Without the edit, the unlocked tags come from the site again.
orchestrator.edit_metadata(story_id, {"tags": None})
assert orchestrator.store.get_story(story_id)["pinned"] == {"title": "Edited"}
def test_unlocking_a_field_drops_its_kept_value(fake, jobs):
story_id = add_story()
orchestrator.update_story_settings(story_id, locked_fields=[])
assert orchestrator.store.get_story(story_id)["pinned"] == {}
def test_metadata_edit_of_a_story_that_cwa_does_not_have_waits_for_the_next_build(fake, jobs):
story_id = add_story()
assert orchestrator.edit_metadata(story_id, {"series": "Saga [2]"})["job"] is None
assert jobs.get_next() is None
with pytest.raises(ValueError, match="Unknown metadata fields"):
orchestrator.edit_metadata(story_id, {"publisher": "x"})
# Restarts
class Crash(BaseException):
"""Bindery stops in the middle of a job."""
def test_a_download_that_stops_keeps_its_chapters_and_resumes(fake, story_epub, tmp_path, monkeypatch):
store = orchestrator.store
story_id = store.create_story(STORY_URL, "www.royalroad.com", "Test Story", "Writer", ["title"])
store.update_story(story_id, metahtml=metahtml(), ingested_at="2026-01-01T00:00:00")
one, two, three = chapters()
store.add_chapter(story_id, one["url"], "One", "<p>1</p>", "h", "site", 0)
four = three["url"].replace("15/c15", "16/c16")
site = [one, two, three, {"url": four, "title": "Four"}]
image = {"url": IMG_URL, "ext": "png", "mime": "image/png", "data": PNG}
monkeypatch.setattr(
fff,
"fetch_story",
lambda adapter: SiteSnapshot(STORY_URL, "www.royalroad.com", "Test Story", "Writer", metahtml(), site, {}),
)
monkeypatch.setattr(fff, "collected_images", lambda adapter: [image])
crash_at = [three["url"]]
def fetch(adapter, url, index):
if url in crash_at:
raise Crash
return f"<div><p>{url}</p></div>"
monkeypatch.setattr(fff, "fetch_chapter", fetch)
queue = JobQueue(tmp_path / "jobs.db")
monkeypatch.setattr(orchestrator, "job_queue", queue)
job = queue.add(Job(mode=DownloadMode.NEW, story_id=story_id))
queue.update(queue.get_next(), status=QueueStatus.RUNNING)
with pytest.raises(Crash):
orchestrator._run_site_job(job)
# Chapter Two and its image are stored, in their place.
assert titles(story_id) == ["One", "Two"]
assert store.image_urls(story_id) == {IMG_URL}
queue = JobQueue(tmp_path / "jobs.db")
monkeypatch.setattr(orchestrator, "job_queue", queue)
crash_at.clear()
resumed = queue.get_next()
assert (resumed.id, resumed.restarts, resumed.result["new"]) == (job.id, 1, 1)
message = orchestrator._run_site_job(resumed)
assert message.startswith("3 new chapters") and "library book 1" in message
assert titles(story_id) == ["One", "Two", "Three", "Four"]
def test_an_upload_that_is_gone_fails_clearly(fake, jobs, story_epub, tmp_path):
path = upload(tmp_path, story_epub)
orchestrator.queue_epub_file(path, "story.epub")
path.unlink()
with pytest.raises(StoryError, match="Uploaded file story.epub is gone. Upload it again."):
orchestrator._run_import_file(jobs.get_next())
# Upload holds
def test_a_new_story_waits_for_approval_of_its_first_upload(fake, jobs, monkeypatch):
orchestrator.save_settings(hold_new_links=True)
fake.books = []
site(monkeypatch, [(1, "One", "a")])
job = Job(mode=DownloadMode.NEW, url=STORY_URL)
assert "Approve as new book" in orchestrator._run_site_job(job)
assert ingest_files() == []
story = orchestrator.story_details(job.story_id)["story"]
assert (story["upload_held"], story["needs_approval"], story["ingested_at"]) == (True, True, None)
assert orchestrator.list_stories()[0]["upload_held"]
queued = orchestrator.approve_upload(job.story_id, None)
assert jobs.get_next().id == queued["id"]
assert rebuild(job.story_id) == "Sent to CWA"
story = orchestrator.story_details(job.story_id)["story"]
assert (story["upload_held"], story["needs_approval"]) == (False, False)
def test_a_new_story_linked_to_a_library_book_waits_for_approval(fake, jobs, monkeypatch):
orchestrator.save_settings(hold_new_links=True)
site(monkeypatch, [(1, "One", "a")])
job = Job(mode=DownloadMode.NEW, url=STORY_URL)
assert "Approve link to library book 1" in orchestrator._run_site_job(job)
assert ingest_files() == []
with pytest.raises(ValueError, match="link of this story changed"):
orchestrator.approve_upload(job.story_id, None)
orchestrator.approve_upload(job.story_id, "uuid-1")
assert "library book 1" in rebuild(job.story_id)
assert len(ingest_files()) == 2
def test_a_changed_link_waits_for_approval(fake, jobs):
orchestrator.save_settings(hold_new_links=True)
fake.books.append(FakeBook(2, "uuid-2", "Other Story", epub=b"x"))
story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=1, calibre_uuid="uuid-1")
assert "library book 1" in rebuild(story_id)
orchestrator.link_story(story_id, "uuid-2")
assert "Approve link to library book 2" in rebuild(story_id)
assert len(ingest_files()) == 2
orchestrator.approve_upload(story_id, "uuid-2")
assert "library book 2" in rebuild(story_id)
def test_a_book_found_after_the_linked_book_is_gone_waits_for_approval(fake, jobs):
orchestrator.save_settings(hold_new_links=True)
story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=5, calibre_uuid="uuid-gone")
assert "Approve link to library book 1" in rebuild(story_id)
assert ingest_files() == []
def test_the_book_that_cwa_imports_from_the_first_upload_needs_no_approval(fake, jobs):
orchestrator.save_settings(hold_new_links=True)
books, fake.books = fake.books, []
story_id = add_story()
assert rebuild(story_id) == "Sent to CWA"
fake.books = books
assert "library book 1" in rebuild(story_id)
def test_a_link_that_went_to_cwa_before_the_setting_needs_no_approval(fake, jobs):
story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=1, calibre_uuid="uuid-1")
orchestrator.link_story(story_id, None)
orchestrator.link_story(story_id, "uuid-1")
assert "library book 1" in rebuild(story_id)
orchestrator.save_settings(hold_new_links=True)
assert "library book 1" in rebuild(story_id)
def test_paused_uploads_go_to_cwa_when_the_pause_ends(fake, jobs):
story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=1, calibre_uuid="uuid-1")
orchestrator.save_settings(pause_uploads=True)
assert "pause" in rebuild(story_id)
assert ingest_files() == []
assert orchestrator.approve_upload(story_id, "uuid-1")["mode"] == DownloadMode.REBUILD.value
assert "pause" in orchestrator._run_rebuild(jobs.get_next())
orchestrator.save_settings(pause_uploads=True)
assert jobs.get_next() is None
orchestrator.save_settings(pause_uploads=False)
assert "library book 1" in orchestrator._run_rebuild(jobs.get_next())
assert len(ingest_files()) == 2
assert not orchestrator.story_details(story_id)["story"]["upload_held"]
orchestrator.save_settings(pause_uploads=False)
assert jobs.get_next() is None
tests/test_jobs.py
+163
View File
@@ -0,0 +1,163 @@
"""The job queue: shared jobs, order, cancelling and the job history."""
from datetime import datetime, timedelta
from bindery import config
from bindery.jobs import DownloadMode, Job, JobQueue, QueueStatus
def job(mode=DownloadMode.NEW, **fields):
return Job(mode=mode, **fields)
def test_a_waiting_job_is_shared():
queue = JobQueue()
first = queue.add(job(story_id=1))
assert queue.add(job(story_id=1)) is first
assert queue.add(job(url="https://example.com/s/1")) is not first
assert queue.add(job(url="https://example.com/s/1")).url == "https://example.com/s/1"
assert queue.add(job(DownloadMode.REBUILD, story_id=1)) is not first
assert queue.add(job(story_id=2)) is not first
assert queue.add(job(story_id=1, force_fields=["title"])) is not first
def test_rechecks_of_other_chapters_are_separate_jobs():
queue = JobQueue()
one = queue.add(job(DownloadMode.RECHECK, story_id=1, chapter_ids=[1]))
assert queue.add(job(DownloadMode.RECHECK, story_id=1, chapter_ids=[2])) is not one
assert queue.add(job(DownloadMode.RECHECK, story_id=1, chapter_ids=[1])) is one
def test_uploads_are_separate_jobs():
queue = JobQueue()
one = queue.add(job(DownloadMode.IMPORT_FILE, story_id=1, options={"path": "a.epub"}))
assert queue.add(job(DownloadMode.IMPORT_FILE, story_id=1, options={"path": "b.epub"})) is not one
def test_a_started_job_is_not_shared():
queue = JobQueue()
first = queue.add(job(story_id=1))
assert queue.get_next() is first
assert queue.add(job(story_id=1)) is not first
def test_jobs_run_in_order_and_cancelled_jobs_are_skipped():
queue = JobQueue()
a, b, c = (queue.add(job(story_id=n)) for n in (1, 2, 3))
assert queue.cancel(b.id)
assert b.status == QueueStatus.CANCELLED
assert queue.get_next() is a
assert queue.get_next() is c
assert queue.get_next() is None
def test_a_cancelled_job_is_not_shared():
queue = JobQueue()
cancelled = queue.add(job(story_id=1))
queue.cancel(cancelled.id)
again = queue.add(job(story_id=1))
assert again is not cancelled and again.status == QueueStatus.QUEUED
assert queue.get_next() is again
def test_a_cancelled_job_can_expire_before_the_queue_reaches_it():
queue = JobQueue()
queue.add(job(story_id=1))
running = queue.get_next()
cancelled = queue.add(job(story_id=2))
queue.cancel(cancelled.id)
cancelled.updated = datetime.now() - timedelta(seconds=config.JOB_HISTORY + 1)
assert cancelled.id not in [j["id"] for j in queue.get_status()]
queue.update(running, status=QueueStatus.DONE)
later = queue.add(job(story_id=3))
assert queue.get_next() is later
def test_only_waiting_jobs_can_be_cancelled():
queue = JobQueue()
running = queue.add(job(story_id=1))
queue.update(queue.get_next(), status=QueueStatus.RUNNING)
assert not queue.cancel(running.id)
assert not queue.cancel("no-such-job")
def test_has_active():
queue = JobQueue()
first = queue.add(job(story_id=1))
assert queue.has_active(1) and not queue.has_active(2)
queue.update(first, status=QueueStatus.RUNNING)
assert queue.has_active(1)
queue.update(first, status=QueueStatus.DONE)
assert not queue.has_active(1)
def test_status_is_newest_first_and_filters_by_story():
queue = JobQueue()
old = queue.add(job(story_id=1, created=datetime.now() - timedelta(minutes=5)))
new = queue.add(job(DownloadMode.REBUILD, story_id=1))
other = queue.add(job(story_id=2))
assert [j["id"] for j in queue.get_status(1)] == [new.id, old.id]
assert {j["id"] for j in queue.get_status()} == {old.id, new.id, other.id}
data = queue.get_status(2)[0]
assert data["mode"] == "new" and data["status"] == "queued"
assert data["mode_label"] == "Download new chapters (default)"
assert datetime.fromisoformat(data["created"])
def test_finished_jobs_expire():
queue = JobQueue()
done = queue.add(job(story_id=1))
waiting = queue.add(job(story_id=2))
long_ago = datetime.now() - timedelta(seconds=config.JOB_HISTORY + 1)
queue.update(done, status=QueueStatus.DONE)
done.updated = long_ago
waiting.updated = long_ago
assert [j["id"] for j in queue.get_status()] == [waiting.id]
def test_jobs_outlive_a_restart(tmp_path):
queue = JobQueue(tmp_path / "db.sqlite")
running = queue.add(job(story_id=1))
waiting = queue.add(job(story_id=2, options={"x": [1]}))
done = queue.add(job(story_id=3))
cancelled = queue.add(job(story_id=4))
queue.update(queue.get_next(), status=QueueStatus.RUNNING, result={"new": 2})
queue.update(done, status=QueueStatus.DONE)
queue.cancel(cancelled.id)
again = JobQueue(tmp_path / "db.sqlite")
status = {j["id"]: j for j in again.get_status()}
assert (status[running.id]["status"], status[running.id]["restarts"]) == ("queued", 1)
assert status[running.id]["message"] == "Resumed"
assert status[running.id]["result"] == {"new": 2}
assert status[done.id]["status"] == "done" and status[cancelled.id]["status"] == "cancelled"
# The interrupted job goes first.
assert again.get_next().id == running.id
nxt = again.get_next()
assert (nxt.id, nxt.options, nxt.mode) == (waiting.id, {"x": [1]}, DownloadMode.NEW)
assert again.get_next() is None
def test_a_job_that_stops_bindery_again_and_again_is_given_up(tmp_path):
path = tmp_path / "db.sqlite"
queue = JobQueue(path)
first = queue.add(job(story_id=1))
queue.update(queue.get_next(), status=QueueStatus.RUNNING)
for restarts in (1, 2):
queue = JobQueue(path)
started = queue.get_next()
assert (started.id, started.restarts) == (first.id, restarts)
queue.update(started, status=QueueStatus.RUNNING)
queue = JobQueue(path)
assert queue.get_next() is None
(gone,) = queue.get_status()
assert gone["status"] == "error" and "3 times" in gone["message"]
def test_finished_jobs_leave_the_database(tmp_path, monkeypatch):
queue = JobQueue(tmp_path / "db.sqlite")
old = queue.add(job(story_id=1))
queue.update(old, status=QueueStatus.DONE)
old.updated = datetime.now() - timedelta(seconds=config.JOB_HISTORY + 1)
queue.get_status()
assert JobQueue(tmp_path / "db.sqlite").get_status() == []
tests/test_metadata.py
+74
View File
@@ -0,0 +1,74 @@
import pytest
from bindery import orchestrator as backend
SITE = {
"title": "Site Title",
"authors": ["Writer"],
"comments": "<p>New blurb</p>",
"tags": ["Fantasy", "Completed"],
"series": "Saga",
"series_index": 2.0,
"pubdate": "2020-01-01T00:00:00",
"languages": "en",
}
def story(**extra):
base = {"locked_fields": ["title"], "pinned": {}, "site_values": {}}
base.update(extra)
return base
def test_first_download_keeps_site_value():
assert backend._resolve_metadata(story(), SITE, []) == {"title": "Site Title"}
def test_locked_field_keeps_its_value():
assert backend._resolve_metadata(story(pinned={"title": "Old"}), SITE, []) == {"title": "Old"}
def test_forced_field_takes_site_value_and_stays_locked():
s = story(pinned={"title": "Old"})
assert backend._resolve_metadata(s, SITE, ["title"]) == {"title": "Site Title"}
def test_edited_unlocked_field_stays_until_the_site_changes_it():
s = story(pinned={"title": "Old", "tags": ["mine"]}, site_values=SITE)
assert backend._resolve_metadata(s, SITE, []) == {"title": "Old", "tags": ["mine"]}
changed = {**SITE, "tags": ["Fantasy", "Completed", "LitRPG"]}
assert backend._resolve_metadata(s, changed, []) == {"title": "Old"}
# A change to another field keeps the edit.
assert backend._resolve_metadata(s, {**SITE, "comments": "<p>x</p>"}, []) == {"title": "Old", "tags": ["mine"]}
def test_forced_update_replaces_an_edited_unlocked_field():
s = story(pinned={"tags": ["mine"]}, site_values=SITE)
assert backend._resolve_metadata(s, SITE, ["tags"]) == {"title": "Site Title"}
def test_edit_before_the_first_site_values_stays():
s = story(pinned={"tags": ["mine"]})
assert backend._resolve_metadata(s, SITE, []) == {"title": "Site Title", "tags": ["mine"]}
def test_edited_values_are_checked_and_normalized():
assert backend._edited_value("authors", "A & B ") == ["A", "B"]
assert backend._edited_value("tags", "x, y,,") == ["x", "y"]
assert backend._edited_value("series", "Saga [2]") == ["Saga", 2.0]
assert backend._edited_value("series", ["Saga", "3"]) == ["Saga", 3.0]
assert backend._edited_value("series", "") == ["", None]
assert backend._edited_value("pubdate", "2024-01-31") == "2024-01-31"
assert backend._edited_value("title", " New ") == "New"
for field, value in (("title", " "), ("authors", "&"), ("pubdate", "soon")):
with pytest.raises(ValueError):
backend._edited_value(field, value)
def test_series_keeps_name_and_index():
s = story(locked_fields=["series"])
assert backend._resolve_metadata(s, SITE, []) == {"series": ["Saga", 2.0]}
def test_resolve_without_site_values_does_not_fail():
assert backend._resolve_metadata(story(pinned={"title": "Old"}), {}, []) == {"title": "Old"}
tests/test_worker_flows.py
+267
View File
@@ -0,0 +1,267 @@
"""Worker and scheduler transitions, with real temporary stores and no background threads."""
from datetime import datetime, timedelta
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
from bindery import config, fff
from bindery import orchestrator as o
from bindery.diffing import text_hash
from bindery.jobs import DownloadMode, Job, JobQueue, QueueStatus
from bindery.store import Store
class StopLoop(BaseException):
"""Stop an infinite loop without entering its application error handler."""
@pytest.fixture
def state(monkeypatch, tmp_path):
store = Store(tmp_path / "stories.db")
queue = JobQueue(tmp_path / "jobs.db")
monkeypatch.setattr(o, "store", store)
monkeypatch.setattr(o, "job_queue", queue)
monkeypatch.setattr(config, "CWA_URL", "")
monkeypatch.setattr(config, "SITE_DELAY", 7)
return store, queue
def add_story(store, url="https://www.royalroad.com/fiction/1234", **values):
sid = store.create_story(url, "www.royalroad.com", "Title", "Writer", [])
store.update_story(sid, **values)
return sid
@pytest.mark.parametrize("mode", list(DownloadMode))
def test_dispatches_each_download_mode(state, monkeypatch, mode):
handlers = {
name: Mock(return_value=name) for name in ("_run_import", "_run_import_file", "_run_rebuild", "_run_site_job")
}
for name, handler in handlers.items():
monkeypatch.setattr(o, name, handler)
expected = {
DownloadMode.IMPORT: "_run_import",
DownloadMode.IMPORT_FILE: "_run_import_file",
DownloadMode.REBUILD: "_run_rebuild",
}.get(mode, "_run_site_job")
job = Job(mode=mode)
assert o._run_job(job) == expected
handlers[expected].assert_called_once_with(job)
assert sum(h.call_count for h in handlers.values()) == 1
@pytest.mark.parametrize("error", [None, o.StoryError("site refused"), RuntimeError("broken")])
@pytest.mark.parametrize("with_story", [False, True])
def test_worker_persists_outcomes_and_continues_after_failure(state, monkeypatch, tmp_path, error, with_story):
store, queue = state
sid = add_story(store) if with_story else None
first = queue.add(Job(mode=DownloadMode.NEW, story_id=sid, url="first"))
second = queue.add(Job(mode=DownloadMode.REBUILD, story_id=sid))
monkeypatch.setattr(queue, "get_next", Mock(side_effect=[None, first, second, StopLoop()]))
sleeps = Mock()
monkeypatch.setattr(o.time, "sleep", sleeps)
def run(job):
assert job.status == QueueStatus.RUNNING
if job is first and error:
raise error
return "Sent"
monkeypatch.setattr(o, "_run_job", run)
with pytest.raises(StopLoop):
o.download_loop()
saved = {j["id"]: j for j in JobQueue(tmp_path / "jobs.db").get_status()}
assert saved[second.id]["status"] == "done"
assert saved[second.id]["progress"] == 1.0
assert saved[first.id]["status"] == ("error" if error else "done")
expected = "RuntimeError: broken" if isinstance(error, RuntimeError) else str(error) if error else "Sent"
assert saved[first.id]["message"] == expected
if error and sid:
story = store.get_story(sid)
assert (story["status"], story["message"]) == ("error", expected)
assert story["last_checked"]
assert [c.args[0] for c in sleeps.call_args_list] == [2, 7]
def test_scheduler_skips_disabled_busy_and_recent_stories(state, monkeypatch):
store, queue = state
now = datetime(2026, 6, 1, 12)
class Clock(datetime):
@classmethod
def now(cls):
return now
monkeypatch.setattr(o, "datetime", Clock)
monkeypatch.setattr(config, "UPDATE_INTERVAL_HOURS", 2)
ids = [add_story(store, f"url-{n}") for n in range(7)]
store.update_story(ids[0], auto_update=0)
queue.add(Job(mode=DownloadMode.NEW, story_id=ids[1]))
store.update_story(ids[2], last_checked=(now - timedelta(hours=1)).isoformat())
store.update_story(ids[3], last_checked=(now - timedelta(hours=2)).isoformat())
for sid in ids[4:]:
store.set_setting(sid, "recheck_days", 3)
store.set_setting(ids[5], "last_full_recheck", (now - timedelta(days=3)).isoformat())
store.set_setting(ids[6], "last_full_recheck", (now - timedelta(days=1)).isoformat())
o._queue_due_updates()
queued = {j["story_id"]: j["mode"] for j in queue.get_status()}
assert queued == {ids[1]: "new", ids[3]: "new", ids[4]: "recheck_all", ids[5]: "recheck_all", ids[6]: "new"}
o._queue_due_updates()
assert len(queue.get_status()) == 5
monkeypatch.setattr(config, "UPDATE_INTERVAL_HOURS", 0)
extra = add_story(store, "extra")
o._queue_due_updates()
assert not queue.has_active(extra)
def test_scheduler_recovers_from_backup_and_queue_errors(state, monkeypatch):
backup = Mock(side_effect=[OSError("disk full"), None])
due = Mock(side_effect=[ValueError("bad date"), None])
sleep = Mock(side_effect=[None, StopLoop()])
monkeypatch.setattr(o, "daily_backup", backup)
monkeypatch.setattr(o, "_queue_due_updates", due)
monkeypatch.setattr(o.time, "sleep", sleep)
with pytest.raises(StopLoop):
o.scheduler_loop()
assert backup.call_count == due.call_count == 2
assert [c.args for c in sleep.call_args_list] == [(60,), (60,)]
@pytest.fixture
def site(state, monkeypatch):
store, queue = state
sid = add_story(store, ingested_at="2026-01-01", site_values={"title": "Title"})
for i in range(1, 4):
html = f"<p>old {i}</p>"
store.add_chapter(sid, f"chapter-{i}", f"Chapter {i}", html, text_hash(html), "new", i)
adapter = SimpleNamespace(normalize_chapterurl=lambda url: url)
snap = fff.SiteSnapshot(
url=store.get_story(sid)["url"],
site="www.royalroad.com",
title="Title",
author="Writer",
metahtml="metadata",
chapters=[
{"url": "chapter-1", "title": "Renamed"},
{"url": "chapter-2", "title": "Chapter 2"},
{"url": "chapter-4", "title": "New"},
],
calibre={"title": "Title"},
)
fetch = Mock(side_effect=lambda a, url, i: f"<p>fresh {url}</p>")
build, deliver = Mock(), Mock(return_value="Sent")
monkeypatch.setattr(fff, "get_adapter", lambda url: adapter)
monkeypatch.setattr(fff, "fetch_story", lambda a: snap)
monkeypatch.setattr(fff, "fetch_chapter", fetch)
monkeypatch.setattr(fff, "collected_images", lambda a: [])
monkeypatch.setattr(o, "_build", build)
monkeypatch.setattr(o, "_linked_book", lambda story: None)
monkeypatch.setattr(o, "_deliver", deliver)
return SimpleNamespace(
store=store, queue=queue, id=sid, snap=snap, adapter=adapter, fetch=fetch, build=build, deliver=deliver
)
@pytest.mark.parametrize("mode", [DownloadMode.METADATA, DownloadMode.FULL, DownloadMode.RECHECK])
def test_download_modes_preserve_history_and_select_the_right_chapters(site, mode):
original = site.store.list_chapters(site.id)
job = Job(mode=mode, story_id=site.id, chapter_ids=[original[1]["id"], original[2]["id"]])
result = o._run_site_job(job)
chapters = site.store.list_chapters(site.id)
assert chapters[0]["title"] == "Renamed"
assert all(site.store.revision_html(c["id"], 1) == f"<p>old {n}</p>" for n, c in enumerate(original, 1))
fetched = [call.args[1] for call in site.fetch.call_args_list]
removed = site.store.get_chapter(original[2]["id"])
assert not removed["on_site"]
if mode == DownloadMode.METADATA:
assert fetched == []
assert len(chapters) == 3
assert removed["included"]
elif mode == DownloadMode.FULL:
assert set(fetched) == {"chapter-1", "chapter-2", "chapter-4"}
assert not removed["included"]
assert "1 taken out" in result
assert site.store.get_setting(site.id, "last_full_recheck")
else:
assert set(fetched) == {"chapter-1", "chapter-2"}
assert job.result["skipped"] == ["Chapter 3"]
assert removed["included"]
site.build.assert_called_once()
site.deliver.assert_called_once()
def test_failed_new_chapter_keeps_successful_chapters_and_images(site, monkeypatch):
site.snap.chapters.append({"url": "chapter-5", "title": "Broken"})
def fetch(adapter, url, index):
if url == "chapter-5":
raise OSError("timeout")
return "<p>ok</p>"
site.fetch.side_effect = fetch
image = {"url": "image", "ext": "png", "mime": "image/png", "data": b"image"}
site.snap.cover = {**image, "url": "cover"}
monkeypatch.setattr(fff, "collected_images", lambda a: [image, image])
job = Job(mode=DownloadMode.NEW, story_id=site.id)
assert "1 failed" in o._run_site_job(job)
assert job.result["failed"] == ["Broken: timeout"]
assert len(site.store.list_chapters(site.id)) == 4
assert site.store.get_story(site.id)["status"] == "error"
assert site.store.image_urls(site.id) == {"image"}
assert site.store.cover(site.id)["url"] == "cover"
def test_new_story_with_no_downloaded_chapters_fails_without_delivery(site):
site.fetch.side_effect = OSError("offline")
job = Job(mode=DownloadMode.NEW, url="new-url")
site.snap.url = "new-url"
with pytest.raises(o.StoryError, match="No chapters downloaded"):
o._run_site_job(job)
assert job.story_id != site.id
assert site.store.list_chapters(job.story_id) == []
site.deliver.assert_not_called()
def test_normalized_chapter_urls_keep_the_existing_identity(site):
original = site.store.list_chapters(site.id)[0]
site.adapter.normalize_chapterurl = lambda url: "renamed-url" if url == "chapter-1" else url
site.snap.chapters[0]["url"] = "renamed-url"
o._run_site_job(Job(mode=DownloadMode.RECHECK_ALL, story_id=site.id))
updated = site.store.get_chapter(original["id"])
assert updated["url"] == "renamed-url"
assert updated["current_rev"] == 2
assert len(site.store.list_chapters(site.id)) == 4
def test_inclusion_changes_only_the_requested_story_and_queues_a_rebuild(state):
store, queue = state
sid, other = add_story(store), add_story(store, "other")
cid = store.add_chapter(sid, "chapter", "One", "<p>text</p>", "h", "new", 0)
foreign = store.add_chapter(other, "foreign", "Other", "<p>other</p>", "h", "new", 0)
result = o.set_chapters_included(sid, [cid, foreign, 999], False)
assert result["mode"] == "rebuild"
assert not store.get_chapter(cid)["included"]
assert store.get_chapter(foreign)["included"]
assert store.revision_html(cid, 1) == "<p>text</p>"
o.set_chapters_included(sid, [cid], True)
assert store.get_chapter(cid)["included"]
assert len(queue.get_status()) == 1
@pytest.mark.parametrize(
"mode,chapters,fields,message",
[
("import", [], [], "library import"),
("recheck", [], [], "at least one chapter"),
("new", [], ["unknown"], "Unknown metadata"),
("invalid", [], [], "not a valid"),
],
)
def test_invalid_job_requests_do_not_enter_the_queue(state, mode, chapters, fields, message):
store, queue = state
sid = add_story(store)
with pytest.raises(ValueError, match=message):
o.queue_story(sid, mode, chapters, fields)
assert queue.get_status() == []