Run download jobs and deliver books to CWA
Jobs download new chapters, recheck chapters, replace the book, update metadata or rebuild from stored chapters. Locked metadata fields keep their values. A scheduler queues updates. An update finds the library book of a story through CWA, and fails with an error if it cannot, so that it never makes a duplicate book.
This commit is contained in:
@@ -0,0 +1,267 @@
|
||||
"""Worker and scheduler transitions, with real temporary stores and no background threads."""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from bindery import config, fff
|
||||
from bindery import orchestrator as o
|
||||
from bindery.diffing import text_hash
|
||||
from bindery.jobs import DownloadMode, Job, JobQueue, QueueStatus
|
||||
from bindery.store import Store
|
||||
|
||||
|
||||
class StopLoop(BaseException):
|
||||
"""Stop an infinite loop without entering its application error handler."""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def state(monkeypatch, tmp_path):
|
||||
store = Store(tmp_path / "stories.db")
|
||||
queue = JobQueue(tmp_path / "jobs.db")
|
||||
monkeypatch.setattr(o, "store", store)
|
||||
monkeypatch.setattr(o, "job_queue", queue)
|
||||
monkeypatch.setattr(config, "CWA_URL", "")
|
||||
monkeypatch.setattr(config, "SITE_DELAY", 7)
|
||||
return store, queue
|
||||
|
||||
|
||||
def add_story(store, url="https://www.royalroad.com/fiction/1234", **values):
|
||||
sid = store.create_story(url, "www.royalroad.com", "Title", "Writer", [])
|
||||
store.update_story(sid, **values)
|
||||
return sid
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", list(DownloadMode))
|
||||
def test_dispatches_each_download_mode(state, monkeypatch, mode):
|
||||
handlers = {
|
||||
name: Mock(return_value=name) for name in ("_run_import", "_run_import_file", "_run_rebuild", "_run_site_job")
|
||||
}
|
||||
for name, handler in handlers.items():
|
||||
monkeypatch.setattr(o, name, handler)
|
||||
expected = {
|
||||
DownloadMode.IMPORT: "_run_import",
|
||||
DownloadMode.IMPORT_FILE: "_run_import_file",
|
||||
DownloadMode.REBUILD: "_run_rebuild",
|
||||
}.get(mode, "_run_site_job")
|
||||
job = Job(mode=mode)
|
||||
assert o._run_job(job) == expected
|
||||
handlers[expected].assert_called_once_with(job)
|
||||
assert sum(h.call_count for h in handlers.values()) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("error", [None, o.StoryError("site refused"), RuntimeError("broken")])
|
||||
@pytest.mark.parametrize("with_story", [False, True])
|
||||
def test_worker_persists_outcomes_and_continues_after_failure(state, monkeypatch, tmp_path, error, with_story):
|
||||
store, queue = state
|
||||
sid = add_story(store) if with_story else None
|
||||
first = queue.add(Job(mode=DownloadMode.NEW, story_id=sid, url="first"))
|
||||
second = queue.add(Job(mode=DownloadMode.REBUILD, story_id=sid))
|
||||
monkeypatch.setattr(queue, "get_next", Mock(side_effect=[None, first, second, StopLoop()]))
|
||||
sleeps = Mock()
|
||||
monkeypatch.setattr(o.time, "sleep", sleeps)
|
||||
|
||||
def run(job):
|
||||
assert job.status == QueueStatus.RUNNING
|
||||
if job is first and error:
|
||||
raise error
|
||||
return "Sent"
|
||||
|
||||
monkeypatch.setattr(o, "_run_job", run)
|
||||
with pytest.raises(StopLoop):
|
||||
o.download_loop()
|
||||
saved = {j["id"]: j for j in JobQueue(tmp_path / "jobs.db").get_status()}
|
||||
assert saved[second.id]["status"] == "done"
|
||||
assert saved[second.id]["progress"] == 1.0
|
||||
assert saved[first.id]["status"] == ("error" if error else "done")
|
||||
expected = "RuntimeError: broken" if isinstance(error, RuntimeError) else str(error) if error else "Sent"
|
||||
assert saved[first.id]["message"] == expected
|
||||
if error and sid:
|
||||
story = store.get_story(sid)
|
||||
assert (story["status"], story["message"]) == ("error", expected)
|
||||
assert story["last_checked"]
|
||||
assert [c.args[0] for c in sleeps.call_args_list] == [2, 7]
|
||||
|
||||
|
||||
def test_scheduler_skips_disabled_busy_and_recent_stories(state, monkeypatch):
|
||||
store, queue = state
|
||||
now = datetime(2026, 6, 1, 12)
|
||||
|
||||
class Clock(datetime):
|
||||
@classmethod
|
||||
def now(cls):
|
||||
return now
|
||||
|
||||
monkeypatch.setattr(o, "datetime", Clock)
|
||||
monkeypatch.setattr(config, "UPDATE_INTERVAL_HOURS", 2)
|
||||
ids = [add_story(store, f"url-{n}") for n in range(7)]
|
||||
store.update_story(ids[0], auto_update=0)
|
||||
queue.add(Job(mode=DownloadMode.NEW, story_id=ids[1]))
|
||||
store.update_story(ids[2], last_checked=(now - timedelta(hours=1)).isoformat())
|
||||
store.update_story(ids[3], last_checked=(now - timedelta(hours=2)).isoformat())
|
||||
for sid in ids[4:]:
|
||||
store.set_setting(sid, "recheck_days", 3)
|
||||
store.set_setting(ids[5], "last_full_recheck", (now - timedelta(days=3)).isoformat())
|
||||
store.set_setting(ids[6], "last_full_recheck", (now - timedelta(days=1)).isoformat())
|
||||
o._queue_due_updates()
|
||||
queued = {j["story_id"]: j["mode"] for j in queue.get_status()}
|
||||
assert queued == {ids[1]: "new", ids[3]: "new", ids[4]: "recheck_all", ids[5]: "recheck_all", ids[6]: "new"}
|
||||
o._queue_due_updates()
|
||||
assert len(queue.get_status()) == 5
|
||||
monkeypatch.setattr(config, "UPDATE_INTERVAL_HOURS", 0)
|
||||
extra = add_story(store, "extra")
|
||||
o._queue_due_updates()
|
||||
assert not queue.has_active(extra)
|
||||
|
||||
|
||||
def test_scheduler_recovers_from_backup_and_queue_errors(state, monkeypatch):
|
||||
backup = Mock(side_effect=[OSError("disk full"), None])
|
||||
due = Mock(side_effect=[ValueError("bad date"), None])
|
||||
sleep = Mock(side_effect=[None, StopLoop()])
|
||||
monkeypatch.setattr(o, "daily_backup", backup)
|
||||
monkeypatch.setattr(o, "_queue_due_updates", due)
|
||||
monkeypatch.setattr(o.time, "sleep", sleep)
|
||||
with pytest.raises(StopLoop):
|
||||
o.scheduler_loop()
|
||||
assert backup.call_count == due.call_count == 2
|
||||
assert [c.args for c in sleep.call_args_list] == [(60,), (60,)]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def site(state, monkeypatch):
|
||||
store, queue = state
|
||||
sid = add_story(store, ingested_at="2026-01-01", site_values={"title": "Title"})
|
||||
for i in range(1, 4):
|
||||
html = f"<p>old {i}</p>"
|
||||
store.add_chapter(sid, f"chapter-{i}", f"Chapter {i}", html, text_hash(html), "new", i)
|
||||
adapter = SimpleNamespace(normalize_chapterurl=lambda url: url)
|
||||
snap = fff.SiteSnapshot(
|
||||
url=store.get_story(sid)["url"],
|
||||
site="www.royalroad.com",
|
||||
title="Title",
|
||||
author="Writer",
|
||||
metahtml="metadata",
|
||||
chapters=[
|
||||
{"url": "chapter-1", "title": "Renamed"},
|
||||
{"url": "chapter-2", "title": "Chapter 2"},
|
||||
{"url": "chapter-4", "title": "New"},
|
||||
],
|
||||
calibre={"title": "Title"},
|
||||
)
|
||||
fetch = Mock(side_effect=lambda a, url, i: f"<p>fresh {url}</p>")
|
||||
build, deliver = Mock(), Mock(return_value="Sent")
|
||||
monkeypatch.setattr(fff, "get_adapter", lambda url: adapter)
|
||||
monkeypatch.setattr(fff, "fetch_story", lambda a: snap)
|
||||
monkeypatch.setattr(fff, "fetch_chapter", fetch)
|
||||
monkeypatch.setattr(fff, "collected_images", lambda a: [])
|
||||
monkeypatch.setattr(o, "_build", build)
|
||||
monkeypatch.setattr(o, "_linked_book", lambda story: None)
|
||||
monkeypatch.setattr(o, "_deliver", deliver)
|
||||
return SimpleNamespace(
|
||||
store=store, queue=queue, id=sid, snap=snap, adapter=adapter, fetch=fetch, build=build, deliver=deliver
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", [DownloadMode.METADATA, DownloadMode.FULL, DownloadMode.RECHECK])
|
||||
def test_download_modes_preserve_history_and_select_the_right_chapters(site, mode):
|
||||
original = site.store.list_chapters(site.id)
|
||||
job = Job(mode=mode, story_id=site.id, chapter_ids=[original[1]["id"], original[2]["id"]])
|
||||
result = o._run_site_job(job)
|
||||
chapters = site.store.list_chapters(site.id)
|
||||
assert chapters[0]["title"] == "Renamed"
|
||||
assert all(site.store.revision_html(c["id"], 1) == f"<p>old {n}</p>" for n, c in enumerate(original, 1))
|
||||
fetched = [call.args[1] for call in site.fetch.call_args_list]
|
||||
removed = site.store.get_chapter(original[2]["id"])
|
||||
assert not removed["on_site"]
|
||||
if mode == DownloadMode.METADATA:
|
||||
assert fetched == []
|
||||
assert len(chapters) == 3
|
||||
assert removed["included"]
|
||||
elif mode == DownloadMode.FULL:
|
||||
assert set(fetched) == {"chapter-1", "chapter-2", "chapter-4"}
|
||||
assert not removed["included"]
|
||||
assert "1 taken out" in result
|
||||
assert site.store.get_setting(site.id, "last_full_recheck")
|
||||
else:
|
||||
assert set(fetched) == {"chapter-1", "chapter-2"}
|
||||
assert job.result["skipped"] == ["Chapter 3"]
|
||||
assert removed["included"]
|
||||
site.build.assert_called_once()
|
||||
site.deliver.assert_called_once()
|
||||
|
||||
|
||||
def test_failed_new_chapter_keeps_successful_chapters_and_images(site, monkeypatch):
|
||||
site.snap.chapters.append({"url": "chapter-5", "title": "Broken"})
|
||||
|
||||
def fetch(adapter, url, index):
|
||||
if url == "chapter-5":
|
||||
raise OSError("timeout")
|
||||
return "<p>ok</p>"
|
||||
|
||||
site.fetch.side_effect = fetch
|
||||
image = {"url": "image", "ext": "png", "mime": "image/png", "data": b"image"}
|
||||
site.snap.cover = {**image, "url": "cover"}
|
||||
monkeypatch.setattr(fff, "collected_images", lambda a: [image, image])
|
||||
job = Job(mode=DownloadMode.NEW, story_id=site.id)
|
||||
assert "1 failed" in o._run_site_job(job)
|
||||
assert job.result["failed"] == ["Broken: timeout"]
|
||||
assert len(site.store.list_chapters(site.id)) == 4
|
||||
assert site.store.get_story(site.id)["status"] == "error"
|
||||
assert site.store.image_urls(site.id) == {"image"}
|
||||
assert site.store.cover(site.id)["url"] == "cover"
|
||||
|
||||
|
||||
def test_new_story_with_no_downloaded_chapters_fails_without_delivery(site):
|
||||
site.fetch.side_effect = OSError("offline")
|
||||
job = Job(mode=DownloadMode.NEW, url="new-url")
|
||||
site.snap.url = "new-url"
|
||||
with pytest.raises(o.StoryError, match="No chapters downloaded"):
|
||||
o._run_site_job(job)
|
||||
assert job.story_id != site.id
|
||||
assert site.store.list_chapters(job.story_id) == []
|
||||
site.deliver.assert_not_called()
|
||||
|
||||
|
||||
def test_normalized_chapter_urls_keep_the_existing_identity(site):
|
||||
original = site.store.list_chapters(site.id)[0]
|
||||
site.adapter.normalize_chapterurl = lambda url: "renamed-url" if url == "chapter-1" else url
|
||||
site.snap.chapters[0]["url"] = "renamed-url"
|
||||
o._run_site_job(Job(mode=DownloadMode.RECHECK_ALL, story_id=site.id))
|
||||
updated = site.store.get_chapter(original["id"])
|
||||
assert updated["url"] == "renamed-url"
|
||||
assert updated["current_rev"] == 2
|
||||
assert len(site.store.list_chapters(site.id)) == 4
|
||||
|
||||
|
||||
def test_inclusion_changes_only_the_requested_story_and_queues_a_rebuild(state):
|
||||
store, queue = state
|
||||
sid, other = add_story(store), add_story(store, "other")
|
||||
cid = store.add_chapter(sid, "chapter", "One", "<p>text</p>", "h", "new", 0)
|
||||
foreign = store.add_chapter(other, "foreign", "Other", "<p>other</p>", "h", "new", 0)
|
||||
result = o.set_chapters_included(sid, [cid, foreign, 999], False)
|
||||
assert result["mode"] == "rebuild"
|
||||
assert not store.get_chapter(cid)["included"]
|
||||
assert store.get_chapter(foreign)["included"]
|
||||
assert store.revision_html(cid, 1) == "<p>text</p>"
|
||||
o.set_chapters_included(sid, [cid], True)
|
||||
assert store.get_chapter(cid)["included"]
|
||||
assert len(queue.get_status()) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mode,chapters,fields,message",
|
||||
[
|
||||
("import", [], [], "library import"),
|
||||
("recheck", [], [], "at least one chapter"),
|
||||
("new", [], ["unknown"], "Unknown metadata"),
|
||||
("invalid", [], [], "not a valid"),
|
||||
],
|
||||
)
|
||||
def test_invalid_job_requests_do_not_enter_the_queue(state, mode, chapters, fields, message):
|
||||
store, queue = state
|
||||
sid = add_story(store)
|
||||
with pytest.raises(ValueError, match=message):
|
||||
o.queue_story(sid, mode, chapters, fields)
|
||||
assert queue.get_status() == []
|
||||
Reference in New Issue
Block a user