tim
8faab2463b
The merge cannot always tell where new chapters go. For example, a site can list an announcement before the chapters of an old book. You can now select chapters and move them, in their order, to after another chapter or to the start. You can also delete them with their revisions. A story keeps at least one chapter. The book is then rebuilt. A deleted chapter that the site still lists comes back with the next download. The Move… and Delete… buttons open a dialog that names the selected chapters before anything changes. A shift-click selects a range of chapters. The Select all checkbox moves into the first heading of the chapter list. On a phone, that heading shows with the text "Select all".
309 lines
13 KiB
Python
309 lines
13 KiB
Python
"""Worker and scheduler transitions, with real temporary stores and no background threads."""
|
|
|
|
from datetime import datetime, timedelta
|
|
from types import SimpleNamespace
|
|
from unittest.mock import Mock
|
|
|
|
import pytest
|
|
|
|
from bindery import config, fff
|
|
from bindery import orchestrator as o
|
|
from bindery.diffing import text_hash
|
|
from bindery.jobs import DownloadMode, Job, JobQueue, QueueStatus
|
|
from bindery.store import Store
|
|
|
|
|
|
class StopLoop(BaseException):
|
|
"""Stop an infinite loop without entering its application error handler."""
|
|
|
|
|
|
@pytest.fixture
|
|
def state(monkeypatch, tmp_path):
|
|
store = Store(tmp_path / "stories.db")
|
|
queue = JobQueue(tmp_path / "jobs.db")
|
|
monkeypatch.setattr(o, "store", store)
|
|
monkeypatch.setattr(o, "job_queue", queue)
|
|
monkeypatch.setattr(config, "CWA_URL", "")
|
|
monkeypatch.setattr(config, "SITE_DELAY", 7)
|
|
return store, queue
|
|
|
|
|
|
def add_story(store, url="https://www.royalroad.com/fiction/1234", **values):
|
|
sid = store.create_story(url, "www.royalroad.com", "Title", "Writer", [])
|
|
store.update_story(sid, **values)
|
|
return sid
|
|
|
|
|
|
@pytest.mark.parametrize("mode", list(DownloadMode))
|
|
def test_dispatches_each_download_mode(state, monkeypatch, mode):
|
|
handlers = {
|
|
name: Mock(return_value=name) for name in ("_run_import", "_run_import_file", "_run_rebuild", "_run_site_job")
|
|
}
|
|
for name, handler in handlers.items():
|
|
monkeypatch.setattr(o, name, handler)
|
|
expected = {
|
|
DownloadMode.IMPORT: "_run_import",
|
|
DownloadMode.IMPORT_FILE: "_run_import_file",
|
|
DownloadMode.REBUILD: "_run_rebuild",
|
|
}.get(mode, "_run_site_job")
|
|
job = Job(mode=mode)
|
|
assert o._run_job(job) == expected
|
|
handlers[expected].assert_called_once_with(job)
|
|
assert sum(h.call_count for h in handlers.values()) == 1
|
|
|
|
|
|
@pytest.mark.parametrize("error", [None, o.StoryError("site refused"), RuntimeError("broken")])
|
|
@pytest.mark.parametrize("with_story", [False, True])
|
|
def test_worker_persists_outcomes_and_continues_after_failure(state, monkeypatch, tmp_path, error, with_story):
|
|
store, queue = state
|
|
sid = add_story(store) if with_story else None
|
|
first = queue.add(Job(mode=DownloadMode.NEW, story_id=sid, url="first"))
|
|
second = queue.add(Job(mode=DownloadMode.REBUILD, story_id=sid))
|
|
monkeypatch.setattr(queue, "get_next", Mock(side_effect=[None, first, second, StopLoop()]))
|
|
sleeps = Mock()
|
|
monkeypatch.setattr(o.time, "sleep", sleeps)
|
|
|
|
def run(job):
|
|
assert job.status == QueueStatus.RUNNING
|
|
if job is first and error:
|
|
raise error
|
|
return "Sent"
|
|
|
|
monkeypatch.setattr(o, "_run_job", run)
|
|
with pytest.raises(StopLoop):
|
|
o.download_loop()
|
|
saved = {j["id"]: j for j in JobQueue(tmp_path / "jobs.db").get_status()}
|
|
assert saved[second.id]["status"] == "done"
|
|
assert saved[second.id]["progress"] == 1.0
|
|
assert saved[first.id]["status"] == ("error" if error else "done")
|
|
expected = "RuntimeError: broken" if isinstance(error, RuntimeError) else str(error) if error else "Sent"
|
|
assert saved[first.id]["message"] == expected
|
|
if error and sid:
|
|
story = store.get_story(sid)
|
|
assert (story["status"], story["message"]) == ("error", expected)
|
|
assert story["last_checked"]
|
|
assert [c.args[0] for c in sleeps.call_args_list] == [2, 7]
|
|
|
|
|
|
def test_scheduler_skips_disabled_busy_and_recent_stories(state, monkeypatch):
|
|
store, queue = state
|
|
now = datetime(2026, 6, 1, 12)
|
|
|
|
class Clock(datetime):
|
|
@classmethod
|
|
def now(cls):
|
|
return now
|
|
|
|
monkeypatch.setattr(o, "datetime", Clock)
|
|
monkeypatch.setattr(config, "UPDATE_INTERVAL_HOURS", 2)
|
|
ids = [add_story(store, f"url-{n}") for n in range(7)]
|
|
store.update_story(ids[0], auto_update=0)
|
|
queue.add(Job(mode=DownloadMode.NEW, story_id=ids[1]))
|
|
store.update_story(ids[2], last_checked=(now - timedelta(hours=1)).isoformat())
|
|
store.update_story(ids[3], last_checked=(now - timedelta(hours=2)).isoformat())
|
|
for sid in ids[4:]:
|
|
store.set_setting(sid, "recheck_days", 3)
|
|
store.set_setting(ids[5], "last_full_recheck", (now - timedelta(days=3)).isoformat())
|
|
store.set_setting(ids[6], "last_full_recheck", (now - timedelta(days=1)).isoformat())
|
|
o._queue_due_updates()
|
|
queued = {j["story_id"]: j["mode"] for j in queue.get_status()}
|
|
assert queued == {ids[1]: "new", ids[3]: "new", ids[4]: "recheck_all", ids[5]: "recheck_all", ids[6]: "new"}
|
|
o._queue_due_updates()
|
|
assert len(queue.get_status()) == 5
|
|
monkeypatch.setattr(config, "UPDATE_INTERVAL_HOURS", 0)
|
|
extra = add_story(store, "extra")
|
|
o._queue_due_updates()
|
|
assert not queue.has_active(extra)
|
|
|
|
|
|
def test_scheduler_recovers_from_backup_and_queue_errors(state, monkeypatch):
|
|
backup = Mock(side_effect=[OSError("disk full"), None])
|
|
due = Mock(side_effect=[ValueError("bad date"), None])
|
|
sleep = Mock(side_effect=[None, StopLoop()])
|
|
monkeypatch.setattr(o, "daily_backup", backup)
|
|
monkeypatch.setattr(o, "_queue_due_updates", due)
|
|
monkeypatch.setattr(o.time, "sleep", sleep)
|
|
with pytest.raises(StopLoop):
|
|
o.scheduler_loop()
|
|
assert backup.call_count == due.call_count == 2
|
|
assert [c.args for c in sleep.call_args_list] == [(60,), (60,)]
|
|
|
|
|
|
@pytest.fixture
|
|
def site(state, monkeypatch):
|
|
store, queue = state
|
|
sid = add_story(store, ingested_at="2026-01-01", site_values={"title": "Title"})
|
|
for i in range(1, 4):
|
|
html = f"<p>old {i}</p>"
|
|
store.add_chapter(sid, f"chapter-{i}", f"Chapter {i}", html, text_hash(html), "new", i)
|
|
adapter = SimpleNamespace(normalize_chapterurl=lambda url: url)
|
|
snap = fff.SiteSnapshot(
|
|
url=store.get_story(sid)["url"],
|
|
site="www.royalroad.com",
|
|
title="Title",
|
|
author="Writer",
|
|
metahtml="metadata",
|
|
chapters=[
|
|
{"url": "chapter-1", "title": "Renamed"},
|
|
{"url": "chapter-2", "title": "Chapter 2"},
|
|
{"url": "chapter-4", "title": "New"},
|
|
],
|
|
calibre={"title": "Title"},
|
|
)
|
|
fetch = Mock(side_effect=lambda a, url, i: f"<p>fresh {url}</p>")
|
|
build, deliver = Mock(), Mock(return_value="Sent")
|
|
monkeypatch.setattr(fff, "get_adapter", lambda url: adapter)
|
|
monkeypatch.setattr(fff, "fetch_story", lambda a: snap)
|
|
monkeypatch.setattr(fff, "fetch_chapter", fetch)
|
|
monkeypatch.setattr(fff, "collected_images", lambda a: [])
|
|
monkeypatch.setattr(o, "_build", build)
|
|
monkeypatch.setattr(o, "_linked_book", lambda story: None)
|
|
monkeypatch.setattr(o, "_deliver", deliver)
|
|
return SimpleNamespace(
|
|
store=store, queue=queue, id=sid, snap=snap, adapter=adapter, fetch=fetch, build=build, deliver=deliver
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("mode", [DownloadMode.METADATA, DownloadMode.FULL, DownloadMode.RECHECK])
|
|
def test_download_modes_preserve_history_and_select_the_right_chapters(site, mode):
|
|
original = site.store.list_chapters(site.id)
|
|
job = Job(mode=mode, story_id=site.id, chapter_ids=[original[1]["id"], original[2]["id"]])
|
|
result = o._run_site_job(job)
|
|
chapters = site.store.list_chapters(site.id)
|
|
assert chapters[0]["title"] == "Renamed"
|
|
assert all(site.store.revision_html(c["id"], 1) == f"<p>old {n}</p>" for n, c in enumerate(original, 1))
|
|
fetched = [call.args[1] for call in site.fetch.call_args_list]
|
|
removed = site.store.get_chapter(original[2]["id"])
|
|
assert not removed["on_site"]
|
|
if mode == DownloadMode.METADATA:
|
|
assert fetched == []
|
|
assert len(chapters) == 3
|
|
assert removed["included"]
|
|
elif mode == DownloadMode.FULL:
|
|
assert set(fetched) == {"chapter-1", "chapter-2", "chapter-4"}
|
|
assert not removed["included"]
|
|
assert "1 taken out" in result
|
|
assert site.store.get_setting(site.id, "last_full_recheck")
|
|
else:
|
|
assert set(fetched) == {"chapter-1", "chapter-2"}
|
|
assert job.result["skipped"] == ["Chapter 3"]
|
|
assert removed["included"]
|
|
site.build.assert_called_once()
|
|
site.deliver.assert_called_once()
|
|
|
|
|
|
def test_failed_new_chapter_keeps_successful_chapters_and_images(site, monkeypatch):
|
|
site.snap.chapters.append({"url": "chapter-5", "title": "Broken"})
|
|
|
|
def fetch(adapter, url, index):
|
|
if url == "chapter-5":
|
|
raise OSError("timeout")
|
|
return "<p>ok</p>"
|
|
|
|
site.fetch.side_effect = fetch
|
|
image = {"url": "image", "ext": "png", "mime": "image/png", "data": b"image"}
|
|
site.snap.cover = {**image, "url": "cover"}
|
|
monkeypatch.setattr(fff, "collected_images", lambda a: [image, image])
|
|
job = Job(mode=DownloadMode.NEW, story_id=site.id)
|
|
assert "1 failed" in o._run_site_job(job)
|
|
assert job.result["failed"] == ["Broken: timeout"]
|
|
assert len(site.store.list_chapters(site.id)) == 4
|
|
assert site.store.get_story(site.id)["status"] == "error"
|
|
assert site.store.image_urls(site.id) == {"image"}
|
|
assert site.store.cover(site.id)["url"] == "cover"
|
|
|
|
|
|
def test_new_story_with_no_downloaded_chapters_fails_without_delivery(site):
|
|
site.fetch.side_effect = OSError("offline")
|
|
job = Job(mode=DownloadMode.NEW, url="new-url")
|
|
site.snap.url = "new-url"
|
|
with pytest.raises(o.StoryError, match="No chapters downloaded"):
|
|
o._run_site_job(job)
|
|
assert job.story_id != site.id
|
|
assert site.store.list_chapters(job.story_id) == []
|
|
site.deliver.assert_not_called()
|
|
|
|
|
|
def test_normalized_chapter_urls_keep_the_existing_identity(site):
|
|
original = site.store.list_chapters(site.id)[0]
|
|
site.adapter.normalize_chapterurl = lambda url: "renamed-url" if url == "chapter-1" else url
|
|
site.snap.chapters[0]["url"] = "renamed-url"
|
|
o._run_site_job(Job(mode=DownloadMode.RECHECK_ALL, story_id=site.id))
|
|
updated = site.store.get_chapter(original["id"])
|
|
assert updated["url"] == "renamed-url"
|
|
assert updated["current_rev"] == 2
|
|
assert len(site.store.list_chapters(site.id)) == 4
|
|
|
|
|
|
def test_inclusion_changes_only_the_requested_story_and_queues_a_rebuild(state):
|
|
store, queue = state
|
|
sid, other = add_story(store), add_story(store, "other")
|
|
cid = store.add_chapter(sid, "chapter", "One", "<p>text</p>", "h", "new", 0)
|
|
foreign = store.add_chapter(other, "foreign", "Other", "<p>other</p>", "h", "new", 0)
|
|
result = o.set_chapters_included(sid, [cid, foreign, 999], False)
|
|
assert result["mode"] == "rebuild"
|
|
assert not store.get_chapter(cid)["included"]
|
|
assert store.get_chapter(foreign)["included"]
|
|
assert store.revision_html(cid, 1) == "<p>text</p>"
|
|
o.set_chapters_included(sid, [cid], True)
|
|
assert store.get_chapter(cid)["included"]
|
|
assert len(queue.get_status()) == 1
|
|
|
|
|
|
def test_move_chapters_keeps_their_order_and_queues_a_rebuild(state):
|
|
store, queue = state
|
|
sid, other = add_story(store), add_story(store, "other")
|
|
ids = [store.add_chapter(sid, f"c{n}", str(n), "<p>x</p>", "h", "new", n) for n in range(5)]
|
|
foreign = store.add_chapter(other, "foreign", "Other", "<p>x</p>", "h", "new", 0)
|
|
|
|
def titles():
|
|
return [c["title"] for c in store.list_chapters(sid)]
|
|
|
|
# The selection order does not count, and a chapter of another story is ignored.
|
|
assert o.move_chapters(sid, [ids[3], ids[1], foreign], ids[4])["mode"] == "rebuild"
|
|
assert titles() == ["0", "2", "4", "1", "3"]
|
|
o.move_chapters(sid, [ids[4], ids[3]], None)
|
|
assert titles() == ["4", "3", "0", "2", "1"]
|
|
assert len(queue.get_status()) == 1
|
|
with pytest.raises(ValueError, match="Select"):
|
|
o.move_chapters(sid, [foreign], None)
|
|
with pytest.raises(ValueError, match="moves with them"):
|
|
o.move_chapters(sid, [ids[0], ids[1]], ids[1])
|
|
with pytest.raises(KeyError):
|
|
o.move_chapters(sid, [ids[0]], foreign)
|
|
|
|
|
|
def test_delete_chapters_removes_their_revisions_and_keeps_one_chapter(state):
|
|
store, queue = state
|
|
sid, other = add_story(store), add_story(store, "other")
|
|
ids = [store.add_chapter(sid, f"c{n}", str(n), "<p>x</p>", "h", "new", n) for n in range(3)]
|
|
store.add_revision(ids[1], "<p>y</p>", "h2", "recheck")
|
|
foreign = store.add_chapter(other, "foreign", "Other", "<p>x</p>", "h", "new", 0)
|
|
|
|
assert o.delete_chapters(sid, [ids[1], foreign])["mode"] == "rebuild"
|
|
assert [c["id"] for c in store.list_chapters(sid)] == [ids[0], ids[2]]
|
|
assert store.list_revisions(ids[1]) == []
|
|
assert store.get_chapter(foreign)
|
|
with pytest.raises(ValueError, match="Select"):
|
|
o.delete_chapters(sid, [foreign])
|
|
with pytest.raises(ValueError, match="at least one chapter"):
|
|
o.delete_chapters(sid, [ids[0], ids[2]])
|
|
assert len(store.list_chapters(sid)) == 2
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mode,chapters,fields,message",
|
|
[
|
|
("import", [], [], "library import"),
|
|
("recheck", [], [], "at least one chapter"),
|
|
("new", [], ["unknown"], "Unknown metadata"),
|
|
("invalid", [], [], "not a valid"),
|
|
],
|
|
)
|
|
def test_invalid_job_requests_do_not_enter_the_queue(state, mode, chapters, fields, message):
|
|
store, queue = state
|
|
sid = add_story(store)
|
|
with pytest.raises(ValueError, match=message):
|
|
o.queue_story(sid, mode, chapters, fields)
|
|
assert queue.get_status() == []
|