tim
81efc03b75
An error outside a job, such as a full disk, stopped the thread, and no job ran until a restart. The health check now answers 503 when a job thread stopped.
348 lines
15 KiB
Python
348 lines
15 KiB
Python
"""Worker and scheduler transitions, with real temporary stores and no background threads."""
|
|
|
|
from datetime import datetime, timedelta
|
|
from types import SimpleNamespace
|
|
from unittest.mock import Mock
|
|
|
|
import pytest
|
|
|
|
from bindery import config, fff, web
|
|
from bindery import orchestrator as o
|
|
from bindery.diffing import text_hash
|
|
from bindery.jobs import DownloadMode, Job, JobQueue, QueueStatus
|
|
from bindery.store import Store
|
|
|
|
|
|
class StopLoop(BaseException):
|
|
"""Stop an infinite loop without entering its application error handler."""
|
|
|
|
|
|
@pytest.fixture
|
|
def state(monkeypatch, tmp_path):
|
|
store = Store(tmp_path / "stories.db")
|
|
queue = JobQueue(tmp_path / "jobs.db")
|
|
monkeypatch.setattr(o, "store", store)
|
|
monkeypatch.setattr(o, "job_queue", queue)
|
|
monkeypatch.setattr(config, "CWA_URL", "")
|
|
monkeypatch.setattr(config, "SITE_DELAY", 7)
|
|
monkeypatch.setattr(config, "TMP_DIR", tmp_path / "tmp")
|
|
config.TMP_DIR.mkdir()
|
|
return store, queue
|
|
|
|
|
|
def add_story(store, url="https://www.royalroad.com/fiction/1234", **values):
|
|
sid = store.create_story(url, "www.royalroad.com", "Title", "Writer", [])
|
|
store.update_story(sid, **values)
|
|
return sid
|
|
|
|
|
|
@pytest.mark.parametrize("mode", list(DownloadMode))
|
|
def test_dispatches_each_download_mode(state, monkeypatch, mode):
|
|
handlers = {
|
|
name: Mock(return_value=name) for name in ("_run_import", "_run_import_file", "_run_rebuild", "_run_site_job")
|
|
}
|
|
for name, handler in handlers.items():
|
|
monkeypatch.setattr(o, name, handler)
|
|
expected = {
|
|
DownloadMode.IMPORT: "_run_import",
|
|
DownloadMode.IMPORT_FILE: "_run_import_file",
|
|
DownloadMode.REBUILD: "_run_rebuild",
|
|
}.get(mode, "_run_site_job")
|
|
job = Job(mode=mode)
|
|
assert o._run_job(job) == expected
|
|
handlers[expected].assert_called_once_with(job)
|
|
assert sum(h.call_count for h in handlers.values()) == 1
|
|
|
|
|
|
@pytest.mark.parametrize("error", [None, o.StoryError("site refused"), RuntimeError("broken")])
|
|
@pytest.mark.parametrize("with_story", [False, True])
|
|
def test_worker_persists_outcomes_and_continues_after_failure(state, monkeypatch, tmp_path, error, with_story):
|
|
store, queue = state
|
|
sid = add_story(store) if with_story else None
|
|
first = queue.add(Job(mode=DownloadMode.NEW, story_id=sid, url="first"))
|
|
second = queue.add(Job(mode=DownloadMode.REBUILD, story_id=sid))
|
|
monkeypatch.setattr(queue, "get_next", Mock(side_effect=[None, first, second, StopLoop()]))
|
|
sleeps = Mock()
|
|
monkeypatch.setattr(o.time, "sleep", sleeps)
|
|
|
|
def run(job):
|
|
assert job.status == QueueStatus.RUNNING
|
|
if job is first and error:
|
|
raise error
|
|
return "Sent"
|
|
|
|
monkeypatch.setattr(o, "_run_job", run)
|
|
with pytest.raises(StopLoop):
|
|
o.download_loop()
|
|
saved = {j["id"]: j for j in JobQueue(tmp_path / "jobs.db").get_status()}
|
|
assert saved[second.id]["status"] == "done"
|
|
assert saved[second.id]["progress"] == 1.0
|
|
assert saved[first.id]["status"] == ("error" if error else "done")
|
|
expected = "RuntimeError: broken" if isinstance(error, RuntimeError) else str(error) if error else "Sent"
|
|
assert saved[first.id]["message"] == expected
|
|
if error and sid:
|
|
story = store.get_story(sid)
|
|
assert (story["status"], story["message"]) == ("error", expected)
|
|
assert story["last_checked"]
|
|
assert [c.args[0] for c in sleeps.call_args_list] == [2, 7]
|
|
|
|
|
|
def test_worker_goes_on_after_an_error_of_the_queue(state, monkeypatch, caplog):
|
|
store, queue = state
|
|
job = queue.add(Job(mode=DownloadMode.REBUILD))
|
|
monkeypatch.setattr(queue, "get_next", Mock(side_effect=[OSError("disk full"), job, StopLoop()]))
|
|
monkeypatch.setattr(queue, "update", Mock(side_effect=[OSError("disk full"), None, None]))
|
|
sleeps = Mock()
|
|
monkeypatch.setattr(o.time, "sleep", sleeps)
|
|
with pytest.raises(StopLoop):
|
|
o.download_loop()
|
|
assert caplog.text.count("Job loop error: disk full") == 2
|
|
assert [c.args[0] for c in sleeps.call_args_list] == [2, 2]
|
|
|
|
|
|
def test_health_reports_a_stopped_job_thread(state, monkeypatch):
|
|
client = web.app.test_client()
|
|
assert o.stopped_threads() == []
|
|
alive, dead = Mock(), Mock()
|
|
alive.is_alive.return_value, dead.is_alive.return_value, dead.name = True, False, "download_loop"
|
|
monkeypatch.setattr(o, "_threads", [alive, dead])
|
|
response = client.get("/api/health")
|
|
assert response.status_code == 503
|
|
assert response.json == {"status": "error", "error": "Stopped: download_loop. Restart Bindery."}
|
|
monkeypatch.setattr(o, "_threads", [alive])
|
|
assert client.get("/api/health").json == {"status": "ok"}
|
|
|
|
|
|
def test_start_threads_keeps_the_threads_for_the_health_check(state, monkeypatch):
|
|
monkeypatch.setattr(o, "_threads", [])
|
|
monkeypatch.setattr(o, "download_loop", lambda: None)
|
|
monkeypatch.setattr(o, "scheduler_loop", lambda: None)
|
|
o.start_threads()
|
|
for thread in o._threads:
|
|
thread.join()
|
|
assert o.stopped_threads() == ["<lambda>", "<lambda>"]
|
|
|
|
|
|
def test_scheduler_skips_disabled_busy_and_recent_stories(state, monkeypatch):
|
|
store, queue = state
|
|
now = datetime(2026, 6, 1, 12)
|
|
|
|
class Clock(datetime):
|
|
@classmethod
|
|
def now(cls):
|
|
return now
|
|
|
|
monkeypatch.setattr(o, "datetime", Clock)
|
|
monkeypatch.setattr(config, "UPDATE_INTERVAL_HOURS", 2)
|
|
ids = [add_story(store, f"url-{n}") for n in range(7)]
|
|
store.update_story(ids[0], auto_update=0)
|
|
queue.add(Job(mode=DownloadMode.NEW, story_id=ids[1]))
|
|
store.update_story(ids[2], last_checked=(now - timedelta(hours=1)).isoformat())
|
|
store.update_story(ids[3], last_checked=(now - timedelta(hours=2)).isoformat())
|
|
for sid in ids[4:]:
|
|
store.set_setting(sid, "recheck_days", 3)
|
|
store.set_setting(ids[5], "last_full_recheck", (now - timedelta(days=3)).isoformat())
|
|
store.set_setting(ids[6], "last_full_recheck", (now - timedelta(days=1)).isoformat())
|
|
o._queue_due_updates()
|
|
queued = {j["story_id"]: j["mode"] for j in queue.get_status()}
|
|
assert queued == {ids[1]: "new", ids[3]: "new", ids[4]: "recheck_all", ids[5]: "recheck_all", ids[6]: "new"}
|
|
o._queue_due_updates()
|
|
assert len(queue.get_status()) == 5
|
|
monkeypatch.setattr(config, "UPDATE_INTERVAL_HOURS", 0)
|
|
extra = add_story(store, "extra")
|
|
o._queue_due_updates()
|
|
assert not queue.has_active(extra)
|
|
|
|
|
|
def test_scheduler_recovers_from_backup_and_queue_errors(state, monkeypatch):
|
|
backup = Mock(side_effect=[OSError("disk full"), None])
|
|
due = Mock(side_effect=[ValueError("bad date"), None])
|
|
sleep = Mock(side_effect=[None, StopLoop()])
|
|
monkeypatch.setattr(o, "daily_backup", backup)
|
|
monkeypatch.setattr(o, "_queue_due_updates", due)
|
|
monkeypatch.setattr(o.time, "sleep", sleep)
|
|
with pytest.raises(StopLoop):
|
|
o.scheduler_loop()
|
|
assert backup.call_count == due.call_count == 2
|
|
assert [c.args for c in sleep.call_args_list] == [(60,), (60,)]
|
|
|
|
|
|
@pytest.fixture
|
|
def site(state, monkeypatch):
|
|
store, queue = state
|
|
sid = add_story(store, ingested_at="2026-01-01", site_values={"title": "Title"})
|
|
for i in range(1, 4):
|
|
html = f"<p>old {i}</p>"
|
|
store.add_chapter(sid, f"chapter-{i}", f"Chapter {i}", html, text_hash(html), "new", i)
|
|
adapter = SimpleNamespace(normalize_chapterurl=lambda url: url)
|
|
snap = fff.SiteSnapshot(
|
|
url=store.get_story(sid)["url"],
|
|
site="www.royalroad.com",
|
|
title="Title",
|
|
author="Writer",
|
|
metahtml="metadata",
|
|
chapters=[
|
|
{"url": "chapter-1", "title": "Renamed"},
|
|
{"url": "chapter-2", "title": "Chapter 2"},
|
|
{"url": "chapter-4", "title": "New"},
|
|
],
|
|
calibre={"title": "Title"},
|
|
)
|
|
fetch = Mock(side_effect=lambda a, url, i: f"<p>fresh {url}</p>")
|
|
build = Mock(side_effect=lambda story, overrides, path: path.write_bytes(b"epub"))
|
|
deliver = Mock(return_value="Sent")
|
|
monkeypatch.setattr(fff, "get_adapter", lambda url: adapter)
|
|
monkeypatch.setattr(fff, "fetch_story", lambda a: snap)
|
|
monkeypatch.setattr(fff, "fetch_chapter", fetch)
|
|
monkeypatch.setattr(fff, "collected_images", lambda a: [])
|
|
monkeypatch.setattr(o, "_build", build)
|
|
monkeypatch.setattr(o, "_linked_book", lambda story: None)
|
|
monkeypatch.setattr(o, "_deliver", deliver)
|
|
return SimpleNamespace(
|
|
store=store, queue=queue, id=sid, snap=snap, adapter=adapter, fetch=fetch, build=build, deliver=deliver
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("mode", [DownloadMode.METADATA, DownloadMode.FULL, DownloadMode.RECHECK])
|
|
def test_download_modes_preserve_history_and_select_the_right_chapters(site, mode):
|
|
original = site.store.list_chapters(site.id)
|
|
job = Job(mode=mode, story_id=site.id, chapter_ids=[original[1]["id"], original[2]["id"]])
|
|
result = o._run_site_job(job)
|
|
chapters = site.store.list_chapters(site.id)
|
|
assert chapters[0]["title"] == "Renamed"
|
|
assert all(site.store.revision_html(c["id"], 1) == f"<p>old {n}</p>" for n, c in enumerate(original, 1))
|
|
fetched = [call.args[1] for call in site.fetch.call_args_list]
|
|
removed = site.store.get_chapter(original[2]["id"])
|
|
assert not removed["on_site"]
|
|
if mode == DownloadMode.METADATA:
|
|
assert fetched == []
|
|
assert len(chapters) == 3
|
|
assert removed["included"]
|
|
elif mode == DownloadMode.FULL:
|
|
assert set(fetched) == {"chapter-1", "chapter-2", "chapter-4"}
|
|
assert not removed["included"]
|
|
assert "1 taken out" in result
|
|
assert site.store.get_setting(site.id, "last_full_recheck")
|
|
else:
|
|
assert set(fetched) == {"chapter-1", "chapter-2"}
|
|
assert job.result["skipped"] == ["Chapter 3"]
|
|
assert removed["included"]
|
|
site.build.assert_called_once()
|
|
site.deliver.assert_called_once()
|
|
|
|
|
|
def test_failed_new_chapter_keeps_successful_chapters_and_images(site, monkeypatch):
|
|
site.snap.chapters.append({"url": "chapter-5", "title": "Broken"})
|
|
|
|
def fetch(adapter, url, index):
|
|
if url == "chapter-5":
|
|
raise OSError("timeout")
|
|
return "<p>ok</p>"
|
|
|
|
site.fetch.side_effect = fetch
|
|
image = {"url": "image", "ext": "png", "mime": "image/png", "data": b"image"}
|
|
site.snap.cover = {**image, "url": "cover"}
|
|
monkeypatch.setattr(fff, "collected_images", lambda a: [image, image])
|
|
job = Job(mode=DownloadMode.NEW, story_id=site.id)
|
|
assert "1 failed" in o._run_site_job(job)
|
|
assert job.result["failed"] == ["Broken: timeout"]
|
|
assert len(site.store.list_chapters(site.id)) == 4
|
|
assert site.store.get_story(site.id)["status"] == "error"
|
|
assert site.store.image_urls(site.id) == {"image"}
|
|
assert site.store.cover(site.id)["url"] == "cover"
|
|
|
|
|
|
def test_new_story_with_no_downloaded_chapters_fails_without_delivery(site):
|
|
site.fetch.side_effect = OSError("offline")
|
|
job = Job(mode=DownloadMode.NEW, url="new-url")
|
|
site.snap.url = "new-url"
|
|
with pytest.raises(o.StoryError, match="No chapters downloaded"):
|
|
o._run_site_job(job)
|
|
assert job.story_id != site.id
|
|
assert site.store.list_chapters(job.story_id) == []
|
|
site.deliver.assert_not_called()
|
|
|
|
|
|
def test_normalized_chapter_urls_keep_the_existing_identity(site):
|
|
original = site.store.list_chapters(site.id)[0]
|
|
site.adapter.normalize_chapterurl = lambda url: "renamed-url" if url == "chapter-1" else url
|
|
site.snap.chapters[0]["url"] = "renamed-url"
|
|
o._run_site_job(Job(mode=DownloadMode.RECHECK_ALL, story_id=site.id))
|
|
updated = site.store.get_chapter(original["id"])
|
|
assert updated["url"] == "renamed-url"
|
|
assert updated["current_rev"] == 2
|
|
assert len(site.store.list_chapters(site.id)) == 4
|
|
|
|
|
|
def test_inclusion_changes_only_the_requested_story_and_queues_a_rebuild(state):
|
|
store, queue = state
|
|
sid, other = add_story(store), add_story(store, "other")
|
|
cid = store.add_chapter(sid, "chapter", "One", "<p>text</p>", "h", "new", 0)
|
|
foreign = store.add_chapter(other, "foreign", "Other", "<p>other</p>", "h", "new", 0)
|
|
result = o.set_chapters_included(sid, [cid, foreign, 999], False)
|
|
assert result["mode"] == "rebuild"
|
|
assert not store.get_chapter(cid)["included"]
|
|
assert store.get_chapter(foreign)["included"]
|
|
assert store.revision_html(cid, 1) == "<p>text</p>"
|
|
o.set_chapters_included(sid, [cid], True)
|
|
assert store.get_chapter(cid)["included"]
|
|
assert len(queue.get_status()) == 1
|
|
|
|
|
|
def test_move_chapters_keeps_their_order_and_queues_a_rebuild(state):
|
|
store, queue = state
|
|
sid, other = add_story(store), add_story(store, "other")
|
|
ids = [store.add_chapter(sid, f"c{n}", str(n), "<p>x</p>", "h", "new", n) for n in range(5)]
|
|
foreign = store.add_chapter(other, "foreign", "Other", "<p>x</p>", "h", "new", 0)
|
|
|
|
def titles():
|
|
return [c["title"] for c in store.list_chapters(sid)]
|
|
|
|
# The selection order does not count, and a chapter of another story is ignored.
|
|
assert o.move_chapters(sid, [ids[3], ids[1], foreign], ids[4])["mode"] == "rebuild"
|
|
assert titles() == ["0", "2", "4", "1", "3"]
|
|
o.move_chapters(sid, [ids[4], ids[3]], None)
|
|
assert titles() == ["4", "3", "0", "2", "1"]
|
|
assert len(queue.get_status()) == 1
|
|
with pytest.raises(ValueError, match="Select"):
|
|
o.move_chapters(sid, [foreign], None)
|
|
with pytest.raises(ValueError, match="moves with them"):
|
|
o.move_chapters(sid, [ids[0], ids[1]], ids[1])
|
|
with pytest.raises(KeyError):
|
|
o.move_chapters(sid, [ids[0]], foreign)
|
|
|
|
|
|
def test_delete_chapters_removes_their_revisions_and_keeps_one_chapter(state):
|
|
store, queue = state
|
|
sid, other = add_story(store), add_story(store, "other")
|
|
ids = [store.add_chapter(sid, f"c{n}", str(n), "<p>x</p>", "h", "new", n) for n in range(3)]
|
|
store.add_revision(ids[1], "<p>y</p>", "h2", "recheck")
|
|
foreign = store.add_chapter(other, "foreign", "Other", "<p>x</p>", "h", "new", 0)
|
|
|
|
assert o.delete_chapters(sid, [ids[1], foreign])["mode"] == "rebuild"
|
|
assert [c["id"] for c in store.list_chapters(sid)] == [ids[0], ids[2]]
|
|
assert store.list_revisions(ids[1]) == []
|
|
assert store.get_chapter(foreign)
|
|
with pytest.raises(ValueError, match="Select"):
|
|
o.delete_chapters(sid, [foreign])
|
|
with pytest.raises(ValueError, match="at least one chapter"):
|
|
o.delete_chapters(sid, [ids[0], ids[2]])
|
|
assert len(store.list_chapters(sid)) == 2
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mode,chapters,fields,message",
|
|
[
|
|
("import", [], [], "library import"),
|
|
("recheck", [], [], "at least one chapter"),
|
|
("new", [], ["unknown"], "Unknown metadata"),
|
|
("invalid", [], [], "not a valid"),
|
|
],
|
|
)
|
|
def test_invalid_job_requests_do_not_enter_the_queue(state, mode, chapters, fields, message):
|
|
store, queue = state
|
|
sid = add_story(store)
|
|
with pytest.raises(ValueError, match=message):
|
|
o.queue_story(sid, mode, chapters, fields)
|
|
assert queue.get_status() == []
|