Keep the job thread alive after an error of the queue

An error outside a job, such as a full disk, stopped the thread, and no
job ran until a restart. The health check now answers 503 when a job
thread stopped.
This commit is contained in:
tim
2026-10-02 23:37:33 +02:00
parent c0504135b2
commit 81efc03b75
4 changed files with 80 additions and 15 deletions
+37 -1
View File
@@ -6,7 +6,7 @@ from unittest.mock import Mock
import pytest
from bindery import config, fff
from bindery import config, fff, web
from bindery import orchestrator as o
from bindery.diffing import text_hash
from bindery.jobs import DownloadMode, Job, JobQueue, QueueStatus
@@ -87,6 +87,42 @@ def test_worker_persists_outcomes_and_continues_after_failure(state, monkeypatch
assert [c.args[0] for c in sleeps.call_args_list] == [2, 7]
def test_worker_goes_on_after_an_error_of_the_queue(state, monkeypatch, caplog):
store, queue = state
job = queue.add(Job(mode=DownloadMode.REBUILD))
monkeypatch.setattr(queue, "get_next", Mock(side_effect=[OSError("disk full"), job, StopLoop()]))
monkeypatch.setattr(queue, "update", Mock(side_effect=[OSError("disk full"), None, None]))
sleeps = Mock()
monkeypatch.setattr(o.time, "sleep", sleeps)
with pytest.raises(StopLoop):
o.download_loop()
assert caplog.text.count("Job loop error: disk full") == 2
assert [c.args[0] for c in sleeps.call_args_list] == [2, 2]
def test_health_reports_a_stopped_job_thread(state, monkeypatch):
client = web.app.test_client()
assert o.stopped_threads() == []
alive, dead = Mock(), Mock()
alive.is_alive.return_value, dead.is_alive.return_value, dead.name = True, False, "download_loop"
monkeypatch.setattr(o, "_threads", [alive, dead])
response = client.get("/api/health")
assert response.status_code == 503
assert response.json == {"status": "error", "error": "Stopped: download_loop. Restart Bindery."}
monkeypatch.setattr(o, "_threads", [alive])
assert client.get("/api/health").json == {"status": "ok"}
def test_start_threads_keeps_the_threads_for_the_health_check(state, monkeypatch):
monkeypatch.setattr(o, "_threads", [])
monkeypatch.setattr(o, "download_loop", lambda: None)
monkeypatch.setattr(o, "scheduler_loop", lambda: None)
o.start_threads()
for thread in o._threads:
thread.join()
assert o.stopped_threads() == ["<lambda>", "<lambda>"]
def test_scheduler_skips_disabled_busy_and_recent_stories(state, monkeypatch):
store, queue = state
now = datetime(2026, 6, 1, 12)