Keep the job thread alive after an error of the queue
An error outside a job, such as a full disk, stopped the thread, and no job ran until a restart. The health check now answers 503 when a job thread stopped.
This commit is contained in:
@@ -6,7 +6,7 @@ from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from bindery import config, fff
|
||||
from bindery import config, fff, web
|
||||
from bindery import orchestrator as o
|
||||
from bindery.diffing import text_hash
|
||||
from bindery.jobs import DownloadMode, Job, JobQueue, QueueStatus
|
||||
@@ -87,6 +87,42 @@ def test_worker_persists_outcomes_and_continues_after_failure(state, monkeypatch
|
||||
assert [c.args[0] for c in sleeps.call_args_list] == [2, 7]
|
||||
|
||||
|
||||
def test_worker_goes_on_after_an_error_of_the_queue(state, monkeypatch, caplog):
|
||||
store, queue = state
|
||||
job = queue.add(Job(mode=DownloadMode.REBUILD))
|
||||
monkeypatch.setattr(queue, "get_next", Mock(side_effect=[OSError("disk full"), job, StopLoop()]))
|
||||
monkeypatch.setattr(queue, "update", Mock(side_effect=[OSError("disk full"), None, None]))
|
||||
sleeps = Mock()
|
||||
monkeypatch.setattr(o.time, "sleep", sleeps)
|
||||
with pytest.raises(StopLoop):
|
||||
o.download_loop()
|
||||
assert caplog.text.count("Job loop error: disk full") == 2
|
||||
assert [c.args[0] for c in sleeps.call_args_list] == [2, 2]
|
||||
|
||||
|
||||
def test_health_reports_a_stopped_job_thread(state, monkeypatch):
|
||||
client = web.app.test_client()
|
||||
assert o.stopped_threads() == []
|
||||
alive, dead = Mock(), Mock()
|
||||
alive.is_alive.return_value, dead.is_alive.return_value, dead.name = True, False, "download_loop"
|
||||
monkeypatch.setattr(o, "_threads", [alive, dead])
|
||||
response = client.get("/api/health")
|
||||
assert response.status_code == 503
|
||||
assert response.json == {"status": "error", "error": "Stopped: download_loop. Restart Bindery."}
|
||||
monkeypatch.setattr(o, "_threads", [alive])
|
||||
assert client.get("/api/health").json == {"status": "ok"}
|
||||
|
||||
|
||||
def test_start_threads_keeps_the_threads_for_the_health_check(state, monkeypatch):
|
||||
monkeypatch.setattr(o, "_threads", [])
|
||||
monkeypatch.setattr(o, "download_loop", lambda: None)
|
||||
monkeypatch.setattr(o, "scheduler_loop", lambda: None)
|
||||
o.start_threads()
|
||||
for thread in o._threads:
|
||||
thread.join()
|
||||
assert o.stopped_threads() == ["<lambda>", "<lambda>"]
|
||||
|
||||
|
||||
def test_scheduler_skips_disabled_busy_and_recent_stories(state, monkeypatch):
|
||||
store, queue = state
|
||||
now = datetime(2026, 6, 1, 12)
|
||||
|
||||
Reference in New Issue
Block a user