tim
eea6adcc91
Update Last Changed only when Bindery stores a new chapter or adds a new chapter revision. Manual ePub generation, rebuilds, metadata edits and rule changes leave it unchanged. Save the timestamp in the same transaction as the chapter or revision. This preserves the date if a later build fails and prevents retries from moving it forward without another chapter change. Add regression tests for new chapters, revisions, forced rebuilds, failed builds, retries and transaction rollback. Adjust OPDS test data setup and document what Last Changed records.
390 lines
16 KiB
Python
390 lines
16 KiB
Python
"""A book is built and sent only when its chapters, rules or metadata changed."""
|
|
|
|
import dataclasses
|
|
from unittest.mock import Mock
|
|
|
|
import pytest
|
|
from test_api import H
|
|
from test_api import client as client
|
|
from test_deliver import STORY_URL, add_story, ingest_files, linked_story, site, update
|
|
from test_deliver import fake as fake
|
|
from test_deliver import jobs as jobs
|
|
from test_deliver import story_epub as story_epub
|
|
|
|
from bindery import config, orchestrator, pages
|
|
from bindery import store as storage
|
|
from bindery.jobs import DownloadMode, Job
|
|
|
|
ONE = [(1, "One", "a")]
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def builds(fake, jobs, monkeypatch, tmp_path):
|
|
"""Count the builds, in a folder of built books that starts empty."""
|
|
monkeypatch.setattr(config, "TMP_DIR", tmp_path / "tmp")
|
|
config.TMP_DIR.mkdir()
|
|
spy = Mock(side_effect=orchestrator._build)
|
|
monkeypatch.setattr(orchestrator, "_build", spy)
|
|
return spy
|
|
|
|
|
|
def story(story_id):
|
|
return orchestrator.store.get_story(story_id)
|
|
|
|
|
|
def test_an_unchanged_story_is_not_built_or_sent_again(builds, monkeypatch):
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, ONE)
|
|
assert "library book 1" in update(story_id)
|
|
changed = story(story_id)["last_changed"]
|
|
orchestrator.store.update_story(story_id, last_changed="2020-01-01T00:00:00")
|
|
for mode in (DownloadMode.NEW, DownloadMode.RECHECK_ALL, DownloadMode.METADATA, DownloadMode.FULL):
|
|
assert update(story_id, mode).startswith("No changes")
|
|
assert changed and story(story_id)["last_changed"] == "2020-01-01T00:00:00"
|
|
assert builds.call_count == 1
|
|
assert len(ingest_files()) == 2 # one epub and its sidecar
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"change", ["chapter", "revision", "renamed", "rule", "global rule", "paragraphs", "metadata", "included"]
|
|
)
|
|
def test_each_change_builds_and_sends_the_book(builds, monkeypatch, change):
|
|
story_id = linked_story([(1, "One", "a"), (2, "Two", "b")])
|
|
site(monkeypatch, [(1, "One", "a"), (2, "Two", "b")])
|
|
update(story_id)
|
|
previous = "2020-01-01T00:00:00"
|
|
orchestrator.store.update_story(story_id, last_changed=previous)
|
|
if change == "chapter":
|
|
site(monkeypatch, [(1, "One", "a"), (2, "Two", "b"), (3, "Three", "c")])
|
|
elif change == "revision":
|
|
site(monkeypatch, [(1, "One", "a"), (2, "Two", "new text")])
|
|
elif change == "renamed":
|
|
site(monkeypatch, [(1, "One", "a"), (2, "Renamed", "b")])
|
|
elif change == "rule":
|
|
orchestrator.save_rules(story_id, [{"pattern": "a", "replacement": "b"}])
|
|
elif change == "global rule":
|
|
orchestrator.save_rules(None, [{"pattern": "a", "replacement": "b"}])
|
|
elif change == "paragraphs":
|
|
orchestrator.save_rules(story_id, [], paragraphs=True)
|
|
elif change == "metadata":
|
|
orchestrator.store.update_story(story_id, pinned={"title": "Edited"})
|
|
else:
|
|
chapter = orchestrator.store.list_chapters(story_id)[0]
|
|
orchestrator.store.update_chapter(chapter["id"], included=0)
|
|
mode = DownloadMode.RECHECK_ALL if change == "revision" else DownloadMode.NEW
|
|
assert "library book 1" in update(story_id, mode)
|
|
assert builds.call_count == 2
|
|
assert len(ingest_files()) == 4
|
|
changed_at = story(story_id)["last_changed"]
|
|
assert (changed_at != previous) == (change in ("chapter", "revision"))
|
|
assert update(story_id).startswith("No changes")
|
|
assert story(story_id)["last_changed"] == changed_at
|
|
|
|
|
|
def test_a_held_book_that_did_not_change_is_not_built_again(builds, monkeypatch):
|
|
monkeypatch.setattr(config, "INGEST_DIR", None)
|
|
site(monkeypatch, ONE)
|
|
job = Job(mode=DownloadMode.NEW, url=STORY_URL)
|
|
assert "Not sent. INGEST_DIR is not set." in orchestrator._run_site_job(job)
|
|
changed = story(job.story_id)["last_changed"]
|
|
orchestrator.store.update_story(job.story_id, last_changed="2020-01-01T00:00:00")
|
|
assert "Not sent. INGEST_DIR is not set." in update(job.story_id)
|
|
assert changed and story(job.story_id)["last_changed"] == "2020-01-01T00:00:00"
|
|
assert builds.call_count == 1
|
|
# The book changes. Bindery builds it, also while it cannot send it.
|
|
site(monkeypatch, [*ONE, (2, "Two", "b")])
|
|
update(job.story_id)
|
|
assert builds.call_count == 2
|
|
assert story(job.story_id)["last_changed"] != "2020-01-01T00:00:00"
|
|
|
|
|
|
def test_a_released_book_goes_out_without_another_build(builds, jobs):
|
|
story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=1, calibre_uuid="uuid-1")
|
|
orchestrator.save_settings(pause_uploads=True)
|
|
orchestrator._run_rebuild(Job(mode=DownloadMode.REBUILD, story_id=story_id))
|
|
assert builds.call_count == 1 and ingest_files() == []
|
|
orchestrator.save_settings(pause_uploads=False)
|
|
assert "library book 1" in orchestrator._run_rebuild(jobs.get_next())
|
|
assert builds.call_count == 1
|
|
assert len(ingest_files()) == 2
|
|
|
|
|
|
def test_a_lost_file_is_built_again_only_to_send_it(builds):
|
|
story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=1, calibre_uuid="uuid-1")
|
|
orchestrator.save_settings(pause_uploads=True)
|
|
job = Job(mode=DownloadMode.REBUILD, story_id=story_id)
|
|
orchestrator._run_rebuild(job)
|
|
for path in config.TMP_DIR.iterdir():
|
|
path.unlink()
|
|
assert "pause" in orchestrator._run_rebuild(job)
|
|
assert builds.call_count == 1
|
|
orchestrator.store.set_setting(None, "pause_uploads", None)
|
|
assert "library book 1" in orchestrator._run_rebuild(job)
|
|
assert builds.call_count == 2
|
|
|
|
|
|
@pytest.mark.parametrize("last_changed", [None, "2020-01-01T00:00:00"])
|
|
def test_a_rebuild_without_a_change_does_nothing_unless_the_user_asks_for_it(builds, client, jobs, last_changed):
|
|
story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=1, calibre_uuid="uuid-1")
|
|
job = Job(mode=DownloadMode.REBUILD, story_id=story_id)
|
|
assert "library book 1" in orchestrator._run_rebuild(job)
|
|
orchestrator.store.update_story(story_id, last_changed=last_changed, message="before")
|
|
assert orchestrator._run_rebuild(job) == "No changes"
|
|
assert (story(story_id)["last_changed"], story(story_id)["message"]) == (last_changed, "before")
|
|
assert builds.call_count == 1
|
|
# The Rebuild button of the story page.
|
|
queued = client.post(f"/api/stories/{story_id}/download", json={"mode": "rebuild"}, headers=H).json
|
|
assert queued["options"] == {"force": True}
|
|
assert "library book 1" in orchestrator._run_rebuild(jobs.get_next())
|
|
assert builds.call_count == 2
|
|
assert len(ingest_files()) == 4
|
|
assert story(story_id)["last_changed"] == last_changed
|
|
# Other modes take no such option.
|
|
queued = client.post(f"/api/stories/{story_id}/download", json={"mode": "new"}, headers=H).json
|
|
assert queued["options"] == {}
|
|
|
|
|
|
def test_a_changed_link_gets_the_book_without_a_build(builds, fake):
|
|
story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=1, calibre_uuid="uuid-1")
|
|
job = Job(mode=DownloadMode.REBUILD, story_id=story_id)
|
|
orchestrator._run_rebuild(job)
|
|
fake.books.append(dataclasses.replace(fake.books[0], id=2, uuid="uuid-2", title="Other"))
|
|
orchestrator.link_story(story_id, "uuid-2")
|
|
assert "library book 2" in orchestrator._run_rebuild(job)
|
|
assert builds.call_count == 1
|
|
|
|
|
|
def test_a_failed_delivery_is_tried_again_without_a_build(builds, fake, monkeypatch):
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, ONE)
|
|
send = orchestrator.IngestFolder.send_update
|
|
monkeypatch.setattr(orchestrator.IngestFolder, "send_update", Mock(side_effect=OSError("disk full")))
|
|
with pytest.raises(OSError, match="disk full"):
|
|
update(story_id)
|
|
monkeypatch.setattr(orchestrator.IngestFolder, "send_update", send)
|
|
assert "library book 1" in update(story_id)
|
|
assert builds.call_count == 1
|
|
|
|
|
|
@pytest.mark.parametrize("change", ["chapter", "revision"])
|
|
def test_chapter_timestamp_survives_a_failed_build_and_retry(builds, monkeypatch, change):
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, ONE)
|
|
update(story_id)
|
|
previous = "2020-01-01T00:00:00"
|
|
orchestrator.store.update_story(story_id, last_changed=previous)
|
|
clock = ["2024-01-01T00:00:00"]
|
|
monkeypatch.setattr(storage, "now", lambda: clock[0])
|
|
site(monkeypatch, [*ONE, (2, "Two", "b")] if change == "chapter" else [(1, "One", "revised")])
|
|
job = Job(mode=DownloadMode.RECHECK_ALL, story_id=story_id)
|
|
build = orchestrator._build
|
|
monkeypatch.setattr(orchestrator, "_build", Mock(side_effect=OSError("disk full")))
|
|
with pytest.raises(OSError, match="disk full"):
|
|
orchestrator._run_site_job(job)
|
|
assert story(story_id)["last_changed"] == clock[0]
|
|
assert job.result["new"] or job.result["changed"]
|
|
clock[0] = "2024-02-01T00:00:00"
|
|
monkeypatch.setattr(orchestrator, "_build", build)
|
|
assert "library book 1" in orchestrator._run_site_job(job)
|
|
assert story(story_id)["last_changed"] == "2024-01-01T00:00:00"
|
|
assert update(story_id).startswith("No changes")
|
|
assert story(story_id)["last_changed"] == "2024-01-01T00:00:00"
|
|
|
|
|
|
def test_a_new_build_removes_the_older_file(builds, monkeypatch):
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, ONE)
|
|
update(story_id)
|
|
(first,) = config.TMP_DIR.iterdir()
|
|
site(monkeypatch, [*ONE, (2, "Two", "b")])
|
|
update(story_id)
|
|
(second,) = config.TMP_DIR.iterdir()
|
|
assert first != second and second.name.startswith(f"story-{story_id}-")
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("changes", "held", "built", "sent"),
|
|
[
|
|
({"calibre_id": 1, "calibre_uuid": "uuid-1"}, False, True, True),
|
|
({"ingested_at": "2026-01-01"}, False, True, True),
|
|
({"ingested_at": "2026-01-01"}, True, True, False),
|
|
({}, False, True, False),
|
|
({"ingested_at": "2026-01-01", "status": "error"}, False, False, False),
|
|
({"ingested_at": "2026-01-01", "no_metadata": True}, False, False, False),
|
|
],
|
|
)
|
|
def test_stories_of_an_older_bindery_get_their_keys(builds, changes, held, built, sent):
|
|
no_metadata = changes.pop("no_metadata", False)
|
|
story_id = add_story(**changes)
|
|
if no_metadata:
|
|
orchestrator.store.update_story(story_id, metahtml=None)
|
|
orchestrator.store.set_setting(story_id, "upload_held", held or None)
|
|
orchestrator.adopt_build_keys()
|
|
settings = orchestrator.store
|
|
assert bool(settings.get_setting(story_id, "built_key")) is built
|
|
assert bool(settings.get_setting(story_id, "sent_key")) is sent
|
|
if sent:
|
|
assert orchestrator._run_rebuild(Job(mode=DownloadMode.REBUILD, story_id=story_id)) == "No changes"
|
|
assert builds.call_count == 0
|
|
|
|
|
|
def test_a_story_with_a_waiting_job_gets_no_key(builds, jobs):
|
|
story_id = add_story(ingested_at="2026-01-01")
|
|
jobs.add(Job(mode=DownloadMode.NEW, story_id=story_id))
|
|
orchestrator.adopt_build_keys()
|
|
assert orchestrator.store.get_setting(story_id, "built_key") is None
|
|
|
|
|
|
def test_the_first_check_after_an_upgrade_sends_nothing(builds, monkeypatch):
|
|
story_id = linked_story(ONE)
|
|
orchestrator.store.update_story(story_id, pinned={"title": "My Title"}, site_values={})
|
|
orchestrator.adopt_build_keys()
|
|
site(monkeypatch, ONE)
|
|
assert update(story_id).startswith("No changes")
|
|
assert builds.call_count == 0 and ingest_files() == []
|
|
|
|
|
|
# Downloads in the browser
|
|
|
|
|
|
def download(client, story_id):
|
|
response = client.get(f"/api/stories/{story_id}/epub")
|
|
assert response.status_code == 200 and response.data.startswith(b"PK")
|
|
assert response.content_length == len(response.data)
|
|
response.close()
|
|
return response.data
|
|
|
|
|
|
def test_a_download_uses_the_built_book(builds, client, monkeypatch):
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, ONE)
|
|
update(story_id)
|
|
(built,) = config.TMP_DIR.iterdir()
|
|
assert download(client, story_id) == download(client, story_id) == built.read_bytes()
|
|
assert builds.call_count == 1
|
|
|
|
|
|
def test_a_download_builds_a_book_that_has_no_file_or_that_changed(builds, client, monkeypatch):
|
|
story_id = add_story()
|
|
changed_at = story(story_id)["last_changed"]
|
|
first = download(client, story_id)
|
|
download(client, story_id)
|
|
assert builds.call_count == 1
|
|
orchestrator.save_rules(story_id, [{"pattern": "Zorian", "replacement": "Zach"}])
|
|
assert download(client, story_id) != first
|
|
assert builds.call_count == 2
|
|
assert len(list(config.TMP_DIR.iterdir())) == 1
|
|
# The build of a download is not the build of a job. The job sends the book.
|
|
assert orchestrator.store.get_setting(story_id, "built_key") is None
|
|
job = Job(mode=DownloadMode.REBUILD, story_id=story_id)
|
|
assert "library book 1" in orchestrator._run_rebuild(job)
|
|
assert builds.call_count == 2
|
|
assert story(story_id)["last_changed"] == changed_at
|
|
|
|
|
|
# Page counts
|
|
|
|
LONG = "A line of the story. " * 20
|
|
|
|
|
|
def listed_pages(story_id):
|
|
return {s["id"]: s["page_count"] for s in orchestrator.list_stories()}[story_id]
|
|
|
|
|
|
def test_a_story_has_no_page_count_until_its_book_is_built(builds, monkeypatch):
|
|
story_id = linked_story(ONE)
|
|
assert listed_pages(story_id) is None
|
|
site(monkeypatch, ONE)
|
|
update(story_id)
|
|
(built,) = config.TMP_DIR.iterdir()
|
|
assert listed_pages(story_id) == pages.epub_pages(built) == 1
|
|
|
|
|
|
def test_each_build_counts_the_pages_of_the_new_book(builds, client, monkeypatch):
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, [*ONE, *[(n, f"Chapter {n}", "<br/>".join([LONG] * 31)) for n in range(2, 6)]])
|
|
update(story_id)
|
|
long_book = listed_pages(story_id)
|
|
# Each of the 124 lines is a paragraph and 6 lines of text.
|
|
assert long_book >= 4 * 31 * 7 // 31
|
|
# The count is of the built book, so a rule that removes text changes it without a download.
|
|
orchestrator.save_rules(story_id, [{"pattern": "(A line of the story. ){19}", "replacement": ""}])
|
|
download(client, story_id)
|
|
assert builds.call_count == 2
|
|
assert 0 < listed_pages(story_id) < long_book / 3
|
|
|
|
|
|
def test_a_book_from_before_the_page_count_is_counted_without_a_build(builds, client, monkeypatch):
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, ONE)
|
|
update(story_id)
|
|
orchestrator.store.set_setting(story_id, "page_count", None)
|
|
download(client, story_id)
|
|
assert builds.call_count == 1
|
|
assert listed_pages(story_id) == 1
|
|
|
|
|
|
def test_a_book_that_cannot_be_counted_is_still_built(builds, monkeypatch):
|
|
def fail(path):
|
|
raise orchestrator.EpubError("unreadable")
|
|
|
|
monkeypatch.setattr(pages, "epub_pages", fail)
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, ONE)
|
|
assert "library book 1" in update(story_id)
|
|
assert listed_pages(story_id) is None
|
|
|
|
|
|
def test_stop_tracking_deletes_the_built_book(builds, client):
|
|
story_id, other = add_story(), orchestrator.store.create_story("https://other", "s", "Other", "W", [])
|
|
download(client, story_id)
|
|
kept = config.TMP_DIR / f"story-{other}-0123456789abcdef.epub"
|
|
kept.write_bytes(b"other")
|
|
orchestrator.delete_story(story_id)
|
|
assert list(config.TMP_DIR.iterdir()) == [kept]
|
|
|
|
|
|
# Uploaded files
|
|
|
|
|
|
@pytest.fixture
|
|
def uploads(monkeypatch, tmp_path):
|
|
monkeypatch.setattr(config, "UPLOAD_DIR", tmp_path / "uploads")
|
|
config.UPLOAD_DIR.mkdir()
|
|
return config.UPLOAD_DIR
|
|
|
|
|
|
def test_a_cancelled_import_deletes_its_uploaded_file(builds, jobs, uploads, story_epub):
|
|
path = uploads / "one.epub"
|
|
path.write_bytes(story_epub)
|
|
queued = orchestrator.queue_epub_file(path, "story.epub")
|
|
assert "path" not in queued["options"]
|
|
assert orchestrator.cancel_job(queued["id"]) is True
|
|
assert not path.exists()
|
|
assert orchestrator.cancel_job(queued["id"]) is False
|
|
assert orchestrator.cancel_job("no-such-job") is False
|
|
|
|
|
|
def test_the_start_deletes_uploads_that_no_job_needs(builds, jobs, uploads, story_epub):
|
|
waiting, stale = uploads / "waiting.epub", uploads / "stale.epub"
|
|
for path in (waiting, stale):
|
|
path.write_bytes(story_epub)
|
|
orchestrator.queue_epub_file(waiting, "story.epub")
|
|
orchestrator.remove_stale_uploads()
|
|
assert list(uploads.iterdir()) == [waiting]
|
|
|
|
|
|
def test_a_missing_ingest_folder_fails_the_job_until_the_folder_exists(builds, jobs, monkeypatch, tmp_path):
|
|
monkeypatch.setattr(config, "INGEST_DIR", tmp_path / "not-mounted")
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, ONE)
|
|
monkeypatch.setattr(orchestrator.time, "sleep", Mock())
|
|
jobs.add(Job(mode=DownloadMode.NEW, story_id=story_id))
|
|
job = orchestrator._run_next_job()
|
|
assert job.status == "error" and "does not exist. Check INGEST_DIR and the volume mount." in job.message
|
|
assert story(story_id)["status"] == "error"
|
|
assert not config.INGEST_DIR.exists()
|
|
config.INGEST_DIR.mkdir()
|
|
assert "library book 1" in update(story_id)
|
|
assert builds.call_count == 1
|