tim
46de77c228
Bindery counts the pages as the last step of a build, from the text of the ePub. The story list and the OPDS catalog show the count. A story has no count until its book is built. The method follows "Paragraphs (APNX Accurate)" of the Count Pages plugin for calibre. Bindery counts each paragraph that a reader sees, so the same text gives the same count as <p> elements, as <div> elements and as lines with <br> between them. An empty paragraph is not counted.
350 lines
14 KiB
Python
350 lines
14 KiB
Python
"""A book is built and sent only when its chapters, rules or metadata changed."""
|
|
|
|
import dataclasses
|
|
from unittest.mock import Mock
|
|
|
|
import pytest
|
|
from test_api import H
|
|
from test_api import client as client
|
|
from test_deliver import STORY_URL, add_story, ingest_files, linked_story, site, update
|
|
from test_deliver import fake as fake
|
|
from test_deliver import jobs as jobs
|
|
from test_deliver import story_epub as story_epub
|
|
|
|
from bindery import config, orchestrator, pages
|
|
from bindery.jobs import DownloadMode, Job
|
|
|
|
ONE = [(1, "One", "a")]
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def builds(fake, jobs, monkeypatch, tmp_path):
|
|
"""Count the builds, in a folder of built books that starts empty."""
|
|
monkeypatch.setattr(config, "TMP_DIR", tmp_path / "tmp")
|
|
config.TMP_DIR.mkdir()
|
|
spy = Mock(side_effect=orchestrator._build)
|
|
monkeypatch.setattr(orchestrator, "_build", spy)
|
|
return spy
|
|
|
|
|
|
def story(story_id):
|
|
return orchestrator.store.get_story(story_id)
|
|
|
|
|
|
def test_an_unchanged_story_is_not_built_or_sent_again(builds, monkeypatch):
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, ONE)
|
|
assert "library book 1" in update(story_id)
|
|
changed = story(story_id)["last_changed"]
|
|
orchestrator.store.update_story(story_id, last_changed="2020-01-01T00:00:00")
|
|
for mode in (DownloadMode.NEW, DownloadMode.RECHECK_ALL, DownloadMode.METADATA, DownloadMode.FULL):
|
|
assert update(story_id, mode).startswith("No changes")
|
|
assert changed and story(story_id)["last_changed"] == "2020-01-01T00:00:00"
|
|
assert builds.call_count == 1
|
|
assert len(ingest_files()) == 2 # one epub and its sidecar
|
|
|
|
|
|
@pytest.mark.parametrize("change", ["chapter", "rule", "global rule", "paragraphs", "metadata", "included"])
|
|
def test_each_change_builds_and_sends_the_book(builds, monkeypatch, change):
|
|
story_id = linked_story([(1, "One", "a"), (2, "Two", "b")])
|
|
site(monkeypatch, [(1, "One", "a"), (2, "Two", "b")])
|
|
update(story_id)
|
|
if change == "chapter":
|
|
site(monkeypatch, [(1, "One", "a"), (2, "Two", "b"), (3, "Three", "c")])
|
|
elif change == "rule":
|
|
orchestrator.save_rules(story_id, [{"pattern": "a", "replacement": "b"}])
|
|
elif change == "global rule":
|
|
orchestrator.save_rules(None, [{"pattern": "a", "replacement": "b"}])
|
|
elif change == "paragraphs":
|
|
orchestrator.save_rules(story_id, [], paragraphs=True)
|
|
elif change == "metadata":
|
|
orchestrator.store.update_story(story_id, pinned={"title": "Edited"})
|
|
else:
|
|
chapter = orchestrator.store.list_chapters(story_id)[0]
|
|
orchestrator.store.update_chapter(chapter["id"], included=0)
|
|
assert "library book 1" in update(story_id)
|
|
assert builds.call_count == 2
|
|
assert len(ingest_files()) == 4
|
|
assert update(story_id).startswith("No changes")
|
|
|
|
|
|
def test_a_held_book_that_did_not_change_is_not_built_again(builds, monkeypatch):
|
|
monkeypatch.setattr(config, "INGEST_DIR", None)
|
|
site(monkeypatch, ONE)
|
|
job = Job(mode=DownloadMode.NEW, url=STORY_URL)
|
|
assert "Not sent. INGEST_DIR is not set." in orchestrator._run_site_job(job)
|
|
changed = story(job.story_id)["last_changed"]
|
|
orchestrator.store.update_story(job.story_id, last_changed="2020-01-01T00:00:00")
|
|
assert "Not sent. INGEST_DIR is not set." in update(job.story_id)
|
|
assert changed and story(job.story_id)["last_changed"] == "2020-01-01T00:00:00"
|
|
assert builds.call_count == 1
|
|
# The book changes. Bindery builds it, also while it cannot send it.
|
|
site(monkeypatch, [*ONE, (2, "Two", "b")])
|
|
update(job.story_id)
|
|
assert builds.call_count == 2
|
|
assert story(job.story_id)["last_changed"] != "2020-01-01T00:00:00"
|
|
|
|
|
|
def test_a_released_book_goes_out_without_another_build(builds, jobs):
|
|
story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=1, calibre_uuid="uuid-1")
|
|
orchestrator.save_settings(pause_uploads=True)
|
|
orchestrator._run_rebuild(Job(mode=DownloadMode.REBUILD, story_id=story_id))
|
|
assert builds.call_count == 1 and ingest_files() == []
|
|
orchestrator.save_settings(pause_uploads=False)
|
|
assert "library book 1" in orchestrator._run_rebuild(jobs.get_next())
|
|
assert builds.call_count == 1
|
|
assert len(ingest_files()) == 2
|
|
|
|
|
|
def test_a_lost_file_is_built_again_only_to_send_it(builds):
|
|
story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=1, calibre_uuid="uuid-1")
|
|
orchestrator.save_settings(pause_uploads=True)
|
|
job = Job(mode=DownloadMode.REBUILD, story_id=story_id)
|
|
orchestrator._run_rebuild(job)
|
|
for path in config.TMP_DIR.iterdir():
|
|
path.unlink()
|
|
assert "pause" in orchestrator._run_rebuild(job)
|
|
assert builds.call_count == 1
|
|
orchestrator.store.set_setting(None, "pause_uploads", None)
|
|
assert "library book 1" in orchestrator._run_rebuild(job)
|
|
assert builds.call_count == 2
|
|
|
|
|
|
def test_a_rebuild_without_a_change_does_nothing_unless_the_user_asks_for_it(builds, client, jobs):
|
|
story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=1, calibre_uuid="uuid-1")
|
|
job = Job(mode=DownloadMode.REBUILD, story_id=story_id)
|
|
assert "library book 1" in orchestrator._run_rebuild(job)
|
|
orchestrator.store.update_story(story_id, last_changed=None, message="before")
|
|
assert orchestrator._run_rebuild(job) == "No changes"
|
|
assert (story(story_id)["last_changed"], story(story_id)["message"]) == (None, "before")
|
|
assert builds.call_count == 1
|
|
# The Rebuild button of the story page.
|
|
queued = client.post(f"/api/stories/{story_id}/download", json={"mode": "rebuild"}, headers=H).json
|
|
assert queued["options"] == {"force": True}
|
|
assert "library book 1" in orchestrator._run_rebuild(jobs.get_next())
|
|
assert builds.call_count == 2
|
|
assert len(ingest_files()) == 4
|
|
assert story(story_id)["last_changed"]
|
|
# Other modes take no such option.
|
|
queued = client.post(f"/api/stories/{story_id}/download", json={"mode": "new"}, headers=H).json
|
|
assert queued["options"] == {}
|
|
|
|
|
|
def test_a_changed_link_gets_the_book_without_a_build(builds, fake):
|
|
story_id = add_story(ingested_at="2026-01-01T00:00:00", calibre_id=1, calibre_uuid="uuid-1")
|
|
job = Job(mode=DownloadMode.REBUILD, story_id=story_id)
|
|
orchestrator._run_rebuild(job)
|
|
fake.books.append(dataclasses.replace(fake.books[0], id=2, uuid="uuid-2", title="Other"))
|
|
orchestrator.link_story(story_id, "uuid-2")
|
|
assert "library book 2" in orchestrator._run_rebuild(job)
|
|
assert builds.call_count == 1
|
|
|
|
|
|
def test_a_failed_delivery_is_tried_again_without_a_build(builds, fake, monkeypatch):
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, ONE)
|
|
send = orchestrator.IngestFolder.send_update
|
|
monkeypatch.setattr(orchestrator.IngestFolder, "send_update", Mock(side_effect=OSError("disk full")))
|
|
with pytest.raises(OSError, match="disk full"):
|
|
update(story_id)
|
|
monkeypatch.setattr(orchestrator.IngestFolder, "send_update", send)
|
|
assert "library book 1" in update(story_id)
|
|
assert builds.call_count == 1
|
|
|
|
|
|
def test_a_new_build_removes_the_older_file(builds, monkeypatch):
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, ONE)
|
|
update(story_id)
|
|
(first,) = config.TMP_DIR.iterdir()
|
|
site(monkeypatch, [*ONE, (2, "Two", "b")])
|
|
update(story_id)
|
|
(second,) = config.TMP_DIR.iterdir()
|
|
assert first != second and second.name.startswith(f"story-{story_id}-")
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("changes", "held", "built", "sent"),
|
|
[
|
|
({"calibre_id": 1, "calibre_uuid": "uuid-1"}, False, True, True),
|
|
({"ingested_at": "2026-01-01"}, False, True, True),
|
|
({"ingested_at": "2026-01-01"}, True, True, False),
|
|
({}, False, True, False),
|
|
({"ingested_at": "2026-01-01", "status": "error"}, False, False, False),
|
|
({"ingested_at": "2026-01-01", "no_metadata": True}, False, False, False),
|
|
],
|
|
)
|
|
def test_stories_of_an_older_bindery_get_their_keys(builds, changes, held, built, sent):
|
|
no_metadata = changes.pop("no_metadata", False)
|
|
story_id = add_story(**changes)
|
|
if no_metadata:
|
|
orchestrator.store.update_story(story_id, metahtml=None)
|
|
orchestrator.store.set_setting(story_id, "upload_held", held or None)
|
|
orchestrator.adopt_build_keys()
|
|
settings = orchestrator.store
|
|
assert bool(settings.get_setting(story_id, "built_key")) is built
|
|
assert bool(settings.get_setting(story_id, "sent_key")) is sent
|
|
if sent:
|
|
assert orchestrator._run_rebuild(Job(mode=DownloadMode.REBUILD, story_id=story_id)) == "No changes"
|
|
assert builds.call_count == 0
|
|
|
|
|
|
def test_a_story_with_a_waiting_job_gets_no_key(builds, jobs):
|
|
story_id = add_story(ingested_at="2026-01-01")
|
|
jobs.add(Job(mode=DownloadMode.NEW, story_id=story_id))
|
|
orchestrator.adopt_build_keys()
|
|
assert orchestrator.store.get_setting(story_id, "built_key") is None
|
|
|
|
|
|
def test_the_first_check_after_an_upgrade_sends_nothing(builds, monkeypatch):
|
|
story_id = linked_story(ONE)
|
|
orchestrator.store.update_story(story_id, pinned={"title": "My Title"}, site_values={})
|
|
orchestrator.adopt_build_keys()
|
|
site(monkeypatch, ONE)
|
|
assert update(story_id).startswith("No changes")
|
|
assert builds.call_count == 0 and ingest_files() == []
|
|
|
|
|
|
# Downloads in the browser
|
|
|
|
|
|
def download(client, story_id):
|
|
response = client.get(f"/api/stories/{story_id}/epub")
|
|
assert response.status_code == 200 and response.data.startswith(b"PK")
|
|
assert response.content_length == len(response.data)
|
|
response.close()
|
|
return response.data
|
|
|
|
|
|
def test_a_download_uses_the_built_book(builds, client, monkeypatch):
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, ONE)
|
|
update(story_id)
|
|
(built,) = config.TMP_DIR.iterdir()
|
|
assert download(client, story_id) == download(client, story_id) == built.read_bytes()
|
|
assert builds.call_count == 1
|
|
|
|
|
|
def test_a_download_builds_a_book_that_has_no_file_or_that_changed(builds, client, monkeypatch):
|
|
story_id = add_story()
|
|
first = download(client, story_id)
|
|
download(client, story_id)
|
|
assert builds.call_count == 1
|
|
orchestrator.save_rules(story_id, [{"pattern": "Zorian", "replacement": "Zach"}])
|
|
assert download(client, story_id) != first
|
|
assert builds.call_count == 2
|
|
assert len(list(config.TMP_DIR.iterdir())) == 1
|
|
# The build of a download is not the build of a job. The job sends the book, and marks the change.
|
|
assert orchestrator.store.get_setting(story_id, "built_key") is None
|
|
job = Job(mode=DownloadMode.REBUILD, story_id=story_id)
|
|
assert "library book 1" in orchestrator._run_rebuild(job)
|
|
assert builds.call_count == 2
|
|
assert story(story_id)["last_changed"]
|
|
|
|
|
|
# Page counts
|
|
|
|
LONG = "A line of the story. " * 20
|
|
|
|
|
|
def listed_pages(story_id):
|
|
return {s["id"]: s["page_count"] for s in orchestrator.list_stories()}[story_id]
|
|
|
|
|
|
def test_a_story_has_no_page_count_until_its_book_is_built(builds, monkeypatch):
|
|
story_id = linked_story(ONE)
|
|
assert listed_pages(story_id) is None
|
|
site(monkeypatch, ONE)
|
|
update(story_id)
|
|
(built,) = config.TMP_DIR.iterdir()
|
|
assert listed_pages(story_id) == pages.epub_pages(built) == 1
|
|
|
|
|
|
def test_each_build_counts_the_pages_of_the_new_book(builds, client, monkeypatch):
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, [*ONE, *[(n, f"Chapter {n}", "<br/>".join([LONG] * 31)) for n in range(2, 6)]])
|
|
update(story_id)
|
|
long_book = listed_pages(story_id)
|
|
# Each of the 124 lines is a paragraph and 6 lines of text.
|
|
assert long_book >= 4 * 31 * 7 // 31
|
|
# The count is of the built book, so a rule that removes text changes it without a download.
|
|
orchestrator.save_rules(story_id, [{"pattern": "(A line of the story. ){19}", "replacement": ""}])
|
|
download(client, story_id)
|
|
assert builds.call_count == 2
|
|
assert 0 < listed_pages(story_id) < long_book / 3
|
|
|
|
|
|
def test_a_book_from_before_the_page_count_is_counted_without_a_build(builds, client, monkeypatch):
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, ONE)
|
|
update(story_id)
|
|
orchestrator.store.set_setting(story_id, "page_count", None)
|
|
download(client, story_id)
|
|
assert builds.call_count == 1
|
|
assert listed_pages(story_id) == 1
|
|
|
|
|
|
def test_a_book_that_cannot_be_counted_is_still_built(builds, monkeypatch):
|
|
def fail(path):
|
|
raise orchestrator.EpubError("unreadable")
|
|
|
|
monkeypatch.setattr(pages, "epub_pages", fail)
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, ONE)
|
|
assert "library book 1" in update(story_id)
|
|
assert listed_pages(story_id) is None
|
|
|
|
|
|
def test_stop_tracking_deletes_the_built_book(builds, client):
|
|
story_id, other = add_story(), orchestrator.store.create_story("https://other", "s", "Other", "W", [])
|
|
download(client, story_id)
|
|
kept = config.TMP_DIR / f"story-{other}-0123456789abcdef.epub"
|
|
kept.write_bytes(b"other")
|
|
orchestrator.delete_story(story_id)
|
|
assert list(config.TMP_DIR.iterdir()) == [kept]
|
|
|
|
|
|
# Uploaded files
|
|
|
|
|
|
@pytest.fixture
|
|
def uploads(monkeypatch, tmp_path):
|
|
monkeypatch.setattr(config, "UPLOAD_DIR", tmp_path / "uploads")
|
|
config.UPLOAD_DIR.mkdir()
|
|
return config.UPLOAD_DIR
|
|
|
|
|
|
def test_a_cancelled_import_deletes_its_uploaded_file(builds, jobs, uploads, story_epub):
|
|
path = uploads / "one.epub"
|
|
path.write_bytes(story_epub)
|
|
queued = orchestrator.queue_epub_file(path, "story.epub")
|
|
assert "path" not in queued["options"]
|
|
assert orchestrator.cancel_job(queued["id"]) is True
|
|
assert not path.exists()
|
|
assert orchestrator.cancel_job(queued["id"]) is False
|
|
assert orchestrator.cancel_job("no-such-job") is False
|
|
|
|
|
|
def test_the_start_deletes_uploads_that_no_job_needs(builds, jobs, uploads, story_epub):
|
|
waiting, stale = uploads / "waiting.epub", uploads / "stale.epub"
|
|
for path in (waiting, stale):
|
|
path.write_bytes(story_epub)
|
|
orchestrator.queue_epub_file(waiting, "story.epub")
|
|
orchestrator.remove_stale_uploads()
|
|
assert list(uploads.iterdir()) == [waiting]
|
|
|
|
|
|
def test_a_missing_ingest_folder_fails_the_job_until_the_folder_exists(builds, jobs, monkeypatch, tmp_path):
|
|
monkeypatch.setattr(config, "INGEST_DIR", tmp_path / "not-mounted")
|
|
story_id = linked_story(ONE)
|
|
site(monkeypatch, ONE)
|
|
monkeypatch.setattr(orchestrator.time, "sleep", Mock())
|
|
jobs.add(Job(mode=DownloadMode.NEW, story_id=story_id))
|
|
job = orchestrator._run_next_job()
|
|
assert job.status == "error" and "does not exist. Check INGEST_DIR and the volume mount." in job.message
|
|
assert story(story_id)["status"] == "error"
|
|
assert not config.INGEST_DIR.exists()
|
|
config.INGEST_DIR.mkdir()
|
|
assert "library book 1" in update(story_id)
|
|
assert builds.call_count == 1
|