Build and send a book only when it changed

A key of the chapters, rules and metadata of a story tells if its book
changed. A check without a change no longer builds the book again, also
for a book that waits for an upload. A held book goes out without a new
build. The Rebuild button still builds and sends in each case.

bindery/fff.py

@@ -238,16 +238,21 @@ def word_count(chapters: list[dict[str, Any]]) -> int:
     return sum(len(block.split()) for c in chapters for block in html_to_blocks(c["html"]))
 
 
+def effective_overrides(overrides: dict[str, Any]) -> dict[str, Any]:
+    """The kept values that replace a site value. An empty title, author list or language does not."""
+    return {k: v for k, v in overrides.items() if v or k not in ("title", "authors", "languages")}
+
+
 def apply_overrides(story, overrides: dict[str, Any]) -> None:
     """Put the kept values of locked fields into FanFicFare's story metadata.
 
     The epub writer takes the OPF metadata from these entries, and CWA takes
     the book record from the OPF on the first import.
     """
-    for key, value in overrides.items():
-        if key == "title" and value:
+    for key, value in effective_overrides(overrides).items():
+        if key == "title":
             story.setMetadata("title", value)
-        elif key == "authors" and value:
+        elif key == "authors":
             for n, author in enumerate(value):
                 story.addToList("author", author, clear=(n == 0))
         elif key == "comments":
@@ -263,7 +268,7 @@ def apply_overrides(story, overrides: dict[str, Any]) -> None:
                 story.setMetadata("datePublished", datetime.fromisoformat(value) if value else None)
             except ValueError:
                 logger.warning(f"Ignoring a kept publication date that is not a date: {value!r}")
-        elif key == "languages" and value:
+        elif key == "languages":
             story.setMetadata("langcode", value)
         elif key == "tags":
             tags = list(value or [])

bindery/orchestrator.py

@@ -1,6 +1,7 @@
 """Download modes, metadata locks, delivery to CWA, and the job threads."""
 
 import hashlib
+import json
 import logging
 import re
 import threading
@@ -94,8 +95,13 @@ def queue_urls(urls: list[str]) -> tuple[list[dict[str, Any]], list[str]]:
 
 
 def queue_story(
-    story_id: int, mode: str, chapter_ids: list[int] | None = None, force_fields: list[str] | None = None
+    story_id: int,
+    mode: str,
+    chapter_ids: list[int] | None = None,
+    force_fields: list[str] | None = None,
+    force: bool = False,
 ) -> dict[str, Any]:
+    """Queue a job for a story. With `force`, a rebuild also builds and sends a book that did not change."""
     story = _require_story(story_id)
     mode_enum = DownloadMode(mode)
     if mode_enum == DownloadMode.IMPORT:
@@ -111,6 +117,7 @@ def queue_story(
         title=story["title"],
         chapter_ids=sorted(int(c) for c in chapter_ids or []),
         force_fields=sorted(force_fields or []),
+        options={"force": True} if force and mode_enum == DownloadMode.REBUILD else {},
     )
     return job_queue.add(job).to_dict()
 
@@ -674,6 +681,60 @@ def _build(story: dict[str, Any], overrides: dict[str, Any], out_path: Path) ->
     )
 
 
+# Built books. The key of a story is a hash of all that goes into its epub, so the book changed when the key
+# changed. The name of the built file holds the key. The story settings "built_key" and "sent_key" are the keys of
+# the last build and of the book that went to CWA.
+
+_build_lock = threading.Lock()
+
+
+def _book_metadata(site: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any]:
+    """The metadata of the book: the site values, with the kept values in their place."""
+    meta = dict(site)
+    for field, value in fff.effective_overrides(overrides).items():
+        if field == "series":
+            meta["series"], meta["series_index"] = value or ("", None)
+        else:
+            meta[field] = value
+    return meta
+
+
+def _build_key(story: dict[str, Any], overrides: dict[str, Any], site: dict[str, Any]) -> str:
+    story_id = story["id"]
+    digest = hashlib.sha256()
+    state = [
+        story["url"],
+        _book_metadata(site, overrides),
+        store.rules(None),
+        store.rules(story_id),
+        _paragraphs(story_id),
+        store.image_sizes(story_id),
+    ]
+    digest.update(json.dumps(state, sort_keys=True, default=str).encode())
+    for c in store.chapters_for_build(story_id):
+        digest.update(json.dumps([c["url"], c["title"], c["html"]]).encode())
+    return digest.hexdigest()
+
+
+def _built_epub(story: dict[str, Any], overrides: dict[str, Any], key: str, force: bool = False) -> Path:
+    """The epub of the story for this key. Bindery builds it only if it has no file for the key."""
+    path = config.TMP_DIR / f"story-{story['id']}-{key[:16]}.epub"
+    with _build_lock:
+        if force or not path.exists():
+            # A build under another name first, so a download never reads half a book.
+            partial = config.TMP_DIR / f"building-{uuid.uuid4().hex}.epub"
+            try:
+                _build(story, overrides, partial)
+                partial.replace(path)
+            finally:
+                partial.unlink(missing_ok=True)
+            for old in config.TMP_DIR.glob(f"story-{story['id']}-*.epub"):
+                if old != path:
+                    old.unlink(missing_ok=True)
+        store.set_setting(story["id"], "built_key", key)
+    return path
+
+
 def _find_book(story_url: str, titles: list[str | None]) -> tuple[Book, Path] | None:
     """Search CWA for the book of a story. Returns the book and its downloaded epub."""
     linked = {s["calibre_id"] for s in store.list_stories() if s["calibre_id"]}
@@ -744,6 +805,8 @@ def _uploads_paused() -> bool:
 
 def _link_changed(story_id: int) -> None:
     store.set_setting(story_id, "unapproved_link", True)
+    # Another book, or no book, does not have the epub that Bindery sent.
+    store.set_setting(story_id, "sent_key", None)
 
 
 def _needs_approval(story_id: int) -> bool:
@@ -773,10 +836,6 @@ def _hold_reason(story: dict[str, Any], book: dict[str, Any] | None) -> str | No
 
 
 def _deliver(story: dict[str, Any], epub: Path, book: dict[str, Any] | None, title: str, author: str) -> str:
-    held = _hold_reason(story, book)
-    if held:
-        store.set_setting(story["id"], "upload_held", True)
-        return held
     # With linking on, do not create a duplicate when the library book cannot be found.
     if _link_books() and not book and story["ingested_at"]:
         raise StoryError(NOT_FOUND)
@@ -794,6 +853,44 @@ def _deliver(story: dict[str, Any], epub: Path, book: dict[str, Any] | None, tit
     return message
 
 
+def _send(
+    story: dict[str, Any], overrides: dict[str, Any], key: str, title: str, author: str, force: bool = False
+) -> tuple[str, bool]:
+    """Build the book if it changed, and deliver it. Returns the message, and whether the book changed.
+
+    A held book that did not change is not built again. With `force`, Bindery builds the book in each case.
+    """
+    story_id = story["id"]
+    book = _linked_book(story)
+    held = _hold_reason(story, book)
+    changed = key != store.get_setting(story_id, "built_key")
+    if held:
+        if changed or force:
+            _built_epub(story, overrides, key, force)
+        store.set_setting(story_id, "upload_held", True)
+        return held, changed
+    message = _deliver(story, _built_epub(story, overrides, key, force), book, title, author)
+    store.set_setting(story_id, "sent_key", key)
+    return message, changed
+
+
+def adopt_build_keys() -> None:
+    """Give the stories of an older Bindery their keys, so the first check does not send each book again."""
+    built = store.story_settings("built_key")
+    held = store.story_settings("upload_held")
+    for listed in store.list_stories():
+        story_id = listed["id"]
+        if story_id in built or listed["status"] == "error" or job_queue.has_active(story_id):
+            continue
+        story = _require_story(story_id)
+        if not story["metahtml"]:
+            continue
+        key = _build_key(story, _epub_overrides(story, story["pinned"]), story["site_values"])
+        store.set_setting(story_id, "built_key", key)
+        if (story["calibre_id"] or story["ingested_at"]) and not held.get(story_id):
+            store.set_setting(story_id, "sent_key", key)
+
+
 # Jobs
 
 
@@ -920,12 +1017,15 @@ def _run_import_file(job: Job) -> str:
 def _run_rebuild(job: Job) -> str:
     story = _require_story(job.story_id)
     job_queue.update(job, title=story["title"])
-    book = _linked_book(story)
     pinned = _resolve_metadata(story, story["site_values"], [])
-    epub = config.TMP_DIR / f"story-{story['id']}.epub"
-    _build(story, _epub_overrides(story, pinned), epub)
-    message = _deliver(story, epub, book, story["title"], story["author"])
-    store.update_story(story["id"], pinned=pinned, status="ok", message=message, last_changed=now())
+    overrides = _epub_overrides(story, pinned)
+    key = _build_key(story, overrides, story["site_values"])
+    force = bool(job.options.get("force"))
+    if not force and key == store.get_setting(story["id"], "sent_key"):
+        return "No changes"
+    message, changed = _send(story, overrides, key, story["title"], story["author"], force)
+    changes: dict[str, Any] = {"last_changed": now()} if changed or force else {}
+    store.update_story(story["id"], pinned=pinned, status="ok", message=message, **changes)
     return message
 
 
@@ -1149,46 +1249,27 @@ def _run_site_job(job: Job) -> str:
         job, result={"new": new_count, "changed": changed, "failed": failed, "skipped": skipped, "removed": removed}
     )
 
-    metadata_changed = snap.calibre != story["site_values"]
-    delivered = bool(story["calibre_id"] or story["ingested_at"])
-    # A resumed job builds the book, because the run before the restart can have stored chapters.
-    needs_build = (
-        new_count
-        or changed
-        or renamed_count
-        or job.force_fields
-        or metadata_changed
-        or not delivered
-        or job.options.get("build")
-        or job.restarts
-        or mode in (DownloadMode.FULL, DownloadMode.METADATA)
-        or (mode == DownloadMode.NEW and story["status"] == "error" and not failed)
-    )
     if mode in (DownloadMode.RECHECK_ALL, DownloadMode.FULL):
         store.set_setting(story_id, "last_full_recheck", now())
     if not store.chapters_for_build(story_id):
         raise StoryError("; ".join(summary) + ". No chapters downloaded.")
-    if not needs_build:
-        store.update_story(story_id, last_checked=now(), status="ok", message="No changes")
-        return "No changes. " + ", ".join(summary)
 
-    book = _linked_book(story)
+    # The book is built and sent only when its chapters, rules or metadata changed. A job that resumes, or that
+    # follows a failed job, finds the changes of the earlier run through the key.
     pinned = _resolve_metadata(story, snap.calibre, job.force_fields)
     title = _display_title(story, pinned, snap.title)
+    overrides = _epub_overrides(story, pinned)
+    key = _build_key(story, overrides, snap.calibre)
+    checked: dict[str, Any] = {"pinned": pinned, "site_values": snap.calibre, "title": title, "last_checked": now()}
+    if key == store.get_setting(story_id, "sent_key"):
+        store.update_story(story_id, **checked, status="ok", message="No changes")
+        return "No changes. " + ", ".join(summary)
+
     _progress(job, "Building epub", 1.0)
-    epub = config.TMP_DIR / f"story-{story_id}.epub"
-    _build(story, _epub_overrides(story, pinned), epub)
-    message = _deliver(story, epub, book, title, snap.author)
-    store.update_story(
-        story_id,
-        pinned=pinned,
-        site_values=snap.calibre,
-        title=title,
-        last_checked=now(),
-        last_changed=now(),
-        message=message,
-        status="error" if failed else "ok",
-    )
+    message, book_changed = _send(story, overrides, key, title, snap.author)
+    if book_changed:
+        checked["last_changed"] = now()
+    store.update_story(story_id, **checked, message=message, status="error" if failed else "ok")
     return ", ".join(summary) + ". " + message
 
 
@@ -1267,5 +1348,9 @@ def scheduler_loop() -> None:
 
 
 def start_threads() -> None:
+    try:
+        adopt_build_keys()
+    except Exception as e:
+        logger.exception(f"Cannot check the built books: {e}")
     for target in (download_loop, scheduler_loop):
         threading.Thread(target=target, daemon=True, name=target.__name__).start()

bindery/store.py

@@ -388,6 +388,14 @@ class Store:
             rows = conn.execute("SELECT url FROM images WHERE story_id = ? AND is_cover = 0", (story_id,))
             return {r["url"] for r in rows}
 
+    def image_sizes(self, story_id: int) -> list[list[Any]]:
+        """The URL, the cover mark and the size of each image, without its data."""
+        with self._connect() as conn:
+            rows = conn.execute(
+                "SELECT url, is_cover, LENGTH(data) FROM images WHERE story_id = ? ORDER BY url, is_cover", (story_id,)
+            )
+            return [list(r) for r in rows]
+
     def images(self, story_id: int) -> list[dict[str, Any]]:
         with self._connect() as conn:
             rows = conn.execute(

bindery/web.py

@@ -291,8 +291,9 @@ def delete_story(story_id: int) -> Any:
 @app.post("/api/stories/<int:story_id>/download")
 def download(story_id: int) -> Any:
     data = _json()
+    # A rebuild that the user asks for also builds and sends a book that did not change.
     return orchestrator.queue_story(
-        story_id, data.get("mode", DownloadMode.NEW.value), data.get("chapter_ids"), data.get("force_fields")
+        story_id, data.get("mode", DownloadMode.NEW.value), data.get("chapter_ids"), data.get("force_fields"), True
     )