diff --git a/README.md b/README.md index 332aee7..ac4bb6c 100644 --- a/README.md +++ b/README.md @@ -650,6 +650,37 @@ always the site name. replaces only the ePub file. So edits that you make in CWA stay, and later changes on the site show in the ePub but not in the CWA catalog. +## Page count + +The story list shows an estimated page count beside the chapter count. The OPDS catalog shows it in the +description of a book. Bindery counts the pages as the last step when it builds an ePub, so the count +includes your replacement rules and the line-break setting. A story shows no page count until Bindery +builds its book. + +The method comes from "Paragraphs (APNX Accurate)" in the +[Count Pages plugin](https://github.com/kiwidude68/calibre_plugins/wiki/Count-Pages) for calibre: + +- A paragraph start is one line. Each 70 characters of text is one more line. +- A page is 31 lines. +- A page is at most 2400 characters. This limit sets the count of a book with almost no paragraphs. + +The plugin counts either the `
` elements or the `
`, + `
` elements, as `
` elements and has no empty paragraphs, the count is within about 1% +of the count of the plugin. The plugin counts each empty paragraph as a line, so its count is higher for a +book that has many of them. +The count is an estimate and differs from the page count of a printed book. + ## Replacement rules A rule is a regular expression, a replacement and optional flags (`i`, `s`, `m`, `x`). Bindery uses diff --git a/bindery/epub.py b/bindery/epub.py index 34f0847..d5af57e 100644 --- a/bindery/epub.py +++ b/bindery/epub.py @@ -1,9 +1,11 @@ """Read the OPF metadata of an epub that came from CWA.""" +import posixpath import re import zipfile from pathlib import Path from typing import Any +from urllib.parse import unquote from xml.etree.ElementTree import Element from defusedxml import ElementTree @@ -17,14 +19,19 @@ class EpubError(Exception): pass +def _opf(path: Path, z: zipfile.ZipFile) -> tuple[str, Element]: + """The name of the OPF file in the epub, and its content.""" + container = ElementTree.fromstring(z.read("META-INF/container.xml")) + rootfile = container.find(f".//{CONTAINER}rootfile") + if rootfile is None or not rootfile.get("full-path"): + raise EpubError(f"{path.name} has no OPF file") + return rootfile.get("full-path"), ElementTree.fromstring(z.read(rootfile.get("full-path"))) + + def _metadata(path: Path) -> Element: try: with zipfile.ZipFile(path) as z: - container = ElementTree.fromstring(z.read("META-INF/container.xml")) - rootfile = container.find(f".//{CONTAINER}rootfile") - if rootfile is None or not rootfile.get("full-path"): - raise EpubError(f"{path.name} has no OPF file") - opf = ElementTree.fromstring(z.read(rootfile.get("full-path"))) + opf = _opf(path, z)[1] except (zipfile.BadZipFile, KeyError, ElementTree.ParseError) as e: raise EpubError(f"{path.name} is not a readable epub: {e}") from e metadata = opf.find(f"{OPF}metadata") @@ -33,6 +40,21 @@ def _metadata(path: Path) -> Element: return metadata +def spine_documents(path: Path) -> list[str]: + """The text of each document in the spine of an epub, in reading order.""" + try: + with zipfile.ZipFile(path) as z: + name, opf = _opf(path, z) + hrefs = {item.get("id"): item.get("href") for item in opf.iter(f"{OPF}item")} + files = [ + posixpath.join(posixpath.dirname(name), unquote(hrefs[ref.get("idref")])) + for ref in opf.iter(f"{OPF}itemref") + ] + return [z.read(file).decode("utf-8", "replace") for file in files] + except (zipfile.BadZipFile, KeyError, ElementTree.ParseError) as e: + raise EpubError(f"{path.name} is not a readable epub: {e}") from e + + def epub_urls(path: Path) -> list[str]: """Every URL in the identifiers and the source of an epub's OPF. diff --git a/bindery/opds.py b/bindery/opds.py index e435f75..87168a7 100644 --- a/bindery/opds.py +++ b/bindery/opds.py @@ -77,6 +77,7 @@ def _languages(value: Any) -> list[str]: def _books() -> list[dict[str, Any]]: """The books of the catalog, in title order.""" books = [] + page_counts = orchestrator.store.story_settings("page_count") for story in orchestrator.store.catalog(): meta = orchestrator.book_metadata(story) title = (meta["title"] or "").strip() or story["url"] @@ -89,6 +90,7 @@ def _books() -> list[dict[str, Any]]: "title": title, "authors": authors, "updated": _updated(story), + "page_count": page_counts.get(story["id"]), "cover": hashlib.sha256(cover.encode()).hexdigest()[:16] if cover else None, } ) @@ -107,6 +109,8 @@ def _content(book: dict[str, Any]) -> str: index = filename_template.series_index(meta.get("series_index")) lines.append(f"Series: {meta['series']}" + (f", book {index}" if index else "")) lines.append(f"Chapters: {book['chapter_count']}") + if book["page_count"] is not None: + lines.append(f"Pages: {book['page_count']}") description = "\n\n".join(diffing.html_to_blocks(meta.get("comments") or "")) return "\n".join(lines) + (f"\n\n{description}" if description else "") diff --git a/bindery/orchestrator.py b/bindery/orchestrator.py index 1d33f13..e481fef 100644 --- a/bindery/orchestrator.py +++ b/bindery/orchestrator.py @@ -13,7 +13,7 @@ from typing import Any, BinaryIO from werkzeug.security import generate_password_hash -from bindery import config, fff, filename_template +from bindery import config, fff, filename_template, pages from bindery.backup import daily_backup from bindery.cwa import Book, BookFinder, CwaClient, CwaError from bindery.diffing import diff_html, text_hash @@ -226,9 +226,11 @@ def list_stories() -> list[dict[str, Any]]: stories = store.list_stories() held = store.story_settings("upload_held") unapproved = store.story_settings("unapproved_link") + page_counts = store.story_settings("page_count") show_holds = _link_books() and store.get_setting(None, "hold_new_links") and _can_upload() for s in stories: s["busy"] = job_queue.has_active(s["id"]) + s["page_count"] = page_counts.get(s["id"]) s["upload_held"] = bool(held.get(s["id"])) s["approval_held"] = bool( show_holds @@ -837,6 +839,14 @@ def _build_key(story: dict[str, Any], overrides: dict[str, Any], site: dict[str, return digest.hexdigest() +def _count_pages(story_id: int, path: Path) -> None: + """Store the estimated pages of a built epub, for the story list and the OPDS catalog.""" + try: + store.set_setting(story_id, "page_count", pages.epub_pages(path)) + except EpubError as e: + logger.warning(f"Cannot count the pages of story {story_id}: {e}") + + def _built_epub( story: dict[str, Any], overrides: dict[str, Any], key: str, force: bool = False, record: bool = True ) -> Path: @@ -846,7 +856,8 @@ def _built_epub( """ path = config.TMP_DIR / f"story-{story['id']}-{key[:16]}.epub" with _build_lock: - if force or not path.exists(): + built = force or not path.exists() + if built: # A build under another name first, so a download never reads half a book. partial = config.TMP_DIR / f"building-{uuid.uuid4().hex}.epub" try: @@ -857,6 +868,9 @@ def _built_epub( for old in config.TMP_DIR.glob(f"story-{story['id']}-*.epub"): if old != path: old.unlink(missing_ok=True) + # A book from before the page count has no count yet. + if built or store.get_setting(story["id"], "page_count") is None: + _count_pages(story["id"], path) if record: store.set_setting(story["id"], "built_key", key) return path diff --git a/bindery/pages.py b/bindery/pages.py new file mode 100644 index 0000000..e56c7ab --- /dev/null +++ b/bindery/pages.py @@ -0,0 +1,78 @@ +"""An estimate of the pages of a built epub, after the "Paragraphs (APNX Accurate)" method of the Count Pages +plugin for calibre. + +The method counts lines. A paragraph start is one line, and so is each 70 characters of text. A page is 31 +lines. A page is also at most 2400 characters, which sets the count of a book with almost no paragraphs. + +The plugin counts either the
or the
elements, as
One
Two
") == Measure(2, 6) + assert pages.measure("One
MiddleTwo
EndThree") == Measure( + 4, 16 + ) + + +@pytest.mark.parametrize( + "book", + [ + "".join(f"
{line}
" for line in TEXT), + "".join(f'{line}
\n' for line in TEXT), + "{TEXT[1]}
{TEXT[2]}abcdef gh
') == Measure(1, 9) + assert pages.measure("| Name | Value |
| a | b |
|---|
Before
after
Shown & counted
\n\n" + ) + assert pages.measure(document) == Measure(1, len("Shown & counted")) + assert pages.measure("") == Measure(0, 0) + assert pages.measure("
\n
") == Measure(0, 0) + + +def test_a_paragraph_that_only_spaces_the_text_is_not_counted(): + spaced = "One
\xa0
Two
" + assert pages.measure(spaced) == pages.measure("One
Two
") == Measure(2, 6) + assert pages.measure("One two\xa0three
") == Measure(1, 13) + + +def test_a_page_is_31_lines_and_a_line_is_a_paragraph_or_70_characters(): + assert pages.page_count(Measure(62, 620)) == (62 + 620 // 70) // 31 == 2 + assert pages.page_count(Measure(31 * 9, 31 * 9)) == 9 + assert pages.page_count(Measure(1000, 177_000)) == (1000 + 177_000 // 70) // 31 + + +def test_a_short_book_has_one_page_and_a_book_with_no_text_has_none(): + assert pages.page_count(Measure(1, 10)) == 1 + assert pages.page_count(Measure(30, 30)) == 1 + assert pages.page_count(Measure()) == 0 + # One block of text: 2400 characters start the second page before 31 lines do. + assert pages.page_count(Measure(1, 2399)) == 1 + assert pages.page_count(Measure(1, 2400)) == 2 + + +def epub(path, documents, spine=None): + names = [f"file {n}.xhtml" for n in range(len(documents))] + items = "".join(f'x
" * 31 + "" + book = epub(tmp_path / "book.epub", [chapter, "Not in the spine
", chapter], [2, 0]) + assert spine_documents(book) == [chapter, chapter] + # 62 paragraphs of one character. + assert pages.epub_pages(book) == 2 + assert pages.epub_pages(epub(tmp_path / "empty.epub", ["
"])) == 0
+
+
+def test_a_file_that_is_not_an_epub_cannot_be_counted(tmp_path):
+ path = tmp_path / "bad.epub"
+ path.write_bytes(b"not a zip")
+ with pytest.raises(EpubError):
+ pages.epub_pages(path)
+ with zipfile.ZipFile(path, "w") as z:
+ z.writestr("META-INF/container.xml", "