diff --git a/README.md b/README.md index 332aee7..ac4bb6c 100644 --- a/README.md +++ b/README.md @@ -650,6 +650,37 @@ always the site name. replaces only the ePub file. So edits that you make in CWA stay, and later changes on the site show in the ePub but not in the CWA catalog. +## Page count + +The story list shows an estimated page count beside the chapter count. The OPDS catalog shows it in the +description of a book. Bindery counts the pages as the last step when it builds an ePub, so the count +includes your replacement rules and the line-break setting. A story shows no page count until Bindery +builds its book. + +The method comes from "Paragraphs (APNX Accurate)" in the +[Count Pages plugin](https://github.com/kiwidude68/calibre_plugins/wiki/Count-Pages) for calibre: + +- A paragraph start is one line. Each 70 characters of text is one more line. +- A page is 31 lines. +- A page is at most 2400 characters. This limit sets the count of a book with almost no paragraphs. + +The plugin counts either the `

` elements or the `

` elements of a book as its paragraphs. Bindery +counts each paragraph that a reader sees: + +- Text starts a paragraph after a `
`, and after the start or the end of a block element such as `

`, + `

`, `
  • ` or a heading. +- A run of `
    ` elements is one paragraph break. +- An element that only holds other blocks is not a paragraph. +- The cells of a table row share one line. +- A paragraph with no text is not counted. This includes a paragraph that holds only a no-break space, + which some stories put between paragraphs as a blank line. + +So the same text gives the same count as `

    ` elements, as `

    ` elements, and as lines with `
    ` +between them. For a book that uses `

    ` elements and has no empty paragraphs, the count is within about 1% +of the count of the plugin. The plugin counts each empty paragraph as a line, so its count is higher for a +book that has many of them. +The count is an estimate and differs from the page count of a printed book. + ## Replacement rules A rule is a regular expression, a replacement and optional flags (`i`, `s`, `m`, `x`). Bindery uses diff --git a/bindery/epub.py b/bindery/epub.py index 34f0847..d5af57e 100644 --- a/bindery/epub.py +++ b/bindery/epub.py @@ -1,9 +1,11 @@ """Read the OPF metadata of an epub that came from CWA.""" +import posixpath import re import zipfile from pathlib import Path from typing import Any +from urllib.parse import unquote from xml.etree.ElementTree import Element from defusedxml import ElementTree @@ -17,14 +19,19 @@ class EpubError(Exception): pass +def _opf(path: Path, z: zipfile.ZipFile) -> tuple[str, Element]: + """The name of the OPF file in the epub, and its content.""" + container = ElementTree.fromstring(z.read("META-INF/container.xml")) + rootfile = container.find(f".//{CONTAINER}rootfile") + if rootfile is None or not rootfile.get("full-path"): + raise EpubError(f"{path.name} has no OPF file") + return rootfile.get("full-path"), ElementTree.fromstring(z.read(rootfile.get("full-path"))) + + def _metadata(path: Path) -> Element: try: with zipfile.ZipFile(path) as z: - container = ElementTree.fromstring(z.read("META-INF/container.xml")) - rootfile = container.find(f".//{CONTAINER}rootfile") - if rootfile is None or not rootfile.get("full-path"): - raise EpubError(f"{path.name} has no OPF file") - opf = ElementTree.fromstring(z.read(rootfile.get("full-path"))) + opf = _opf(path, z)[1] except (zipfile.BadZipFile, KeyError, ElementTree.ParseError) as e: raise EpubError(f"{path.name} is not a readable epub: {e}") from e metadata = opf.find(f"{OPF}metadata") @@ -33,6 +40,21 @@ def _metadata(path: Path) -> Element: return metadata +def spine_documents(path: Path) -> list[str]: + """The text of each document in the spine of an epub, in reading order.""" + try: + with zipfile.ZipFile(path) as z: + name, opf = _opf(path, z) + hrefs = {item.get("id"): item.get("href") for item in opf.iter(f"{OPF}item")} + files = [ + posixpath.join(posixpath.dirname(name), unquote(hrefs[ref.get("idref")])) + for ref in opf.iter(f"{OPF}itemref") + ] + return [z.read(file).decode("utf-8", "replace") for file in files] + except (zipfile.BadZipFile, KeyError, ElementTree.ParseError) as e: + raise EpubError(f"{path.name} is not a readable epub: {e}") from e + + def epub_urls(path: Path) -> list[str]: """Every URL in the identifiers and the source of an epub's OPF. diff --git a/bindery/opds.py b/bindery/opds.py index e435f75..87168a7 100644 --- a/bindery/opds.py +++ b/bindery/opds.py @@ -77,6 +77,7 @@ def _languages(value: Any) -> list[str]: def _books() -> list[dict[str, Any]]: """The books of the catalog, in title order.""" books = [] + page_counts = orchestrator.store.story_settings("page_count") for story in orchestrator.store.catalog(): meta = orchestrator.book_metadata(story) title = (meta["title"] or "").strip() or story["url"] @@ -89,6 +90,7 @@ def _books() -> list[dict[str, Any]]: "title": title, "authors": authors, "updated": _updated(story), + "page_count": page_counts.get(story["id"]), "cover": hashlib.sha256(cover.encode()).hexdigest()[:16] if cover else None, } ) @@ -107,6 +109,8 @@ def _content(book: dict[str, Any]) -> str: index = filename_template.series_index(meta.get("series_index")) lines.append(f"Series: {meta['series']}" + (f", book {index}" if index else "")) lines.append(f"Chapters: {book['chapter_count']}") + if book["page_count"] is not None: + lines.append(f"Pages: {book['page_count']}") description = "\n\n".join(diffing.html_to_blocks(meta.get("comments") or "")) return "\n".join(lines) + (f"\n\n{description}" if description else "") diff --git a/bindery/orchestrator.py b/bindery/orchestrator.py index 1d33f13..e481fef 100644 --- a/bindery/orchestrator.py +++ b/bindery/orchestrator.py @@ -13,7 +13,7 @@ from typing import Any, BinaryIO from werkzeug.security import generate_password_hash -from bindery import config, fff, filename_template +from bindery import config, fff, filename_template, pages from bindery.backup import daily_backup from bindery.cwa import Book, BookFinder, CwaClient, CwaError from bindery.diffing import diff_html, text_hash @@ -226,9 +226,11 @@ def list_stories() -> list[dict[str, Any]]: stories = store.list_stories() held = store.story_settings("upload_held") unapproved = store.story_settings("unapproved_link") + page_counts = store.story_settings("page_count") show_holds = _link_books() and store.get_setting(None, "hold_new_links") and _can_upload() for s in stories: s["busy"] = job_queue.has_active(s["id"]) + s["page_count"] = page_counts.get(s["id"]) s["upload_held"] = bool(held.get(s["id"])) s["approval_held"] = bool( show_holds @@ -837,6 +839,14 @@ def _build_key(story: dict[str, Any], overrides: dict[str, Any], site: dict[str, return digest.hexdigest() +def _count_pages(story_id: int, path: Path) -> None: + """Store the estimated pages of a built epub, for the story list and the OPDS catalog.""" + try: + store.set_setting(story_id, "page_count", pages.epub_pages(path)) + except EpubError as e: + logger.warning(f"Cannot count the pages of story {story_id}: {e}") + + def _built_epub( story: dict[str, Any], overrides: dict[str, Any], key: str, force: bool = False, record: bool = True ) -> Path: @@ -846,7 +856,8 @@ def _built_epub( """ path = config.TMP_DIR / f"story-{story['id']}-{key[:16]}.epub" with _build_lock: - if force or not path.exists(): + built = force or not path.exists() + if built: # A build under another name first, so a download never reads half a book. partial = config.TMP_DIR / f"building-{uuid.uuid4().hex}.epub" try: @@ -857,6 +868,9 @@ def _built_epub( for old in config.TMP_DIR.glob(f"story-{story['id']}-*.epub"): if old != path: old.unlink(missing_ok=True) + # A book from before the page count has no count yet. + if built or store.get_setting(story["id"], "page_count") is None: + _count_pages(story["id"], path) if record: store.set_setting(story["id"], "built_key", key) return path diff --git a/bindery/pages.py b/bindery/pages.py new file mode 100644 index 0000000..e56c7ab --- /dev/null +++ b/bindery/pages.py @@ -0,0 +1,78 @@ +"""An estimate of the pages of a built epub, after the "Paragraphs (APNX Accurate)" method of the Count Pages +plugin for calibre. + +The method counts lines. A paragraph start is one line, and so is each 70 characters of text. A page is 31 +lines. A page is also at most 2400 characters, which sets the count of a book with almost no paragraphs. + +The plugin counts either the

    or the

    elements of a book as its paragraphs. Bindery counts what a +reader sees as a paragraph: text that follows a
    or the start or end of a block element. So the same text +gives the same count as

    elements, as

    elements, and as lines with
    between them. A paragraph with +no text, or with only a no-break space, is not counted. +""" + +import html as htmllib +import re +from pathlib import Path +from typing import NamedTuple + +from bindery.epub import spine_documents + +LINE_CHARS = 70 +LINES_PER_PAGE = 31 +CHARS_PER_PAGE = 2400 +# Elements that stay in the line of the text around them. Each other element, and
    , ends a paragraph. +# The cells of a table row share a line. +INLINE = frozenset( + "a abbr acronym b bdi bdo big cite code data del dfn em font i img ins kbd mark q rp rt ruby s samp small " + "span strike strong sub sup td th time tt u var wbr".split() +) +# Elements whose text is not text of the book. +HIDDEN = frozenset(("head", "script", "style")) +_TOKEN = re.compile(r"<(/?)([a-zA-Z][a-zA-Z0-9]*)[^>]*>|<[^>]*>|([^<]+)") + + +class Measure(NamedTuple): + paragraphs: int = 0 + chars: int = 0 + + +def measure(html: str) -> Measure: + """The paragraphs and the characters of text of one document.""" + paragraphs = chars = 0 + hidden = False + new_paragraph = True + for match in _TOKEN.finditer(html): + closing, name, text = match.groups() + if name: + name = name.lower() + if name in HIDDEN: + hidden = not closing and not match.group().endswith("/>") + elif name not in INLINE: + new_paragraph = True + elif text and not hidden: + if "&" in text: + text = htmllib.unescape(text) + # A no-break space is white space here, so a paragraph that only spaces the text is no paragraph. + if text.strip(): + if new_paragraph: + paragraphs += 1 + new_paragraph = False + text = text.lstrip() + chars += len(text) + elif not new_paragraph: + chars += 1 # the space between two inline elements + return Measure(paragraphs, chars) + + +def page_count(total: Measure) -> int: + """The pages of a book, from the sums of its documents. A book with no text has no pages.""" + if not total.chars: + return 0 + lines = total.paragraphs + total.chars // LINE_CHARS + return max(lines // LINES_PER_PAGE, total.chars // CHARS_PER_PAGE + 1) + + +def epub_pages(path: Path) -> int: + """The estimated pages of an epub, from each document in its spine.""" + measures = [measure(text) for text in spine_documents(path)] + return page_count(Measure(sum(m.paragraphs for m in measures), sum(m.chars for m in measures))) diff --git a/bindery/static/js/stories.js b/bindery/static/js/stories.js index 0e8285c..2949f53 100644 --- a/bindery/static/js/stories.js +++ b/bindery/static/js/stories.js @@ -9,6 +9,7 @@ const STORY_COLUMNS = [ { label: 'Author', key: 'author', type: 'text' }, { label: 'Site', key: 'site', type: 'text' }, { label: 'Chapters', key: 'chapter_count', type: 'number' }, + { label: 'Pages', key: 'page_count', type: 'number' }, { label: 'Status', key: 'status', type: 'text' }, { label: 'Last change', key: 'last_changed', type: 'date' }, { label: 'Last check', key: 'last_checked', type: 'date' }, @@ -231,6 +232,8 @@ function storyRow(s) { s.offsite_count > 0 && h('small', { title: 'Chapters the site no longer lists' }, `${s.offsite_count} removed`), s.revised_count > 0 && h('small', { title: 'Chapters the site changed since Bindery added them' }, `${s.revised_count} revised`), ), + // A story has no page count until Bindery builds its book. + h('td', { 'data-label': 'Pages', title: 'Estimated from the built book' }, s.page_count != null && String(s.page_count)), h( 'td', { 'data-label': 'Status' }, @@ -276,6 +279,8 @@ export function storiesPage(main, _params, config) { const order = (left - right) * direction; if (order) return order; } else if (sort !== 'title') { + // A story with no number goes last in each direction. + if (sortType === 'number' && (a[sort] == null) !== (b[sort] == null)) return a[sort] == null ? 1 : -1; const order = sortType === 'number' ? a[sort] - b[sort] : (a[sort] || '').localeCompare(b[sort] || '', undefined, { sensitivity: 'base' }); diff --git a/bindery/static/style.css b/bindery/static/style.css index 1a4a2e4..574e23a 100644 --- a/bindery/static/style.css +++ b/bindery/static/style.css @@ -940,6 +940,11 @@ textarea { color: var(--muted); } + /* A story with no page count yet shows no line for it. */ + .stories-table td:empty { + display: none; + } + /* The chapter list has far more rows than the story list, so each row packs its cells onto three lines instead of one line per cell: checkbox and number, title, then the rest. */ .chapters-table tbody tr { diff --git a/tests/js/flows.test.mjs b/tests/js/flows.test.mjs index 1d7ce70..0a59183 100644 --- a/tests/js/flows.test.mjs +++ b/tests/js/flows.test.mjs @@ -380,7 +380,7 @@ test('story list stacks on a narrow screen instead of scrolling sideways', async const wrap = nodes(main).find((el) => el.className === 'table-wrap stories-table'); assert.ok(wrap); const dataLabels = nodes(wrap).filter((el) => el.tagName === 'td').map((el) => el.attributes['data-label']); - assert.deepEqual(dataLabels, [undefined, 'Author', 'Site', 'Chapters', 'Status', 'Last change', 'Last check']); + assert.deepEqual(dataLabels, [undefined, 'Author', 'Site', 'Chapters', 'Pages', 'Status', 'Last change', 'Last check']); }); test('the story list shows removed and revised chapter counts, only when there are any', async () => { diff --git a/tests/js/layout.test.mjs b/tests/js/layout.test.mjs index f71bd3b..74fda30 100644 --- a/tests/js/layout.test.mjs +++ b/tests/js/layout.test.mjs @@ -114,6 +114,12 @@ test('sortable headings show direction and remain usable above story cards on na assert.doesNotMatch(css, /\.stories-table thead\s*\{[^}]*clip-path/); }); +test('on narrow screens a story cell with no value shows no line', () => { + const start = css.indexOf('@media (max-width: 600px)', css.indexOf('.stories-table th button')); + assert.equal(declarations('.stories-table td:empty', css.slice(start)).display, 'none'); + assert.doesNotMatch(css.slice(0, start), /\.stories-table td:empty/); +}); + test('on narrow screens the headings are sort chips, and only the sorted chip is tinted with an arrow', () => { const start = css.indexOf('@media (max-width: 600px)', css.indexOf('.stories-table th button')); const mobile = css.slice(start); diff --git a/tests/js/story-list.test.mjs b/tests/js/story-list.test.mjs index 0ffe8df..12975ee 100644 --- a/tests/js/story-list.test.mjs +++ b/tests/js/story-list.test.mjs @@ -21,7 +21,7 @@ const story = (id, title, changes = {}) => ({ id, title, url: `https://example.com/${id}`, sort_title: (title || `https://example.com/${id}`).trim().replace(/^(?:the|an?)\s+/i, '').toLowerCase(), author: 'Reader', site: 'example.com', - chapter_count: 42, offsite_count: 0, revised_count: 0, status: 'ok', message: 'No changes', + chapter_count: 42, page_count: 120, offsite_count: 0, revised_count: 0, status: 'ok', message: 'No changes', last_changed: null, last_checked: null, ...changes, }); const rows = () => { @@ -106,6 +106,39 @@ test('chapter sorting compares counts numerically and keeps equal counts in titl assert.equal(heading('Chapters').attributes['aria-sort'], 'ascending'); }); +test('pages show beside the chapters and sort numerically', async (t) => { + await mount(t, [ + story(1, 'Zeta', { page_count: 9 }), + story(2, 'Alpha', { page_count: 1200 }), + story(3, 'Beta', { page_count: 0 }), + story(4, 'Gamma', { page_count: 9 }), + ]); + const cells = nodes(rows()[0]).filter((el) => el.tagName === 'td').map((el) => el.attributes['data-label']); + assert.deepEqual(cells.slice(3, 5), ['Chapters', 'Pages']); + assert.equal(find(app, 'button', 'Pages').attributes['aria-label'], 'Sort by pages, most first'); + await click(app, 'Pages'); + assert.deepEqual(titles(), ['Alpha', 'Gamma', 'Zeta', 'Beta']); + await click(app, 'Pages'); + assert.deepEqual(titles(), ['Beta', 'Gamma', 'Zeta', 'Alpha']); + assert.deepEqual(JSON.parse(window.localStorage.getItem(SORT_KEY)), { column: 'page_count', direction: 1 }); +}); + +test('a story with no built book shows no page count and sorts last in each direction', async (t) => { + await mount(t, [ + story(1, 'Alpha', { page_count: null }), + story(2, 'Beta', { page_count: 300 }), + story(3, 'Gamma', { page_count: 0 }), + story(4, 'Delta', { page_count: null }), + ]); + const pages = () => rows().map((row) => cell(row, 'Pages')); + assert.deepEqual(pages().map((el) => el.textContent), ['', '300', '', '0']); + assert.deepEqual(pages().map((el) => el.children.length), [0, 1, 0, 1], 'An empty cell hides on a narrow screen'); + await click(app, 'Pages'); + assert.deepEqual(titles(), ['Beta', 'Gamma', 'Alpha', 'Delta']); + await click(app, 'Pages'); + assert.deepEqual(titles(), ['Gamma', 'Beta', 'Alpha', 'Delta']); +}); + for (const [label, key] of [['Author', 'author'], ['Site', 'site'], ['Status', 'status']]) { test(`${label} sorts alphabetically in either direction and uses titles to break ties`, async (t) => { await mount(t, [ @@ -248,7 +281,7 @@ test('date columns keep each timestamp on one line, with last change before last const checked = '2026-01-03T05:06:00Z'; await mount(t, [story(1, 'Alpha', { last_changed: changed, last_checked: checked })]); const headings = nodes(app).filter((el) => el.tagName === 'th').map((el) => el.textContent); - assert.deepEqual(headings, ['Title', 'Author', 'Site', 'Chapters', 'Status', 'Last change', 'Last check']); + assert.deepEqual(headings, ['Title', 'Author', 'Site', 'Chapters', 'Pages', 'Status', 'Last change', 'Last check']); for (const [label, value] of [['Last change', changed], ['Last check', checked]]) { const dateCell = cell(rows()[0], label); const time = nodes(dateCell).find((el) => el.tagName === 'time'); diff --git a/tests/test_build_keys.py b/tests/test_build_keys.py index 611076e..9d5dd67 100644 --- a/tests/test_build_keys.py +++ b/tests/test_build_keys.py @@ -11,7 +11,7 @@ from test_deliver import fake as fake from test_deliver import jobs as jobs from test_deliver import story_epub as story_epub -from bindery import config, orchestrator +from bindery import config, orchestrator, pages from bindery.jobs import DownloadMode, Job ONE = [(1, "One", "a")] @@ -242,6 +242,59 @@ def test_a_download_builds_a_book_that_has_no_file_or_that_changed(builds, clien assert story(story_id)["last_changed"] +# Page counts + +LONG = "A line of the story. " * 20 + + +def listed_pages(story_id): + return {s["id"]: s["page_count"] for s in orchestrator.list_stories()}[story_id] + + +def test_a_story_has_no_page_count_until_its_book_is_built(builds, monkeypatch): + story_id = linked_story(ONE) + assert listed_pages(story_id) is None + site(monkeypatch, ONE) + update(story_id) + (built,) = config.TMP_DIR.iterdir() + assert listed_pages(story_id) == pages.epub_pages(built) == 1 + + +def test_each_build_counts_the_pages_of_the_new_book(builds, client, monkeypatch): + story_id = linked_story(ONE) + site(monkeypatch, [*ONE, *[(n, f"Chapter {n}", "
    ".join([LONG] * 31)) for n in range(2, 6)]]) + update(story_id) + long_book = listed_pages(story_id) + # Each of the 124 lines is a paragraph and 6 lines of text. + assert long_book >= 4 * 31 * 7 // 31 + # The count is of the built book, so a rule that removes text changes it without a download. + orchestrator.save_rules(story_id, [{"pattern": "(A line of the story. ){19}", "replacement": ""}]) + download(client, story_id) + assert builds.call_count == 2 + assert 0 < listed_pages(story_id) < long_book / 3 + + +def test_a_book_from_before_the_page_count_is_counted_without_a_build(builds, client, monkeypatch): + story_id = linked_story(ONE) + site(monkeypatch, ONE) + update(story_id) + orchestrator.store.set_setting(story_id, "page_count", None) + download(client, story_id) + assert builds.call_count == 1 + assert listed_pages(story_id) == 1 + + +def test_a_book_that_cannot_be_counted_is_still_built(builds, monkeypatch): + def fail(path): + raise orchestrator.EpubError("unreadable") + + monkeypatch.setattr(pages, "epub_pages", fail) + story_id = linked_story(ONE) + site(monkeypatch, ONE) + assert "library book 1" in update(story_id) + assert listed_pages(story_id) is None + + def test_stop_tracking_deletes_the_built_book(builds, client): story_id, other = add_story(), orchestrator.store.create_story("https://other", "s", "Other", "W", []) download(client, story_id) diff --git a/tests/test_opds.py b/tests/test_opds.py index 0bf7f0a..398997a 100644 --- a/tests/test_opds.py +++ b/tests/test_opds.py @@ -205,6 +205,17 @@ def test_entry_shows_the_book_metadata_and_cover(client): assert client.get(f"/opds/books/{add_book()}/cover").status_code == 404 +def test_entry_shows_the_page_count_of_a_built_book(client): + sid = add_book() + orchestrator.store.set_setting(sid, "page_count", 321) + add_book(title="Not built") + contents = { + e.findtext("a:title", namespaces=NS): e.findtext("a:content", namespaces=NS) + for e in entries(feed(client.get("/opds/title"))) + } + assert contents == {"A book": "Chapters: 1\nPages: 321", "Not built": "Chapters: 1"} + + def test_cover_of_an_unsafe_type_is_left_out(client): sid = add_book() orchestrator.store.put_image(sid, "https://example.com/c.svg", "svg", "image/svg+xml", b"", is_cover=True) diff --git a/tests/test_pages.py b/tests/test_pages.py new file mode 100644 index 0000000..a183b59 --- /dev/null +++ b/tests/test_pages.py @@ -0,0 +1,122 @@ +"""The page estimate of a document and of a built epub.""" + +import zipfile + +import pytest + +from bindery import pages +from bindery.epub import EpubError, spine_documents +from bindery.pages import Measure + +TEXT = ["One line of the story.", "Another, longer line of the story that goes on for a while.", "The end."] +CHARS = sum(map(len, TEXT)) + + +def test_a_paragraph_is_text_after_a_block_or_a_line_break(): + assert pages.measure("

    One

    Two

    ") == Measure(2, 6) + assert pages.measure("
    One

    Two
    Three
    ") == Measure(3, 11) + assert pages.measure("
    One
    Two
    ") == Measure(2, 6) + # Text before, between and after the blocks of an element counts too. + assert pages.measure("
    Intro

    One

    Middle

    Two

    End
    ") == Measure(5, 20) + assert pages.measure("

    Title

    • One
    • Two
    Three
    ") == Measure( + 4, 16 + ) + + +@pytest.mark.parametrize( + "book", + [ + "".join(f"

    {line}

    " for line in TEXT), + "".join(f'

    {line}

    \n' for line in TEXT), + "
    " + "".join(f"
    {line}
    " for line in TEXT) + "
    ", + "
    " + "

    ".join(TEXT) + "
    ", + "
    " + "
    \n".join(TEXT) + "
    ", + f"
    {TEXT[0]}

    {TEXT[1]}

    {TEXT[2]}
    ", + ], +) +def test_the_same_text_gives_the_same_measure_in_each_markup(book): + assert pages.measure(book) == Measure(3, CHARS) + + +def test_inline_elements_and_table_cells_stay_in_the_line(): + assert pages.measure('

    abcdef gh

    ') == Measure(1, 9) + assert pages.measure("
    NameValue
    ab
    ") == Measure( + 2, 11 + ) + assert pages.measure('

    Beforeafter

    ') == Measure(1, 11) + + +def test_only_the_text_of_the_book_counts(): + document = ( + 'Hidden title' + '