Show an estimated page count for each built book
Bindery counts the pages as the last step of a build, from the text of the ePub. The story list and the OPDS catalog show the count. A story has no count until its book is built. The method follows "Paragraphs (APNX Accurate)" of the Count Pages plugin for calibre. Bindery counts each paragraph that a reader sees, so the same text gives the same count as <p> elements, as <div> elements and as lines with <br> between them. An empty paragraph is not counted.
This commit is contained in:
@@ -650,6 +650,37 @@ always the site name.
|
|||||||
replaces only the ePub file. So edits that you make in CWA stay, and later changes on the site show
|
replaces only the ePub file. So edits that you make in CWA stay, and later changes on the site show
|
||||||
in the ePub but not in the CWA catalog.
|
in the ePub but not in the CWA catalog.
|
||||||
|
|
||||||
|
## Page count
|
||||||
|
|
||||||
|
The story list shows an estimated page count beside the chapter count. The OPDS catalog shows it in the
|
||||||
|
description of a book. Bindery counts the pages as the last step when it builds an ePub, so the count
|
||||||
|
includes your replacement rules and the line-break setting. A story shows no page count until Bindery
|
||||||
|
builds its book.
|
||||||
|
|
||||||
|
The method comes from "Paragraphs (APNX Accurate)" in the
|
||||||
|
[Count Pages plugin](https://github.com/kiwidude68/calibre_plugins/wiki/Count-Pages) for calibre:
|
||||||
|
|
||||||
|
- A paragraph start is one line. Each 70 characters of text is one more line.
|
||||||
|
- A page is 31 lines.
|
||||||
|
- A page is at most 2400 characters. This limit sets the count of a book with almost no paragraphs.
|
||||||
|
|
||||||
|
The plugin counts either the `<p>` elements or the `<div>` elements of a book as its paragraphs. Bindery
|
||||||
|
counts each paragraph that a reader sees:
|
||||||
|
|
||||||
|
- Text starts a paragraph after a `<br>`, and after the start or the end of a block element such as `<p>`,
|
||||||
|
`<div>`, `<li>` or a heading.
|
||||||
|
- A run of `<br>` elements is one paragraph break.
|
||||||
|
- An element that only holds other blocks is not a paragraph.
|
||||||
|
- The cells of a table row share one line.
|
||||||
|
- A paragraph with no text is not counted. This includes a paragraph that holds only a no-break space,
|
||||||
|
which some stories put between paragraphs as a blank line.
|
||||||
|
|
||||||
|
So the same text gives the same count as `<p>` elements, as `<div>` elements, and as lines with `<br>`
|
||||||
|
between them. For a book that uses `<p>` elements and has no empty paragraphs, the count is within about 1%
|
||||||
|
of the count of the plugin. The plugin counts each empty paragraph as a line, so its count is higher for a
|
||||||
|
book that has many of them.
|
||||||
|
The count is an estimate and differs from the page count of a printed book.
|
||||||
|
|
||||||
## Replacement rules
|
## Replacement rules
|
||||||
|
|
||||||
A rule is a regular expression, a replacement and optional flags (`i`, `s`, `m`, `x`). Bindery uses
|
A rule is a regular expression, a replacement and optional flags (`i`, `s`, `m`, `x`). Bindery uses
|
||||||
|
|||||||
+27
-5
@@ -1,9 +1,11 @@
|
|||||||
"""Read the OPF metadata of an epub that came from CWA."""
|
"""Read the OPF metadata of an epub that came from CWA."""
|
||||||
|
|
||||||
|
import posixpath
|
||||||
import re
|
import re
|
||||||
import zipfile
|
import zipfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
from urllib.parse import unquote
|
||||||
from xml.etree.ElementTree import Element
|
from xml.etree.ElementTree import Element
|
||||||
|
|
||||||
from defusedxml import ElementTree
|
from defusedxml import ElementTree
|
||||||
@@ -17,14 +19,19 @@ class EpubError(Exception):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _opf(path: Path, z: zipfile.ZipFile) -> tuple[str, Element]:
|
||||||
|
"""The name of the OPF file in the epub, and its content."""
|
||||||
|
container = ElementTree.fromstring(z.read("META-INF/container.xml"))
|
||||||
|
rootfile = container.find(f".//{CONTAINER}rootfile")
|
||||||
|
if rootfile is None or not rootfile.get("full-path"):
|
||||||
|
raise EpubError(f"{path.name} has no OPF file")
|
||||||
|
return rootfile.get("full-path"), ElementTree.fromstring(z.read(rootfile.get("full-path")))
|
||||||
|
|
||||||
|
|
||||||
def _metadata(path: Path) -> Element:
|
def _metadata(path: Path) -> Element:
|
||||||
try:
|
try:
|
||||||
with zipfile.ZipFile(path) as z:
|
with zipfile.ZipFile(path) as z:
|
||||||
container = ElementTree.fromstring(z.read("META-INF/container.xml"))
|
opf = _opf(path, z)[1]
|
||||||
rootfile = container.find(f".//{CONTAINER}rootfile")
|
|
||||||
if rootfile is None or not rootfile.get("full-path"):
|
|
||||||
raise EpubError(f"{path.name} has no OPF file")
|
|
||||||
opf = ElementTree.fromstring(z.read(rootfile.get("full-path")))
|
|
||||||
except (zipfile.BadZipFile, KeyError, ElementTree.ParseError) as e:
|
except (zipfile.BadZipFile, KeyError, ElementTree.ParseError) as e:
|
||||||
raise EpubError(f"{path.name} is not a readable epub: {e}") from e
|
raise EpubError(f"{path.name} is not a readable epub: {e}") from e
|
||||||
metadata = opf.find(f"{OPF}metadata")
|
metadata = opf.find(f"{OPF}metadata")
|
||||||
@@ -33,6 +40,21 @@ def _metadata(path: Path) -> Element:
|
|||||||
return metadata
|
return metadata
|
||||||
|
|
||||||
|
|
||||||
|
def spine_documents(path: Path) -> list[str]:
|
||||||
|
"""The text of each document in the spine of an epub, in reading order."""
|
||||||
|
try:
|
||||||
|
with zipfile.ZipFile(path) as z:
|
||||||
|
name, opf = _opf(path, z)
|
||||||
|
hrefs = {item.get("id"): item.get("href") for item in opf.iter(f"{OPF}item")}
|
||||||
|
files = [
|
||||||
|
posixpath.join(posixpath.dirname(name), unquote(hrefs[ref.get("idref")]))
|
||||||
|
for ref in opf.iter(f"{OPF}itemref")
|
||||||
|
]
|
||||||
|
return [z.read(file).decode("utf-8", "replace") for file in files]
|
||||||
|
except (zipfile.BadZipFile, KeyError, ElementTree.ParseError) as e:
|
||||||
|
raise EpubError(f"{path.name} is not a readable epub: {e}") from e
|
||||||
|
|
||||||
|
|
||||||
def epub_urls(path: Path) -> list[str]:
|
def epub_urls(path: Path) -> list[str]:
|
||||||
"""Every URL in the identifiers and the source of an epub's OPF.
|
"""Every URL in the identifiers and the source of an epub's OPF.
|
||||||
|
|
||||||
|
|||||||
@@ -77,6 +77,7 @@ def _languages(value: Any) -> list[str]:
|
|||||||
def _books() -> list[dict[str, Any]]:
|
def _books() -> list[dict[str, Any]]:
|
||||||
"""The books of the catalog, in title order."""
|
"""The books of the catalog, in title order."""
|
||||||
books = []
|
books = []
|
||||||
|
page_counts = orchestrator.store.story_settings("page_count")
|
||||||
for story in orchestrator.store.catalog():
|
for story in orchestrator.store.catalog():
|
||||||
meta = orchestrator.book_metadata(story)
|
meta = orchestrator.book_metadata(story)
|
||||||
title = (meta["title"] or "").strip() or story["url"]
|
title = (meta["title"] or "").strip() or story["url"]
|
||||||
@@ -89,6 +90,7 @@ def _books() -> list[dict[str, Any]]:
|
|||||||
"title": title,
|
"title": title,
|
||||||
"authors": authors,
|
"authors": authors,
|
||||||
"updated": _updated(story),
|
"updated": _updated(story),
|
||||||
|
"page_count": page_counts.get(story["id"]),
|
||||||
"cover": hashlib.sha256(cover.encode()).hexdigest()[:16] if cover else None,
|
"cover": hashlib.sha256(cover.encode()).hexdigest()[:16] if cover else None,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
@@ -107,6 +109,8 @@ def _content(book: dict[str, Any]) -> str:
|
|||||||
index = filename_template.series_index(meta.get("series_index"))
|
index = filename_template.series_index(meta.get("series_index"))
|
||||||
lines.append(f"Series: {meta['series']}" + (f", book {index}" if index else ""))
|
lines.append(f"Series: {meta['series']}" + (f", book {index}" if index else ""))
|
||||||
lines.append(f"Chapters: {book['chapter_count']}")
|
lines.append(f"Chapters: {book['chapter_count']}")
|
||||||
|
if book["page_count"] is not None:
|
||||||
|
lines.append(f"Pages: {book['page_count']}")
|
||||||
description = "\n\n".join(diffing.html_to_blocks(meta.get("comments") or ""))
|
description = "\n\n".join(diffing.html_to_blocks(meta.get("comments") or ""))
|
||||||
return "\n".join(lines) + (f"\n\n{description}" if description else "")
|
return "\n".join(lines) + (f"\n\n{description}" if description else "")
|
||||||
|
|
||||||
|
|||||||
+16
-2
@@ -13,7 +13,7 @@ from typing import Any, BinaryIO
|
|||||||
|
|
||||||
from werkzeug.security import generate_password_hash
|
from werkzeug.security import generate_password_hash
|
||||||
|
|
||||||
from bindery import config, fff, filename_template
|
from bindery import config, fff, filename_template, pages
|
||||||
from bindery.backup import daily_backup
|
from bindery.backup import daily_backup
|
||||||
from bindery.cwa import Book, BookFinder, CwaClient, CwaError
|
from bindery.cwa import Book, BookFinder, CwaClient, CwaError
|
||||||
from bindery.diffing import diff_html, text_hash
|
from bindery.diffing import diff_html, text_hash
|
||||||
@@ -226,9 +226,11 @@ def list_stories() -> list[dict[str, Any]]:
|
|||||||
stories = store.list_stories()
|
stories = store.list_stories()
|
||||||
held = store.story_settings("upload_held")
|
held = store.story_settings("upload_held")
|
||||||
unapproved = store.story_settings("unapproved_link")
|
unapproved = store.story_settings("unapproved_link")
|
||||||
|
page_counts = store.story_settings("page_count")
|
||||||
show_holds = _link_books() and store.get_setting(None, "hold_new_links") and _can_upload()
|
show_holds = _link_books() and store.get_setting(None, "hold_new_links") and _can_upload()
|
||||||
for s in stories:
|
for s in stories:
|
||||||
s["busy"] = job_queue.has_active(s["id"])
|
s["busy"] = job_queue.has_active(s["id"])
|
||||||
|
s["page_count"] = page_counts.get(s["id"])
|
||||||
s["upload_held"] = bool(held.get(s["id"]))
|
s["upload_held"] = bool(held.get(s["id"]))
|
||||||
s["approval_held"] = bool(
|
s["approval_held"] = bool(
|
||||||
show_holds
|
show_holds
|
||||||
@@ -837,6 +839,14 @@ def _build_key(story: dict[str, Any], overrides: dict[str, Any], site: dict[str,
|
|||||||
return digest.hexdigest()
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _count_pages(story_id: int, path: Path) -> None:
|
||||||
|
"""Store the estimated pages of a built epub, for the story list and the OPDS catalog."""
|
||||||
|
try:
|
||||||
|
store.set_setting(story_id, "page_count", pages.epub_pages(path))
|
||||||
|
except EpubError as e:
|
||||||
|
logger.warning(f"Cannot count the pages of story {story_id}: {e}")
|
||||||
|
|
||||||
|
|
||||||
def _built_epub(
|
def _built_epub(
|
||||||
story: dict[str, Any], overrides: dict[str, Any], key: str, force: bool = False, record: bool = True
|
story: dict[str, Any], overrides: dict[str, Any], key: str, force: bool = False, record: bool = True
|
||||||
) -> Path:
|
) -> Path:
|
||||||
@@ -846,7 +856,8 @@ def _built_epub(
|
|||||||
"""
|
"""
|
||||||
path = config.TMP_DIR / f"story-{story['id']}-{key[:16]}.epub"
|
path = config.TMP_DIR / f"story-{story['id']}-{key[:16]}.epub"
|
||||||
with _build_lock:
|
with _build_lock:
|
||||||
if force or not path.exists():
|
built = force or not path.exists()
|
||||||
|
if built:
|
||||||
# A build under another name first, so a download never reads half a book.
|
# A build under another name first, so a download never reads half a book.
|
||||||
partial = config.TMP_DIR / f"building-{uuid.uuid4().hex}.epub"
|
partial = config.TMP_DIR / f"building-{uuid.uuid4().hex}.epub"
|
||||||
try:
|
try:
|
||||||
@@ -857,6 +868,9 @@ def _built_epub(
|
|||||||
for old in config.TMP_DIR.glob(f"story-{story['id']}-*.epub"):
|
for old in config.TMP_DIR.glob(f"story-{story['id']}-*.epub"):
|
||||||
if old != path:
|
if old != path:
|
||||||
old.unlink(missing_ok=True)
|
old.unlink(missing_ok=True)
|
||||||
|
# A book from before the page count has no count yet.
|
||||||
|
if built or store.get_setting(story["id"], "page_count") is None:
|
||||||
|
_count_pages(story["id"], path)
|
||||||
if record:
|
if record:
|
||||||
store.set_setting(story["id"], "built_key", key)
|
store.set_setting(story["id"], "built_key", key)
|
||||||
return path
|
return path
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
"""An estimate of the pages of a built epub, after the "Paragraphs (APNX Accurate)" method of the Count Pages
|
||||||
|
plugin for calibre.
|
||||||
|
|
||||||
|
The method counts lines. A paragraph start is one line, and so is each 70 characters of text. A page is 31
|
||||||
|
lines. A page is also at most 2400 characters, which sets the count of a book with almost no paragraphs.
|
||||||
|
|
||||||
|
The plugin counts either the <p> or the <div> elements of a book as its paragraphs. Bindery counts what a
|
||||||
|
reader sees as a paragraph: text that follows a <br> or the start or end of a block element. So the same text
|
||||||
|
gives the same count as <p> elements, as <div> elements, and as lines with <br> between them. A paragraph with
|
||||||
|
no text, or with only a no-break space, is not counted.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import html as htmllib
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import NamedTuple
|
||||||
|
|
||||||
|
from bindery.epub import spine_documents
|
||||||
|
|
||||||
|
LINE_CHARS = 70
|
||||||
|
LINES_PER_PAGE = 31
|
||||||
|
CHARS_PER_PAGE = 2400
|
||||||
|
# Elements that stay in the line of the text around them. Each other element, and <br>, ends a paragraph.
|
||||||
|
# The cells of a table row share a line.
|
||||||
|
INLINE = frozenset(
|
||||||
|
"a abbr acronym b bdi bdo big cite code data del dfn em font i img ins kbd mark q rp rt ruby s samp small "
|
||||||
|
"span strike strong sub sup td th time tt u var wbr".split()
|
||||||
|
)
|
||||||
|
# Elements whose text is not text of the book.
|
||||||
|
HIDDEN = frozenset(("head", "script", "style"))
|
||||||
|
_TOKEN = re.compile(r"<(/?)([a-zA-Z][a-zA-Z0-9]*)[^>]*>|<[^>]*>|([^<]+)")
|
||||||
|
|
||||||
|
|
||||||
|
class Measure(NamedTuple):
|
||||||
|
paragraphs: int = 0
|
||||||
|
chars: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
def measure(html: str) -> Measure:
|
||||||
|
"""The paragraphs and the characters of text of one document."""
|
||||||
|
paragraphs = chars = 0
|
||||||
|
hidden = False
|
||||||
|
new_paragraph = True
|
||||||
|
for match in _TOKEN.finditer(html):
|
||||||
|
closing, name, text = match.groups()
|
||||||
|
if name:
|
||||||
|
name = name.lower()
|
||||||
|
if name in HIDDEN:
|
||||||
|
hidden = not closing and not match.group().endswith("/>")
|
||||||
|
elif name not in INLINE:
|
||||||
|
new_paragraph = True
|
||||||
|
elif text and not hidden:
|
||||||
|
if "&" in text:
|
||||||
|
text = htmllib.unescape(text)
|
||||||
|
# A no-break space is white space here, so a paragraph that only spaces the text is no paragraph.
|
||||||
|
if text.strip():
|
||||||
|
if new_paragraph:
|
||||||
|
paragraphs += 1
|
||||||
|
new_paragraph = False
|
||||||
|
text = text.lstrip()
|
||||||
|
chars += len(text)
|
||||||
|
elif not new_paragraph:
|
||||||
|
chars += 1 # the space between two inline elements
|
||||||
|
return Measure(paragraphs, chars)
|
||||||
|
|
||||||
|
|
||||||
|
def page_count(total: Measure) -> int:
|
||||||
|
"""The pages of a book, from the sums of its documents. A book with no text has no pages."""
|
||||||
|
if not total.chars:
|
||||||
|
return 0
|
||||||
|
lines = total.paragraphs + total.chars // LINE_CHARS
|
||||||
|
return max(lines // LINES_PER_PAGE, total.chars // CHARS_PER_PAGE + 1)
|
||||||
|
|
||||||
|
|
||||||
|
def epub_pages(path: Path) -> int:
|
||||||
|
"""The estimated pages of an epub, from each document in its spine."""
|
||||||
|
measures = [measure(text) for text in spine_documents(path)]
|
||||||
|
return page_count(Measure(sum(m.paragraphs for m in measures), sum(m.chars for m in measures)))
|
||||||
@@ -9,6 +9,7 @@ const STORY_COLUMNS = [
|
|||||||
{ label: 'Author', key: 'author', type: 'text' },
|
{ label: 'Author', key: 'author', type: 'text' },
|
||||||
{ label: 'Site', key: 'site', type: 'text' },
|
{ label: 'Site', key: 'site', type: 'text' },
|
||||||
{ label: 'Chapters', key: 'chapter_count', type: 'number' },
|
{ label: 'Chapters', key: 'chapter_count', type: 'number' },
|
||||||
|
{ label: 'Pages', key: 'page_count', type: 'number' },
|
||||||
{ label: 'Status', key: 'status', type: 'text' },
|
{ label: 'Status', key: 'status', type: 'text' },
|
||||||
{ label: 'Last change', key: 'last_changed', type: 'date' },
|
{ label: 'Last change', key: 'last_changed', type: 'date' },
|
||||||
{ label: 'Last check', key: 'last_checked', type: 'date' },
|
{ label: 'Last check', key: 'last_checked', type: 'date' },
|
||||||
@@ -231,6 +232,8 @@ function storyRow(s) {
|
|||||||
s.offsite_count > 0 && h('small', { title: 'Chapters the site no longer lists' }, `${s.offsite_count} removed`),
|
s.offsite_count > 0 && h('small', { title: 'Chapters the site no longer lists' }, `${s.offsite_count} removed`),
|
||||||
s.revised_count > 0 && h('small', { title: 'Chapters the site changed since Bindery added them' }, `${s.revised_count} revised`),
|
s.revised_count > 0 && h('small', { title: 'Chapters the site changed since Bindery added them' }, `${s.revised_count} revised`),
|
||||||
),
|
),
|
||||||
|
// A story has no page count until Bindery builds its book.
|
||||||
|
h('td', { 'data-label': 'Pages', title: 'Estimated from the built book' }, s.page_count != null && String(s.page_count)),
|
||||||
h(
|
h(
|
||||||
'td',
|
'td',
|
||||||
{ 'data-label': 'Status' },
|
{ 'data-label': 'Status' },
|
||||||
@@ -276,6 +279,8 @@ export function storiesPage(main, _params, config) {
|
|||||||
const order = (left - right) * direction;
|
const order = (left - right) * direction;
|
||||||
if (order) return order;
|
if (order) return order;
|
||||||
} else if (sort !== 'title') {
|
} else if (sort !== 'title') {
|
||||||
|
// A story with no number goes last in each direction.
|
||||||
|
if (sortType === 'number' && (a[sort] == null) !== (b[sort] == null)) return a[sort] == null ? 1 : -1;
|
||||||
const order = sortType === 'number'
|
const order = sortType === 'number'
|
||||||
? a[sort] - b[sort]
|
? a[sort] - b[sort]
|
||||||
: (a[sort] || '').localeCompare(b[sort] || '', undefined, { sensitivity: 'base' });
|
: (a[sort] || '').localeCompare(b[sort] || '', undefined, { sensitivity: 'base' });
|
||||||
|
|||||||
@@ -940,6 +940,11 @@ textarea {
|
|||||||
color: var(--muted);
|
color: var(--muted);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* A story with no page count yet shows no line for it. */
|
||||||
|
.stories-table td:empty {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
/* The chapter list has far more rows than the story list, so each row packs its cells onto
|
/* The chapter list has far more rows than the story list, so each row packs its cells onto
|
||||||
three lines instead of one line per cell: checkbox and number, title, then the rest. */
|
three lines instead of one line per cell: checkbox and number, title, then the rest. */
|
||||||
.chapters-table tbody tr {
|
.chapters-table tbody tr {
|
||||||
|
|||||||
@@ -380,7 +380,7 @@ test('story list stacks on a narrow screen instead of scrolling sideways', async
|
|||||||
const wrap = nodes(main).find((el) => el.className === 'table-wrap stories-table');
|
const wrap = nodes(main).find((el) => el.className === 'table-wrap stories-table');
|
||||||
assert.ok(wrap);
|
assert.ok(wrap);
|
||||||
const dataLabels = nodes(wrap).filter((el) => el.tagName === 'td').map((el) => el.attributes['data-label']);
|
const dataLabels = nodes(wrap).filter((el) => el.tagName === 'td').map((el) => el.attributes['data-label']);
|
||||||
assert.deepEqual(dataLabels, [undefined, 'Author', 'Site', 'Chapters', 'Status', 'Last change', 'Last check']);
|
assert.deepEqual(dataLabels, [undefined, 'Author', 'Site', 'Chapters', 'Pages', 'Status', 'Last change', 'Last check']);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('the story list shows removed and revised chapter counts, only when there are any', async () => {
|
test('the story list shows removed and revised chapter counts, only when there are any', async () => {
|
||||||
|
|||||||
@@ -114,6 +114,12 @@ test('sortable headings show direction and remain usable above story cards on na
|
|||||||
assert.doesNotMatch(css, /\.stories-table thead\s*\{[^}]*clip-path/);
|
assert.doesNotMatch(css, /\.stories-table thead\s*\{[^}]*clip-path/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('on narrow screens a story cell with no value shows no line', () => {
|
||||||
|
const start = css.indexOf('@media (max-width: 600px)', css.indexOf('.stories-table th button'));
|
||||||
|
assert.equal(declarations('.stories-table td:empty', css.slice(start)).display, 'none');
|
||||||
|
assert.doesNotMatch(css.slice(0, start), /\.stories-table td:empty/);
|
||||||
|
});
|
||||||
|
|
||||||
test('on narrow screens the headings are sort chips, and only the sorted chip is tinted with an arrow', () => {
|
test('on narrow screens the headings are sort chips, and only the sorted chip is tinted with an arrow', () => {
|
||||||
const start = css.indexOf('@media (max-width: 600px)', css.indexOf('.stories-table th button'));
|
const start = css.indexOf('@media (max-width: 600px)', css.indexOf('.stories-table th button'));
|
||||||
const mobile = css.slice(start);
|
const mobile = css.slice(start);
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ const story = (id, title, changes = {}) => ({
|
|||||||
id, title, url: `https://example.com/${id}`,
|
id, title, url: `https://example.com/${id}`,
|
||||||
sort_title: (title || `https://example.com/${id}`).trim().replace(/^(?:the|an?)\s+/i, '').toLowerCase(),
|
sort_title: (title || `https://example.com/${id}`).trim().replace(/^(?:the|an?)\s+/i, '').toLowerCase(),
|
||||||
author: 'Reader', site: 'example.com',
|
author: 'Reader', site: 'example.com',
|
||||||
chapter_count: 42, offsite_count: 0, revised_count: 0, status: 'ok', message: 'No changes',
|
chapter_count: 42, page_count: 120, offsite_count: 0, revised_count: 0, status: 'ok', message: 'No changes',
|
||||||
last_changed: null, last_checked: null, ...changes,
|
last_changed: null, last_checked: null, ...changes,
|
||||||
});
|
});
|
||||||
const rows = () => {
|
const rows = () => {
|
||||||
@@ -106,6 +106,39 @@ test('chapter sorting compares counts numerically and keeps equal counts in titl
|
|||||||
assert.equal(heading('Chapters').attributes['aria-sort'], 'ascending');
|
assert.equal(heading('Chapters').attributes['aria-sort'], 'ascending');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('pages show beside the chapters and sort numerically', async (t) => {
|
||||||
|
await mount(t, [
|
||||||
|
story(1, 'Zeta', { page_count: 9 }),
|
||||||
|
story(2, 'Alpha', { page_count: 1200 }),
|
||||||
|
story(3, 'Beta', { page_count: 0 }),
|
||||||
|
story(4, 'Gamma', { page_count: 9 }),
|
||||||
|
]);
|
||||||
|
const cells = nodes(rows()[0]).filter((el) => el.tagName === 'td').map((el) => el.attributes['data-label']);
|
||||||
|
assert.deepEqual(cells.slice(3, 5), ['Chapters', 'Pages']);
|
||||||
|
assert.equal(find(app, 'button', 'Pages').attributes['aria-label'], 'Sort by pages, most first');
|
||||||
|
await click(app, 'Pages');
|
||||||
|
assert.deepEqual(titles(), ['Alpha', 'Gamma', 'Zeta', 'Beta']);
|
||||||
|
await click(app, 'Pages');
|
||||||
|
assert.deepEqual(titles(), ['Beta', 'Gamma', 'Zeta', 'Alpha']);
|
||||||
|
assert.deepEqual(JSON.parse(window.localStorage.getItem(SORT_KEY)), { column: 'page_count', direction: 1 });
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a story with no built book shows no page count and sorts last in each direction', async (t) => {
|
||||||
|
await mount(t, [
|
||||||
|
story(1, 'Alpha', { page_count: null }),
|
||||||
|
story(2, 'Beta', { page_count: 300 }),
|
||||||
|
story(3, 'Gamma', { page_count: 0 }),
|
||||||
|
story(4, 'Delta', { page_count: null }),
|
||||||
|
]);
|
||||||
|
const pages = () => rows().map((row) => cell(row, 'Pages'));
|
||||||
|
assert.deepEqual(pages().map((el) => el.textContent), ['', '300', '', '0']);
|
||||||
|
assert.deepEqual(pages().map((el) => el.children.length), [0, 1, 0, 1], 'An empty cell hides on a narrow screen');
|
||||||
|
await click(app, 'Pages');
|
||||||
|
assert.deepEqual(titles(), ['Beta', 'Gamma', 'Alpha', 'Delta']);
|
||||||
|
await click(app, 'Pages');
|
||||||
|
assert.deepEqual(titles(), ['Gamma', 'Beta', 'Alpha', 'Delta']);
|
||||||
|
});
|
||||||
|
|
||||||
for (const [label, key] of [['Author', 'author'], ['Site', 'site'], ['Status', 'status']]) {
|
for (const [label, key] of [['Author', 'author'], ['Site', 'site'], ['Status', 'status']]) {
|
||||||
test(`${label} sorts alphabetically in either direction and uses titles to break ties`, async (t) => {
|
test(`${label} sorts alphabetically in either direction and uses titles to break ties`, async (t) => {
|
||||||
await mount(t, [
|
await mount(t, [
|
||||||
@@ -248,7 +281,7 @@ test('date columns keep each timestamp on one line, with last change before last
|
|||||||
const checked = '2026-01-03T05:06:00Z';
|
const checked = '2026-01-03T05:06:00Z';
|
||||||
await mount(t, [story(1, 'Alpha', { last_changed: changed, last_checked: checked })]);
|
await mount(t, [story(1, 'Alpha', { last_changed: changed, last_checked: checked })]);
|
||||||
const headings = nodes(app).filter((el) => el.tagName === 'th').map((el) => el.textContent);
|
const headings = nodes(app).filter((el) => el.tagName === 'th').map((el) => el.textContent);
|
||||||
assert.deepEqual(headings, ['Title', 'Author', 'Site', 'Chapters', 'Status', 'Last change', 'Last check']);
|
assert.deepEqual(headings, ['Title', 'Author', 'Site', 'Chapters', 'Pages', 'Status', 'Last change', 'Last check']);
|
||||||
for (const [label, value] of [['Last change', changed], ['Last check', checked]]) {
|
for (const [label, value] of [['Last change', changed], ['Last check', checked]]) {
|
||||||
const dateCell = cell(rows()[0], label);
|
const dateCell = cell(rows()[0], label);
|
||||||
const time = nodes(dateCell).find((el) => el.tagName === 'time');
|
const time = nodes(dateCell).find((el) => el.tagName === 'time');
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from test_deliver import fake as fake
|
|||||||
from test_deliver import jobs as jobs
|
from test_deliver import jobs as jobs
|
||||||
from test_deliver import story_epub as story_epub
|
from test_deliver import story_epub as story_epub
|
||||||
|
|
||||||
from bindery import config, orchestrator
|
from bindery import config, orchestrator, pages
|
||||||
from bindery.jobs import DownloadMode, Job
|
from bindery.jobs import DownloadMode, Job
|
||||||
|
|
||||||
ONE = [(1, "One", "a")]
|
ONE = [(1, "One", "a")]
|
||||||
@@ -242,6 +242,59 @@ def test_a_download_builds_a_book_that_has_no_file_or_that_changed(builds, clien
|
|||||||
assert story(story_id)["last_changed"]
|
assert story(story_id)["last_changed"]
|
||||||
|
|
||||||
|
|
||||||
|
# Page counts
|
||||||
|
|
||||||
|
LONG = "A line of the story. " * 20
|
||||||
|
|
||||||
|
|
||||||
|
def listed_pages(story_id):
|
||||||
|
return {s["id"]: s["page_count"] for s in orchestrator.list_stories()}[story_id]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_story_has_no_page_count_until_its_book_is_built(builds, monkeypatch):
|
||||||
|
story_id = linked_story(ONE)
|
||||||
|
assert listed_pages(story_id) is None
|
||||||
|
site(monkeypatch, ONE)
|
||||||
|
update(story_id)
|
||||||
|
(built,) = config.TMP_DIR.iterdir()
|
||||||
|
assert listed_pages(story_id) == pages.epub_pages(built) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_each_build_counts_the_pages_of_the_new_book(builds, client, monkeypatch):
|
||||||
|
story_id = linked_story(ONE)
|
||||||
|
site(monkeypatch, [*ONE, *[(n, f"Chapter {n}", "<br/>".join([LONG] * 31)) for n in range(2, 6)]])
|
||||||
|
update(story_id)
|
||||||
|
long_book = listed_pages(story_id)
|
||||||
|
# Each of the 124 lines is a paragraph and 6 lines of text.
|
||||||
|
assert long_book >= 4 * 31 * 7 // 31
|
||||||
|
# The count is of the built book, so a rule that removes text changes it without a download.
|
||||||
|
orchestrator.save_rules(story_id, [{"pattern": "(A line of the story. ){19}", "replacement": ""}])
|
||||||
|
download(client, story_id)
|
||||||
|
assert builds.call_count == 2
|
||||||
|
assert 0 < listed_pages(story_id) < long_book / 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_book_from_before_the_page_count_is_counted_without_a_build(builds, client, monkeypatch):
|
||||||
|
story_id = linked_story(ONE)
|
||||||
|
site(monkeypatch, ONE)
|
||||||
|
update(story_id)
|
||||||
|
orchestrator.store.set_setting(story_id, "page_count", None)
|
||||||
|
download(client, story_id)
|
||||||
|
assert builds.call_count == 1
|
||||||
|
assert listed_pages(story_id) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_book_that_cannot_be_counted_is_still_built(builds, monkeypatch):
|
||||||
|
def fail(path):
|
||||||
|
raise orchestrator.EpubError("unreadable")
|
||||||
|
|
||||||
|
monkeypatch.setattr(pages, "epub_pages", fail)
|
||||||
|
story_id = linked_story(ONE)
|
||||||
|
site(monkeypatch, ONE)
|
||||||
|
assert "library book 1" in update(story_id)
|
||||||
|
assert listed_pages(story_id) is None
|
||||||
|
|
||||||
|
|
||||||
def test_stop_tracking_deletes_the_built_book(builds, client):
|
def test_stop_tracking_deletes_the_built_book(builds, client):
|
||||||
story_id, other = add_story(), orchestrator.store.create_story("https://other", "s", "Other", "W", [])
|
story_id, other = add_story(), orchestrator.store.create_story("https://other", "s", "Other", "W", [])
|
||||||
download(client, story_id)
|
download(client, story_id)
|
||||||
|
|||||||
@@ -205,6 +205,17 @@ def test_entry_shows_the_book_metadata_and_cover(client):
|
|||||||
assert client.get(f"/opds/books/{add_book()}/cover").status_code == 404
|
assert client.get(f"/opds/books/{add_book()}/cover").status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
def test_entry_shows_the_page_count_of_a_built_book(client):
|
||||||
|
sid = add_book()
|
||||||
|
orchestrator.store.set_setting(sid, "page_count", 321)
|
||||||
|
add_book(title="Not built")
|
||||||
|
contents = {
|
||||||
|
e.findtext("a:title", namespaces=NS): e.findtext("a:content", namespaces=NS)
|
||||||
|
for e in entries(feed(client.get("/opds/title")))
|
||||||
|
}
|
||||||
|
assert contents == {"A book": "Chapters: 1\nPages: 321", "Not built": "Chapters: 1"}
|
||||||
|
|
||||||
|
|
||||||
def test_cover_of_an_unsafe_type_is_left_out(client):
|
def test_cover_of_an_unsafe_type_is_left_out(client):
|
||||||
sid = add_book()
|
sid = add_book()
|
||||||
orchestrator.store.put_image(sid, "https://example.com/c.svg", "svg", "image/svg+xml", b"<svg/>", is_cover=True)
|
orchestrator.store.put_image(sid, "https://example.com/c.svg", "svg", "image/svg+xml", b"<svg/>", is_cover=True)
|
||||||
|
|||||||
@@ -0,0 +1,122 @@
|
|||||||
|
"""The page estimate of a document and of a built epub."""
|
||||||
|
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from bindery import pages
|
||||||
|
from bindery.epub import EpubError, spine_documents
|
||||||
|
from bindery.pages import Measure
|
||||||
|
|
||||||
|
TEXT = ["One line of the story.", "Another, longer line of the story that goes on for a while.", "The end."]
|
||||||
|
CHARS = sum(map(len, TEXT))
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_paragraph_is_text_after_a_block_or_a_line_break():
|
||||||
|
assert pages.measure("<p>One</p><p>Two</p>") == Measure(2, 6)
|
||||||
|
assert pages.measure("<div>One<br/><br/>Two<br>Three</div>") == Measure(3, 11)
|
||||||
|
assert pages.measure("<div><div>One</div><div>Two</div></div>") == Measure(2, 6)
|
||||||
|
# Text before, between and after the blocks of an element counts too.
|
||||||
|
assert pages.measure("<div>Intro<p>One</p>Middle<p>Two</p>End</div>") == Measure(5, 20)
|
||||||
|
assert pages.measure("<h3>Title</h3><ul><li>One</li><li>Two</li></ul><blockquote>Three</blockquote>") == Measure(
|
||||||
|
4, 16
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"book",
|
||||||
|
[
|
||||||
|
"".join(f"<p>{line}</p>" for line in TEXT),
|
||||||
|
"".join(f'<p class="a">{line}</p>\n' for line in TEXT),
|
||||||
|
"<div>" + "".join(f"<div>{line}</div>" for line in TEXT) + "</div>",
|
||||||
|
"<div>" + "<br/><br/>".join(TEXT) + "</div>",
|
||||||
|
"<div>" + "<br>\n".join(TEXT) + "</div>",
|
||||||
|
f"<div>{TEXT[0]}<p>{TEXT[1]}</p>{TEXT[2]}</div>",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_the_same_text_gives_the_same_measure_in_each_markup(book):
|
||||||
|
assert pages.measure(book) == Measure(3, CHARS)
|
||||||
|
|
||||||
|
|
||||||
|
def test_inline_elements_and_table_cells_stay_in_the_line():
|
||||||
|
assert pages.measure('<p>ab<b>cd</b><a href="x">ef</a> <span><i>gh</i></span></p>') == Measure(1, 9)
|
||||||
|
assert pages.measure("<table><tr><td>Name</td><td>Value</td></tr><tr><th>a</th><td>b</td></tr></table>") == Measure(
|
||||||
|
2, 11
|
||||||
|
)
|
||||||
|
assert pages.measure('<p>Before<img src="a.png"/>after</p>') == Measure(1, 11)
|
||||||
|
|
||||||
|
|
||||||
|
def test_only_the_text_of_the_book_counts():
|
||||||
|
document = (
|
||||||
|
'<?xml version="1.0"?><html><head><title>Hidden title</title><style>p { color: red }</style>'
|
||||||
|
'<link href="a.css"/></head><body><script>var hidden;</script><script src="x.js"/>'
|
||||||
|
"<!-- a note --><p>Shown & counted</p>\n\n<p> </p><p></p></body></html>"
|
||||||
|
)
|
||||||
|
assert pages.measure(document) == Measure(1, len("Shown & counted"))
|
||||||
|
assert pages.measure("") == Measure(0, 0)
|
||||||
|
assert pages.measure("<p>\n</p>") == Measure(0, 0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_paragraph_that_only_spaces_the_text_is_not_counted():
|
||||||
|
spaced = "<p>One</p><p> </p><p>\xa0</p><p>   </p><p><br/></p><p><span> </span></p><p>Two</p>"
|
||||||
|
assert pages.measure(spaced) == pages.measure("<p>One</p><p>Two</p>") == Measure(2, 6)
|
||||||
|
assert pages.measure("<div>One<br/> <br/>Two</div>") == Measure(2, 6)
|
||||||
|
# A no-break space inside a paragraph is a character of its text.
|
||||||
|
assert pages.measure("<p>One <b>two</b>\xa0three</p>") == Measure(1, 13)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_page_is_31_lines_and_a_line_is_a_paragraph_or_70_characters():
|
||||||
|
assert pages.page_count(Measure(62, 620)) == (62 + 620 // 70) // 31 == 2
|
||||||
|
assert pages.page_count(Measure(31 * 9, 31 * 9)) == 9
|
||||||
|
assert pages.page_count(Measure(1000, 177_000)) == (1000 + 177_000 // 70) // 31
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_short_book_has_one_page_and_a_book_with_no_text_has_none():
|
||||||
|
assert pages.page_count(Measure(1, 10)) == 1
|
||||||
|
assert pages.page_count(Measure(30, 30)) == 1
|
||||||
|
assert pages.page_count(Measure()) == 0
|
||||||
|
# One block of text: 2400 characters start the second page before 31 lines do.
|
||||||
|
assert pages.page_count(Measure(1, 2399)) == 1
|
||||||
|
assert pages.page_count(Measure(1, 2400)) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def epub(path, documents, spine=None):
|
||||||
|
names = [f"file {n}.xhtml" for n in range(len(documents))]
|
||||||
|
items = "".join(f'<item id="f{n}" href="{name.replace(" ", "%20")}"/>' for n, name in enumerate(names))
|
||||||
|
items += '<item id="css" href="stylesheet.css"/>'
|
||||||
|
refs = "".join(f'<itemref idref="f{n}"/>' for n in (spine or range(len(documents))))
|
||||||
|
with zipfile.ZipFile(path, "w") as z:
|
||||||
|
z.writestr(
|
||||||
|
"META-INF/container.xml",
|
||||||
|
'<container xmlns="urn:oasis:names:tc:opendocument:xmlns:container"><rootfiles>'
|
||||||
|
'<rootfile full-path="OEBPS/content.opf"/></rootfiles></container>',
|
||||||
|
)
|
||||||
|
z.writestr(
|
||||||
|
"OEBPS/content.opf",
|
||||||
|
f'<package xmlns="http://www.idpf.org/2007/opf"><metadata/><manifest>{items}</manifest>'
|
||||||
|
f"<spine>{refs}</spine></package>",
|
||||||
|
)
|
||||||
|
z.writestr("OEBPS/stylesheet.css", "p { margin: 0 }")
|
||||||
|
for name, document in zip(names, documents, strict=True):
|
||||||
|
z.writestr(f"OEBPS/{name}", document)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_epub_is_counted_from_the_documents_of_its_spine(tmp_path):
|
||||||
|
chapter = "<html><head><title>T</title></head><body>" + "<p>x</p>" * 31 + "</body></html>"
|
||||||
|
book = epub(tmp_path / "book.epub", [chapter, "<html><body><p>Not in the spine</p></body></html>", chapter], [2, 0])
|
||||||
|
assert spine_documents(book) == [chapter, chapter]
|
||||||
|
# 62 paragraphs of one character.
|
||||||
|
assert pages.epub_pages(book) == 2
|
||||||
|
assert pages.epub_pages(epub(tmp_path / "empty.epub", ["<html><body><img src='a.png'/></body></html>"])) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_file_that_is_not_an_epub_cannot_be_counted(tmp_path):
|
||||||
|
path = tmp_path / "bad.epub"
|
||||||
|
path.write_bytes(b"not a zip")
|
||||||
|
with pytest.raises(EpubError):
|
||||||
|
pages.epub_pages(path)
|
||||||
|
with zipfile.ZipFile(path, "w") as z:
|
||||||
|
z.writestr("META-INF/container.xml", "<container/>")
|
||||||
|
with pytest.raises(EpubError):
|
||||||
|
pages.epub_pages(path)
|
||||||
Reference in New Issue
Block a user