Add Calibre's file name templates

The module prepares the metadata of a book and expands a template as Calibre 9.2.1 does with
its default tweaks. It takes fields, formats, indexes, and text before and after a field. It
refuses template functions and the program modes.
This commit is contained in:
tim
2026-10-03 08:09:41 +02:00
parent ccd3b18791
commit 9e7a488b50
3 changed files with 517 additions and 0 deletions
+1
View File
@@ -28,6 +28,7 @@ The server has no reloader. If you change Python code, restart it. Flask reads `
- `bindery/fff.py` is the only module that calls FanFicFare.
- `bindery/cwa.py` talks to CWA through OPDS. Bindery never opens the CWA database.
- `bindery/diffing.py` hashes and diffs text, not HTML.
- `bindery/filename_template.py` expands Calibre's file name templates. It never runs template text as code.
- `bindery/jobs.py` keeps the job queue in SQLite, so jobs survive a restart.
- `tests/fake_cwa.py` is a fake CWA server.
+301
View File
@@ -0,0 +1,301 @@
"""Calibre's templates for file names, for the name of a downloaded book.
Bindery follows Calibre 9.2.1 with its default tweaks: titles and series in library order with the English
articles, and dates in the form "%b, %Y". It takes {field}, {field[0]}, {field:format|prefix|suffix} and {{ }}.
It refuses template functions and the program modes. It never runs template text as code.
The tags sort close to Calibre's ICU rules, but their order can differ for characters outside ASCII.
"""
import re
import unicodedata
from datetime import datetime
from string import Formatter
from typing import Any
from bindery.ingest import sanitize_filename
MAX_TEMPLATE_LENGTH = 1024
# The most characters that a format can ask for, as a width or a precision.
MAX_WIDTH = 255
MAX_NAME_BYTES = 200
# The fields that Bindery has, with the help text for each. Any other field is empty, as it is in Calibre for
# a book without that value.
FIELDS = {
"title": "The title",
"title_sort": "The title in sort order",
"authors": "The authors",
"author": "The authors",
"author_sort": "The author sort string. To use only the first letter of the name, use {author_sort[0]}",
"series": "The series",
"series_index": "The series number. To get leading zeros, use {series_index:0>3s}",
"tags": "The tags",
"languages": "The language",
"publisher": "The story site",
"pubdate": "The published date",
"timestamp": "The date when Bindery added the story",
"last_modified": "The date when the chapters last changed",
"id": "The Bindery story ID",
}
class TemplateError(ValueError):
"""A template that Bindery cannot use, or that fails for a book."""
# Metadata, as Calibre prepares it for a file name
_ARTICLE = re.compile(r"^(A\s+|The\s+|An\s+)", re.IGNORECASE)
# Calibre's quote_pairs: each opening quote, with the quotes that close it.
_QUOTE_PAIRS = {
'"': '"',
"'": "'",
"\u201c": "\u201d\u201c",
"\u201d": "\u201d",
"\u201e": "\u201d\u201c",
"\u201a": "\u2019\u2018",
"\u2019": "\u2019\u2018",
"\u2018": "\u2019\u2018",
"\u2039": "\u203a",
"\u203a": "\u2039",
"\u300a": "\u300b",
"\u3008": "\u3009",
"\u00bb": "\u00ab\u00bb",
"\u00ab": "\u00ab\u00bb",
"\u300c": "\u300d",
"\u300e": "\u300f",
}
# The ASCII punctuation in the order of Calibre's sort. Digits come next, then letters.
_PUNCTUATION = " _-,;:!?.'\"()[]{}@*/\\&#%`^+<=>|~$"
_MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")
# Calibre's default tweaks for author names.
_NAME_PREFIXES = {"mr", "mrs", "ms", "dr", "prof"}
_NAME_SUFFIXES = {"jr", "sr", "inc", "ph.d", "phd", "md", "m.d", "i", "ii", "iii", "iv", "junior", "senior"}
_COPY_WORDS = {
"agency",
"corporation",
"company",
"co.",
"council",
"committee",
"inc.",
"institute",
"national",
"society",
"club",
"team",
"software",
"games",
"entertainment",
"media",
"studios",
}
_BRACKETS = {"(": ")", "[": "]", "{": "}"}
def _unquote(title: str) -> str:
closers = _QUOTE_PAIRS.get(title[:1])
if closers:
title = title[1:]
if title and title[-1] in closers:
title = title[:-1]
return title
def title_sort(title: str) -> str:
"""Calibre's title_sort(): "The Book" gives "Book, The"."""
title = _unquote(title.strip())
match = _ARTICLE.search(title)
if match:
title = _unquote(f"{title[match.end() :]}, {match.group(1)}")
return title.strip()
def _without_brackets(text: str) -> str:
depth = dict.fromkeys(_BRACKETS, 0)
closers = {close: opener for opener, close in _BRACKETS.items()}
kept = []
for char in text:
if char in depth:
depth[char] += 1
elif char in closers:
depth[closers[char]] = max(0, depth[closers[char]] - 1)
elif not any(depth.values()):
kept.append(char)
return "".join(kept)
def author_sort(author: str) -> str:
"""Calibre's author_to_author_sort() with its default tweaks: "Ann Writer" gives "Writer, Ann"."""
plain = _without_brackets(author).strip()
tokens = plain.split()
lower = [t.lower().removesuffix(".") for t in tokens]
if "," in plain or len(tokens) < 2 or _COPY_WORDS.intersection(t.lower() for t in tokens):
return author
first = next((n for n, t in enumerate(lower) if t not in _NAME_PREFIXES), None)
if first is None:
return author
last = next((n for n in range(len(tokens) - 1, first - 1, -1) if lower[n] not in _NAME_SUFFIXES), None)
if last is None:
return author
names = [tokens[last], *tokens[first:last]]
if len(names) > 1:
names[0] += ","
return " ".join(names + tokens[last + 1 :])
def series_index(index: Any) -> str:
"""Calibre's fmt_sidx(): 2.0 gives "2", and 2.345 gives "2.35"."""
try:
index = float(index)
except (TypeError, ValueError):
index = 1.0
if index.is_integer():
return str(int(index))
return f"{index:.2f}".rstrip("0")
def _date(value: Any) -> str:
try:
date = datetime.fromisoformat(value)
except (TypeError, ValueError):
return ""
return f"{_MONTHS[date.month - 1]}, {date.year:04}"
def _tag_key(tag: str) -> tuple[list[tuple[int, str]], str]:
"""Close to the key of Calibre's sort: no case, punctuation before digits before letters, accents last."""
text = unicodedata.normalize("NFKD", tag.casefold())
ranks = len(_PUNCTUATION)
letters = [
(_PUNCTUATION.index(c), "") if c in _PUNCTUATION else (ranks + c.isalpha(), c)
for c in text
if not unicodedata.combining(c)
]
return letters, text
def prepare(meta: dict[str, Any], book_id: int, order: str = "library_order") -> dict[str, Any]:
"""Calibre's get_component_metadata(): the values of a book in the form that a file name takes.
`order` is Calibre's tweak save_template_title_series_sorting. With "strictly_alphabetic", the title and the
series keep a leading article.
"""
library = order == "library_order"
values: dict[str, Any] = {"id": str(book_id), "publisher": meta.get("publisher") or ""}
title, authors, series = meta.get("title"), [a for a in meta.get("authors") or [] if a], meta.get("series")
if title:
values["title_sort"] = title_sort(title)
values["title"] = values["title_sort"] if library else title
if authors:
values["authors"] = values["author"] = " & ".join(a.replace("&", "&&") for a in authors)
values["author_sort"] = " & ".join(author_sort(a) for a in authors)
if series:
values["series"] = title_sort(series) if library else series.strip()
if meta.get("series_index") is not None:
values["series_index"] = series_index(meta["series_index"])
values["tags"] = ", ".join(sorted(meta.get("tags") or [], key=_tag_key)).removeprefix("/")
values["languages"] = meta.get("languages") or ""
for name in ("pubdate", "timestamp", "last_modified"):
values[name] = _date(meta.get(name))
return values
# Templates
_FIELD = re.compile(r"([^.\[\]]*)((?:\[\d+\])*)")
_AFFIXES = re.compile(r"^(.*)\|([^|]*)\|(.*)$", re.DOTALL)
_INT_CODES, _FLOAT_CODES = "bcdoxXn", "eEfFgG%"
def _format(value: str, fmt: str) -> str:
"""Calibre's _do_format(): a number code takes the text as a number."""
if not fmt or not value:
return value
code = fmt[-1]
try:
number = int(value) if code in _INT_CODES else float(value) if code in _FLOAT_CODES else value
return format(number, fmt)
except (ValueError, OverflowError) as e:
raise TemplateError(f"The format {fmt!r} cannot take the value {value!r}: {e}") from e
def _parse(template: str) -> list[tuple[str, str | None, list[int], str, str, str]]:
"""The parts of a template. Each part is the text before a field, then the name of the field, its indexes,
its format, its prefix and its suffix. The name is None for text with no field after it."""
if len(template) > MAX_TEMPLATE_LENGTH:
raise TemplateError(f"A template can have {MAX_TEMPLATE_LENGTH} characters at most")
if template.startswith(("program:", "python:")):
raise TemplateError("Bindery does not support the program modes of Calibre")
try:
fields = list(Formatter().parse(template))
except ValueError as e:
raise TemplateError(f"The braces of the template do not match: {e}") from e
parts = []
for text, name, spec, conversion in fields:
if name is None:
parts.append((text, None, [], "", "", ""))
continue
field = _FIELD.fullmatch(name)
if not field or conversion or "{" in spec:
raise TemplateError(f"Bindery does not support the field {{{name}}} in this form")
if not field.group(1) or field.group(1).isdigit():
raise TemplateError("A field needs a name, such as {title}")
affixes = _AFFIXES.match(spec)
fmt, prefix, suffix = affixes.groups() if affixes else (spec, "", "")
if fmt.startswith("'") or (":'" in fmt and fmt.endswith("'")) or ("(" in fmt and fmt.endswith(")")):
raise TemplateError("Bindery does not support template functions, such as {title:uppercase()}")
if any(int(n) > MAX_WIDTH for n in re.findall(r"\d+", fmt)):
raise TemplateError(f"A format can ask for {MAX_WIDTH} characters at most")
_format("1", fmt)
indexes = [int(n) for n in re.findall(r"\d+", field.group(2))]
parts.append((text, field.group(1).lower(), indexes, fmt, prefix, suffix))
return parts
def check(template: str) -> None:
"""Raise TemplateError if Bindery cannot use the template."""
_parse(template)
def expand(template: str, values: dict[str, Any]) -> str:
"""The text of a template for the prepared values of a book, as Calibre gives it before it makes a path."""
out = []
for text, name, indexes, fmt, prefix, suffix in _parse(template):
out.append(text)
if name is None:
continue
value = values.get(name, "")
if isinstance(value, list):
value = ",".join(value)
value = value.replace("/", "_").replace("\\", "_")
try:
for index in indexes:
value = value[index]
except IndexError as e:
raise TemplateError(f"The value of {{{name}}} is too short for [{index}]") from e
if value:
value = _format(value, fmt)
if value:
out += prefix, value, suffix
return re.sub(r"\s+", " ", "".join(out)).strip(" ")
_RESERVED = re.compile(r"(?:CON|PRN|AUX|NUL|COM\d|LPT\d)(?:\.|$)", re.IGNORECASE)
def file_name(template: str, values: dict[str, Any]) -> str:
"""The name of one file from a template, without the extension. Empty if the template gives no name.
A download has one name, so Bindery uses the last part of a template with folders. That is the name that
Calibre gives the file. Windows cannot use some names, such as CON. Those get a leading underscore.
"""
parts = [part.strip() for part in expand(template, values).split("/")]
name = next((part for part in reversed(parts) if part), "")
if not name:
return ""
name = sanitize_filename(name)
if _RESERVED.match(name):
name = f"_{name}"
return name.encode()[:MAX_NAME_BYTES].decode(errors="ignore").rstrip(" .")
+215
View File
@@ -0,0 +1,215 @@
"""The file name templates, against the results of Calibre 9.2.1.
Each expected text is what Calibre gives for the same book and template. Bindery has no rating and no custom
columns, and it has one date format, so there are no cases for those.
"""
import pytest
from bindery import filename_template as ft
BOOK = {
"title": "The Book",
"authors": ["Ann Writer", "Ben Reader"],
"series": "The Saga",
"series_index": 2.345,
"languages": ["eng", "fra"],
"tags": ["Space", "Fiction"],
"pubdate": "2020-05-06T12:00:00+00:00",
"publisher": "site.example",
}
NO_SERIES = {**BOOK, "series": "", "series_index": None}
def expand(template, order="library_order", **changes):
return ft.expand(template, ft.prepare({**BOOK, **changes}, 42, order))
# name, template, text from Calibre, series index
REFERENCE = [
("title", "{title}", "Book, The", 2.345),
("series", "{series}", "Saga, The", 2.345),
("fraction", "{series_index}", "2.35", 2.345),
("languages", "{languages}", "eng,fra", 2.345),
("tags", "{tags}", "Fiction, Space", 2.345),
("send_date", "{pubdate}", "May, 2020", 2.345),
("space_padding", "x{series_index:>5s}x", "x 2x", 2),
("zero_padding", "{series_index:0>3s}", "002", 2),
("numeric_format", "{series_index:03d}", "002", 2),
("first_character", "{author_sort[0]}", "W", 2),
("conditional", "{series:|| - }{title}", "Saga, The - Book, The", 2),
("case_insensitive", "{TITLE}", "Book, The", 2),
("unknown", "x{not_a_field}x", "xx", 2),
("literal_braces", "{{title}}", "{title}", 2),
]
@pytest.mark.parametrize("name,template,expected,index", REFERENCE, ids=[case[0] for case in REFERENCE])
def test_reference_cases(name, template, expected, index):
assert expand(template, series_index=index) == expected
def test_reference_cases_for_a_missing_series_and_for_alphabetic_order():
values = ft.prepare(NO_SERIES, 42)
assert ft.expand("{series}{series_index:| - | - }{title}", values) == "Book, The"
assert expand("{title}", "strictly_alphabetic") == "The Book"
assert expand("{series}", "strictly_alphabetic") == "The Saga"
@pytest.mark.parametrize(
"template,expected",
[
("{authors}", "Ann Writer & Ben Reader"),
("{author}", "Ann Writer & Ben Reader"),
("{author_sort}", "Writer, Ann & Reader, Ben"),
("{title_sort}", "Book, The"),
("{publisher} {id}", "site.example 42"),
("{title:.3s|<|>}", "<Boo>"),
("{title:*^15}", "***Book, The***"),
("{id:05d} {id:x} {id:.2f}", "00042 2a 42.00"),
("{title[0]}{title[1]}", "Bo"),
("{title} {{{id}}}", "Book, The {42}"),
(" {title} -\t{authors} ", "Book, The - Ann Writer & Ben Reader"),
("{ title }{rating}{isbn}{#custom}", ""),
("a|b|c{title:|-|}", "a|b|c-Book, The"),
("{series}/{series_index:0>2s| | - }{title}", "Saga, The/ 2.35 - Book, The"),
],
)
def test_more_templates_give_the_text_of_calibre(template, expected):
assert expand(template) == expected
def test_values_follow_the_rules_of_calibre():
assert expand("{title} - {authors}", title="A/B: C\\D", authors=["X/Y"]) == "A_B: C_D - X_Y"
assert expand("{authors}", authors=["Rock & Roll", "", "Solo"]) == "Rock && Roll & Solo"
assert expand("{tags}", tags=["/root", "alpha", "Beta", "1st", "_x"]) == "_x, _root, 1st, alpha, Beta"
assert expand("{tags}", tags=["F/M", "Hurt-Comfort", "hurt comfort", "Hurt", "x2", "x10", "é", "e", "f"]) == (
"e, é, f, F_M, Hurt, hurt comfort, Hurt-Comfort, x10, x2"
)
assert expand("{tags}|{languages}|{pubdate}", tags=[], languages="", pubdate="") == "||"
assert expand("{languages}", languages="en") == "en"
assert expand("{pubdate} {timestamp}", pubdate="1999-12-31T23:59:59", timestamp="2026-10-03") == (
"Dec, 1999 Oct, 2026"
)
assert expand("{pubdate}", pubdate="not a date") == ""
# A series without a number has no series_index, and a book without a series has neither.
assert expand("{series}{series_index:| [|]}", series_index=None) == "Saga, The"
assert ft.expand("{series_index:03d}|{series}", ft.prepare(NO_SERIES, 42)) == "|"
@pytest.mark.parametrize(
"title,expected",
[
("A Tale", "Tale, A"),
("An Apple", "Apple, An"),
("the lower", "lower, the"),
("Theory", "Theory"),
("The", "The"),
(" The Spaced ", "Spaced, The"),
("'The Quoted'", "Quoted, The"),
("“The Curly”", "Curly, The"),
("„The Low“", "Low, The"),
("The 'Inner", "Inner, The"),
('""', ""),
("Le Monde", "Le Monde"),
],
)
def test_title_sort(title, expected):
assert ft.title_sort(title) == expected
@pytest.mark.parametrize(
"author,expected",
[
("Ann Writer", "Writer, Ann"),
("Dr. John Smith Jr.", "Smith, John Jr."),
("John Smith MD PhD", "Smith, John MD PhD"),
("Mr Smith", "Smith"),
("Dr Mr", "Dr Mr"),
("Jr Sr", "Jr Sr"),
("xX_Dark_Xx", "xX_Dark_Xx"),
("Smith, John", "Smith, John"),
("John (Jack) Doe", "Doe, John"),
("A (b c", "A (b c"),
("Acme Media", "Acme Media"),
("Ludwig van Beethoven", "Beethoven, Ludwig van"),
("J. R. R. Tolkien", "Tolkien, J. R. R."),
],
)
def test_author_sort(author, expected):
assert ft.author_sort(author) == expected
@pytest.mark.parametrize(
"index,expected",
[(1, "1"), (2.0, "2"), (0, "0"), (2.5, "2.5"), (1.10, "1.1"), (3.14159, "3.14"), (2.001, "2."), (10.999, "11.")],
)
def test_series_index(index, expected):
assert ft.series_index(index) == expected
@pytest.mark.parametrize(
"template,error",
[
("{title:uppercase()}", "does not support template functions"),
("{series:ifempty(none)}", "does not support template functions"),
("{title:'uppercase($)'}", "does not support template functions"),
("{title:.3s:'uppercase($)'}", "does not support template functions"),
("program: field('title')", "does not support the program modes"),
("python: def evaluate(book, context): return ''", "does not support the program modes"),
("{title!r}", "does not support the field"),
("{title.upper}", "does not support the field"),
("{title[x]}", "does not support the field"),
("{title:{id}}", "does not support the field"),
("{}", "A field needs a name"),
("{0}", "A field needs a name"),
("{title", "The braces of the template do not match"),
("title}", "The braces of the template do not match"),
("{title:5q}", "The format '5q' cannot take"),
("{series:|a|b|c}", "The format '|a' cannot take"),
("{id:.3d}", "The format '.3d' cannot take"),
("{title:>256s}", "A format can ask for 255 characters at most"),
("{id:.999f}", "A format can ask for 255 characters at most"),
("x" * 1025, "A template can have 1024 characters at most"),
],
)
def test_check_refuses_what_bindery_does_not_support(template, error):
with pytest.raises(ft.TemplateError, match=error.replace("|", "\\|")):
ft.check(template)
def test_check_accepts_the_templates_of_calibre():
templates = ("", "{author_sort}/{title}/{title} - {authors}", "{title:>255s}", "x" * 1024, "{#saga_index:0>3s}")
for template in templates:
ft.check(template)
@pytest.mark.parametrize("template", ["{series_index:03d}", "{title:d}", "{title[50]}", "{id:c}{title:c}"])
def test_a_template_can_fail_for_one_book(template):
ft.check(template)
with pytest.raises(ft.TemplateError):
expand(template)
def file_name(template, **changes):
return ft.file_name(template, ft.prepare({**BOOK, **changes}, 42))
def test_file_name_is_the_last_part_of_the_path():
assert file_name("{author_sort}/{title}/{title} - {authors}") == "Book, The - Ann Writer & Ben Reader"
assert file_name("{series}//{title} /") == "Book, The"
assert file_name("{author_sort[0]}/{series}", series="") == "W"
assert file_name("{series}/{rating}", series="") == ""
assert file_name(" / ") == ""
def test_file_name_is_safe():
assert file_name("{title}?", title="A/B: C\\D") == "A_B_ C_D_"
assert file_name("<{id:c}>") == "___"
assert file_name("{title}", title="con") == "_con"
assert file_name("{title}.{id}", title="NUL") == "_NUL.42"
assert file_name("{title}", title="Console") == "Console"
assert file_name("{title}...") == "Book, The"
long = file_name("{title}", title="é" * 150)
assert long == "é" * 100
assert len(file_name("{title}", title="\U0001f600" * 150).encode()) == 200