From 9e7a488b50808f366f5b8307175111dbb8ca99ea Mon Sep 17 00:00:00 2001 From: Timothy Allen Date: Sat, 3 Oct 2026 00:28:52 +0200 Subject: [PATCH] Add Calibre's file name templates The module prepares the metadata of a book and expands a template as Calibre 9.2.1 does with its default tweaks. It takes fields, formats, indexes, and text before and after a field. It refuses template functions and the program modes. --- AGENTS.md | 1 + bindery/filename_template.py | 301 ++++++++++++++++++++++++++++++++ tests/test_filename_template.py | 215 +++++++++++++++++++++++ 3 files changed, 517 insertions(+) create mode 100644 bindery/filename_template.py create mode 100644 tests/test_filename_template.py diff --git a/AGENTS.md b/AGENTS.md index e8823dc..aa6faa9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,6 +28,7 @@ The server has no reloader. If you change Python code, restart it. Flask reads ` - `bindery/fff.py` is the only module that calls FanFicFare. - `bindery/cwa.py` talks to CWA through OPDS. Bindery never opens the CWA database. - `bindery/diffing.py` hashes and diffs text, not HTML. +- `bindery/filename_template.py` expands Calibre's file name templates. It never runs template text as code. - `bindery/jobs.py` keeps the job queue in SQLite, so jobs survive a restart. - `tests/fake_cwa.py` is a fake CWA server. diff --git a/bindery/filename_template.py b/bindery/filename_template.py new file mode 100644 index 0000000..ac871ec --- /dev/null +++ b/bindery/filename_template.py @@ -0,0 +1,301 @@ +"""Calibre's templates for file names, for the name of a downloaded book. + +Bindery follows Calibre 9.2.1 with its default tweaks: titles and series in library order with the English +articles, and dates in the form "%b, %Y". It takes {field}, {field[0]}, {field:format|prefix|suffix} and {{ }}. +It refuses template functions and the program modes. It never runs template text as code. + +The tags sort close to Calibre's ICU rules, but their order can differ for characters outside ASCII. +""" + +import re +import unicodedata +from datetime import datetime +from string import Formatter +from typing import Any + +from bindery.ingest import sanitize_filename + +MAX_TEMPLATE_LENGTH = 1024 +# The most characters that a format can ask for, as a width or a precision. +MAX_WIDTH = 255 +MAX_NAME_BYTES = 200 + +# The fields that Bindery has, with the help text for each. Any other field is empty, as it is in Calibre for +# a book without that value. +FIELDS = { + "title": "The title", + "title_sort": "The title in sort order", + "authors": "The authors", + "author": "The authors", + "author_sort": "The author sort string. To use only the first letter of the name, use {author_sort[0]}", + "series": "The series", + "series_index": "The series number. To get leading zeros, use {series_index:0>3s}", + "tags": "The tags", + "languages": "The language", + "publisher": "The story site", + "pubdate": "The published date", + "timestamp": "The date when Bindery added the story", + "last_modified": "The date when the chapters last changed", + "id": "The Bindery story ID", +} + + +class TemplateError(ValueError): + """A template that Bindery cannot use, or that fails for a book.""" + + +# Metadata, as Calibre prepares it for a file name + +_ARTICLE = re.compile(r"^(A\s+|The\s+|An\s+)", re.IGNORECASE) +# Calibre's quote_pairs: each opening quote, with the quotes that close it. +_QUOTE_PAIRS = { + '"': '"', + "'": "'", + "\u201c": "\u201d\u201c", + "\u201d": "\u201d", + "\u201e": "\u201d\u201c", + "\u201a": "\u2019\u2018", + "\u2019": "\u2019\u2018", + "\u2018": "\u2019\u2018", + "\u2039": "\u203a", + "\u203a": "\u2039", + "\u300a": "\u300b", + "\u3008": "\u3009", + "\u00bb": "\u00ab\u00bb", + "\u00ab": "\u00ab\u00bb", + "\u300c": "\u300d", + "\u300e": "\u300f", +} +# The ASCII punctuation in the order of Calibre's sort. Digits come next, then letters. +_PUNCTUATION = " _-,;:!?.'\"()[]{}@*/\\&#%`^+<=>|~$" +_MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec") +# Calibre's default tweaks for author names. +_NAME_PREFIXES = {"mr", "mrs", "ms", "dr", "prof"} +_NAME_SUFFIXES = {"jr", "sr", "inc", "ph.d", "phd", "md", "m.d", "i", "ii", "iii", "iv", "junior", "senior"} +_COPY_WORDS = { + "agency", + "corporation", + "company", + "co.", + "council", + "committee", + "inc.", + "institute", + "national", + "society", + "club", + "team", + "software", + "games", + "entertainment", + "media", + "studios", +} +_BRACKETS = {"(": ")", "[": "]", "{": "}"} + + +def _unquote(title: str) -> str: + closers = _QUOTE_PAIRS.get(title[:1]) + if closers: + title = title[1:] + if title and title[-1] in closers: + title = title[:-1] + return title + + +def title_sort(title: str) -> str: + """Calibre's title_sort(): "The Book" gives "Book, The".""" + title = _unquote(title.strip()) + match = _ARTICLE.search(title) + if match: + title = _unquote(f"{title[match.end() :]}, {match.group(1)}") + return title.strip() + + +def _without_brackets(text: str) -> str: + depth = dict.fromkeys(_BRACKETS, 0) + closers = {close: opener for opener, close in _BRACKETS.items()} + kept = [] + for char in text: + if char in depth: + depth[char] += 1 + elif char in closers: + depth[closers[char]] = max(0, depth[closers[char]] - 1) + elif not any(depth.values()): + kept.append(char) + return "".join(kept) + + +def author_sort(author: str) -> str: + """Calibre's author_to_author_sort() with its default tweaks: "Ann Writer" gives "Writer, Ann".""" + plain = _without_brackets(author).strip() + tokens = plain.split() + lower = [t.lower().removesuffix(".") for t in tokens] + if "," in plain or len(tokens) < 2 or _COPY_WORDS.intersection(t.lower() for t in tokens): + return author + first = next((n for n, t in enumerate(lower) if t not in _NAME_PREFIXES), None) + if first is None: + return author + last = next((n for n in range(len(tokens) - 1, first - 1, -1) if lower[n] not in _NAME_SUFFIXES), None) + if last is None: + return author + names = [tokens[last], *tokens[first:last]] + if len(names) > 1: + names[0] += "," + return " ".join(names + tokens[last + 1 :]) + + +def series_index(index: Any) -> str: + """Calibre's fmt_sidx(): 2.0 gives "2", and 2.345 gives "2.35".""" + try: + index = float(index) + except (TypeError, ValueError): + index = 1.0 + if index.is_integer(): + return str(int(index)) + return f"{index:.2f}".rstrip("0") + + +def _date(value: Any) -> str: + try: + date = datetime.fromisoformat(value) + except (TypeError, ValueError): + return "" + return f"{_MONTHS[date.month - 1]}, {date.year:04}" + + +def _tag_key(tag: str) -> tuple[list[tuple[int, str]], str]: + """Close to the key of Calibre's sort: no case, punctuation before digits before letters, accents last.""" + text = unicodedata.normalize("NFKD", tag.casefold()) + ranks = len(_PUNCTUATION) + letters = [ + (_PUNCTUATION.index(c), "") if c in _PUNCTUATION else (ranks + c.isalpha(), c) + for c in text + if not unicodedata.combining(c) + ] + return letters, text + + +def prepare(meta: dict[str, Any], book_id: int, order: str = "library_order") -> dict[str, Any]: + """Calibre's get_component_metadata(): the values of a book in the form that a file name takes. + + `order` is Calibre's tweak save_template_title_series_sorting. With "strictly_alphabetic", the title and the + series keep a leading article. + """ + library = order == "library_order" + values: dict[str, Any] = {"id": str(book_id), "publisher": meta.get("publisher") or ""} + title, authors, series = meta.get("title"), [a for a in meta.get("authors") or [] if a], meta.get("series") + if title: + values["title_sort"] = title_sort(title) + values["title"] = values["title_sort"] if library else title + if authors: + values["authors"] = values["author"] = " & ".join(a.replace("&", "&&") for a in authors) + values["author_sort"] = " & ".join(author_sort(a) for a in authors) + if series: + values["series"] = title_sort(series) if library else series.strip() + if meta.get("series_index") is not None: + values["series_index"] = series_index(meta["series_index"]) + values["tags"] = ", ".join(sorted(meta.get("tags") or [], key=_tag_key)).removeprefix("/") + values["languages"] = meta.get("languages") or "" + for name in ("pubdate", "timestamp", "last_modified"): + values[name] = _date(meta.get(name)) + return values + + +# Templates + +_FIELD = re.compile(r"([^.\[\]]*)((?:\[\d+\])*)") +_AFFIXES = re.compile(r"^(.*)\|([^|]*)\|(.*)$", re.DOTALL) +_INT_CODES, _FLOAT_CODES = "bcdoxXn", "eEfFgG%" + + +def _format(value: str, fmt: str) -> str: + """Calibre's _do_format(): a number code takes the text as a number.""" + if not fmt or not value: + return value + code = fmt[-1] + try: + number = int(value) if code in _INT_CODES else float(value) if code in _FLOAT_CODES else value + return format(number, fmt) + except (ValueError, OverflowError) as e: + raise TemplateError(f"The format {fmt!r} cannot take the value {value!r}: {e}") from e + + +def _parse(template: str) -> list[tuple[str, str | None, list[int], str, str, str]]: + """The parts of a template. Each part is the text before a field, then the name of the field, its indexes, + its format, its prefix and its suffix. The name is None for text with no field after it.""" + if len(template) > MAX_TEMPLATE_LENGTH: + raise TemplateError(f"A template can have {MAX_TEMPLATE_LENGTH} characters at most") + if template.startswith(("program:", "python:")): + raise TemplateError("Bindery does not support the program modes of Calibre") + try: + fields = list(Formatter().parse(template)) + except ValueError as e: + raise TemplateError(f"The braces of the template do not match: {e}") from e + parts = [] + for text, name, spec, conversion in fields: + if name is None: + parts.append((text, None, [], "", "", "")) + continue + field = _FIELD.fullmatch(name) + if not field or conversion or "{" in spec: + raise TemplateError(f"Bindery does not support the field {{{name}}} in this form") + if not field.group(1) or field.group(1).isdigit(): + raise TemplateError("A field needs a name, such as {title}") + affixes = _AFFIXES.match(spec) + fmt, prefix, suffix = affixes.groups() if affixes else (spec, "", "") + if fmt.startswith("'") or (":'" in fmt and fmt.endswith("'")) or ("(" in fmt and fmt.endswith(")")): + raise TemplateError("Bindery does not support template functions, such as {title:uppercase()}") + if any(int(n) > MAX_WIDTH for n in re.findall(r"\d+", fmt)): + raise TemplateError(f"A format can ask for {MAX_WIDTH} characters at most") + _format("1", fmt) + indexes = [int(n) for n in re.findall(r"\d+", field.group(2))] + parts.append((text, field.group(1).lower(), indexes, fmt, prefix, suffix)) + return parts + + +def check(template: str) -> None: + """Raise TemplateError if Bindery cannot use the template.""" + _parse(template) + + +def expand(template: str, values: dict[str, Any]) -> str: + """The text of a template for the prepared values of a book, as Calibre gives it before it makes a path.""" + out = [] + for text, name, indexes, fmt, prefix, suffix in _parse(template): + out.append(text) + if name is None: + continue + value = values.get(name, "") + if isinstance(value, list): + value = ",".join(value) + value = value.replace("/", "_").replace("\\", "_") + try: + for index in indexes: + value = value[index] + except IndexError as e: + raise TemplateError(f"The value of {{{name}}} is too short for [{index}]") from e + if value: + value = _format(value, fmt) + if value: + out += prefix, value, suffix + return re.sub(r"\s+", " ", "".join(out)).strip(" ") + + +_RESERVED = re.compile(r"(?:CON|PRN|AUX|NUL|COM\d|LPT\d)(?:\.|$)", re.IGNORECASE) + + +def file_name(template: str, values: dict[str, Any]) -> str: + """The name of one file from a template, without the extension. Empty if the template gives no name. + + A download has one name, so Bindery uses the last part of a template with folders. That is the name that + Calibre gives the file. Windows cannot use some names, such as CON. Those get a leading underscore. + """ + parts = [part.strip() for part in expand(template, values).split("/")] + name = next((part for part in reversed(parts) if part), "") + if not name: + return "" + name = sanitize_filename(name) + if _RESERVED.match(name): + name = f"_{name}" + return name.encode()[:MAX_NAME_BYTES].decode(errors="ignore").rstrip(" .") diff --git a/tests/test_filename_template.py b/tests/test_filename_template.py new file mode 100644 index 0000000..369576d --- /dev/null +++ b/tests/test_filename_template.py @@ -0,0 +1,215 @@ +"""The file name templates, against the results of Calibre 9.2.1. + +Each expected text is what Calibre gives for the same book and template. Bindery has no rating and no custom +columns, and it has one date format, so there are no cases for those. +""" + +import pytest + +from bindery import filename_template as ft + +BOOK = { + "title": "The Book", + "authors": ["Ann Writer", "Ben Reader"], + "series": "The Saga", + "series_index": 2.345, + "languages": ["eng", "fra"], + "tags": ["Space", "Fiction"], + "pubdate": "2020-05-06T12:00:00+00:00", + "publisher": "site.example", +} +NO_SERIES = {**BOOK, "series": "", "series_index": None} + + +def expand(template, order="library_order", **changes): + return ft.expand(template, ft.prepare({**BOOK, **changes}, 42, order)) + + +# name, template, text from Calibre, series index +REFERENCE = [ + ("title", "{title}", "Book, The", 2.345), + ("series", "{series}", "Saga, The", 2.345), + ("fraction", "{series_index}", "2.35", 2.345), + ("languages", "{languages}", "eng,fra", 2.345), + ("tags", "{tags}", "Fiction, Space", 2.345), + ("send_date", "{pubdate}", "May, 2020", 2.345), + ("space_padding", "x{series_index:>5s}x", "x 2x", 2), + ("zero_padding", "{series_index:0>3s}", "002", 2), + ("numeric_format", "{series_index:03d}", "002", 2), + ("first_character", "{author_sort[0]}", "W", 2), + ("conditional", "{series:|| - }{title}", "Saga, The - Book, The", 2), + ("case_insensitive", "{TITLE}", "Book, The", 2), + ("unknown", "x{not_a_field}x", "xx", 2), + ("literal_braces", "{{title}}", "{title}", 2), +] + + +@pytest.mark.parametrize("name,template,expected,index", REFERENCE, ids=[case[0] for case in REFERENCE]) +def test_reference_cases(name, template, expected, index): + assert expand(template, series_index=index) == expected + + +def test_reference_cases_for_a_missing_series_and_for_alphabetic_order(): + values = ft.prepare(NO_SERIES, 42) + assert ft.expand("{series}{series_index:| - | - }{title}", values) == "Book, The" + assert expand("{title}", "strictly_alphabetic") == "The Book" + assert expand("{series}", "strictly_alphabetic") == "The Saga" + + +@pytest.mark.parametrize( + "template,expected", + [ + ("{authors}", "Ann Writer & Ben Reader"), + ("{author}", "Ann Writer & Ben Reader"), + ("{author_sort}", "Writer, Ann & Reader, Ben"), + ("{title_sort}", "Book, The"), + ("{publisher} {id}", "site.example 42"), + ("{title:.3s|<|>}", ""), + ("{title:*^15}", "***Book, The***"), + ("{id:05d} {id:x} {id:.2f}", "00042 2a 42.00"), + ("{title[0]}{title[1]}", "Bo"), + ("{title} {{{id}}}", "Book, The {42}"), + (" {title} -\t{authors} ", "Book, The - Ann Writer & Ben Reader"), + ("{ title }{rating}{isbn}{#custom}", ""), + ("a|b|c{title:|-|}", "a|b|c-Book, The"), + ("{series}/{series_index:0>2s| | - }{title}", "Saga, The/ 2.35 - Book, The"), + ], +) +def test_more_templates_give_the_text_of_calibre(template, expected): + assert expand(template) == expected + + +def test_values_follow_the_rules_of_calibre(): + assert expand("{title} - {authors}", title="A/B: C\\D", authors=["X/Y"]) == "A_B: C_D - X_Y" + assert expand("{authors}", authors=["Rock & Roll", "", "Solo"]) == "Rock && Roll & Solo" + assert expand("{tags}", tags=["/root", "alpha", "Beta", "1st", "_x"]) == "_x, _root, 1st, alpha, Beta" + assert expand("{tags}", tags=["F/M", "Hurt-Comfort", "hurt comfort", "Hurt", "x2", "x10", "é", "e", "f"]) == ( + "e, é, f, F_M, Hurt, hurt comfort, Hurt-Comfort, x10, x2" + ) + assert expand("{tags}|{languages}|{pubdate}", tags=[], languages="", pubdate="") == "||" + assert expand("{languages}", languages="en") == "en" + assert expand("{pubdate} {timestamp}", pubdate="1999-12-31T23:59:59", timestamp="2026-10-03") == ( + "Dec, 1999 Oct, 2026" + ) + assert expand("{pubdate}", pubdate="not a date") == "" + # A series without a number has no series_index, and a book without a series has neither. + assert expand("{series}{series_index:| [|]}", series_index=None) == "Saga, The" + assert ft.expand("{series_index:03d}|{series}", ft.prepare(NO_SERIES, 42)) == "|" + + +@pytest.mark.parametrize( + "title,expected", + [ + ("A Tale", "Tale, A"), + ("An Apple", "Apple, An"), + ("the lower", "lower, the"), + ("Theory", "Theory"), + ("The", "The"), + (" The Spaced ", "Spaced, The"), + ("'The Quoted'", "Quoted, The"), + ("“The Curly”", "Curly, The"), + ("„The Low“", "Low, The"), + ("The 'Inner", "Inner, The"), + ('""', ""), + ("Le Monde", "Le Monde"), + ], +) +def test_title_sort(title, expected): + assert ft.title_sort(title) == expected + + +@pytest.mark.parametrize( + "author,expected", + [ + ("Ann Writer", "Writer, Ann"), + ("Dr. John Smith Jr.", "Smith, John Jr."), + ("John Smith MD PhD", "Smith, John MD PhD"), + ("Mr Smith", "Smith"), + ("Dr Mr", "Dr Mr"), + ("Jr Sr", "Jr Sr"), + ("xX_Dark_Xx", "xX_Dark_Xx"), + ("Smith, John", "Smith, John"), + ("John (Jack) Doe", "Doe, John"), + ("A (b c", "A (b c"), + ("Acme Media", "Acme Media"), + ("Ludwig van Beethoven", "Beethoven, Ludwig van"), + ("J. R. R. Tolkien", "Tolkien, J. R. R."), + ], +) +def test_author_sort(author, expected): + assert ft.author_sort(author) == expected + + +@pytest.mark.parametrize( + "index,expected", + [(1, "1"), (2.0, "2"), (0, "0"), (2.5, "2.5"), (1.10, "1.1"), (3.14159, "3.14"), (2.001, "2."), (10.999, "11.")], +) +def test_series_index(index, expected): + assert ft.series_index(index) == expected + + +@pytest.mark.parametrize( + "template,error", + [ + ("{title:uppercase()}", "does not support template functions"), + ("{series:ifempty(none)}", "does not support template functions"), + ("{title:'uppercase($)'}", "does not support template functions"), + ("{title:.3s:'uppercase($)'}", "does not support template functions"), + ("program: field('title')", "does not support the program modes"), + ("python: def evaluate(book, context): return ''", "does not support the program modes"), + ("{title!r}", "does not support the field"), + ("{title.upper}", "does not support the field"), + ("{title[x]}", "does not support the field"), + ("{title:{id}}", "does not support the field"), + ("{}", "A field needs a name"), + ("{0}", "A field needs a name"), + ("{title", "The braces of the template do not match"), + ("title}", "The braces of the template do not match"), + ("{title:5q}", "The format '5q' cannot take"), + ("{series:|a|b|c}", "The format '|a' cannot take"), + ("{id:.3d}", "The format '.3d' cannot take"), + ("{title:>256s}", "A format can ask for 255 characters at most"), + ("{id:.999f}", "A format can ask for 255 characters at most"), + ("x" * 1025, "A template can have 1024 characters at most"), + ], +) +def test_check_refuses_what_bindery_does_not_support(template, error): + with pytest.raises(ft.TemplateError, match=error.replace("|", "\\|")): + ft.check(template) + + +def test_check_accepts_the_templates_of_calibre(): + templates = ("", "{author_sort}/{title}/{title} - {authors}", "{title:>255s}", "x" * 1024, "{#saga_index:0>3s}") + for template in templates: + ft.check(template) + + +@pytest.mark.parametrize("template", ["{series_index:03d}", "{title:d}", "{title[50]}", "{id:c}{title:c}"]) +def test_a_template_can_fail_for_one_book(template): + ft.check(template) + with pytest.raises(ft.TemplateError): + expand(template) + + +def file_name(template, **changes): + return ft.file_name(template, ft.prepare({**BOOK, **changes}, 42)) + + +def test_file_name_is_the_last_part_of_the_path(): + assert file_name("{author_sort}/{title}/{title} - {authors}") == "Book, The - Ann Writer & Ben Reader" + assert file_name("{series}//{title} /") == "Book, The" + assert file_name("{author_sort[0]}/{series}", series="") == "W" + assert file_name("{series}/{rating}", series="") == "" + assert file_name(" / ") == "" + + +def test_file_name_is_safe(): + assert file_name("{title}?", title="A/B: C\\D") == "A_B_ C_D_" + assert file_name("<{id:c}>") == "___" + assert file_name("{title}", title="con") == "_con" + assert file_name("{title}.{id}", title="NUL") == "_NUL.42" + assert file_name("{title}", title="Console") == "Console" + assert file_name("{title}...") == "Book, The" + long = file_name("{title}", title="é" * 150) + assert long == "é" * 100 + assert len(file_name("{title}", title="\U0001f600" * 150).encode()) == 200