Store chapters, merge chapter lists, diff and apply rules

The SQLite store keeps every chapter revision, so a chapter that the site removes stays in the book. The merge puts new chapters after the chapter that comes before them on the site. Diffs compare text, not HTML, and replacement rules run on chapter HTML when a book is built.
2026-09-27 21:23:20 +02:00
parent ee59917257
commit 6c2574cf0f
10 changed files with 1054 additions and 0 deletions
@@ -0,0 +1,9 @@
+"""Point every directory at a temporary folder before the app modules load."""
+
+import os
+import tempfile
+from pathlib import Path
+
+_tmp = Path(tempfile.mkdtemp(prefix="bindery-tests-"))
+for name, sub in [("CONFIG_DIR", "config"), ("INGEST_DIR", "ingest"), ("TMP_DIR", "tmp")]:
+    os.environ[name] = str(_tmp / sub)
@@ -0,0 +1,33 @@
+from bindery.diffing import diff_html, html_to_blocks, text_hash
+
+
+def test_random_class_names_do_not_count_as_changes():
+    a = '<div><p class="cnMwZG">Hello <em>there</em>.</p><p class="x1">Second</p></div>'
+    b = '<div><p class="zzQ99">Hello <em>there</em>.</p><p class="y2">Second</p></div>'
+    assert text_hash(a) == text_hash(b)
+    assert html_to_blocks(a) == ["Hello there.", "Second"]
+
+
+def test_text_changes_change_the_hash():
+    assert text_hash("<p>one</p>") != text_hash("<p>two</p>")
+
+
+def test_diff_counts_and_word_level_changes():
+    old = "<p>Same.</p><p>The cat sat.</p><p>Gone.</p>"
+    new = "<p>Same.</p><p>The dog sat.</p><p>Gone.</p><p>Added.</p>"
+    result = diff_html(old, new)
+    assert (result["changed"], result["added"], result["removed"]) == (1, 1, 0)
+    assert "<del>cat</del><ins>dog</ins>" in result["html"]
+    assert not result["identical"]
+
+
+def test_diff_output_is_escaped():
+    result = diff_html("<p>a</p>", "<p>&lt;script&gt;alert(1)&lt;/script&gt;</p>")
+    assert "<script>" not in result["html"]
+    assert "&lt;script&gt;" in result["html"]
+
+
+def test_source_diff_sees_markup_changes():
+    old, new = "<p>Hi</p>", '<p class="x">Hi</p>'
+    assert diff_html(old, new)["identical"]
+    assert not diff_html(old, new, source=True)["identical"]
@@ -0,0 +1,55 @@
+from bindery.merge import merge_chapter_lists
+
+
+def urls(*numbers):
+    return [f"c{n}" for n in numbers]
+
+
+def test_stubbed_chapters_stay_and_new_chapter_goes_last():
+    existing = urls(*range(1, 61))
+    site = urls(1, 2, 3, 4, *range(46, 62))
+    assert merge_chapter_lists(existing, site) == urls(*range(1, 62))
+
+
+def test_new_chapter_goes_after_its_site_predecessor():
+    assert merge_chapter_lists(urls(1, 2, 3), ["c1", "c2", "x", "c3"]) == ["c1", "c2", "x", "c3"]
+
+
+def test_new_chapter_after_a_gap_goes_before_removed_chapters():
+    # The author replaced 2-3 with a new chapter x.
+    assert merge_chapter_lists(urls(1, 2, 3, 4), ["c1", "x", "c4"]) == ["c1", "x", "c2", "c3", "c4"]
+
+
+def test_new_chapter_before_everything_goes_first():
+    assert merge_chapter_lists(urls(1, 2), ["p", "c1", "c2"]) == ["p", "c1", "c2"]
+
+
+def test_failed_chapter_later_lands_in_order():
+    # c61 failed last time, c62 was added. The next run adds c61 before c62.
+    assert merge_chapter_lists(urls(59, 60, 62), urls(59, 60, 61, 62)) == urls(59, 60, 61, 62)
+
+
+def test_existing_order_wins_and_duplicates_are_dropped():
+    # The site moved b before a. Our order stays, and c is newer than every chapter we have.
+    assert merge_chapter_lists(["a", "b", "b"], ["b", "a", "a", "c"]) == ["a", "b", "c"]
+
+
+def test_new_chapter_after_a_stubbed_tail_goes_last():
+    # The site kept 1-5, removed 6-10 and added 11 and 12.
+    assert merge_chapter_lists(urls(*range(1, 11)), urls(1, 2, 3, 4, 5, 11, 12)) == urls(*range(1, 13))
+
+
+def test_new_chapters_after_every_old_chapter_is_gone_go_last():
+    assert merge_chapter_lists(urls(1, 2, 3), urls(7, 8)) == urls(1, 2, 3, 7, 8)
+
+
+def test_empty_lists():
+    assert merge_chapter_lists([], urls(1, 2)) == urls(1, 2)
+    assert merge_chapter_lists(urls(1), []) == urls(1)
+
+
+def test_older_copy_fills_the_gap_before_newer_chapters():
+    existing = urls(*range(1, 6), *range(50, 61))
+    epub = urls(*range(1, 46))
+    assert merge_chapter_lists(existing, epub, newest_last=False) == urls(*range(1, 46), *range(50, 61))
+    assert merge_chapter_lists(urls(1, 2), urls(1, 2, 3), newest_last=False) == urls(1, 2, 3)
@@ -0,0 +1,90 @@
+import pytest
+
+from bindery import rules as rules_module
+from bindery.rules import RuleError, apply_rules, breaks_to_paragraphs, compile_rules, normalize_rules
+
+
+def test_rules_apply_in_order_and_skip_disabled():
+    rules = normalize_rules(
+        [
+            {"pattern": "cat", "replacement": "dog"},
+            {"pattern": "dog", "replacement": "wolf", "enabled": False},
+            {"pattern": r"(\w+) house", "replacement": r"\1 home", "flags": "I"},
+        ]
+    )
+    assert apply_rules("The cat is in the HOUSE", compile_rules(rules)) == "The dog is in the home"
+    assert apply_rules("a cat House", compile_rules(rules)) == "a dog home"
+
+
+def test_global_rules_run_before_story_rules():
+    global_rules = normalize_rules([{"pattern": "a", "replacement": "b"}])
+    story_rules = normalize_rules([{"pattern": "b", "replacement": "c"}])
+    assert apply_rules("a", compile_rules(global_rules, story_rules)) == "c"
+
+
+def test_flags_are_cleaned():
+    assert normalize_rules([{"pattern": "x", "flags": "smi"}])[0]["flags"] == "ims"
+
+
+@pytest.mark.parametrize(
+    "rule, message",
+    [
+        ({"pattern": ""}, "empty"),
+        ({"pattern": "(open"}, "missing"),
+        ({"pattern": "a", "replacement": r"\3"}, "group"),
+        ({"pattern": "a", "flags": "q"}, "flag"),
+    ],
+)
+def test_bad_rules_name_the_rule(rule, message):
+    with pytest.raises(RuleError) as info:
+        normalize_rules([{"pattern": "fine"}, rule])
+    assert info.value.index == 1
+    assert message in str(info.value)
+
+
+@pytest.mark.parametrize(
+    "rule",
+    [
+        {"pattern": r"(a)(?P<b>b)", "replacement": r"\2 \g<b> \1"},
+        {"pattern": r"(?i)(a)", "replacement": r"\1"},
+        {"pattern": "(a) # a comment", "replacement": r"\1", "flags": "x"},
+        {"pattern": "a", "replacement": r"x\\y\n"},
+    ],
+)
+def test_good_replacements_pass(rule):
+    assert normalize_rules([rule])[0]["replacement"] == rule["replacement"]
+
+
+@pytest.mark.parametrize("replacement", [r"\g<nope>", "\\", r"\q"])
+def test_bad_replacements_fail_before_a_match(replacement):
+    with pytest.raises(RuleError):
+        normalize_rules([{"pattern": "(a)", "replacement": replacement}])
+
+
+def test_a_slow_rule_stops_at_the_timeout(monkeypatch):
+    monkeypatch.setattr(rules_module, "TIMEOUT", 0.1)
+    steps = compile_rules(normalize_rules([{"pattern": "(a|aa)+$"}]))
+    with pytest.raises(ValueError, match=r"takes longer than 0.1 seconds"):
+        apply_rules("a" * 60 + "b", steps)
+
+
+@pytest.mark.parametrize(
+    ("html", "expected"),
+    [
+        ("<div>One<br>Two<br/><br/>Three</div>", "<div><p>One</p><p>Two</p><p>Three</p></div>"),
+        ('<div><p class="x" id="p1">A<br>B</p></div>', '<div><p class="x" id="p1">A</p><p class="x">B</p></div>'),
+        ("<div>Loose<br><p>Block</p>After</div>", "<div><p>Loose</p><p>Block</p><p>After</p></div>"),
+        ("<div><p><em>a<br/>b</em></p></div>", "<div><p><em>a<br/>b</em></p></div>"),
+        ('<div>Text<br>&nbsp;<br><img src="i.png"/><br> </div>', '<div><p>Text</p><p><img src="i.png"/></p></div>'),
+        ("<div><p>No breaks</p></div>", "<div><p>No breaks</p></div>"),
+    ],
+)
+def test_breaks_become_paragraphs(html, expected):
+    assert breaks_to_paragraphs(html) == expected
+
+
+def test_paragraphs_come_before_the_rules():
+    rules = normalize_rules([{"pattern": "<p>", "replacement": '<p class="x">'}])
+    steps = compile_rules(rules, paragraphs=True)
+    assert apply_rules("<div>a<br>b</div>", steps) == '<div><p class="x">a</p><p class="x">b</p></div>'
+    assert apply_rules("<div>a<br>b</div>", compile_rules(rules)) == "<div>a<br>b</div>"
@@ -0,0 +1,71 @@
+from bindery.store import Store
+
+
+def test_chapters_revisions_and_rules(tmp_path):
+    store = Store(tmp_path / "db.sqlite")
+    sid = store.create_story("https://example/s/1", "example", "T", "A", ["title"])
+    c1 = store.add_chapter(sid, "u1", "One", "<p>1</p>", "h1", "new", 0)
+    c2 = store.add_chapter(sid, "u2", "Two", "<p>2</p>", "h2", "new", 1)
+    store.set_positions(sid, [c2, c1])
+    assert [c["url"] for c in store.chapters_for_build(sid)] == ["u2", "u1"]
+
+    assert store.add_revision(c1, "<p>1b</p>", "h1b", "recheck") == 2
+    assert store.current_revisions(sid)[c1] == {"rev": 2, "text_hash": "h1b", "source": "recheck"}
+    store.replace_revision(c1, 2, "<p>1c</p>", "h1c")
+    assert store.current_revisions(sid)[c1]["text_hash"] == "h1c"
+    assert store.revision_html(c1, 1) == "<p>1</p>"
+    store.update_chapter(c1, current_rev=1, included=0)
+    assert [c["url"] for c in store.chapters_for_build(sid)] == ["u2"]
+
+    store.set_rules(None, [{"pattern": "g"}])
+    store.set_rules(sid, [{"pattern": "s", "flags": "i"}])
+    assert [r["pattern"] for r in store.rules(None)] == ["g"]
+    assert store.rules(sid)[0]["flags"] == "i"
+
+    story = store.get_story(sid)
+    assert story["locked_fields"] == ["title"] and story["pinned"] == {}
+    store.update_story(sid, pinned={"title": "Kept"})
+    assert store.get_story_by_url("https://example/s/1")["pinned"] == {"title": "Kept"}
+
+    store.delete_story(sid)
+    assert store.get_chapter(c1) is None
+    assert store.rules(sid) == []
+    assert store.rules(None) != []
+
+
+def test_cover_replaces_previous_cover(tmp_path):
+    store = Store(tmp_path / "db.sqlite")
+    sid = store.create_story("u", "s", "T", "A", [])
+    store.put_image(sid, "c1", "jpg", "image/jpeg", b"1", is_cover=True)
+    store.put_image(sid, "c2", "jpg", "image/jpeg", b"2", is_cover=True)
+    store.put_image(sid, "i1", "png", "image/png", b"3")
+    assert store.cover(sid)["url"] == "c2"
+    assert [i["url"] for i in store.images(sid)] == ["i1"]
+
+
+def test_add_chapters_places_them_in_one_step(tmp_path):
+    store = Store(tmp_path / "db.sqlite")
+    sid = store.create_story("u", "s", "T", "A", [])
+    store.add_chapter(sid, "u1", "One", "<p>1</p>", "h1", "new", 0)
+    store.add_chapter(sid, "u3", "Three", "<p>3</p>", "h3", "new", 1)
+    store.add_chapter(sid, "u9", "Nine", "<p>9</p>", "h9", "new", 2)
+    new = [{"url": u, "title": u, "html": "<p/>", "text_hash": "h"} for u in ("u2", "u4")]
+    ids = store.add_chapters(sid, new, "import", ["u1", "u2", "u3", "u4"])
+    chapters = store.list_chapters(sid)
+    assert [c["url"] for c in chapters] == ["u1", "u2", "u3", "u4", "u9"]
+    assert [c["position"] for c in chapters] == [0, 1, 2, 3, 4]
+    assert [c["id"] for c in chapters if c["url"] in ("u2", "u4")] == ids
+
+
+def test_settings(tmp_path):
+    store = Store(tmp_path / "db.sqlite")
+    sid = store.create_story("u", "s", "T", "A", [])
+    assert store.get_setting(None, "paragraphs") is None
+    store.set_setting(None, "paragraphs", True)
+    store.set_setting(sid, "paragraphs", False)
+    assert (store.get_setting(None, "paragraphs"), store.get_setting(sid, "paragraphs")) == (True, False)
+    store.set_setting(sid, "paragraphs", None)
+    assert store.get_setting(sid, "paragraphs") is None
+    store.set_setting(sid, "paragraphs", True)
+    store.delete_story(sid)
+    assert store.get_setting(sid, "paragraphs") is None