tim
df9959a078
Some revisions differ from others only by curly quotes, curly apostrophes and the ellipsis. The script deletes those revisions and keeps the one with the least smart punctuation. In revisions with other text changes, it makes the punctuation straight and stores the new text hash. It does not change the newest revision if it came from the site. The script uses only the Python standard library, so you can run it on the host against bindery.db. It is a dry run unless you give --apply, and --apply makes a backup first.
138 lines
6.1 KiB
Python
138 lines
6.1 KiB
Python
"""Tests for scripts/remove_smart_quote_revisions.py, a manual cleanup tool."""
|
||
|
||
import pytest
|
||
|
||
from bindery import diffing
|
||
from bindery.store import Store
|
||
from scripts.remove_smart_quote_revisions import main, straighten_html, text_blocks, text_hash
|
||
|
||
CURLY = "<div><p class='a1'>“Hi,” he said.</p><p>It’s ‘fine’…</p></div>"
|
||
STRAIGHT = "<div><p class='b2'>\"Hi,\" he said.</p><p>It's 'fine'...</p></div>"
|
||
# The same text as CURLY, with the smart punctuation as entities.
|
||
ENTITIES = "<div><p>“Hi,” he said.</p><p>It’s ‘fine’…</p></div>"
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"html",
|
||
[
|
||
CURLY,
|
||
ENTITIES,
|
||
"<p>One<br/>Two</p><script>x()</script><style>p{}</style>",
|
||
"<div>Loose <em>text</em><ul><li>a</li><li>b & c</li></ul> tail </div><hr/>",
|
||
"",
|
||
],
|
||
)
|
||
def test_text_and_hash_match_bindery(html):
|
||
assert text_blocks(html) == diffing.html_to_blocks(html)
|
||
assert text_hash(html) == diffing.text_hash(html)
|
||
|
||
|
||
def test_straighten_html_changes_only_the_text():
|
||
html = '<p title="“x”">“Hi,” it’s ‘fine’… & ’<!-- “note” --></p>'
|
||
assert straighten_html(html) == "<p title=\"“x”\">\"Hi,\" it's 'fine'... & '<!-- “note” --></p>"
|
||
assert diffing.html_to_blocks(straighten_html(ENTITIES)) == diffing.html_to_blocks(STRAIGHT)
|
||
|
||
|
||
@pytest.fixture
|
||
def db(tmp_path):
|
||
path = tmp_path / "bindery.db"
|
||
store = Store(path)
|
||
story_id = store.create_story("https://example.com/s/1", "example.com", "Story", "Writer", [])
|
||
return path, store, story_id
|
||
|
||
|
||
def chapter(store, story_id, n, revisions, current=None):
|
||
"""A chapter with revisions given as (html, source). The last one is in the book, unless `current` says."""
|
||
html, source = revisions[0]
|
||
cid = store.add_chapter(story_id, f"u{n}", f"Chapter {n}", html, diffing.text_hash(html), source, n)
|
||
for html, source in revisions[1:]:
|
||
store.add_revision(cid, html, diffing.text_hash(html), source)
|
||
if current:
|
||
store.update_chapter(cid, current_rev=current)
|
||
return cid
|
||
|
||
|
||
def revs(store, cid):
|
||
return [r["rev"] for r in store.list_revisions(cid)]
|
||
|
||
|
||
def test_dry_run_changes_nothing(db, capsys):
|
||
path, store, story_id = db
|
||
cid = chapter(store, story_id, 1, [(CURLY, "import"), (STRAIGHT, "recheck")])
|
||
assert main([str(path)]) == 0
|
||
assert revs(store, cid) == [1, 2]
|
||
out = capsys.readouterr().out
|
||
assert "delete revision 1 (import), keep 2 (recheck)" in out
|
||
assert "Run again with --apply" in out
|
||
assert not list(path.parent.glob("*before-quote-cleanup*"))
|
||
|
||
|
||
def test_apply_deletes_revisions_that_differ_only_by_smart_punctuation(db, capsys):
|
||
path, store, story_id = db
|
||
# The smart import, then the straight site text.
|
||
older = chapter(store, story_id, 1, [(CURLY, "import"), (STRAIGHT, "recheck")])
|
||
# The same, but the user put the smart revision in the book.
|
||
chosen = chapter(store, story_id, 2, [(CURLY, "import"), (STRAIGHT, "recheck")], current=1)
|
||
# A smart import after the site text. It is the newest, but not from the site.
|
||
imported = chapter(store, story_id, 3, [(STRAIGHT, "new"), (ENTITIES, "import")])
|
||
# The site sends smart punctuation now. The next recheck would add it again.
|
||
site = chapter(store, story_id, 4, [(STRAIGHT, "new"), (CURLY, "recheck")])
|
||
|
||
assert main([str(path), "--apply"]) == 0
|
||
assert revs(store, older) == [2]
|
||
assert revs(store, chosen) == [2]
|
||
assert store.get_chapter(chosen)["current_rev"] == 2
|
||
assert revs(store, imported) == [1]
|
||
assert store.get_chapter(imported)["current_rev"] == 1
|
||
assert revs(store, site) == [1, 2]
|
||
out = capsys.readouterr().out
|
||
assert "3 revisions to delete, and 0 revisions to make straight" in out
|
||
assert "not changed: revision 2 (recheck) is the newest text from the site" in out
|
||
assert "Rebuild them:\n - Story" in out
|
||
(backup,) = path.parent.glob("bindery.before-quote-cleanup-*.db")
|
||
assert revs(Store(backup), older) == [1, 2]
|
||
|
||
|
||
def test_apply_makes_revisions_with_other_changes_straight(db, capsys):
|
||
path, store, story_id = db
|
||
changed = CURLY.replace("fine", "good")
|
||
# Three different texts. The newest is not from the site, so all of them get straight.
|
||
both = chapter(
|
||
store, story_id, 1, [(CURLY, "import"), (changed, "recheck"), (ENTITIES.replace("Hi", "Yo"), "import")]
|
||
)
|
||
# The newest is from the site, so only the older one gets straight.
|
||
older = chapter(store, story_id, 2, [(CURLY, "import"), (changed, "recheck")])
|
||
# A smart revision that also matches a straight one goes, and the other text gets straight.
|
||
mixed = chapter(store, story_id, 3, [(CURLY, "import"), (STRAIGHT, "new"), (changed, "import")], current=1)
|
||
|
||
assert main([str(path), "--apply"]) == 0
|
||
|
||
def smart(cid, rev):
|
||
text = "\n".join(diffing.html_to_blocks(store.revision_html(cid, rev)))
|
||
return any(c in text for c in "‘’“”…")
|
||
|
||
assert [smart(both, rev) for rev in (1, 2, 3)] == [False, False, False]
|
||
assert [smart(older, rev) for rev in (1, 2)] == [False, True]
|
||
assert revs(store, mixed) == [2, 3]
|
||
assert [smart(mixed, rev) for rev in (2, 3)] == [False, False]
|
||
assert store.get_chapter(mixed)["current_rev"] == 2
|
||
# Bindery compares the next site text with the stored hash, so it must be the hash of the new text.
|
||
for cid in (both, older, mixed):
|
||
for r in store.list_revisions(cid):
|
||
assert r["text_hash"] == diffing.text_hash(store.revision_html(cid, r["rev"]))
|
||
assert "class='a1'" in store.revision_html(both, 1)
|
||
out = capsys.readouterr().out
|
||
assert "make the punctuation straight in revision 1 (import), 2 (recheck), 3 (import)" in out
|
||
assert "1 revisions to delete, and 5 revisions to make straight" in out
|
||
# A second run finds nothing to do.
|
||
assert main([str(path)]) == 0
|
||
assert "not changed: revision 2 (recheck)" in capsys.readouterr().out
|
||
|
||
|
||
def test_no_work_and_a_missing_file(db, tmp_path, capsys):
|
||
path, store, story_id = db
|
||
chapter(store, story_id, 1, [(STRAIGHT, "new")])
|
||
assert main([str(path), "--apply"]) == 0
|
||
assert "No revisions to change." in capsys.readouterr().out
|
||
assert main([str(tmp_path / "missing.db")]) == 1
|