Add a script that fills a gap in a story from an ePub without chapter links
A FanFicFare ePub records the site address of each chapter, so Bindery can import it. Other ePubs, such as the author's own ebook of a story, have no chapter links. The script asks which stored chapters come before and after the gap, and which ePub chapters match them. It suggests matches by title and puts the suggestion on the input line. It then adds the ePub chapters between the matches, with their images, in the order of the ePub. Each added chapter gets a stable URL with the #bindery-import- marker and the revision source "epub-import". A second run with the same ePub does not add the same chapters again. --remove deletes the chapters that the script added.
This commit is contained in:
@@ -0,0 +1,274 @@
|
||||
"""Tests for scripts/import_unlinked_epub_chapters.py, a manual tool."""
|
||||
|
||||
import zipfile
|
||||
|
||||
import pytest
|
||||
|
||||
from bindery import config
|
||||
from bindery.store import Store
|
||||
from scripts import import_unlinked_epub_chapters as script
|
||||
from scripts.import_unlinked_epub_chapters import (
|
||||
ScriptError,
|
||||
chapter_url,
|
||||
main,
|
||||
read_epub,
|
||||
splice_order,
|
||||
suggest_matches,
|
||||
)
|
||||
|
||||
STORY_URL = "https://www.royalroad.com/fiction/1234"
|
||||
|
||||
CONTAINER_XML = """<?xml version="1.0"?>
|
||||
<container xmlns="urn:oasis:names:tc:opendocument:xmlns:container" version="1.0">
|
||||
<rootfiles><rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/></rootfiles>
|
||||
</container>"""
|
||||
|
||||
|
||||
def chapter_html(title: str, text: str) -> str:
|
||||
return f"<html><head><title>{title}</title></head><body><h1>{title}</h1><p>{text}</p></body></html>"
|
||||
|
||||
|
||||
def make_opf(count: int, identifier: str) -> str:
|
||||
items = "\n".join(
|
||||
f' <item id="c{i}" href="chap{i}.xhtml" media-type="application/xhtml+xml"/>' for i in range(1, count + 1)
|
||||
)
|
||||
refs = "\n".join(f' <itemref idref="c{i}"/>' for i in range(1, count + 1))
|
||||
return f"""<?xml version="1.0"?>
|
||||
<package xmlns="http://www.idpf.org/2007/opf" version="3.0">
|
||||
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:identifier>{identifier}</dc:identifier></metadata>
|
||||
<manifest>
|
||||
<item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>
|
||||
{items}
|
||||
</manifest>
|
||||
<spine>
|
||||
<itemref idref="nav" linear="no"/>
|
||||
{refs}
|
||||
</spine>
|
||||
</package>"""
|
||||
|
||||
|
||||
def make_epub(path, chapters, identifier="book-1", files=None):
|
||||
"""An ePub with a chapter file for each (title, text). `files` adds more files, such as images."""
|
||||
with zipfile.ZipFile(path, "w") as z:
|
||||
z.writestr("META-INF/container.xml", CONTAINER_XML)
|
||||
z.writestr("OEBPS/content.opf", make_opf(len(chapters), identifier))
|
||||
z.writestr("OEBPS/nav.xhtml", "<html><body><nav></nav></body></html>")
|
||||
for i, (title, text) in enumerate(chapters, start=1):
|
||||
z.writestr(f"OEBPS/chap{i}.xhtml", chapter_html(title, text))
|
||||
for name, data in (files or {}).items():
|
||||
z.writestr(name, data)
|
||||
return path
|
||||
|
||||
|
||||
def test_read_epub_reads_spine_order_and_skips_nonlinear_items(tmp_path):
|
||||
path = make_epub(tmp_path / "book.epub", [("Chapter One", "First text"), ("Chapter Two", "Second text")])
|
||||
chapters = read_epub(path)
|
||||
assert [c["title"] for c in chapters] == ["Chapter One", "Chapter Two"]
|
||||
assert "First text" in chapters[0]["html"]
|
||||
assert "<body" not in chapters[0]["html"]
|
||||
assert chapters[0]["key"] != chapters[1]["key"]
|
||||
|
||||
|
||||
NCX_OPF = """<?xml version="1.0"?>
|
||||
<package xmlns="http://www.idpf.org/2007/opf" version="2.0">
|
||||
<metadata/>
|
||||
<manifest>
|
||||
<item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml"/>
|
||||
<item id="contents" href="text/contents.xhtml" media-type="application/xhtml+xml"/>
|
||||
<item id="c1" href="text/one.xhtml" media-type="application/xhtml+xml"/>
|
||||
<item id="c2" href="text/two.xhtml" media-type="application/xhtml+xml"/>
|
||||
</manifest>
|
||||
<spine toc="ncx"><itemref idref="contents"/><itemref idref="c1"/><itemref idref="c2"/></spine>
|
||||
</package>"""
|
||||
|
||||
NCX_TOC = """<?xml version="1.0"?>
|
||||
<ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" version="2005-1"><navMap>
|
||||
<navPoint id="p1"><navLabel><text>CHAPTER ONE: Arrival</text></navLabel><content src="text/one.xhtml"/></navPoint>
|
||||
<navPoint id="p2"><navLabel><text>12</text></navLabel><content src="text/two.xhtml#page_12"/></navPoint>
|
||||
</navMap></ncx>"""
|
||||
|
||||
|
||||
def test_read_epub_skips_a_contents_page_and_takes_titles_from_the_ncx(tmp_path):
|
||||
# As in a calibre conversion: a contents page in the reading order, a title in two headings,
|
||||
# and a page number in the NCX. Curly quotes stay.
|
||||
path = tmp_path / "book.epub"
|
||||
with zipfile.ZipFile(path, "w") as z:
|
||||
z.writestr("META-INF/container.xml", CONTAINER_XML)
|
||||
z.writestr("OEBPS/content.opf", NCX_OPF)
|
||||
z.writestr("OEBPS/toc.ncx", NCX_TOC)
|
||||
z.writestr(
|
||||
"OEBPS/text/contents.xhtml",
|
||||
'<html><body><h1><a href="one.xhtml">CHAPTER ONE: Arrival</a></h1>'
|
||||
'<p><a href="two.xhtml">CHAPTER TWO: Decision</a></p></body></html>',
|
||||
)
|
||||
z.writestr(
|
||||
"OEBPS/text/one.xhtml", "<html><body><h1>CHAPTER ONE</h1><h1>Arrival</h1><p>“I’m here.”</p></body></html>"
|
||||
)
|
||||
z.writestr("OEBPS/text/two.xhtml", "<html><body><h1>CHAPTER TWO</h1><h1>Decision</h1><p>Text</p></body></html>")
|
||||
chapters = read_epub(path)
|
||||
assert [c["title"] for c in chapters] == ["CHAPTER ONE: Arrival", "CHAPTER TWO"]
|
||||
assert "<p>“I’m here.”</p>" in chapters[0]["html"]
|
||||
|
||||
|
||||
def test_read_epub_points_images_at_their_own_urls_and_drops_missing_ones(tmp_path):
|
||||
path = make_epub(
|
||||
tmp_path / "book.epub",
|
||||
[("Map", '<img src="images/map.png"/><img src="images/gone.png"/><img src="images/notes.txt"/>')],
|
||||
files={"OEBPS/images/map.png": b"png data", "OEBPS/images/notes.txt": b"text"},
|
||||
)
|
||||
(chapter,) = read_epub(path)
|
||||
(image,) = chapter["images"]
|
||||
assert image["url"].startswith("file:///bindery-epub-import/")
|
||||
assert image["url"].endswith("/OEBPS/images/map.png")
|
||||
assert (image["ext"], image["mime"], image["data"]) == ("png", "image/png", b"png data")
|
||||
assert f'src="{image["url"]}"' in chapter["html"]
|
||||
assert "gone.png" not in chapter["html"]
|
||||
assert "notes.txt" not in chapter["html"]
|
||||
|
||||
|
||||
def test_read_epub_rejects_a_bad_file(tmp_path):
|
||||
path = tmp_path / "not-an-epub.epub"
|
||||
path.write_text("nope")
|
||||
with pytest.raises(ScriptError):
|
||||
read_epub(path)
|
||||
|
||||
|
||||
def test_suggest_matches_ranks_the_closest_title_first_and_drops_weak_ones():
|
||||
candidates = ["Chapter One: A New Start", "Interlude", "Chapter Two: Trouble Brews"]
|
||||
assert suggest_matches("Chapter Two", candidates)[0] == 2
|
||||
assert suggest_matches("Chapter One", ["Completely unrelated title"]) == []
|
||||
|
||||
|
||||
def test_chapter_url_is_stable_and_differs_between_epubs(tmp_path):
|
||||
one = read_epub(make_epub(tmp_path / "one.epub", [("Prologue", "a")], identifier="book-1"))[0]
|
||||
again = read_epub(make_epub(tmp_path / "again.epub", [("Prologue", "a")], identifier="book-1"))[0]
|
||||
other = read_epub(make_epub(tmp_path / "other.epub", [("Prologue", "a")], identifier="book-3"))[0]
|
||||
url = chapter_url(STORY_URL, one)
|
||||
assert url.startswith(f"{STORY_URL}#bindery-import-")
|
||||
assert chapter_url(STORY_URL, again) == url
|
||||
assert chapter_url(STORY_URL, other) != url
|
||||
|
||||
|
||||
def test_splice_order_puts_the_new_urls_after_the_given_index():
|
||||
assert splice_order(["a", "b", "c"], 0, ["x", "y"]) == ["a", "x", "y", "b", "c"]
|
||||
assert splice_order(["a", "b"], -1, ["x"]) == ["x", "a", "b"]
|
||||
assert splice_order(["a", "b"], 1, ["x"]) == ["a", "b", "x"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def story(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(config, "DB_PATH", tmp_path / "bindery.db")
|
||||
db_store = Store(config.DB_PATH)
|
||||
story_id = db_store.create_story(STORY_URL, "www.royalroad.com", "Story", "Writer", [])
|
||||
db_store.add_chapter(story_id, f"{STORY_URL}/1", "Chapter 1", "<p>one</p>", "h1", "new", 0)
|
||||
db_store.add_chapter(story_id, f"{STORY_URL}/4", "Chapter 4", "<p>four</p>", "h4", "new", 1)
|
||||
epub = make_epub(
|
||||
tmp_path / "book.epub",
|
||||
[
|
||||
("Cover", "cover"),
|
||||
("Chapter 1", "one text"),
|
||||
("Chapter 2", '<img src="map.png"/>two text'),
|
||||
("Chapter 3", "three text"),
|
||||
("Chapter 4", "four text"),
|
||||
("About the Author", "author"),
|
||||
],
|
||||
files={"OEBPS/map.png": b"png"},
|
||||
)
|
||||
return db_store, story_id, epub
|
||||
|
||||
|
||||
def answer(monkeypatch, *answers):
|
||||
"""Answer the prompts in order. A prompt that shows a default in [brackets] gets it for an empty answer."""
|
||||
replies = iter(answers)
|
||||
prompts = []
|
||||
|
||||
def fake_input(prompt=""):
|
||||
prompts.append(prompt)
|
||||
return next(replies)
|
||||
|
||||
monkeypatch.setattr("builtins.input", fake_input)
|
||||
return prompts
|
||||
|
||||
|
||||
def titles(db_store, story_id):
|
||||
return [c["title"] for c in db_store.list_chapters(story_id)]
|
||||
|
||||
|
||||
def test_main_fills_a_gap_with_the_suggested_matches(story, monkeypatch, capsys):
|
||||
db_store, story_id, epub = story
|
||||
# Before the gap: stored chapter 1. Then Enter takes each suggestion.
|
||||
prompts = answer(monkeypatch, "1", "", "", "", "y")
|
||||
main([STORY_URL, str(epub)])
|
||||
|
||||
# Each prompt says what it asks for, and shows the suggestion. Numbers start at 1.
|
||||
assert prompts[:4] == [
|
||||
"Stored chapter just BEFORE the gap (or 'start'): ",
|
||||
'ePub chapter that matches "Chapter 1" [2]: ',
|
||||
"Stored chapter just AFTER the gap (or 'end') [2]: ",
|
||||
'ePub chapter that matches "Chapter 4" [5]: ',
|
||||
]
|
||||
assert titles(db_store, story_id) == ["Chapter 1", "Chapter 2", "Chapter 3", "Chapter 4"]
|
||||
added = db_store.list_chapters(story_id)[1]
|
||||
assert added["url"].startswith(f"{STORY_URL}#bindery-import-")
|
||||
assert added["on_site"] == 0
|
||||
assert db_store.list_revisions(added["id"])[0]["source"] == "epub-import"
|
||||
html = db_store.revision_html(added["id"], 1)
|
||||
(image,) = db_store.images(story_id)
|
||||
assert f'src="{image["url"]}"' in html
|
||||
assert 'The script will add these 2 chapters after "Chapter 1"' in capsys.readouterr().out
|
||||
|
||||
# A second run over the same gap finds the same chapters in the story, and adds nothing.
|
||||
answer(monkeypatch, "1", "", "4", "")
|
||||
main([STORY_URL, str(epub)])
|
||||
assert len(db_store.list_chapters(story_id)) == 4
|
||||
assert "2 ePub chapters of the gap are in the story already" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_main_asks_again_for_a_bad_answer_and_can_show_the_whole_list(story, monkeypatch, capsys):
|
||||
db_store, story_id, epub = story
|
||||
monkeypatch.setattr(script, "WINDOW", 2)
|
||||
answer(monkeypatch, "9", "list", "1", "", "1", "", "", "n")
|
||||
main([STORY_URL, str(epub)])
|
||||
out = capsys.readouterr().out
|
||||
assert "Type a number from 1 to 2, or 'list' or 'start'." in out
|
||||
# The AFTER question cannot go before the BEFORE chapter.
|
||||
assert "Type a number from 2 to 2, or 'list' or 'end'." in out
|
||||
assert titles(db_store, story_id) == ["Chapter 1", "Chapter 4"]
|
||||
|
||||
|
||||
def test_main_asks_for_the_first_and_last_epub_chapters_at_the_start_and_end(story, monkeypatch):
|
||||
db_store, story_id, epub = story
|
||||
# The gap is at the start: add from ePub chapter 1 up to the match of stored Chapter 1.
|
||||
prompts = answer(monkeypatch, "start", "1", "", "", "y")
|
||||
main([STORY_URL, str(epub)])
|
||||
assert prompts[1] == "FIRST ePub chapter to add: "
|
||||
assert titles(db_store, story_id) == ["Cover", "Chapter 1", "Chapter 4"]
|
||||
|
||||
# The gap is after the last stored chapter, so the script does not ask for the chapter after it.
|
||||
prompts = answer(monkeypatch, "3", "", "", "y")
|
||||
main([STORY_URL, str(epub)])
|
||||
assert prompts[2] == "LAST ePub chapter to add [6]: "
|
||||
assert titles(db_store, story_id) == ["Cover", "Chapter 1", "Chapter 4", "About the Author"]
|
||||
|
||||
|
||||
def test_main_changes_nothing_when_the_user_says_no(story, monkeypatch):
|
||||
db_store, story_id, epub = story
|
||||
answer(monkeypatch, "1", "", "", "", "n")
|
||||
main([STORY_URL, str(epub)])
|
||||
assert titles(db_store, story_id) == ["Chapter 1", "Chapter 4"]
|
||||
assert db_store.images(story_id) == []
|
||||
|
||||
|
||||
def test_remove_deletes_only_the_added_chapters_and_their_revisions(story, monkeypatch):
|
||||
db_store, story_id, _ = story
|
||||
added = db_store.add_chapter(story_id, f"{STORY_URL}#bindery-import-x", "Two", "<p>2</p>", "h2", "epub-import", 1)
|
||||
db_store.add_revision(added, "<p>2b</p>", "h2b", "recheck")
|
||||
|
||||
answer(monkeypatch, "n")
|
||||
main([STORY_URL, "--remove"])
|
||||
assert len(db_store.list_chapters(story_id)) == 3
|
||||
|
||||
answer(monkeypatch, "y")
|
||||
main([STORY_URL, "--remove"])
|
||||
assert titles(db_store, story_id) == ["Chapter 1", "Chapter 4"]
|
||||
assert db_store.list_revisions(added) == []
|
||||
Reference in New Issue
Block a user