From 96cf1e4f0331684ac54abdf22d05ce1db30759d6 Mon Sep 17 00:00:00 2001 From: Timothy Allen Date: Fri, 2 Oct 2026 23:49:04 +0200 Subject: [PATCH] Split the site job into smaller functions A new story, the chapter list of the site and the choice of chapters to download each have a function. One helper downloads a chapter for both loops. --- bindery/orchestrator.py | 220 +++++++++++++++++++++------------------- 1 file changed, 114 insertions(+), 106 deletions(-) diff --git a/bindery/orchestrator.py b/bindery/orchestrator.py index 76abeb2..da938b5 100644 --- a/bindery/orchestrator.py +++ b/bindery/orchestrator.py @@ -1116,48 +1116,39 @@ def _recent_chapters(story_id: int) -> int: return _setting_recent_chapters() -def _run_site_job(job: Job) -> str: - story = store.get_story(job.story_id) if job.story_id else None - url = story["url"] if story else job.url - adapter = fff.get_adapter(url) - _progress(job, "Reading story page") - snap = fff.fetch_story(adapter) - if story is None: - story = store.get_story_by_url(snap.url) - if story is None: - # A new story. If the library already has it (for example from the - # FanFicFare calibre plugin), link it and keep its chapters. - found = _find_book(snap.url, [snap.title]) if _link_books() and library_available() else None - book_id, book_uuid = (found[0].id, found[0].uuid) if found else (None, None) - contents = None - if found: - _progress(job, "Reading chapters of library book") - try: - contents = _read_epub(found[1], found[0].title) - except StoryError as e: - logger.warning(str(e)) - finally: - found[1].unlink(missing_ok=True) - story_id = store.create_story( - snap.url, snap.site, snap.title, snap.author, default_locked_fields(), book_id, book_uuid - ) - _link_changed(story_id) - if found: - store.update_story(story_id, ingested_at=now()) - if contents: - _import_epub(story_id, contents) - story = _require_story(story_id) - job.story_id = story["id"] - job_queue.update(job, title=snap.title) - story_id = story["id"] +def _new_story(job: Job, snap: fff.SiteSnapshot) -> dict[str, Any]: + """Create the story of a site job. If the library already has it (for example from the FanFicFare calibre + plugin), link it and keep its chapters.""" + found = _find_book(snap.url, [snap.title]) if _link_books() and library_available() else None + book_id, book_uuid = (found[0].id, found[0].uuid) if found else (None, None) + contents = None + if found: + _progress(job, "Reading chapters of library book") + try: + contents = _read_epub(found[1], found[0].title) + except StoryError as e: + logger.warning(str(e)) + finally: + found[1].unlink(missing_ok=True) + story_id = store.create_story( + snap.url, snap.site, snap.title, snap.author, default_locked_fields(), book_id, book_uuid + ) + _link_changed(story_id) + if found: + store.update_story(story_id, ingested_at=now()) + if contents: + _import_epub(story_id, contents) + return _require_story(story_id) - store.update_story(story_id, metahtml=snap.metahtml, site=snap.site, author=snap.author) - story["metahtml"] = snap.metahtml - if snap.cover: - store.put_image( - story_id, snap.cover["url"], snap.cover["ext"], snap.cover["mime"], snap.cover["data"], is_cover=True - ) +def _sync_chapters( + story_id: int, adapter: Any, site_titles: dict[str, str], full: bool +) -> tuple[list[dict[str, Any]], dict[int, str], int]: + """Mark each stored chapter as the site lists it now. + + Returns the chapters, the new site title of each renamed chapter by its id, and how many chapters a full + download took out of the book. + """ # Chapter URLs can change (RoyalRoad changes the slug when a title # changes). FanFicFare maps old forms to the current one. chapters = store.list_chapters(story_id) @@ -1170,11 +1161,8 @@ def _run_site_job(job: Job) -> str: known.add(normal) c["url"] = normal - site_titles = {c["url"]: c["title"] for c in snap.chapters} - site_index = {url: i for i, url in enumerate(site_titles)} - by_url = {c["url"]: c for c in chapters} dropped = 0 - renamed: dict[int, str] = {} # chapter id: new title on the site + renamed: dict[int, str] = {} for c in chapters: on_site = int(c["url"] in site_titles) changes = {} @@ -1182,46 +1170,73 @@ def _run_site_job(job: Job) -> str: changes["on_site"] = on_site if on_site and site_titles[c["url"]] != c["title"]: renamed[c["id"]] = site_titles[c["url"]] - if job.mode == DownloadMode.FULL and c["included"] != on_site: + if full and c["included"] != on_site: changes["included"] = on_site dropped += 1 - on_site if changes: store.update_chapter(c["id"], **changes) c.update(changes) + return chapters, renamed, dropped + +def _chapters_to_fetch( + job: Job, story_id: int, chapters: list[dict[str, Any]], site_titles: dict[str, str], renamed: dict[int, str] +) -> tuple[list[str], list[dict[str, Any]], list[str]]: + """What a site job downloads: the URLs of the new chapters, and the stored chapters to check again. + + Also returns the titles of the selected chapters that the site no longer lists. + """ mode = job.mode - new_urls = [u for u in site_titles if u not in by_url] - if mode == DownloadMode.METADATA: + stored = {c["url"] for c in chapters} + new_urls = [u for u in site_titles if u not in stored] + if mode in (DownloadMode.METADATA, DownloadMode.RECHECK): new_urls = [] - refetch: list[dict[str, Any]] = [] - skipped: list[str] = [] + selected = [c for c in chapters if c["id"] in job.chapter_ids] if mode == DownloadMode.RECHECK else [] + skipped = [fff.plain_title(c["title"]) for c in selected if not c["on_site"]] if mode in (DownloadMode.RECHECK_ALL, DownloadMode.FULL): refetch = [c for c in chapters if c["on_site"]] - elif mode == DownloadMode.RECHECK: - new_urls = [] - for c in chapters: - if c["id"] not in job.chapter_ids: - continue - if c["on_site"]: - refetch.append(c) - else: - skipped.append(fff.plain_title(c["title"])) + else: + refetch = [c for c in selected if c["on_site"]] # A new title can come with new text. On a site that numbers its chapter # URLs, it can also mean that the chapters moved up. So a renamed chapter is # checked again, and it takes the new title only when that works. A metadata # update downloads no chapters, so it only takes the new titles. - renamed_count = job.result.get("renamed", 0) - if mode == DownloadMode.METADATA: - for chapter_id, title in renamed.items(): - store.update_chapter(chapter_id, title=title) - renamed_count += len(renamed) - else: + if mode != DownloadMode.METADATA: refetch += [c for c in chapters if c["id"] in renamed and c not in refetch] # Authors most often fix their latest chapters, so a new-chapter download can check those again. recent = _recent_chapters(story_id) if mode == DownloadMode.NEW else 0 if recent: latest = [c for c in chapters if c["on_site"] and c["included"]][-recent:] refetch += [c for c in latest if c not in refetch] + return new_urls, refetch, skipped + + +def _run_site_job(job: Job) -> str: + story = store.get_story(job.story_id) if job.story_id else None + adapter = fff.get_adapter(story["url"] if story else job.url) + _progress(job, "Reading story page") + snap = fff.fetch_story(adapter) + story = story or store.get_story_by_url(snap.url) or _new_story(job, snap) + story_id = job.story_id = story["id"] + job_queue.update(job, title=snap.title) + + store.update_story(story_id, metahtml=snap.metahtml, site=snap.site, author=snap.author) + story["metahtml"] = snap.metahtml + if snap.cover: + store.put_image( + story_id, snap.cover["url"], snap.cover["ext"], snap.cover["mime"], snap.cover["data"], is_cover=True + ) + + mode = job.mode + site_titles = {c["url"]: c["title"] for c in snap.chapters} + site_index = {url: i for i, url in enumerate(site_titles)} + chapters, renamed, dropped = _sync_chapters(story_id, adapter, site_titles, mode == DownloadMode.FULL) + new_urls, refetch, skipped = _chapters_to_fetch(job, story_id, chapters, site_titles, renamed) + renamed_count = job.result.get("renamed", 0) + if mode == DownloadMode.METADATA: + for chapter_id, title in renamed.items(): + store.update_chapter(chapter_id, title=title) + renamed_count += len(renamed) # Each chapter is stored as soon as it arrives, after its images. The job # result keeps what this job did, so a job that resumes after a restart @@ -1236,11 +1251,23 @@ def _run_site_job(job: Job) -> str: failed: list[str] = [] stored_images = store.image_urls(story_id) - def store_images() -> None: + def fetch(url: str, title: str, action: str) -> str | None: + """Download a chapter and store its images. A chapter that fails goes in `failed`, and gives None.""" + nonlocal done + name = fff.plain_title(title) + _progress(job, f"{action} {done + 1} of {total}: {name}", done / max(total, 1)) + done += 1 + try: + html = fff.fetch_chapter(adapter, url, site_index[url]) + except Exception as e: + logger.warning(f"Chapter {url} failed: {e}") + failed.append(f"{name}: {e}") + return None for img in fff.collected_images(adapter): if img["url"] not in stored_images: store.put_image(story_id, img["url"], img["ext"], img["mime"], img["data"]) stored_images.add(img["url"]) + return html def save_progress() -> None: job_queue.update( @@ -1248,15 +1275,9 @@ def _run_site_job(job: Job) -> str: ) for url in new_urls: - _progress(job, f"New chapter {done + 1} of {total}: {fff.plain_title(site_titles[url])}", done / max(total, 1)) - done += 1 - try: - html = fff.fetch_chapter(adapter, url, site_index[url]) - except Exception as e: - logger.warning(f"Chapter {url} failed: {e}") - failed.append(f"{fff.plain_title(site_titles[url])}: {e}") + html = fetch(url, site_titles[url], "New chapter") + if html is None: continue - store_images() chapter = {"url": url, "title": site_titles[url], "html": html, "text_hash": text_hash(html)} # The chapter goes in the order that the story has now. A move or a delete by hand during this job stays. store.add_chapters(story_id, [chapter], mode.value, lambda have: merge_chapter_lists(have, list(site_titles))) @@ -1264,18 +1285,9 @@ def _run_site_job(job: Job) -> str: save_progress() for c in refetch: - _progress(job, f"Rechecking chapter {done + 1} of {total}: {fff.plain_title(c['title'])}", done / max(total, 1)) - try: - html = fff.fetch_chapter(adapter, c["url"], site_index[c["url"]]) - except Exception as e: - logger.warning(f"Chapter {c['url']} failed: {e}") - failed.append(f"{fff.plain_title(c['title'])}: {e}") - done += 1 - continue - store_images() - revisions = store.list_revisions(c["id"]) - if not revisions: # the user deleted the chapter during this job - done += 1 + html = fetch(c["url"], c["title"], "Rechecking chapter") + revisions = store.list_revisions(c["id"]) if html is not None else [] + if not revisions: # the download failed, or the user deleted the chapter during this job continue if c["id"] in renamed: c["title"] = renamed[c["id"]] @@ -1293,22 +1305,18 @@ def _run_site_job(job: Job) -> str: changed.append({"chapter_id": c["id"], "title": c["title"], "rev": rev}) checked.append(c["id"]) save_progress() - done += 1 removed = sum(1 for c in chapters if not c["on_site"] and c["included"]) - summary = [f"{new_count} new chapter{'s' if new_count != 1 else ''}"] - if renamed_count: - summary.append(f"{renamed_count} renamed") - if rechecked: - summary.append(f"{len(changed)} of {rechecked} rechecked chapters changed") - if removed: - summary.append(f"{removed} kept after removal from site") - if dropped: - summary.append(f"{dropped} taken out because site removed them") - if skipped: - summary.append(f"{len(skipped)} skipped, no longer on site") - if failed: - summary.append(f"{len(failed)} failed") + parts = [ + (True, f"{new_count} new chapter{'s' if new_count != 1 else ''}"), + (renamed_count, f"{renamed_count} renamed"), + (rechecked, f"{len(changed)} of {rechecked} rechecked chapters changed"), + (removed, f"{removed} kept after removal from site"), + (dropped, f"{dropped} taken out because site removed them"), + (skipped, f"{len(skipped)} skipped, no longer on site"), + (failed, f"{len(failed)} failed"), + ] + summary = ", ".join(text for show, text in parts if show) job_queue.update( job, result={"new": new_count, "changed": changed, "failed": failed, "skipped": skipped, "removed": removed} ) @@ -1316,7 +1324,7 @@ def _run_site_job(job: Job) -> str: if mode in (DownloadMode.RECHECK_ALL, DownloadMode.FULL): store.set_setting(story_id, "last_full_recheck", now()) if not store.chapters_for_build(story_id): - raise StoryError(", ".join(summary) + ". No chapters downloaded.") + raise StoryError(f"{summary}. No chapters downloaded.") # The book is built and sent only when its chapters, rules or metadata changed. A job that resumes, or that # follows a failed job, finds the changes of the earlier run through the key. @@ -1324,17 +1332,17 @@ def _run_site_job(job: Job) -> str: title = _display_title(pinned, snap.title) overrides = _epub_overrides(pinned) key = _build_key(story, overrides, snap.calibre) - checked: dict[str, Any] = {"pinned": pinned, "site_values": snap.calibre, "title": title, "last_checked": now()} + changes: dict[str, Any] = {"pinned": pinned, "site_values": snap.calibre, "title": title, "last_checked": now()} if key == _done_key(story_id): - store.update_story(story_id, **checked, status="ok", message="No changes") - return "No changes. " + ", ".join(summary) + store.update_story(story_id, **changes, status="ok", message="No changes") + return f"No changes. {summary}" _progress(job, "Building epub", 1.0) message, book_changed = _send(story, overrides, key, title, snap.author) if book_changed: - checked["last_changed"] = now() - store.update_story(story_id, **checked, message=message, status="error" if failed else "ok") - return ", ".join(summary) + ". " + message + changes["last_changed"] = now() + store.update_story(story_id, **changes, message=message, status="error" if failed else "ok") + return f"{summary}. {message}" def _run_job(job: Job) -> str: