1272 lines
52 KiB
Python
1272 lines
52 KiB
Python
"""Download modes, metadata locks, delivery to CWA, and the job threads."""
|
|
|
|
import hashlib
|
|
import logging
|
|
import re
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from bindery import config, fff
|
|
from bindery.backup import daily_backup
|
|
from bindery.cwa import Book, BookFinder, CwaClient, CwaError
|
|
from bindery.diffing import diff_html, text_hash
|
|
from bindery.epub import EpubError, epub_metadata, epub_urls
|
|
from bindery.fff import StoryError
|
|
from bindery.ingest import IngestFolder
|
|
from bindery.jobs import METADATA_FIELDS, DownloadMode, Job, QueueStatus, job_queue
|
|
from bindery.merge import merge_chapter_lists
|
|
from bindery.rules import RuleError, Step, apply_rules, compile_rules, normalize_rules
|
|
from bindery.store import Store, now
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
store = Store(config.DB_PATH)
|
|
finder = BookFinder()
|
|
|
|
SITE_MODES = (
|
|
DownloadMode.NEW,
|
|
DownloadMode.RECHECK,
|
|
DownloadMode.RECHECK_ALL,
|
|
DownloadMode.FULL,
|
|
DownloadMode.METADATA,
|
|
)
|
|
|
|
_client_lock = threading.Lock()
|
|
_client: tuple[tuple[str, str, str], CwaClient | None] = (("", "", ""), None)
|
|
|
|
|
|
def cwa() -> CwaClient:
|
|
"""The CWA client for the configured account."""
|
|
global _client
|
|
key = (config.CWA_URL, config.CWA_USERNAME, config.CWA_PASSWORD)
|
|
with _client_lock:
|
|
if _client[0] != key or _client[1] is None:
|
|
_client = (key, CwaClient(*key))
|
|
return _client[1]
|
|
|
|
|
|
def library_available() -> bool:
|
|
return bool(config.CWA_URL)
|
|
|
|
|
|
def _linking_available() -> bool:
|
|
return bool(config.CWA_URL or config.CALIBRE_DB_PATH)
|
|
|
|
|
|
def _link_books() -> bool:
|
|
return _linking_available() and store.get_setting(None, "link_books") is not False
|
|
|
|
|
|
def _can_match_books() -> bool:
|
|
return _link_books()
|
|
|
|
|
|
def default_locked_fields() -> list[str]:
|
|
return [f for f in config.DEFAULT_LOCKED_FIELDS if f in METADATA_FIELDS]
|
|
|
|
|
|
# Queueing
|
|
|
|
|
|
def queue_urls(urls: list[str]) -> tuple[list[dict[str, Any]], list[str]]:
|
|
"""Queue a "new chapters" download for each URL. Returns (jobs, errors)."""
|
|
jobs, errors = [], []
|
|
for url in (u.strip() for u in urls):
|
|
if not url:
|
|
continue
|
|
normal = fff.normalize_url(url)
|
|
if not normal:
|
|
errors.append(f"Not a supported story URL: {url}")
|
|
continue
|
|
story = store.get_story_by_url(normal[0])
|
|
job = Job(
|
|
mode=DownloadMode.NEW,
|
|
story_id=story["id"] if story else None,
|
|
url=normal[0],
|
|
title=story["title"] if story else normal[0],
|
|
)
|
|
jobs.append(job_queue.add(job).to_dict())
|
|
return jobs, errors
|
|
|
|
|
|
def queue_story(
|
|
story_id: int, mode: str, chapter_ids: list[int] | None = None, force_fields: list[str] | None = None
|
|
) -> dict[str, Any]:
|
|
story = _require_story(story_id)
|
|
mode_enum = DownloadMode(mode)
|
|
if mode_enum == DownloadMode.IMPORT:
|
|
raise ValueError("Use the library import for this mode")
|
|
if mode_enum == DownloadMode.RECHECK and not chapter_ids:
|
|
raise ValueError("Select at least one chapter to recheck")
|
|
bad = [f for f in force_fields or [] if f not in METADATA_FIELDS]
|
|
if bad:
|
|
raise ValueError(f"Unknown metadata fields: {bad}")
|
|
job = Job(
|
|
mode=mode_enum,
|
|
story_id=story_id,
|
|
title=story["title"],
|
|
chapter_ids=sorted(int(c) for c in chapter_ids or []),
|
|
force_fields=sorted(force_fields or []),
|
|
)
|
|
return job_queue.add(job).to_dict()
|
|
|
|
|
|
def queue_import(books: list[dict[str, Any]], check: bool = True) -> list[dict[str, Any]]:
|
|
"""Queue imports of library books, given as {"id": ..., "uuid": ..., "title": ...}."""
|
|
jobs = []
|
|
for b in books:
|
|
book_id = int(b["id"])
|
|
jobs.append(
|
|
job_queue.add(
|
|
Job(
|
|
mode=DownloadMode.IMPORT,
|
|
calibre_id=book_id,
|
|
title=b.get("title") or f"Library book {book_id}",
|
|
options={"check": check, "uuid": str(b.get("uuid") or "")},
|
|
)
|
|
).to_dict()
|
|
)
|
|
return jobs
|
|
|
|
|
|
def queue_epub_file(path: Path, filename: str, story_id: int | None = None, check: bool = True) -> dict[str, Any]:
|
|
"""Queue an import of an uploaded epub. A file without a story URL, or of another story, fails at once."""
|
|
try:
|
|
normal = next((n for n in map(fff.normalize_url, epub_urls(path)) if n), None)
|
|
if not normal:
|
|
raise StoryError(f"{filename} has no supported story URL. Only epubs from FanFicFare can be imported.")
|
|
story = store.get_story_by_url(normal[0])
|
|
if story_id is not None:
|
|
story = _require_story(story_id)
|
|
if story["url"] != normal[0]:
|
|
raise StoryError(f"{filename} is an epub of another story: {normal[0]}")
|
|
except EpubError as e:
|
|
path.unlink(missing_ok=True)
|
|
raise EpubError(str(e).replace(path.name, filename)) from e
|
|
except Exception:
|
|
path.unlink(missing_ok=True)
|
|
raise
|
|
job = Job(
|
|
mode=DownloadMode.IMPORT_FILE,
|
|
story_id=story["id"] if story else None,
|
|
url=normal[0],
|
|
title=story["title"] if story else filename,
|
|
options={"path": str(path), "name": filename, "check": check},
|
|
)
|
|
return job_queue.add(job).to_dict()
|
|
|
|
|
|
def queue_status(story_id: int | None = None) -> list[dict[str, Any]]:
|
|
return job_queue.get_status(story_id)
|
|
|
|
|
|
def cancel_job(job_id: str) -> bool:
|
|
return job_queue.cancel(job_id)
|
|
|
|
|
|
# Story, chapter and rule management for the UI
|
|
|
|
|
|
def _require_story(story_id: int) -> dict[str, Any]:
|
|
story = store.get_story(story_id)
|
|
if not story:
|
|
raise KeyError(f"No story with id {story_id}")
|
|
return story
|
|
|
|
|
|
def list_stories() -> list[dict[str, Any]]:
|
|
stories = store.list_stories()
|
|
held = store.story_settings("upload_held")
|
|
unapproved = store.story_settings("unapproved_link")
|
|
show_holds = _link_books() and store.get_setting(None, "hold_new_links") and not _uploads_paused()
|
|
for s in stories:
|
|
s["busy"] = job_queue.has_active(s["id"])
|
|
s["upload_held"] = bool(held.get(s["id"]))
|
|
s["approval_held"] = bool(
|
|
show_holds
|
|
and s["upload_held"]
|
|
and unapproved.get(s["id"])
|
|
and (s["calibre_id"] is not None or not s["ingested_at"])
|
|
)
|
|
return stories
|
|
|
|
|
|
# Only raster images. An SVG cover could hold a script that runs on Bindery's origin.
|
|
COVER_TYPES = frozenset({"image/jpeg", "image/png", "image/gif", "image/webp"})
|
|
|
|
|
|
def story_cover(story_id: int) -> dict[str, Any] | None:
|
|
"""The site cover of a story, if a browser can show it safely."""
|
|
_require_story(story_id)
|
|
cover = store.cover(story_id)
|
|
return cover if cover and cover["mime"] in COVER_TYPES else None
|
|
|
|
|
|
def story_details(story_id: int) -> dict[str, Any]:
|
|
story = _require_story(story_id)
|
|
story.pop("metahtml", None)
|
|
cover = story_cover(story_id)
|
|
# The version changes with the image, so a browser can keep a cover until the site changes it.
|
|
story["cover"] = hashlib.sha256(cover["data"]).hexdigest()[:16] if cover else None
|
|
book, library_error = None, None
|
|
if story["calibre_uuid"] and _can_match_books():
|
|
try:
|
|
book = cwa().get_book(story["calibre_uuid"])
|
|
except CwaError as e:
|
|
library_error = str(e)
|
|
story["recheck_days"] = store.get_setting(story_id, "recheck_days")
|
|
story["last_full_recheck"] = store.get_setting(story_id, "last_full_recheck")
|
|
story["upload_held"] = bool(store.get_setting(story_id, "upload_held"))
|
|
story["needs_approval"] = _needs_approval(story_id)
|
|
return {
|
|
"recent_chapters": _setting_recent_chapters(),
|
|
"uploads_paused": _uploads_paused(),
|
|
"ingest_available": config.INGEST_DIR is not None,
|
|
"link_books": _link_books(),
|
|
"library_matching_available": _can_match_books(),
|
|
"story": story,
|
|
"chapters": store.list_chapters(story_id),
|
|
"rules": store.rules(story_id),
|
|
"book": book,
|
|
"library_available": library_available(),
|
|
"library_error": library_error,
|
|
"jobs": job_queue.get_status(story_id),
|
|
}
|
|
|
|
|
|
def update_story_settings(
|
|
story_id: int,
|
|
locked_fields: list[str] | None = None,
|
|
auto_update: bool | None = None,
|
|
recheck_days: int | None = None,
|
|
) -> None:
|
|
"""Change the settings that are given. A recheck_days of 0 turns the full rechecks off."""
|
|
story = _require_story(story_id)
|
|
if recheck_days is not None:
|
|
days = int(recheck_days)
|
|
if days < 0:
|
|
raise ValueError("The number of days cannot be negative")
|
|
store.set_setting(story_id, "recheck_days", days or None)
|
|
changes: dict[str, Any] = {}
|
|
if locked_fields is not None:
|
|
bad = [f for f in locked_fields if f not in METADATA_FIELDS]
|
|
if bad:
|
|
raise ValueError(f"Unknown metadata fields: {bad}")
|
|
changes["locked_fields"] = sorted(set(locked_fields))
|
|
# An unlocked field takes the site value again.
|
|
unlocked = set(story["locked_fields"]) - set(locked_fields)
|
|
changes["pinned"] = {f: v for f, v in story["pinned"].items() if f not in unlocked}
|
|
if auto_update is not None:
|
|
changes["auto_update"] = int(bool(auto_update))
|
|
store.update_story(story_id, **changes)
|
|
|
|
|
|
def _edited_value(field: str, value: Any) -> Any:
|
|
"""Check an edited metadata value and put it in the stored form."""
|
|
if field in ("authors", "tags"):
|
|
items = value if isinstance(value, list) else re.split("&" if field == "authors" else ",", str(value))
|
|
value = [str(i).strip() for i in items if str(i).strip()]
|
|
if field == "authors" and not value:
|
|
raise ValueError("A story needs at least one author")
|
|
return value
|
|
if field == "series":
|
|
if isinstance(value, list):
|
|
name, index = (value + [None, None])[:2]
|
|
name, index = str(name or "").strip(), float(index) if index not in (None, "") else None
|
|
else:
|
|
name, index = fff.parse_series(str(value).strip())
|
|
return [name, index if name else None]
|
|
value = str(value).strip()
|
|
if field == "title" and not value:
|
|
raise ValueError("A story needs a title")
|
|
if field == "pubdate" and value:
|
|
try:
|
|
datetime.fromisoformat(value)
|
|
except ValueError as e:
|
|
raise ValueError(f"The published date must be a date, for example 2024-01-31: {value}") from e
|
|
return value
|
|
|
|
|
|
def edit_metadata(story_id: int, values: dict[str, Any]) -> dict[str, Any]:
|
|
"""Keep edited metadata values. A value of None removes the edit, so the site value comes back.
|
|
|
|
The book is rebuilt when Bindery sent it to CWA before. Otherwise the next
|
|
build uses the values.
|
|
"""
|
|
story = _require_story(story_id)
|
|
bad = [f for f in values if f not in METADATA_FIELDS]
|
|
if bad:
|
|
raise ValueError(f"Unknown metadata fields: {bad}")
|
|
pinned = dict(story["pinned"])
|
|
for field, value in values.items():
|
|
if value is None:
|
|
pinned.pop(field, None)
|
|
else:
|
|
pinned[field] = _edited_value(field, value)
|
|
title = _display_title(story, pinned, story["site_values"].get("title") or story["title"])
|
|
store.update_story(story_id, pinned=pinned, title=title)
|
|
queued = None
|
|
if story["metahtml"] and (story["calibre_id"] or story["ingested_at"]):
|
|
queued = job_queue.add(Job(mode=DownloadMode.REBUILD, story_id=story_id, title=title)).to_dict()
|
|
return {"story": story_details(story_id)["story"], "job": queued}
|
|
|
|
|
|
# Defaults for the job list settings. Each pair is how many hours of finished jobs the Jobs
|
|
# panel shows, and the most it shows, before a queued job (which the panel always shows).
|
|
JOB_LIST_DEFAULTS = {
|
|
"story_jobs_hours": 36,
|
|
"story_jobs_count": 10,
|
|
"main_jobs_hours": 6,
|
|
"main_jobs_count": 25,
|
|
}
|
|
|
|
|
|
def job_list_settings() -> dict[str, int]:
|
|
result = {}
|
|
for name, default in JOB_LIST_DEFAULTS.items():
|
|
value = store.get_setting(None, name)
|
|
result[name] = int(value) if value is not None else default
|
|
return result
|
|
|
|
|
|
def get_settings() -> dict[str, Any]:
|
|
ini = config.PERSONAL_INI
|
|
return {
|
|
"recent_chapters": _setting_recent_chapters(),
|
|
"link_books": _link_books(),
|
|
"linking_available": _linking_available(),
|
|
"hold_new_links": bool(store.get_setting(None, "hold_new_links")),
|
|
"pause_uploads": _uploads_paused(),
|
|
"ingest_available": config.INGEST_DIR is not None,
|
|
"personal_ini": ini.read_text(encoding="utf-8") if ini.exists() else "",
|
|
**job_list_settings(),
|
|
}
|
|
|
|
|
|
def _validate_job_list_value(value: Any, name: str, maximum: int) -> int:
|
|
try:
|
|
n = int(value)
|
|
except (TypeError, ValueError) as e:
|
|
raise ValueError(f"{name} must be a whole number") from e
|
|
if n < 0:
|
|
raise ValueError(f"{name} cannot be negative")
|
|
if n > maximum:
|
|
raise ValueError(f"{name} cannot be more than {maximum}")
|
|
return n
|
|
|
|
|
|
def save_settings(
|
|
recent_chapters: Any = None,
|
|
personal_ini: str | None = None,
|
|
hold_new_links: bool | None = None,
|
|
pause_uploads: bool | None = None,
|
|
story_jobs_hours: Any = None,
|
|
story_jobs_count: Any = None,
|
|
main_jobs_hours: Any = None,
|
|
main_jobs_count: Any = None,
|
|
link_books: bool | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Change the settings that are given. Return all settings, and FanFicFare's warnings about personal.ini."""
|
|
warnings: list[str] = []
|
|
released = False
|
|
if link_books is not None:
|
|
if not isinstance(link_books, bool):
|
|
raise ValueError("Link books to CWA library must be true or false")
|
|
if _linking_available():
|
|
released = _link_books() != link_books
|
|
store.set_setting(None, "link_books", link_books)
|
|
for name, value in (("hold_new_links", hold_new_links), ("pause_uploads", pause_uploads)):
|
|
if value is None or (name == "pause_uploads" and config.INGEST_DIR is None):
|
|
continue
|
|
released = released or (bool(store.get_setting(None, name)) and not value)
|
|
store.set_setting(None, name, True if value else None)
|
|
if released and not _uploads_paused():
|
|
_release_held_uploads()
|
|
if recent_chapters is not None:
|
|
try:
|
|
count = int(recent_chapters)
|
|
except (TypeError, ValueError) as e:
|
|
raise ValueError("The number of recent chapters must be a whole number") from e
|
|
if count < 0:
|
|
raise ValueError("The number of recent chapters cannot be negative")
|
|
store.set_setting(None, "recent_chapters", count)
|
|
for name, value, maximum in (
|
|
("story_jobs_hours", story_jobs_hours, config.JOB_LIST_MAX_HOURS),
|
|
("story_jobs_count", story_jobs_count, config.JOB_LIST_MAX_COUNT),
|
|
("main_jobs_hours", main_jobs_hours, config.JOB_LIST_MAX_HOURS),
|
|
("main_jobs_count", main_jobs_count, config.JOB_LIST_MAX_COUNT),
|
|
):
|
|
if value is not None:
|
|
store.set_setting(None, name, _validate_job_list_value(value, name, maximum))
|
|
if personal_ini is not None:
|
|
warnings = fff.check_personal_ini(personal_ini)
|
|
# Write a new file, then replace the old one, so a job never reads half a file.
|
|
partial = config.PERSONAL_INI.with_name("personal.ini.part")
|
|
partial.write_text(personal_ini, encoding="utf-8")
|
|
partial.replace(config.PERSONAL_INI)
|
|
return {**get_settings(), "warnings": warnings}
|
|
|
|
|
|
def get_rules(story_id: int | None) -> dict[str, Any]:
|
|
"""The rules of a story, or the global rules for None, with the paragraph setting.
|
|
|
|
For a story, `paragraphs` is None when the story follows the global setting.
|
|
"""
|
|
if story_id is not None:
|
|
_require_story(story_id)
|
|
return {
|
|
"rules": store.rules(story_id),
|
|
"paragraphs": store.get_setting(story_id, "paragraphs"),
|
|
"global_paragraphs": bool(store.get_setting(None, "paragraphs")),
|
|
}
|
|
|
|
|
|
def save_rules(story_id: int | None, rules: list[dict[str, Any]], paragraphs: Any = ...) -> dict[str, Any]:
|
|
"""Store the rules. `paragraphs` is True, False, None (a story follows the global setting), or left out."""
|
|
if story_id is not None:
|
|
_require_story(story_id)
|
|
clean = normalize_rules(rules)
|
|
store.set_rules(story_id, clean)
|
|
if paragraphs is not ...:
|
|
if paragraphs is None and story_id is None:
|
|
paragraphs = False
|
|
store.set_setting(story_id, "paragraphs", None if paragraphs is None else bool(paragraphs))
|
|
return get_rules(story_id)
|
|
|
|
|
|
def preview_rules(
|
|
story_id: int | None,
|
|
rules: list[dict[str, Any]],
|
|
chapter_id: int,
|
|
source: bool = False,
|
|
paragraphs: bool | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Diff one chapter before and after the given rules (plus the global rules for a story).
|
|
|
|
`paragraphs` is the setting in the editor. None follows the global setting.
|
|
"""
|
|
chapter = store.get_chapter(chapter_id)
|
|
if not chapter or (story_id is not None and chapter["story_id"] != story_id):
|
|
raise KeyError(f"No chapter with id {chapter_id}")
|
|
clean = normalize_rules(rules)
|
|
lists = [clean] if story_id is None else [store.rules(None), clean]
|
|
if paragraphs is None:
|
|
paragraphs = bool(store.get_setting(None, "paragraphs"))
|
|
html = store.revision_html(chapter_id, chapter["current_rev"]) or ""
|
|
result = diff_html(html, apply_rules(html, compile_rules(*lists, paragraphs=bool(paragraphs))), source=source)
|
|
result["chapter"] = chapter
|
|
return result
|
|
|
|
|
|
def chapter_diff(chapter_id: int, a: int | None = None, b: int | None = None, source: bool = False) -> dict[str, Any]:
|
|
chapter = store.get_chapter(chapter_id)
|
|
if not chapter:
|
|
raise KeyError(f"No chapter with id {chapter_id}")
|
|
revisions = store.list_revisions(chapter_id)
|
|
b = b or chapter["current_rev"]
|
|
if not a:
|
|
# The revision before b. Deleted revisions leave gaps in the numbers.
|
|
others = [r["rev"] for r in revisions if r["rev"] != b]
|
|
a = max((r for r in others if r < b), default=min(others, default=b))
|
|
a, b = min(a, b), max(a, b)
|
|
old, new = store.revision_html(chapter_id, a), store.revision_html(chapter_id, b)
|
|
if old is None or new is None:
|
|
raise KeyError("No such revision")
|
|
result = diff_html(old, new, source=source)
|
|
story = store.get_story(chapter["story_id"])
|
|
result.update(chapter=chapter, revisions=revisions, a=a, b=b, story_title=story["title"] if story else "")
|
|
return result
|
|
|
|
|
|
def set_chapter_revision(chapter_id: int, rev: int) -> dict[str, Any]:
|
|
chapter = store.get_chapter(chapter_id)
|
|
if not chapter or store.revision_html(chapter_id, rev) is None:
|
|
raise KeyError("No such chapter revision")
|
|
store.update_chapter(chapter_id, current_rev=rev, changed_at=now())
|
|
return queue_story(chapter["story_id"], DownloadMode.REBUILD.value)
|
|
|
|
|
|
def delete_revision(chapter_id: int, rev: int) -> list[dict[str, Any]]:
|
|
"""Delete a revision that is not in the book and not the newest, and return the revisions that are left."""
|
|
chapter = store.get_chapter(chapter_id)
|
|
if not chapter or store.revision_html(chapter_id, rev) is None:
|
|
raise KeyError("No such chapter revision")
|
|
if rev == chapter["current_rev"]:
|
|
raise ValueError("This revision is in the book. Put another revision in the book first.")
|
|
if rev == store.list_revisions(chapter_id)[-1]["rev"]:
|
|
raise ValueError(
|
|
"This revision is the newest text from the site. "
|
|
"The next recheck compares with it, so you cannot delete it."
|
|
)
|
|
store.delete_revision(chapter_id, rev)
|
|
return store.list_revisions(chapter_id)
|
|
|
|
|
|
def set_chapters_included(story_id: int, chapter_ids: list[int], included: bool) -> dict[str, Any]:
|
|
for cid in chapter_ids:
|
|
chapter = store.get_chapter(cid)
|
|
if chapter and chapter["story_id"] == story_id:
|
|
store.update_chapter(cid, included=int(included))
|
|
return queue_story(story_id, DownloadMode.REBUILD.value)
|
|
|
|
|
|
def move_chapters(story_id: int, chapter_ids: list[int], after_id: int | None) -> dict[str, Any]:
|
|
"""Move chapters, in their current order, to directly after the chapter `after_id`, or to the start with None."""
|
|
order = [c["id"] for c in store.list_chapters(story_id)]
|
|
moving = [cid for cid in order if cid in set(chapter_ids)]
|
|
if not moving:
|
|
raise ValueError("Select the chapters to move")
|
|
if after_id is not None and after_id not in order:
|
|
raise KeyError(f"No chapter {after_id} in this story")
|
|
if after_id in moving:
|
|
raise ValueError("The chapters cannot go after a chapter that moves with them")
|
|
rest = [cid for cid in order if cid not in moving]
|
|
at = rest.index(after_id) + 1 if after_id is not None else 0
|
|
store.set_positions(story_id, rest[:at] + moving + rest[at:])
|
|
return queue_story(story_id, DownloadMode.REBUILD.value)
|
|
|
|
|
|
def delete_chapters(story_id: int, chapter_ids: list[int]) -> dict[str, Any]:
|
|
"""Delete chapters and their revisions. A chapter that the site still lists comes back with the next download."""
|
|
order = [c["id"] for c in store.list_chapters(story_id)]
|
|
deleting = [cid for cid in order if cid in set(chapter_ids)]
|
|
if not deleting:
|
|
raise ValueError("Select the chapters to delete")
|
|
if len(deleting) == len(order):
|
|
raise ValueError("A story must keep at least one chapter. To remove the story, stop tracking it")
|
|
store.delete_chapters(story_id, deleting)
|
|
return queue_story(story_id, DownloadMode.REBUILD.value)
|
|
|
|
|
|
def link_story(story_id: int, book_uuid: str | None) -> None:
|
|
"""Link a story to a library book by its uuid, or unlink it with None."""
|
|
story = _require_story(story_id)
|
|
if not book_uuid:
|
|
store.update_story(story_id, calibre_id=None, calibre_uuid=None, ingested_at=None)
|
|
_link_changed(story_id)
|
|
return
|
|
book = cwa().get_book(book_uuid)
|
|
if not book:
|
|
raise KeyError(f"No library book with uuid {book_uuid}")
|
|
other = store.get_story_by_calibre_id(book["id"])
|
|
if other and other["id"] != story_id:
|
|
raise ValueError(f"Library book {book['id']} is already linked to {other['title']}")
|
|
store.update_story(
|
|
story_id, calibre_id=book["id"], calibre_uuid=book_uuid, ingested_at=story["ingested_at"] or now()
|
|
)
|
|
if book_uuid != story["calibre_uuid"]:
|
|
_link_changed(story_id)
|
|
|
|
|
|
def approve_upload(story_id: int, book_uuid: str | None) -> dict[str, Any] | None:
|
|
"""Approve the library link of a story: the uuid of its book, or None for a new book.
|
|
|
|
Returns the queued rebuild when a built book waits for the approval.
|
|
"""
|
|
story = _require_story(story_id)
|
|
if (book_uuid or None) != story["calibre_uuid"]:
|
|
raise ValueError("The library link of this story changed. Reload the page, and check the link again.")
|
|
store.set_setting(story_id, "unapproved_link", None)
|
|
if not _uploads_paused() and store.get_setting(story_id, "upload_held") and story["metahtml"]:
|
|
return job_queue.add(Job(mode=DownloadMode.REBUILD, story_id=story_id, title=story["title"])).to_dict()
|
|
return None
|
|
|
|
|
|
def delete_story(story_id: int) -> None:
|
|
store.delete_story(story_id)
|
|
|
|
|
|
def search_library(query: str) -> list[dict[str, Any]]:
|
|
linked = {s["calibre_id"]: s["title"] for s in store.list_stories() if s["calibre_id"]}
|
|
results = []
|
|
for book in cwa().search(query)[:50]:
|
|
item = book.to_dict()
|
|
item["tracked_as"] = linked.get(book.id)
|
|
results.append(item)
|
|
return results
|
|
|
|
|
|
def library_candidates() -> list[dict[str, Any]]:
|
|
"""Library books from a story site that are not tracked yet."""
|
|
tracked_ids = {s["calibre_id"] for s in store.list_stories() if s["calibre_id"]}
|
|
return [
|
|
{"calibre_id": b.id, "uuid": b.uuid, "title": b.title, "authors": " & ".join(b.authors), "site": b.publisher}
|
|
for b in cwa().site_books(fff.is_supported_site)
|
|
if b.id not in tracked_ids
|
|
]
|
|
|
|
|
|
def build_for_download(story_id: int) -> tuple[Path, str]:
|
|
"""Build the epub from stored data, for a browser download."""
|
|
story = _require_story(story_id)
|
|
path = config.TMP_DIR / f"download-{story_id}.epub"
|
|
_build(story, _epub_overrides(story, story["pinned"]), path)
|
|
return path, story["title"]
|
|
|
|
|
|
# Metadata locks
|
|
|
|
|
|
def _field_value(field: str, meta: dict[str, Any]) -> Any:
|
|
if field == "series":
|
|
return [meta.get("series") or "", meta.get("series_index")]
|
|
return meta.get(field)
|
|
|
|
|
|
def _resolve_metadata(story: dict[str, Any], site: dict[str, Any], force_fields: list[str]) -> dict[str, Any]:
|
|
"""Return the values that replace the site values in this build.
|
|
|
|
A locked field keeps its stored value. The first download, a library
|
|
import, an edit on the story page, or a forced update sets that value.
|
|
An unlocked field keeps an edited value until the site changes that field,
|
|
or until a forced update. The newer value wins.
|
|
"""
|
|
locked = set(story["locked_fields"])
|
|
before = story["site_values"]
|
|
pinned = {}
|
|
for f, value in story["pinned"].items():
|
|
site_changed = bool(before) and _field_value(f, site) != _field_value(f, before)
|
|
if f in locked or (f not in force_fields and not site_changed):
|
|
pinned[f] = value
|
|
for f in locked:
|
|
if f in force_fields or f not in pinned:
|
|
pinned[f] = _field_value(f, site)
|
|
return pinned
|
|
|
|
|
|
def _epub_overrides(story: dict[str, Any], pinned: dict[str, Any]) -> dict[str, Any]:
|
|
return {f: v for f, v in pinned.items() if f in METADATA_FIELDS}
|
|
|
|
|
|
def _display_title(story: dict[str, Any], pinned: dict[str, Any], site_title: str) -> str:
|
|
return pinned.get("title") or site_title
|
|
|
|
|
|
# Building and delivery
|
|
|
|
|
|
def _paragraphs(story_id: int | None) -> bool:
|
|
"""Whether line breaks become paragraphs before the rules run. A story setting wins over the global one."""
|
|
value = store.get_setting(story_id, "paragraphs") if story_id is not None else None
|
|
return bool(store.get_setting(None, "paragraphs") if value is None else value)
|
|
|
|
|
|
def _steps(story_id: int) -> list[Step]:
|
|
return compile_rules(store.rules(None), store.rules(story_id), paragraphs=_paragraphs(story_id))
|
|
|
|
|
|
def _build(story: dict[str, Any], overrides: dict[str, Any], out_path: Path) -> dict[str, Any]:
|
|
if not story.get("metahtml"):
|
|
raise StoryError("Story has no site metadata yet. Download new chapters first.")
|
|
return fff.build_epub(
|
|
story["url"],
|
|
story["metahtml"],
|
|
store.chapters_for_build(story["id"]),
|
|
store.images(story["id"]),
|
|
store.cover(story["id"]),
|
|
_steps(story["id"]),
|
|
overrides,
|
|
out_path,
|
|
)
|
|
|
|
|
|
def _find_book(story_url: str, titles: list[str | None]) -> tuple[Book, Path] | None:
|
|
"""Search CWA for the book of a story. Returns the book and its downloaded epub."""
|
|
linked = {s["calibre_id"] for s in store.list_stories() if s["calibre_id"]}
|
|
return finder.find(cwa(), story_url, [t for t in titles if t], fff.normalize_url, config.TMP_DIR, linked)
|
|
|
|
|
|
def _link(story: dict[str, Any], book_id: int, book_uuid: str) -> None:
|
|
# A book that CWA imported from the first epub of the story is the book that Bindery sent. Other links need
|
|
# an approval when the setting asks for one.
|
|
if book_uuid != story["calibre_uuid"] and (story["calibre_uuid"] or not story["ingested_at"]):
|
|
_link_changed(story["id"])
|
|
story.update(calibre_id=book_id, calibre_uuid=book_uuid, ingested_at=story["ingested_at"] or now())
|
|
store.update_story(story["id"], calibre_id=book_id, calibre_uuid=book_uuid, ingested_at=story["ingested_at"])
|
|
|
|
|
|
def _linked_book(story: dict[str, Any]) -> dict[str, Any] | None:
|
|
"""The library book for a story. Links the story if CWA has a book with its URL."""
|
|
if not _can_match_books():
|
|
if story["calibre_id"] is not None:
|
|
return {"id": story["calibre_id"], "uuid": story["calibre_uuid"], "title": story["title"]}
|
|
return None
|
|
if story["calibre_uuid"]:
|
|
try:
|
|
book = cwa().get_book(story["calibre_uuid"])
|
|
except CwaError as e:
|
|
# CWA checks the book id itself, so an update can still go out.
|
|
logger.warning(f"Cannot check library book {story['calibre_id']}: {e}")
|
|
return {"id": story["calibre_id"], "uuid": story["calibre_uuid"], "title": story["title"]}
|
|
if book:
|
|
if book["id"] != story["calibre_id"]:
|
|
_link(story, book["id"], book["uuid"])
|
|
return book
|
|
logger.warning(f"Library book {story['calibre_id']} for {story['url']} is gone")
|
|
store.update_story(story["id"], calibre_id=None, calibre_uuid=None)
|
|
_link_changed(story["id"])
|
|
story.update(calibre_id=None, calibre_uuid=None)
|
|
if config.CALIBRE_DB_PATH:
|
|
book = cwa().book_by_url(story["url"])
|
|
if book:
|
|
other = store.get_story_by_calibre_id(book["id"])
|
|
if not other or other["id"] == story["id"]:
|
|
_link(story, book["id"], book["uuid"])
|
|
return book
|
|
if not library_available():
|
|
return None
|
|
# CWA keeps the title of its first import, so an edited title can differ from it.
|
|
titles = [story["title"], story["pinned"].get("title"), story["site_values"].get("title")]
|
|
found = _find_book(story["url"], list(dict.fromkeys(titles)))
|
|
if not found:
|
|
return None
|
|
book, path = found
|
|
path.unlink(missing_ok=True)
|
|
_link(story, book.id, book.uuid)
|
|
logger.info(f"Linked {story['title']} to library book {book.id}")
|
|
return book.to_dict()
|
|
|
|
|
|
NOT_FOUND = "Cannot find this story in CWA. Link to library book, or send as new book."
|
|
|
|
|
|
# Upload holds. A story gets the per-story setting "unapproved_link" when its library link changes, and loses it
|
|
# with the next upload or an approval. "upload_held" marks a built book that did not go to CWA.
|
|
|
|
|
|
def _uploads_paused() -> bool:
|
|
return config.INGEST_DIR is None or bool(store.get_setting(None, "pause_uploads"))
|
|
|
|
|
|
def _link_changed(story_id: int) -> None:
|
|
store.set_setting(story_id, "unapproved_link", True)
|
|
|
|
|
|
def _needs_approval(story_id: int) -> bool:
|
|
return bool(
|
|
_link_books() and store.get_setting(None, "hold_new_links") and store.get_setting(story_id, "unapproved_link")
|
|
)
|
|
|
|
|
|
def _release_held_uploads() -> None:
|
|
"""Rebuild the held books, so that each one goes to CWA or waits for its own reason."""
|
|
for story_id in store.story_settings("upload_held"):
|
|
story = store.get_story(story_id)
|
|
if story and story["metahtml"]:
|
|
job_queue.add(Job(mode=DownloadMode.REBUILD, story_id=story_id, title=story["title"]))
|
|
|
|
|
|
def _hold_reason(story: dict[str, Any], book: dict[str, Any] | None) -> str | None:
|
|
if _uploads_paused():
|
|
if config.INGEST_DIR is None:
|
|
return "Not sent. INGEST_DIR is not set."
|
|
return "Not sent. Turn off pause to send."
|
|
if not _needs_approval(story["id"]):
|
|
return None
|
|
if book:
|
|
return f"Not sent. Approve link to library book {book['id']}."
|
|
return "Not sent. Approve as new book."
|
|
|
|
|
|
def _deliver(story: dict[str, Any], epub: Path, book: dict[str, Any] | None, title: str, author: str) -> str:
|
|
held = _hold_reason(story, book)
|
|
if held:
|
|
store.set_setting(story["id"], "upload_held", True)
|
|
return held
|
|
# With linking on, do not create a duplicate when the library book cannot be found.
|
|
if _link_books() and not book and story["ingested_at"]:
|
|
raise StoryError(NOT_FOUND)
|
|
ingest_folder = IngestFolder(config.INGEST_DIR)
|
|
if book:
|
|
ingest_folder.send_update(epub, book["id"], title)
|
|
message = f"Updated library book {book['id']}"
|
|
else:
|
|
ingest_folder.ingest(epub, f"{title} - {author}" if author else title)
|
|
store.update_story(story["id"], ingested_at=now())
|
|
story["ingested_at"] = now()
|
|
message = "Sent to CWA" if _link_books() else "Sent to ingest folder"
|
|
store.set_setting(story["id"], "upload_held", None)
|
|
store.set_setting(story["id"], "unapproved_link", None)
|
|
return message
|
|
|
|
|
|
# Jobs
|
|
|
|
|
|
def _progress(job: Job, message: str, progress: float | None = None) -> None:
|
|
changes: dict[str, Any] = {"message": message}
|
|
if progress is not None:
|
|
changes["progress"] = progress
|
|
job_queue.update(job, **changes)
|
|
|
|
|
|
def _read_epub(epub: Path, name: str) -> fff.EpubContents:
|
|
try:
|
|
contents = fff.read_epub(epub)
|
|
except Exception as e:
|
|
raise StoryError(f"Cannot read chapters of {name}: {e}") from e
|
|
if not contents.chapters:
|
|
raise StoryError(f"{name} has no FanFicFare chapter data")
|
|
return contents
|
|
|
|
|
|
def _import_epub(story_id: int, contents: fff.EpubContents) -> int:
|
|
"""Add the chapters and images of a FanFicFare epub that the story does not have yet.
|
|
|
|
Each chapter is stored as if it had been downloaded. Chapters that the story
|
|
has already keep their stored text. The chapter order of the epub places the
|
|
new chapters, in the same way as a site update, so chapters that the site
|
|
removed go back to their place. The chapters go in with one transaction, so
|
|
an import that stops leaves no part of the epub behind.
|
|
"""
|
|
# FanFicFare maps old forms of a chapter URL to the current one.
|
|
normalize = fff.get_adapter(_require_story(story_id)["url"]).normalize_chapterurl
|
|
stored = {normalize(c["url"]): c["url"] for c in store.list_chapters(story_id)}
|
|
epub_order = []
|
|
new = []
|
|
for c in contents.chapters:
|
|
url = normalize(c["url"])
|
|
if url not in stored and url not in epub_order:
|
|
new.append({"url": url, "title": c["title"], "html": c["html"], "text_hash": text_hash(c["html"])})
|
|
epub_order.append(url)
|
|
for img in contents.images:
|
|
store.put_image(story_id, img["url"], img["ext"], img["mime"], img["data"])
|
|
if new:
|
|
order = merge_chapter_lists(list(stored), epub_order, newest_last=False)
|
|
store.add_chapters(story_id, new, "import", [stored.get(u, u) for u in order])
|
|
return len(new)
|
|
|
|
|
|
def _run_import(job: Job) -> str:
|
|
tracked = store.get_story_by_calibre_id(job.calibre_id)
|
|
# A tracked story without chapters is left from an import that stopped, so the import goes on.
|
|
if tracked and store.list_chapters(tracked["id"]):
|
|
return "Book already tracked"
|
|
book_uuid = job.options.get("uuid") or ""
|
|
source = Book(
|
|
id=job.calibre_id, uuid=book_uuid, title=job.title, formats={"epub": f"opds/download/{job.calibre_id}/epub/"}
|
|
)
|
|
_progress(job, "Downloading epub from CWA")
|
|
epub = cwa().download_epub(source, config.TMP_DIR / f"import-{job.calibre_id}-{uuid.uuid4().hex[:8]}.epub")
|
|
try:
|
|
normal = next((n for n in map(fff.normalize_url, epub_urls(epub)) if n), None)
|
|
if not normal:
|
|
raise StoryError(f"{job.title} has no supported story URL")
|
|
meta = epub_metadata(epub)
|
|
title = meta["title"] or job.title
|
|
job_queue.update(job, title=title)
|
|
story = store.get_story_by_url(normal[0])
|
|
if story and story["calibre_id"] != job.calibre_id:
|
|
_link(story, job.calibre_id, book_uuid)
|
|
return f"Linked to tracked story {story['title']}"
|
|
contents = _read_epub(epub, title)
|
|
if story is None:
|
|
locked = default_locked_fields()
|
|
story_id = store.create_story(
|
|
normal[0], normal[1], title, " & ".join(meta["authors"]), locked, job.calibre_id, book_uuid or None
|
|
)
|
|
pinned = {f: _field_value(f, meta) for f in locked}
|
|
store.update_story(story_id, pinned=pinned, ingested_at=now(), status="ok")
|
|
story = _require_story(story_id)
|
|
story_id = story["id"]
|
|
job_queue.update(job, story_id=story_id)
|
|
added = _import_epub(story_id, contents)
|
|
finally:
|
|
epub.unlink(missing_ok=True)
|
|
if job.options.get("check", True):
|
|
job_queue.add(Job(mode=DownloadMode.NEW, story_id=story_id, title=title))
|
|
return f"Imported {added} chapters"
|
|
|
|
|
|
def _run_import_file(job: Job) -> str:
|
|
path, name = Path(job.options["path"]), job.options["name"]
|
|
if not path.exists():
|
|
raise StoryError(f"Uploaded file {name} is gone. Upload it again.")
|
|
try:
|
|
contents = _read_epub(path, name)
|
|
story = store.get_story(job.story_id) if job.story_id else store.get_story_by_url(job.url)
|
|
if story is None:
|
|
meta = epub_metadata(path)
|
|
locked = default_locked_fields()
|
|
story_id = store.create_story(
|
|
job.url, fff.normalize_url(job.url)[1], meta["title"] or job.url, " & ".join(meta["authors"]), locked
|
|
)
|
|
store.update_story(story_id, pinned={f: _field_value(f, meta) for f in locked}, status="ok")
|
|
_link_changed(story_id)
|
|
story = _require_story(story_id)
|
|
job_queue.update(job, story_id=story["id"], title=story["title"])
|
|
added = _import_epub(story["id"], contents)
|
|
finally:
|
|
path.unlink(missing_ok=True)
|
|
message = f"Imported {added} chapter{'s' if added != 1 else ''} from {name}"
|
|
if not added:
|
|
message += " Already had every chapter."
|
|
store.update_story(story["id"], message=message)
|
|
# The book gets the imported chapters on the next build. A story without site
|
|
# metadata cannot be built before its first download.
|
|
if job.options.get("check", True):
|
|
job_queue.add(
|
|
Job(mode=DownloadMode.NEW, story_id=story["id"], title=story["title"], options={"build": bool(added)})
|
|
)
|
|
elif added and story["metahtml"]:
|
|
job_queue.add(Job(mode=DownloadMode.REBUILD, story_id=story["id"], title=story["title"]))
|
|
return message
|
|
|
|
|
|
def _run_rebuild(job: Job) -> str:
|
|
story = _require_story(job.story_id)
|
|
job_queue.update(job, title=story["title"])
|
|
book = _linked_book(story)
|
|
pinned = _resolve_metadata(story, story["site_values"], [])
|
|
epub = config.TMP_DIR / f"story-{story['id']}.epub"
|
|
_build(story, _epub_overrides(story, pinned), epub)
|
|
message = _deliver(story, epub, book, story["title"], story["author"])
|
|
store.update_story(story["id"], pinned=pinned, status="ok", message=message, last_changed=now())
|
|
return message
|
|
|
|
|
|
def _rules_explain_change(story_id: int, newest: dict[str, Any], html: str) -> bool:
|
|
"""Whether an imported chapter differs from the site text only by the rules.
|
|
|
|
An imported epub has the rules of its build in its text, and the site text
|
|
has none. If the rules turn the site text into the imported text, the
|
|
chapter did not change.
|
|
"""
|
|
if newest["source"] != "import":
|
|
return False
|
|
return text_hash(apply_rules(html, _steps(story_id))) == newest["text_hash"]
|
|
|
|
|
|
def _setting_recent_chapters() -> int:
|
|
return int(store.get_setting(None, "recent_chapters") or 0)
|
|
|
|
|
|
def _recent_chapters(story_id: int) -> int:
|
|
"""How many of the last chapters a new-chapter download checks again.
|
|
|
|
A story with its own interval for full rechecks checks none.
|
|
"""
|
|
if store.get_setting(story_id, "recheck_days"):
|
|
return 0
|
|
return _setting_recent_chapters()
|
|
|
|
|
|
def _run_site_job(job: Job) -> str:
|
|
story = store.get_story(job.story_id) if job.story_id else None
|
|
url = story["url"] if story else job.url
|
|
adapter = fff.get_adapter(url)
|
|
_progress(job, "Reading story page")
|
|
snap = fff.fetch_story(adapter)
|
|
if story is None:
|
|
story = store.get_story_by_url(snap.url)
|
|
if story is None:
|
|
# A new story. If the library already has it (for example from the
|
|
# FanFicFare calibre plugin), link it and keep its chapters.
|
|
found = _find_book(snap.url, [snap.title]) if _link_books() and library_available() else None
|
|
book_id, book_uuid = (found[0].id, found[0].uuid) if found else (None, None)
|
|
contents = None
|
|
if found:
|
|
_progress(job, "Reading chapters of library book")
|
|
try:
|
|
contents = _read_epub(found[1], found[0].title)
|
|
except StoryError as e:
|
|
logger.warning(str(e))
|
|
finally:
|
|
found[1].unlink(missing_ok=True)
|
|
story_id = store.create_story(
|
|
snap.url, snap.site, snap.title, snap.author, default_locked_fields(), book_id, book_uuid
|
|
)
|
|
_link_changed(story_id)
|
|
if found:
|
|
store.update_story(story_id, ingested_at=now())
|
|
if contents:
|
|
_import_epub(story_id, contents)
|
|
story = _require_story(story_id)
|
|
job.story_id = story["id"]
|
|
job_queue.update(job, title=snap.title)
|
|
story_id = story["id"]
|
|
|
|
store.update_story(story_id, metahtml=snap.metahtml, site=snap.site, author=snap.author)
|
|
story["metahtml"] = snap.metahtml
|
|
if snap.cover:
|
|
store.put_image(
|
|
story_id, snap.cover["url"], snap.cover["ext"], snap.cover["mime"], snap.cover["data"], is_cover=True
|
|
)
|
|
|
|
# Chapter URLs can change (RoyalRoad changes the slug when a title
|
|
# changes). FanFicFare maps old forms to the current one.
|
|
chapters = store.list_chapters(story_id)
|
|
known = {c["url"] for c in chapters}
|
|
for c in chapters:
|
|
normal = adapter.normalize_chapterurl(c["url"])
|
|
if normal != c["url"] and normal not in known:
|
|
store.update_chapter(c["id"], url=normal)
|
|
known.discard(c["url"])
|
|
known.add(normal)
|
|
c["url"] = normal
|
|
|
|
site_titles = {c["url"]: c["title"] for c in snap.chapters}
|
|
site_index = {url: i for i, url in enumerate(site_titles)}
|
|
by_url = {c["url"]: c for c in chapters}
|
|
dropped = 0
|
|
renamed: dict[int, str] = {} # chapter id: new title on the site
|
|
for c in chapters:
|
|
on_site = int(c["url"] in site_titles)
|
|
changes = {}
|
|
if on_site != c["on_site"]:
|
|
changes["on_site"] = on_site
|
|
if on_site and site_titles[c["url"]] != c["title"]:
|
|
renamed[c["id"]] = site_titles[c["url"]]
|
|
if job.mode == DownloadMode.FULL and c["included"] != on_site:
|
|
changes["included"] = on_site
|
|
dropped += 1 - on_site
|
|
if changes:
|
|
store.update_chapter(c["id"], **changes)
|
|
c.update(changes)
|
|
|
|
mode = job.mode
|
|
new_urls = [u for u in site_titles if u not in by_url]
|
|
if mode == DownloadMode.METADATA:
|
|
new_urls = []
|
|
refetch: list[dict[str, Any]] = []
|
|
skipped: list[str] = []
|
|
if mode in (DownloadMode.RECHECK_ALL, DownloadMode.FULL):
|
|
refetch = [c for c in chapters if c["on_site"]]
|
|
elif mode == DownloadMode.RECHECK:
|
|
new_urls = []
|
|
for c in chapters:
|
|
if c["id"] not in job.chapter_ids:
|
|
continue
|
|
if c["on_site"]:
|
|
refetch.append(c)
|
|
else:
|
|
skipped.append(fff.plain_title(c["title"]))
|
|
# A new title can come with new text. On a site that numbers its chapter
|
|
# URLs, it can also mean that the chapters moved up. So a renamed chapter is
|
|
# checked again, and it takes the new title only when that works. A metadata
|
|
# update downloads no chapters, so it only takes the new titles.
|
|
renamed_count = job.result.get("renamed", 0)
|
|
if mode == DownloadMode.METADATA:
|
|
for chapter_id, title in renamed.items():
|
|
store.update_chapter(chapter_id, title=title)
|
|
renamed_count += len(renamed)
|
|
else:
|
|
refetch += [c for c in chapters if c["id"] in renamed and c not in refetch]
|
|
# Authors most often fix their latest chapters, so a new-chapter download can check those again.
|
|
recent = _recent_chapters(story_id) if mode == DownloadMode.NEW else 0
|
|
if recent:
|
|
latest = [c for c in chapters if c["on_site"] and c["included"]][-recent:]
|
|
refetch += [c for c in latest if c not in refetch]
|
|
|
|
# Each chapter is stored as soon as it arrives, after its images. The job
|
|
# result keeps what this job did, so a job that resumes after a restart
|
|
# does not recheck the same chapters again, and its summary counts them.
|
|
new_count = job.result.get("new", 0)
|
|
changed: list[dict[str, Any]] = job.result.get("changed", [])
|
|
checked: list[int] = job.result.get("checked", [])
|
|
rechecked = len(refetch)
|
|
refetch = [c for c in refetch if c["id"] not in checked]
|
|
total = len(new_urls) + len(refetch)
|
|
done = 0
|
|
failed: list[str] = []
|
|
order = merge_chapter_lists([c["url"] for c in chapters], list(site_titles))
|
|
stored_images = store.image_urls(story_id)
|
|
|
|
def store_images() -> None:
|
|
for img in fff.collected_images(adapter):
|
|
if img["url"] not in stored_images:
|
|
store.put_image(story_id, img["url"], img["ext"], img["mime"], img["data"])
|
|
stored_images.add(img["url"])
|
|
|
|
def save_progress() -> None:
|
|
job_queue.update(
|
|
job, result={"new": new_count, "changed": changed, "checked": checked, "renamed": renamed_count}
|
|
)
|
|
|
|
for url in new_urls:
|
|
_progress(job, f"New chapter {done + 1} of {total}: {fff.plain_title(site_titles[url])}", done / max(total, 1))
|
|
done += 1
|
|
try:
|
|
html = fff.fetch_chapter(adapter, url, site_index[url])
|
|
except Exception as e:
|
|
logger.warning(f"Chapter {url} failed: {e}")
|
|
failed.append(f"{fff.plain_title(site_titles[url])}: {e}")
|
|
continue
|
|
store_images()
|
|
chapter = {"url": url, "title": site_titles[url], "html": html, "text_hash": text_hash(html)}
|
|
by_url[url] = chapter
|
|
store.add_chapters(story_id, [chapter], mode.value, [u for u in order if u in by_url])
|
|
new_count += 1
|
|
save_progress()
|
|
|
|
for c in refetch:
|
|
_progress(job, f"Rechecking chapter {done + 1} of {total}: {fff.plain_title(c['title'])}", done / max(total, 1))
|
|
try:
|
|
html = fff.fetch_chapter(adapter, c["url"], site_index[c["url"]])
|
|
except Exception as e:
|
|
logger.warning(f"Chapter {c['url']} failed: {e}")
|
|
failed.append(f"{fff.plain_title(c['title'])}: {e}")
|
|
done += 1
|
|
continue
|
|
store_images()
|
|
if c["id"] in renamed:
|
|
c["title"] = renamed[c["id"]]
|
|
store.update_chapter(c["id"], title=c["title"])
|
|
renamed_count += 1
|
|
new_hash = text_hash(html)
|
|
# The site changed when its text differs from the text that it sent last. Until then, the book keeps
|
|
# a revision that the user chose.
|
|
newest = store.list_revisions(c["id"])[-1]
|
|
if new_hash != newest["text_hash"]:
|
|
if _rules_explain_change(story_id, newest, html):
|
|
store.replace_revision(c["id"], newest["rev"], html, new_hash)
|
|
else:
|
|
rev = store.add_revision(c["id"], html, new_hash, mode.value)
|
|
changed.append({"chapter_id": c["id"], "title": c["title"], "rev": rev})
|
|
checked.append(c["id"])
|
|
save_progress()
|
|
done += 1
|
|
|
|
removed = sum(1 for c in chapters if not c["on_site"] and c["included"])
|
|
summary = [f"{new_count} new chapter{'s' if new_count != 1 else ''}"]
|
|
if renamed_count:
|
|
summary.append(f"{renamed_count} renamed")
|
|
if rechecked:
|
|
summary.append(f"{len(changed)} of {rechecked} rechecked chapters changed")
|
|
if removed:
|
|
summary.append(f"{removed} kept after removal from site")
|
|
if dropped:
|
|
summary.append(f"{dropped} taken out because site removed them")
|
|
if skipped:
|
|
summary.append(f"{len(skipped)} skipped, no longer on site")
|
|
if failed:
|
|
summary.append(f"{len(failed)} failed")
|
|
job_queue.update(
|
|
job, result={"new": new_count, "changed": changed, "failed": failed, "skipped": skipped, "removed": removed}
|
|
)
|
|
|
|
metadata_changed = snap.calibre != story["site_values"]
|
|
delivered = bool(story["calibre_id"] or story["ingested_at"])
|
|
# A resumed job builds the book, because the run before the restart can have stored chapters.
|
|
needs_build = (
|
|
new_count
|
|
or changed
|
|
or renamed_count
|
|
or job.force_fields
|
|
or metadata_changed
|
|
or not delivered
|
|
or job.options.get("build")
|
|
or job.restarts
|
|
or mode in (DownloadMode.FULL, DownloadMode.METADATA)
|
|
or (mode == DownloadMode.NEW and story["status"] == "error" and not failed)
|
|
)
|
|
if mode in (DownloadMode.RECHECK_ALL, DownloadMode.FULL):
|
|
store.set_setting(story_id, "last_full_recheck", now())
|
|
if not store.chapters_for_build(story_id):
|
|
raise StoryError("; ".join(summary) + ". No chapters downloaded.")
|
|
if not needs_build:
|
|
store.update_story(story_id, last_checked=now(), status="ok", message="No changes")
|
|
return "No changes. " + ", ".join(summary)
|
|
|
|
book = _linked_book(story)
|
|
pinned = _resolve_metadata(story, snap.calibre, job.force_fields)
|
|
title = _display_title(story, pinned, snap.title)
|
|
_progress(job, "Building epub", 1.0)
|
|
epub = config.TMP_DIR / f"story-{story_id}.epub"
|
|
_build(story, _epub_overrides(story, pinned), epub)
|
|
message = _deliver(story, epub, book, title, snap.author)
|
|
store.update_story(
|
|
story_id,
|
|
pinned=pinned,
|
|
site_values=snap.calibre,
|
|
title=title,
|
|
last_checked=now(),
|
|
last_changed=now(),
|
|
message=message,
|
|
status="error" if failed else "ok",
|
|
)
|
|
return ", ".join(summary) + ". " + message
|
|
|
|
|
|
def _run_job(job: Job) -> str:
|
|
if job.mode == DownloadMode.IMPORT:
|
|
return _run_import(job)
|
|
if job.mode == DownloadMode.REBUILD:
|
|
return _run_rebuild(job)
|
|
if job.mode == DownloadMode.IMPORT_FILE:
|
|
return _run_import_file(job)
|
|
return _run_site_job(job)
|
|
|
|
|
|
def download_loop() -> None:
|
|
"""Background thread for processing the job queue."""
|
|
logger.info("Starting download loop")
|
|
while True:
|
|
job = job_queue.get_next()
|
|
if not job:
|
|
time.sleep(2)
|
|
continue
|
|
job_queue.update(job, status=QueueStatus.RUNNING, message="Starting")
|
|
try:
|
|
message = _run_job(job)
|
|
job_queue.update(job, status=QueueStatus.DONE, message=message, progress=1.0)
|
|
logger.info(f"Job {job.id} ({job.mode.value}) for {job.title}: {message}")
|
|
except (StoryError, CwaError, EpubError, RuleError, KeyError, ValueError) as e:
|
|
_fail(job, str(e))
|
|
except Exception as e:
|
|
logger.exception(f"Job {job.id} failed: {e}")
|
|
_fail(job, f"{type(e).__name__}: {e}")
|
|
if job.mode in SITE_MODES:
|
|
time.sleep(config.SITE_DELAY)
|
|
|
|
|
|
def _fail(job: Job, message: str) -> None:
|
|
logger.warning(f"Job {job.id} ({job.mode.value}) for {job.title} failed: {message}")
|
|
job_queue.update(job, status=QueueStatus.ERROR, message=message)
|
|
if job.story_id:
|
|
store.update_story(job.story_id, status="error", message=message, last_checked=now())
|
|
|
|
|
|
def _queue_due_updates() -> None:
|
|
interval = config.UPDATE_INTERVAL_HOURS
|
|
if interval <= 0:
|
|
return
|
|
due = datetime.now() - timedelta(hours=interval)
|
|
recheck_days = store.story_settings("recheck_days")
|
|
last_full = store.story_settings("last_full_recheck")
|
|
for s in store.list_stories():
|
|
if not s["auto_update"] or job_queue.has_active(s["id"]):
|
|
continue
|
|
if s["last_checked"] and datetime.fromisoformat(s["last_checked"]) > due:
|
|
continue
|
|
# A story with a recheck interval gets a full recheck in place of its update when one is due.
|
|
mode = DownloadMode.NEW
|
|
days = recheck_days.get(s["id"])
|
|
last = last_full.get(s["id"])
|
|
if days and (not last or datetime.fromisoformat(last) <= datetime.now() - timedelta(days=days)):
|
|
mode = DownloadMode.RECHECK_ALL
|
|
job_queue.add(Job(mode=mode, story_id=s["id"], title=s["title"]))
|
|
|
|
|
|
def scheduler_loop() -> None:
|
|
logger.info(f"Starting scheduler, update interval {config.UPDATE_INTERVAL_HOURS} hours")
|
|
while True:
|
|
try:
|
|
daily_backup(store, config.BACKUP_DIR, config.PERSONAL_INI, config.BACKUP_COUNT)
|
|
except Exception as e:
|
|
logger.exception(f"Backup error: {e}")
|
|
try:
|
|
_queue_due_updates()
|
|
except Exception as e:
|
|
logger.exception(f"Scheduler error: {e}")
|
|
time.sleep(60)
|
|
|
|
|
|
def start_threads() -> None:
|
|
for target in (download_loop, scheduler_loop):
|
|
threading.Thread(target=target, daemon=True, name=target.__name__).start()
|