tim
b73b2808ca
The search shows the matches from a story site that Bindery does not track yet. The button that lists every such book stays.
1555 lines
64 KiB
Python
1555 lines
64 KiB
Python
"""Download modes, metadata locks, delivery to CWA, and the job threads."""
|
|
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import re
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any, BinaryIO
|
|
|
|
from werkzeug.security import generate_password_hash
|
|
|
|
from bindery import config, fff, filename_template
|
|
from bindery.backup import daily_backup
|
|
from bindery.cwa import Book, BookFinder, CwaClient, CwaError
|
|
from bindery.diffing import diff_html, text_hash
|
|
from bindery.epub import EpubError, epub_metadata, epub_urls
|
|
from bindery.fff import StoryError
|
|
from bindery.ingest import IngestError, IngestFolder, sanitize_filename
|
|
from bindery.jobs import METADATA_FIELDS, DownloadMode, Job, QueueStatus, job_queue
|
|
from bindery.merge import merge_chapter_lists
|
|
from bindery.rules import RuleError, Step, apply_rules, compile_rules, normalize_rules
|
|
from bindery.store import Store, now
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
store = Store(config.DB_PATH)
|
|
finder = BookFinder()
|
|
|
|
SITE_MODES = (
|
|
DownloadMode.NEW,
|
|
DownloadMode.RECHECK,
|
|
DownloadMode.RECHECK_ALL,
|
|
DownloadMode.FULL,
|
|
DownloadMode.METADATA,
|
|
)
|
|
|
|
_client_lock = threading.Lock()
|
|
_client: tuple[tuple[str, str, str], CwaClient | None] = (("", "", ""), None)
|
|
|
|
|
|
def cwa() -> CwaClient:
|
|
"""The CWA client for the configured account."""
|
|
global _client
|
|
key = (config.CWA_URL, config.CWA_USERNAME, config.CWA_PASSWORD)
|
|
with _client_lock:
|
|
if _client[0] != key or _client[1] is None:
|
|
_client = (key, CwaClient(*key))
|
|
return _client[1]
|
|
|
|
|
|
def library_available() -> bool:
|
|
return bool(config.CWA_URL)
|
|
|
|
|
|
def _linking_available() -> bool:
|
|
return bool(config.CWA_URL or config.CALIBRE_DB_PATH)
|
|
|
|
|
|
def _link_books() -> bool:
|
|
return _linking_available() and store.get_setting(None, "link_books") is not False
|
|
|
|
|
|
def default_locked_fields() -> list[str]:
|
|
return [f for f in config.DEFAULT_LOCKED_FIELDS if f in METADATA_FIELDS]
|
|
|
|
|
|
def whole_number(value: Any, name: str, most: int | None = None, negative: bool = False) -> int:
|
|
"""A value of a request as a whole number. A wrong value is an error of the request."""
|
|
try:
|
|
number = int(value)
|
|
except (TypeError, ValueError):
|
|
raise ValueError(f"{name} must be a whole number") from None
|
|
if number < 0 and not negative:
|
|
raise ValueError(f"{name} cannot be negative")
|
|
if most is not None and number > most:
|
|
raise ValueError(f"{name} cannot be more than {most}")
|
|
return number
|
|
|
|
|
|
def _check_fields(fields: Any) -> None:
|
|
bad = [f for f in fields if f not in METADATA_FIELDS]
|
|
if bad:
|
|
raise ValueError(f"Unknown metadata fields: {bad}")
|
|
|
|
|
|
# Queueing
|
|
|
|
|
|
def queue_urls(urls: list[str]) -> tuple[list[dict[str, Any]], list[str]]:
|
|
"""Queue a "new chapters" download for each URL. Returns (jobs, errors)."""
|
|
jobs, errors = [], []
|
|
for url in (u.strip() for u in urls):
|
|
if not url:
|
|
continue
|
|
normal = fff.normalize_url(url)
|
|
if not normal:
|
|
errors.append(f"Not a supported story URL: {url}")
|
|
continue
|
|
story = store.get_story_by_url(normal[0])
|
|
job = Job(
|
|
mode=DownloadMode.NEW,
|
|
story_id=story["id"] if story else None,
|
|
url=normal[0],
|
|
title=story["title"] if story else normal[0],
|
|
)
|
|
jobs.append(job_queue.add(job).to_dict())
|
|
return jobs, errors
|
|
|
|
|
|
def queue_story(
|
|
story_id: int,
|
|
mode: str,
|
|
chapter_ids: list[int] | None = None,
|
|
force_fields: list[str] | None = None,
|
|
force: bool = False,
|
|
) -> dict[str, Any]:
|
|
"""Queue a job for a story. With `force`, a rebuild also builds and sends a book that did not change."""
|
|
story = _require_story(story_id)
|
|
mode_enum = DownloadMode(mode)
|
|
if mode_enum == DownloadMode.IMPORT:
|
|
raise ValueError("Use the library import for this mode")
|
|
if mode_enum == DownloadMode.RECHECK and not chapter_ids:
|
|
raise ValueError("Select at least one chapter to recheck")
|
|
_check_fields(force_fields or [])
|
|
job = Job(
|
|
mode=mode_enum,
|
|
story_id=story_id,
|
|
title=story["title"],
|
|
chapter_ids=sorted(int(c) for c in chapter_ids or []),
|
|
force_fields=sorted(force_fields or []),
|
|
options={"force": True} if force and mode_enum == DownloadMode.REBUILD else {},
|
|
)
|
|
return job_queue.add(job).to_dict()
|
|
|
|
|
|
def _queue_rebuild(story: dict[str, Any]) -> dict[str, Any]:
|
|
return job_queue.add(Job(mode=DownloadMode.REBUILD, story_id=story["id"], title=story["title"])).to_dict()
|
|
|
|
|
|
def queue_import(books: list[dict[str, Any]], check: bool = True) -> list[dict[str, Any]]:
|
|
"""Queue imports of library books, given as {"id": ..., "uuid": ..., "title": ...}."""
|
|
jobs = []
|
|
for b in books:
|
|
book_id = int(b["id"])
|
|
jobs.append(
|
|
job_queue.add(
|
|
Job(
|
|
mode=DownloadMode.IMPORT,
|
|
calibre_id=book_id,
|
|
title=b.get("title") or f"Library book {book_id}",
|
|
options={"check": check, "uuid": str(b.get("uuid") or "")},
|
|
)
|
|
).to_dict()
|
|
)
|
|
return jobs
|
|
|
|
|
|
def _epub_story_url(path: Path) -> tuple[str, str] | None:
|
|
"""The story URL and the site of an epub from FanFicFare, or None."""
|
|
return next((n for n in map(fff.normalize_url, epub_urls(path)) if n), None)
|
|
|
|
|
|
def queue_epub_file(path: Path, filename: str, story_id: int | None = None, check: bool = True) -> dict[str, Any]:
|
|
"""Queue an import of an uploaded epub. A file without a story URL, or of another story, fails at once."""
|
|
try:
|
|
normal = _epub_story_url(path)
|
|
if not normal:
|
|
raise StoryError(f"{filename} has no supported story URL. Only epubs from FanFicFare can be imported.")
|
|
story = store.get_story_by_url(normal[0])
|
|
if story_id is not None:
|
|
story = _require_story(story_id)
|
|
if story["url"] != normal[0]:
|
|
raise StoryError(f"{filename} is an epub of another story: {normal[0]}")
|
|
except EpubError as e:
|
|
path.unlink(missing_ok=True)
|
|
raise EpubError(str(e).replace(path.name, filename)) from e
|
|
except Exception:
|
|
path.unlink(missing_ok=True)
|
|
raise
|
|
job = Job(
|
|
mode=DownloadMode.IMPORT_FILE,
|
|
story_id=story["id"] if story else None,
|
|
url=normal[0],
|
|
title=story["title"] if story else filename,
|
|
options={"path": str(path), "name": filename, "check": check},
|
|
)
|
|
return job_queue.add(job).to_dict()
|
|
|
|
|
|
def queue_status(story_id: int | None = None) -> list[dict[str, Any]]:
|
|
return job_queue.get_status(story_id)
|
|
|
|
|
|
def cancel_job(job_id: str) -> bool:
|
|
job = job_queue.get(job_id)
|
|
cancelled = job_queue.cancel(job_id)
|
|
if cancelled and job and job.mode == DownloadMode.IMPORT_FILE:
|
|
Path(job.options["path"]).unlink(missing_ok=True)
|
|
return cancelled
|
|
|
|
|
|
def remove_stale_uploads() -> None:
|
|
"""Delete the uploaded files that no waiting job needs, for example of a job that failed at each restart."""
|
|
needed = {Path(j.options["path"]).name for j in job_queue.active() if j.mode == DownloadMode.IMPORT_FILE}
|
|
for path in config.UPLOAD_DIR.glob("*.epub"):
|
|
if path.name not in needed:
|
|
logger.info(f"Deleting the uploaded file {path.name}, which no job needs")
|
|
path.unlink(missing_ok=True)
|
|
|
|
|
|
# Story, chapter and rule management for the UI
|
|
|
|
|
|
def _require_story(story_id: int) -> dict[str, Any]:
|
|
story = store.get_story(story_id)
|
|
if not story:
|
|
raise KeyError(f"No story with id {story_id}")
|
|
return story
|
|
|
|
|
|
def list_stories() -> list[dict[str, Any]]:
|
|
stories = store.list_stories()
|
|
held = store.story_settings("upload_held")
|
|
unapproved = store.story_settings("unapproved_link")
|
|
show_holds = _link_books() and store.get_setting(None, "hold_new_links") and _can_upload()
|
|
for s in stories:
|
|
s["busy"] = job_queue.has_active(s["id"])
|
|
s["upload_held"] = bool(held.get(s["id"]))
|
|
s["approval_held"] = bool(
|
|
show_holds
|
|
and s["upload_held"]
|
|
and unapproved.get(s["id"])
|
|
and (s["calibre_id"] is not None or not s["ingested_at"])
|
|
)
|
|
return stories
|
|
|
|
|
|
# The seconds that the story page waits for CWA. A job waits for the full timeout of the client.
|
|
PAGE_TIMEOUT = 5
|
|
|
|
# Only raster images. An SVG cover could hold a script that runs on Bindery's origin.
|
|
COVER_TYPES = frozenset({"image/jpeg", "image/png", "image/gif", "image/webp"})
|
|
|
|
|
|
def story_cover(story_id: int) -> dict[str, Any] | None:
|
|
"""The site cover of a story, if a browser can show it safely."""
|
|
_require_story(story_id)
|
|
cover = store.cover(story_id)
|
|
return cover if cover and cover["mime"] in COVER_TYPES else None
|
|
|
|
|
|
def story_view(story_id: int) -> dict[str, Any]:
|
|
"""A story as the story page shows it, without its chapters and its library book."""
|
|
story = _require_story(story_id)
|
|
story.pop("metahtml", None)
|
|
cover = story_cover(story_id)
|
|
# The version changes with the image, so a browser can keep a cover until the site changes it.
|
|
story["cover"] = hashlib.sha256(cover["data"]).hexdigest()[:16] if cover else None
|
|
story["recheck_days"] = store.get_setting(story_id, "recheck_days")
|
|
story["last_full_recheck"] = store.get_setting(story_id, "last_full_recheck")
|
|
story["upload_held"] = bool(store.get_setting(story_id, "upload_held"))
|
|
story["needs_approval"] = _needs_approval(story_id)
|
|
return story
|
|
|
|
|
|
def story_details(story_id: int) -> dict[str, Any]:
|
|
story = story_view(story_id)
|
|
book, library_error = None, None
|
|
if story["calibre_uuid"] and _link_books():
|
|
try:
|
|
book = cwa().get_book(story["calibre_uuid"], timeout=PAGE_TIMEOUT)
|
|
except CwaError as e:
|
|
library_error = str(e)
|
|
return {
|
|
"recent_chapters": _setting_recent_chapters(),
|
|
**_upload_state(),
|
|
"story": story,
|
|
"chapters": store.list_chapters(story_id),
|
|
"book": book,
|
|
"library_error": library_error,
|
|
}
|
|
|
|
|
|
def update_story_settings(
|
|
story_id: int,
|
|
locked_fields: list[str] | None = None,
|
|
auto_update: bool | None = None,
|
|
recheck_days: int | None = None,
|
|
) -> None:
|
|
"""Change the settings that are given. A recheck_days of 0 turns the full rechecks off."""
|
|
story = _require_story(story_id)
|
|
if recheck_days is not None:
|
|
days = whole_number(recheck_days, "The number of days")
|
|
store.set_setting(story_id, "recheck_days", days or None)
|
|
changes: dict[str, Any] = {}
|
|
if locked_fields is not None:
|
|
_check_fields(locked_fields)
|
|
changes["locked_fields"] = sorted(set(locked_fields))
|
|
# An unlocked field takes the site value again.
|
|
unlocked = set(story["locked_fields"]) - set(locked_fields)
|
|
changes["pinned"] = {f: v for f, v in story["pinned"].items() if f not in unlocked}
|
|
if auto_update is not None:
|
|
changes["auto_update"] = int(bool(auto_update))
|
|
store.update_story(story_id, **changes)
|
|
|
|
|
|
def _edited_value(field: str, value: Any) -> Any:
|
|
"""Check an edited metadata value and put it in the stored form."""
|
|
if field in ("authors", "tags"):
|
|
items = value if isinstance(value, list) else re.split("&" if field == "authors" else ",", str(value))
|
|
value = [str(i).strip() for i in items if str(i).strip()]
|
|
if field == "authors" and not value:
|
|
raise ValueError("A story needs at least one author")
|
|
return value
|
|
if field == "series":
|
|
if isinstance(value, list):
|
|
name, index = (value + [None, None])[:2]
|
|
name, index = str(name or "").strip(), float(index) if index not in (None, "") else None
|
|
else:
|
|
name, index = fff.parse_series(str(value).strip())
|
|
return [name, index if name else None]
|
|
value = str(value).strip()
|
|
if field == "title" and not value:
|
|
raise ValueError("A story needs a title")
|
|
if field == "pubdate" and value:
|
|
try:
|
|
datetime.fromisoformat(value)
|
|
except ValueError as e:
|
|
raise ValueError(f"The published date must be a date, for example 2024-01-31: {value}") from e
|
|
return value
|
|
|
|
|
|
def edit_metadata(story_id: int, values: dict[str, Any]) -> dict[str, Any]:
|
|
"""Keep edited metadata values. A value of None removes the edit, so the site value comes back.
|
|
|
|
The book is rebuilt when Bindery sent it to CWA before. Otherwise the next
|
|
build uses the values.
|
|
"""
|
|
story = _require_story(story_id)
|
|
_check_fields(values)
|
|
pinned = dict(story["pinned"])
|
|
for field, value in values.items():
|
|
if value is None:
|
|
pinned.pop(field, None)
|
|
else:
|
|
pinned[field] = _edited_value(field, value)
|
|
title = _display_title(pinned, story["site_values"].get("title") or story["title"])
|
|
store.update_story(story_id, pinned=pinned, title=title)
|
|
queued = None
|
|
if story["metahtml"] and (story["calibre_id"] or story["ingested_at"]):
|
|
queued = _queue_rebuild({**story, "title": title})
|
|
return {"story": story_view(story_id), "job": queued}
|
|
|
|
|
|
# Defaults for the job list settings. Each pair is how many hours of finished jobs the Jobs
|
|
# panel shows, and the most it shows, before a queued job (which the panel always shows).
|
|
JOB_LIST_DEFAULTS = {
|
|
"story_jobs_hours": 36,
|
|
"story_jobs_count": 10,
|
|
"main_jobs_hours": 6,
|
|
"main_jobs_count": 25,
|
|
}
|
|
|
|
|
|
def job_list_settings() -> dict[str, int]:
|
|
result = {}
|
|
for name, default in JOB_LIST_DEFAULTS.items():
|
|
value = store.get_setting(None, name)
|
|
result[name] = int(value) if value is not None else default
|
|
return result
|
|
|
|
|
|
def get_settings() -> dict[str, Any]:
|
|
ini = config.PERSONAL_INI
|
|
return {
|
|
"recent_chapters": _setting_recent_chapters(),
|
|
**_upload_state(),
|
|
"linking_available": _linking_available(),
|
|
"hold_new_links": bool(store.get_setting(None, "hold_new_links")),
|
|
"opds_enabled": bool(store.get_setting(None, "opds_enabled")),
|
|
"personal_ini": ini.read_text(encoding="utf-8") if ini.exists() else "",
|
|
**{name: _download_setting(name) for name in DOWNLOAD_DEFAULTS},
|
|
**job_list_settings(),
|
|
}
|
|
|
|
|
|
# The settings that save_settings takes.
|
|
SETTINGS = (
|
|
"recent_chapters",
|
|
"personal_ini",
|
|
"link_books",
|
|
"hold_new_links",
|
|
"pause_uploads",
|
|
"opds_enabled",
|
|
"download_template",
|
|
"download_date_format",
|
|
*JOB_LIST_DEFAULTS,
|
|
)
|
|
|
|
|
|
def save_settings(**changes: Any) -> dict[str, Any]:
|
|
"""Change the settings that are given, by the names in SETTINGS. A value of None changes nothing.
|
|
|
|
Returns all settings, and FanFicFare's warnings about personal.ini. Every value is checked before the first
|
|
one is stored, so a bad value changes nothing.
|
|
"""
|
|
unknown = set(changes) - set(SETTINGS)
|
|
if unknown:
|
|
raise ValueError(f"Unknown settings: {sorted(unknown)}")
|
|
changes = {name: value for name, value in changes.items() if value is not None}
|
|
link_books, personal_ini = changes.get("link_books"), changes.get("personal_ini")
|
|
if link_books is not None and not isinstance(link_books, bool):
|
|
raise ValueError("Link books to CWA library must be true or false")
|
|
opds_enabled = changes.get("opds_enabled")
|
|
if opds_enabled is not None and not isinstance(opds_enabled, bool):
|
|
raise ValueError("Enable OPDS must be true or false")
|
|
numbers: dict[str, int] = {}
|
|
if "recent_chapters" in changes:
|
|
numbers["recent_chapters"] = whole_number(changes["recent_chapters"], "The number of recent chapters")
|
|
for name in JOB_LIST_DEFAULTS:
|
|
if name in changes:
|
|
most = config.JOB_LIST_MAX_HOURS if name.endswith("hours") else config.JOB_LIST_MAX_COUNT
|
|
numbers[name] = whole_number(changes[name], name, most)
|
|
texts: dict[str, str] = {}
|
|
for name, label, check in (
|
|
("download_template", "The download template", filename_template.check),
|
|
("download_date_format", "The date format", filename_template.check_date_format),
|
|
):
|
|
if name in changes:
|
|
if not isinstance(changes[name], str):
|
|
raise ValueError(f"{label} must be text")
|
|
texts[name] = changes[name].strip()
|
|
check(texts[name])
|
|
warnings = fff.check_personal_ini(personal_ini) if personal_ini is not None else []
|
|
|
|
if opds_enabled is not None:
|
|
store.set_setting(None, "opds_enabled", opds_enabled)
|
|
released = False
|
|
if link_books is not None and _linking_available():
|
|
released = _link_books() != link_books
|
|
store.set_setting(None, "link_books", link_books)
|
|
for name in ("hold_new_links", "pause_uploads"):
|
|
value = changes.get(name)
|
|
if value is None or (name == "pause_uploads" and config.INGEST_DIR is None):
|
|
continue
|
|
released = released or (bool(store.get_setting(None, name)) and not value)
|
|
store.set_setting(None, name, True if value else None)
|
|
for name, number in numbers.items():
|
|
store.set_setting(None, name, number)
|
|
# An empty value restores the default.
|
|
for name, text in texts.items():
|
|
store.set_setting(None, name, text or None)
|
|
if personal_ini is not None:
|
|
# Write a new file, then replace the old one, so a job never reads half a file.
|
|
partial = config.PERSONAL_INI.with_name("personal.ini.part")
|
|
partial.write_text(personal_ini, encoding="utf-8")
|
|
partial.replace(config.PERSONAL_INI)
|
|
if released and _can_upload():
|
|
_release_held_uploads()
|
|
return {**get_settings(), "warnings": warnings}
|
|
|
|
|
|
def get_rules(story_id: int | None) -> dict[str, Any]:
|
|
"""The rules of a story, or the global rules for None, with the paragraph setting.
|
|
|
|
For a story, `paragraphs` is None when the story follows the global setting.
|
|
"""
|
|
if story_id is not None:
|
|
_require_story(story_id)
|
|
return {
|
|
"rules": store.rules(story_id),
|
|
"paragraphs": store.get_setting(story_id, "paragraphs"),
|
|
"global_paragraphs": bool(store.get_setting(None, "paragraphs")),
|
|
}
|
|
|
|
|
|
def save_rules(story_id: int | None, rules: list[dict[str, Any]], paragraphs: Any = ...) -> dict[str, Any]:
|
|
"""Store the rules. `paragraphs` is True, False, None (a story follows the global setting), or left out."""
|
|
if story_id is not None:
|
|
_require_story(story_id)
|
|
clean = normalize_rules(rules)
|
|
store.set_rules(story_id, clean)
|
|
if paragraphs is not ...:
|
|
if paragraphs is None and story_id is None:
|
|
paragraphs = False
|
|
store.set_setting(story_id, "paragraphs", None if paragraphs is None else bool(paragraphs))
|
|
return get_rules(story_id)
|
|
|
|
|
|
def preview_rules(
|
|
story_id: int | None,
|
|
rules: list[dict[str, Any]],
|
|
chapter_id: int,
|
|
source: bool = False,
|
|
paragraphs: bool | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Diff one chapter before and after the given rules (plus the global rules for a story).
|
|
|
|
`paragraphs` is the setting in the editor. None follows the global setting.
|
|
"""
|
|
chapter = store.get_chapter(chapter_id)
|
|
if not chapter or (story_id is not None and chapter["story_id"] != story_id):
|
|
raise KeyError(f"No chapter with id {chapter_id}")
|
|
clean = normalize_rules(rules)
|
|
lists = [clean] if story_id is None else [store.rules(None), clean]
|
|
if paragraphs is None:
|
|
paragraphs = bool(store.get_setting(None, "paragraphs"))
|
|
html = store.revision_html(chapter_id, chapter["current_rev"]) or ""
|
|
result = diff_html(html, apply_rules(html, compile_rules(*lists, paragraphs=bool(paragraphs))), source=source)
|
|
result["chapter"] = chapter
|
|
return result
|
|
|
|
|
|
def chapter_diff(chapter_id: int, a: int | None = None, b: int | None = None, source: bool = False) -> dict[str, Any]:
|
|
chapter = store.get_chapter(chapter_id)
|
|
if not chapter:
|
|
raise KeyError(f"No chapter with id {chapter_id}")
|
|
revisions = store.list_revisions(chapter_id)
|
|
b = b or chapter["current_rev"]
|
|
if not a:
|
|
# The revision before b. Deleted revisions leave gaps in the numbers.
|
|
others = [r["rev"] for r in revisions if r["rev"] != b]
|
|
a = max((r for r in others if r < b), default=min(others, default=b))
|
|
a, b = min(a, b), max(a, b)
|
|
old, new = store.revision_html(chapter_id, a), store.revision_html(chapter_id, b)
|
|
if old is None or new is None:
|
|
raise KeyError("No such revision")
|
|
result = diff_html(old, new, source=source)
|
|
story = store.get_story(chapter["story_id"])
|
|
result.update(chapter=chapter, revisions=revisions, a=a, b=b, story_title=story["title"] if story else "")
|
|
return result
|
|
|
|
|
|
def set_chapter_revision(chapter_id: int, rev: int) -> dict[str, Any]:
|
|
chapter = store.get_chapter(chapter_id)
|
|
if not chapter or store.revision_html(chapter_id, rev) is None:
|
|
raise KeyError("No such chapter revision")
|
|
store.update_chapter(chapter_id, current_rev=rev, changed_at=now())
|
|
return _queue_rebuild(_require_story(chapter["story_id"]))
|
|
|
|
|
|
def delete_revision(chapter_id: int, rev: int) -> list[dict[str, Any]]:
|
|
"""Delete a revision that is not in the book and not the newest, and return the revisions that are left."""
|
|
chapter = store.get_chapter(chapter_id)
|
|
if not chapter or store.revision_html(chapter_id, rev) is None:
|
|
raise KeyError("No such chapter revision")
|
|
if rev == chapter["current_rev"]:
|
|
raise ValueError("This revision is in the book. Put another revision in the book first.")
|
|
if rev == store.list_revisions(chapter_id)[-1]["rev"]:
|
|
raise ValueError(
|
|
"This revision is the newest text from the site. "
|
|
"The next recheck compares with it, so you cannot delete it."
|
|
)
|
|
store.delete_revision(chapter_id, rev)
|
|
return store.list_revisions(chapter_id)
|
|
|
|
|
|
def set_chapters_included(story_id: int, chapter_ids: list[int], included: bool) -> dict[str, Any]:
|
|
for cid in chapter_ids:
|
|
chapter = store.get_chapter(cid)
|
|
if chapter and chapter["story_id"] == story_id:
|
|
store.update_chapter(cid, included=int(included))
|
|
return _queue_rebuild(_require_story(story_id))
|
|
|
|
|
|
def move_chapters(story_id: int, chapter_ids: list[int], after_id: int | None) -> dict[str, Any]:
|
|
"""Move chapters, in their current order, to directly after the chapter `after_id`, or to the start with None."""
|
|
|
|
def arrange(order: list[int]) -> list[int]:
|
|
moving = [cid for cid in order if cid in set(chapter_ids)]
|
|
if not moving:
|
|
raise ValueError("Select the chapters to move")
|
|
if after_id is not None and after_id not in order:
|
|
raise KeyError(f"No chapter {after_id} in this story")
|
|
if after_id in moving:
|
|
raise ValueError("The chapters cannot go after a chapter that moves with them")
|
|
rest = [cid for cid in order if cid not in moving]
|
|
at = rest.index(after_id) + 1 if after_id is not None else 0
|
|
return rest[:at] + moving + rest[at:]
|
|
|
|
store.reorder_chapters(story_id, arrange)
|
|
return _queue_rebuild(_require_story(story_id))
|
|
|
|
|
|
def delete_chapters(story_id: int, chapter_ids: list[int]) -> dict[str, Any]:
|
|
"""Delete chapters and their revisions. A chapter that the site still lists comes back with the next download."""
|
|
order = [c["id"] for c in store.list_chapters(story_id)]
|
|
deleting = [cid for cid in order if cid in set(chapter_ids)]
|
|
if not deleting:
|
|
raise ValueError("Select the chapters to delete")
|
|
if len(deleting) == len(order):
|
|
raise ValueError("A story must keep at least one chapter. To remove the story, stop tracking it")
|
|
store.delete_chapters(story_id, deleting)
|
|
return _queue_rebuild(_require_story(story_id))
|
|
|
|
|
|
def link_story(story_id: int, book_uuid: str | None) -> None:
|
|
"""Link a story to a library book by its uuid, or unlink it with None."""
|
|
story = _require_story(story_id)
|
|
if not book_uuid:
|
|
store.update_story(story_id, calibre_id=None, calibre_uuid=None, ingested_at=None)
|
|
_link_changed(story_id)
|
|
return
|
|
book = cwa().get_book(book_uuid)
|
|
if not book:
|
|
raise KeyError(f"No library book with uuid {book_uuid}")
|
|
other = store.get_story_by_calibre_id(book["id"])
|
|
if other and other["id"] != story_id:
|
|
raise ValueError(f"Library book {book['id']} is already linked to {other['title']}")
|
|
store.update_story(
|
|
story_id, calibre_id=book["id"], calibre_uuid=book_uuid, ingested_at=story["ingested_at"] or now()
|
|
)
|
|
if book_uuid != story["calibre_uuid"]:
|
|
_link_changed(story_id)
|
|
|
|
|
|
def approve_upload(story_id: int, book_uuid: str | None) -> dict[str, Any] | None:
|
|
"""Approve the library link of a story: the uuid of its book, or None for a new book.
|
|
|
|
Returns the queued rebuild when a built book waits for the approval.
|
|
"""
|
|
story = _require_story(story_id)
|
|
if (book_uuid or None) != story["calibre_uuid"]:
|
|
raise ValueError("The library link of this story changed. Reload the page, and check the link again.")
|
|
store.set_setting(story_id, "unapproved_link", None)
|
|
if _can_upload() and store.get_setting(story_id, "upload_held") and story["metahtml"]:
|
|
return _queue_rebuild(story)
|
|
return None
|
|
|
|
|
|
def delete_story(story_id: int) -> None:
|
|
store.delete_story(story_id)
|
|
with _build_lock:
|
|
for built in config.TMP_DIR.glob(f"story-{story_id}-*.epub"):
|
|
built.unlink(missing_ok=True)
|
|
|
|
|
|
def search_library(query: str) -> list[dict[str, Any]]:
|
|
linked = {s["calibre_id"]: s["title"] for s in store.list_stories() if s["calibre_id"]}
|
|
results = []
|
|
for book in cwa().search(query)[:50]:
|
|
item = book.to_dict()
|
|
item["tracked_as"] = linked.get(book.id)
|
|
results.append(item)
|
|
return results
|
|
|
|
|
|
def library_candidates(query: str | None = None) -> list[dict[str, Any]]:
|
|
"""Library books from a story site that are not tracked yet. With `query`, only the books that CWA finds for it."""
|
|
tracked_ids = {s["calibre_id"] for s in store.list_stories() if s["calibre_id"]}
|
|
if query is None:
|
|
books = cwa().site_books(fff.is_supported_site)
|
|
else:
|
|
books = [b for b in cwa().search(query) if fff.is_supported_site(b.publisher) and "epub" in b.formats]
|
|
return [b.to_dict() for b in books if b.id not in tracked_ids]
|
|
|
|
|
|
def download_fields() -> dict[str, str]:
|
|
"""The fields of the download template, with the help text for each."""
|
|
fields = dict(filename_template.FIELDS)
|
|
if config.CWA_ENABLED:
|
|
fields["id"] = "The CWA book ID. For a story with no library book, the Bindery story ID"
|
|
return fields
|
|
|
|
|
|
DOWNLOAD_DEFAULTS = {
|
|
"download_template": filename_template.DEFAULT_TEMPLATE,
|
|
"download_date_format": filename_template.DEFAULT_DATE_FORMAT,
|
|
}
|
|
|
|
|
|
def _download_setting(name: str) -> str:
|
|
return store.get_setting(None, name) or DOWNLOAD_DEFAULTS[name]
|
|
|
|
|
|
OPDS_PASSWORD_LENGTH = (8, 256)
|
|
|
|
|
|
def _opds_password_name(username: str) -> str:
|
|
return f"opds_password_hash:{username.strip().casefold()}"
|
|
|
|
|
|
def opds_password_hash(username: str) -> str | None:
|
|
"""The hash of the OPDS password of a user in the proxy mode, or None."""
|
|
return store.get_setting(None, _opds_password_name(username))
|
|
|
|
|
|
def set_opds_password(username: str, password: Any) -> None:
|
|
"""Set the OPDS password of a user. Bindery keeps only the hash."""
|
|
least, most = OPDS_PASSWORD_LENGTH
|
|
if not isinstance(password, str) or not least <= len(password) <= most:
|
|
raise ValueError(f"The OPDS password must be text of {least} to {most} characters")
|
|
store.set_setting(None, _opds_password_name(username), generate_password_hash(password))
|
|
|
|
|
|
def book_metadata(story: dict[str, Any]) -> dict[str, Any]:
|
|
"""The metadata of the book of a story, with the stored title and author if the site gave none."""
|
|
meta = _book_metadata(story["site_values"], _epub_overrides(story["pinned"]))
|
|
meta["title"] = meta.get("title") or story["title"]
|
|
meta["authors"] = meta.get("authors") or [story["author"]]
|
|
return meta
|
|
|
|
|
|
def _download_name(story: dict[str, Any]) -> str:
|
|
"""The file name of a downloaded book, from the download template. If the template fails or gives no name for
|
|
this book, the name is the title."""
|
|
meta = book_metadata(story)
|
|
meta.update(publisher=story["site"], timestamp=story["created_at"], last_modified=story["last_changed"])
|
|
# Without CWA, the saved CWA book IDs do not apply.
|
|
book_id = story["calibre_id"] if config.CWA_ENABLED and story["calibre_id"] is not None else story["id"]
|
|
name = ""
|
|
try:
|
|
# {title} and {series} keep a leading article, as in Calibre with this value of its tweak
|
|
# save_template_title_series_sorting. {title_sort} moves it to the end.
|
|
date_format = _download_setting("download_date_format")
|
|
values = filename_template.prepare(meta, book_id, order="strictly_alphabetic", date_format=date_format)
|
|
name = filename_template.file_name(_download_setting("download_template"), values)
|
|
except filename_template.TemplateError as e:
|
|
logger.warning(f"The download template fails for {story['url']}: {e}")
|
|
return f"{name or sanitize_filename(story['title'])}.epub"
|
|
|
|
|
|
def build_for_download(story_id: int) -> tuple[BinaryIO, str]:
|
|
"""The epub of a story for a browser download, as an open file with its file name. Bindery builds it only if
|
|
the book changed after the last build."""
|
|
story = _require_story(story_id)
|
|
overrides = _epub_overrides(story["pinned"])
|
|
key = _build_key(story, overrides, story["site_values"])
|
|
name = _download_name(story)
|
|
# The file is opened under the lock, so a new build cannot delete it first.
|
|
with _build_lock:
|
|
return open(_built_epub(story, overrides, key, record=False), "rb"), name
|
|
|
|
|
|
# Metadata locks
|
|
|
|
|
|
def _field_value(field: str, meta: dict[str, Any]) -> Any:
|
|
if field == "series":
|
|
return [meta.get("series") or "", meta.get("series_index")]
|
|
return meta.get(field)
|
|
|
|
|
|
def _resolve_metadata(story: dict[str, Any], site: dict[str, Any], force_fields: list[str]) -> dict[str, Any]:
|
|
"""Return the values that replace the site values in this build.
|
|
|
|
A locked field keeps its stored value. The first download, a library
|
|
import, an edit on the story page, or a forced update sets that value.
|
|
An unlocked field keeps an edited value until the site changes that field,
|
|
or until a forced update. The newer value wins.
|
|
"""
|
|
locked = set(story["locked_fields"])
|
|
before = story["site_values"]
|
|
pinned = {}
|
|
for f, value in story["pinned"].items():
|
|
site_changed = bool(before) and _field_value(f, site) != _field_value(f, before)
|
|
if f in locked or (f not in force_fields and not site_changed):
|
|
pinned[f] = value
|
|
for f in locked:
|
|
if f in force_fields or f not in pinned:
|
|
pinned[f] = _field_value(f, site)
|
|
return pinned
|
|
|
|
|
|
def _epub_overrides(pinned: dict[str, Any]) -> dict[str, Any]:
|
|
return {f: v for f, v in pinned.items() if f in METADATA_FIELDS}
|
|
|
|
|
|
def _display_title(pinned: dict[str, Any], site_title: str) -> str:
|
|
return pinned.get("title") or site_title
|
|
|
|
|
|
# Building and delivery
|
|
|
|
|
|
def _paragraphs(story_id: int | None) -> bool:
|
|
"""Whether line breaks become paragraphs before the rules run. A story setting wins over the global one."""
|
|
value = store.get_setting(story_id, "paragraphs") if story_id is not None else None
|
|
return bool(store.get_setting(None, "paragraphs") if value is None else value)
|
|
|
|
|
|
def _steps(story_id: int) -> list[Step]:
|
|
return compile_rules(store.rules(None), store.rules(story_id), paragraphs=_paragraphs(story_id))
|
|
|
|
|
|
def _build(story: dict[str, Any], overrides: dict[str, Any], out_path: Path) -> dict[str, Any]:
|
|
if not story.get("metahtml"):
|
|
raise StoryError("Story has no site metadata yet. Download new chapters first.")
|
|
return fff.build_epub(
|
|
story["url"],
|
|
story["metahtml"],
|
|
store.chapters_for_build(story["id"]),
|
|
store.images(story["id"]),
|
|
store.cover(story["id"]),
|
|
_steps(story["id"]),
|
|
overrides,
|
|
out_path,
|
|
)
|
|
|
|
|
|
# Built books. The key of a story is a hash of all that goes into its epub, so the book changed when the key
|
|
# changed. The name of the built file holds the key. The story settings "built_key" and "sent_key" are the keys of
|
|
# the last build and of the book that went to CWA.
|
|
|
|
_build_lock = threading.RLock()
|
|
|
|
|
|
def _book_metadata(site: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any]:
|
|
"""The metadata of the book: the site values, with the kept values in their place."""
|
|
meta = dict(site)
|
|
for field, value in fff.effective_overrides(overrides).items():
|
|
if field == "series":
|
|
meta["series"], meta["series_index"] = value or ("", None)
|
|
else:
|
|
meta[field] = value
|
|
return meta
|
|
|
|
|
|
def _build_key(story: dict[str, Any], overrides: dict[str, Any], site: dict[str, Any]) -> str:
|
|
story_id = story["id"]
|
|
digest = hashlib.sha256()
|
|
state = [
|
|
story["url"],
|
|
_book_metadata(site, overrides),
|
|
store.rules(None),
|
|
store.rules(story_id),
|
|
_paragraphs(story_id),
|
|
store.image_sizes(story_id),
|
|
]
|
|
digest.update(json.dumps(state, sort_keys=True, default=str).encode())
|
|
for c in store.chapters_for_build(story_id):
|
|
digest.update(json.dumps([c["url"], c["title"], c["html"]]).encode())
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _built_epub(
|
|
story: dict[str, Any], overrides: dict[str, Any], key: str, force: bool = False, record: bool = True
|
|
) -> Path:
|
|
"""The epub of the story for this key. Bindery builds it only if it has no file for the key.
|
|
|
|
A download passes record=False. Its build is not a build of a job, so the next job still sees the change.
|
|
"""
|
|
path = config.TMP_DIR / f"story-{story['id']}-{key[:16]}.epub"
|
|
with _build_lock:
|
|
if force or not path.exists():
|
|
# A build under another name first, so a download never reads half a book.
|
|
partial = config.TMP_DIR / f"building-{uuid.uuid4().hex}.epub"
|
|
try:
|
|
_build(story, overrides, partial)
|
|
partial.replace(path)
|
|
finally:
|
|
partial.unlink(missing_ok=True)
|
|
for old in config.TMP_DIR.glob(f"story-{story['id']}-*.epub"):
|
|
if old != path:
|
|
old.unlink(missing_ok=True)
|
|
if record:
|
|
store.set_setting(story["id"], "built_key", key)
|
|
return path
|
|
|
|
|
|
def _find_book(story_url: str, titles: list[str | None]) -> tuple[Book, Path] | None:
|
|
"""Search CWA for the book of a story. Returns the book and its downloaded epub."""
|
|
linked = {s["calibre_id"] for s in store.list_stories() if s["calibre_id"]}
|
|
return finder.find(cwa(), story_url, [t for t in titles if t], fff.normalize_url, config.TMP_DIR, linked)
|
|
|
|
|
|
def _link(story: dict[str, Any], book_id: int, book_uuid: str) -> None:
|
|
# A book that CWA imported from the first epub of the story is the book that Bindery sent. Other links need
|
|
# an approval when the setting asks for one.
|
|
if book_uuid != story["calibre_uuid"] and (story["calibre_uuid"] or not story["ingested_at"]):
|
|
_link_changed(story["id"])
|
|
story.update(calibre_id=book_id, calibre_uuid=book_uuid, ingested_at=story["ingested_at"] or now())
|
|
store.update_story(story["id"], calibre_id=book_id, calibre_uuid=book_uuid, ingested_at=story["ingested_at"])
|
|
|
|
|
|
def _saved_book(story: dict[str, Any]) -> dict[str, Any] | None:
|
|
"""The library book that the story names, without a check that CWA still has it."""
|
|
if story["calibre_id"] is None:
|
|
return None
|
|
return {"id": story["calibre_id"], "uuid": story["calibre_uuid"], "title": story["title"], "authors": []}
|
|
|
|
|
|
def _linked_book(story: dict[str, Any]) -> dict[str, Any] | None:
|
|
"""The library book for a story. Links the story if CWA has a book with its URL."""
|
|
if not _link_books():
|
|
return _saved_book(story)
|
|
if story["calibre_uuid"]:
|
|
try:
|
|
book = cwa().get_book(story["calibre_uuid"])
|
|
except CwaError as e:
|
|
# CWA checks the book id itself, so an update can still go out.
|
|
logger.warning(f"Cannot check library book {story['calibre_id']}: {e}")
|
|
return _saved_book(story)
|
|
if book:
|
|
if book["id"] != story["calibre_id"]:
|
|
_link(story, book["id"], book["uuid"])
|
|
return book
|
|
logger.warning(f"Library book {story['calibre_id']} for {story['url']} is gone")
|
|
store.update_story(story["id"], calibre_id=None, calibre_uuid=None)
|
|
_link_changed(story["id"])
|
|
story.update(calibre_id=None, calibre_uuid=None)
|
|
if config.CALIBRE_DB_PATH:
|
|
book = cwa().book_by_url(story["url"])
|
|
if book:
|
|
other = store.get_story_by_calibre_id(book["id"])
|
|
if not other or other["id"] == story["id"]:
|
|
_link(story, book["id"], book["uuid"])
|
|
return book
|
|
if not library_available():
|
|
return None
|
|
# CWA keeps the title of its first import, so an edited title can differ from it.
|
|
titles = [story["title"], story["pinned"].get("title"), story["site_values"].get("title")]
|
|
found = _find_book(story["url"], list(dict.fromkeys(titles)))
|
|
if not found:
|
|
return None
|
|
book, path = found
|
|
path.unlink(missing_ok=True)
|
|
_link(story, book.id, book.uuid)
|
|
logger.info(f"Linked {story['title']} to library book {book.id}")
|
|
return book.to_dict()
|
|
|
|
|
|
NOT_FOUND = "Cannot find this story in CWA. Link to library book, or send as new book."
|
|
|
|
|
|
# Upload holds. A story gets the per-story setting "unapproved_link" when its library link changes, and loses it
|
|
# with the next upload or an approval. "upload_held" marks a built book that did not go to CWA.
|
|
|
|
|
|
def _paused() -> bool:
|
|
"""Whether the user paused the uploads."""
|
|
return bool(store.get_setting(None, "pause_uploads"))
|
|
|
|
|
|
def _can_upload() -> bool:
|
|
return config.INGEST_DIR is not None and not _paused()
|
|
|
|
|
|
def _upload_state() -> dict[str, bool]:
|
|
"""What the pages show about uploads. Without an ingest folder, the uploads count as paused."""
|
|
return {
|
|
"link_books": _link_books(),
|
|
"pause_uploads": not _can_upload(),
|
|
"ingest_available": config.INGEST_DIR is not None,
|
|
}
|
|
|
|
|
|
def _link_changed(story_id: int) -> None:
|
|
store.set_setting(story_id, "unapproved_link", True)
|
|
# Another book, or no book, does not have the epub that Bindery sent.
|
|
store.set_setting(story_id, "sent_key", None)
|
|
|
|
|
|
def _needs_approval(story_id: int) -> bool:
|
|
return bool(
|
|
_link_books() and store.get_setting(None, "hold_new_links") and store.get_setting(story_id, "unapproved_link")
|
|
)
|
|
|
|
|
|
def _release_held_uploads() -> None:
|
|
"""Rebuild the held books, so that each one goes to CWA or waits for its own reason."""
|
|
for story_id in store.story_settings("upload_held"):
|
|
story = store.get_story(story_id)
|
|
if story and story["metahtml"]:
|
|
_queue_rebuild(story)
|
|
|
|
|
|
def _hold_reason(story: dict[str, Any], book: dict[str, Any] | None) -> str | None:
|
|
if config.INGEST_DIR is None:
|
|
return "Not sent. INGEST_DIR is not set."
|
|
if _paused():
|
|
return "Not sent. Turn off pause to send."
|
|
if not _needs_approval(story["id"]):
|
|
return None
|
|
if book:
|
|
return f"Not sent. Approve link to library book {book['id']}."
|
|
return "Not sent. Approve as new book."
|
|
|
|
|
|
def _deliver(story: dict[str, Any], epub: Path, book: dict[str, Any] | None, title: str, author: str) -> str:
|
|
# With linking on, do not create a duplicate when the library book cannot be found.
|
|
if _link_books() and not book and story["ingested_at"]:
|
|
raise StoryError(NOT_FOUND)
|
|
ingest_folder = IngestFolder(config.INGEST_DIR)
|
|
if book:
|
|
ingest_folder.send_update(epub, book["id"], title)
|
|
message = f"Updated library book {book['id']}"
|
|
else:
|
|
ingest_folder.ingest(epub, f"{title} - {author}" if author else title)
|
|
message = "Sent to CWA" if _link_books() else "Sent to ingest folder"
|
|
# Without CWA, the ingest folder is a plain folder, so the story did not go to CWA.
|
|
if config.CWA_ENABLED:
|
|
if not book:
|
|
store.update_story(story["id"], ingested_at=now())
|
|
story["ingested_at"] = now()
|
|
store.set_setting(story["id"], "unapproved_link", None)
|
|
return message
|
|
|
|
|
|
def _send(
|
|
story: dict[str, Any], overrides: dict[str, Any], key: str, title: str, author: str, force: bool = False
|
|
) -> tuple[str, bool]:
|
|
"""Build the book if it changed, and deliver it. Returns the message, and whether the book changed.
|
|
|
|
A held book that did not change is not built again. With `force`, Bindery builds the book in each case.
|
|
"""
|
|
story_id = story["id"]
|
|
changed = key != store.get_setting(story_id, "built_key")
|
|
# Without CWA, a story has no library book, and nothing holds a book.
|
|
book = _linked_book(story) if config.CWA_ENABLED else None
|
|
held = _hold_reason(story, book) if config.CWA_ENABLED else None
|
|
if held:
|
|
if changed or force:
|
|
_built_epub(story, overrides, key, force)
|
|
store.set_setting(story_id, "upload_held", True)
|
|
return held, changed
|
|
epub = _built_epub(story, overrides, key, force)
|
|
if config.INGEST_DIR is None:
|
|
message = "Built the book"
|
|
else:
|
|
message = _deliver(story, epub, book, title, author)
|
|
store.set_setting(story_id, "sent_key", key)
|
|
store.set_setting(story_id, "upload_held", None)
|
|
return message, changed
|
|
|
|
|
|
def _done_key(story_id: int) -> Any:
|
|
"""The key of the book that is at its destination: CWA, the ingest folder or, without both, the built file."""
|
|
plain = not config.CWA_ENABLED and config.INGEST_DIR is None
|
|
return store.get_setting(story_id, "built_key" if plain else "sent_key")
|
|
|
|
|
|
def adopt_build_keys() -> None:
|
|
"""Give the stories of an older Bindery their keys, so the first check does not send each book again."""
|
|
built = store.story_settings("built_key")
|
|
held = store.story_settings("upload_held")
|
|
for listed in store.list_stories():
|
|
story_id = listed["id"]
|
|
if story_id in built or listed["status"] == "error" or job_queue.has_active(story_id):
|
|
continue
|
|
story = _require_story(story_id)
|
|
if not story["metahtml"]:
|
|
continue
|
|
key = _build_key(story, _epub_overrides(story["pinned"]), story["site_values"])
|
|
store.set_setting(story_id, "built_key", key)
|
|
if (story["calibre_id"] or story["ingested_at"]) and not held.get(story_id):
|
|
store.set_setting(story_id, "sent_key", key)
|
|
|
|
|
|
# Jobs
|
|
|
|
|
|
def _progress(job: Job, message: str, progress: float | None = None) -> None:
|
|
changes: dict[str, Any] = {"message": message}
|
|
if progress is not None:
|
|
changes["progress"] = progress
|
|
job_queue.update(job, **changes)
|
|
|
|
|
|
def _read_epub(epub: Path, name: str) -> fff.EpubContents:
|
|
try:
|
|
contents = fff.read_epub(epub)
|
|
except Exception as e:
|
|
raise StoryError(f"Cannot read chapters of {name}: {e}") from e
|
|
if not contents.chapters:
|
|
raise StoryError(f"{name} has no FanFicFare chapter data")
|
|
return contents
|
|
|
|
|
|
def _import_epub(story_id: int, contents: fff.EpubContents) -> int:
|
|
"""Add the chapters and images of a FanFicFare epub that the story does not have yet.
|
|
|
|
Each chapter is stored as if it had been downloaded. Chapters that the story
|
|
has already keep their stored text. The chapter order of the epub places the
|
|
new chapters, in the same way as a site update, so chapters that the site
|
|
removed go back to their place. The chapters go in with one transaction, so
|
|
an import that stops leaves no part of the epub behind.
|
|
"""
|
|
# FanFicFare maps old forms of a chapter URL to the current one.
|
|
normalize = fff.get_adapter(_require_story(story_id)["url"]).normalize_chapterurl
|
|
stored = {normalize(c["url"]): c["url"] for c in store.list_chapters(story_id)}
|
|
epub_order = []
|
|
new = []
|
|
for c in contents.chapters:
|
|
url = normalize(c["url"])
|
|
if url not in stored and url not in epub_order:
|
|
new.append({"url": url, "title": c["title"], "html": c["html"], "text_hash": text_hash(c["html"])})
|
|
epub_order.append(url)
|
|
for img in contents.images:
|
|
store.put_image(story_id, img["url"], img["ext"], img["mime"], img["data"])
|
|
if new:
|
|
# The store keeps a chapter under the URL that it had at its download.
|
|
in_epub = [stored.get(u, u) for u in epub_order]
|
|
store.add_chapters(story_id, new, "import", lambda have: merge_chapter_lists(have, in_epub, newest_last=False))
|
|
return len(new)
|
|
|
|
|
|
def _story_from_epub(
|
|
url: str, site: str, title: str, meta: dict[str, Any], book_id: int | None = None, book_uuid: str | None = None
|
|
) -> int:
|
|
"""Create a story with the metadata of its epub. The locked fields keep the values of the epub."""
|
|
locked = default_locked_fields()
|
|
story_id = store.create_story(url, site, title, " & ".join(meta["authors"]), locked, book_id, book_uuid)
|
|
store.update_story(story_id, pinned={f: _field_value(f, meta) for f in locked}, status="ok")
|
|
return story_id
|
|
|
|
|
|
def _run_import(job: Job) -> str:
|
|
tracked = store.get_story_by_calibre_id(job.calibre_id)
|
|
# A tracked story without chapters is left from an import that stopped, so the import goes on.
|
|
if tracked and store.list_chapters(tracked["id"]):
|
|
return "Book already tracked"
|
|
book_uuid = job.options.get("uuid") or ""
|
|
source = Book(
|
|
id=job.calibre_id, uuid=book_uuid, title=job.title, formats={"epub": f"opds/download/{job.calibre_id}/epub/"}
|
|
)
|
|
_progress(job, "Downloading epub from CWA")
|
|
epub = cwa().download_epub(source, config.TMP_DIR / f"import-{job.calibre_id}-{uuid.uuid4().hex[:8]}.epub")
|
|
try:
|
|
normal = _epub_story_url(epub)
|
|
if not normal:
|
|
raise StoryError(f"{job.title} has no supported story URL")
|
|
meta = epub_metadata(epub)
|
|
title = meta["title"] or job.title
|
|
job_queue.update(job, title=title)
|
|
story = store.get_story_by_url(normal[0])
|
|
if story and story["calibre_id"] != job.calibre_id:
|
|
_link(story, job.calibre_id, book_uuid)
|
|
return f"Linked to tracked story {story['title']}"
|
|
contents = _read_epub(epub, title)
|
|
if story is None:
|
|
story_id = _story_from_epub(normal[0], normal[1], title, meta, job.calibre_id, book_uuid or None)
|
|
store.update_story(story_id, ingested_at=now())
|
|
story = _require_story(story_id)
|
|
story_id = story["id"]
|
|
job_queue.update(job, story_id=story_id)
|
|
added = _import_epub(story_id, contents)
|
|
finally:
|
|
epub.unlink(missing_ok=True)
|
|
if job.options.get("check", True):
|
|
job_queue.add(Job(mode=DownloadMode.NEW, story_id=story_id, title=title))
|
|
return f"Imported {added} chapters"
|
|
|
|
|
|
def _run_import_file(job: Job) -> str:
|
|
path, name = Path(job.options["path"]), job.options["name"]
|
|
if not path.exists():
|
|
raise StoryError(f"Uploaded file {name} is gone. Upload it again.")
|
|
try:
|
|
contents = _read_epub(path, name)
|
|
story = store.get_story(job.story_id) if job.story_id else store.get_story_by_url(job.url)
|
|
if story is None:
|
|
meta = epub_metadata(path)
|
|
story_id = _story_from_epub(job.url, fff.normalize_url(job.url)[1], meta["title"] or job.url, meta)
|
|
_link_changed(story_id)
|
|
story = _require_story(story_id)
|
|
job_queue.update(job, story_id=story["id"], title=story["title"])
|
|
added = _import_epub(story["id"], contents)
|
|
finally:
|
|
path.unlink(missing_ok=True)
|
|
message = f"Imported {added} chapter{'s' if added != 1 else ''} from {name}"
|
|
if not added:
|
|
message += " Already had every chapter."
|
|
store.update_story(story["id"], message=message)
|
|
# The book gets the imported chapters on the next build. A story without site
|
|
# metadata cannot be built before its first download.
|
|
if job.options.get("check", True):
|
|
job_queue.add(
|
|
Job(mode=DownloadMode.NEW, story_id=story["id"], title=story["title"], options={"build": bool(added)})
|
|
)
|
|
elif added and story["metahtml"]:
|
|
_queue_rebuild(story)
|
|
return message
|
|
|
|
|
|
def _run_rebuild(job: Job) -> str:
|
|
story = _require_story(job.story_id)
|
|
job_queue.update(job, title=story["title"])
|
|
pinned = _resolve_metadata(story, story["site_values"], [])
|
|
overrides = _epub_overrides(pinned)
|
|
key = _build_key(story, overrides, story["site_values"])
|
|
force = bool(job.options.get("force"))
|
|
if not force and key == _done_key(story["id"]):
|
|
return "No changes"
|
|
message, changed = _send(story, overrides, key, story["title"], story["author"], force)
|
|
changes: dict[str, Any] = {"last_changed": now()} if changed or force else {}
|
|
store.update_story(story["id"], pinned=pinned, status="ok", message=message, **changes)
|
|
return message
|
|
|
|
|
|
def _rules_explain_change(story_id: int, newest: dict[str, Any], html: str) -> bool:
|
|
"""Whether an imported chapter differs from the site text only by the rules.
|
|
|
|
An imported epub has the rules of its build in its text, and the site text
|
|
has none. If the rules turn the site text into the imported text, the
|
|
chapter did not change.
|
|
"""
|
|
if newest["source"] != "import":
|
|
return False
|
|
return text_hash(apply_rules(html, _steps(story_id))) == newest["text_hash"]
|
|
|
|
|
|
def _setting_recent_chapters() -> int:
|
|
return int(store.get_setting(None, "recent_chapters") or 0)
|
|
|
|
|
|
def _recent_chapters(story_id: int) -> int:
|
|
"""How many of the last chapters a new-chapter download checks again.
|
|
|
|
A story with its own interval for full rechecks checks none.
|
|
"""
|
|
if store.get_setting(story_id, "recheck_days"):
|
|
return 0
|
|
return _setting_recent_chapters()
|
|
|
|
|
|
def _new_story(job: Job, snap: fff.SiteSnapshot) -> dict[str, Any]:
|
|
"""Create the story of a site job. If the library already has it (for example from the FanFicFare calibre
|
|
plugin), link it and keep its chapters."""
|
|
found = _find_book(snap.url, [snap.title]) if _link_books() and library_available() else None
|
|
book_id, book_uuid = (found[0].id, found[0].uuid) if found else (None, None)
|
|
contents = None
|
|
if found:
|
|
_progress(job, "Reading chapters of library book")
|
|
try:
|
|
contents = _read_epub(found[1], found[0].title)
|
|
except StoryError as e:
|
|
logger.warning(str(e))
|
|
finally:
|
|
found[1].unlink(missing_ok=True)
|
|
story_id = store.create_story(
|
|
snap.url, snap.site, snap.title, snap.author, default_locked_fields(), book_id, book_uuid
|
|
)
|
|
_link_changed(story_id)
|
|
if found:
|
|
store.update_story(story_id, ingested_at=now())
|
|
if contents:
|
|
_import_epub(story_id, contents)
|
|
return _require_story(story_id)
|
|
|
|
|
|
def _sync_chapters(
|
|
story_id: int, adapter: Any, site_titles: dict[str, str], full: bool
|
|
) -> tuple[list[dict[str, Any]], dict[int, str], int]:
|
|
"""Mark each stored chapter as the site lists it now.
|
|
|
|
Returns the chapters, the new site title of each renamed chapter by its id, and how many chapters a full
|
|
download took out of the book.
|
|
"""
|
|
# Chapter URLs can change (RoyalRoad changes the slug when a title
|
|
# changes). FanFicFare maps old forms to the current one.
|
|
chapters = store.list_chapters(story_id)
|
|
known = {c["url"] for c in chapters}
|
|
for c in chapters:
|
|
normal = adapter.normalize_chapterurl(c["url"])
|
|
if normal != c["url"] and normal not in known:
|
|
store.update_chapter(c["id"], url=normal)
|
|
known.discard(c["url"])
|
|
known.add(normal)
|
|
c["url"] = normal
|
|
|
|
dropped = 0
|
|
renamed: dict[int, str] = {}
|
|
for c in chapters:
|
|
on_site = int(c["url"] in site_titles)
|
|
changes = {}
|
|
if on_site != c["on_site"]:
|
|
changes["on_site"] = on_site
|
|
if on_site and site_titles[c["url"]] != c["title"]:
|
|
renamed[c["id"]] = site_titles[c["url"]]
|
|
if full and c["included"] != on_site:
|
|
changes["included"] = on_site
|
|
dropped += 1 - on_site
|
|
if changes:
|
|
store.update_chapter(c["id"], **changes)
|
|
c.update(changes)
|
|
return chapters, renamed, dropped
|
|
|
|
|
|
def _chapters_to_fetch(
|
|
job: Job, story_id: int, chapters: list[dict[str, Any]], site_titles: dict[str, str], renamed: dict[int, str]
|
|
) -> tuple[list[str], list[dict[str, Any]], list[str]]:
|
|
"""What a site job downloads: the URLs of the new chapters, and the stored chapters to check again.
|
|
|
|
Also returns the titles of the selected chapters that the site no longer lists.
|
|
"""
|
|
mode = job.mode
|
|
stored = {c["url"] for c in chapters}
|
|
new_urls = [u for u in site_titles if u not in stored]
|
|
if mode in (DownloadMode.METADATA, DownloadMode.RECHECK):
|
|
new_urls = []
|
|
selected = [c for c in chapters if c["id"] in job.chapter_ids] if mode == DownloadMode.RECHECK else []
|
|
skipped = [fff.unescape_title(c["title"]) for c in selected if not c["on_site"]]
|
|
if mode in (DownloadMode.RECHECK_ALL, DownloadMode.FULL):
|
|
refetch = [c for c in chapters if c["on_site"]]
|
|
else:
|
|
refetch = [c for c in selected if c["on_site"]]
|
|
# A new title can come with new text. On a site that numbers its chapter
|
|
# URLs, it can also mean that the chapters moved up. So a renamed chapter is
|
|
# checked again, and it takes the new title only when that works. A metadata
|
|
# update downloads no chapters, so it only takes the new titles.
|
|
if mode != DownloadMode.METADATA:
|
|
refetch += [c for c in chapters if c["id"] in renamed and c not in refetch]
|
|
# Authors most often fix their latest chapters, so a new-chapter download can check those again.
|
|
recent = _recent_chapters(story_id) if mode == DownloadMode.NEW else 0
|
|
if recent:
|
|
latest = [c for c in chapters if c["on_site"] and c["included"]][-recent:]
|
|
refetch += [c for c in latest if c not in refetch]
|
|
return new_urls, refetch, skipped
|
|
|
|
|
|
def _run_site_job(job: Job) -> str:
|
|
story = store.get_story(job.story_id) if job.story_id else None
|
|
adapter = fff.get_adapter(story["url"] if story else job.url)
|
|
_progress(job, "Reading story page")
|
|
snap = fff.fetch_story(adapter)
|
|
story = story or store.get_story_by_url(snap.url) or _new_story(job, snap)
|
|
story_id = job.story_id = story["id"]
|
|
job_queue.update(job, title=snap.title)
|
|
|
|
store.update_story(story_id, metahtml=snap.metahtml, site=snap.site, author=snap.author)
|
|
story["metahtml"] = snap.metahtml
|
|
if snap.cover:
|
|
store.put_image(
|
|
story_id, snap.cover["url"], snap.cover["ext"], snap.cover["mime"], snap.cover["data"], is_cover=True
|
|
)
|
|
|
|
mode = job.mode
|
|
site_titles = {c["url"]: c["title"] for c in snap.chapters}
|
|
site_index = {url: i for i, url in enumerate(site_titles)}
|
|
chapters, renamed, dropped = _sync_chapters(story_id, adapter, site_titles, mode == DownloadMode.FULL)
|
|
new_urls, refetch, skipped = _chapters_to_fetch(job, story_id, chapters, site_titles, renamed)
|
|
renamed_count = job.result.get("renamed", 0)
|
|
if mode == DownloadMode.METADATA:
|
|
for chapter_id, title in renamed.items():
|
|
store.update_chapter(chapter_id, title=title)
|
|
renamed_count += len(renamed)
|
|
|
|
# Each chapter is stored as soon as it arrives, after its images. The job
|
|
# result keeps what this job did, so a job that resumes after a restart
|
|
# does not recheck the same chapters again, and its summary counts them.
|
|
new_count = job.result.get("new", 0)
|
|
changed: list[dict[str, Any]] = job.result.get("changed", [])
|
|
checked: list[int] = job.result.get("checked", [])
|
|
rechecked = len(refetch)
|
|
refetch = [c for c in refetch if c["id"] not in checked]
|
|
total = len(new_urls) + len(refetch)
|
|
done = 0
|
|
failed: list[str] = []
|
|
stored_images = store.image_urls(story_id)
|
|
|
|
def fetch(url: str, title: str, action: str) -> str | None:
|
|
"""Download a chapter and store its images. A chapter that fails goes in `failed`, and gives None."""
|
|
nonlocal done
|
|
name = fff.unescape_title(title)
|
|
_progress(job, f"{action} {done + 1} of {total}: {name}", done / max(total, 1))
|
|
done += 1
|
|
try:
|
|
html = fff.fetch_chapter(adapter, url, site_index[url])
|
|
except Exception as e:
|
|
logger.warning(f"Chapter {url} failed: {e}")
|
|
failed.append(f"{name}: {e}")
|
|
return None
|
|
for img in fff.collected_images(adapter):
|
|
if img["url"] not in stored_images:
|
|
store.put_image(story_id, img["url"], img["ext"], img["mime"], img["data"])
|
|
stored_images.add(img["url"])
|
|
return html
|
|
|
|
def save_progress() -> None:
|
|
job_queue.update(
|
|
job, result={"new": new_count, "changed": changed, "checked": checked, "renamed": renamed_count}
|
|
)
|
|
|
|
for url in new_urls:
|
|
html = fetch(url, site_titles[url], "New chapter")
|
|
if html is None:
|
|
continue
|
|
chapter = {"url": url, "title": site_titles[url], "html": html, "text_hash": text_hash(html)}
|
|
# The chapter goes in the order that the story has now. A move or a delete by hand during this job stays.
|
|
store.add_chapters(story_id, [chapter], mode.value, lambda have: merge_chapter_lists(have, list(site_titles)))
|
|
new_count += 1
|
|
save_progress()
|
|
|
|
for c in refetch:
|
|
html = fetch(c["url"], c["title"], "Rechecking chapter")
|
|
revisions = store.list_revisions(c["id"]) if html is not None else []
|
|
if not revisions: # the download failed, or the user deleted the chapter during this job
|
|
continue
|
|
if c["id"] in renamed:
|
|
c["title"] = renamed[c["id"]]
|
|
store.update_chapter(c["id"], title=c["title"])
|
|
renamed_count += 1
|
|
new_hash = text_hash(html)
|
|
# The site changed when its text differs from the text that it sent last. Until then, the book keeps
|
|
# a revision that the user chose.
|
|
newest = revisions[-1]
|
|
if new_hash != newest["text_hash"]:
|
|
if _rules_explain_change(story_id, newest, html):
|
|
store.replace_revision(c["id"], newest["rev"], html, new_hash)
|
|
else:
|
|
rev = store.add_revision(c["id"], html, new_hash, mode.value)
|
|
changed.append({"chapter_id": c["id"], "title": c["title"], "rev": rev})
|
|
checked.append(c["id"])
|
|
save_progress()
|
|
|
|
removed = sum(1 for c in chapters if not c["on_site"] and c["included"])
|
|
parts = [
|
|
(True, f"{new_count} new chapter{'s' if new_count != 1 else ''}"),
|
|
(renamed_count, f"{renamed_count} renamed"),
|
|
(rechecked, f"{len(changed)} of {rechecked} rechecked chapters changed"),
|
|
(removed, f"{removed} kept after removal from site"),
|
|
(dropped, f"{dropped} taken out because site removed them"),
|
|
(skipped, f"{len(skipped)} skipped, no longer on site"),
|
|
(failed, f"{len(failed)} failed"),
|
|
]
|
|
summary = ", ".join(text for show, text in parts if show)
|
|
job_queue.update(
|
|
job, result={"new": new_count, "changed": changed, "failed": failed, "skipped": skipped, "removed": removed}
|
|
)
|
|
|
|
if mode in (DownloadMode.RECHECK_ALL, DownloadMode.FULL):
|
|
store.set_setting(story_id, "last_full_recheck", now())
|
|
if not store.chapters_for_build(story_id):
|
|
raise StoryError(f"{summary}. No chapters downloaded.")
|
|
|
|
# The book is built and sent only when its chapters, rules or metadata changed. A job that resumes, or that
|
|
# follows a failed job, finds the changes of the earlier run through the key.
|
|
pinned = _resolve_metadata(story, snap.calibre, job.force_fields)
|
|
title = _display_title(pinned, snap.title)
|
|
overrides = _epub_overrides(pinned)
|
|
key = _build_key(story, overrides, snap.calibre)
|
|
changes: dict[str, Any] = {"pinned": pinned, "site_values": snap.calibre, "title": title, "last_checked": now()}
|
|
if key == _done_key(story_id):
|
|
store.update_story(story_id, **changes, status="ok", message="No changes")
|
|
return f"No changes. {summary}"
|
|
|
|
_progress(job, "Building epub", 1.0)
|
|
message, book_changed = _send(story, overrides, key, title, snap.author)
|
|
if book_changed:
|
|
changes["last_changed"] = now()
|
|
store.update_story(story_id, **changes, message=message, status="error" if failed else "ok")
|
|
return f"{summary}. {message}"
|
|
|
|
|
|
def _run_job(job: Job) -> str:
|
|
if job.mode == DownloadMode.IMPORT:
|
|
return _run_import(job)
|
|
if job.mode == DownloadMode.REBUILD:
|
|
return _run_rebuild(job)
|
|
if job.mode == DownloadMode.IMPORT_FILE:
|
|
return _run_import_file(job)
|
|
return _run_site_job(job)
|
|
|
|
|
|
def _run_next_job() -> Job | None:
|
|
"""Run the job at the front of the queue. Returns the job, or None if no job waits."""
|
|
job = job_queue.get_next()
|
|
if not job:
|
|
return None
|
|
job_queue.update(job, status=QueueStatus.RUNNING, message="Starting")
|
|
try:
|
|
message = _run_job(job)
|
|
job_queue.update(job, status=QueueStatus.DONE, message=message, progress=1.0)
|
|
logger.info(f"Job {job.id} ({job.mode.value}) for {job.title}: {message}")
|
|
except (StoryError, CwaError, EpubError, IngestError, RuleError, KeyError, ValueError) as e:
|
|
_fail(job, str(e))
|
|
except Exception as e:
|
|
logger.exception(f"Job {job.id} failed: {e}")
|
|
_fail(job, f"{type(e).__name__}: {e}")
|
|
return job
|
|
|
|
|
|
def download_loop() -> None:
|
|
"""Background thread for processing the job queue."""
|
|
logger.info("Starting download loop")
|
|
while True:
|
|
try:
|
|
job = _run_next_job()
|
|
except Exception as e:
|
|
# The queue or the store failed, for example on a full disk. The loop goes on, because no job runs
|
|
# after it stops.
|
|
logger.exception(f"Job loop error: {e}")
|
|
job = None
|
|
if job is None:
|
|
time.sleep(2)
|
|
elif job.mode in SITE_MODES:
|
|
time.sleep(config.SITE_DELAY)
|
|
|
|
|
|
def _fail(job: Job, message: str) -> None:
|
|
logger.warning(f"Job {job.id} ({job.mode.value}) for {job.title} failed: {message}")
|
|
job_queue.update(job, status=QueueStatus.ERROR, message=message)
|
|
if job.story_id:
|
|
store.update_story(job.story_id, status="error", message=message, last_checked=now())
|
|
|
|
|
|
def _queue_due_updates() -> None:
|
|
interval = config.UPDATE_INTERVAL_HOURS
|
|
if interval <= 0:
|
|
return
|
|
due = datetime.now() - timedelta(hours=interval)
|
|
recheck_days = store.story_settings("recheck_days")
|
|
last_full = store.story_settings("last_full_recheck")
|
|
for s in store.list_stories():
|
|
if not s["auto_update"] or job_queue.has_active(s["id"]):
|
|
continue
|
|
if s["last_checked"] and datetime.fromisoformat(s["last_checked"]) > due:
|
|
continue
|
|
# A story with a recheck interval gets a full recheck in place of its update when one is due.
|
|
mode = DownloadMode.NEW
|
|
days = recheck_days.get(s["id"])
|
|
last = last_full.get(s["id"])
|
|
if days and (not last or datetime.fromisoformat(last) <= datetime.now() - timedelta(days=days)):
|
|
mode = DownloadMode.RECHECK_ALL
|
|
job_queue.add(Job(mode=mode, story_id=s["id"], title=s["title"]))
|
|
|
|
|
|
def scheduler_loop() -> None:
|
|
logger.info(f"Starting scheduler, update interval {config.UPDATE_INTERVAL_HOURS} hours")
|
|
while True:
|
|
try:
|
|
daily_backup(store, config.BACKUP_DIR, config.PERSONAL_INI, config.BACKUP_COUNT)
|
|
except Exception as e:
|
|
logger.exception(f"Backup error: {e}")
|
|
try:
|
|
_queue_due_updates()
|
|
except Exception as e:
|
|
logger.exception(f"Scheduler error: {e}")
|
|
time.sleep(60)
|
|
|
|
|
|
_threads: list[threading.Thread] = []
|
|
|
|
|
|
def start_threads() -> None:
|
|
for prepare in (adopt_build_keys, remove_stale_uploads):
|
|
try:
|
|
prepare()
|
|
except Exception as e:
|
|
logger.exception(f"Error in {prepare.__name__}: {e}")
|
|
for target in (download_loop, scheduler_loop):
|
|
thread = threading.Thread(target=target, daemon=True, name=target.__name__)
|
|
thread.start()
|
|
_threads.append(thread)
|
|
|
|
|
|
def stopped_threads() -> list[str]:
|
|
"""The names of the job threads that stopped. Bindery must restart to start them again."""
|
|
return [t.name for t in _threads if not t.is_alive()]
|