diff --git a/bindery/config.py b/bindery/config.py index d1134d3..846571f 100644 --- a/bindery/config.py +++ b/bindery/config.py @@ -68,7 +68,9 @@ TRUST_PROXY = _bool("TRUST_PROXY") START_THREADS = _bool("BINDERY_START_THREADS", "true") SITE_DELAY = 5 # seconds between two jobs that go to a story site -JOB_HISTORY = 60 * 60 # seconds that finished jobs stay in the job list +JOB_LIST_MAX_HOURS = 24 * 30 # the highest allowed value of a "jobs from the last N hours" setting +JOB_LIST_MAX_COUNT = 500 # the highest allowed value of a "to a maximum of N jobs" setting +JOB_HISTORY = JOB_LIST_MAX_HOURS * 60 * 60 # seconds; finished jobs stay this long, for the settings above DB_PATH = CONFIG_DIR / "bindery.db" UPLOAD_DIR = CONFIG_DIR / "uploads" # uploaded epubs wait here for their job, also over a restart diff --git a/bindery/orchestrator.py b/bindery/orchestrator.py index cce27d8..22510b2 100644 --- a/bindery/orchestrator.py +++ b/bindery/orchestrator.py @@ -293,6 +293,24 @@ def edit_metadata(story_id: int, values: dict[str, Any]) -> dict[str, Any]: return {"story": story_details(story_id)["story"], "job": queued} +# Defaults for the job list settings. Each pair is how many hours of finished jobs the Jobs +# panel shows, and the most it shows, before a queued job (which the panel always shows). +JOB_LIST_DEFAULTS = { + "story_jobs_hours": 36, + "story_jobs_count": 10, + "main_jobs_hours": 6, + "main_jobs_count": 25, +} + + +def job_list_settings() -> dict[str, int]: + result = {} + for name, default in JOB_LIST_DEFAULTS.items(): + value = store.get_setting(None, name) + result[name] = int(value) if value is not None else default + return result + + def get_settings() -> dict[str, Any]: ini = config.PERSONAL_INI return { @@ -300,14 +318,31 @@ def get_settings() -> dict[str, Any]: "hold_new_links": bool(store.get_setting(None, "hold_new_links")), "pause_uploads": _uploads_paused(), "personal_ini": ini.read_text(encoding="utf-8") if ini.exists() else "", + **job_list_settings(), } +def _validate_job_list_value(value: Any, name: str, maximum: int) -> int: + try: + n = int(value) + except (TypeError, ValueError) as e: + raise ValueError(f"{name} must be a whole number") from e + if n < 0: + raise ValueError(f"{name} cannot be negative") + if n > maximum: + raise ValueError(f"{name} cannot be more than {maximum}") + return n + + def save_settings( recent_chapters: Any = None, personal_ini: str | None = None, hold_new_links: bool | None = None, pause_uploads: bool | None = None, + story_jobs_hours: Any = None, + story_jobs_count: Any = None, + main_jobs_hours: Any = None, + main_jobs_count: Any = None, ) -> dict[str, Any]: """Change the settings that are given. Return all settings, and FanFicFare's warnings about personal.ini.""" warnings: list[str] = [] @@ -327,6 +362,14 @@ def save_settings( if count < 0: raise ValueError("The number of recent chapters cannot be negative") store.set_setting(None, "recent_chapters", count) + for name, value, maximum in ( + ("story_jobs_hours", story_jobs_hours, config.JOB_LIST_MAX_HOURS), + ("story_jobs_count", story_jobs_count, config.JOB_LIST_MAX_COUNT), + ("main_jobs_hours", main_jobs_hours, config.JOB_LIST_MAX_HOURS), + ("main_jobs_count", main_jobs_count, config.JOB_LIST_MAX_COUNT), + ): + if value is not None: + store.set_setting(None, name, _validate_job_list_value(value, name, maximum)) if personal_ini is not None: warnings = fff.check_personal_ini(personal_ini) # Write a new file, then replace the old one, so a job never reads half a file. diff --git a/bindery/static/js/jobs.js b/bindery/static/js/jobs.js index 3a25865..2706f9f 100644 --- a/bindery/static/js/jobs.js +++ b/bindery/static/js/jobs.js @@ -95,20 +95,37 @@ function jobItem(job) { ); } +// A queued or running job always shows, however old or far down the list. A job that is done, +// failed or cancelled shows only within the last `hours` hours, and at most `count` of them, +// newest first. +function visibleJobs(list, { hours, count }) { + const byNewest = (a, b) => new Date(b.created || 0) - new Date(a.created || 0); + const active = list.filter((j) => ACTIVE.has(j.status)); + const others = list.filter((j) => !ACTIVE.has(j.status)); + const cutoff = Date.now() - hours * 3600000; + // A missing updated time never happens outside a test fixture. Showing the job is the safer default. + const recent = others + .filter((j) => !j.updated || new Date(j.updated).getTime() >= cutoff) + .sort(byNewest) + .slice(0, count); + return [...active, ...recent].sort(byNewest); +} + // A job list for a page that polls. The list keeps its scroll box and changes only the jobs in it, so an update // does not move the list back to the top. A poll without changes leaves the list alone. -export function jobPanel() { +export function jobPanel(limits = { hours: 24 * 30, count: 500 }) { const box = h('div', { class: 'jobs' }); const empty = h('p', { class: 'empty' }, 'No jobs.'); const el = h('div'); let shown = null; let child = null; const render = (list) => { - const key = JSON.stringify(list); + const visible = visibleJobs(list, limits); + const key = JSON.stringify(visible); if (key === shown) return; shown = key; - fill(box, list.map(jobItem)); - const next = list.length ? box : empty; + fill(box, visible.map(jobItem)); + const next = visible.length ? box : empty; if (child !== next) { child = next; fill(el, child); diff --git a/bindery/static/js/settings.js b/bindery/static/js/settings.js index 1fbea25..b17a0bd 100644 --- a/bindery/static/js/settings.js +++ b/bindery/static/js/settings.js @@ -6,11 +6,17 @@ import { globalRulesCard } from './rules.js'; const INI_DOCS = 'https://github.com/JimmXinu/FanFicFare/wiki/INI-File'; const DEFAULTS_INI = 'https://github.com/JimmXinu/FanFicFare/blob/main/fanficfare/defaults.ini'; -export function settingsPage(main) { +export function settingsPage(main, _params, config) { document.title = 'Settings · Bindery'; const recent = h('input', { id: 'recent-chapters', type: 'number', min: 0, step: 1 }); const holdNewLinks = checkbox(false, () => {}); const pauseUploads = checkbox(false, () => {}); + const hoursField = (id) => h('input', { id, type: 'number', min: 0, max: config.job_list_max_hours, step: 1 }); + const countField = (id) => h('input', { id, type: 'number', min: 0, max: config.job_list_max_count, step: 1 }); + const storyJobsHours = hoursField('story-jobs-hours'); + const storyJobsCount = countField('story-jobs-count'); + const mainJobsHours = hoursField('main-jobs-hours'); + const mainJobsCount = countField('main-jobs-count'); const ini = h('textarea', { class: 'ini', rows: 24, @@ -37,6 +43,10 @@ export function settingsPage(main) { holdNewLinks.checked = settings.hold_new_links; pauseUploads.checked = settings.pause_uploads; ini.value = settings.personal_ini; + storyJobsHours.value = settings.story_jobs_hours; + storyJobsCount.value = settings.story_jobs_count; + mainJobsHours.value = settings.main_jobs_hours; + mainJobsCount.value = settings.main_jobs_count; }) .catch(showError); @@ -97,6 +107,48 @@ export function settingsPage(main) { }, { class: 'primary' }), ), ), + h( + 'section', + { class: 'card' }, + h('h2', {}, 'Jobs list'), + h( + 'div', + { class: 'controls' }, + h('label', { for: 'story-jobs-hours' }, "A story page's job list shows jobs from the last"), + storyJobsHours, + h('label', { for: 'story-jobs-count' }, 'hours, to a maximum of'), + storyJobsCount, + 'jobs.', + ), + h( + 'div', + { class: 'controls' }, + h('label', { for: 'main-jobs-hours' }, "The story list's job list shows jobs from the last"), + mainJobsHours, + h('label', { for: 'main-jobs-count' }, 'hours, to a maximum of'), + mainJobsCount, + 'jobs.', + ), + h( + 'p', + { class: 'hint' }, + `A queued or running job always shows. The most either field allows is ${config.job_list_max_hours} hours ` + + `and ${config.job_list_max_count} jobs.`, + ), + h( + 'div', + { class: 'controls' }, + button('Save', async () => { + await api('settings', 'PUT', { + story_jobs_hours: storyJobsHours.value, + story_jobs_count: storyJobsCount.value, + main_jobs_hours: mainJobsHours.value, + main_jobs_count: mainJobsCount.value, + }); + toast('Saved'); + }, { class: 'primary' }), + ), + ), globalRulesCard(), h( 'section', diff --git a/bindery/static/js/stories.js b/bindery/static/js/stories.js index dac29b4..bdc5101 100644 --- a/bindery/static/js/stories.js +++ b/bindery/static/js/stories.js @@ -191,7 +191,7 @@ export function storiesPage(main, _params, config) { let stories = null; let filter = ''; const table = h('div', {}, h('p', { class: 'empty' }, 'Loading…')); - const jobs = jobPanel(); + const jobs = jobPanel({ hours: config.main_jobs_hours, count: config.main_jobs_count }); const render = () => { const q = filter.trim().toLowerCase(); diff --git a/bindery/static/js/story.js b/bindery/static/js/story.js index 47d2c71..6c83854 100644 --- a/bindery/static/js/story.js +++ b/bindery/static/js/story.js @@ -88,7 +88,7 @@ export function storyPage(main, { storyId }, config) { const updates = h('div'); const chapters = h('div'); const selectAll = h('span', { class: 'chapter-select-all' }); - const jobs = jobPanel(); + const jobs = jobPanel({ hours: config.story_jobs_hours, count: config.story_jobs_count }); const linked = h('div'); const searchResults = h('div'); diff --git a/bindery/web.py b/bindery/web.py index f60d52e..7b41f2a 100644 --- a/bindery/web.py +++ b/bindery/web.py @@ -211,6 +211,9 @@ def app_config() -> Any: "mode_labels": MODE_LABELS, "library_available": orchestrator.library_available(), "update_interval": config.UPDATE_INTERVAL_HOURS, + "job_list_max_hours": config.JOB_LIST_MAX_HOURS, + "job_list_max_count": config.JOB_LIST_MAX_COUNT, + **orchestrator.job_list_settings(), } @@ -219,7 +222,14 @@ def settings() -> Any: if request.method == "PUT": data = _json() return orchestrator.save_settings( - data.get("recent_chapters"), data.get("personal_ini"), data.get("hold_new_links"), data.get("pause_uploads") + data.get("recent_chapters"), + data.get("personal_ini"), + data.get("hold_new_links"), + data.get("pause_uploads"), + data.get("story_jobs_hours"), + data.get("story_jobs_count"), + data.get("main_jobs_hours"), + data.get("main_jobs_count"), ) return orchestrator.get_settings() diff --git a/tests/js/flows.test.mjs b/tests/js/flows.test.mjs index 0c0148b..5b57d54 100644 --- a/tests/js/flows.test.mjs +++ b/tests/js/flows.test.mjs @@ -15,6 +15,7 @@ let api, main, details, cleanups; const config = { metadata_fields: { title: 'Title', authors: 'Authors', comments: 'Description', series: 'Series', pubdate: 'Published', tags: 'Tags', languages: 'Language' }, mode_labels: { new: 'New chapters' }, library_available: true, update_interval: 2, + job_list_max_hours: 720, job_list_max_count: 500, }; const rule = { pattern: 'before', replacement: 'after', flags: 'i', note: '', enabled: true }; const emptyRules = { rules: [], paragraphs: null, global_paragraphs: true }; @@ -49,7 +50,7 @@ const writes = () => api.requests.filter((r) => r.method !== 'GET'); for (const value of ['7', '']) { test(`settings save converts ${JSON.stringify(value)} to a count`, async () => { - settingsPage(main); + settingsPage(main, {}, config); await settle(); api.on('PUT', 'settings', {}); field(main, 'recent-chapters').value = value; @@ -60,7 +61,7 @@ for (const value of ['7', '']) { } test('personal.ini save reports warnings and clears them after a clean save', async () => { - settingsPage(main); + settingsPage(main, {}, config); await settle(); const card = section(main, 'FanFicFare settings (personal.ini)'); field(main, 'personal.ini').value = '[defaults]\nis_adult: true'; @@ -74,7 +75,7 @@ test('personal.ini save reports warnings and clears them after a clean save', as }); test('failed settings saves retain edits and re-enable the action', async () => { - settingsPage(main); + settingsPage(main, {}, config); await settle(); const card = section(main, 'FanFicFare settings (personal.ini)'); field(main, 'personal.ini').value = 'invalid ini'; @@ -85,6 +86,28 @@ test('failed settings saves retain edits and re-enable the action', async () => assert.match(ui.toasts.textContent, /Invalid settings/); }); +test('the jobs list settings load and save all four fields', async () => { + api.on('GET', 'settings', { + recent_chapters: 0, personal_ini: '', + story_jobs_hours: 36, story_jobs_count: 10, main_jobs_hours: 6, main_jobs_count: 25, + }); + settingsPage(main, {}, config); + await settle(); + const card = section(main, 'Jobs list'); + assert.equal(field(main, 'story-jobs-hours').value, 36); + assert.equal(field(main, 'main-jobs-count').value, 25); + field(main, 'story-jobs-hours').value = '48'; + field(main, 'story-jobs-count').value = '20'; + field(main, 'main-jobs-hours').value = '12'; + field(main, 'main-jobs-count').value = '50'; + api.on('PUT', 'settings', {}); + await click(card, 'Save'); + assert.deepEqual(api.sent('PUT', 'settings')[0].body, { + story_jobs_hours: '48', story_jobs_count: '20', main_jobs_hours: '12', main_jobs_count: '50', + }); + assert.match(ui.toasts.textContent, /Saved/); +}); + test('adding stories keeps input on rejection and clears it after a mixed result', async () => { await mountStories({ ...config, library_available: false }); const urls = nodes(main).find((el) => el.tagName === 'textarea'); @@ -491,6 +514,46 @@ test('chapter comparisons and revision changes send the chosen revision numbers' assert.equal(api.sent('GET', 'chapters/2/diff?source=1').length, 1); }); +test('a job panel shows a queued job however old, but limits others by age', () => { + const { el, render } = jobPanel({ hours: 1, count: 10 }); + const now = Date.now(); + const old = new Date(now - 2 * 3600000).toISOString(); + const recent = new Date(now - 30 * 60000).toISOString(); + render([ + { id: 'a', status: 'queued', title: 'Old queued', updated: old, created: old }, + { id: 'b', status: 'done', title: 'Old done', updated: old, created: old }, + { id: 'c', status: 'done', title: 'Recent done', updated: recent, created: recent }, + ]); + assert.match(el.textContent, /Old queued/); + assert.doesNotMatch(el.textContent, /Old done/); + assert.match(el.textContent, /Recent done/); +}); + +test('a job panel caps jobs other than queued by count, newest first', () => { + const { el, render } = jobPanel({ hours: 24, count: 2 }); + const now = Date.now(); + const at = (minsAgo) => new Date(now - minsAgo * 60000).toISOString(); + render([ + { id: 'a', status: 'done', title: 'Oldest', updated: at(30), created: at(30) }, + { id: 'b', status: 'done', title: 'Middle', updated: at(20), created: at(20) }, + { id: 'c', status: 'done', title: 'Newest', updated: at(10), created: at(10) }, + ]); + assert.match(el.textContent, /Newest/); + assert.match(el.textContent, /Middle/); + assert.doesNotMatch(el.textContent, /Oldest/); +}); + +test('a job panel shows a running job however old, like a queued job', () => { + const { el, render } = jobPanel({ hours: 1, count: 10 }); + const old = new Date(Date.now() - 2 * 3600000).toISOString(); + render([ + { id: 'a', status: 'running', title: 'Old running', updated: old, created: old }, + { id: 'b', status: 'done', title: 'Old done', updated: old, created: old }, + ]); + assert.match(el.textContent, /Old running/); + assert.doesNotMatch(el.textContent, /Old done/); +}); + test('job cancellation posts the job ID and refreshes polling', async () => { startPolling(); await settle(); diff --git a/tests/js/frontend.test.mjs b/tests/js/frontend.test.mjs index ce5389a..aac564a 100644 --- a/tests/js/frontend.test.mjs +++ b/tests/js/frontend.test.mjs @@ -45,7 +45,10 @@ const elements = (node) => node instanceof Element ? [node, ...node.children.fla const text = (node) => node instanceof Element ? node.children.map(text).join('') : String(node); const find = (root, tag, label) => elements(root).find((el) => el.tagName === tag && text(el) === label); const settle = () => new Promise((resolve) => setImmediate(resolve)); -const config = { metadata_fields: { title: 'Title' }, mode_labels: {}, library_available: true, update_interval: 0 }; +const config = { + metadata_fields: { title: 'Title' }, mode_labels: {}, library_available: true, update_interval: 0, + job_list_max_hours: 720, job_list_max_count: 500, +}; const story = { id: 1, title: 'Story', author: 'Author', url: 'https://example.com/story', site: 'example.com', locked_fields: [], pinned: { title: 'Edited title' }, site_values: { title: 'Site title' }, @@ -101,7 +104,7 @@ test('button variants and attributes do not need a base class', () => { }); for (const [name, render] of [ - ['settings', (main) => settingsPage(main)], + ['settings', (main) => settingsPage(main, {}, config)], ['stories', (main) => storiesPage(main, {}, config)], ['story', (main) => storyPage(main, { storyId: '1' }, config)], ['chapter', (main) => chapterPage(main, { storyId: '1', chapterId: '2' })], diff --git a/tests/js/upload-hold.test.mjs b/tests/js/upload-hold.test.mjs index a41a73d..7bb07ca 100644 --- a/tests/js/upload-hold.test.mjs +++ b/tests/js/upload-hold.test.mjs @@ -7,6 +7,7 @@ const { settingsPage } = await import('../../bindery/static/js/settings.js'); const { storyPage } = await import('../../bindery/static/js/story.js'); const { storiesPage } = await import('../../bindery/static/js/stories.js'); +const config = { job_list_max_hours: 720, job_list_max_count: 500 }; const book = { id: 7, uuid: 'book-7', title: 'Library Title', authors: ['Writer'] }; const details = (story, extra = {}) => ({ story: { id: 1, title: 'Story', author: 'Writer', url: 'https://example.com/story', site: 'example.com', @@ -40,7 +41,7 @@ test('the upload switches load and save on the settings page', async (t) => { backend.on('GET', 'rules', { rules: [], paragraphs: false }); backend.on('GET', 'stories', []); const main = new Element('main'); - settingsPage(main); + settingsPage(main, {}, config); await settle(); const card = section(main, 'Uploads to CWA'); const [hold, pause] = nodes(card).filter((el) => el.attributes.type === 'checkbox'); diff --git a/tests/test_api.py b/tests/test_api.py index b3c733d..8bdc629 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -289,6 +289,10 @@ def test_settings(client, monkeypatch, tmp_path): "hold_new_links": False, "pause_uploads": False, "personal_ini": "[defaults]\nis_adult:true\n", + "story_jobs_hours": 36, + "story_jobs_count": 10, + "main_jobs_hours": 6, + "main_jobs_count": 25, } saved = client.put("/api/settings", json={"recent_chapters": 3}, headers=H).json assert (saved["recent_chapters"], saved["warnings"]) == (3, []) @@ -301,6 +305,16 @@ def test_settings(client, monkeypatch, tmp_path): assert bad.status_code == 400 and "line 1" in bad.json["error"] assert ini.read_text() == "[www.notasite.com]\nx:1\n" assert client.put("/api/settings", json={"recent_chapters": -1}, headers=H).status_code == 400 + saved = client.put( + "/api/settings", + json={"story_jobs_hours": 48, "story_jobs_count": 20, "main_jobs_hours": 12, "main_jobs_count": 50}, + headers=H, + ).json + assert (saved["story_jobs_hours"], saved["story_jobs_count"]) == (48, 20) + assert (saved["main_jobs_hours"], saved["main_jobs_count"]) == (12, 50) + assert client.put("/api/settings", json={"story_jobs_hours": -1}, headers=H).status_code == 400 + assert client.put("/api/settings", json={"story_jobs_hours": 721}, headers=H).status_code == 400 + assert client.put("/api/settings", json={"main_jobs_count": 501}, headers=H).status_code == 400 def test_metadata_edit(client): diff --git a/tests/test_api_flows.py b/tests/test_api_flows.py index 7e45b6f..1ee55d6 100644 --- a/tests/test_api_flows.py +++ b/tests/test_api_flows.py @@ -44,7 +44,11 @@ def test_add_list_download_and_cancel_jobs(client, story): assert result.status_code == 200 assert result.json["force_fields"] == ["title"] assert result.json["mode"] == "metadata" - assert client.get("/api/config").json["metadata_fields"]["title"] + app_config = client.get("/api/config").json + assert app_config["metadata_fields"]["title"] + assert (app_config["story_jobs_hours"], app_config["story_jobs_count"]) == (36, 10) + assert (app_config["main_jobs_hours"], app_config["main_jobs_count"]) == (6, 25) + assert (app_config["job_list_max_hours"], app_config["job_list_max_count"]) == (720, 500) def test_settings_and_inclusion_persist_and_delete_cascades(client, story):