From 61670a7ee9b70a67691bc1338441e5c41e599796 Mon Sep 17 00:00:00 2001 From: Junwei Zhao Date: Sun, 22 Mar 2026 14:05:10 +1100 Subject: [PATCH] Update style --- data/db.sqlite3 | Bin 479232 -> 479232 bytes k8s/manifest.yaml | 4 +- ..._screenshot_job_timeout_to_sitesettings.py | 18 ++++ links/models.py | 8 ++ links/tasks.py | 101 ++++++++++-------- links/templates/links/jobs.html | 2 +- 6 files changed, 86 insertions(+), 47 deletions(-) create mode 100644 links/migrations/0043_add_screenshot_job_timeout_to_sitesettings.py diff --git a/data/db.sqlite3 b/data/db.sqlite3 index cd054bb7b63779e6f0078026234524d1b5174314..1df67b1cef2c51dfec07779af731e36b949943d4 100644 GIT binary patch delta 662 zcmYk2T}V@57{}k|Js+=|^Bhl7lIC1lS+I?>Gv{)qI1B_8Luv$hv90Z7%XSu?BY0Ce z#gCi7aCYd!7zMitin4qOgMtd9fX6UsMv`WXS~t zRR93Sqod$mN^lJw4uctInL^ZC#H~zL&y84dD;GC2mTu~n zl`#fQP1TxxYS5=?9yJ(I)rc1Mhr=zQU}(Cu!O_wNKL8DmF66{(sO(Oe;<_Oe$Xjxe z|HD~$t-Pw7filz!?1=Eg0bw^d#Joq`oFO^`!V|*F33x+$9Ks{0nb?;1kO_r_>eg16 zoFJQ3AOMps_8bWNq1lcmVe9+@;gx+I3!V16*POKHe|S?*<&3ns%abuIeNZ3v^c_Cx z>5KLD(gm0B3|BJOBB_oAA>WoU()y?!UE!+iRUMS+f-KO2EM|X-yoPGft$9KkF-s~j zlzjZ46XlPyfVPi^qEWE(S4Fp7x-8yQ0$QXg6lv1@YM6aCc!;B8K<;B+Tt?kA`9<^r JO$lV7`yT~GxU>KO delta 459 zcmZozAltA&c7n9vD+UIJQXsYlVlE(lF;T~u@zutJ3H)rlK(2c3=5PGMa*VcgNj!giemCJV;t?~gEhGdG$RPv4lr zB)mP~D6<~pbk8&<`RT`#n9QcvCNuhMpOVZ}aEFwfm_Ck`-&GjE^A!+MBmdfhSR1#JSX+XUDcKd`awW{_fRn99zs&ctFg)qp*3 ba)803?UJ|IHZg6Fy~k$AxILMbeXbV(FC>LL diff --git a/k8s/manifest.yaml b/k8s/manifest.yaml index fd07ee2..a6afb2f 100644 --- a/k8s/manifest.yaml +++ b/k8s/manifest.yaml @@ -177,8 +177,8 @@ spec: cpu: 200m memory: 400Mi limits: - cpu: 1000m - memory: 1000Mi + cpu: 1200m + memory: 2Gi imagePullSecrets: - name: github-image-pull-secret --- diff --git a/links/migrations/0043_add_screenshot_job_timeout_to_sitesettings.py b/links/migrations/0043_add_screenshot_job_timeout_to_sitesettings.py new file mode 100644 index 0000000..2a28a91 --- /dev/null +++ b/links/migrations/0043_add_screenshot_job_timeout_to_sitesettings.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.11 on 2026-03-22 03:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('links', '0042_sitesettings_scheduler_intervals'), + ] + + operations = [ + migrations.AddField( + model_name='sitesettings', + name='screenshot_job_timeout_seconds', + field=models.IntegerField(default=300, help_text='Maximum time in seconds a single screenshot job may run before it is cancelled and marked as failed. Default is 300 (5 minutes).', verbose_name='Screenshot Job Timeout (seconds)'), + ), + ] diff --git a/links/models.py b/links/models.py index cb55354..0aa3999 100644 --- a/links/models.py +++ b/links/models.py @@ -442,6 +442,14 @@ class SiteSettings(models.Model): default=120, help_text=_('How often (in seconds) to check for stuck or pending screenshots and retry them.'), ) + screenshot_job_timeout_seconds = models.IntegerField( + _('Screenshot Job Timeout (seconds)'), + default=300, + help_text=_( + 'Maximum time in seconds a single screenshot job may run before it is cancelled and ' + 'marked as failed. Default is 300 (5 minutes).' + ), + ) class Meta: verbose_name = _('Site Settings') diff --git a/links/tasks.py b/links/tasks.py index e3a3b94..0e2ac1b 100644 --- a/links/tasks.py +++ b/links/tasks.py @@ -54,55 +54,59 @@ def extract_description_from_soup(soup): return text return None -async def _capture_screenshot_async(page_url, full_path): - """Async function to capture screenshot using Playwright""" - async with async_playwright() as p: - browser = await p.chromium.launch( - headless=True, - args=[ - '--no-sandbox', - '--disable-setuid-sandbox', - '--disable-dev-shm-usage', - ] - ) - - context = await browser.new_context( - viewport={'width': 1366, 'height': 768}, - locale='zh-CN', - user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - extra_http_headers={ - 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', - }, - ignore_https_errors=True, - ) - - page = await context.new_page() - - try: - # Navigate and wait for network idle - await page.goto( - page_url, - wait_until='networkidle', - timeout=SCREENSHOT_TIMEOUT +async def _capture_screenshot_async(page_url, full_path, job_timeout_seconds=300): + """Async function to capture screenshot using Playwright. + + Raises asyncio.TimeoutError if the entire operation exceeds job_timeout_seconds. + """ + async def _run(): + async with async_playwright() as p: + browser = await p.chromium.launch( + headless=True, + args=[ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + ] ) - - # Take full-page screenshot - await page.screenshot( - path=full_path, - full_page=True, - type='png' + + context = await browser.new_context( + viewport={'width': 1366, 'height': 768}, + locale='zh-CN', + user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + extra_http_headers={ + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', + }, + ignore_https_errors=True, ) - - logger.info(f"Successfully captured screenshot: {full_path}") - - finally: - await context.close() - await browser.close() + + page = await context.new_page() + + try: + await page.goto( + page_url, + wait_until='networkidle', + timeout=SCREENSHOT_TIMEOUT + ) + + await page.screenshot( + path=full_path, + full_page=True, + type='png' + ) + + logger.info(f"Successfully captured screenshot: {full_path}") + + finally: + await context.close() + await browser.close() + + await asyncio.wait_for(_run(), timeout=job_timeout_seconds) def capture_screenshot(page_id, screenshot_id, retry_count=0): """Task to capture full-page screenshot with Playwright""" - from .models import Page, Screenshot + from .models import Page, Screenshot, SiteSettings screenshot = None try: @@ -122,6 +126,8 @@ def capture_screenshot(page_id, screenshot_id, retry_count=0): screenshot.save() return + job_timeout = SiteSettings.get().screenshot_job_timeout_seconds + screenshot.status = Screenshot.Status.PROCESSING screenshot.error = None screenshot.save() @@ -138,7 +144,7 @@ def capture_screenshot(page_id, screenshot_id, retry_count=0): # Run async screenshot capture in sync context try: - asyncio.run(_capture_screenshot_async(page.url, full_path)) + asyncio.run(_capture_screenshot_async(page.url, full_path, job_timeout_seconds=job_timeout)) # Update screenshot record screenshot.path = filepath @@ -147,6 +153,13 @@ def capture_screenshot(page_id, screenshot_id, retry_count=0): logger.info(f"Successfully captured screenshot for page {page_id}") + except asyncio.TimeoutError: + msg = f"Job timed out after {job_timeout}s" + logger.error(f"Screenshot {screenshot_id}: {msg}") + screenshot.status = Screenshot.Status.FAILED + screenshot.error = msg + screenshot.save() + return except PlaywrightTimeoutError as e: logger.error(f"Timeout during screenshot capture: {str(e)}") screenshot.status = Screenshot.Status.FAILED diff --git a/links/templates/links/jobs.html b/links/templates/links/jobs.html index b3fe224..6285383 100644 --- a/links/templates/links/jobs.html +++ b/links/templates/links/jobs.html @@ -400,7 +400,7 @@