diff --git a/data/db.sqlite3 b/data/db.sqlite3 index cd054bb..1df67b1 100644 Binary files a/data/db.sqlite3 and b/data/db.sqlite3 differ diff --git a/k8s/manifest.yaml b/k8s/manifest.yaml index fd07ee2..a6afb2f 100644 --- a/k8s/manifest.yaml +++ b/k8s/manifest.yaml @@ -177,8 +177,8 @@ spec: cpu: 200m memory: 400Mi limits: - cpu: 1000m - memory: 1000Mi + cpu: 1200m + memory: 2Gi imagePullSecrets: - name: github-image-pull-secret --- diff --git a/links/migrations/0043_add_screenshot_job_timeout_to_sitesettings.py b/links/migrations/0043_add_screenshot_job_timeout_to_sitesettings.py new file mode 100644 index 0000000..2a28a91 --- /dev/null +++ b/links/migrations/0043_add_screenshot_job_timeout_to_sitesettings.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.11 on 2026-03-22 03:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('links', '0042_sitesettings_scheduler_intervals'), + ] + + operations = [ + migrations.AddField( + model_name='sitesettings', + name='screenshot_job_timeout_seconds', + field=models.IntegerField(default=300, help_text='Maximum time in seconds a single screenshot job may run before it is cancelled and marked as failed. Default is 300 (5 minutes).', verbose_name='Screenshot Job Timeout (seconds)'), + ), + ] diff --git a/links/models.py b/links/models.py index cb55354..0aa3999 100644 --- a/links/models.py +++ b/links/models.py @@ -442,6 +442,14 @@ class SiteSettings(models.Model): default=120, help_text=_('How often (in seconds) to check for stuck or pending screenshots and retry them.'), ) + screenshot_job_timeout_seconds = models.IntegerField( + _('Screenshot Job Timeout (seconds)'), + default=300, + help_text=_( + 'Maximum time in seconds a single screenshot job may run before it is cancelled and ' + 'marked as failed. Default is 300 (5 minutes).' + ), + ) class Meta: verbose_name = _('Site Settings') diff --git a/links/tasks.py b/links/tasks.py index e3a3b94..0e2ac1b 100644 --- a/links/tasks.py +++ b/links/tasks.py @@ -54,55 +54,59 @@ def extract_description_from_soup(soup): return text return None -async def _capture_screenshot_async(page_url, full_path): - """Async function to capture screenshot using Playwright""" - async with async_playwright() as p: - browser = await p.chromium.launch( - headless=True, - args=[ - '--no-sandbox', - '--disable-setuid-sandbox', - '--disable-dev-shm-usage', - ] - ) - - context = await browser.new_context( - viewport={'width': 1366, 'height': 768}, - locale='zh-CN', - user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - extra_http_headers={ - 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', - }, - ignore_https_errors=True, - ) - - page = await context.new_page() - - try: - # Navigate and wait for network idle - await page.goto( - page_url, - wait_until='networkidle', - timeout=SCREENSHOT_TIMEOUT +async def _capture_screenshot_async(page_url, full_path, job_timeout_seconds=300): + """Async function to capture screenshot using Playwright. + + Raises asyncio.TimeoutError if the entire operation exceeds job_timeout_seconds. + """ + async def _run(): + async with async_playwright() as p: + browser = await p.chromium.launch( + headless=True, + args=[ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + ] ) - - # Take full-page screenshot - await page.screenshot( - path=full_path, - full_page=True, - type='png' + + context = await browser.new_context( + viewport={'width': 1366, 'height': 768}, + locale='zh-CN', + user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + extra_http_headers={ + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', + }, + ignore_https_errors=True, ) - - logger.info(f"Successfully captured screenshot: {full_path}") - - finally: - await context.close() - await browser.close() + + page = await context.new_page() + + try: + await page.goto( + page_url, + wait_until='networkidle', + timeout=SCREENSHOT_TIMEOUT + ) + + await page.screenshot( + path=full_path, + full_page=True, + type='png' + ) + + logger.info(f"Successfully captured screenshot: {full_path}") + + finally: + await context.close() + await browser.close() + + await asyncio.wait_for(_run(), timeout=job_timeout_seconds) def capture_screenshot(page_id, screenshot_id, retry_count=0): """Task to capture full-page screenshot with Playwright""" - from .models import Page, Screenshot + from .models import Page, Screenshot, SiteSettings screenshot = None try: @@ -122,6 +126,8 @@ def capture_screenshot(page_id, screenshot_id, retry_count=0): screenshot.save() return + job_timeout = SiteSettings.get().screenshot_job_timeout_seconds + screenshot.status = Screenshot.Status.PROCESSING screenshot.error = None screenshot.save() @@ -138,7 +144,7 @@ def capture_screenshot(page_id, screenshot_id, retry_count=0): # Run async screenshot capture in sync context try: - asyncio.run(_capture_screenshot_async(page.url, full_path)) + asyncio.run(_capture_screenshot_async(page.url, full_path, job_timeout_seconds=job_timeout)) # Update screenshot record screenshot.path = filepath @@ -147,6 +153,13 @@ def capture_screenshot(page_id, screenshot_id, retry_count=0): logger.info(f"Successfully captured screenshot for page {page_id}") + except asyncio.TimeoutError: + msg = f"Job timed out after {job_timeout}s" + logger.error(f"Screenshot {screenshot_id}: {msg}") + screenshot.status = Screenshot.Status.FAILED + screenshot.error = msg + screenshot.save() + return except PlaywrightTimeoutError as e: logger.error(f"Timeout during screenshot capture: {str(e)}") screenshot.status = Screenshot.Status.FAILED diff --git a/links/templates/links/jobs.html b/links/templates/links/jobs.html index b3fe224..6285383 100644 --- a/links/templates/links/jobs.html +++ b/links/templates/links/jobs.html @@ -400,7 +400,7 @@