diff --git a/Dockerfile b/Dockerfile index 53a49ec..1ba534b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,6 +62,7 @@ RUN apt-get update && apt-get install -y \ gnupg \ chromium \ chromium-driver \ + fonts-arphic-ukai \ && rm -rf /var/lib/apt/lists/* # Create a non-root user diff --git a/Dockerfile.local b/Dockerfile.local index 72828a3..936f12b 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -17,6 +17,11 @@ RUN apt-get update && apt-get install -y \ nodejs \ npm \ gettext \ + gnupg \ + chromium \ + chromium-driver \ + fonts-wqy-zenhei \ + fonts-arphic-ukai \ && rm -rf /var/lib/apt/lists/* # Copy project files diff --git a/data/db.sqlite3 b/data/db.sqlite3 index cffe83f..4bafebf 100644 Binary files a/data/db.sqlite3 and b/data/db.sqlite3 differ diff --git a/links/migrations/0014_page_screenshot_error_page_screenshot_last_attempt.py b/links/migrations/0014_page_screenshot_error_page_screenshot_last_attempt.py new file mode 100644 index 0000000..47024fa --- /dev/null +++ b/links/migrations/0014_page_screenshot_error_page_screenshot_last_attempt.py @@ -0,0 +1,23 @@ +# Generated by Django 5.0.9 on 2024-11-09 21:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('links', '0013_newsletter'), + ] + + operations = [ + migrations.AddField( + model_name='page', + name='screenshot_error', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='page', + name='screenshot_last_attempt', + field=models.DateTimeField(null=True), + ), + ] diff --git a/links/models.py b/links/models.py index 5615c35..78904f5 100644 --- a/links/models.py +++ b/links/models.py @@ -156,6 +156,10 @@ class Page(models.Model): # New field for screenshot screenshot_path = models.CharField(max_length=255, blank=True) + # New fields for screenshot errors + screenshot_error = models.TextField(blank=True, null=True) + screenshot_last_attempt = models.DateTimeField(null=True) + class Meta: ordering = ['-updated_at'] verbose_name = _('Page') diff --git a/links/tasks.py b/links/tasks.py index e398e25..898bb5e 100644 --- a/links/tasks.py +++ b/links/tasks.py @@ -11,6 +11,7 @@ from django.conf import settings from urllib.parse import quote import time from celery.exceptions import MaxRetriesExceededError +import random logger = logging.getLogger(__name__) SCREENSHOT_TIMEOUT = 20 @@ -57,20 +58,36 @@ def extract_description_from_soup(soup): @shared_task(bind=True, max_retries=3) def capture_screenshot(self, page_id): - """Task to capture screenshot of a page""" + """Task to capture full-page screenshot with improved Chinese website support""" from .models import Page try: page = Page.objects.get(id=page_id) + page.screenshot_error = None + page.screenshot_last_attempt = timezone.now() + page.save() chrome_options = Options() - chrome_options.add_argument('--headless') + chrome_options.add_argument('--headless') # Use new headless mode chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--disable-dev-shm-usage') chrome_options.add_argument('--disable-gpu') - chrome_options.add_argument('--window-size=1366,768') + chrome_options.add_argument('--window-size=1920,1080') chrome_options.add_argument('--disable-software-rasterizer') chrome_options.add_argument('--disable-extensions') + chrome_options.add_argument('--ignore-certificate-errors') + chrome_options.add_argument('--start-maximized') + chrome_options.add_argument('--ignore-ssl-errors') + chrome_options.add_argument('--disable-web-security') + chrome_options.add_argument('--allow-running-insecure-content') + + + chrome_options.add_argument('--disable-blink-features=AutomationControlled') + chrome_options.add_argument('--lang=zh-CN,zh;q=0.9,en;q=0.8') + chrome_options.add_argument( + 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' + ) + chrome_options.binary_location = os.getenv('CHROME_BIN', '/usr/bin/chromium') chrome_service = webdriver.ChromeService( @@ -83,41 +100,117 @@ def capture_screenshot(self, page_id): ) try: + + driver.execute_cdp_cmd('Network.setExtraHTTPHeaders', { + 'headers': { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-AU,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,en-GB;q=0.6,en-US;q=0.5', + 'Accept-Encoding': 'gzip, deflate, br', + } + }) + driver.set_page_load_timeout(SCREENSHOT_TIMEOUT) driver.get(page.url) - time.sleep(2) # Wait for dynamic content + + # Wait for dynamic content and fonts to load + time.sleep(3 + random.uniform(1, 3)) + + # Get the total height including any hidden elements with fallback + total_height = driver.execute_script(""" + try { + const height = Math.max( + document.documentElement.scrollHeight || 0, + document.body.scrollHeight || 0, + document.documentElement.offsetHeight || 0, + document.body.offsetHeight || 0, + document.documentElement.clientHeight || 0, + document.body.clientHeight || 0, + Array.from(document.getElementsByTagName('*')).reduce((max, el) => + Math.max(max, el.offsetTop + el.offsetHeight), 0) + ); + return height || 1080; // Fallback to 1080 if calculation fails + } catch (e) { + return 1080; // Default height if script fails + } + """) + + # Ensure we have a valid height + if not total_height or total_height < 1080: + total_height = 1080 + + # Add padding for footer + total_height += 100 + + # Set window size with the new height + driver.set_window_size(1920, total_height) + + # Scroll through the page gradually + current_height = 0 + step = 200 # Smaller step size for smoother scrolling + while current_height < total_height: + driver.execute_script(f"window.scrollTo(0, {current_height});") + current_height += step + time.sleep(0.2) # Longer delay for content to load + + # Scroll to very bottom to ensure footer is loaded + driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") + time.sleep(1) + + # Scroll back to top + driver.execute_script("window.scrollTo(0, 0);") + time.sleep(1) # Wait longer for any scrolling animations + + # # Force all elements to be visible + # driver.execute_script(""" + # document.querySelectorAll('*').forEach(el => { + # if (window.getComputedStyle(el).display === 'none') { + # el.style.display = 'block'; + # } + # }); + # """) # Create screenshots directory screenshots_dir = os.path.join(settings.MEDIA_ROOT, 'screenshots') os.makedirs(screenshots_dir, exist_ok=True) - # Create filename - filename = f"page_{page_id}.png" + # Create filename with timestamp + timestamp = timezone.now().strftime('%Y%m%d_%H%M%S') + filename = f"page_{page_id}_{timestamp}.png" filepath = os.path.join('screenshots', filename) full_path = os.path.join(settings.MEDIA_ROOT, filepath) - # Take screenshot + # Take full page screenshot driver.save_screenshot(full_path) # Update page model page.screenshot_path = filepath page.save() - logger.info(f"Successfully captured screenshot for page {page_id}") + logger.info(f"Successfully captured full page screenshot for page {page_id}") + except Exception as e: + logger.error(f"Error during screenshot capture: {str(e)}") + raise e finally: try: + driver.execute_script("window.localStorage.clear();") + driver.execute_script("window.sessionStorage.clear();") + driver.delete_all_cookies() driver.quit() except Exception as e: - logger.warning(f"Error closing Chrome driver: {e}") + logger.warning(f"Error cleaning up Chrome driver: {e}") except Exception as exc: logger.error(f"Failed to capture screenshot for page {page_id}: {exc}") try: - self.retry(exc=exc, countdown=fibonacci(self.request.retries)) + retry_delay = fibonacci(self.request.retries) + page.screenshot_error = f"Error: {str(exc)}. Retrying in {retry_delay} seconds..." + page.save() + self.retry(exc=exc, countdown=retry_delay) except MaxRetriesExceededError: logger.error(f"Max retries exceeded for page {page_id}") page.error_message = f"Screenshot capture failed: {str(exc)}" + page.screenshot_error = f"Failed after {self.request.retries} attempts. Last error: {str(exc)}" page.save() @shared_task(bind=True, max_retries=3) diff --git a/links/templates/links/page_detail.html b/links/templates/links/page_detail.html index c695fb1..6601abd 100644 --- a/links/templates/links/page_detail.html +++ b/links/templates/links/page_detail.html @@ -109,22 +109,44 @@ {% endif %} - -
-

{% trans "Screenshot" %}

-
- {% if page.screenshot_path %} - Page screenshot - {% else %} - Default screenshot - {% endif %} + + {% if page.screenshot_path %} +
+

{% trans "Screenshot" %}

+ Page screenshot
-
+ {% elif page.screenshot_error %} +
+
+
+
+ + + +
+
+

+ {% trans "Screenshot Error" %} +

+
+

{{ page.screenshot_error }}

+
+ {% if page.screenshot_last_attempt %} +
+ {% trans "Last attempt" %}: + + {{ page.screenshot_last_attempt|date:"Y-m-d H:i" }} + +
+ {% endif %} +
+
+
+
+ {% endif %}