diff --git a/data/db.sqlite3 b/data/db.sqlite3 index 4bafebf..c6a861f 100644 Binary files a/data/db.sqlite3 and b/data/db.sqlite3 differ diff --git a/docker.sh b/docker.sh index 759e364..0ac489b 100755 --- a/docker.sh +++ b/docker.sh @@ -15,6 +15,9 @@ case "$1" in "restart") docker-compose restart ;; + "restart-worker") + docker-compose restart celery_worker celery_beat + ;; "logs") if [ "$2" ]; then docker-compose logs -f "$2" diff --git a/links/tasks.py b/links/tasks.py index 898bb5e..20fea97 100644 --- a/links/tasks.py +++ b/links/tasks.py @@ -60,6 +60,8 @@ def extract_description_from_soup(soup): def capture_screenshot(self, page_id): """Task to capture full-page screenshot with improved Chinese website support""" from .models import Page + from selenium.webdriver.support.ui import WebDriverWait + from selenium.webdriver.support import expected_conditions as EC try: page = Page.objects.get(id=page_id) @@ -68,21 +70,17 @@ def capture_screenshot(self, page_id): page.save() chrome_options = Options() - chrome_options.add_argument('--headless') # Use new headless mode + chrome_options.add_argument('--headless=new') # Use new headless mode chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--disable-dev-shm-usage') chrome_options.add_argument('--disable-gpu') - chrome_options.add_argument('--window-size=1920,1080') + chrome_options.add_argument('--start-maximized') chrome_options.add_argument('--disable-software-rasterizer') chrome_options.add_argument('--disable-extensions') chrome_options.add_argument('--ignore-certificate-errors') - chrome_options.add_argument('--start-maximized') chrome_options.add_argument('--ignore-ssl-errors') chrome_options.add_argument('--disable-web-security') chrome_options.add_argument('--allow-running-insecure-content') - - - chrome_options.add_argument('--disable-blink-features=AutomationControlled') chrome_options.add_argument('--lang=zh-CN,zh;q=0.9,en;q=0.8') chrome_options.add_argument( 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' @@ -100,7 +98,6 @@ def capture_screenshot(self, page_id): ) try: - driver.execute_cdp_cmd('Network.setExtraHTTPHeaders', { 'headers': { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', @@ -110,64 +107,60 @@ def capture_screenshot(self, page_id): }) driver.set_page_load_timeout(SCREENSHOT_TIMEOUT) + + # Initial page load driver.get(page.url) + time.sleep(3 + random.uniform(1, 3)) # Wait for initial load - # Wait for dynamic content and fonts to load - time.sleep(3 + random.uniform(1, 3)) + # Get initial window size and document height + original_size = driver.get_window_size() - # Get the total height including any hidden elements with fallback + # Get total height of the page total_height = driver.execute_script(""" - try { - const height = Math.max( - document.documentElement.scrollHeight || 0, - document.body.scrollHeight || 0, - document.documentElement.offsetHeight || 0, - document.body.offsetHeight || 0, - document.documentElement.clientHeight || 0, - document.body.clientHeight || 0, - Array.from(document.getElementsByTagName('*')).reduce((max, el) => - Math.max(max, el.offsetTop + el.offsetHeight), 0) - ); - return height || 1080; // Fallback to 1080 if calculation fails - } catch (e) { - return 1080; // Default height if script fails - } + return Math.max( + document.documentElement.scrollHeight, + document.body.scrollHeight, + document.documentElement.offsetHeight, + document.body.offsetHeight, + document.documentElement.clientHeight, + document.body.clientHeight + ); """) - # Ensure we have a valid height - if not total_height or total_height < 1080: - total_height = 1080 + # Set viewport to match the full height + driver.set_window_size(original_size['width'], total_height) + time.sleep(1) # Wait for resize - # Add padding for footer - total_height += 100 - - # Set window size with the new height - driver.set_window_size(1920, total_height) - - # Scroll through the page gradually - current_height = 0 - step = 200 # Smaller step size for smoother scrolling - while current_height < total_height: - driver.execute_script(f"window.scrollTo(0, {current_height});") - current_height += step - time.sleep(0.2) # Longer delay for content to load - - # Scroll to very bottom to ensure footer is loaded - driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") - time.sleep(1) + # Scroll in smaller increments + for i in range(0, total_height, 200): + driver.execute_script(f"window.scrollTo(0, {i});") + time.sleep(0.1) # Short pause while scrolling # Scroll back to top driver.execute_script("window.scrollTo(0, 0);") - time.sleep(1) # Wait longer for any scrolling animations + time.sleep(1) - # # Force all elements to be visible - # driver.execute_script(""" - # document.querySelectorAll('*').forEach(el => { - # if (window.getComputedStyle(el).display === 'none') { - # el.style.display = 'block'; - # } - # }); - # """) + # Ensure all content is loaded + driver.execute_script(""" + return new Promise((resolve) => { + let totalHeight = 0; + let distance = 200; + let timer = setInterval(() => { + let scrollHeight = document.body.scrollHeight; + window.scrollBy(0, distance); + totalHeight += distance; + + if(totalHeight >= scrollHeight){ + clearInterval(timer); + window.scrollTo(0, 0); + resolve(); + } + }, 100); + }); + """) + + # Final wait for any animations + time.sleep(2) # Create screenshots directory screenshots_dir = os.path.join(settings.MEDIA_ROOT, 'screenshots') @@ -179,7 +172,12 @@ def capture_screenshot(self, page_id): filepath = os.path.join('screenshots', filename) full_path = os.path.join(settings.MEDIA_ROOT, filepath) - # Take full page screenshot + # Take the full page screenshot + required_height = driver.execute_script('return document.body.parentNode.scrollHeight') + driver.set_window_size(original_size['width'], required_height) + time.sleep(1) # Wait for resize + + # Take screenshot driver.save_screenshot(full_path) # Update page model @@ -209,7 +207,6 @@ def capture_screenshot(self, page_id): self.retry(exc=exc, countdown=retry_delay) except MaxRetriesExceededError: logger.error(f"Max retries exceeded for page {page_id}") - page.error_message = f"Screenshot capture failed: {str(exc)}" page.screenshot_error = f"Failed after {self.request.retries} attempts. Last error: {str(exc)}" page.save()