From 879bc2ef7d679dbf6040a903289b850b9926937d Mon Sep 17 00:00:00 2001
From: Junwei Zhao <wahyd4@gmail.com>
Date: Sun, 10 Nov 2024 10:18:04 +1100
Subject: [PATCH] Update logic to scrol and screenshot the entire page

---
 data/db.sqlite3 | Bin 212992 -> 212992 bytes
 docker.sh       |   3 ++
 links/tasks.py  | 109 +++++++++++++++++++++++-------------------------
 3 files changed, 56 insertions(+), 56 deletions(-)

diff --git a/data/db.sqlite3 b/data/db.sqlite3
index 4bafebf7901204bd64c415fd88fb103d99147651..c6a861f627fdb4b6d467af6f774febd23e30c769 100644
GIT binary patch
delta 163
zcmZo@;B9E&ogmE^JW<A(F?eIbbv;JYW()mx3w=f)W}0rH&wPf9nQJEl?@68yTst=_
zDy-&WZZR?4e#Vyh2&0Cvm7$rHv6-HUg{g(5v5|q1iLRlcu7Rb35m3h5%D`C9*wED2
y#A14*EwiM)aXd(cp`n3gypge?nW?2-L0&qVCZKWVdWMFU29_q<KiDt}FarQsB`MYb

delta 57
zcmZo@;B9E&ogmE^Fj2;tF<@iDbv;ItW()mx3w=f)W}0rH&wPf1iRS|Y?@68yn-vwF
M@NEBJ!z{oI00GAl?f?J)

diff --git a/docker.sh b/docker.sh
index 759e364..0ac489b 100755
--- a/docker.sh
+++ b/docker.sh
@@ -15,6 +15,9 @@ case "$1" in
     "restart")
         docker-compose restart
         ;;
+    "restart-worker")
+        docker-compose restart celery_worker celery_beat
+        ;;
     "logs")
         if [ "$2" ]; then
             docker-compose logs -f "$2"
diff --git a/links/tasks.py b/links/tasks.py
index 898bb5e..20fea97 100644
--- a/links/tasks.py
+++ b/links/tasks.py
@@ -60,6 +60,8 @@ def extract_description_from_soup(soup):
 def capture_screenshot(self, page_id):
     """Task to capture full-page screenshot with improved Chinese website support"""
     from .models import Page
+    from selenium.webdriver.support.ui import WebDriverWait
+    from selenium.webdriver.support import expected_conditions as EC
 
     try:
         page = Page.objects.get(id=page_id)
@@ -68,21 +70,17 @@ def capture_screenshot(self, page_id):
         page.save()
 
         chrome_options = Options()
-        chrome_options.add_argument('--headless')  # Use new headless mode
+        chrome_options.add_argument('--headless=new')  # Use new headless mode
         chrome_options.add_argument('--no-sandbox')
         chrome_options.add_argument('--disable-dev-shm-usage')
         chrome_options.add_argument('--disable-gpu')
-        chrome_options.add_argument('--window-size=1920,1080')
+        chrome_options.add_argument('--start-maximized')
         chrome_options.add_argument('--disable-software-rasterizer')
         chrome_options.add_argument('--disable-extensions')
         chrome_options.add_argument('--ignore-certificate-errors')
-        chrome_options.add_argument('--start-maximized')
         chrome_options.add_argument('--ignore-ssl-errors')
         chrome_options.add_argument('--disable-web-security')
         chrome_options.add_argument('--allow-running-insecure-content')
-
-
-        chrome_options.add_argument('--disable-blink-features=AutomationControlled')
         chrome_options.add_argument('--lang=zh-CN,zh;q=0.9,en;q=0.8')
         chrome_options.add_argument(
             'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
@@ -100,7 +98,6 @@ def capture_screenshot(self, page_id):
         )
 
         try:
-
             driver.execute_cdp_cmd('Network.setExtraHTTPHeaders', {
                 'headers': {
                     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
@@ -110,64 +107,60 @@ def capture_screenshot(self, page_id):
             })
 
             driver.set_page_load_timeout(SCREENSHOT_TIMEOUT)
+
+            # Initial page load
             driver.get(page.url)
+            time.sleep(3 + random.uniform(1, 3))  # Wait for initial load
 
-            # Wait for dynamic content and fonts to load
-            time.sleep(3 + random.uniform(1, 3))
+            # Get initial window size and document height
+            original_size = driver.get_window_size()
 
-            # Get the total height including any hidden elements with fallback
+            # Get total height of the page
             total_height = driver.execute_script("""
-                try {
-                    const height = Math.max(
-                        document.documentElement.scrollHeight || 0,
-                        document.body.scrollHeight || 0,
-                        document.documentElement.offsetHeight || 0,
-                        document.body.offsetHeight || 0,
-                        document.documentElement.clientHeight || 0,
-                        document.body.clientHeight || 0,
-                        Array.from(document.getElementsByTagName('*')).reduce((max, el) =>
-                            Math.max(max, el.offsetTop + el.offsetHeight), 0)
-                    );
-                    return height || 1080;  // Fallback to 1080 if calculation fails
-                } catch (e) {
-                    return 1080;  // Default height if script fails
-                }
+                return Math.max(
+                    document.documentElement.scrollHeight,
+                    document.body.scrollHeight,
+                    document.documentElement.offsetHeight,
+                    document.body.offsetHeight,
+                    document.documentElement.clientHeight,
+                    document.body.clientHeight
+                );
             """)
 
-            # Ensure we have a valid height
-            if not total_height or total_height < 1080:
-                total_height = 1080
+            # Set viewport to match the full height
+            driver.set_window_size(original_size['width'], total_height)
+            time.sleep(1)  # Wait for resize
 
-            # Add padding for footer
-            total_height += 100
-
-            # Set window size with the new height
-            driver.set_window_size(1920, total_height)
-
-            # Scroll through the page gradually
-            current_height = 0
-            step = 200  # Smaller step size for smoother scrolling
-            while current_height < total_height:
-                driver.execute_script(f"window.scrollTo(0, {current_height});")
-                current_height += step
-                time.sleep(0.2)  # Longer delay for content to load
-
-            # Scroll to very bottom to ensure footer is loaded
-            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
-            time.sleep(1)
+            # Scroll in smaller increments
+            for i in range(0, total_height, 200):
+                driver.execute_script(f"window.scrollTo(0, {i});")
+                time.sleep(0.1)  # Short pause while scrolling
 
             # Scroll back to top
             driver.execute_script("window.scrollTo(0, 0);")
-            time.sleep(1)  # Wait longer for any scrolling animations
+            time.sleep(1)
 
-            # # Force all elements to be visible
-            # driver.execute_script("""
-            #     document.querySelectorAll('*').forEach(el => {
-            #         if (window.getComputedStyle(el).display === 'none') {
-            #             el.style.display = 'block';
-            #         }
-            #     });
-            # """)
+            # Ensure all content is loaded
+            driver.execute_script("""
+                return new Promise((resolve) => {
+                    let totalHeight = 0;
+                    let distance = 200;
+                    let timer = setInterval(() => {
+                        let scrollHeight = document.body.scrollHeight;
+                        window.scrollBy(0, distance);
+                        totalHeight += distance;
+
+                        if(totalHeight >= scrollHeight){
+                            clearInterval(timer);
+                            window.scrollTo(0, 0);
+                            resolve();
+                        }
+                    }, 100);
+                });
+            """)
+
+            # Final wait for any animations
+            time.sleep(2)
 
             # Create screenshots directory
             screenshots_dir = os.path.join(settings.MEDIA_ROOT, 'screenshots')
@@ -179,7 +172,12 @@ def capture_screenshot(self, page_id):
             filepath = os.path.join('screenshots', filename)
             full_path = os.path.join(settings.MEDIA_ROOT, filepath)
 
-            # Take full page screenshot
+            # Take the full page screenshot
+            required_height = driver.execute_script('return document.body.parentNode.scrollHeight')
+            driver.set_window_size(original_size['width'], required_height)
+            time.sleep(1)  # Wait for resize
+
+            # Take screenshot
             driver.save_screenshot(full_path)
 
             # Update page model
@@ -209,7 +207,6 @@ def capture_screenshot(self, page_id):
             self.retry(exc=exc, countdown=retry_delay)
         except MaxRetriesExceededError:
             logger.error(f"Max retries exceeded for page {page_id}")
-            page.error_message = f"Screenshot capture failed: {str(exc)}"
             page.screenshot_error = f"Failed after {self.request.retries} attempts. Last error: {str(exc)}"
             page.save()