home/links
2
0
Fork 0
mirror of https://github.com/wahyd4/links.git synced 2026-08-09 05:06:16 +10:00
Code Issues Packages Projects Releases Wiki Activity

Update logic

Browse Source
This commit is contained in:
junv
2024-11-10 10:09:03 +11:00
parent 5b025571da
commit e18bafe60d
7 changed files with 173 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+1
View File
@@ -62,6 +62,7 @@ RUN apt-get update && apt-get install -y \
gnupg \
chromium \
chromium-driver \
fonts-arphic-ukai \
&& rm -rf /var/lib/apt/lists/*
# Create a non-root user
Dockerfile.local
+5
View File
@@ -17,6 +17,11 @@ RUN apt-get update && apt-get install -y \
nodejs \
npm \
gettext \
gnupg \
chromium \
chromium-driver \
fonts-wqy-zenhei \
fonts-arphic-ukai \
&& rm -rf /var/lib/apt/lists/*
# Copy project files
data/db.sqlite3
BIN
View File
Binary file not shown.
links/migrations/0014_page_screenshot_error_page_screenshot_last_attempt.py
+23
View File
@@ -0,0 +1,23 @@
# Generated by Django 5.0.9 on 2024-11-09 21:49
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('links', '0013_newsletter'),
]
operations = [
migrations.AddField(
model_name='page',
name='screenshot_error',
field=models.TextField(blank=True, null=True),
),
migrations.AddField(
model_name='page',
name='screenshot_last_attempt',
field=models.DateTimeField(null=True),
),
]
links/models.py
+4
View File
@@ -156,6 +156,10 @@ class Page(models.Model):
# New field for screenshot
screenshot_path = models.CharField(max_length=255, blank=True)
# New fields for screenshot errors
screenshot_error = models.TextField(blank=True, null=True)
screenshot_last_attempt = models.DateTimeField(null=True)
class Meta:
ordering = ['-updated_at']
verbose_name = _('Page')
links/tasks.py
+103 -10
View File
@@ -11,6 +11,7 @@ from django.conf import settings
from urllib.parse import quote
import time
from celery.exceptions import MaxRetriesExceededError
import random
logger = logging.getLogger(__name__)
SCREENSHOT_TIMEOUT = 20
@@ -57,20 +58,36 @@ def extract_description_from_soup(soup):
@shared_task(bind=True, max_retries=3)
def capture_screenshot(self, page_id):
"""Task to capture screenshot of a page"""
"""Task to capture full-page screenshot with improved Chinese website support"""
from .models import Page
try:
page = Page.objects.get(id=page_id)
page.screenshot_error = None
page.screenshot_last_attempt = timezone.now()
page.save()
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--headless') # Use new headless mode
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('--window-size=1366,768')
chrome_options.add_argument('--window-size=1920,1080')
chrome_options.add_argument('--disable-software-rasterizer')
chrome_options.add_argument('--disable-extensions')
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_argument('--start-maximized')
chrome_options.add_argument('--ignore-ssl-errors')
chrome_options.add_argument('--disable-web-security')
chrome_options.add_argument('--allow-running-insecure-content')
chrome_options.add_argument('--disable-blink-features=AutomationControlled')
chrome_options.add_argument('--lang=zh-CN,zh;q=0.9,en;q=0.8')
chrome_options.add_argument(
'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
)
chrome_options.binary_location = os.getenv('CHROME_BIN', '/usr/bin/chromium')
chrome_service = webdriver.ChromeService(
@@ -83,41 +100,117 @@ def capture_screenshot(self, page_id):
)
try:
driver.execute_cdp_cmd('Network.setExtraHTTPHeaders', {
'headers': {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-AU,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,en-GB;q=0.6,en-US;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
}
})
driver.set_page_load_timeout(SCREENSHOT_TIMEOUT)
driver.get(page.url)
time.sleep(2) # Wait for dynamic content
# Wait for dynamic content and fonts to load
time.sleep(3 + random.uniform(1, 3))
# Get the total height including any hidden elements with fallback
total_height = driver.execute_script("""
try {
const height = Math.max(
document.documentElement.scrollHeight || 0,
document.body.scrollHeight || 0,
document.documentElement.offsetHeight || 0,
document.body.offsetHeight || 0,
document.documentElement.clientHeight || 0,
document.body.clientHeight || 0,
Array.from(document.getElementsByTagName('*')).reduce((max, el) =>
Math.max(max, el.offsetTop + el.offsetHeight), 0)
);
return height || 1080; // Fallback to 1080 if calculation fails
} catch (e) {
return 1080; // Default height if script fails
}
""")
# Ensure we have a valid height
if not total_height or total_height < 1080:
total_height = 1080
# Add padding for footer
total_height += 100
# Set window size with the new height
driver.set_window_size(1920, total_height)
# Scroll through the page gradually
current_height = 0
step = 200 # Smaller step size for smoother scrolling
while current_height < total_height:
driver.execute_script(f"window.scrollTo(0, {current_height});")
current_height += step
time.sleep(0.2) # Longer delay for content to load
# Scroll to very bottom to ensure footer is loaded
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(1)
# Scroll back to top
driver.execute_script("window.scrollTo(0, 0);")
time.sleep(1) # Wait longer for any scrolling animations
# # Force all elements to be visible
# driver.execute_script("""
# document.querySelectorAll('*').forEach(el => {
# if (window.getComputedStyle(el).display === 'none') {
# el.style.display = 'block';
# }
# });
# """)
# Create screenshots directory
screenshots_dir = os.path.join(settings.MEDIA_ROOT, 'screenshots')
os.makedirs(screenshots_dir, exist_ok=True)
# Create filename
filename = f"page_{page_id}.png"
# Create filename with timestamp
timestamp = timezone.now().strftime('%Y%m%d_%H%M%S')
filename = f"page_{page_id}_{timestamp}.png"
filepath = os.path.join('screenshots', filename)
full_path = os.path.join(settings.MEDIA_ROOT, filepath)
# Take screenshot
# Take full page screenshot
driver.save_screenshot(full_path)
# Update page model
page.screenshot_path = filepath
page.save()
logger.info(f"Successfully captured screenshot for page {page_id}")
logger.info(f"Successfully captured full page screenshot for page {page_id}")
except Exception as e:
logger.error(f"Error during screenshot capture: {str(e)}")
raise e
finally:
try:
driver.execute_script("window.localStorage.clear();")
driver.execute_script("window.sessionStorage.clear();")
driver.delete_all_cookies()
driver.quit()
except Exception as e:
logger.warning(f"Error closing Chrome driver: {e}")
logger.warning(f"Error cleaning up Chrome driver: {e}")
except Exception as exc:
logger.error(f"Failed to capture screenshot for page {page_id}: {exc}")
try:
self.retry(exc=exc, countdown=fibonacci(self.request.retries))
retry_delay = fibonacci(self.request.retries)
page.screenshot_error = f"Error: {str(exc)}. Retrying in {retry_delay} seconds..."
page.save()
self.retry(exc=exc, countdown=retry_delay)
except MaxRetriesExceededError:
logger.error(f"Max retries exceeded for page {page_id}")
page.error_message = f"Screenshot capture failed: {str(exc)}"
page.screenshot_error = f"Failed after {self.request.retries} attempts. Last error: {str(exc)}"
page.save()
@shared_task(bind=True, max_retries=3)
links/templates/links/page_detail.html
+37 -15
View File
@@ -109,22 +109,44 @@
</div>
{% endif %}
<!-- Screenshot -->
<div class="mb-6">
<h2 class="text-sm font-medium text-gray-500 mb-2">{% trans "Screenshot" %}</h2>
<div class="rounded-lg overflow-hidden border border-gray-200">
{% if page.screenshot_path %}
<img src="{{ page.get_screenshot_url }}"
alt="Page screenshot"
class="w-full h-auto"
loading="lazy">
{% else %}
<img src="{% static 'images/default-screenshot.png' %}"
alt="Default screenshot"
class="w-full h-auto">
{% endif %}
<!-- Screenshot Status Section -->
{% if page.screenshot_path %}
<div class="mt-4">
<h3 class="text-lg font-medium text-gray-900 mb-2">{% trans "Screenshot" %}</h3>
<img src="{{ page.get_screenshot_url }}"
alt="Page screenshot"
class="rounded-lg shadow-sm max-w-full h-auto">
</div>
</div>
{% elif page.screenshot_error %}
<div class="mt-4">
<div class="rounded-md bg-red-50 p-4">
<div class="flex">
<div class="flex-shrink-0">
<svg class="h-5 w-5 text-red-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
d="M12 8v4m0 4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z"/>
</svg>
</div>
<div class="ml-3">
<h3 class="text-sm font-medium text-red-800">
{% trans "Screenshot Error" %}
</h3>
<div class="mt-2 text-sm text-red-700">
<p>{{ page.screenshot_error }}</p>
</div>
{% if page.screenshot_last_attempt %}
<div class="mt-1 text-xs text-red-600">
{% trans "Last attempt" %}:
<span class="local-datetime" data-timestamp="{{ page.screenshot_last_attempt|date:'c' }}">
{{ page.screenshot_last_attempt|date:"Y-m-d H:i" }}
</span>
</div>
{% endif %}
</div>
</div>
</div>
</div>
{% endif %}
</div>
</div>
</div>