From 5eb404702894df1bf863c592d602e32ec8ad77d9 Mon Sep 17 00:00:00 2001 From: Junwei Zhao Date: Mon, 4 Nov 2024 16:39:15 +1100 Subject: [PATCH] Update --- celerybeat-schedule | Bin 16384 -> 0 bytes core/celery.py | 6 + core/settings.py | 13 +- data/db.sqlite3 | Bin 204800 -> 204800 bytes docker-compose.yml | 40 +---- init.sh | 5 +- k8s/manifest.yaml | 4 + links/__init__.py | 1 + links/page_views.py | 64 ++++--- links/tasks.py | 159 +++++------------- links/templates/links/page_list.html | 27 +-- links/templates/links/screenshot_gallery.html | 116 +++++++++++++ links/urls.py | 16 +- templates/base.html | 9 + 14 files changed, 258 insertions(+), 202 deletions(-) delete mode 100644 celerybeat-schedule create mode 100644 links/__init__.py create mode 100644 links/templates/links/screenshot_gallery.html diff --git a/celerybeat-schedule b/celerybeat-schedule deleted file mode 100644 index 36b459f5c4863b2f394ba03a115442e65417109b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI(&ubGw6bJB0yPCwb&>9bdAkvG9c4?&*3W9~AAd5X{MeyLV*_|d!)=9`t3JnB; z2emM_PA~oeo<;ENSug$pp1gVTFA(%iXKF)f6l)B)51)B)51)B)51)B)6i|Evy-+Pj*KxyWX0GFPy1*{xq6x9cCS*|>MzZvA!Z z(eCf`+V3cAw<>Ue!|?rq`EWe*;nVxi7sGIkkGXnRq06431^&n54`(=A0XV>6Sit7z z9=|!?m2r4;s6OL)a@Ps<(^|mca1PPN*D4pX^K{mpU!<@|-vQxyIL~*lSI){FV^>_g z8li4Z)AF}@n$JAt`M~RI-#q_z^0XzwaNuoi{|kUdtWM1PbN-C>WskO+d0}2^^ufxK z_DxCVBC2IP8F(-JpS^8Ox!(`^T2}hAQ=1f~R7&Md>3-@0gUfm&2}Tr*Sjm`kB6`?RyOhCO1pK z)Px)T&Zwy4(oRl2G)}6r8B@YD8PR&c?p`ef>^Yshv25DQecjiRl6=T){-|aLP#@jV un_L*V{q~PLuKw1sxYsebaR8%l=OtPQKmY;|fB*y_009U<00I!$U*HE_g6hEl diff --git a/core/celery.py b/core/celery.py index 1e3d331..e05efd8 100644 --- a/core/celery.py +++ b/core/celery.py @@ -1,8 +1,14 @@ import os from celery import Celery +from django.conf import settings os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings') app = Celery('core') app.config_from_object('django.conf:settings', namespace='CELERY') app.autodiscover_tasks() + +# Optional: For debugging +@app.task(bind=True) +def debug_task(self): + print(f'Request: {self.request!r}') diff --git a/core/settings.py b/core/settings.py index 0f38c4e..24253a0 100644 --- a/core/settings.py +++ b/core/settings.py @@ -1,7 +1,6 @@ import os from pathlib import Path from django.utils.translation import gettext_lazy as _ -import links.patches # 添加这一行在文件最上方 from django.urls import re_path # 构建路径，如 BASE_DIR / 'subdir' @@ -124,7 +123,7 @@ SIMPLEMDE_OPTIONS = { } MEDIA_URL = '/media/' -MEDIA_ROOT = os.path.join(BASE_DIR, 'media') +MEDIA_ROOT = os.path.join(BASE_DIR, 'data', 'media') LOGGING = { 'version': 1, @@ -205,17 +204,23 @@ CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' CELERY_TIMEZONE = TIME_ZONE # Now TIME_ZONE is defined +# Celery Beat settings +CELERYBEAT_SCHEDULE_FILENAME = os.path.join(BASE_DIR, 'data', 'celery', 'celerybeat-schedule') + # Celery Beat Schedule CELERY_BEAT_SCHEDULE = { 'check-pending-pages': { 'task': 'links.tasks.schedule_pending_pages', - 'schedule': 60.0, # 每60秒运行一次 + 'schedule': 60.0, # Run every 60 seconds }, } # Media files configuration MEDIA_URL = '/media/' -MEDIA_ROOT = os.path.join(BASE_DIR, 'media') +MEDIA_ROOT = os.path.join(BASE_DIR, 'data', 'media') # Ensure media files are served in all environments SERVE_MEDIA = True + +# Celery Beat settings +CELERYBEAT_SCHEDULE_FILENAME = os.path.join(BASE_DIR, 'data', 'celerybeat-schedule') diff --git a/data/db.sqlite3 b/data/db.sqlite3 index 9d4ba2e09eae53a2fdaf4105c1bbb07c0721d72c..a23cfcbd14a053782222debf47de5e1a9992fc53 100644 GIT binary patch delta 863 zcmZXS&rj4q6vuZ3T^eddlIurt7*Pj}tVGRqCza%8Arqy^83C1z0cAEQS|oi`@78HfuJS*9LF7 z>j&Lt#|f6bPRqY?y%!DRy|_np|n?Y93Kb*9>@;F`EvMSBd-2&D~z)o)TcOu#BAQ#s1G1X+U(p{evGX+}+_ zlxrmkC{Ul1OO&<(RCt{x!h*3p%m}U^)TAG1B%lHa(by%RSwX524CO~$>vV`?Z9?Vr zrS^csHm5boluIHY#;BM`5?l^p2xt)!E=y%8J7Wj9w#%K cD0|-KK!LxkEF7m8BY~_?#_f6POp|^9071zXH~;_u diff --git a/docker-compose.yml b/docker-compose.yml index 4d408c6..199d16e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,8 @@ services: python manage.py runserver 0.0.0.0:8000" volumes: - .:/app - - ./media:/app/media + - ./data/media:/app/data/media + - ./data/celery:/app/data/celery ports: - "8000:8000" environment: @@ -17,7 +18,6 @@ services: - CELERY_RESULT_BACKEND=redis://redis:6379/0 depends_on: - redis - - tailwind networks: - app-network @@ -28,7 +28,8 @@ services: command: celery -A core worker --loglevel=info volumes: - .:/app - - ./media:/app/media + - ./data/media:/app/data/media + - ./data/celery:/app/data/celery environment: - DJANGO_SETTINGS_MODULE=core.settings - CELERY_BROKER_URL=redis://redis:6379/0 @@ -40,16 +41,18 @@ services: celery_beat: build: . - command: celery -A core beat --loglevel=info + command: celery -A core beat -s /app/data/celerybeat-schedule --loglevel=info volumes: - .:/app + - ./data/media:/app/data/media + - ./data/celery:/app/data/celery environment: - DJANGO_SETTINGS_MODULE=core.settings - CELERY_BROKER_URL=redis://redis:6379/0 - CELERY_RESULT_BACKEND=redis://redis:6379/0 depends_on: - redis - - celery_worker + - web networks: - app-network @@ -60,33 +63,6 @@ services: networks: - app-network - tailwind: - build: . - command: python manage.py tailwind start - volumes: - - .:/app - ports: - - "3000:3000" - networks: - - app-network - - flower: - build: . - command: celery -A core flower --port=5555 - volumes: - - .:/app - ports: - - "5555:5555" - environment: - - DJANGO_SETTINGS_MODULE=core.settings - - CELERY_BROKER_URL=redis://redis:6379/0 - - CELERY_RESULT_BACKEND=redis://redis:6379/0 - depends_on: - - redis - - celery_worker - networks: - - app-network - volumes: media_volume: static_volume: diff --git a/init.sh b/init.sh index 910249a..b3d77a5 100644 --- a/init.sh +++ b/init.sh @@ -1,3 +1,4 @@ #!/bin/bash -mkdir -p media/screenshots -chmod -R 777 media +mkdir -p data/celery +mkdir -p data/media/screenshots +chmod -R 777 data diff --git a/k8s/manifest.yaml b/k8s/manifest.yaml index 21d7ec4..da26982 100644 --- a/k8s/manifest.yaml +++ b/k8s/manifest.yaml @@ -61,6 +61,10 @@ spec: # { secretKeyRef: { name: database-credentials, key: password } } - name: CSRF_TRUSTED_ORIGINS value: "https://to.junv.cc,https://go.junv.cc,http://go" + - name: CELERY_BROKER_URL + value: "redis://redis-master.db.svc.cluster.local:6379" + - name: CELERY_RESULT_BACKEND + value: "redis://redis-master.db.svc.cluster.local:6379" ports: - containerPort: 8000 name: links-port diff --git a/links/__init__.py b/links/__init__.py new file mode 100644 index 0000000..ef61f61 --- /dev/null +++ b/links/__init__.py @@ -0,0 +1 @@ +default_app_config = 'links.apps.LinksConfig' diff --git a/links/page_views.py b/links/page_views.py index b6c68c3..d9a76c7 100644 --- a/links/page_views.py +++ b/links/page_views.py @@ -1,4 +1,4 @@ -from django.views.generic import ListView, DetailView, CreateView, UpdateView, DeleteView +from django.views.generic import ListView, DetailView, CreateView, UpdateView, DeleteView, TemplateView from django.urls import reverse_lazy from django.http import JsonResponse import requests @@ -18,6 +18,9 @@ from django.conf import settings import time import base64 from django.core.files.base import ContentFile +from .tasks import capture_screenshot +from pathlib import Path +from datetime import datetime class PageListView(ListView): model = Page @@ -36,31 +39,10 @@ class PageCreateView(CreateView): success_url = reverse_lazy('page-list') def form_valid(self, form): - # Handle screenshot data - screenshot_data = self.request.POST.get('screenshot_data') - if screenshot_data and screenshot_data.startswith('data:image/png;base64,'): - # Extract the base64 data - image_data = screenshot_data.split(',')[1] - # Create filename - filename = f"page_screenshot_{form.instance.id}.png" - # Save the image - image_content = ContentFile(base64.b64decode(image_data)) - form.instance.screenshot_path = f'screenshots/{filename}' - - # Ensure directory exists - os.makedirs(os.path.join(settings.MEDIA_ROOT, 'screenshots'), exist_ok=True) - - # Save the file - with open(os.path.join(settings.MEDIA_ROOT, form.instance.screenshot_path), 'wb') as f: - f.write(base64.b64decode(image_data)) - - # Set status - if form.instance.title and form.instance.summary: - form.instance.process_status = Page.ProcessStatus.COMPLETED - else: - form.instance.process_status = Page.ProcessStatus.PENDING - - return super().form_valid(form) + response = super().form_valid(form) + # Use delay() to call the task asynchronously + capture_screenshot.delay(self.object.id) + return response class PageUpdateView(UpdateView): model = Page @@ -176,3 +158,33 @@ class PageViewSet(viewsets.ModelViewSet): self.perform_create(serializer) headers = self.get_success_headers(serializer.data) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) + +class ScreenshotGalleryView(TemplateView): + template_name = 'links/screenshot_gallery.html' + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + screenshots_dir = os.path.join(settings.MEDIA_ROOT, 'screenshots') + screenshots = [] + + if os.path.exists(screenshots_dir): + for filename in os.listdir(screenshots_dir): + if filename.endswith('.png'): + file_path = os.path.join(screenshots_dir, filename) + # Extract page ID and clean up filename + name_parts = filename.split('_', 2) # Split into ['page', id, 'rest_of_name'] + if len(name_parts) >= 3: + display_name = name_parts[2].replace('.png', '') # Remove .png extension + else: + display_name = filename.replace('.png', '') + + screenshots.append({ + 'url': f'/media/screenshots/{filename}', + 'created': datetime.fromtimestamp(os.path.getctime(file_path)), + 'display_name': display_name + }) + + # Sort screenshots by creation time, newest first + screenshots.sort(key=lambda x: x['created'], reverse=True) + context['screenshots'] = screenshots + return context diff --git a/links/tasks.py b/links/tasks.py index 6c72306..c188bf3 100644 --- a/links/tasks.py +++ b/links/tasks.py @@ -4,14 +4,13 @@ from datetime import timedelta import requests from bs4 import BeautifulSoup import logging -from .models import Page from selenium import webdriver from selenium.webdriver.chrome.options import Options import os from django.conf import settings from urllib.parse import quote import time -from core.celery import app +from celery.exceptions import MaxRetriesExceededError logger = logging.getLogger(__name__) @@ -26,135 +25,53 @@ def fibonacci(n): a, b = b, a + b return b -def take_screenshot(url, page_id): - chrome_options = Options() - chrome_options.add_argument('--headless') - chrome_options.add_argument('--no-sandbox') - chrome_options.add_argument('--disable-dev-shm-usage') - chrome_options.add_argument('--window-size=1920,1080') - - driver = webdriver.Chrome(options=chrome_options) - try: - driver.get(url) - # Wait for page to load - time.sleep(5) - - # Create screenshots directory if it doesn't exist - screenshots_dir = os.path.join(settings.MEDIA_ROOT, 'screenshots') - os.makedirs(screenshots_dir, exist_ok=True) - - # Create a safe filename using the page ID and encoded URL - safe_url = quote(url, safe='')[:50] # Limit URL length in filename - filename = f"page_{page_id}_{safe_url}.png" - filepath = os.path.join('screenshots', filename) - full_path = os.path.join(settings.MEDIA_ROOT, filepath) - - # Take screenshot - driver.save_screenshot(full_path) - return filepath - finally: - driver.quit() - @shared_task(bind=True, max_retries=3) -def process_page(self, page_id): +def capture_screenshot(self, page_id): + # Import here to avoid circular import + from .models import Page + try: page = Page.objects.get(id=page_id) - if not page.needs_processing(): - return + chrome_options = Options() + chrome_options.add_argument('--headless') + chrome_options.add_argument('--no-sandbox') + chrome_options.add_argument('--disable-dev-shm-usage') + chrome_options.add_argument('--window-size=1920,1080') - page.process_status = Page.ProcessStatus.PROCESSING - page.save() + driver = webdriver.Chrome(options=chrome_options) + try: + driver.set_page_load_timeout(10) + driver.get(page.url) + time.sleep(2) # Wait for dynamic content - headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.5', - } + # Create screenshots directory + screenshots_dir = os.path.join(settings.MEDIA_ROOT, 'screenshots') + os.makedirs(screenshots_dir, exist_ok=True) - # Fetch page content - response = requests.get(page.url, headers=headers, timeout=10, verify=False) - response.raise_for_status() + # Create filename + safe_url = quote(page.url, safe='')[:50] + filename = f"page_{page_id}_{safe_url}.png" + filepath = os.path.join('screenshots', filename) + full_path = os.path.join(settings.MEDIA_ROOT, filepath) - if response.encoding == 'ISO-8859-1': - response.encoding = response.apparent_encoding or 'utf-8' + # Take screenshot + driver.save_screenshot(full_path) - soup = BeautifulSoup(response.text, 'html.parser') + # Update page model + page.screenshot_path = filepath + page.save() - # Extract title if not present - if not page.title: - title = None - if soup.title: - title = soup.title.string - elif soup.find('h1'): - title = soup.find('h1').get_text(strip=True) - elif soup.find('meta', property='og:title'): - title = soup.find('meta', property='og:title').get('content') + logger.info(f"Successfully captured screenshot for page {page_id}") - if title: - page.title = title.strip() - - # Extract summary if not present - if not page.summary: - description = None - meta_desc = soup.find('meta', {'name': 'description'}) or soup.find('meta', {'property': 'og:description'}) - if meta_desc: - description = meta_desc.get('content') - - if not description: - for tag in soup(['script', 'style', 'nav', 'header', 'footer']): - tag.decompose() - - paragraphs = soup.find_all(['p', 'div']) - for p in paragraphs: - text = p.get_text(strip=True) - if len(text) > 100: - description = text - break - - if description: - page.summary = description[:500] - - # Take screenshot if not present - if not page.screenshot_path: - try: - screenshot_path = take_screenshot(page.url, page.id) - page.screenshot_path = screenshot_path - except Exception as e: - logger.error(f"Failed to take screenshot for page {page_id}: {e}") - - page.process_status = Page.ProcessStatus.COMPLETED - page.save() - logger.info(f"Successfully processed page {page_id}") + finally: + driver.quit() except Exception as exc: - page.retry_count += 1 - page.last_retry_at = timezone.now() - - if page.retry_count >= 3: - page.process_status = Page.ProcessStatus.FAILED - page.error_message = str(exc) - logger.error(f"Failed to process page {page_id} after 3 retries: {exc}") - else: - page.process_status = Page.ProcessStatus.PENDING - retry_delay = fibonacci(page.retry_count) - logger.info(f"Scheduling retry for page {page_id} in {retry_delay} seconds") - self.retry(exc=exc, countdown=retry_delay) - - page.save() - raise exc - -@shared_task -def schedule_pending_pages(): - """Periodic task to schedule processing of pending pages""" - logger.info("Checking for pending pages...") - pending_pages = Page.objects.filter( - process_status=Page.ProcessStatus.PENDING, - retry_count__lt=3 - ).exclude( - last_retry_at__gte=timezone.now() - timedelta(seconds=fibonacci(3)) - ) - - for page in pending_pages: - logger.info(f"Scheduling processing for page {page.id}") - process_page.delay(page.id) + logger.error(f"Failed to capture screenshot for page {page_id}: {exc}") + try: + self.retry(exc=exc, countdown=fibonacci(self.request.retries)) + except MaxRetriesExceededError: + logger.error(f"Max retries exceeded for page {page_id}") + page.error_message = f"Screenshot capture failed: {str(exc)}" + page.save() diff --git a/links/templates/links/page_list.html b/links/templates/links/page_list.html index ed8cf19..bf1dd22 100644 --- a/links/templates/links/page_list.html +++ b/links/templates/links/page_list.html @@ -17,17 +17,22 @@
  • -
    - {% if page.screenshot_path %} - Page thumbnail - {% else %} - Default thumbnail - {% endif %} +
    +
    + {% if page.screenshot_path %} + Page thumbnail + {% else %} +
    + Default thumbnail +
    + {% endif %} +
    diff --git a/links/templates/links/screenshot_gallery.html b/links/templates/links/screenshot_gallery.html new file mode 100644 index 0000000..28c197d --- /dev/null +++ b/links/templates/links/screenshot_gallery.html @@ -0,0 +1,116 @@ +{% extends 'base.html' %} +{% load i18n %} +{% load static %} + +{% block extra_css %} + +{% endblock %} + +{% block content %} + + + +
    +
    +

    {% trans "Screenshots" %}

    + {{ screenshots|length }} {% trans "images" %} +
    + + {% if screenshots %} +
    + {% for screenshot in screenshots %} +
    + +
    + Screenshot +
    + +
    +
    + {{ screenshot.display_name }} +
    +
    +
    + {% endfor %} +
    + {% else %} +
    + {% trans "No screenshots available yet." %} +
    + {% endif %} +
    +{% endblock %} + +{% block extra_js %} + +{% endblock %} diff --git a/links/urls.py b/links/urls.py index 58eb224..a2ac91b 100644 --- a/links/urls.py +++ b/links/urls.py @@ -2,6 +2,7 @@ from django.urls import path, include from rest_framework.routers import DefaultRouter from django.urls import re_path from . import views +from . import page_views router = DefaultRouter(trailing_slash=False) router.register(r'pages', views.PageViewSet, basename='api-page') @@ -23,12 +24,13 @@ urlpatterns = [ path('export/', views.export_links, name='export_links'), # Pages - path('ui/pages/', views.PageListView.as_view(), name='page-list'), - path('ui/pages/new/', views.PageCreateView.as_view(), name='page-create'), - path('ui/pages//', views.PageDetailView.as_view(), name='page-detail'), - path('ui/pages//edit/', views.PageUpdateView.as_view(), name='page-update'), - path('ui/pages//delete/', views.PageDeleteView.as_view(), name='page-delete'), - path('fetch-page-info/', views.fetch_page_info, name='fetch-page-info'), + path('ui/pages/', page_views.PageListView.as_view(), name='page-list'), + path('ui/pages/new/', page_views.PageCreateView.as_view(), name='page-create'), + path('ui/pages//', page_views.PageDetailView.as_view(), name='page-detail'), + path('ui/pages//edit/', page_views.PageUpdateView.as_view(), name='page-update'), + path('ui/pages//delete/', page_views.PageDeleteView.as_view(), name='page-delete'), + path('fetch-page-info/', page_views.fetch_page_info, name='fetch-page-info'), + path('ui/screenshots/', page_views.ScreenshotGalleryView.as_view(), name='screenshot-gallery'), # API URLs - place before catch-all alias routes path('api/', include(router.urls)), @@ -37,4 +39,6 @@ urlpatterns = [ path('/', views.redirect_to_original, name='redirect_to_original'), path('//', views.redirect_to_original, name='redirect_to_original_with_param'), path('alias//', views.LinkDetailView.as_view(), name='link_detail_by_alias'), + + ] diff --git a/templates/base.html b/templates/base.html index fba7e5e..a68d88c 100644 --- a/templates/base.html +++ b/templates/base.html @@ -101,6 +101,15 @@ {% trans "Help" %}
    + +
    + + + + {% trans "Screenshots" %} +
    +