diff --git a/Dockerfile.worker b/Dockerfile.worker new file mode 100644 index 0000000..2e93377 --- /dev/null +++ b/Dockerfile.worker @@ -0,0 +1,23 @@ +FROM python:3.12-slim + +# Install Chrome and dependencies +RUN apt-get update && apt-get install -y \ + wget \ + gnupg \ + chromium \ + chromium-driver \ + && rm -rf /var/lib/apt/lists/* + +# Set environment variables +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +# Set work directory +WORKDIR /app + +# Install Python dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy project +COPY . . diff --git a/README.md b/README.md index 54772c9..b482471 100644 --- a/README.md +++ b/README.md @@ -180,3 +180,34 @@ Flower provides: - Runtime - Result - Stack traces for failed tasks + + +## Why we have `core` folder and what it is? + +The core folder is the Django project's root configuration directory, created when you first run django-admin startproject core. It serves several essential purposes: +Project-wide settings (settings.py): +Database configuration +Installed apps +Middleware +Static/media files settings +Other global settings +Root URL configuration (urls.py): +Main URL routing +Admin site URLs +Including app-specific URLs +3. WSGI/ASGI configuration: +For production deployment +Server integration +Celery configuration (celery.py): +Project-wide Celery settings +Task discovery +We can't completely remove it because Django requires this project-level configuration. The links folder is an app within the project, and you could have multiple apps in one project. +Think of it this way: +core/ = Project configuration (can contain multiple apps) +links/ = One specific app's code +This separation allows you to: +1. Have multiple apps in one project +Reuse apps across different projects +Keep project-wide settings separate from app-specific code +If you want to simplify, you could rename core to something shorter like config or core, but you can't eliminate it entirely as it's a fundamental part of Django's project structure. +Would you like me to show you how to rename it to something shorter and cleaner? diff --git a/celery_beat.sh b/celery_beat.sh index fcb4c02..ec500db 100755 --- a/celery_beat.sh +++ b/celery_beat.sh @@ -2,4 +2,4 @@ set -eu source 3.12/bin/activate echo "Starting Django development server..." -celery -A url_manager beat -l info +celery -A core beat -l info diff --git a/celerybeat-schedule b/celerybeat-schedule index c3f159d..36b459f 100644 Binary files a/celerybeat-schedule and b/celerybeat-schedule differ diff --git a/url_manager/celery.py b/core/celery.py similarity index 56% rename from url_manager/celery.py rename to core/celery.py index d6a0c58..1e3d331 100644 --- a/url_manager/celery.py +++ b/core/celery.py @@ -1,8 +1,8 @@ import os from celery import Celery -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'url_manager.settings') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings') -app = Celery('url_manager') +app = Celery('core') app.config_from_object('django.conf:settings', namespace='CELERY') app.autodiscover_tasks() diff --git a/url_manager/middleware.py b/core/middleware.py similarity index 100% rename from url_manager/middleware.py rename to core/middleware.py diff --git a/url_manager/settings.py b/core/settings.py similarity index 93% rename from url_manager/settings.py rename to core/settings.py index 715b446..0f38c4e 100644 --- a/url_manager/settings.py +++ b/core/settings.py @@ -22,12 +22,12 @@ INSTALLED_APPS = [ 'markdown', # 只需要基本的markdown包 ] -ROOT_URLCONF = 'url_manager.urls' +ROOT_URLCONF = 'core.urls' MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', - 'url_manager.middleware.CustomLocaleMiddleware', # 替换原来的 LocaleMiddleware + 'core.middleware.CustomLocaleMiddleware', # 替换原来的 LocaleMiddleware 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', @@ -97,7 +97,7 @@ USE_TZ = True # ... 其他设置 ... -ROOT_URLCONF = 'url_manager.urls' +ROOT_URLCONF = 'core.urls' # ... 其他设置 ... @@ -176,7 +176,7 @@ LOCALE_INDEPENDENT_PATHS = [ MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', - 'url_manager.middleware.CustomLocaleMiddleware', # 使用自定义中间件 + 'core.middleware.CustomLocaleMiddleware', # 使用自定义中间件 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', @@ -212,3 +212,10 @@ CELERY_BEAT_SCHEDULE = { 'schedule': 60.0, # 每60秒运行一次 }, } + +# Media files configuration +MEDIA_URL = '/media/' +MEDIA_ROOT = os.path.join(BASE_DIR, 'media') + +# Ensure media files are served in all environments +SERVE_MEDIA = True diff --git a/url_manager/tailwind.config.js b/core/tailwind.config.js similarity index 100% rename from url_manager/tailwind.config.js rename to core/tailwind.config.js diff --git a/url_manager/urls.py b/core/urls.py similarity index 80% rename from url_manager/urls.py rename to core/urls.py index 8705d68..6cb7a12 100644 --- a/url_manager/urls.py +++ b/core/urls.py @@ -1,9 +1,10 @@ from django.contrib import admin -from django.urls import path, include +from django.urls import path, include, re_path from django.conf.urls.i18n import i18n_patterns from django.conf import settings from django.conf.urls.static import static from links.views import LinkDetailView, LinkUpdateView, CustomLinkView +from django.views.static import serve urlpatterns = [ path('i18n/', include('django.conf.urls.i18n')), @@ -11,8 +12,11 @@ urlpatterns = [ path('link//', LinkDetailView.as_view(), name='link_detail'), path('link//edit/', LinkUpdateView.as_view(), name='link_update'), path('custom//', CustomLinkView.as_view(), name='custom_link'), + # Serve media files directly + re_path(r'^media/(?P.*)$', serve, { + 'document_root': settings.MEDIA_ROOT, + }), ] - urlpatterns += i18n_patterns( path('admin/', admin.site.urls), path('', include('links.urls')), diff --git a/url_manager/wsgi.py b/core/wsgi.py similarity index 57% rename from url_manager/wsgi.py rename to core/wsgi.py index 2800bf9..4157369 100644 --- a/url_manager/wsgi.py +++ b/core/wsgi.py @@ -1,6 +1,6 @@ import os from django.core.wsgi import get_wsgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'url_manager.settings') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings') application = get_wsgi_application() diff --git a/data/db.sqlite3 b/data/db.sqlite3 index 72f7ca2..9d4ba2e 100644 Binary files a/data/db.sqlite3 and b/data/db.sqlite3 differ diff --git a/docker-compose.yml b/docker-compose.yml index 411cba3..4d408c6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,5 @@ +version: '3.8' + services: web: build: . @@ -6,10 +8,11 @@ services: python manage.py runserver 0.0.0.0:8000" volumes: - .:/app + - ./media:/app/media ports: - "8000:8000" environment: - - DJANGO_SETTINGS_MODULE=url_manager.settings + - DJANGO_SETTINGS_MODULE=core.settings - CELERY_BROKER_URL=redis://redis:6379/0 - CELERY_RESULT_BACKEND=redis://redis:6379/0 depends_on: @@ -19,27 +22,29 @@ services: - app-network celery_worker: - build: . - command: celery -A url_manager worker --loglevel=info + build: + context: . + dockerfile: Dockerfile.worker + command: celery -A core worker --loglevel=info volumes: - .:/app + - ./media:/app/media environment: - - DJANGO_SETTINGS_MODULE=url_manager.settings + - DJANGO_SETTINGS_MODULE=core.settings - CELERY_BROKER_URL=redis://redis:6379/0 - CELERY_RESULT_BACKEND=redis://redis:6379/0 depends_on: - redis - - web networks: - app-network celery_beat: build: . - command: celery -A url_manager beat --loglevel=info + command: celery -A core beat --loglevel=info volumes: - .:/app environment: - - DJANGO_SETTINGS_MODULE=url_manager.settings + - DJANGO_SETTINGS_MODULE=core.settings - CELERY_BROKER_URL=redis://redis:6379/0 - CELERY_RESULT_BACKEND=redis://redis:6379/0 depends_on: @@ -67,13 +72,13 @@ services: flower: build: . - command: celery -A url_manager flower --port=5555 + command: celery -A core flower --port=5555 volumes: - .:/app ports: - "5555:5555" environment: - - DJANGO_SETTINGS_MODULE=url_manager.settings + - DJANGO_SETTINGS_MODULE=core.settings - CELERY_BROKER_URL=redis://redis:6379/0 - CELERY_RESULT_BACKEND=redis://redis:6379/0 depends_on: @@ -82,6 +87,10 @@ services: networks: - app-network +volumes: + media_volume: + static_volume: + networks: app-network: driver: bridge diff --git a/init.sh b/init.sh new file mode 100644 index 0000000..910249a --- /dev/null +++ b/init.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p media/screenshots +chmod -R 777 media diff --git a/links/migrations/0012_page_screenshot_path.py b/links/migrations/0012_page_screenshot_path.py new file mode 100644 index 0000000..81775f9 --- /dev/null +++ b/links/migrations/0012_page_screenshot_path.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.9 on 2024-11-04 03:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('links', '0011_page_error_message_page_last_retry_at_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='page', + name='screenshot_path', + field=models.CharField(blank=True, max_length=255), + ), + ] diff --git a/links/models.py b/links/models.py index 0884907..a5ec904 100644 --- a/links/models.py +++ b/links/models.py @@ -4,6 +4,8 @@ from django.urls import reverse import logging import re from django.utils import timezone +from django.conf import settings +import os logger = logging.getLogger(__name__) @@ -151,6 +153,9 @@ class Page(models.Model): last_retry_at = models.DateTimeField(null=True, blank=True) error_message = models.TextField(blank=True) + # New field for screenshot + screenshot_path = models.CharField(max_length=255, blank=True) + class Meta: ordering = ['-updated_at'] verbose_name = _('Page') @@ -163,6 +168,11 @@ class Page(models.Model): return reverse('page-detail', kwargs={'pk': self.pk}) def needs_processing(self): - return (not self.title or not self.summary) and \ + return (not self.title or not self.summary or not self.screenshot_path) and \ self.process_status != self.ProcessStatus.FAILED and \ self.retry_count < 3 + + def get_screenshot_url(self): + if self.screenshot_path: + return f'/media/{self.screenshot_path}' + return None diff --git a/links/page_views.py b/links/page_views.py index 57f3f9c..b6c68c3 100644 --- a/links/page_views.py +++ b/links/page_views.py @@ -3,7 +3,7 @@ from django.urls import reverse_lazy from django.http import JsonResponse import requests from bs4 import BeautifulSoup -from urllib.parse import urlparse +from urllib.parse import urlparse, quote import re from rest_framework import viewsets, status from rest_framework.response import Response @@ -11,6 +11,13 @@ from rest_framework.pagination import PageNumberPagination from .models import Page from .forms import PageForm from .serializers import PageSerializer +import os +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from django.conf import settings +import time +import base64 +from django.core.files.base import ContentFile class PageListView(ListView): model = Page @@ -29,7 +36,25 @@ class PageCreateView(CreateView): success_url = reverse_lazy('page-list') def form_valid(self, form): - # 如果标题和摘要都已经自动填充，则将状态设置为已完成 + # Handle screenshot data + screenshot_data = self.request.POST.get('screenshot_data') + if screenshot_data and screenshot_data.startswith('data:image/png;base64,'): + # Extract the base64 data + image_data = screenshot_data.split(',')[1] + # Create filename + filename = f"page_screenshot_{form.instance.id}.png" + # Save the image + image_content = ContentFile(base64.b64decode(image_data)) + form.instance.screenshot_path = f'screenshots/{filename}' + + # Ensure directory exists + os.makedirs(os.path.join(settings.MEDIA_ROOT, 'screenshots'), exist_ok=True) + + # Save the file + with open(os.path.join(settings.MEDIA_ROOT, form.instance.screenshot_path), 'wb') as f: + f.write(base64.b64decode(image_data)) + + # Set status if form.instance.title and form.instance.summary: form.instance.process_status = Page.ProcessStatus.COMPLETED else: @@ -48,12 +73,30 @@ class PageDeleteView(DeleteView): template_name = 'links/page_confirm_delete.html' success_url = reverse_lazy('page-list') +def take_screenshot(url): + chrome_options = Options() + chrome_options.add_argument('--headless') + chrome_options.add_argument('--no-sandbox') + chrome_options.add_argument('--disable-dev-shm-usage') + chrome_options.add_argument('--window-size=1920,1080') + + driver = webdriver.Chrome(options=chrome_options) + try: + driver.set_page_load_timeout(10) # Set page load timeout + driver.get(url) + time.sleep(2) # Wait for any dynamic content + + # Take screenshot and convert to base64 + screenshot = driver.get_screenshot_as_base64() + return f"data:image/png;base64,{screenshot}" + finally: + driver.quit() + def fetch_page_info(request): url = request.GET.get('url') if not url: return JsonResponse({'error': 'URL is required'}, status=400) - # 移除URL开头可能的@符号 url = url.lstrip('@') try: @@ -63,14 +106,13 @@ def fetch_page_info(request): 'Accept-Language': 'en-US,en;q=0.5', } - response = requests.get(url, headers=headers, timeout=10, verify=False) - + response = requests.get(url, headers=headers, timeout=5, verify=False) # Reduced timeout to 5 seconds if response.encoding == 'ISO-8859-1': response.encoding = response.apparent_encoding or 'utf-8' soup = BeautifulSoup(response.text, 'html.parser') - # 获取标题 + # Get title title = None if soup.title: title = soup.title.string @@ -85,7 +127,7 @@ def fetch_page_info(request): else: title = urlparse(url).netloc - # 获取描述 + # Get description description = None meta_desc = soup.find('meta', {'name': 'description'}) or soup.find('meta', {'property': 'og:description'}) if meta_desc: @@ -113,10 +155,6 @@ def fetch_page_info(request): 'summary': description }) - except requests.exceptions.RequestException as e: - return JsonResponse({ - 'error': f'Failed to fetch page: {str(e)}' - }, status=400) except Exception as e: return JsonResponse({ 'error': f'Error processing page: {str(e)}' diff --git a/links/tasks.py b/links/tasks.py index da475c9..6c72306 100644 --- a/links/tasks.py +++ b/links/tasks.py @@ -5,6 +5,13 @@ import requests from bs4 import BeautifulSoup import logging from .models import Page +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +import os +from django.conf import settings +from urllib.parse import quote +import time +from core.celery import app logger = logging.getLogger(__name__) @@ -19,16 +26,43 @@ def fibonacci(n): a, b = b, a + b return b +def take_screenshot(url, page_id): + chrome_options = Options() + chrome_options.add_argument('--headless') + chrome_options.add_argument('--no-sandbox') + chrome_options.add_argument('--disable-dev-shm-usage') + chrome_options.add_argument('--window-size=1920,1080') + + driver = webdriver.Chrome(options=chrome_options) + try: + driver.get(url) + # Wait for page to load + time.sleep(5) + + # Create screenshots directory if it doesn't exist + screenshots_dir = os.path.join(settings.MEDIA_ROOT, 'screenshots') + os.makedirs(screenshots_dir, exist_ok=True) + + # Create a safe filename using the page ID and encoded URL + safe_url = quote(url, safe='')[:50] # Limit URL length in filename + filename = f"page_{page_id}_{safe_url}.png" + filepath = os.path.join('screenshots', filename) + full_path = os.path.join(settings.MEDIA_ROOT, filepath) + + # Take screenshot + driver.save_screenshot(full_path) + return filepath + finally: + driver.quit() + @shared_task(bind=True, max_retries=3) def process_page(self, page_id): try: page = Page.objects.get(id=page_id) - # Check if page still needs processing if not page.needs_processing(): return - # Update status to processing page.process_status = Page.ProcessStatus.PROCESSING page.save() @@ -38,6 +72,7 @@ def process_page(self, page_id): 'Accept-Language': 'en-US,en;q=0.5', } + # Fetch page content response = requests.get(page.url, headers=headers, timeout=10, verify=False) response.raise_for_status() @@ -80,6 +115,14 @@ def process_page(self, page_id): if description: page.summary = description[:500] + # Take screenshot if not present + if not page.screenshot_path: + try: + screenshot_path = take_screenshot(page.url, page.id) + page.screenshot_path = screenshot_path + except Exception as e: + logger.error(f"Failed to take screenshot for page {page_id}: {e}") + page.process_status = Page.ProcessStatus.COMPLETED page.save() logger.info(f"Successfully processed page {page_id}") @@ -94,7 +137,6 @@ def process_page(self, page_id): logger.error(f"Failed to process page {page_id} after 3 retries: {exc}") else: page.process_status = Page.ProcessStatus.PENDING - # Schedule retry with fibonacci backoff retry_delay = fibonacci(page.retry_count) logger.info(f"Scheduling retry for page {page_id} in {retry_delay} seconds") self.retry(exc=exc, countdown=retry_delay) diff --git a/links/templates/links/page_detail.html b/links/templates/links/page_detail.html index b580008..bd2df2c 100644 --- a/links/templates/links/page_detail.html +++ b/links/templates/links/page_detail.html @@ -1,6 +1,7 @@ {% extends 'base.html' %} {% load i18n %} {% load markdown_extras %} +{% load static %} {% block content %}
@@ -108,6 +109,23 @@
{% endif %} + + +
+

{% trans "Screenshot" %}

+
+ {% if page.screenshot_path %} + Page screenshot + {% else %} + Default screenshot + {% endif %} +
+
diff --git a/links/templates/links/page_form.html b/links/templates/links/page_form.html index d0e87ce..05fea62 100644 --- a/links/templates/links/page_form.html +++ b/links/templates/links/page_form.html @@ -3,14 +3,32 @@ {% block extra_css %}