home/links
2
0
Fork 0
mirror of https://github.com/wahyd4/links.git synced 2026-08-08 21:04:53 +10:00
Code Issues Packages Projects Releases Wiki Activity

Update

Browse Source
This commit is contained in:
junv
2024-11-04 16:39:15 +11:00
parent ea08f337d7
commit 5eb4047028
14 changed files with 258 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files
celerybeat-schedule
BIN
View File
Binary file not shown.
core/celery.py
+6
View File
@@ -1,8 +1,14 @@
import os
from celery import Celery
from django.conf import settings
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
app = Celery('core')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()
# Optional: For debugging
@app.task(bind=True)
def debug_task(self):
print(f'Request: {self.request!r}')
core/settings.py
+9 -4
View File
@@ -1,7 +1,6 @@
import os
from pathlib import Path
from django.utils.translation import gettext_lazy as _
import links.patches # 添加这一行在文件最上方
from django.urls import re_path
# 构建路径，如 BASE_DIR / 'subdir'
@@ -124,7 +123,7 @@ SIMPLEMDE_OPTIONS = {
}
MEDIA_URL = '/media/'
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
MEDIA_ROOT = os.path.join(BASE_DIR, 'data', 'media')
LOGGING = {
'version': 1,
@@ -205,17 +204,23 @@ CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = TIME_ZONE # Now TIME_ZONE is defined
# Celery Beat settings
CELERYBEAT_SCHEDULE_FILENAME = os.path.join(BASE_DIR, 'data', 'celery', 'celerybeat-schedule')
# Celery Beat Schedule
CELERY_BEAT_SCHEDULE = {
'check-pending-pages': {
'task': 'links.tasks.schedule_pending_pages',
'schedule': 60.0, # 每60秒运行一次
'schedule': 60.0, # Run every 60 seconds
},
}
# Media files configuration
MEDIA_URL = '/media/'
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
MEDIA_ROOT = os.path.join(BASE_DIR, 'data', 'media')
# Ensure media files are served in all environments
SERVE_MEDIA = True
# Celery Beat settings
CELERYBEAT_SCHEDULE_FILENAME = os.path.join(BASE_DIR, 'data', 'celerybeat-schedule')
data/db.sqlite3
BIN
View File
Binary file not shown.
docker-compose.yml
+8 -32
View File
@@ -8,7 +8,8 @@ services:
python manage.py runserver 0.0.0.0:8000"
volumes:
- .:/app
- ./media:/app/media
- ./data/media:/app/data/media
- ./data/celery:/app/data/celery
ports:
- "8000:8000"
environment:
@@ -17,7 +18,6 @@ services:
- CELERY_RESULT_BACKEND=redis://redis:6379/0
depends_on:
- redis
- tailwind
networks:
- app-network
@@ -28,7 +28,8 @@ services:
command: celery -A core worker --loglevel=info
volumes:
- .:/app
- ./media:/app/media
- ./data/media:/app/data/media
- ./data/celery:/app/data/celery
environment:
- DJANGO_SETTINGS_MODULE=core.settings
- CELERY_BROKER_URL=redis://redis:6379/0
@@ -40,16 +41,18 @@ services:
celery_beat:
build: .
command: celery -A core beat --loglevel=info
command: celery -A core beat -s /app/data/celerybeat-schedule --loglevel=info
volumes:
- .:/app
- ./data/media:/app/data/media
- ./data/celery:/app/data/celery
environment:
- DJANGO_SETTINGS_MODULE=core.settings
- CELERY_BROKER_URL=redis://redis:6379/0
- CELERY_RESULT_BACKEND=redis://redis:6379/0
depends_on:
- redis
- celery_worker
- web
networks:
- app-network
@@ -60,33 +63,6 @@ services:
networks:
- app-network
tailwind:
build: .
command: python manage.py tailwind start
volumes:
- .:/app
ports:
- "3000:3000"
networks:
- app-network
flower:
build: .
command: celery -A core flower --port=5555
volumes:
- .:/app
ports:
- "5555:5555"
environment:
- DJANGO_SETTINGS_MODULE=core.settings
- CELERY_BROKER_URL=redis://redis:6379/0
- CELERY_RESULT_BACKEND=redis://redis:6379/0
depends_on:
- redis
- celery_worker
networks:
- app-network
volumes:
media_volume:
static_volume:
init.sh
+3 -2
View File
@@ -1,3 +1,4 @@
#!/bin/bash
mkdir -p media/screenshots
chmod -R 777 media
mkdir -p data/celery
mkdir -p data/media/screenshots
chmod -R 777 data
k8s/manifest.yaml
+4
View File
@@ -61,6 +61,10 @@ spec:
# { secretKeyRef: { name: database-credentials, key: password } }
- name: CSRF_TRUSTED_ORIGINS
value: "https://to.junv.cc,https://go.junv.cc,http://go"
- name: CELERY_BROKER_URL
value: "redis://redis-master.db.svc.cluster.local:6379"
- name: CELERY_RESULT_BACKEND
value: "redis://redis-master.db.svc.cluster.local:6379"
ports:
- containerPort: 8000
name: links-port
links/__init__.py
+1
View File
@@ -0,0 +1 @@
default_app_config = 'links.apps.LinksConfig'
links/page_views.py
+38 -26
View File
@@ -1,4 +1,4 @@
from django.views.generic import ListView, DetailView, CreateView, UpdateView, DeleteView
from django.views.generic import ListView, DetailView, CreateView, UpdateView, DeleteView, TemplateView
from django.urls import reverse_lazy
from django.http import JsonResponse
import requests
@@ -18,6 +18,9 @@ from django.conf import settings
import time
import base64
from django.core.files.base import ContentFile
from .tasks import capture_screenshot
from pathlib import Path
from datetime import datetime
class PageListView(ListView):
model = Page
@@ -36,31 +39,10 @@ class PageCreateView(CreateView):
success_url = reverse_lazy('page-list')
def form_valid(self, form):
# Handle screenshot data
screenshot_data = self.request.POST.get('screenshot_data')
if screenshot_data and screenshot_data.startswith('data:image/png;base64,'):
# Extract the base64 data
image_data = screenshot_data.split(',')[1]
# Create filename
filename = f"page_screenshot_{form.instance.id}.png"
# Save the image
image_content = ContentFile(base64.b64decode(image_data))
form.instance.screenshot_path = f'screenshots/{filename}'
# Ensure directory exists
os.makedirs(os.path.join(settings.MEDIA_ROOT, 'screenshots'), exist_ok=True)
# Save the file
with open(os.path.join(settings.MEDIA_ROOT, form.instance.screenshot_path), 'wb') as f:
f.write(base64.b64decode(image_data))
# Set status
if form.instance.title and form.instance.summary:
form.instance.process_status = Page.ProcessStatus.COMPLETED
else:
form.instance.process_status = Page.ProcessStatus.PENDING
return super().form_valid(form)
response = super().form_valid(form)
# Use delay() to call the task asynchronously
capture_screenshot.delay(self.object.id)
return response
class PageUpdateView(UpdateView):
model = Page
@@ -176,3 +158,33 @@ class PageViewSet(viewsets.ModelViewSet):
self.perform_create(serializer)
headers = self.get_success_headers(serializer.data)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
class ScreenshotGalleryView(TemplateView):
template_name = 'links/screenshot_gallery.html'
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
screenshots_dir = os.path.join(settings.MEDIA_ROOT, 'screenshots')
screenshots = []
if os.path.exists(screenshots_dir):
for filename in os.listdir(screenshots_dir):
if filename.endswith('.png'):
file_path = os.path.join(screenshots_dir, filename)
# Extract page ID and clean up filename
name_parts = filename.split('_', 2) # Split into ['page', id, 'rest_of_name']
if len(name_parts) >= 3:
display_name = name_parts[2].replace('.png', '') # Remove .png extension
else:
display_name = filename.replace('.png', '')
screenshots.append({
'url': f'/media/screenshots/{filename}',
'created': datetime.fromtimestamp(os.path.getctime(file_path)),
'display_name': display_name
})
# Sort screenshots by creation time, newest first
screenshots.sort(key=lambda x: x['created'], reverse=True)
context['screenshots'] = screenshots
return context
links/tasks.py
+38 -121
View File
@@ -4,14 +4,13 @@ from datetime import timedelta
import requests
from bs4 import BeautifulSoup
import logging
from .models import Page
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import os
from django.conf import settings
from urllib.parse import quote
import time
from core.celery import app
from celery.exceptions import MaxRetriesExceededError
logger = logging.getLogger(__name__)
@@ -26,135 +25,53 @@ def fibonacci(n):
a, b = b, a + b
return b
def take_screenshot(url, page_id):
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('--window-size=1920,1080')
driver = webdriver.Chrome(options=chrome_options)
try:
driver.get(url)
# Wait for page to load
time.sleep(5)
# Create screenshots directory if it doesn't exist
screenshots_dir = os.path.join(settings.MEDIA_ROOT, 'screenshots')
os.makedirs(screenshots_dir, exist_ok=True)
# Create a safe filename using the page ID and encoded URL
safe_url = quote(url, safe='')[:50] # Limit URL length in filename
filename = f"page_{page_id}_{safe_url}.png"
filepath = os.path.join('screenshots', filename)
full_path = os.path.join(settings.MEDIA_ROOT, filepath)
# Take screenshot
driver.save_screenshot(full_path)
return filepath
finally:
driver.quit()
@shared_task(bind=True, max_retries=3)
def process_page(self, page_id):
def capture_screenshot(self, page_id):
# Import here to avoid circular import
from .models import Page
try:
page = Page.objects.get(id=page_id)
if not page.needs_processing():
return
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('--window-size=1920,1080')
page.process_status = Page.ProcessStatus.PROCESSING
page.save()
driver = webdriver.Chrome(options=chrome_options)
try:
driver.set_page_load_timeout(10)
driver.get(page.url)
time.sleep(2) # Wait for dynamic content
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
}
# Create screenshots directory
screenshots_dir = os.path.join(settings.MEDIA_ROOT, 'screenshots')
os.makedirs(screenshots_dir, exist_ok=True)
# Fetch page content
response = requests.get(page.url, headers=headers, timeout=10, verify=False)
response.raise_for_status()
# Create filename
safe_url = quote(page.url, safe='')[:50]
filename = f"page_{page_id}_{safe_url}.png"
filepath = os.path.join('screenshots', filename)
full_path = os.path.join(settings.MEDIA_ROOT, filepath)
if response.encoding == 'ISO-8859-1':
response.encoding = response.apparent_encoding or 'utf-8'
# Take screenshot
driver.save_screenshot(full_path)
soup = BeautifulSoup(response.text, 'html.parser')
# Update page model
page.screenshot_path = filepath
page.save()
# Extract title if not present
if not page.title:
title = None
if soup.title:
title = soup.title.string
elif soup.find('h1'):
title = soup.find('h1').get_text(strip=True)
elif soup.find('meta', property='og:title'):
title = soup.find('meta', property='og:title').get('content')
logger.info(f"Successfully captured screenshot for page {page_id}")
if title:
page.title = title.strip()
# Extract summary if not present
if not page.summary:
description = None
meta_desc = soup.find('meta', {'name': 'description'}) or soup.find('meta', {'property': 'og:description'})
if meta_desc:
description = meta_desc.get('content')
if not description:
for tag in soup(['script', 'style', 'nav', 'header', 'footer']):
tag.decompose()
paragraphs = soup.find_all(['p', 'div'])
for p in paragraphs:
text = p.get_text(strip=True)
if len(text) > 100:
description = text
break
if description:
page.summary = description[:500]
# Take screenshot if not present
if not page.screenshot_path:
try:
screenshot_path = take_screenshot(page.url, page.id)
page.screenshot_path = screenshot_path
except Exception as e:
logger.error(f"Failed to take screenshot for page {page_id}: {e}")
page.process_status = Page.ProcessStatus.COMPLETED
page.save()
logger.info(f"Successfully processed page {page_id}")
finally:
driver.quit()
except Exception as exc:
page.retry_count += 1
page.last_retry_at = timezone.now()
if page.retry_count >= 3:
page.process_status = Page.ProcessStatus.FAILED
page.error_message = str(exc)
logger.error(f"Failed to process page {page_id} after 3 retries: {exc}")
else:
page.process_status = Page.ProcessStatus.PENDING
retry_delay = fibonacci(page.retry_count)
logger.info(f"Scheduling retry for page {page_id} in {retry_delay} seconds")
self.retry(exc=exc, countdown=retry_delay)
page.save()
raise exc
@shared_task
def schedule_pending_pages():
"""Periodic task to schedule processing of pending pages"""
logger.info("Checking for pending pages...")
pending_pages = Page.objects.filter(
process_status=Page.ProcessStatus.PENDING,
retry_count__lt=3
).exclude(
last_retry_at__gte=timezone.now() - timedelta(seconds=fibonacci(3))
)
for page in pending_pages:
logger.info(f"Scheduling processing for page {page.id}")
process_page.delay(page.id)
logger.error(f"Failed to capture screenshot for page {page_id}: {exc}")
try:
self.retry(exc=exc, countdown=fibonacci(self.request.retries))
except MaxRetriesExceededError:
logger.error(f"Max retries exceeded for page {page_id}")
page.error_message = f"Screenshot capture failed: {str(exc)}"
page.save()
links/templates/links/page_list.html
+16 -11
View File
@@ -17,17 +17,22 @@
<li class="p-4">
<div class="flex space-x-4">
<!-- Thumbnail Container -->
<div class="flex-shrink-0 w-[200px] h-[200px] rounded-lg overflow-hidden bg-gray-100">
{% if page.screenshot_path %}
<img src="{{ page.get_screenshot_url }}"
alt="Page thumbnail"
class="w-[200px] h-[200px] object-cover"
loading="lazy">
{% else %}
<img src="{% static 'images/default-screenshot.png' %}"
alt="Default thumbnail"
class="w-[200px] h-[200px] object-cover">
{% endif %}
<div class="flex-shrink-0 w-48 h-48 rounded-lg overflow-hidden bg-gray-100 border border-gray-200">
<div class="w-full h-full flex items-center justify-center">
{% if page.screenshot_path %}
<img src="{{ page.get_screenshot_url }}"
alt="Page thumbnail"
class="w-full h-full object-cover"
loading="lazy"
onerror="this.onerror=null; this.src='{% static 'images/default-screenshot.png' %}';">
{% else %}
<div class="w-full h-full flex items-center justify-center">
<img src="{% static 'images/default-screenshot.png' %}"
alt="Default thumbnail"
class="w-3/4 h-3/4 object-contain">
</div>
{% endif %}
</div>
</div>
<!-- Content Container -->
links/templates/links/screenshot_gallery.html
+116
View File
@@ -0,0 +1,116 @@
{% extends 'base.html' %}
{% load i18n %}
{% load static %}
{% block extra_css %}
<style>
.modal {
display: none;
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
background-color: rgba(0, 0, 0, 0.9);
z-index: 1000;
cursor: zoom-out;
}
.modal-content {
max-width: 90%;
max-height: 90vh;
margin: auto;
display: block;
position: relative;
top: 50%;
transform: translateY(-50%);
object-fit: contain;
}
.zoom-in {
cursor: zoom-in;
}
.filename {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
</style>
{% endblock %}
{% block content %}
<!-- Image Modal -->
<div id="imageModal" class="modal" onclick="closeModal()">
<img id="modalImage" class="modal-content" src="" alt="Zoomed screenshot">
</div>
<div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
<div class="flex justify-between items-center mb-6">
<h1 class="text-2xl font-bold text-gray-900">{% trans "Screenshots" %}</h1>
<span class="text-gray-500">{{ screenshots|length }} {% trans "images" %}</span>
</div>
{% if screenshots %}
<div class="grid grid-cols-2 md:grid-cols-3 lg:grid-cols-3 gap-4">
{% for screenshot in screenshots %}
<div class="relative group bg-white p-3 rounded-lg shadow-sm">
<!-- Image Container -->
<div class="aspect-w-16 aspect-h-9 rounded-lg overflow-hidden bg-gray-100">
<img src="{{ screenshot.url }}"
alt="Screenshot"
class="w-full h-full object-cover transition-transform duration-200 group-hover:scale-105 zoom-in"
loading="lazy"
onclick="openModal(this.src)">
</div>
<!-- Filename -->
<div class="mt-3">
<div class="text-sm text-gray-600 filename" title="{{ screenshot.display_name }}">
{{ screenshot.display_name }}
</div>
</div>
</div>
{% endfor %}
</div>
{% else %}
<div class="text-center text-gray-500 py-12">
{% trans "No screenshots available yet." %}
</div>
{% endif %}
</div>
{% endblock %}
{% block extra_js %}
<script>
const modal = document.getElementById('imageModal');
const modalImg = document.getElementById('modalImage');
function openModal(imgSrc) {
modal.style.display = "block";
modalImg.src = imgSrc;
document.body.style.overflow = 'hidden';
}
function closeModal() {
modal.style.display = "none";
document.body.style.overflow = 'auto';
}
document.addEventListener('keydown', function(event) {
if (event.key === 'Escape') {
closeModal();
}
});
modalImg.onclick = function(event) {
event.stopPropagation();
};
modalImg.onload = function() {
this.style.opacity = '1';
};
modalImg.style.opacity = '0';
modalImg.style.transition = 'opacity 0.3s ease-in-out';
</script>
{% endblock %}
links/urls.py
+10 -6
View File
@@ -2,6 +2,7 @@ from django.urls import path, include
from rest_framework.routers import DefaultRouter
from django.urls import re_path
from . import views
from . import page_views
router = DefaultRouter(trailing_slash=False)
router.register(r'pages', views.PageViewSet, basename='api-page')
@@ -23,12 +24,13 @@ urlpatterns = [
path('export/', views.export_links, name='export_links'),
# Pages
path('ui/pages/', views.PageListView.as_view(), name='page-list'),
path('ui/pages/new/', views.PageCreateView.as_view(), name='page-create'),
path('ui/pages/<int:pk>/', views.PageDetailView.as_view(), name='page-detail'),
path('ui/pages/<int:pk>/edit/', views.PageUpdateView.as_view(), name='page-update'),
path('ui/pages/<int:pk>/delete/', views.PageDeleteView.as_view(), name='page-delete'),
path('fetch-page-info/', views.fetch_page_info, name='fetch-page-info'),
path('ui/pages/', page_views.PageListView.as_view(), name='page-list'),
path('ui/pages/new/', page_views.PageCreateView.as_view(), name='page-create'),
path('ui/pages/<int:pk>/', page_views.PageDetailView.as_view(), name='page-detail'),
path('ui/pages/<int:pk>/edit/', page_views.PageUpdateView.as_view(), name='page-update'),
path('ui/pages/<int:pk>/delete/', page_views.PageDeleteView.as_view(), name='page-delete'),
path('fetch-page-info/', page_views.fetch_page_info, name='fetch-page-info'),
path('ui/screenshots/', page_views.ScreenshotGalleryView.as_view(), name='screenshot-gallery'),
# API URLs - place before catch-all alias routes
path('api/', include(router.urls)),
@@ -37,4 +39,6 @@ urlpatterns = [
path('<str:alias>/', views.redirect_to_original, name='redirect_to_original'),
path('<str:alias>/<str:param>/', views.redirect_to_original, name='redirect_to_original_with_param'),
path('alias/<str:alias>/', views.LinkDetailView.as_view(), name='link_detail_by_alias'),
]
templates/base.html
+9
View File
@@ -101,6 +101,15 @@
{% trans "Help" %}
</div>
</a>
<a href="{% url 'screenshot-gallery' %}" class="block px-4 py-2 text-sm text-gray-700 hover:bg-gray-100">
<div class="flex items-center">
<svg class="w-5 h-5 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z"/>
</svg>
{% trans "Screenshots" %}
</div>
</a>
</div>
</div>
<!-- 语言选择器 -->